@cosyte/bench 0.1.1 → 0.1.3

package/dist/cli.mjs

@@ -7,6 +7,7 @@ import 'perf_hooks';
 import { fileURLToPath, pathToFileURL } from 'url';
 import { gzipSync } from 'zlib';
 import { spawn } from 'child_process';
+import 'v8';
 
 function readCgroupCpuMax(path = "/sys/fs/cgroup/cpu.max") {
   try {
@@ -59,6 +60,7 @@ var BENCH_CONTRACT = Object.freeze({
   ALLOC_WINDOW_MS: 100,
   ALLOC_SEMI_SPACE_MIB: 64,
   PEAK_CALLS: 20,
+  FLOOR_INPUT_BYTES: 64 * 1024,
   LONG_RUN_ROUNDS: 30,
   LONG_RUN_SETTLE_ROUNDS: 5,
   LONG_RUN_ROUND_MS: 100,
@@ -67,6 +69,10 @@ var BENCH_CONTRACT = Object.freeze({
   GC_SETTLE_ROUNDS: 2,
   GC_MAX_ROUNDS: 7,
   IMPORT_RUNS: 20,
+  LARGE_PASS_MS: 50,
+  LARGE_WARMUP_WINDOW: 5,
+  LARGE_WARMUP_MAX_MS: 3e4,
+  LONG_RUN_MAX_MS: 6e4,
   REQUIRED_FALSE_ALARM_RATE: 0.01,
   REQUIRED_CONFIDENCE: 0.95
 });
@@ -110,11 +116,13 @@ function runNode(entry, args, execArgv, timeoutMs, execPath = process.execPath)
 }
 function flagsFor(job) {
   if (job.mode === "timing") return [];
-  if (job.mode === "heap") return ["--expose-gc"];
+  const memory = ["--expose-gc", "--no-concurrent-array-buffer-sweeping"];
+  if (job.mode === "heap" || job.mode === "floor") return memory;
   const mib = String(BENCH_CONTRACT.ALLOC_SEMI_SPACE_MIB);
-  return ["--expose-gc", `--min-semi-space-size=${mib}`, `--max-semi-space-size=${mib}`];
+  return [...memory, `--min-semi-space-size=${mib}`, `--max-semi-space-size=${mib}`];
 }
 function describe(job) {
+  if (job.mode === "floor") return "floor job";
   return `${job.mode} job for "${job.case.benchmark} @ ${job.case.size}"`;
 }
 async function runInChild(job, options = {}) {
@@ -380,11 +388,18 @@ async function compareBuilds(options) {
           s.skip = `${side}: ${result.reason}`;
           break;
         }
-        got.set(side, result.samplesMsPerOp);
+        got.set(side, { samples: result.samplesMsPerOp, plan: result.warmup.plan });
       }
       const base = got.get("base");
       const head = got.get("head");
-      if (base !== void 0 && head !== void 0) s.rounds.push({ base, head });
+      if (base !== void 0 && head !== void 0) {
+        s.rounds.push({
+          base: base.samples,
+          head: head.samples,
+          basePlan: base.plan,
+          headPlan: head.plan
+        });
+      }
     }
     options.onProgress?.(`round ${String(r + 1)} of ${String(BENCH_CONTRACT.ROUNDS)} done`);
   }
@@ -398,7 +413,9 @@ async function compareBuilds(options) {
         baseMsPerOp: rounds.map((p) => Math.min(...p.base)),
         headMsPerOp: rounds.map((p) => Math.min(...p.head)),
         baseSamples: rounds.map((p) => p.base),
-        headSamples: rounds.map((p) => p.head)
+        headSamples: rounds.map((p) => p.head),
+        basePlans: rounds.map((p) => p.basePlan),
+        headPlans: rounds.map((p) => p.headPlan)
       };
       if (skip !== null) {
         return {
@@ -540,18 +557,55 @@ async function calibrateComparison(options) {
   }
   return summarizeCalibration(options.inject ?? 0, runs);
 }
+function aboveFloor(peak, floor) {
+  if (peak.status !== "measured" || floor === null || floor.status !== "measured") return peak;
+  const less = (value, base) => value === void 0 || base === void 0 ? void 0 : Math.max(0, value - base);
+  const out = {
+    ...peak,
+    peakHeapAboveFloorBytes: less(peak.peakHeapBytes, floor.medianPeakHeapBytes),
+    medianPeakHeapAboveFloorBytes: less(peak.medianPeakHeapBytes, floor.medianPeakHeapBytes),
+    peakExternalAboveFloorBytes: less(peak.peakExternalBytes, floor.medianPeakExternalBytes),
+    medianPeakExternalAboveFloorBytes: less(
+      peak.medianPeakExternalBytes,
+      floor.medianPeakExternalBytes
+    ),
+    peakTotalAboveFloorBytes: less(peak.peakTotalBytes, floor.medianPeakTotalBytes),
+    medianPeakTotalAboveFloorBytes: less(peak.medianPeakTotalBytes, floor.medianPeakTotalBytes)
+  };
+  return Object.fromEntries(
+    Object.entries(out).filter(([, v]) => v !== void 0)
+  );
+}
 
 // src/reading.ts
+var WrongKind = class extends Error {
+};
 function expectTiming(result) {
   if (result.mode !== "timing")
-    throw new Error("@cosyte/bench: a timing job returned another kind");
+    throw new WrongKind("@cosyte/bench: a timing job returned another kind");
   return result;
 }
 function expectMemory(result) {
-  if (result.mode === "timing")
-    throw new Error("@cosyte/bench: a memory job returned a timing result");
+  if (result.mode === "timing" || result.mode === "floor")
+    throw new WrongKind("@cosyte/bench: a memory job returned another kind");
   return result;
 }
+function expectFloor(result) {
+  if (result.mode !== "floor")
+    throw new WrongKind("@cosyte/bench: a floor job returned another kind");
+  return result.peak;
+}
+function firstLine(error) {
+  const text = error instanceof Error ? error.message : String(error);
+  return text.split("\n")[0];
+}
+function budgetOf(c, p) {
+  const raw = p.peakTotalBytes ?? p.peakHeapBytes;
+  if (c.bytesPerPass === null) return raw;
+  const perInput = c.bytesPerPass / c.inputsPerPass;
+  if (perInput >= BENCH_CONTRACT.FLOOR_INPUT_BYTES) return raw;
+  return p.medianPeakTotalAboveFloorBytes ?? raw;
+}
 function summarizeThroughput(first, rest) {
   const measured = [];
   for (const t of [first, ...rest]) {
@@ -560,7 +614,8 @@ function summarizeThroughput(first, rest) {
   }
   const per = measured.map((t) => ({
     samplesMsPerOp: t.samplesMsPerOp,
-    medianMsPerOp: t.medianMsPerOp
+    medianMsPerOp: t.medianMsPerOp,
+    plan: t.warmup.plan
   }));
   const mid = median(per.map((x) => x.medianMsPerOp));
   const perOpBytes = first.bytesPerPass === null ? null : first.bytesPerPass / first.inputsPerPass;
@@ -572,48 +627,102 @@ function summarizeThroughput(first, rest) {
     bytesPerSecond: perOpBytes === null ? null : round(perOpBytes * (1e3 / mid), 0)
   };
 }
+function longRunFits(throughput, inputsPerPass) {
+  if (throughput.status !== "measured") return false;
+  const roundMs = Math.max(
+    BENCH_CONTRACT.LONG_RUN_ROUND_MS,
+    throughput.medianMsPerOp * inputsPerPass
+  );
+  return BENCH_CONTRACT.LONG_RUN_ROUNDS * roundMs <= BENCH_CONTRACT.LONG_RUN_MAX_MS;
+}
 async function readBuild(options) {
   const runJob = options.runJob;
   const processes = options.processes ?? 3;
   if (!Number.isInteger(processes) || processes < 1) {
     throw new TypeError("@cosyte/bench: processes must be a positive integer");
   }
+  const memory = options.memory ?? true;
   const base = { suite: options.suiteFile, target: options.target };
+  const listed = listCases(options.suite, options.only);
   const cases = [];
-  for (const c of listCases(options.suite, options.only)) {
-    const timing = async (latency2) => expectTiming(await runJob({ mode: "timing", ...base, case: c, latency: latency2, inject: 0 }));
-    const first = await timing(true);
-    const rest = [];
-    for (let p = 1; p < processes; p++) rest.push(await timing(false));
-    const throughput = summarizeThroughput(first, rest);
-    const latency = first.status === "measured" ? first.latency : null;
-    let allocations = null;
-    let peak = null;
-    let longRun = null;
-    if (options.memory ?? true) {
-      allocations = expectMemory(await runJob({ mode: "alloc", ...base, case: c })).allocations;
-      const { benchmark } = findCase(options.suite, c);
-      const longRunSize = benchmark.longRun ?? benchmark.sizes[0].label;
-      const heap = expectMemory(
-        await runJob({ mode: "heap", ...base, case: c, longRun: c.size === longRunSize })
-      );
-      peak = heap.peak;
-      longRun = heap.longRun;
+  const failures = [];
+  const fail2 = (phase, c, error) => {
+    if (error instanceof WrongKind) throw error;
+    const message = firstLine(error);
+    failures.push({ phase, benchmark: c?.benchmark ?? null, size: c?.size ?? null, message });
+    const what = c === null ? "the memory floor" : `${phase} of ${c.benchmark} @ ${c.size}`;
+    options.onProgress?.(
+      `could not measure ${what}: ${error instanceof Error ? error.message : String(error)}`
+    );
+  };
+  for (const name of new Set(listed.map((c) => c.benchmark))) {
+    const timed = [];
+    for (const c of listed.filter((l) => l.benchmark === name)) {
+      const timing = async (latency) => expectTiming(await runJob({ mode: "timing", ...base, case: c, latency, inject: 0 }));
+      try {
+        const first = await timing(true);
+        const rest = [];
+        for (let p = 1; p < processes; p++) rest.push(await timing(false));
+        timed.push({ c, first, throughput: summarizeThroughput(first, rest) });
+      } catch (error) {
+        fail2("timing", c, error);
+      }
+    }
+    const largest = timed.filter((t) => longRunFits(t.throughput, t.first.inputsPerPass)).pop();
+    for (const { c, first, throughput } of timed) {
+      let allocations = null;
+      let peak = null;
+      let longRun = null;
+      if (memory) {
+        try {
+          allocations = expectMemory(await runJob({ mode: "alloc", ...base, case: c })).allocations;
+          const { benchmark } = findCase(options.suite, c);
+          const longRunSize = benchmark.longRun ?? benchmark.sizes[0].label;
+          const heap = expectMemory(
+            await runJob({
+              mode: "heap",
+              ...base,
+              case: c,
+              longRun: c.size === longRunSize || c.size === largest?.c.size
+            })
+          );
+          peak = heap.peak;
+          longRun = heap.longRun;
+        } catch (error) {
+          fail2("memory", c, error);
+        }
+      }
+      cases.push({
+        ...c,
+        inputsPerPass: first.inputsPerPass,
+        bytesPerPass: first.bytesPerPass,
+        throughput,
+        latency: first.status === "measured" ? first.latency : null,
+        allocations,
+        peak,
+        longRun
+      });
+      options.onProgress?.(`read ${c.benchmark} @ ${c.size}`);
+    }
+  }
+  let memoryFloor = null;
+  if (memory) {
+    try {
+      memoryFloor = expectFloor(await runJob({ mode: "floor" }));
+    } catch (error) {
+      fail2("floor", null, error);
     }
-    cases.push({
-      ...c,
-      inputsPerPass: first.inputsPerPass,
-      bytesPerPass: first.bytesPerPass,
-      throughput,
-      latency,
-      allocations,
-      peak,
-      longRun
-    });
-    options.onProgress?.(`read ${c.benchmark} @ ${c.size}`);
   }
   const importCost = options.importCost ?? true ? await options.measureImport(options.target) : null;
-  return { processes, cases, importCost };
+  return {
+    processes,
+    cases: cases.map(
+      (c) => c.peak === null ? c : { ...c, peak: aboveFloor(c.peak, memoryFloor) }
+    ),
+    memoryFloor,
+    failures,
+    importCost
+  };
 }
 
 // src/report.ts
@@ -698,26 +807,54 @@ function renderMarkdown(doc) {
     const r = doc.reading;
     lines.push(...header(doc, "Reading"));
     lines.push(
-      `Throughput is the median of ${String(r.processes)} fresh processes. Latency is per call, nearest rank. Memory: allocations per operation, the peak heap of one call above the settled heap, and whether the settled heap stays flat over a long run.`,
-      "",
-      "| Benchmark | Size | Per op | Ops/s | Bytes/s | p50 | p95 | p99 | Alloc/op | Peak heap | Long run |",
-      "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
+      `Throughput is the median of ${String(r.processes)} fresh processes. "Large-case plan" marks a case whose calls each outlast a warmup batch. Latency is per call, nearest rank. Memory: allocations per operation ("collections counted" when every window collected), the peak of one call above what was in use before it (the V8 heap, external memory such as Buffers, and the two together), the total above the empty-call floor, the peak a memory budget reads (the median call's total above the floor for an input under ${bytes(BENCH_CONTRACT.FLOOR_INPUT_BYTES)}, the highest raw total for any other), and whether the settled heap stays flat over a long run.`,
+      ""
+    );
+    const f = r.memoryFloor;
+    if (f !== void 0 && f !== null) {
+      lines.push(
+        f.status === "measured" ? `Empty-call floor, what the peak probe itself costs: ${bytes(f.medianPeakHeapBytes)} heap, ${f.medianPeakTotalBytes === void 0 ? "n/a" : bytes(f.medianPeakTotalBytes)} total (the median of ${String(f.calls)} calls).` : `Empty-call floor: skipped (${f.reason}).`,
+        ""
+      );
+    }
+    lines.push(
+      "| Benchmark | Size | Per op | Ops/s | Bytes/s | p50 | p95 | p99 | Alloc/op | Peak heap | Peak external | Peak total | Above floor | Budget reads | Long run |",
+      "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
     );
     for (const c of r.cases) {
       const t = c.throughput;
-      const perOp = t.status === "measured" ? time(t.medianMsPerOp) : `skipped (${t.reason})`;
+      const large = t.status === "measured" && t.processes.some((p2) => p2.plan === "large") ? " (large-case plan)" : "";
+      const perOp = t.status === "measured" ? `${time(t.medianMsPerOp)}${large}` : `skipped (${t.reason})`;
       const ops = t.status === "measured" ? String(t.opsPerSecond) : "n/a";
       const bps = t.status === "measured" && t.bytesPerSecond !== null ? `${bytes(t.bytesPerSecond)}/s` : "n/a";
       const l = c.latency;
       const lat = l === null ? ["n/a", "n/a", "n/a"] : l.status === "measured" ? [ns(l.p50Ns), ns(l.p95Ns), ns(l.p99Ns)] : [`skipped (${l.reason})`, "n/a", "n/a"];
       const a = c.allocations;
-      const alloc = a === null ? "n/a" : a.status === "measured" ? bytes(a.heapBytesPerOp + a.externalBytesPerOp) : `skipped (${a.reason})`;
+      const alloc = a === null ? "n/a" : a.status === "measured" ? `${bytes(a.heapBytesPerOp + a.externalBytesPerOp)}${a.method === "gc-accounted" ? " (collections counted)" : ""}` : `skipped (${a.reason})`;
       const p = c.peak;
-      const peak = p === null ? "n/a" : p.status === "measured" ? bytes(p.peakHeapBytes) : `skipped (${p.reason})`;
+      const optional = (n) => n === void 0 ? "n/a" : bytes(n);
+      const peak = p === null ? ["n/a", "n/a", "n/a", "n/a", "n/a"] : p.status === "measured" ? [
+        bytes(p.peakHeapBytes),
+        optional(p.peakExternalBytes),
+        optional(p.peakTotalBytes),
+        optional(p.peakTotalAboveFloorBytes),
+        bytes(budgetOf(c, p))
+      ] : Array.from({ length: 5 }, () => `skipped (${p.reason})`);
       const lr = c.longRun;
       const long = lr === null ? "" : lr.status === "measured" ? `${lr.verdict} (${bytes(lr.growthBytes)} over ${String(lr.ops)} ops, ${String(lr.slopeBytesPerOp)} B/op)` : `skipped (${lr.reason})`;
       lines.push(
-        `| ${c.benchmark} | ${c.size} | ${perOp} | ${ops} | ${bps} | ${lat.join(" | ")} | ${alloc} | ${peak} | ${long} |`
+        `| ${c.benchmark} | ${c.size} | ${perOp} | ${ops} | ${bps} | ${lat.join(" | ")} | ${alloc} | ${peak.join(" | ")} | ${long} |`
+      );
+    }
+    const failures = r.failures ?? [];
+    if (failures.length > 0) {
+      lines.push(
+        "",
+        `Could not measure ${String(failures.length)} part(s) of the run; every other case above was measured and written:`,
+        "",
+        ...failures.map(
+          (x) => x.benchmark === null ? `- the empty-call floor: ${x.message}` : `- ${x.phase} of ${x.benchmark} @ ${String(x.size)}: ${x.message}`
+        )
       );
     }
     if (r.importCost !== null) {
@@ -783,6 +920,7 @@ var skipped = {
   properties: { status: { const: "skipped" }, reason: str }
 };
 var measuredOrSkipped = (measured) => ({ oneOf: [measured, skipped] });
+var plan = { enum: ["standard", "large"] };
 var orNull = (s) => ({ oneOf: [{ type: "null" }, s] });
 var RESULTS_SCHEMA = {
   $schema: "https://json-schema.org/draft/2020-12/schema",
@@ -899,6 +1037,9 @@ var RESULTS_SCHEMA = {
         headMsPerOp: nums,
         ratios: nums,
         medianRatio: numOrNull,
+        // Added within version 1, so not required: a document written before them is still valid.
+        basePlans: { type: "array", items: plan },
+        headPlans: { type: "array", items: plan },
         interval: { $ref: "#/$defs/interval" },
         verdict: {
           enum: ["regression", "improvement", "no-regression", "inconclusive", "skipped"]
@@ -960,12 +1101,46 @@ var RESULTS_SCHEMA = {
         }
       }
     },
+    peak: measuredOrSkipped({
+      type: "object",
+      required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
+      properties: {
+        status: { const: "measured" },
+        calls: int,
+        peakHeapBytes: num,
+        medianPeakHeapBytes: num,
+        // Added within version 1, so not required.
+        peakExternalBytes: num,
+        medianPeakExternalBytes: num,
+        peakTotalBytes: num,
+        medianPeakTotalBytes: num,
+        peakHeapAboveFloorBytes: num,
+        medianPeakHeapAboveFloorBytes: num,
+        peakExternalAboveFloorBytes: num,
+        medianPeakExternalAboveFloorBytes: num,
+        peakTotalAboveFloorBytes: num,
+        medianPeakTotalAboveFloorBytes: num
+      }
+    }),
+    failure: {
+      type: "object",
+      required: ["phase", "benchmark", "size", "message"],
+      properties: {
+        phase: { enum: ["timing", "memory", "floor"] },
+        benchmark: strOrNull,
+        size: strOrNull,
+        message: str
+      }
+    },
     reading: {
       type: "object",
       required: ["processes", "cases", "importCost"],
       properties: {
         processes: int,
         cases: { type: "array", items: { $ref: "#/$defs/caseReading" } },
+        // Added within version 1, so not required.
+        memoryFloor: orNull({ $ref: "#/$defs/peak" }),
+        failures: { type: "array", items: { $ref: "#/$defs/failure" } },
         importCost: orNull({
           type: "object",
           required: [
@@ -1028,7 +1203,8 @@ var RESULTS_SCHEMA = {
               items: {
                 type: "object",
                 required: ["samplesMsPerOp", "medianMsPerOp"],
-                properties: { samplesMsPerOp: nums, medianMsPerOp: num }
+                // `plan` was added within version 1, so it is not required.
+                properties: { samplesMsPerOp: nums, medianMsPerOp: num, plan }
               }
             },
             medianMsPerOp: num,
@@ -1055,24 +1231,16 @@ var RESULTS_SCHEMA = {
             required: ["status", "windowOps", "heapBytesPerOp", "externalBytesPerOp"],
             properties: {
               status: { const: "measured" },
+              // Added within version 1, so not required.
+              method: { enum: ["no-gc-window", "gc-accounted"] },
+              collections: int,
               windowOps: int,
               heapBytesPerOp: num,
               externalBytesPerOp: num
             }
           })
         ),
-        peak: orNull(
-          measuredOrSkipped({
-            type: "object",
-            required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
-            properties: {
-              status: { const: "measured" },
-              calls: int,
-              peakHeapBytes: num,
-              medianPeakHeapBytes: num
-            }
-          })
-        ),
+        peak: orNull({ $ref: "#/$defs/peak" }),
         longRun: orNull(
           measuredOrSkipped({
             type: "object",
@@ -1186,7 +1354,8 @@ var USAGE = `Usage:
   cosyte-bench check <results.json>
 
 Exit codes: 0 clean, 1 a finding (a regression, or results that do not match the schema),
-2 could not measure or could not read its input.
+2 could not measure or could not read its input. A run that could not measure a case still
+writes every other case, lists what failed, and exits 2.
 `;
 var UsageError = class extends Error {
 };
@@ -1374,7 +1543,8 @@ async function main(argv, io) {
           subject: { target: rel(target), commit },
           reading
         },
-        0
+        // Whatever measured is written; a part that could not be is still exit 2.
+        reading.failures.length > 0 ? 2 : 0
       );
     }
     {