@cosyte/bench 0.1.1 → 0.1.2

package/dist/cli.mjs

@@ -67,6 +67,10 @@ var BENCH_CONTRACT = Object.freeze({
   GC_SETTLE_ROUNDS: 2,
   GC_MAX_ROUNDS: 7,
   IMPORT_RUNS: 20,
+  LARGE_PASS_MS: 50,
+  LARGE_WARMUP_WINDOW: 5,
+  LARGE_WARMUP_MAX_MS: 3e4,
+  LONG_RUN_MAX_MS: 6e4,
   REQUIRED_FALSE_ALARM_RATE: 0.01,
   REQUIRED_CONFIDENCE: 0.95
 });
@@ -110,9 +114,10 @@ function runNode(entry, args, execArgv, timeoutMs, execPath = process.execPath)
 }
 function flagsFor(job) {
   if (job.mode === "timing") return [];
-  if (job.mode === "heap") return ["--expose-gc"];
+  const memory = ["--expose-gc", "--no-concurrent-array-buffer-sweeping"];
+  if (job.mode === "heap") return memory;
   const mib = String(BENCH_CONTRACT.ALLOC_SEMI_SPACE_MIB);
-  return ["--expose-gc", `--min-semi-space-size=${mib}`, `--max-semi-space-size=${mib}`];
+  return [...memory, `--min-semi-space-size=${mib}`, `--max-semi-space-size=${mib}`];
 }
 function describe(job) {
   return `${job.mode} job for "${job.case.benchmark} @ ${job.case.size}"`;
@@ -380,11 +385,18 @@ async function compareBuilds(options) {
           s.skip = `${side}: ${result.reason}`;
           break;
         }
-        got.set(side, result.samplesMsPerOp);
+        got.set(side, { samples: result.samplesMsPerOp, plan: result.warmup.plan });
       }
       const base = got.get("base");
       const head = got.get("head");
-      if (base !== void 0 && head !== void 0) s.rounds.push({ base, head });
+      if (base !== void 0 && head !== void 0) {
+        s.rounds.push({
+          base: base.samples,
+          head: head.samples,
+          basePlan: base.plan,
+          headPlan: head.plan
+        });
+      }
     }
     options.onProgress?.(`round ${String(r + 1)} of ${String(BENCH_CONTRACT.ROUNDS)} done`);
   }
@@ -398,7 +410,9 @@ async function compareBuilds(options) {
         baseMsPerOp: rounds.map((p) => Math.min(...p.base)),
         headMsPerOp: rounds.map((p) => Math.min(...p.head)),
         baseSamples: rounds.map((p) => p.base),
-        headSamples: rounds.map((p) => p.head)
+        headSamples: rounds.map((p) => p.head),
+        basePlans: rounds.map((p) => p.basePlan),
+        headPlans: rounds.map((p) => p.headPlan)
       };
       if (skip !== null) {
         return {
@@ -560,7 +574,8 @@ function summarizeThroughput(first, rest) {
   }
   const per = measured.map((t) => ({
     samplesMsPerOp: t.samplesMsPerOp,
-    medianMsPerOp: t.medianMsPerOp
+    medianMsPerOp: t.medianMsPerOp,
+    plan: t.warmup.plan
   }));
   const mid = median(per.map((x) => x.medianMsPerOp));
   const perOpBytes = first.bytesPerPass === null ? null : first.bytesPerPass / first.inputsPerPass;
@@ -572,6 +587,14 @@ function summarizeThroughput(first, rest) {
     bytesPerSecond: perOpBytes === null ? null : round(perOpBytes * (1e3 / mid), 0)
   };
 }
+function longRunFits(throughput, inputsPerPass) {
+  if (throughput.status !== "measured") return false;
+  const roundMs = Math.max(
+    BENCH_CONTRACT.LONG_RUN_ROUND_MS,
+    throughput.medianMsPerOp * inputsPerPass
+  );
+  return BENCH_CONTRACT.LONG_RUN_ROUNDS * roundMs <= BENCH_CONTRACT.LONG_RUN_MAX_MS;
+}
 async function readBuild(options) {
   const runJob = options.runJob;
   const processes = options.processes ?? 3;
@@ -579,38 +602,49 @@ async function readBuild(options) {
     throw new TypeError("@cosyte/bench: processes must be a positive integer");
   }
   const base = { suite: options.suiteFile, target: options.target };
+  const listed = listCases(options.suite, options.only);
   const cases = [];
-  for (const c of listCases(options.suite, options.only)) {
-    const timing = async (latency2) => expectTiming(await runJob({ mode: "timing", ...base, case: c, latency: latency2, inject: 0 }));
-    const first = await timing(true);
-    const rest = [];
-    for (let p = 1; p < processes; p++) rest.push(await timing(false));
-    const throughput = summarizeThroughput(first, rest);
-    const latency = first.status === "measured" ? first.latency : null;
-    let allocations = null;
-    let peak = null;
-    let longRun = null;
-    if (options.memory ?? true) {
-      allocations = expectMemory(await runJob({ mode: "alloc", ...base, case: c })).allocations;
-      const { benchmark } = findCase(options.suite, c);
-      const longRunSize = benchmark.longRun ?? benchmark.sizes[0].label;
-      const heap = expectMemory(
-        await runJob({ mode: "heap", ...base, case: c, longRun: c.size === longRunSize })
-      );
-      peak = heap.peak;
-      longRun = heap.longRun;
+  for (const name of new Set(listed.map((c) => c.benchmark))) {
+    const timed = [];
+    for (const c of listed.filter((l) => l.benchmark === name)) {
+      const timing = async (latency) => expectTiming(await runJob({ mode: "timing", ...base, case: c, latency, inject: 0 }));
+      const first = await timing(true);
+      const rest = [];
+      for (let p = 1; p < processes; p++) rest.push(await timing(false));
+      timed.push({ c, first, throughput: summarizeThroughput(first, rest) });
+    }
+    const largest = timed.filter((t) => longRunFits(t.throughput, t.first.inputsPerPass)).pop();
+    for (const { c, first, throughput } of timed) {
+      let allocations = null;
+      let peak = null;
+      let longRun = null;
+      if (options.memory ?? true) {
+        allocations = expectMemory(await runJob({ mode: "alloc", ...base, case: c })).allocations;
+        const { benchmark } = findCase(options.suite, c);
+        const longRunSize = benchmark.longRun ?? benchmark.sizes[0].label;
+        const heap = expectMemory(
+          await runJob({
+            mode: "heap",
+            ...base,
+            case: c,
+            longRun: c.size === longRunSize || c.size === largest?.c.size
+          })
+        );
+        peak = heap.peak;
+        longRun = heap.longRun;
+      }
+      cases.push({
+        ...c,
+        inputsPerPass: first.inputsPerPass,
+        bytesPerPass: first.bytesPerPass,
+        throughput,
+        latency: first.status === "measured" ? first.latency : null,
+        allocations,
+        peak,
+        longRun
+      });
+      options.onProgress?.(`read ${c.benchmark} @ ${c.size}`);
     }
-    cases.push({
-      ...c,
-      inputsPerPass: first.inputsPerPass,
-      bytesPerPass: first.bytesPerPass,
-      throughput,
-      latency,
-      allocations,
-      peak,
-      longRun
-    });
-    options.onProgress?.(`read ${c.benchmark} @ ${c.size}`);
   }
   const importCost = options.importCost ?? true ? await options.measureImport(options.target) : null;
   return { processes, cases, importCost };
@@ -698,26 +732,28 @@ function renderMarkdown(doc) {
     const r = doc.reading;
     lines.push(...header(doc, "Reading"));
     lines.push(
-      `Throughput is the median of ${String(r.processes)} fresh processes. Latency is per call, nearest rank. Memory: allocations per operation, the peak heap of one call above the settled heap, and whether the settled heap stays flat over a long run.`,
+      `Throughput is the median of ${String(r.processes)} fresh processes. "Large-case plan" marks a case whose calls each outlast a warmup batch. Latency is per call, nearest rank. Memory: allocations per operation ("collections counted" when every window collected), the peak of one call above what was in use before it (the V8 heap, external memory such as Buffers, and the two together, which a memory budget reads), and whether the settled heap stays flat over a long run.`,
       "",
-      "| Benchmark | Size | Per op | Ops/s | Bytes/s | p50 | p95 | p99 | Alloc/op | Peak heap | Long run |",
-      "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
+      "| Benchmark | Size | Per op | Ops/s | Bytes/s | p50 | p95 | p99 | Alloc/op | Peak heap | Peak external | Peak total | Long run |",
+      "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
     );
     for (const c of r.cases) {
       const t = c.throughput;
-      const perOp = t.status === "measured" ? time(t.medianMsPerOp) : `skipped (${t.reason})`;
+      const large = t.status === "measured" && t.processes.some((p2) => p2.plan === "large") ? " (large-case plan)" : "";
+      const perOp = t.status === "measured" ? `${time(t.medianMsPerOp)}${large}` : `skipped (${t.reason})`;
       const ops = t.status === "measured" ? String(t.opsPerSecond) : "n/a";
       const bps = t.status === "measured" && t.bytesPerSecond !== null ? `${bytes(t.bytesPerSecond)}/s` : "n/a";
       const l = c.latency;
       const lat = l === null ? ["n/a", "n/a", "n/a"] : l.status === "measured" ? [ns(l.p50Ns), ns(l.p95Ns), ns(l.p99Ns)] : [`skipped (${l.reason})`, "n/a", "n/a"];
       const a = c.allocations;
-      const alloc = a === null ? "n/a" : a.status === "measured" ? bytes(a.heapBytesPerOp + a.externalBytesPerOp) : `skipped (${a.reason})`;
+      const alloc = a === null ? "n/a" : a.status === "measured" ? `${bytes(a.heapBytesPerOp + a.externalBytesPerOp)}${a.method === "gc-accounted" ? " (collections counted)" : ""}` : `skipped (${a.reason})`;
       const p = c.peak;
-      const peak = p === null ? "n/a" : p.status === "measured" ? bytes(p.peakHeapBytes) : `skipped (${p.reason})`;
+      const optional = (n) => n === void 0 ? "n/a" : bytes(n);
+      const peak = p === null ? ["n/a", "n/a", "n/a"] : p.status === "measured" ? [bytes(p.peakHeapBytes), optional(p.peakExternalBytes), optional(p.peakTotalBytes)] : Array.from({ length: 3 }, () => `skipped (${p.reason})`);
       const lr = c.longRun;
       const long = lr === null ? "" : lr.status === "measured" ? `${lr.verdict} (${bytes(lr.growthBytes)} over ${String(lr.ops)} ops, ${String(lr.slopeBytesPerOp)} B/op)` : `skipped (${lr.reason})`;
       lines.push(
-        `| ${c.benchmark} | ${c.size} | ${perOp} | ${ops} | ${bps} | ${lat.join(" | ")} | ${alloc} | ${peak} | ${long} |`
+        `| ${c.benchmark} | ${c.size} | ${perOp} | ${ops} | ${bps} | ${lat.join(" | ")} | ${alloc} | ${peak.join(" | ")} | ${long} |`
       );
     }
     if (r.importCost !== null) {
@@ -783,6 +819,7 @@ var skipped = {
   properties: { status: { const: "skipped" }, reason: str }
 };
 var measuredOrSkipped = (measured) => ({ oneOf: [measured, skipped] });
+var plan = { enum: ["standard", "large"] };
 var orNull = (s) => ({ oneOf: [{ type: "null" }, s] });
 var RESULTS_SCHEMA = {
   $schema: "https://json-schema.org/draft/2020-12/schema",
@@ -899,6 +936,9 @@ var RESULTS_SCHEMA = {
         headMsPerOp: nums,
         ratios: nums,
         medianRatio: numOrNull,
+        // Added within version 1, so not required: a document written before them is still valid.
+        basePlans: { type: "array", items: plan },
+        headPlans: { type: "array", items: plan },
         interval: { $ref: "#/$defs/interval" },
         verdict: {
           enum: ["regression", "improvement", "no-regression", "inconclusive", "skipped"]
@@ -1028,7 +1068,8 @@ var RESULTS_SCHEMA = {
               items: {
                 type: "object",
                 required: ["samplesMsPerOp", "medianMsPerOp"],
-                properties: { samplesMsPerOp: nums, medianMsPerOp: num }
+                // `plan` was added within version 1, so it is not required.
+                properties: { samplesMsPerOp: nums, medianMsPerOp: num, plan }
               }
             },
             medianMsPerOp: num,
@@ -1055,6 +1096,9 @@ var RESULTS_SCHEMA = {
             required: ["status", "windowOps", "heapBytesPerOp", "externalBytesPerOp"],
             properties: {
               status: { const: "measured" },
+              // Added within version 1, so not required.
+              method: { enum: ["no-gc-window", "gc-accounted"] },
+              collections: int,
               windowOps: int,
               heapBytesPerOp: num,
               externalBytesPerOp: num
@@ -1069,7 +1113,12 @@ var RESULTS_SCHEMA = {
               status: { const: "measured" },
               calls: int,
               peakHeapBytes: num,
-              medianPeakHeapBytes: num
+              medianPeakHeapBytes: num,
+              // Added within version 1, so not required.
+              peakExternalBytes: num,
+              medianPeakExternalBytes: num,
+              peakTotalBytes: num,
+              medianPeakTotalBytes: num
             }
           })
         ),