@cosyte/bench 0.0.0-stage → 0.1.2

package/dist/child-entry.mjs

@@ -0,0 +1,554 @@
+import { readFileSync, writeFileSync } from 'fs';
+import { performance } from 'perf_hooks';
+import { pathToFileURL } from 'url';
+import { GCProfiler, getHeapStatistics } from 'v8';
+
+// src/child.ts
+
+// src/contract.ts
+var BENCH_CONTRACT = Object.freeze({
+  SAMPLES: 5,
+  SAMPLE_MIN_MS: 50,
+  ROUNDS: 12,
+  ALPHA: 0.05,
+  THRESHOLD: 0.03,
+  LATENCY_BUDGET_MS: 2e3,
+  LATENCY_MAX_CALLS: 5e3,
+  LATENCY_MIN_CALLS: 20,
+  LATENCY_TAIL_CALLS: 10,
+  LATENCY_TIMER_MULTIPLE: 20,
+  ALLOC_WINDOW_OPS: 256,
+  ALLOC_WINDOW_MS: 100,
+  ALLOC_SEMI_SPACE_MIB: 64,
+  PEAK_CALLS: 20,
+  LONG_RUN_ROUNDS: 30,
+  LONG_RUN_SETTLE_ROUNDS: 5,
+  LONG_RUN_ROUND_MS: 100,
+  LONG_RUN_GROWTH_BYTES: 64 * 1024,
+  GC_SETTLE_BYTES: 64 * 1024,
+  GC_SETTLE_ROUNDS: 2,
+  GC_MAX_ROUNDS: 7,
+  IMPORT_RUNS: 20,
+  LARGE_PASS_MS: 50,
+  LARGE_WARMUP_WINDOW: 5,
+  LARGE_WARMUP_MAX_MS: 3e4,
+  LONG_RUN_MAX_MS: 6e4,
+  REQUIRED_FALSE_ALARM_RATE: 0.01,
+  REQUIRED_CONFIDENCE: 0.95
+});
+
+// src/stats.ts
+function sortedCopy(xs) {
+  return [...xs].sort((a, b) => a - b);
+}
+function median(xs) {
+  const s = sortedCopy(xs);
+  const mid = Math.floor(s.length / 2);
+  return s.length % 2 === 1 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
+}
+function nearestRank(xs, q) {
+  if (xs.length === 0) return Number.NaN;
+  const s = sortedCopy(xs);
+  const rank = Math.max(1, Math.ceil(q * s.length));
+  return s[rank - 1];
+}
+function theilSenSlope(xs, ys) {
+  const slopes = [];
+  xs.forEach((xi, i) => {
+    for (let j = i + 1; j < xs.length; j++) {
+      const dx = xs[j] - xi;
+      if (dx !== 0) slopes.push((ys[j] - ys[i]) / dx);
+    }
+  });
+  return median(slopes);
+}
+function round(n, places = 4) {
+  const f = 10 ** places;
+  return Math.round(n * f) / f;
+}
+
+// src/memory.ts
+function realProbe() {
+  const gc = globalThis.gc;
+  if (typeof gc !== "function") return null;
+  return {
+    gc: () => {
+      gc();
+    },
+    usedHeap: () => getHeapStatistics().used_heap_size,
+    external: () => process.memoryUsage().external,
+    profile: (fn) => {
+      const profiler = new GCProfiler();
+      profiler.start();
+      fn();
+      const stats = profiler.stop().statistics;
+      const report = {
+        gcs: stats.length,
+        peakBeforeGc: 0,
+        peakExternalBeforeGc: 0,
+        peakTotalBeforeGc: 0,
+        reclaimedHeap: 0,
+        reclaimedExternal: 0
+      };
+      for (const s of stats) {
+        const before = s.beforeGC.heapStatistics;
+        const after = s.afterGC.heapStatistics;
+        report.peakBeforeGc = Math.max(report.peakBeforeGc, before.usedHeapSize);
+        report.peakExternalBeforeGc = Math.max(report.peakExternalBeforeGc, before.externalMemory);
+        report.peakTotalBeforeGc = Math.max(
+          report.peakTotalBeforeGc,
+          before.usedHeapSize + before.externalMemory
+        );
+        report.reclaimedHeap += before.usedHeapSize - after.usedHeapSize;
+        report.reclaimedExternal += before.externalMemory - after.externalMemory;
+      }
+      return report;
+    }
+  };
+}
+function settle(probe) {
+  let quiet = 0;
+  let before = probe.usedHeap();
+  for (let round2 = 1; round2 <= BENCH_CONTRACT.GC_MAX_ROUNDS; round2++) {
+    probe.gc();
+    const after = probe.usedHeap();
+    quiet = before - after < BENCH_CONTRACT.GC_SETTLE_BYTES ? quiet + 1 : 0;
+    before = after;
+    if (quiet >= BENCH_CONTRACT.GC_SETTLE_ROUNDS)
+      return { settled: true, rounds: round2, heap: after };
+  }
+  return { settled: false, rounds: BENCH_CONTRACT.GC_MAX_ROUNDS, heap: before };
+}
+function measureAllocations(call, probe, msPerOp2) {
+  const window = (ops) => {
+    const s = settle(probe);
+    const h0 = probe.usedHeap();
+    const e0 = probe.external();
+    const p = probe.profile(() => {
+      for (let i = 0; i < ops; i++) call(i);
+    });
+    return {
+      settled: s.settled,
+      gcs: p.gcs,
+      heap: probe.usedHeap() - h0 + p.reclaimedHeap,
+      ext: probe.external() - e0 + p.reclaimedExternal
+    };
+  };
+  const perOp = (method, n, a2, b2) => ({
+    status: "measured",
+    method,
+    collections: a2.gcs + b2.gcs,
+    windowOps: n,
+    heapBytesPerOp: round((b2.heap - a2.heap) / n, 1),
+    externalBytesPerOp: round((b2.ext - a2.ext) / n, 1)
+  });
+  const paced = Math.floor(BENCH_CONTRACT.ALLOC_WINDOW_MS / msPerOp2);
+  const first = Math.max(1, Math.min(BENCH_CONTRACT.ALLOC_WINDOW_OPS, paced));
+  let largest = null;
+  for (let n = first; n >= 1; n = Math.floor(n / 2)) {
+    const a2 = window(n);
+    const b2 = window(2 * n);
+    if (!a2.settled || !b2.settled) return { status: "skipped", reason: "gc-unsettled" };
+    if (a2.gcs === 0 && b2.gcs === 0) return perOp("no-gc-window", n, a2, b2);
+    largest ??= [a2, b2];
+  }
+  const [a, b] = largest;
+  return perOp("gc-accounted", first, a, b);
+}
+function measurePeak(call, probe) {
+  const heap = [];
+  const external = [];
+  const total = [];
+  for (let i = 0; i < BENCH_CONTRACT.PEAK_CALLS; i++) {
+    const s = settle(probe);
+    if (!s.settled) return { status: "skipped", reason: "gc-unsettled" };
+    const e0 = probe.external();
+    const p = probe.profile(() => {
+      call(i);
+    });
+    const h1 = probe.usedHeap();
+    const e1 = probe.external();
+    heap.push(Math.max(p.peakBeforeGc, h1) - s.heap);
+    external.push(Math.max(p.peakExternalBeforeGc, e1) - e0);
+    total.push(Math.max(p.peakTotalBeforeGc, h1 + e1) - (s.heap + e0));
+  }
+  return {
+    status: "measured",
+    calls: heap.length,
+    peakHeapBytes: Math.max(...heap),
+    medianPeakHeapBytes: median(heap),
+    peakExternalBytes: Math.max(...external),
+    medianPeakExternalBytes: median(external),
+    peakTotalBytes: Math.max(...total),
+    medianPeakTotalBytes: median(total)
+  };
+}
+function measureLongRun(pass, opsPerPass, probe, clock) {
+  const t0 = clock();
+  let passesPerRound = 0;
+  do {
+    pass();
+    passesPerRound++;
+  } while (clock() - t0 < BENCH_CONTRACT.LONG_RUN_ROUND_MS);
+  const opsPerRound = passesPerRound * opsPerPass;
+  const rounds = [];
+  let ops = opsPerRound;
+  for (; ; ) {
+    const s = settle(probe);
+    if (!s.settled) return { status: "skipped", reason: "gc-unsettled" };
+    rounds.push({ ops, heap: s.heap });
+    if (rounds.length === BENCH_CONTRACT.LONG_RUN_ROUNDS) break;
+    for (let p = 0; p < passesPerRound; p++) pass();
+    ops += opsPerRound;
+  }
+  const skip = BENCH_CONTRACT.LONG_RUN_SETTLE_ROUNDS;
+  const fitted = rounds.slice(skip);
+  const slope = theilSenSlope(
+    fitted.map((r) => r.ops),
+    fitted.map((r) => r.heap)
+  );
+  const span = (BENCH_CONTRACT.LONG_RUN_ROUNDS - 1 - skip) * opsPerRound;
+  const growth = slope * span;
+  return {
+    status: "measured",
+    // Stryker disable next-line EqualityOperator: growth is a fitted slope times 24 rounds, so exactly 65,536 bytes is not reached in floating point (tried in memory.test.ts history).
+    verdict: growth >= BENCH_CONTRACT.LONG_RUN_GROWTH_BYTES ? "growing" : "flat",
+    ops: BENCH_CONTRACT.LONG_RUN_ROUNDS * opsPerRound,
+    opsPerRound,
+    settledHeap: rounds.map((r) => r.heap),
+    slopeBytesPerOp: round(slope, 4),
+    growthBytes: Math.round(growth),
+    detectableBytesPerOp: round(BENCH_CONTRACT.LONG_RUN_GROWTH_BYTES / span, 4)
+  };
+}
+
+// ../test-utils/src/perf/contract.ts
+var PERF_CONTRACT = Object.freeze({
+  RATIO_CEILING: 8,
+  RATIO_FLOOR: 1.5,
+  REPS: 5,
+  SCALE_STEP: 4,
+  MIN_PHASE_MS: 4,
+  WARMUP_MIN_MS: 500,
+  WARMUP_BATCH_MS: 50,
+  WARMUP_MAX_MS: 5e3,
+  WARMUP_STABLE_TOL: 0.05,
+  WARMUP_STABLE_BATCHES: 3
+});
+
+// src/measure.ts
+var WARMUP = Object.freeze({
+  MIN_MS: PERF_CONTRACT.WARMUP_MIN_MS,
+  BATCH_MS: PERF_CONTRACT.WARMUP_BATCH_MS,
+  MAX_MS: PERF_CONTRACT.WARMUP_MAX_MS,
+  STABLE_TOL: PERF_CONTRACT.WARMUP_STABLE_TOL,
+  STABLE_BATCHES: PERF_CONTRACT.WARMUP_STABLE_BATCHES
+});
+function batchesAreStable(batches) {
+  if (batches.length < WARMUP.STABLE_BATCHES) return false;
+  const window = batches.slice(-WARMUP.STABLE_BATCHES);
+  const mid = median(window);
+  return window.every((b) => Math.abs(b - mid) / mid <= WARMUP.STABLE_TOL);
+}
+function isLargeCase(batches) {
+  if (batches.length < WARMUP.STABLE_BATCHES) return false;
+  return batches.slice(-WARMUP.STABLE_BATCHES).every((b) => b >= BENCH_CONTRACT.LARGE_PASS_MS);
+}
+function windowsAreStable(passes) {
+  const w = BENCH_CONTRACT.LARGE_WARMUP_WINDOW;
+  if (passes.length < 2 * w) return false;
+  const last = median(passes.slice(-w));
+  const before = median(passes.slice(-2 * w, -w));
+  return Math.abs(last - before) / before <= WARMUP.STABLE_TOL;
+}
+function warmUp(pass, clock) {
+  const start = clock();
+  const batches = [];
+  let passes = 0;
+  for (; ; ) {
+    const b0 = clock();
+    let inBatch = 0;
+    let batchMs;
+    do {
+      pass();
+      inBatch++;
+      batchMs = clock() - b0;
+    } while (batchMs < WARMUP.BATCH_MS);
+    passes += inBatch;
+    batches.push(batchMs / inBatch);
+    const elapsedMs = clock() - start;
+    if (elapsedMs >= WARMUP.MIN_MS) {
+      if (batchesAreStable(batches)) {
+        return { stable: true, elapsedMs, passes, batches, plan: "standard" };
+      }
+      if (isLargeCase(batches)) return warmUpLarge(pass, clock, start, batches, passes);
+    }
+    if (elapsedMs >= WARMUP.MAX_MS) {
+      return { stable: false, elapsedMs, passes, batches, plan: "standard" };
+    }
+  }
+}
+function warmUpLarge(pass, clock, start, batches, passesSoFar) {
+  let passes = passesSoFar;
+  for (; ; ) {
+    const p0 = clock();
+    pass();
+    const t = clock();
+    passes++;
+    batches.push(t - p0);
+    const elapsedMs = t - start;
+    if (windowsAreStable(batches))
+      return { stable: true, elapsedMs, passes, batches, plan: "large" };
+    if (elapsedMs >= BENCH_CONTRACT.LARGE_WARMUP_MAX_MS) {
+      return { stable: false, elapsedMs, passes, batches, plan: "large" };
+    }
+  }
+}
+function sampleThroughput(pass, opsPerPass, clock, inject = 0) {
+  const samples = [];
+  for (let s = 0; s < BENCH_CONTRACT.SAMPLES; s++) {
+    const t0 = clock();
+    let t = t0;
+    let passes = 0;
+    do {
+      const p0 = t;
+      pass();
+      t = clock();
+      const until = t + inject * (t - p0);
+      while (t < until) t = clock();
+      passes++;
+    } while (t - t0 < BENCH_CONTRACT.SAMPLE_MIN_MS);
+    samples.push((t - t0) / (passes * opsPerPass));
+  }
+  return samples;
+}
+function timerOverheadNs(ns, reads = 1e3) {
+  const gaps = [];
+  for (let i = 0; i < reads; i++) {
+    const a = ns();
+    const b = ns();
+    gaps.push(Number(b - a));
+  }
+  return median(gaps);
+}
+function tailPercentile(times, q) {
+  return times.length * (1 - q) >= BENCH_CONTRACT.LATENCY_TAIL_CALLS ? nearestRank(times, q) : null;
+}
+function measureLatency(call, ns) {
+  const overhead = timerOverheadNs(ns);
+  const budgetNs = BigInt(BENCH_CONTRACT.LATENCY_BUDGET_MS) * 1000000n;
+  const times = [];
+  const start = ns();
+  let now = start;
+  while (times.length < BENCH_CONTRACT.LATENCY_MAX_CALLS && (times.length < BENCH_CONTRACT.LATENCY_MIN_CALLS || now - start < budgetNs)) {
+    const a = ns();
+    call(times.length);
+    now = ns();
+    times.push(Number(now - a));
+  }
+  const p50 = nearestRank(times, 0.5);
+  if (p50 < BENCH_CONTRACT.LATENCY_TIMER_MULTIPLE * overhead) {
+    return {
+      status: "skipped",
+      reason: "latency-below-timer-resolution",
+      calls: times.length,
+      timerOverheadNs: round(overhead, 1),
+      p50Ns: p50
+    };
+  }
+  return {
+    status: "measured",
+    calls: times.length,
+    timerOverheadNs: round(overhead, 1),
+    p50Ns: tailPercentile(times, 0.5),
+    p95Ns: tailPercentile(times, 0.95),
+    p99Ns: tailPercentile(times, 0.99),
+    maxNs: Math.max(...times)
+  };
+}
+
+// src/suite.ts
+function fail(message) {
+  throw new TypeError(`@cosyte/bench: ${message}`);
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null;
+}
+function checkName(value, what) {
+  if (typeof value !== "string" || value.trim() === "") fail(`${what} must be a non-empty string`);
+  return value;
+}
+function checkSuite(value) {
+  if (!isRecord(value)) fail("a suite must be an object made by defineSuite");
+  const name = checkName(value["name"], "the suite name");
+  const benchmarks = value["benchmarks"];
+  if (!Array.isArray(benchmarks) || benchmarks.length === 0) {
+    fail(`suite "${name}" must have at least one benchmark`);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const b of benchmarks) {
+    if (!isRecord(b)) fail(`suite "${name}" holds a benchmark that is not an object`);
+    const bName = checkName(b["name"], `a benchmark name in suite "${name}"`);
+    if (seen.has(bName)) fail(`suite "${name}" has two benchmarks named "${bName}"`);
+    seen.add(bName);
+    if (typeof b["bind"] !== "function") fail(`benchmark "${bName}" has no bind function`);
+    if (b["weigh"] !== void 0 && typeof b["weigh"] !== "function") {
+      fail(`benchmark "${bName}" has a weigh that is not a function`);
+    }
+    const sizes = b["sizes"];
+    if (!Array.isArray(sizes) || sizes.length === 0) fail(`benchmark "${bName}" has no sizes`);
+    const labels = /* @__PURE__ */ new Set();
+    for (const s of sizes) {
+      if (!isRecord(s)) fail(`benchmark "${bName}" holds a size that is not an object`);
+      const label = checkName(s["label"], `a size label in benchmark "${bName}"`);
+      if (labels.has(label)) fail(`benchmark "${bName}" has two sizes labelled "${label}"`);
+      labels.add(label);
+      if (typeof s["build"] !== "function")
+        fail(`size "${label}" of "${bName}" has no build function`);
+      if (s["compare"] !== void 0 && typeof s["compare"] !== "boolean") {
+        fail(`size "${label}" of "${bName}" has a compare that is not true or false`);
+      }
+    }
+    if (b["longRun"] !== void 0 && !labels.has(b["longRun"])) {
+      fail(`benchmark "${bName}" names a longRun size it does not have`);
+    }
+  }
+  return Object.freeze(value);
+}
+function caseId(c) {
+  return `${c.benchmark} @ ${c.size}`;
+}
+function findCase(suite, c) {
+  const benchmark = suite.benchmarks.find((b) => b.name === c.benchmark);
+  const size = benchmark?.sizes.find((s) => s.label === c.size);
+  if (size === void 0) fail(`suite "${suite.name}" has no case "${caseId(c)}"`);
+  return { benchmark, size };
+}
+
+// src/child.ts
+var realDeps = () => ({
+  clock: () => performance.now(),
+  ns: () => process.hrtime.bigint(),
+  probe: realProbe()
+});
+var defaultWeigh = (result) => result === void 0 || result === null ? 0 : 1;
+var SKIPPED_NO_GC = { status: "skipped", reason: "gc-unavailable" };
+function guard(c, phase, fn) {
+  try {
+    return fn();
+  } catch (error) {
+    const kind = error instanceof Error ? error.name : typeof error;
+    throw new Error(
+      `@cosyte/bench: case "${c.benchmark} @ ${c.size}": the suite's ${phase} threw (${kind}). Its message is not printed, because it may quote an input; run the case directly to read it.`,
+      { cause: error }
+    );
+  }
+}
+function msPerOp(warmup, shape) {
+  return (warmup.batches[warmup.batches.length - 1] ?? 0) / shape.inputsPerPass;
+}
+function measureJob(job, shape, warmup, pass, call, deps) {
+  const base = { case: job.case, ...shape };
+  if (job.mode === "timing") {
+    if (!warmup.stable) {
+      return { mode: "timing", ...base, warmup, status: "skipped", reason: "warmup-unstable" };
+    }
+    const samples = sampleThroughput(pass, shape.inputsPerPass, deps.clock, job.inject);
+    return {
+      mode: "timing",
+      ...base,
+      warmup,
+      status: "measured",
+      samplesMsPerOp: samples,
+      medianMsPerOp: median(samples),
+      latency: job.latency ? measureLatency(call, deps.ns) : null
+    };
+  }
+  const probe = deps.probe;
+  if (job.mode === "alloc") {
+    return {
+      mode: "alloc",
+      ...base,
+      allocations: probe === null ? SKIPPED_NO_GC : measureAllocations(call, probe, msPerOp(warmup, shape)),
+      peak: null,
+      longRun: null
+    };
+  }
+  let longRun = null;
+  if (job.longRun) {
+    longRun = probe === null ? SKIPPED_NO_GC : measureLongRun(pass, shape.inputsPerPass, probe, deps.clock);
+  }
+  return {
+    mode: "heap",
+    ...base,
+    allocations: null,
+    peak: probe === null ? SKIPPED_NO_GC : measurePeak(call, probe),
+    longRun
+  };
+}
+function measuredSomething(result) {
+  if (result.mode === "timing") return result.status === "measured";
+  return [result.allocations, result.peak, result.longRun].some((r) => r?.status === "measured");
+}
+async function runJob(job, deps = realDeps()) {
+  const suiteModule = await import(pathToFileURL(job.suite).href);
+  const suite = checkSuite(suiteModule.default);
+  const target = await import(pathToFileURL(job.target).href);
+  const { benchmark, size } = findCase(suite, job.case);
+  const corpus = guard(job.case, "build", () => size.build());
+  try {
+    const inputs = corpus.inputs;
+    if (!Array.isArray(inputs) || inputs.length === 0) {
+      throw new Error(
+        `@cosyte/bench: case "${job.case.benchmark} @ ${job.case.size}" built an empty corpus`
+      );
+    }
+    const fn = guard(job.case, "bind", () => benchmark.bind(target));
+    const weigh = benchmark.weigh ?? defaultWeigh;
+    const sink = { value: 0 };
+    const pass = () => {
+      let local = 0;
+      for (const input of inputs) local += weigh(fn(input));
+      sink.value += local;
+    };
+    const call = (i) => {
+      sink.value += weigh(fn(inputs[i % inputs.length]));
+    };
+    const shape = {
+      inputsPerPass: inputs.length,
+      bytesPerPass: typeof corpus.bytes === "number" ? corpus.bytes : null
+    };
+    const warmup = guard(job.case, "measured function", () => warmUp(pass, deps.clock));
+    const before = sink.value;
+    const result = guard(
+      job.case,
+      "measured function",
+      () => measureJob(job, shape, warmup, pass, call, deps)
+    );
+    if (!(sink.value > before) && measuredSomething(result)) {
+      throw new Error(
+        `@cosyte/bench: case "${job.case.benchmark} @ ${job.case.size}": the sink did not grow, so the measured loop may have been deleted by the compiler, or \`weigh\` gave nothing positive. Supply a \`weigh\` that returns a positive number for a result.`
+      );
+    }
+    return result;
+  } finally {
+    corpus.teardown?.();
+  }
+}
+async function childMain(jobFile, outFile, deps) {
+  try {
+    const job = JSON.parse(readFileSync(jobFile, "utf8"));
+    const result = await runJob(job, deps);
+    writeFileSync(outFile, JSON.stringify({ ok: true, result }));
+    return 0;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    writeFileSync(outFile, JSON.stringify({ ok: false, error: message }));
+    return 2;
+  }
+}
+
+// src/child-entry.ts
+process.exitCode = await childMain(process.argv[2] ?? "", process.argv[3] ?? "");
+//# sourceMappingURL=child-entry.mjs.map
+//# sourceMappingURL=child-entry.mjs.map