@cosyte/bench 0.0.0-stage → 0.1.1

package/dist/cli.mjs

@@ -0,0 +1,1441 @@
+#!/usr/bin/env node
+import { writeFileSync, readFileSync, mkdtempSync, rmSync } from 'fs';
+import { cpus, loadavg, totalmem, availableParallelism, release, tmpdir } from 'os';
+import { createRequire } from 'module';
+import { resolve, relative, join } from 'path';
+import 'perf_hooks';
+import { fileURLToPath, pathToFileURL } from 'url';
+import { gzipSync } from 'zlib';
+import { spawn } from 'child_process';
+
+function readCgroupCpuMax(path = "/sys/fs/cgroup/cpu.max") {
+  try {
+    return String(readFileSync(path)).trim();
+  } catch {
+    return null;
+  }
+}
+function environment(env = process.env, cgroupCpuMax = () => readCgroupCpuMax(), cpuList = cpus, load = loadavg) {
+  const cpu = cpuList()[0];
+  const provider = env["GITHUB_ACTIONS"] === "true" ? "github-actions" : null;
+  const imageOS = env["ImageOS"] ?? null;
+  const runnerClass = provider === null ? `local-${process.platform}-${process.arch}` : `${provider}-${env["RUNNER_ENVIRONMENT"] ?? "unknown"}-${imageOS ?? "unknown"}-${process.arch}`;
+  return {
+    node: process.version,
+    v8: process.versions.v8,
+    platform: process.platform,
+    arch: process.arch,
+    osRelease: release(),
+    cpuModel: cpu === void 0 ? null : cpu.model.trim(),
+    logicalCpus: availableParallelism(),
+    totalMemoryBytes: totalmem(),
+    cgroupCpuMax: cgroupCpuMax(),
+    nodeOptionsSet: (env["NODE_OPTIONS"] ?? "") !== "",
+    loadAverage: load().map((x) => Math.round(x * 100) / 100),
+    runner: {
+      ci: env["CI"] === "true",
+      provider,
+      environment: env["RUNNER_ENVIRONMENT"] ?? null,
+      imageOS,
+      imageVersion: env["ImageVersion"] ?? null,
+      class: runnerClass
+    }
+  };
+}
+
+// src/contract.ts
+var BENCH_CONTRACT = Object.freeze({
+  SAMPLES: 5,
+  SAMPLE_MIN_MS: 50,
+  ROUNDS: 12,
+  ALPHA: 0.05,
+  THRESHOLD: 0.03,
+  LATENCY_BUDGET_MS: 2e3,
+  LATENCY_MAX_CALLS: 5e3,
+  LATENCY_MIN_CALLS: 20,
+  LATENCY_TAIL_CALLS: 10,
+  LATENCY_TIMER_MULTIPLE: 20,
+  ALLOC_WINDOW_OPS: 256,
+  ALLOC_WINDOW_MS: 100,
+  ALLOC_SEMI_SPACE_MIB: 64,
+  PEAK_CALLS: 20,
+  LONG_RUN_ROUNDS: 30,
+  LONG_RUN_SETTLE_ROUNDS: 5,
+  LONG_RUN_ROUND_MS: 100,
+  LONG_RUN_GROWTH_BYTES: 64 * 1024,
+  GC_SETTLE_BYTES: 64 * 1024,
+  GC_SETTLE_ROUNDS: 2,
+  GC_MAX_ROUNDS: 7,
+  IMPORT_RUNS: 20,
+  REQUIRED_FALSE_ALARM_RATE: 0.01,
+  REQUIRED_CONFIDENCE: 0.95
+});
+var BenchError = class extends Error {
+  name = "BenchError";
+};
+var ENTRIES = Object.freeze({
+  child: fileURLToPath(new URL("./child-entry.mjs", import.meta.url)),
+  importer: fileURLToPath(new URL("./import-entry.mjs", import.meta.url))
+});
+function childEnv(env = process.env) {
+  const out = { ...env };
+  delete out["NODE_V8_COVERAGE"];
+  delete out["NODE_COMPILE_CACHE"];
+  return out;
+}
+function runNode(entry, args, execArgv, timeoutMs, execPath = process.execPath) {
+  return new Promise((resolve2, reject) => {
+    const child = spawn(execPath, [...execArgv, entry, ...args], {
+      stdio: ["ignore", "ignore", "pipe"],
+      env: childEnv()
+    });
+    let stderr = "";
+    let timedOut = false;
+    child.stderr.on("data", (chunk) => {
+      stderr = (stderr + String(chunk)).slice(-16384);
+    });
+    const timer = setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGKILL");
+    }, timeoutMs);
+    child.on("error", (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+    child.on("close", (code, signal) => {
+      clearTimeout(timer);
+      resolve2({ code, signal, stderr, timedOut });
+    });
+  });
+}
+function flagsFor(job) {
+  if (job.mode === "timing") return [];
+  if (job.mode === "heap") return ["--expose-gc"];
+  const mib = String(BENCH_CONTRACT.ALLOC_SEMI_SPACE_MIB);
+  return ["--expose-gc", `--min-semi-space-size=${mib}`, `--max-semi-space-size=${mib}`];
+}
+function describe(job) {
+  return `${job.mode} job for "${job.case.benchmark} @ ${job.case.size}"`;
+}
+async function runInChild(job, options = {}) {
+  const dir = mkdtempSync(join(tmpdir(), "cosyte-bench-"));
+  try {
+    const jobFile = join(dir, "job.json");
+    const outFile = join(dir, "out.json");
+    writeFileSync(jobFile, JSON.stringify(job));
+    const outcome = await runNode(
+      options.entry ?? ENTRIES.child,
+      [jobFile, outFile],
+      options.execArgv ?? flagsFor(job),
+      options.timeoutMs ?? 6e5
+    );
+    if (outcome.timedOut) throw new BenchError(`the ${describe(job)} timed out`);
+    let text;
+    try {
+      text = String(readFileSync(outFile));
+    } catch {
+      const how = outcome.signal === null ? `exit ${String(outcome.code)}` : outcome.signal;
+      throw new BenchError(
+        `the ${describe(job)} ended (${how}) without a result. Its stderr ends:
+${outcome.stderr}`
+      );
+    }
+    const out = JSON.parse(text);
+    if (!out.ok || out.result === void 0) {
+      throw new BenchError(`the ${describe(job)} could not measure: ${String(out.error)}`);
+    }
+    return out.result;
+  } finally {
+    rmSync(dir, { recursive: true });
+  }
+}
+
+// src/stats.ts
+function sortedCopy(xs) {
+  return [...xs].sort((a, b) => a - b);
+}
+function median(xs) {
+  const s = sortedCopy(xs);
+  const mid = Math.floor(s.length / 2);
+  return s.length % 2 === 1 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
+}
+function nearestRank(xs, q) {
+  if (xs.length === 0) return Number.NaN;
+  const s = sortedCopy(xs);
+  const rank = Math.max(1, Math.ceil(q * s.length));
+  return s[rank - 1];
+}
+function binomialCdf(k, n, p) {
+  const logP = Math.log(p);
+  const logQ = Math.log1p(-p);
+  let logCoef = 0;
+  let total = 0;
+  for (let i = 0; i <= Math.min(k, n); i++) {
+    if (i > 0) logCoef += Math.log(n - i + 1) - Math.log(i);
+    total += Math.exp(logCoef + i * logP + (n - i) * logQ);
+  }
+  return Math.min(1, total);
+}
+function signTestRank(n, alpha) {
+  let k = 0;
+  while (k < n && binomialCdf(k, n, 0.5) <= alpha) k++;
+  return k;
+}
+function signTestInterval(xs, alpha) {
+  const rank = signTestRank(xs.length, alpha);
+  if (rank === 0) return null;
+  const s = sortedCopy(xs);
+  return {
+    low: s[rank - 1],
+    high: s[s.length - rank],
+    rank,
+    tail: binomialCdf(rank - 1, xs.length, 0.5)
+  };
+}
+function rateUpperBound(k, n, confidence) {
+  if (n === 0) return 1;
+  const alpha = 1 - confidence;
+  let lo = k / n;
+  let hi = 1;
+  for (let i = 0; i < 100; i++) {
+    const mid = (lo + hi) / 2;
+    if (binomialCdf(k, n, mid) > alpha) lo = mid;
+    else hi = mid;
+  }
+  return hi;
+}
+function trialsForZeroEvents(rate, confidence) {
+  return Math.ceil(Math.log(1 - confidence) / Math.log1p(-rate));
+}
+function round(n, places = 4) {
+  const f = 10 ** places;
+  return Math.round(n * f) / f;
+}
+
+// src/import-cost.ts
+async function oneImport(entry, execArgv, mode, target, dir) {
+  const out = join(dir, "import.json");
+  rmSync(out, { force: true });
+  const outcome = await runNode(entry, [mode, target, out], execArgv, 12e4);
+  try {
+    return JSON.parse(String(readFileSync(out)));
+  } catch {
+    throw new BenchError(
+      `the cold import of ${relative(process.cwd(), target)} wrote no result. Its stderr ends:
+${outcome.stderr}`
+    );
+  }
+}
+async function measureImportCost(target, entry = ENTRIES.importer, execArgv = []) {
+  const dir = mkdtempSync(join(tmpdir(), "cosyte-bench-import-"));
+  try {
+    const empty = join(dir, "empty.mjs");
+    writeFileSync(empty, "export {};\n");
+    const cold = [];
+    const bare = [];
+    for (let i = 0; i < BENCH_CONTRACT.IMPORT_RUNS; i++) {
+      const order = i % 2 === 0 ? [target, empty] : [empty, target];
+      for (const which of order) {
+        const { ms } = await oneImport(entry, execArgv, "time", which, dir);
+        (which === target ? cold : bare).push(round(ms, 3));
+      }
+    }
+    const { files } = await oneImport(entry, execArgv, "files", target, dir);
+    const loaded = files.map((file) => {
+      const bytes2 = readFileSync(file);
+      return {
+        path: relative(process.cwd(), file),
+        bytes: bytes2.length,
+        gzipBytes: gzipSync(bytes2, { level: 9 }).length
+      };
+    }).sort((a, b) => a.path.localeCompare(b.path));
+    return {
+      runs: BENCH_CONTRACT.IMPORT_RUNS,
+      coldImportMs: cold,
+      medianMs: round(median(cold), 3),
+      p95Ms: round(nearestRank(cold, 0.95), 3),
+      emptyImportMs: bare,
+      emptyMedianMs: round(median(bare), 3),
+      files: loaded,
+      totalBytes: loaded.reduce((n, f) => n + f.bytes, 0),
+      totalGzipBytes: loaded.reduce((n, f) => n + f.gzipBytes, 0)
+    };
+  } finally {
+    rmSync(dir, { recursive: true });
+  }
+}
+
+// src/suite.ts
+function fail(message) {
+  throw new TypeError(`@cosyte/bench: ${message}`);
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null;
+}
+function checkName(value, what) {
+  if (typeof value !== "string" || value.trim() === "") fail(`${what} must be a non-empty string`);
+  return value;
+}
+function checkSuite(value) {
+  if (!isRecord(value)) fail("a suite must be an object made by defineSuite");
+  const name = checkName(value["name"], "the suite name");
+  const benchmarks = value["benchmarks"];
+  if (!Array.isArray(benchmarks) || benchmarks.length === 0) {
+    fail(`suite "${name}" must have at least one benchmark`);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const b of benchmarks) {
+    if (!isRecord(b)) fail(`suite "${name}" holds a benchmark that is not an object`);
+    const bName = checkName(b["name"], `a benchmark name in suite "${name}"`);
+    if (seen.has(bName)) fail(`suite "${name}" has two benchmarks named "${bName}"`);
+    seen.add(bName);
+    if (typeof b["bind"] !== "function") fail(`benchmark "${bName}" has no bind function`);
+    if (b["weigh"] !== void 0 && typeof b["weigh"] !== "function") {
+      fail(`benchmark "${bName}" has a weigh that is not a function`);
+    }
+    const sizes = b["sizes"];
+    if (!Array.isArray(sizes) || sizes.length === 0) fail(`benchmark "${bName}" has no sizes`);
+    const labels = /* @__PURE__ */ new Set();
+    for (const s of sizes) {
+      if (!isRecord(s)) fail(`benchmark "${bName}" holds a size that is not an object`);
+      const label = checkName(s["label"], `a size label in benchmark "${bName}"`);
+      if (labels.has(label)) fail(`benchmark "${bName}" has two sizes labelled "${label}"`);
+      labels.add(label);
+      if (typeof s["build"] !== "function")
+        fail(`size "${label}" of "${bName}" has no build function`);
+      if (s["compare"] !== void 0 && typeof s["compare"] !== "boolean") {
+        fail(`size "${label}" of "${bName}" has a compare that is not true or false`);
+      }
+    }
+    if (b["longRun"] !== void 0 && !labels.has(b["longRun"])) {
+      fail(`benchmark "${bName}" names a longRun size it does not have`);
+    }
+  }
+  return Object.freeze(value);
+}
+function listCases(suite, only = [], forComparison = false) {
+  const all = [];
+  for (const b of suite.benchmarks) {
+    for (const s of b.sizes) {
+      if (!forComparison || s.compare !== false) all.push({ benchmark: b.name, size: s.label });
+    }
+  }
+  if (only.length === 0) return all;
+  for (const o of only) {
+    if (!all.some((c) => c.benchmark === o || caseId(c) === o)) {
+      fail(`--only "${o}" names no benchmark or case in suite "${suite.name}"`);
+    }
+  }
+  return all.filter((c) => only.includes(c.benchmark) || only.includes(caseId(c)));
+}
+function caseId(c) {
+  return `${c.benchmark} @ ${c.size}`;
+}
+function findCase(suite, c) {
+  const benchmark = suite.benchmarks.find((b) => b.name === c.benchmark);
+  const size = benchmark?.sizes.find((s) => s.label === c.size);
+  if (size === void 0) fail(`suite "${suite.name}" has no case "${caseId(c)}"`);
+  return { benchmark, size };
+}
+
+// src/compare.ts
+function judge(ratios, threshold = BENCH_CONTRACT.THRESHOLD, alpha = BENCH_CONTRACT.ALPHA) {
+  const interval = signTestInterval(ratios, alpha);
+  const medianRatio = median(ratios);
+  if (interval === null) return { verdict: "inconclusive", interval, medianRatio };
+  const up = 1 + threshold;
+  let verdict;
+  if (interval.low > up) verdict = "regression";
+  else if (interval.high < 1 / up) verdict = "improvement";
+  else if (interval.high <= up) verdict = "no-regression";
+  else verdict = "inconclusive";
+  return { verdict, interval, medianRatio };
+}
+async function compareBuilds(options) {
+  const runJob = options.runJob;
+  const inject = options.inject ?? 0;
+  const cases = listCases(options.suite, options.only, true);
+  const state = cases.map((c) => ({
+    c,
+    rounds: [],
+    skip: null
+  }));
+  for (let r = 0; r < BENCH_CONTRACT.ROUNDS; r++) {
+    for (const s of state) {
+      if (s.skip !== null) continue;
+      const sides = r % 2 === 0 ? ["base", "head"] : ["head", "base"];
+      const got = /* @__PURE__ */ new Map();
+      for (const side of sides) {
+        const result = await runJob({
+          mode: "timing",
+          suite: options.suiteFile,
+          target: side === "base" ? options.base : options.head,
+          case: s.c,
+          latency: false,
+          inject: side === "head" ? inject : 0
+        });
+        if (result.mode !== "timing")
+          throw new Error("@cosyte/bench: a timing job returned another kind");
+        if (result.status === "skipped") {
+          s.skip = `${side}: ${result.reason}`;
+          break;
+        }
+        got.set(side, result.samplesMsPerOp);
+      }
+      const base = got.get("base");
+      const head = got.get("head");
+      if (base !== void 0 && head !== void 0) s.rounds.push({ base, head });
+    }
+    options.onProgress?.(`round ${String(r + 1)} of ${String(BENCH_CONTRACT.ROUNDS)} done`);
+  }
+  return {
+    rounds: BENCH_CONTRACT.ROUNDS,
+    threshold: BENCH_CONTRACT.THRESHOLD,
+    alpha: BENCH_CONTRACT.ALPHA,
+    inject,
+    cases: state.map(({ c, rounds, skip }) => {
+      const recorded = {
+        baseMsPerOp: rounds.map((p) => Math.min(...p.base)),
+        headMsPerOp: rounds.map((p) => Math.min(...p.head)),
+        baseSamples: rounds.map((p) => p.base),
+        headSamples: rounds.map((p) => p.head)
+      };
+      if (skip !== null) {
+        return {
+          ...c,
+          ...recorded,
+          ratios: [],
+          medianRatio: null,
+          interval: null,
+          verdict: "skipped",
+          skipReason: skip
+        };
+      }
+      const ratios = rounds.map((p) => round(Math.min(...p.head) / Math.min(...p.base), 6));
+      const j = judge(ratios);
+      return {
+        ...c,
+        ...recorded,
+        ratios,
+        medianRatio: round(j.medianRatio, 6),
+        interval: j.interval,
+        verdict: j.verdict,
+        skipReason: null
+      };
+    })
+  };
+}
+function comparisonExitCode(body) {
+  if (body.cases.some((c) => c.verdict === "regression")) return 1;
+  if (body.cases.some((c) => c.verdict === "skipped")) return 2;
+  return 0;
+}
+function counted(events, trials) {
+  return {
+    events,
+    trials,
+    rate: trials === 0 ? 0 : round(events / trials, 6),
+    upperBound: round(rateUpperBound(events, trials, BENCH_CONTRACT.REQUIRED_CONFIDENCE), 6)
+  };
+}
+function summarizeCalibration(inject, runs) {
+  const verdicts = {
+    regression: 0,
+    improvement: 0,
+    "no-regression": 0,
+    inconclusive: 0,
+    skipped: 0
+  };
+  const pooled = /* @__PURE__ */ new Map();
+  let caseTrials = 0;
+  let caseEvents = 0;
+  let compTrials = 0;
+  let compEvents = 0;
+  let compSkipped = 0;
+  for (const run of runs) {
+    let alarmed = false;
+    let skipped2 = false;
+    for (const c of run) {
+      verdicts[c.verdict]++;
+      const id = caseId(c);
+      const pool = pooled.get(id) ?? { ref: { benchmark: c.benchmark, size: c.size }, ratios: [] };
+      pooled.set(id, pool);
+      if (c.verdict === "skipped") {
+        skipped2 = true;
+        continue;
+      }
+      pool.ratios.push(...c.ratios);
+      caseTrials++;
+      if (c.verdict === "regression") {
+        caseEvents++;
+        alarmed = true;
+      }
+    }
+    if (skipped2) compSkipped++;
+    if (alarmed || !skipped2) compTrials++;
+    if (alarmed) compEvents++;
+  }
+  const perComparison = counted(compEvents, compTrials);
+  const skippedComparisons = counted(compSkipped, runs.length);
+  const line = BENCH_CONTRACT.REQUIRED_FALSE_ALARM_RATE;
+  const needed = trialsForZeroEvents(line, BENCH_CONTRACT.REQUIRED_CONFIDENCE);
+  const pct2 = (x) => `${String(round(x * 100, 4))}%`;
+  const alarmsUnder = perComparison.upperBound < line;
+  const skipsUnder = skippedComparisons.upperBound < line;
+  const alarms = `${String(compEvents)} false alarm(s) in ${String(compTrials)} A/A comparisons that measured every case or called a regression`;
+  const skips = `${String(compSkipped)} of ${String(runs.length)} comparisons skipped a case`;
+  let requiredRule;
+  if (inject !== 0) {
+    requiredRule = {
+      applies: false,
+      met: false,
+      reason: "a calibration with an injected slowdown measures detection, not false alarms"
+    };
+  } else if (alarmsUnder && skipsUnder) {
+    requiredRule = {
+      applies: true,
+      met: true,
+      reason: `${alarms}, and ${skips}: both rates are under ${pct2(line)} with ${pct2(BENCH_CONTRACT.REQUIRED_CONFIDENCE)} confidence (upper bounds ${pct2(perComparison.upperBound)} and ${pct2(skippedComparisons.upperBound)})`
+    };
+  } else {
+    const why = [];
+    if (!alarmsUnder) {
+      why.push(
+        `${alarms}: the upper bound ${pct2(perComparison.upperBound)} is not under ${pct2(line)}, and a clean record needs at least ${String(needed)} of them`
+      );
+    }
+    if (!skipsUnder) {
+      why.push(
+        `${skips}, and a required check is red on each: the upper bound ${pct2(skippedComparisons.upperBound)} on that rate is not under ${pct2(line)}`
+      );
+    }
+    requiredRule = { applies: true, met: false, reason: why.join("; ") };
+  }
+  return {
+    inject,
+    comparisons: runs.length,
+    runs,
+    perCase: counted(caseEvents, caseTrials),
+    perComparison,
+    skippedComparisons,
+    verdicts,
+    observed: [...pooled.values()].map(({ ref, ratios }) => ({
+      ...ref,
+      rounds: ratios.length,
+      medianRatio: ratios.length === 0 ? null : round(median(ratios), 6)
+    })),
+    requiredRule
+  };
+}
+async function calibrateComparison(options) {
+  const runs = [];
+  for (let k = 0; k < options.comparisons; k++) {
+    const { onProgress: _quiet, ...quiet } = options;
+    const body = await compareBuilds(quiet);
+    runs.push([...body.cases]);
+    const alarms = body.cases.filter((c) => c.verdict === "regression").map(caseId);
+    options.onProgress?.(
+      `comparison ${String(k + 1)} of ${String(options.comparisons)}: ` + (alarms.length === 0 ? "no regression verdict" : `regression on ${alarms.join(", ")}`)
+    );
+  }
+  return summarizeCalibration(options.inject ?? 0, runs);
+}
+
+// src/reading.ts
+function expectTiming(result) {
+  if (result.mode !== "timing")
+    throw new Error("@cosyte/bench: a timing job returned another kind");
+  return result;
+}
+function expectMemory(result) {
+  if (result.mode === "timing")
+    throw new Error("@cosyte/bench: a memory job returned a timing result");
+  return result;
+}
+function summarizeThroughput(first, rest) {
+  const measured = [];
+  for (const t of [first, ...rest]) {
+    if (t.status === "skipped") return { status: "skipped", reason: t.reason };
+    measured.push(t);
+  }
+  const per = measured.map((t) => ({
+    samplesMsPerOp: t.samplesMsPerOp,
+    medianMsPerOp: t.medianMsPerOp
+  }));
+  const mid = median(per.map((x) => x.medianMsPerOp));
+  const perOpBytes = first.bytesPerPass === null ? null : first.bytesPerPass / first.inputsPerPass;
+  return {
+    status: "measured",
+    processes: per,
+    medianMsPerOp: mid,
+    opsPerSecond: round(1e3 / mid, 2),
+    bytesPerSecond: perOpBytes === null ? null : round(perOpBytes * (1e3 / mid), 0)
+  };
+}
+async function readBuild(options) {
+  const runJob = options.runJob;
+  const processes = options.processes ?? 3;
+  if (!Number.isInteger(processes) || processes < 1) {
+    throw new TypeError("@cosyte/bench: processes must be a positive integer");
+  }
+  const base = { suite: options.suiteFile, target: options.target };
+  const cases = [];
+  for (const c of listCases(options.suite, options.only)) {
+    const timing = async (latency2) => expectTiming(await runJob({ mode: "timing", ...base, case: c, latency: latency2, inject: 0 }));
+    const first = await timing(true);
+    const rest = [];
+    for (let p = 1; p < processes; p++) rest.push(await timing(false));
+    const throughput = summarizeThroughput(first, rest);
+    const latency = first.status === "measured" ? first.latency : null;
+    let allocations = null;
+    let peak = null;
+    let longRun = null;
+    if (options.memory ?? true) {
+      allocations = expectMemory(await runJob({ mode: "alloc", ...base, case: c })).allocations;
+      const { benchmark } = findCase(options.suite, c);
+      const longRunSize = benchmark.longRun ?? benchmark.sizes[0].label;
+      const heap = expectMemory(
+        await runJob({ mode: "heap", ...base, case: c, longRun: c.size === longRunSize })
+      );
+      peak = heap.peak;
+      longRun = heap.longRun;
+    }
+    cases.push({
+      ...c,
+      inputsPerPass: first.inputsPerPass,
+      bytesPerPass: first.bytesPerPass,
+      throughput,
+      latency,
+      allocations,
+      peak,
+      longRun
+    });
+    options.onProgress?.(`read ${c.benchmark} @ ${c.size}`);
+  }
+  const importCost = options.importCost ?? true ? await options.measureImport(options.target) : null;
+  return { processes, cases, importCost };
+}
+
+// src/report.ts
+function pct(x) {
+  const v = round(x * 100, 1);
+  return `${v > 0 ? "+" : ""}${String(v)}%`;
+}
+function bytes(n) {
+  if (Math.abs(n) >= 1024 * 1024) return `${String(round(n / (1024 * 1024), 2))} MiB`;
+  if (Math.abs(n) >= 1024) return `${String(round(n / 1024, 1))} KiB`;
+  return `${String(round(n, 1))} B`;
+}
+function share(x) {
+  return `${String(round(x * 100, 2))}%`;
+}
+function time(ms) {
+  if (ms >= 1) return `${String(round(ms, 3))} ms`;
+  if (ms >= 1e-3) return `${String(round(ms * 1e3, 2))} us`;
+  return `${String(round(ms * 1e6, 1))} ns`;
+}
+function ns(n) {
+  return n === null ? "n/a" : time(n / 1e6);
+}
+function header(doc, title) {
+  const e = doc.environment;
+  return [
+    `### ${title}: ${doc.suite.name}`,
+    "",
+    `Node ${e.node} on ${e.runner.class} (${e.cpuModel ?? "unknown CPU"}, ${String(e.logicalCpus)} logical CPUs), @cosyte/bench ${doc.tool.version}, ${doc.createdAt}.`,
+    ""
+  ];
+}
+function comparisonRow(c) {
+  const base = c.baseMsPerOp.length === 0 ? "n/a" : time(c.baseMsPerOp.reduce((a, b) => a + b, 0) / c.baseMsPerOp.length);
+  const head = c.headMsPerOp.length === 0 ? "n/a" : time(c.headMsPerOp.reduce((a, b) => a + b, 0) / c.headMsPerOp.length);
+  const change = c.medianRatio === null ? "n/a" : pct(c.medianRatio - 1);
+  const interval = c.interval === null ? "n/a" : `${pct(c.interval.low - 1)} to ${pct(c.interval.high - 1)}`;
+  const verdict = c.verdict === "skipped" ? `skipped (${String(c.skipReason)})` : c.verdict;
+  return `| ${c.benchmark} | ${c.size} | ${base} | ${head} | ${change} | ${interval} | ${verdict} |`;
+}
+function renderMarkdown(doc) {
+  const lines = [];
+  if (doc.kind === "comparison") {
+    const b = doc.comparison;
+    lines.push(...header(doc, "Base against head"));
+    lines.push(
+      `${String(b.rounds)} interleaved rounds, a fresh process per side per case per round. A regression is a change whose whole interval is above ${pct(b.threshold)}; each end of the interval is wrong at most ${String(round(b.alpha * 100, 1))}% of the time. Base and head times are means of the per-round fastest samples.`,
+      "",
+      "| Benchmark | Size | Base | Head | Change (median) | Interval | Verdict |",
+      "| --- | --- | --- | --- | --- | --- | --- |",
+      ...b.cases.map(comparisonRow)
+    );
+  } else if (doc.kind === "calibration") {
+    const c = doc.calibration;
+    lines.push(
+      ...header(
+        doc,
+        c.inject === 0 ? "A/A calibration" : `Calibration with a ${share(c.inject)} slowdown`
+      )
+    );
+    const what = c.inject === 0 ? "false alarms" : "detections";
+    lines.push(
+      `${String(c.comparisons)} comparisons. Regression verdicts (${what}): ${String(c.perCase.events)} of ${String(c.perCase.trials)} measured case verdicts (${share(c.perCase.rate)}, upper bound ${share(c.perCase.upperBound)}); ${String(c.perComparison.events)} of ${String(c.perComparison.trials)} comparisons that measured every case or called a regression (${share(c.perComparison.rate)}, upper bound ${share(c.perComparison.upperBound)}). Comparisons that skipped a case: ${String(c.skippedComparisons.events)} of ${String(c.skippedComparisons.trials)} (${share(c.skippedComparisons.rate)}, upper bound ${share(c.skippedComparisons.upperBound)}).`,
+      "",
+      `Verdicts: ${Object.entries(c.verdicts).map(([k, v]) => `${k} ${String(v)}`).join(", ")}.`,
+      "",
+      "The change each case was observed at, the median of every measured round's head / base:",
+      "",
+      "| Benchmark | Size | Measured rounds | Observed change |",
+      "| --- | --- | --- | --- |",
+      ...c.observed.map(
+        (o) => `| ${o.benchmark} | ${o.size} | ${String(o.rounds)} | ${o.medianRatio === null ? "n/a" : pct(o.medianRatio - 1)} |`
+      )
+    );
+    if (c.requiredRule.applies) {
+      lines.push(
+        "",
+        `Required check rule: ${c.requiredRule.met ? "met" : "not met"}. ${c.requiredRule.reason}.`
+      );
+    }
+  } else {
+    const r = doc.reading;
+    lines.push(...header(doc, "Reading"));
+    lines.push(
+      `Throughput is the median of ${String(r.processes)} fresh processes. Latency is per call, nearest rank. Memory: allocations per operation, the peak heap of one call above the settled heap, and whether the settled heap stays flat over a long run.`,
+      "",
+      "| Benchmark | Size | Per op | Ops/s | Bytes/s | p50 | p95 | p99 | Alloc/op | Peak heap | Long run |",
+      "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
+    );
+    for (const c of r.cases) {
+      const t = c.throughput;
+      const perOp = t.status === "measured" ? time(t.medianMsPerOp) : `skipped (${t.reason})`;
+      const ops = t.status === "measured" ? String(t.opsPerSecond) : "n/a";
+      const bps = t.status === "measured" && t.bytesPerSecond !== null ? `${bytes(t.bytesPerSecond)}/s` : "n/a";
+      const l = c.latency;
+      const lat = l === null ? ["n/a", "n/a", "n/a"] : l.status === "measured" ? [ns(l.p50Ns), ns(l.p95Ns), ns(l.p99Ns)] : [`skipped (${l.reason})`, "n/a", "n/a"];
+      const a = c.allocations;
+      const alloc = a === null ? "n/a" : a.status === "measured" ? bytes(a.heapBytesPerOp + a.externalBytesPerOp) : `skipped (${a.reason})`;
+      const p = c.peak;
+      const peak = p === null ? "n/a" : p.status === "measured" ? bytes(p.peakHeapBytes) : `skipped (${p.reason})`;
+      const lr = c.longRun;
+      const long = lr === null ? "" : lr.status === "measured" ? `${lr.verdict} (${bytes(lr.growthBytes)} over ${String(lr.ops)} ops, ${String(lr.slopeBytesPerOp)} B/op)` : `skipped (${lr.reason})`;
+      lines.push(
+        `| ${c.benchmark} | ${c.size} | ${perOp} | ${ops} | ${bps} | ${lat.join(" | ")} | ${alloc} | ${peak} | ${long} |`
+      );
+    }
+    if (r.importCost !== null) {
+      const i = r.importCost;
+      lines.push(
+        "",
+        `Cold import: median ${time(i.medianMs)}, p95 ${time(i.p95Ms)} over ${String(i.runs)} fresh processes (an empty module: median ${time(i.emptyMedianMs)}). Loaded: ${String(i.files.length)} file(s), ${bytes(i.totalBytes)}, ${bytes(i.totalGzipBytes)} gzipped.`
+      );
+    }
+  }
+  return `${lines.join("\n")}
+`;
+}
+
+// ../test-utils/src/perf/contract.ts
+var PERF_CONTRACT = Object.freeze({
+  RATIO_CEILING: 8,
+  RATIO_FLOOR: 1.5,
+  REPS: 5,
+  SCALE_STEP: 4,
+  MIN_PHASE_MS: 4,
+  WARMUP_MIN_MS: 500,
+  WARMUP_BATCH_MS: 50,
+  WARMUP_MAX_MS: 5e3,
+  WARMUP_STABLE_TOL: 0.05,
+  WARMUP_STABLE_BATCHES: 3
+});
+
+// src/measure.ts
+var WARMUP = Object.freeze({
+  MIN_MS: PERF_CONTRACT.WARMUP_MIN_MS,
+  BATCH_MS: PERF_CONTRACT.WARMUP_BATCH_MS,
+  MAX_MS: PERF_CONTRACT.WARMUP_MAX_MS,
+  STABLE_TOL: PERF_CONTRACT.WARMUP_STABLE_TOL,
+  STABLE_BATCHES: PERF_CONTRACT.WARMUP_STABLE_BATCHES
+});
+
+// src/results.ts
+var SCHEMA_VERSION = 1;
+function toolVersion() {
+  const require2 = createRequire(import.meta.url);
+  return require2("../package.json").version;
+}
+function documentBase(suite, environment2, now = /* @__PURE__ */ new Date()) {
+  return {
+    schemaVersion: SCHEMA_VERSION,
+    tool: { name: "@cosyte/bench", version: toolVersion() },
+    createdAt: now.toISOString(),
+    suite,
+    environment: environment2,
+    contract: { bench: BENCH_CONTRACT, warmup: WARMUP }
+  };
+}
+var num = { type: "number" };
+var int = { type: "integer", minimum: 0 };
+var str = { type: "string" };
+var strOrNull = { type: ["string", "null"] };
+var numOrNull = { type: ["number", "null"] };
+var nums = { type: "array", items: num };
+var skipped = {
+  type: "object",
+  required: ["status", "reason"],
+  properties: { status: { const: "skipped" }, reason: str }
+};
+var measuredOrSkipped = (measured) => ({ oneOf: [measured, skipped] });
+var orNull = (s) => ({ oneOf: [{ type: "null" }, s] });
+var RESULTS_SCHEMA = {
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  $id: "https://github.com/cosyte/config/blob/main/packages/bench/schema/results-v1.schema.json",
+  title: "@cosyte/bench results, version 1",
+  type: "object",
+  required: ["schemaVersion", "kind", "tool", "createdAt", "suite", "environment", "contract"],
+  properties: {
+    schemaVersion: { const: 1 },
+    kind: { enum: ["reading", "comparison", "calibration"] },
+    tool: {
+      type: "object",
+      required: ["name", "version"],
+      properties: { name: { const: "@cosyte/bench" }, version: str }
+    },
+    createdAt: str,
+    suite: { type: "object", required: ["name", "file"], properties: { name: str, file: str } },
+    environment: { $ref: "#/$defs/environment" },
+    contract: {
+      type: "object",
+      required: ["bench", "warmup"],
+      properties: { bench: { type: "object" }, warmup: { type: "object" } }
+    }
+  },
+  oneOf: [
+    {
+      required: ["subject", "reading"],
+      properties: {
+        kind: { const: "reading" },
+        subject: { $ref: "#/$defs/build" },
+        reading: { $ref: "#/$defs/reading" }
+      }
+    },
+    {
+      required: ["base", "head", "comparison"],
+      properties: {
+        kind: { const: "comparison" },
+        base: { $ref: "#/$defs/build" },
+        head: { $ref: "#/$defs/build" },
+        comparison: { $ref: "#/$defs/comparison" }
+      }
+    },
+    {
+      required: ["base", "head", "calibration"],
+      properties: {
+        kind: { const: "calibration" },
+        base: { $ref: "#/$defs/build" },
+        head: { $ref: "#/$defs/build" },
+        calibration: { $ref: "#/$defs/calibration" }
+      }
+    }
+  ],
+  $defs: {
+    build: {
+      type: "object",
+      required: ["target", "commit"],
+      properties: { target: str, commit: strOrNull }
+    },
+    environment: {
+      type: "object",
+      required: [
+        "node",
+        "v8",
+        "platform",
+        "arch",
+        "osRelease",
+        "cpuModel",
+        "logicalCpus",
+        "totalMemoryBytes",
+        "cgroupCpuMax",
+        "runner"
+      ],
+      properties: {
+        node: str,
+        v8: str,
+        platform: str,
+        arch: str,
+        osRelease: str,
+        cpuModel: strOrNull,
+        logicalCpus: int,
+        totalMemoryBytes: int,
+        cgroupCpuMax: strOrNull,
+        // Added within version 1, so not required: a document written before it is still valid.
+        loadAverage: nums,
+        runner: {
+          type: "object",
+          required: ["ci", "provider", "class"],
+          properties: { ci: { type: "boolean" }, provider: strOrNull, class: str }
+        }
+      }
+    },
+    interval: orNull({
+      type: "object",
+      required: ["low", "high", "rank", "tail"],
+      properties: { low: num, high: num, rank: int, tail: num }
+    }),
+    caseComparison: {
+      type: "object",
+      required: [
+        "benchmark",
+        "size",
+        "baseMsPerOp",
+        "headMsPerOp",
+        "ratios",
+        "medianRatio",
+        "interval",
+        "verdict",
+        "skipReason"
+      ],
+      properties: {
+        benchmark: str,
+        size: str,
+        baseMsPerOp: nums,
+        headMsPerOp: nums,
+        ratios: nums,
+        medianRatio: numOrNull,
+        interval: { $ref: "#/$defs/interval" },
+        verdict: {
+          enum: ["regression", "improvement", "no-regression", "inconclusive", "skipped"]
+        },
+        skipReason: strOrNull
+      }
+    },
+    comparison: {
+      type: "object",
+      required: ["rounds", "threshold", "alpha", "inject", "cases"],
+      properties: {
+        rounds: int,
+        threshold: num,
+        alpha: num,
+        inject: num,
+        cases: { type: "array", items: { $ref: "#/$defs/caseComparison" } }
+      }
+    },
+    rate: {
+      type: "object",
+      required: ["events", "trials", "rate", "upperBound"],
+      properties: { events: int, trials: int, rate: num, upperBound: num }
+    },
+    calibration: {
+      type: "object",
+      required: [
+        "inject",
+        "comparisons",
+        "runs",
+        "perCase",
+        "perComparison",
+        "verdicts",
+        "requiredRule"
+      ],
+      properties: {
+        inject: num,
+        comparisons: int,
+        runs: {
+          type: "array",
+          items: { type: "array", items: { $ref: "#/$defs/caseComparison" } }
+        },
+        perCase: { $ref: "#/$defs/rate" },
+        perComparison: { $ref: "#/$defs/rate" },
+        // Added within version 1, so optional: documents written before it validate unchanged.
+        skippedComparisons: { $ref: "#/$defs/rate" },
+        verdicts: { type: "object" },
+        observed: {
+          type: "array",
+          items: {
+            type: "object",
+            required: ["benchmark", "size", "rounds", "medianRatio"],
+            properties: { benchmark: str, size: str, rounds: int, medianRatio: numOrNull }
+          }
+        },
+        requiredRule: {
+          type: "object",
+          required: ["applies", "met", "reason"],
+          properties: { applies: { type: "boolean" }, met: { type: "boolean" }, reason: str }
+        }
+      }
+    },
+    reading: {
+      type: "object",
+      required: ["processes", "cases", "importCost"],
+      properties: {
+        processes: int,
+        cases: { type: "array", items: { $ref: "#/$defs/caseReading" } },
+        importCost: orNull({
+          type: "object",
+          required: [
+            "runs",
+            "coldImportMs",
+            "medianMs",
+            "p95Ms",
+            "emptyImportMs",
+            "emptyMedianMs",
+            "files",
+            "totalBytes",
+            "totalGzipBytes"
+          ],
+          properties: {
+            runs: int,
+            coldImportMs: nums,
+            medianMs: num,
+            p95Ms: num,
+            emptyImportMs: nums,
+            emptyMedianMs: num,
+            files: {
+              type: "array",
+              items: {
+                type: "object",
+                required: ["path", "bytes", "gzipBytes"],
+                properties: { path: str, bytes: int, gzipBytes: int }
+              }
+            },
+            totalBytes: int,
+            totalGzipBytes: int
+          }
+        })
+      }
+    },
+    caseReading: {
+      type: "object",
+      required: [
+        "benchmark",
+        "size",
+        "inputsPerPass",
+        "bytesPerPass",
+        "throughput",
+        "latency",
+        "allocations",
+        "peak",
+        "longRun"
+      ],
+      properties: {
+        benchmark: str,
+        size: str,
+        inputsPerPass: int,
+        bytesPerPass: numOrNull,
+        throughput: measuredOrSkipped({
+          type: "object",
+          required: ["status", "processes", "medianMsPerOp", "opsPerSecond", "bytesPerSecond"],
+          properties: {
+            status: { const: "measured" },
+            processes: {
+              type: "array",
+              items: {
+                type: "object",
+                required: ["samplesMsPerOp", "medianMsPerOp"],
+                properties: { samplesMsPerOp: nums, medianMsPerOp: num }
+              }
+            },
+            medianMsPerOp: num,
+            opsPerSecond: num,
+            bytesPerSecond: numOrNull
+          }
+        }),
+        latency: orNull({
+          type: "object",
+          required: ["status", "calls", "timerOverheadNs", "p50Ns"],
+          properties: {
+            status: { enum: ["measured", "skipped"] },
+            calls: int,
+            timerOverheadNs: num,
+            p50Ns: numOrNull,
+            p95Ns: numOrNull,
+            p99Ns: numOrNull,
+            maxNs: num
+          }
+        }),
+        allocations: orNull(
+          measuredOrSkipped({
+            type: "object",
+            required: ["status", "windowOps", "heapBytesPerOp", "externalBytesPerOp"],
+            properties: {
+              status: { const: "measured" },
+              windowOps: int,
+              heapBytesPerOp: num,
+              externalBytesPerOp: num
+            }
+          })
+        ),
+        peak: orNull(
+          measuredOrSkipped({
+            type: "object",
+            required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
+            properties: {
+              status: { const: "measured" },
+              calls: int,
+              peakHeapBytes: num,
+              medianPeakHeapBytes: num
+            }
+          })
+        ),
+        longRun: orNull(
+          measuredOrSkipped({
+            type: "object",
+            required: ["status", "verdict", "ops", "opsPerRound", "settledHeap", "slopeBytesPerOp"],
+            properties: {
+              status: { const: "measured" },
+              verdict: { enum: ["flat", "growing"] },
+              ops: int,
+              opsPerRound: int,
+              settledHeap: nums,
+              slopeBytesPerOp: num
+            }
+          })
+        )
+      }
+    }
+  }
+};
+function typeOf(value) {
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "array";
+  if (Number.isInteger(value)) return "integer";
+  return typeof value;
+}
+function typeMatches(value, type) {
+  const actual = typeOf(value);
+  return actual === type || type === "number" && actual === "integer";
+}
+function resolveRef(ref, root) {
+  const target = root.$defs?.[ref.replace("#/$defs/", "")];
+  if (target === void 0) throw new Error(`schema has no definition ${ref}`);
+  return target;
+}
+function validate(value, schema, root = schema, at = "") {
+  const problems = [];
+  const here = at === "" ? "/" : at;
+  if (typeof schema.$ref === "string")
+    return validate(value, resolveRef(schema.$ref, root), root, at);
+  if ("const" in schema && value !== schema.const)
+    problems.push(`${here}: must be ${JSON.stringify(schema.const)}`);
+  if (schema.enum !== void 0 && !schema.enum.includes(value)) {
+    problems.push(
+      `${here}: must be one of ${schema.enum.map((e) => JSON.stringify(e)).join(", ")}`
+    );
+  }
+  if (schema.type !== void 0) {
+    const types = typeof schema.type === "string" ? [schema.type] : schema.type;
+    if (!types.some((t) => typeMatches(value, t))) {
+      problems.push(`${here}: must be ${types.join(" or ")}, is ${typeOf(value)}`);
+      return problems;
+    }
+  }
+  if (typeof value === "number" && value < (schema.minimum ?? -Infinity)) {
+    problems.push(`${here}: must be at least ${String(schema.minimum)}`);
+  }
+  if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+    const record = value;
+    for (const key of schema.required ?? []) {
+      if (!(key in record)) problems.push(`${here}: is missing "${key}"`);
+    }
+    for (const [key, sub] of Object.entries(schema.properties ?? {})) {
+      if (key in record) problems.push(...validate(record[key], sub, root, `${at}/${key}`));
+    }
+  }
+  if (Array.isArray(value) && schema.items !== void 0) {
+    const items = schema.items;
+    value.forEach((item, i) => problems.push(...validate(item, items, root, `${at}/${String(i)}`)));
+  }
+  if (schema.oneOf !== void 0) {
+    const results = schema.oneOf.map((s) => validate(value, s, root, at));
+    const passing = results.filter((r) => r.length === 0).length;
+    if (passing !== 1) {
+      problems.push(
+        `${here}: must match exactly one of ${String(results.length)} shapes, matches ${String(passing)}`
+      );
+      problems.push(...results.reduce((a, b) => b.length < a.length ? b : a));
+    }
+  }
+  return problems;
+}
+function readResults(text) {
+  let value;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    throw new TypeError("@cosyte/bench: the results are not JSON");
+  }
+  const version = value?.schemaVersion;
+  if (version !== SCHEMA_VERSION) {
+    throw new TypeError(
+      `@cosyte/bench: results version ${String(version)} is not one this package reads (${String(SCHEMA_VERSION)})`
+    );
+  }
+  const problems = validate(value, RESULTS_SCHEMA);
+  if (problems.length > 0)
+    throw new TypeError(
+      `@cosyte/bench: the results do not match the schema:
+${problems.join("\n")}`
+    );
+  return value;
+}
+
+// src/main.ts
+var USAGE = `Usage:
+  cosyte-bench run --suite <file> --target <file> [--processes <n>] [--no-memory] [--no-import]
+                   [--commit <sha>] [--only <name>]... [--out <file.json>] [--summary <file.md>]
+  cosyte-bench compare --suite <file> --base <file> --head <file> [--base-commit <sha>]
+                   [--head-commit <sha>] [--only <name>]... [--out <file.json>] [--summary <file.md>]
+  cosyte-bench calibrate --suite <file> --base <file> --head <file> --comparisons <n>
+                   [--inject <fraction>] [--only <name>]... [--out <file.json>] [--summary <file.md>]
+  cosyte-bench check <results.json>
+
+Exit codes: 0 clean, 1 a finding (a regression, or results that do not match the schema),
+2 could not measure or could not read its input.
+`;
+var UsageError = class extends Error {
+};
+var VALUE_FLAGS = /* @__PURE__ */ new Set([
+  "--suite",
+  "--target",
+  "--base",
+  "--head",
+  "--out",
+  "--summary",
+  "--commit",
+  "--base-commit",
+  "--head-commit",
+  "--processes",
+  "--comparisons",
+  "--inject",
+  "--only"
+]);
+var BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["--no-memory", "--no-import"]);
+var FLAGS_FOR = {
+  run: [
+    "--suite",
+    "--target",
+    "--processes",
+    "--no-memory",
+    "--no-import",
+    "--commit",
+    "--only",
+    "--out",
+    "--summary"
+  ],
+  compare: [
+    "--suite",
+    "--base",
+    "--head",
+    "--base-commit",
+    "--head-commit",
+    "--only",
+    "--out",
+    "--summary"
+  ],
+  calibrate: [
+    "--suite",
+    "--base",
+    "--head",
+    "--comparisons",
+    "--inject",
+    "--only",
+    "--out",
+    "--summary"
+  ]
+};
+function parseArgs(argv) {
+  const values = /* @__PURE__ */ new Map();
+  const flags = /* @__PURE__ */ new Set();
+  const positional = [];
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (VALUE_FLAGS.has(arg)) {
+      const value = argv[i + 1];
+      if (value === void 0 || value.startsWith("--"))
+        throw new UsageError(`${arg} needs a value`);
+      values.set(arg, [...values.get(arg) ?? [], value]);
+      i++;
+    } else if (BOOLEAN_FLAGS.has(arg)) {
+      flags.add(arg);
+    } else if (arg.startsWith("--")) {
+      throw new UsageError(`unknown flag ${arg}`);
+    } else {
+      positional.push(arg);
+    }
+  }
+  return { values, flags, positional };
+}
+function one(args, flag) {
+  const all = args.values.get(flag);
+  if (all !== void 0 && all.length > 1) throw new UsageError(`${flag} given more than once`);
+  return all?.[0];
+}
+function required(args, flag) {
+  const value = one(args, flag);
+  if (value === void 0) throw new UsageError(`${flag} is required`);
+  return value;
+}
+function count(args, flag, fallback) {
+  const raw = one(args, flag);
+  if (raw === void 0) return fallback;
+  const n = Number(raw);
+  if (!Number.isInteger(n) || n < 1) throw new UsageError(`${flag} must be a positive integer`);
+  return n;
+}
+function fraction(args, flag) {
+  const raw = one(args, flag);
+  if (raw === void 0) return 0;
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n < 0 || n > 1)
+    throw new UsageError(`${flag} must be a fraction from 0 to 1`);
+  return n;
+}
+async function loadSuite(file) {
+  const module = await import(pathToFileURL(file).href);
+  return checkSuite(module.default);
+}
+async function main(argv, io) {
+  const [command, ...rest] = argv;
+  try {
+    if (command === void 0 || command === "--help" || command === "help") {
+      io.stdout(USAGE);
+      return command === void 0 ? 2 : 0;
+    }
+    const args = parseArgs(rest);
+    const abs = (p) => resolve(io.cwd, p);
+    const rel = (p) => relative(io.cwd, abs(p));
+    const progress = (line) => {
+      io.stderr(`[cosyte-bench] ${line}
+`);
+    };
+    const finish = (doc, code) => {
+      const out = one(args, "--out");
+      const summary = one(args, "--summary");
+      const markdown = renderMarkdown(doc);
+      if (out !== void 0) writeFileSync(abs(out), `${JSON.stringify(doc, null, 2)}
+`);
+      if (summary !== void 0) writeFileSync(abs(summary), markdown);
+      io.stdout(markdown);
+      return code;
+    };
+    const overrides = { runJob: io.runJob, onProgress: progress };
+    if (command === "check") {
+      if (args.values.size > 0 || args.flags.size > 0) throw new UsageError("check takes no flags");
+      const file = args.positional[0];
+      if (file === void 0) throw new UsageError("check needs a results file");
+      let text;
+      try {
+        text = String(readFileSync(abs(file)));
+      } catch {
+        io.stderr(`[cosyte-bench] cannot read ${rel(file)}
+`);
+        return 2;
+      }
+      try {
+        const doc = readResults(text);
+        io.stdout(
+          `${rel(file)}: a valid ${doc.kind} document, schema version ${String(doc.schemaVersion)}
+`
+        );
+        return 0;
+      } catch (error) {
+        io.stderr(`[cosyte-bench] ${rel(file)}: ${error.message}
+`);
+        return 1;
+      }
+    }
+    if (command !== "run" && command !== "compare" && command !== "calibrate") {
+      throw new UsageError(`unknown command ${command}`);
+    }
+    for (const flag of [...args.values.keys(), ...args.flags]) {
+      if (!FLAGS_FOR[command].includes(flag))
+        throw new UsageError(`${flag} does not apply to ${command}`);
+    }
+    if (args.positional.length > 0)
+      throw new UsageError(`unexpected argument ${String(args.positional[0])}`);
+    const suiteFile = abs(required(args, "--suite"));
+    const only = args.values.get("--only") ?? [];
+    if (command === "run") {
+      const target = required(args, "--target");
+      const processes = count(args, "--processes", 3);
+      const suite = await loadSuite(suiteFile);
+      const reading = await readBuild({
+        suiteFile,
+        suite,
+        target: abs(target),
+        only,
+        processes,
+        memory: !args.flags.has("--no-memory"),
+        importCost: !args.flags.has("--no-import"),
+        ...overrides,
+        measureImport: io.measureImport
+      });
+      const commit = one(args, "--commit") ?? io.env["GITHUB_SHA"] ?? null;
+      return finish(
+        {
+          ...documentBase({ name: suite.name, file: rel(suiteFile) }, io.environment(), io.now()),
+          kind: "reading",
+          subject: { target: rel(target), commit },
+          reading
+        },
+        0
+      );
+    }
+    {
+      const base = required(args, "--base");
+      const head = required(args, "--head");
+      const comparisons = count(args, "--comparisons", 0);
+      if (command === "calibrate" && comparisons === 0)
+        throw new UsageError("--comparisons is required");
+      const inject = fraction(args, "--inject");
+      const suite = await loadSuite(suiteFile);
+      const common = {
+        suiteFile,
+        suite,
+        base: abs(base),
+        head: abs(head),
+        only,
+        inject,
+        ...overrides
+      };
+      const sides = {
+        base: { target: rel(base), commit: one(args, "--base-commit") ?? null },
+        head: { target: rel(head), commit: one(args, "--head-commit") ?? null }
+      };
+      const docBase = documentBase(
+        { name: suite.name, file: rel(suiteFile) },
+        io.environment(),
+        io.now()
+      );
+      if (command === "compare") {
+        const comparison = await compareBuilds(common);
+        return finish(
+          { ...docBase, kind: "comparison", ...sides, comparison },
+          comparisonExitCode(comparison)
+        );
+      }
+      const calibration = await calibrateComparison({ ...common, comparisons });
+      return finish({ ...docBase, kind: "calibration", ...sides, calibration }, 0);
+    }
+  } catch (error) {
+    if (error instanceof UsageError) {
+      io.stderr(`[cosyte-bench] ${error.message}
+
+${USAGE}`);
+      return 2;
+    }
+    io.stderr(`[cosyte-bench] ${error instanceof Error ? error.message : String(error)}
+`);
+    return 2;
+  }
+}
+
+// src/cli.ts
+process.exitCode = await main(process.argv.slice(2), {
+  stdout: (text) => process.stdout.write(text),
+  stderr: (text) => process.stderr.write(text),
+  cwd: process.cwd(),
+  env: process.env,
+  runJob: (job) => runInChild(job),
+  measureImport: (target) => measureImportCost(target),
+  environment: () => environment(process.env),
+  now: () => /* @__PURE__ */ new Date()
+});
+//# sourceMappingURL=cli.mjs.map
+//# sourceMappingURL=cli.mjs.map