@cosyte/bench 0.0.0-stage → 0.1.1

package/dist/index.mjs

@@ -0,0 +1,555 @@
+import 'module';
+import 'child_process';
+import 'fs';
+import 'os';
+import 'path';
+import { fileURLToPath } from 'url';
+
+// src/contract.ts
+var BENCH_CONTRACT = Object.freeze({
+  SAMPLES: 5,
+  SAMPLE_MIN_MS: 50,
+  ROUNDS: 12,
+  ALPHA: 0.05,
+  THRESHOLD: 0.03,
+  LATENCY_BUDGET_MS: 2e3,
+  LATENCY_MAX_CALLS: 5e3,
+  LATENCY_MIN_CALLS: 20,
+  LATENCY_TAIL_CALLS: 10,
+  LATENCY_TIMER_MULTIPLE: 20,
+  ALLOC_WINDOW_OPS: 256,
+  ALLOC_WINDOW_MS: 100,
+  ALLOC_SEMI_SPACE_MIB: 64,
+  PEAK_CALLS: 20,
+  LONG_RUN_ROUNDS: 30,
+  LONG_RUN_SETTLE_ROUNDS: 5,
+  LONG_RUN_ROUND_MS: 100,
+  LONG_RUN_GROWTH_BYTES: 64 * 1024,
+  GC_SETTLE_BYTES: 64 * 1024,
+  GC_SETTLE_ROUNDS: 2,
+  GC_MAX_ROUNDS: 7,
+  IMPORT_RUNS: 20,
+  REQUIRED_FALSE_ALARM_RATE: 0.01,
+  REQUIRED_CONFIDENCE: 0.95
+});
+
+// src/suite.ts
+function defineBenchmark(benchmark) {
+  return benchmark;
+}
+function defineSuite(suite) {
+  return checkSuite(suite);
+}
+function fail(message) {
+  throw new TypeError(`@cosyte/bench: ${message}`);
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null;
+}
+function checkName(value, what) {
+  if (typeof value !== "string" || value.trim() === "") fail(`${what} must be a non-empty string`);
+  return value;
+}
+function checkSuite(value) {
+  if (!isRecord(value)) fail("a suite must be an object made by defineSuite");
+  const name = checkName(value["name"], "the suite name");
+  const benchmarks = value["benchmarks"];
+  if (!Array.isArray(benchmarks) || benchmarks.length === 0) {
+    fail(`suite "${name}" must have at least one benchmark`);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const b of benchmarks) {
+    if (!isRecord(b)) fail(`suite "${name}" holds a benchmark that is not an object`);
+    const bName = checkName(b["name"], `a benchmark name in suite "${name}"`);
+    if (seen.has(bName)) fail(`suite "${name}" has two benchmarks named "${bName}"`);
+    seen.add(bName);
+    if (typeof b["bind"] !== "function") fail(`benchmark "${bName}" has no bind function`);
+    if (b["weigh"] !== void 0 && typeof b["weigh"] !== "function") {
+      fail(`benchmark "${bName}" has a weigh that is not a function`);
+    }
+    const sizes = b["sizes"];
+    if (!Array.isArray(sizes) || sizes.length === 0) fail(`benchmark "${bName}" has no sizes`);
+    const labels = /* @__PURE__ */ new Set();
+    for (const s of sizes) {
+      if (!isRecord(s)) fail(`benchmark "${bName}" holds a size that is not an object`);
+      const label = checkName(s["label"], `a size label in benchmark "${bName}"`);
+      if (labels.has(label)) fail(`benchmark "${bName}" has two sizes labelled "${label}"`);
+      labels.add(label);
+      if (typeof s["build"] !== "function")
+        fail(`size "${label}" of "${bName}" has no build function`);
+      if (s["compare"] !== void 0 && typeof s["compare"] !== "boolean") {
+        fail(`size "${label}" of "${bName}" has a compare that is not true or false`);
+      }
+    }
+    if (b["longRun"] !== void 0 && !labels.has(b["longRun"])) {
+      fail(`benchmark "${bName}" names a longRun size it does not have`);
+    }
+  }
+  return Object.freeze(value);
+}
+
+// src/stats.ts
+function sortedCopy(xs) {
+  return [...xs].sort((a, b) => a - b);
+}
+function median(xs) {
+  const s = sortedCopy(xs);
+  const mid = Math.floor(s.length / 2);
+  return s.length % 2 === 1 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
+}
+function binomialCdf(k, n, p) {
+  const logP = Math.log(p);
+  const logQ = Math.log1p(-p);
+  let logCoef = 0;
+  let total = 0;
+  for (let i = 0; i <= Math.min(k, n); i++) {
+    if (i > 0) logCoef += Math.log(n - i + 1) - Math.log(i);
+    total += Math.exp(logCoef + i * logP + (n - i) * logQ);
+  }
+  return Math.min(1, total);
+}
+function signTestRank(n, alpha) {
+  let k = 0;
+  while (k < n && binomialCdf(k, n, 0.5) <= alpha) k++;
+  return k;
+}
+function signTestInterval(xs, alpha) {
+  const rank = signTestRank(xs.length, alpha);
+  if (rank === 0) return null;
+  const s = sortedCopy(xs);
+  return {
+    low: s[rank - 1],
+    high: s[s.length - rank],
+    rank,
+    tail: binomialCdf(rank - 1, xs.length, 0.5)
+  };
+}
+
+// src/compare.ts
+function judge(ratios, threshold = BENCH_CONTRACT.THRESHOLD, alpha = BENCH_CONTRACT.ALPHA) {
+  const interval = signTestInterval(ratios, alpha);
+  const medianRatio = median(ratios);
+  if (interval === null) return { verdict: "inconclusive", interval, medianRatio };
+  const up = 1 + threshold;
+  let verdict;
+  if (interval.low > up) verdict = "regression";
+  else if (interval.high < 1 / up) verdict = "improvement";
+  else if (interval.high <= up) verdict = "no-regression";
+  else verdict = "inconclusive";
+  return { verdict, interval, medianRatio };
+}
+var SCHEMA_VERSION = 1;
+var num = { type: "number" };
+var int = { type: "integer", minimum: 0 };
+var str = { type: "string" };
+var strOrNull = { type: ["string", "null"] };
+var numOrNull = { type: ["number", "null"] };
+var nums = { type: "array", items: num };
+var skipped = {
+  type: "object",
+  required: ["status", "reason"],
+  properties: { status: { const: "skipped" }, reason: str }
+};
+var measuredOrSkipped = (measured) => ({ oneOf: [measured, skipped] });
+var orNull = (s) => ({ oneOf: [{ type: "null" }, s] });
+var RESULTS_SCHEMA = {
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  $id: "https://github.com/cosyte/config/blob/main/packages/bench/schema/results-v1.schema.json",
+  title: "@cosyte/bench results, version 1",
+  type: "object",
+  required: ["schemaVersion", "kind", "tool", "createdAt", "suite", "environment", "contract"],
+  properties: {
+    schemaVersion: { const: 1 },
+    kind: { enum: ["reading", "comparison", "calibration"] },
+    tool: {
+      type: "object",
+      required: ["name", "version"],
+      properties: { name: { const: "@cosyte/bench" }, version: str }
+    },
+    createdAt: str,
+    suite: { type: "object", required: ["name", "file"], properties: { name: str, file: str } },
+    environment: { $ref: "#/$defs/environment" },
+    contract: {
+      type: "object",
+      required: ["bench", "warmup"],
+      properties: { bench: { type: "object" }, warmup: { type: "object" } }
+    }
+  },
+  oneOf: [
+    {
+      required: ["subject", "reading"],
+      properties: {
+        kind: { const: "reading" },
+        subject: { $ref: "#/$defs/build" },
+        reading: { $ref: "#/$defs/reading" }
+      }
+    },
+    {
+      required: ["base", "head", "comparison"],
+      properties: {
+        kind: { const: "comparison" },
+        base: { $ref: "#/$defs/build" },
+        head: { $ref: "#/$defs/build" },
+        comparison: { $ref: "#/$defs/comparison" }
+      }
+    },
+    {
+      required: ["base", "head", "calibration"],
+      properties: {
+        kind: { const: "calibration" },
+        base: { $ref: "#/$defs/build" },
+        head: { $ref: "#/$defs/build" },
+        calibration: { $ref: "#/$defs/calibration" }
+      }
+    }
+  ],
+  $defs: {
+    build: {
+      type: "object",
+      required: ["target", "commit"],
+      properties: { target: str, commit: strOrNull }
+    },
+    environment: {
+      type: "object",
+      required: [
+        "node",
+        "v8",
+        "platform",
+        "arch",
+        "osRelease",
+        "cpuModel",
+        "logicalCpus",
+        "totalMemoryBytes",
+        "cgroupCpuMax",
+        "runner"
+      ],
+      properties: {
+        node: str,
+        v8: str,
+        platform: str,
+        arch: str,
+        osRelease: str,
+        cpuModel: strOrNull,
+        logicalCpus: int,
+        totalMemoryBytes: int,
+        cgroupCpuMax: strOrNull,
+        // Added within version 1, so not required: a document written before it is still valid.
+        loadAverage: nums,
+        runner: {
+          type: "object",
+          required: ["ci", "provider", "class"],
+          properties: { ci: { type: "boolean" }, provider: strOrNull, class: str }
+        }
+      }
+    },
+    interval: orNull({
+      type: "object",
+      required: ["low", "high", "rank", "tail"],
+      properties: { low: num, high: num, rank: int, tail: num }
+    }),
+    caseComparison: {
+      type: "object",
+      required: [
+        "benchmark",
+        "size",
+        "baseMsPerOp",
+        "headMsPerOp",
+        "ratios",
+        "medianRatio",
+        "interval",
+        "verdict",
+        "skipReason"
+      ],
+      properties: {
+        benchmark: str,
+        size: str,
+        baseMsPerOp: nums,
+        headMsPerOp: nums,
+        ratios: nums,
+        medianRatio: numOrNull,
+        interval: { $ref: "#/$defs/interval" },
+        verdict: {
+          enum: ["regression", "improvement", "no-regression", "inconclusive", "skipped"]
+        },
+        skipReason: strOrNull
+      }
+    },
+    comparison: {
+      type: "object",
+      required: ["rounds", "threshold", "alpha", "inject", "cases"],
+      properties: {
+        rounds: int,
+        threshold: num,
+        alpha: num,
+        inject: num,
+        cases: { type: "array", items: { $ref: "#/$defs/caseComparison" } }
+      }
+    },
+    rate: {
+      type: "object",
+      required: ["events", "trials", "rate", "upperBound"],
+      properties: { events: int, trials: int, rate: num, upperBound: num }
+    },
+    calibration: {
+      type: "object",
+      required: [
+        "inject",
+        "comparisons",
+        "runs",
+        "perCase",
+        "perComparison",
+        "verdicts",
+        "requiredRule"
+      ],
+      properties: {
+        inject: num,
+        comparisons: int,
+        runs: {
+          type: "array",
+          items: { type: "array", items: { $ref: "#/$defs/caseComparison" } }
+        },
+        perCase: { $ref: "#/$defs/rate" },
+        perComparison: { $ref: "#/$defs/rate" },
+        // Added within version 1, so optional: documents written before it validate unchanged.
+        skippedComparisons: { $ref: "#/$defs/rate" },
+        verdicts: { type: "object" },
+        observed: {
+          type: "array",
+          items: {
+            type: "object",
+            required: ["benchmark", "size", "rounds", "medianRatio"],
+            properties: { benchmark: str, size: str, rounds: int, medianRatio: numOrNull }
+          }
+        },
+        requiredRule: {
+          type: "object",
+          required: ["applies", "met", "reason"],
+          properties: { applies: { type: "boolean" }, met: { type: "boolean" }, reason: str }
+        }
+      }
+    },
+    reading: {
+      type: "object",
+      required: ["processes", "cases", "importCost"],
+      properties: {
+        processes: int,
+        cases: { type: "array", items: { $ref: "#/$defs/caseReading" } },
+        importCost: orNull({
+          type: "object",
+          required: [
+            "runs",
+            "coldImportMs",
+            "medianMs",
+            "p95Ms",
+            "emptyImportMs",
+            "emptyMedianMs",
+            "files",
+            "totalBytes",
+            "totalGzipBytes"
+          ],
+          properties: {
+            runs: int,
+            coldImportMs: nums,
+            medianMs: num,
+            p95Ms: num,
+            emptyImportMs: nums,
+            emptyMedianMs: num,
+            files: {
+              type: "array",
+              items: {
+                type: "object",
+                required: ["path", "bytes", "gzipBytes"],
+                properties: { path: str, bytes: int, gzipBytes: int }
+              }
+            },
+            totalBytes: int,
+            totalGzipBytes: int
+          }
+        })
+      }
+    },
+    caseReading: {
+      type: "object",
+      required: [
+        "benchmark",
+        "size",
+        "inputsPerPass",
+        "bytesPerPass",
+        "throughput",
+        "latency",
+        "allocations",
+        "peak",
+        "longRun"
+      ],
+      properties: {
+        benchmark: str,
+        size: str,
+        inputsPerPass: int,
+        bytesPerPass: numOrNull,
+        throughput: measuredOrSkipped({
+          type: "object",
+          required: ["status", "processes", "medianMsPerOp", "opsPerSecond", "bytesPerSecond"],
+          properties: {
+            status: { const: "measured" },
+            processes: {
+              type: "array",
+              items: {
+                type: "object",
+                required: ["samplesMsPerOp", "medianMsPerOp"],
+                properties: { samplesMsPerOp: nums, medianMsPerOp: num }
+              }
+            },
+            medianMsPerOp: num,
+            opsPerSecond: num,
+            bytesPerSecond: numOrNull
+          }
+        }),
+        latency: orNull({
+          type: "object",
+          required: ["status", "calls", "timerOverheadNs", "p50Ns"],
+          properties: {
+            status: { enum: ["measured", "skipped"] },
+            calls: int,
+            timerOverheadNs: num,
+            p50Ns: numOrNull,
+            p95Ns: numOrNull,
+            p99Ns: numOrNull,
+            maxNs: num
+          }
+        }),
+        allocations: orNull(
+          measuredOrSkipped({
+            type: "object",
+            required: ["status", "windowOps", "heapBytesPerOp", "externalBytesPerOp"],
+            properties: {
+              status: { const: "measured" },
+              windowOps: int,
+              heapBytesPerOp: num,
+              externalBytesPerOp: num
+            }
+          })
+        ),
+        peak: orNull(
+          measuredOrSkipped({
+            type: "object",
+            required: ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
+            properties: {
+              status: { const: "measured" },
+              calls: int,
+              peakHeapBytes: num,
+              medianPeakHeapBytes: num
+            }
+          })
+        ),
+        longRun: orNull(
+          measuredOrSkipped({
+            type: "object",
+            required: ["status", "verdict", "ops", "opsPerRound", "settledHeap", "slopeBytesPerOp"],
+            properties: {
+              status: { const: "measured" },
+              verdict: { enum: ["flat", "growing"] },
+              ops: int,
+              opsPerRound: int,
+              settledHeap: nums,
+              slopeBytesPerOp: num
+            }
+          })
+        )
+      }
+    }
+  }
+};
+function typeOf(value) {
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "array";
+  if (Number.isInteger(value)) return "integer";
+  return typeof value;
+}
+function typeMatches(value, type) {
+  const actual = typeOf(value);
+  return actual === type || type === "number" && actual === "integer";
+}
+function resolveRef(ref, root) {
+  const target = root.$defs?.[ref.replace("#/$defs/", "")];
+  if (target === void 0) throw new Error(`schema has no definition ${ref}`);
+  return target;
+}
+function validate(value, schema, root = schema, at = "") {
+  const problems = [];
+  const here = at === "" ? "/" : at;
+  if (typeof schema.$ref === "string")
+    return validate(value, resolveRef(schema.$ref, root), root, at);
+  if ("const" in schema && value !== schema.const)
+    problems.push(`${here}: must be ${JSON.stringify(schema.const)}`);
+  if (schema.enum !== void 0 && !schema.enum.includes(value)) {
+    problems.push(
+      `${here}: must be one of ${schema.enum.map((e) => JSON.stringify(e)).join(", ")}`
+    );
+  }
+  if (schema.type !== void 0) {
+    const types = typeof schema.type === "string" ? [schema.type] : schema.type;
+    if (!types.some((t) => typeMatches(value, t))) {
+      problems.push(`${here}: must be ${types.join(" or ")}, is ${typeOf(value)}`);
+      return problems;
+    }
+  }
+  if (typeof value === "number" && value < (schema.minimum ?? -Infinity)) {
+    problems.push(`${here}: must be at least ${String(schema.minimum)}`);
+  }
+  if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+    const record = value;
+    for (const key of schema.required ?? []) {
+      if (!(key in record)) problems.push(`${here}: is missing "${key}"`);
+    }
+    for (const [key, sub] of Object.entries(schema.properties ?? {})) {
+      if (key in record) problems.push(...validate(record[key], sub, root, `${at}/${key}`));
+    }
+  }
+  if (Array.isArray(value) && schema.items !== void 0) {
+    const items = schema.items;
+    value.forEach((item, i) => problems.push(...validate(item, items, root, `${at}/${String(i)}`)));
+  }
+  if (schema.oneOf !== void 0) {
+    const results = schema.oneOf.map((s) => validate(value, s, root, at));
+    const passing = results.filter((r) => r.length === 0).length;
+    if (passing !== 1) {
+      problems.push(
+        `${here}: must match exactly one of ${String(results.length)} shapes, matches ${String(passing)}`
+      );
+      problems.push(...results.reduce((a, b) => b.length < a.length ? b : a));
+    }
+  }
+  return problems;
+}
+function readResults(text) {
+  let value;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    throw new TypeError("@cosyte/bench: the results are not JSON");
+  }
+  const version = value?.schemaVersion;
+  if (version !== SCHEMA_VERSION) {
+    throw new TypeError(
+      `@cosyte/bench: results version ${String(version)} is not one this package reads (${String(SCHEMA_VERSION)})`
+    );
+  }
+  const problems = validate(value, RESULTS_SCHEMA);
+  if (problems.length > 0)
+    throw new TypeError(
+      `@cosyte/bench: the results do not match the schema:
+${problems.join("\n")}`
+    );
+  return value;
+}
+var BenchError = class extends Error {
+  name = "BenchError";
+};
+Object.freeze({
+  child: fileURLToPath(new URL("./child-entry.mjs", import.meta.url)),
+  importer: fileURLToPath(new URL("./import-entry.mjs", import.meta.url))
+});
+
+export { BENCH_CONTRACT, BenchError, RESULTS_SCHEMA, SCHEMA_VERSION, defineBenchmark, defineSuite, judge, readResults };
+//# sourceMappingURL=index.mjs.map
+//# sourceMappingURL=index.mjs.map