benchforge 0.1.0

package/src/test/PermutationTest.test.ts

@@ -0,0 +1,121 @@
+import { expect, test } from "vitest";
+import { compareWithBaseline } from "../PermutationTest.ts";
+import { assertValid, getSampleData } from "./TestUtils.ts";
+
+test("detects 20% performance improvement", () => {
+  const baseline = getSampleData(0, 100);
+  const improved = baseline.map(v => v * 0.8);
+  const result = compareWithBaseline(baseline, improved);
+
+  expect(result.currentMedian).toBeLessThan(result.baselineMedian);
+  expect(result.currentMean).toBeLessThan(result.baselineMean);
+  expect(result.medianChange.percent).toBeCloseTo(-20, 0);
+  expect(result.meanChange.percent).toBeCloseTo(-20, 0);
+  expect(result.medianChange.significant).toBe(true);
+  expect(["good", "strong"]).toContain(result.medianChange.significance);
+  expect(result.meanChange.significant).toBe(true);
+  expect(["good", "strong"]).toContain(result.meanChange.significance);
+});
+
+test("detects 30% performance regression", () => {
+  const baseline = getSampleData(100, 200);
+  const regressed = baseline.map(v => v * 1.3);
+  const result = compareWithBaseline(baseline, regressed);
+
+  expect(result.currentMedian).toBeGreaterThan(result.baselineMedian);
+  expect(result.currentMean).toBeGreaterThan(result.baselineMean);
+  expect(result.medianChange.percent).toBeCloseTo(30, 0);
+  expect(result.meanChange.percent).toBeCloseTo(30, 0);
+  expect(result.medianChange.significant).toBe(true);
+  expect(result.meanChange.significant).toBe(true);
+});
+
+test("detects no change with noise", () => {
+  const baseline = getSampleData(200, 300);
+  const noisy = baseline.map(v => v + (Math.random() - 0.5) * 2);
+  const result = compareWithBaseline(baseline, noisy);
+
+  expect(Math.abs(result.medianChange.percent)).toBeLessThan(5);
+  expect(Math.abs(result.meanChange.percent)).toBeLessThan(5);
+  expect(result.medianChange.significant).toBe(false);
+  expect(result.medianChange.significance).toBe("none");
+  expect(result.meanChange.significant).toBe(false);
+  expect(result.meanChange.significance).toBe("none");
+});
+
+test("compares early vs late benchmark runs", () => {
+  const early = getSampleData(0, 50);
+  const late = getSampleData(560, 610);
+  const result = compareWithBaseline(early, late);
+
+  expect(result.baselineMedian).toBeGreaterThan(40);
+  expect(result.currentMedian).toBeGreaterThan(40);
+  expect(result.baselineMean).toBeGreaterThan(40);
+  expect(result.currentMean).toBeGreaterThan(40);
+  assertValid.pValue(result.medianChange.pValue);
+  assertValid.pValue(result.meanChange.pValue);
+});
+
+test("produces high p-values for identical data", () => {
+  const samples = getSampleData(300, 350);
+  const identical = [...samples];
+  const result = compareWithBaseline(samples, identical);
+
+  expect(result.medianChange.pValue).toBeGreaterThan(0.5);
+  expect(result.meanChange.pValue).toBeGreaterThan(0.5);
+});
+
+test("produces low p-values for 4x performance difference", () => {
+  const fast = getSampleData(400, 450).map(v => v * 0.5);
+  const slow = getSampleData(450, 500).map(v => v * 2.0);
+  const result = compareWithBaseline(fast, slow);
+
+  expect(result.medianChange.pValue).toBeLessThan(0.01);
+  expect(result.meanChange.pValue).toBeLessThan(0.01);
+});
+
+test("handles single value vs identical values", () => {
+  const single = [50];
+  const identical = [50, 50, 50, 50, 50];
+  const result = compareWithBaseline(single, identical);
+
+  expect(result.baselineMedian).toBe(50);
+  expect(result.currentMedian).toBe(50);
+  expect(result.baselineMean).toBe(50);
+  expect(result.currentMean).toBe(50);
+  expect(result.medianChange.percent).toBe(0);
+  expect(result.meanChange.percent).toBe(0);
+  expect(result.medianChange.significant).toBe(false);
+  expect(result.meanChange.significant).toBe(false);
+});
+
+test("categorizes change significance by magnitude", () => {
+  const baseline = getSampleData(0, 100);
+
+  const slight = baseline.map(v => v * 1.05);
+  const moderate = baseline.map(v => v * 1.15);
+  const large = baseline.map(v => v * 1.5);
+
+  const slightResult = compareWithBaseline(baseline, slight);
+  const moderateResult = compareWithBaseline(baseline, moderate);
+  const largeResult = compareWithBaseline(baseline, large);
+
+  assertValid.significance(slightResult.medianChange.significance);
+  assertValid.significance(moderateResult.medianChange.significance);
+  assertValid.significance(largeResult.medianChange.significance);
+
+  const levels = ["none", "weak", "good", "strong"];
+  const slightIdx = levels.indexOf(slightResult.medianChange.significance);
+  const largeIdx = levels.indexOf(largeResult.medianChange.significance);
+  expect(largeIdx).toBeGreaterThanOrEqual(slightIdx);
+});
+
+test("compares warmup vs stable performance", () => {
+  const warmup = getSampleData(0, 20);
+  const stable = getSampleData(100, 120);
+  const result = compareWithBaseline(warmup, stable);
+
+  expect(result.baselineMedian).toBeGreaterThan(result.currentMedian);
+  expect(result.medianChange.percent).toBeLessThan(0);
+  expect(result.medianChange.absolute).toBeLessThan(0);
+});

package/src/test/RunBenchCLI.test.ts

@@ -0,0 +1,166 @@
+import { execSync } from "node:child_process";
+import path from "node:path";
+import { expect, test } from "vitest";
+import type { BenchSuite } from "../Benchmark.ts";
+import { filterBenchmarks } from "../cli/FilterBenchmarks.ts";
+import { runBenchCLITest } from "./TestUtils.ts";
+
+const testSuite: BenchSuite = {
+  name: "Test Suite",
+  groups: [
+    {
+      name: "String Operations",
+      benchmarks: [
+        { name: "concatenation", fn: () => "a" + "b" },
+        { name: "template literal", fn: () => `a${"b"}` },
+      ],
+    },
+    {
+      name: "Math Operations",
+      benchmarks: [
+        { name: "addition", fn: () => 1 + 1 },
+        { name: "multiplication", fn: () => 2 * 2 },
+      ],
+    },
+  ],
+};
+
+const suiteWithSetup: BenchSuite = {
+  name: "Array Suite",
+  groups: [
+    {
+      name: "Array Operations",
+      setup: () => ({
+        numbers: Array.from({ length: 100 }, (_, i) => i),
+        strings: Array.from({ length: 100 }, (_, i) => `item${i}`),
+      }),
+      benchmarks: [
+        {
+          name: "sum numbers",
+          fn: ({ numbers }: any) =>
+            numbers.reduce((a: number, b: number) => a + b, 0),
+        },
+        {
+          name: "join strings",
+          fn: ({ strings }: any) => strings.join(","),
+        },
+      ],
+    },
+  ],
+};
+
+test("runs all benchmarks", { timeout: 30000 }, async () => {
+  const output = await runBenchCLITest(testSuite, "--time 0.1");
+
+  expect(output).toContain("concatenation");
+  expect(output).toContain("template literal");
+  expect(output).toContain("addition");
+  expect(output).toContain("multiplication");
+  expect(output).toContain("mean");
+  expect(output).toContain("runs");
+});
+
+test("filters by substring", { timeout: 15000 }, async () => {
+  const output = await runBenchCLITest(testSuite, "--filter concat --time 0.1");
+
+  expect(output).toContain("concatenation");
+  expect(output).not.toContain("addition");
+});
+
+test("filters by regex", { timeout: 15000 }, async () => {
+  const output = await runBenchCLITest(
+    testSuite,
+    "--filter ^template --time 0.1",
+  );
+  expect(output).toContain("template literal");
+  expect(output).not.toContain("addition");
+});
+
+test("filter preserves suite structure", () => {
+  const filtered = filterBenchmarks(testSuite, "concatenation", false);
+
+  expect(filtered.name).toBe("Test Suite");
+  expect(filtered.groups).toHaveLength(2);
+  expect(filtered.groups[0].name).toBe("String Operations");
+  expect(filtered.groups[0].benchmarks).toHaveLength(1);
+  expect(filtered.groups[0].benchmarks[0].name).toBe("concatenation");
+  expect(filtered.groups[1].benchmarks).toHaveLength(0);
+});
+
+/** Execute test fixture script and return output */
+function executeTestScript(args = ""): string {
+  const script = path.join(
+    import.meta.dirname!,
+    "fixtures/test-bench-script.ts",
+  );
+  return execSync(`node --expose-gc --allow-natives-syntax ${script} ${args}`, {
+    encoding: "utf8",
+  });
+}
+
+test("e2e: runs user script", { timeout: 30000 }, () => {
+  const output = executeTestScript("--time 0.1");
+
+  expect(output).toContain("plus");
+  expect(output).toContain("multiply");
+  expect(output).toContain("mean");
+  expect(output).toContain("runs");
+
+  const lines = output.split("\n");
+  const headerLine = lines.find(l => l.includes("name"));
+  expect(headerLine).toBeTruthy();
+
+  const plusLine = lines.find(l => l.includes("plus"));
+  expect(plusLine).toBeTruthy();
+});
+
+test("e2e: filter flag", { timeout: 30000 }, () => {
+  const output = executeTestScript('--filter "plus" --time 0.1');
+
+  expect(output).toContain("plus");
+  expect(output).not.toContain("multiply");
+});
+
+test("runs benchmarks with setup function", { timeout: 30000 }, async () => {
+  const output = await runBenchCLITest(suiteWithSetup, "--time 0.1");
+
+  expect(output).toContain("sum numbers");
+  expect(output).toContain("join strings");
+  expect(output).toContain("mean");
+  expect(output).toContain("runs");
+});
+
+test(
+  "runs benchmarks with baseline comparison",
+  { timeout: 30000 },
+  async () => {
+    const suiteWithBaseline: BenchSuite = {
+      name: "Baseline Test",
+      groups: [
+        {
+          name: "Sort Comparison",
+          setup: () => ({
+            data: Array.from({ length: 10 }, () => Math.random()),
+          }),
+          baseline: {
+            name: "baseline sort",
+            fn: ({ data }: any) => [...data].sort(),
+          },
+          benchmarks: [
+            {
+              name: "optimized sort",
+              fn: ({ data }: any) => [...data].sort((a, b) => a - b),
+            },
+          ],
+        },
+      ],
+    };
+
+    const output = await runBenchCLITest(suiteWithBaseline, "--time 0.01");
+
+    expect(output).toContain("baseline sort");
+    expect(output).toContain("optimized sort");
+    expect(output).toContain("Δ%"); // Diff column should appear
+    expect(output).toContain("mean");
+  },
+);

package/src/test/RunnerOrchestrator.test.ts

@@ -0,0 +1,102 @@
+import { expect, test } from "vitest";
+import type { BenchmarkSpec } from "../Benchmark.ts";
+import { runBenchmark } from "../runners/RunnerOrchestrator.ts";
+
+/** lightweight function for testing worker communication */
+function simpleTestFunction(): number {
+  let sum = 0;
+  for (let i = 0; i < 1000; i++) sum += Math.sqrt(i);
+  return sum;
+}
+
+test("BasicRunner runs benchmark in worker mode", async () => {
+  const spec: BenchmarkSpec = {
+    name: "basic-worker-test",
+    fn: simpleTestFunction,
+  };
+
+  const results = await runBenchmark({
+    spec,
+    runner: "basic",
+    options: {
+      maxTime: 5,
+      maxIterations: 50,
+    },
+    useWorker: true,
+  });
+
+  expect(results).toHaveLength(1);
+  const result = results[0];
+
+  expect(result.name).toBe("basic-worker-test");
+  expect(result.samples.length).toBeGreaterThan(0);
+  expect(result.samples.length).toBeLessThanOrEqual(500);
+  expect(result.time.min).toBeGreaterThan(0);
+  expect(result.time.max).toBeGreaterThanOrEqual(result.time.min);
+  expect(result.time.avg).toBeGreaterThan(0);
+  expect(result.time.p50).toBeGreaterThan(0);
+  expect(result.time.p99).toBeGreaterThan(0);
+});
+
+test("BasicRunner runs benchmark in non-worker mode", async () => {
+  const spec: BenchmarkSpec = {
+    name: "basic-test",
+    fn: simpleTestFunction,
+  };
+
+  const results = await runBenchmark({
+    spec,
+    runner: "basic",
+    options: {
+      maxTime: 5,
+      maxIterations: 50,
+    },
+    useWorker: false,
+  });
+
+  expect(results).toHaveLength(1);
+  const result = results[0];
+
+  expect(result.name).toBe("basic-test");
+  expect(result.samples.length).toBeGreaterThan(0);
+  expect(result.time.p50).toBeGreaterThan(0);
+});
+
+test("BasicRunner with parameterized benchmark", async () => {
+  const spec: BenchmarkSpec<number> = {
+    name: "parameterized-test",
+    fn: (n: number) => {
+      let sum = 0;
+      for (let i = 0; i < n; i++) sum += i;
+      return sum;
+    },
+  };
+
+  const results = await runBenchmark({
+    spec,
+    runner: "basic",
+    options: { maxTime: 5, maxIterations: 20 },
+    useWorker: false,
+    params: 100,
+  });
+
+  expect(results).toHaveLength(1);
+  expect(results[0].name).toBe("parameterized-test");
+});
+
+test("RunnerOrchestrator propagates errors from worker", async () => {
+  const spec: BenchmarkSpec = {
+    name: "error-test",
+    fn: () => {
+      throw new Error("Test error from benchmark");
+    },
+  };
+
+  const promise = runBenchmark({
+    spec,
+    runner: "basic",
+    options: { maxTime: 1, maxIterations: 1 },
+    useWorker: true,
+  });
+  await expect(promise).rejects.toThrow("Test error from benchmark");
+});

package/src/test/StatisticalUtils.test.ts

@@ -0,0 +1,112 @@
+import { expect, test } from "vitest";
+import {
+  average,
+  bootstrapDifferenceCI,
+  bootstrapMedian,
+  coefficientOfVariation,
+  findOutliers,
+  medianAbsoluteDeviation,
+  percentile,
+  standardDeviation,
+} from "../StatisticalUtils.ts";
+import { assertValid, getSampleData } from "./TestUtils.ts";
+
+test("calculates mean correctly", () => {
+  const subset = getSampleData(0, 10);
+  const expected = subset.reduce((a, b) => a + b, 0) / subset.length;
+  expect(average(subset)).toBeCloseTo(expected, 5);
+  expect(average([10])).toBe(10);
+  expect(average([-5, 5])).toBe(0);
+});
+
+test("calculates standard deviation", () => {
+  const subset = getSampleData(50, 100);
+  const stddev = standardDeviation(subset);
+  expect(stddev).toBeGreaterThan(0);
+  expect(stddev).toBeLessThan(10);
+  expect(standardDeviation([5, 5, 5])).toBe(0);
+  expect(standardDeviation([5])).toBe(0);
+});
+
+test("calculates percentiles in order", () => {
+  const subset = getSampleData(100, 200);
+  const p25 = percentile(subset, 0.25);
+  const p50 = percentile(subset, 0.5);
+  const p75 = percentile(subset, 0.75);
+  const p99 = percentile(subset, 0.99);
+
+  assertValid.percentileOrder(p25, p50, p75, p99);
+  expect(p50).toBeGreaterThan(40);
+  expect(p50).toBeLessThan(60);
+  expect(percentile([42], 0.5)).toBe(42);
+});
+
+test("calculates coefficient of variation", () => {
+  const stable = getSampleData(200, 300);
+  const cv = coefficientOfVariation(stable);
+  expect(cv).toBeGreaterThan(0);
+  expect(cv).toBeLessThan(0.2);
+  expect(coefficientOfVariation([-1, 0, 1])).toBe(0);
+  expect(coefficientOfVariation([5, 5, 5])).toBe(0);
+});
+
+test("calculates median absolute deviation", () => {
+  const warmup = getSampleData(0, 30);
+  const mad = medianAbsoluteDeviation(warmup);
+  expect(mad).toBeGreaterThan(0);
+  expect(mad).toBeLessThan(15);
+  expect(medianAbsoluteDeviation([1, 2, 3, 4, 5])).toBe(1);
+});
+
+test("identifies outliers in mixed data", () => {
+  const mixed = [...getSampleData(0, 50)];
+  mixed.push(200, 5);
+  const outliers = findOutliers(mixed);
+
+  expect(outliers.rate).toBeGreaterThan(0);
+  expect(outliers.indices).toContain(50);
+  expect(outliers.indices).toContain(51);
+});
+
+test("bootstrap estimates median with confidence intervals", () => {
+  const stable = getSampleData(400, 450);
+  const actual = percentile(stable, 0.5);
+  const result = bootstrapMedian(stable, { resamples: 1000 });
+
+  expect(result.estimate).toBeCloseTo(actual, 1);
+  expect(result.ci[0]).toBeLessThanOrEqual(result.estimate);
+  expect(result.ci[1]).toBeGreaterThanOrEqual(result.estimate);
+  expect(result.ci[1] - result.ci[0]).toBeLessThan(5);
+  expect(result.samples).toHaveLength(1000);
+});
+
+test("bootstrapDifferenceCI detects improvement", () => {
+  const baseline = getSampleData(0, 100);
+  const improved = baseline.map(v => v * 0.8);
+  const result = bootstrapDifferenceCI(baseline, improved, { resamples: 1000 });
+
+  expect(result.percent).toBeCloseTo(-20, 0);
+  expect(result.ci[1]).toBeLessThan(0);
+  expect(result.direction).toBe("faster");
+});
+
+test("bootstrapDifferenceCI detects regression", () => {
+  const baseline = getSampleData(0, 100);
+  const slower = baseline.map(v => v * 1.2);
+  const result = bootstrapDifferenceCI(baseline, slower, { resamples: 1000 });
+
+  expect(result.percent).toBeCloseTo(20, 0);
+  expect(result.ci[0]).toBeGreaterThan(0);
+  expect(result.direction).toBe("slower");
+});
+
+test("bootstrapDifferenceCI shows uncertainty for noise", () => {
+  const baseline = getSampleData(0, 100);
+  const noisy = baseline.map(v => v + (Math.random() - 0.5) * 2);
+  const result = bootstrapDifferenceCI(baseline, noisy, { resamples: 1000 });
+
+  // CI should span zero for no real change
+  expect(result.ci[0]).toBeLessThanOrEqual(0);
+  expect(result.ci[1]).toBeGreaterThanOrEqual(0);
+  expect(result.direction).toBe("uncertain");
+});

package/src/test/TestUtils.ts

@@ -0,0 +1,93 @@
+import type { BenchSuite } from "../Benchmark.ts";
+import type { BenchmarkReport } from "../BenchmarkReport.ts";
+import type { Configure, DefaultCliArgs } from "../cli/CliArgs.ts";
+import { parseCliArgs } from "../cli/CliArgs.ts";
+import { defaultReport, runBenchmarks } from "../cli/RunBenchCLI.ts";
+import type { MeasuredResults } from "../MeasuredResults.ts";
+import { average, percentile } from "../StatisticalUtils.ts";
+import { bevy30SamplesMs } from "../tests/fixtures/bevy30-samples.ts";
+
+/** @return formatted benchmark output for CLI testing */
+export async function runBenchCLITest<T = DefaultCliArgs>(
+  suite: BenchSuite,
+  args: string,
+  configureArgs?: Configure<T>,
+): Promise<string> {
+  const argv = args.split(/\s+/).filter(arg => arg.length > 0);
+  const parsedArgs = parseCliArgs(argv, configureArgs) as T & DefaultCliArgs;
+  const results = await runBenchmarks(suite, parsedArgs);
+  return defaultReport(results, parsedArgs);
+}
+
+/** @return slice of bevy30 samples for consistent test data */
+export function getSampleData(start: number, end: number): number[] {
+  return bevy30SamplesMs.slice(start, end);
+}
+
+/** @return test MeasuredResults from bevy30 samples */
+export function createMeasuredResults(
+  sampleRange: [number, number],
+  overrides?: Partial<MeasuredResults>,
+): MeasuredResults {
+  const samples = getSampleData(sampleRange[0], sampleRange[1]);
+  const sorted = [...samples].sort((a, b) => a - b);
+
+  return {
+    name: "test",
+    samples,
+    time: {
+      min: sorted[0],
+      max: sorted[sorted.length - 1],
+      avg: average(samples),
+      p50: percentile(samples, 0.5),
+      p75: percentile(samples, 0.75),
+      p99: percentile(samples, 0.99),
+      p999: percentile(samples, 0.999),
+    },
+    nodeGcTime: {
+      inRun: 0.09,
+      before: 0.01,
+      after: 0.02,
+      total: 0.12,
+      collects: 3,
+      events: [],
+    },
+    ...overrides,
+  };
+}
+
+/** @return test BenchmarkReport from bevy30 samples */
+export function createBenchmarkReport(
+  name: string,
+  sampleRange: [number, number],
+  overrides?: Partial<MeasuredResults>,
+): BenchmarkReport {
+  return {
+    name,
+    measuredResults: createMeasuredResults(sampleRange, overrides),
+  };
+}
+
+/** Validation helpers for statistical tests */
+export const assertValid = {
+  pValue: (value: number) => {
+    if (value < 0 || value > 1) {
+      throw new Error(`Expected p-value between 0 and 1, got ${value}`);
+    }
+  },
+
+  percentileOrder: (p25: number, p50: number, p75: number, p99: number) => {
+    if (!(p25 <= p50 && p50 <= p75 && p75 <= p99)) {
+      throw new Error(
+        `Percentiles not ordered: p25=${p25}, p50=${p50}, p75=${p75}, p99=${p99}`,
+      );
+    }
+  },
+
+  significance: (level: string) => {
+    const valid = ["none", "weak", "good", "strong"];
+    if (!valid.includes(level)) {
+      throw new Error(`Invalid significance level: ${level}`);
+    }
+  },
+};

package/src/test/fixtures/test-bench-script.ts

@@ -0,0 +1,30 @@
+#!/usr/bin/env -S node --expose-gc --allow-natives-syntax
+import { type BenchSuite, runDefaultBench } from "../../index.ts";
+
+const suite: BenchSuite = {
+  name: "Test",
+  groups: [
+    {
+      name: "Math",
+      benchmarks: [
+        { name: "plus", fn: () => 1 + 1 },
+        { name: "multiply", fn: () => 2 * 2 },
+      ],
+    },
+    {
+      name: "Array Math",
+      setup: () => ({
+        nums: [1, 2, 3, 4, 5],
+      }),
+      benchmarks: [
+        {
+          name: "array sum",
+          fn: ({ nums }: any) =>
+            nums.reduce((a: number, b: number) => a + b, 0),
+        },
+      ],
+    },
+  ],
+};
+
+await runDefaultBench(suite);