benchforge 0.1.9 → 0.2.4

package/src/test/RunBenchCLI.test.ts

@@ -1,8 +1,8 @@
 import { execSync } from "node:child_process";
 import path from "node:path";
 import { expect, test } from "vitest";
-import type { BenchSuite } from "../Benchmark.ts";
 import { filterBenchmarks } from "../cli/FilterBenchmarks.ts";
+import type { BenchSuite } from "../runners/BenchmarkSpec.ts";
 import { runBenchCLITest } from "./TestUtils.ts";
 
 const testSuite: BenchSuite = {
@@ -49,8 +49,26 @@ const suiteWithSetup: BenchSuite = {
   ],
 };
 
+/** Execute test fixture script and return output */
+function executeTestScript(args = ""): string {
+  const script = path.join(
+    import.meta.dirname!,
+    "fixtures/test-bench-script.ts",
+  );
+  return execSync(`node --expose-gc --allow-natives-syntax ${script} ${args}`, {
+    encoding: "utf8",
+  });
+}
+
+/** Run a fixture file via bin/benchforge and return output */
+function executeBenchforgeFile(file: string, args = ""): string {
+  const bin = path.join(import.meta.dirname!, "../../bin/benchforge");
+  const fixture = path.join(import.meta.dirname!, "fixtures", file);
+  return execSync(`${bin} ${fixture} ${args}`, { encoding: "utf8" });
+}
+
 test("runs all benchmarks", { timeout: 30000 }, async () => {
-  const output = await runBenchCLITest(testSuite, "--time 0.1");
+  const output = await runBenchCLITest(testSuite, "--duration 0.1");
 
   expect(output).toContain("concatenation");
   expect(output).toContain("template literal");
@@ -61,7 +79,10 @@ test("runs all benchmarks", { timeout: 30000 }, async () => {
 });
 
 test("filters by substring", { timeout: 15000 }, async () => {
-  const output = await runBenchCLITest(testSuite, "--filter concat --time 0.1");
+  const output = await runBenchCLITest(
+    testSuite,
+    "--filter concat --duration 0.1",
+  );
 
   expect(output).toContain("concatenation");
   expect(output).not.toContain("addition");
@@ -70,7 +91,7 @@ test("filters by substring", { timeout: 15000 }, async () => {
 test("filters by regex", { timeout: 15000 }, async () => {
   const output = await runBenchCLITest(
     testSuite,
-    "--filter ^template --time 0.1",
+    "--filter ^template --duration 0.1",
   );
   expect(output).toContain("template literal");
   expect(output).not.toContain("addition");
@@ -87,26 +108,8 @@ test("filter preserves suite structure", () => {
   expect(filtered.groups[1].benchmarks).toHaveLength(0);
 });
 
-/** Execute test fixture script and return output */
-function executeTestScript(args = ""): string {
-  const script = path.join(
-    import.meta.dirname!,
-    "fixtures/test-bench-script.ts",
-  );
-  return execSync(`node --expose-gc --allow-natives-syntax ${script} ${args}`, {
-    encoding: "utf8",
-  });
-}
-
-/** Run a fixture file via bin/benchforge and return output */
-function executeBenchforgeFile(file: string, args = ""): string {
-  const bin = path.join(import.meta.dirname!, "../../bin/benchforge");
-  const fixture = path.join(import.meta.dirname!, "fixtures", file);
-  return execSync(`${bin} ${fixture} ${args}`, { encoding: "utf8" });
-}
-
 test("e2e: runs user script", { timeout: 30000 }, () => {
-  const output = executeTestScript("--time 0.1");
+  const output = executeTestScript("--duration 0.1");
 
   expect(output).toContain("plus");
   expect(output).toContain("multiply");
@@ -122,14 +125,14 @@ test("e2e: runs user script", { timeout: 30000 }, () => {
 });
 
 test("e2e: filter flag", { timeout: 30000 }, () => {
-  const output = executeTestScript('--filter "plus" --time 0.1');
+  const output = executeTestScript('--filter "plus" --duration 0.1');
 
   expect(output).toContain("plus");
   expect(output).not.toContain("multiply");
 });
 
 test("runs benchmarks with setup function", { timeout: 30000 }, async () => {
-  const output = await runBenchCLITest(suiteWithSetup, "--time 0.1");
+  const output = await runBenchCLITest(suiteWithSetup, "--duration 0.1");
 
   expect(output).toContain("sum numbers");
   expect(output).toContain("join strings");
@@ -137,40 +140,38 @@ test("runs benchmarks with setup function", { timeout: 30000 }, async () => {
   expect(output).toContain("runs");
 });
 
-test(
-  "runs benchmarks with baseline comparison",
-  { timeout: 30000 },
-  async () => {
-    const suiteWithBaseline: BenchSuite = {
-      name: "Baseline Test",
-      groups: [
-        {
-          name: "Sort Comparison",
-          setup: () => ({
-            data: Array.from({ length: 10 }, () => Math.random()),
-          }),
-          baseline: {
-            name: "baseline sort",
-            fn: ({ data }: any) => [...data].sort(),
-          },
-          benchmarks: [
-            {
-              name: "optimized sort",
-              fn: ({ data }: any) => [...data].sort((a, b) => a - b),
-            },
-          ],
+test("runs benchmarks with baseline comparison", {
+  timeout: 30000,
+}, async () => {
+  const suiteWithBaseline: BenchSuite = {
+    name: "Baseline Test",
+    groups: [
+      {
+        name: "Sort Comparison",
+        setup: () => ({
+          data: Array.from({ length: 10 }, () => Math.random()),
+        }),
+        baseline: {
+          name: "baseline sort",
+          fn: ({ data }: any) => [...data].sort(),
         },
-      ],
-    };
+        benchmarks: [
+          {
+            name: "optimized sort",
+            fn: ({ data }: any) => [...data].sort((a, b) => a - b),
+          },
+        ],
+      },
+    ],
+  };
 
-    const output = await runBenchCLITest(suiteWithBaseline, "--time 0.01");
+  const output = await runBenchCLITest(suiteWithBaseline, "--iterations 20");
 
-    expect(output).toContain("baseline sort");
-    expect(output).toContain("optimized sort");
-    expect(output).toContain("Δ%"); // Diff column should appear
-    expect(output).toContain("mean");
-  },
-);
+  expect(output).toContain("baseline sort");
+  expect(output).toContain("optimized sort");
+  expect(output).toContain("Δ%"); // Diff column should appear
+  expect(output).toContain("mean");
+});
 
 test("file mode: BenchSuite export", { timeout: 30000 }, () => {
   const output = executeBenchforgeFile(

package/src/test/RunnerOrchestrator.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import type { BenchmarkSpec } from "../Benchmark.ts";
+import type { BenchmarkSpec } from "../runners/BenchmarkSpec.ts";
 import { runBenchmark } from "../runners/RunnerOrchestrator.ts";
 
 /** lightweight function for testing worker communication */
@@ -9,15 +9,15 @@ function simpleTestFunction(): number {
   return sum;
 }
 
-test("BasicRunner runs benchmark in worker mode", async () => {
+test("TimingRunner runs benchmark in worker mode", async () => {
   const spec: BenchmarkSpec = {
-    name: "basic-worker-test",
+    name: "timing-worker-test",
     fn: simpleTestFunction,
   };
 
   const results = await runBenchmark({
     spec,
-    runner: "basic",
+    runner: "timing",
     options: {
       maxTime: 5,
       maxIterations: 50,
@@ -28,7 +28,7 @@ test("BasicRunner runs benchmark in worker mode", async () => {
   expect(results).toHaveLength(1);
   const result = results[0];
 
-  expect(result.name).toBe("basic-worker-test");
+  expect(result.name).toBe("timing-worker-test");
   expect(result.samples.length).toBeGreaterThan(0);
   expect(result.samples.length).toBeLessThanOrEqual(500);
   expect(result.time.min).toBeGreaterThan(0);
@@ -38,15 +38,15 @@ test("BasicRunner runs benchmark in worker mode", async () => {
   expect(result.time.p99).toBeGreaterThan(0);
 });
 
-test("BasicRunner runs benchmark in non-worker mode", async () => {
+test("TimingRunner runs benchmark in non-worker mode", async () => {
   const spec: BenchmarkSpec = {
-    name: "basic-test",
+    name: "timing-test",
     fn: simpleTestFunction,
   };
 
   const results = await runBenchmark({
     spec,
-    runner: "basic",
+    runner: "timing",
     options: {
       maxTime: 5,
       maxIterations: 50,
@@ -57,12 +57,12 @@ test("BasicRunner runs benchmark in non-worker mode", async () => {
   expect(results).toHaveLength(1);
   const result = results[0];
 
-  expect(result.name).toBe("basic-test");
+  expect(result.name).toBe("timing-test");
   expect(result.samples.length).toBeGreaterThan(0);
   expect(result.time.p50).toBeGreaterThan(0);
 });
 
-test("BasicRunner with parameterized benchmark", async () => {
+test("TimingRunner with parameterized benchmark", async () => {
   const spec: BenchmarkSpec<number> = {
     name: "parameterized-test",
     fn: (n: number) => {
@@ -74,7 +74,7 @@ test("BasicRunner with parameterized benchmark", async () => {
 
   const results = await runBenchmark({
     spec,
-    runner: "basic",
+    runner: "timing",
     options: { maxTime: 5, maxIterations: 20 },
     useWorker: false,
     params: 100,
@@ -94,7 +94,7 @@ test("RunnerOrchestrator propagates errors from worker", async () => {
 
   const promise = runBenchmark({
     spec,
-    runner: "basic",
+    runner: "timing",
     options: { maxTime: 1, maxIterations: 1 },
     useWorker: true,
   });

package/src/test/StatisticalUtils.test.ts

@@ -1,14 +1,19 @@
 import { expect, test } from "vitest";
+import {
+  blockDifferenceCI,
+  sampleDifferenceCI,
+} from "../stats/BootstrapDifference.ts";
 import {
   average,
-  bootstrapDifferenceCI,
-  bootstrapMedian,
+  blockBootstrap,
   coefficientOfVariation,
   findOutliers,
+  median,
   medianAbsoluteDeviation,
   percentile,
+  sampleBootstrap,
   standardDeviation,
-} from "../StatisticalUtils.ts";
+} from "../stats/StatisticalUtils.ts";
 import { assertValid, getSampleData } from "./TestUtils.ts";
 
 test("calculates mean correctly", () => {
@@ -68,11 +73,21 @@ test("identifies outliers in mixed data", () => {
   expect(outliers.indices).toContain(51);
 });
 
-test("bootstrap estimates median with confidence intervals", () => {
+test("sampleBootstrap estimates median with CI", () => {
+  const stable = getSampleData(400, 450);
+  const result = sampleBootstrap(stable, median, { resamples: 1000 });
+  expect(result.ciLevel).toBe("sample");
+  expect(result.ci[0]).toBeLessThanOrEqual(result.estimate);
+  expect(result.ci[1]).toBeGreaterThanOrEqual(result.estimate);
+});
+
+test("blockBootstrap estimates median with confidence intervals", () => {
   const stable = getSampleData(400, 450);
   const actual = percentile(stable, 0.5);
-  const result = bootstrapMedian(stable, { resamples: 1000 });
+  const blocks = Array.from({ length: 5 }, (_, i) => i * 10);
+  const result = blockBootstrap(stable, blocks, median, { resamples: 1000 });
 
+  expect(result.ciLevel).toBe("block");
   expect(result.estimate).toBeCloseTo(actual, 1);
   expect(result.ci[0]).toBeLessThanOrEqual(result.estimate);
   expect(result.ci[1]).toBeGreaterThanOrEqual(result.estimate);
@@ -80,32 +95,53 @@ test("bootstrap estimates median with confidence intervals", () => {
   expect(result.samples).toHaveLength(1000);
 });
 
-test("bootstrapDifferenceCI detects improvement", () => {
+test("sampleDifferenceCI detects improvement", () => {
+  const baseline = getSampleData(0, 100);
+  const improved = baseline.map(v => v * 0.8);
+  const result = sampleDifferenceCI(baseline, improved, median, {
+    resamples: 1000,
+  });
+  expect(result.ciLevel).toBe("sample");
+  expect(result.percent).toBeCloseTo(-20, 0);
+});
+
+test("blockDifferenceCI detects improvement", () => {
   const baseline = getSampleData(0, 100);
   const improved = baseline.map(v => v * 0.8);
-  const result = bootstrapDifferenceCI(baseline, improved, { resamples: 1000 });
+  const blocks = Array.from({ length: 10 }, (_, i) => i * 10);
+  const result = blockDifferenceCI(baseline, blocks, improved, median, {
+    resamples: 1000,
+  });
 
+  expect(result.ciLevel).toBe("block");
   expect(result.percent).toBeCloseTo(-20, 0);
   expect(result.ci[1]).toBeLessThan(0);
   expect(result.direction).toBe("faster");
 });
 
-test("bootstrapDifferenceCI detects regression", () => {
+test("blockDifferenceCI detects regression", () => {
   const baseline = getSampleData(0, 100);
   const slower = baseline.map(v => v * 1.2);
-  const result = bootstrapDifferenceCI(baseline, slower, { resamples: 1000 });
+  const blocks = Array.from({ length: 10 }, (_, i) => i * 10);
+  const result = blockDifferenceCI(baseline, blocks, slower, median, {
+    resamples: 1000,
+  });
 
+  expect(result.ciLevel).toBe("block");
   expect(result.percent).toBeCloseTo(20, 0);
   expect(result.ci[0]).toBeGreaterThan(0);
   expect(result.direction).toBe("slower");
 });
 
-test("bootstrapDifferenceCI shows uncertainty for noise", () => {
+test("blockDifferenceCI shows uncertainty for noise", () => {
   const baseline = getSampleData(0, 100);
   const noisy = baseline.map(v => v + (Math.random() - 0.5) * 2);
-  const result = bootstrapDifferenceCI(baseline, noisy, { resamples: 1000 });
+  const blocks = Array.from({ length: 10 }, (_, i) => i * 10);
+  const result = blockDifferenceCI(baseline, blocks, noisy, median, {
+    resamples: 1000,
+  });
 
-  // CI should span zero for no real change
+  expect(result.ciLevel).toBe("block");
   expect(result.ci[0]).toBeLessThanOrEqual(0);
   expect(result.ci[1]).toBeGreaterThanOrEqual(0);
   expect(result.direction).toBe("uncertain");

package/src/{table-util/test → test}/TableReport.test.ts

@@ -1,10 +1,10 @@
 import { expect, test } from "vitest";
-import { integer } from "../Formatters.ts";
+import { integer } from "../report/Formatters.ts";
 import {
   buildTable,
   type ColumnGroup,
   type ResultGroup,
-} from "../TableReport.ts";
+} from "../report/text/TableReport.ts";
 
 interface TestRecord {
   name: string;

package/src/test/TestUtils.ts

@@ -1,11 +1,40 @@
-import type { BenchSuite } from "../Benchmark.ts";
-import type { BenchmarkReport } from "../BenchmarkReport.ts";
 import type { Configure, DefaultCliArgs } from "../cli/CliArgs.ts";
 import { parseCliArgs } from "../cli/CliArgs.ts";
-import { defaultReport, runBenchmarks } from "../cli/RunBenchCLI.ts";
-import type { MeasuredResults } from "../MeasuredResults.ts";
-import { average, percentile } from "../StatisticalUtils.ts";
-import { bevy30SamplesMs } from "../tests/fixtures/bevy30-samples.ts";
+import { defaultReport } from "../cli/CliReport.ts";
+import { runBenchmarks } from "../cli/SuiteRunner.ts";
+import type { BenchmarkReport } from "../report/BenchmarkReport.ts";
+import type { BenchSuite } from "../runners/BenchmarkSpec.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { average, percentile } from "../stats/StatisticalUtils.ts";
+import { bevy30SamplesMs } from "./fixtures/bevy30-samples.ts";
+
+/** Validation helpers for statistical tests */
+export const assertValid: {
+  pValue: (value: number) => void;
+  percentileOrder: (p25: number, p50: number, p75: number, p99: number) => void;
+  significance: (level: string) => void;
+} = {
+  pValue: (value: number) => {
+    if (value < 0 || value > 1) {
+      throw new Error(`Expected p-value between 0 and 1, got ${value}`);
+    }
+  },
+
+  percentileOrder: (p25: number, p50: number, p75: number, p99: number) => {
+    if (!(p25 <= p50 && p50 <= p75 && p75 <= p99)) {
+      throw new Error(
+        `Percentiles not ordered: p25=${p25}, p50=${p50}, p75=${p75}, p99=${p99}`,
+      );
+    }
+  },
+
+  significance: (level: string) => {
+    const valid = ["none", "weak", "good", "strong"];
+    if (!valid.includes(level)) {
+      throw new Error(`Invalid significance level: ${level}`);
+    }
+  },
+};
 
 /** @return formatted benchmark output for CLI testing */
 export async function runBenchCLITest<T = DefaultCliArgs>(
@@ -67,27 +96,3 @@ export function createBenchmarkReport(
     measuredResults: createMeasuredResults(sampleRange, overrides),
   };
 }
-
-/** Validation helpers for statistical tests */
-export const assertValid = {
-  pValue: (value: number) => {
-    if (value < 0 || value > 1) {
-      throw new Error(`Expected p-value between 0 and 1, got ${value}`);
-    }
-  },
-
-  percentileOrder: (p25: number, p50: number, p75: number, p99: number) => {
-    if (!(p25 <= p50 && p50 <= p75 && p75 <= p99)) {
-      throw new Error(
-        `Percentiles not ordered: p25=${p25}, p50=${p50}, p75=${p75}, p99=${p99}`,
-      );
-    }
-  },
-
-  significance: (level: string) => {
-    const valid = ["none", "weak", "good", "strong"];
-    if (!valid.includes(level)) {
-      throw new Error(`Invalid significance level: ${level}`);
-    }
-  },
-};

package/src/test/TimeExport.test.ts

@@ -0,0 +1,139 @@
+import { expect, test } from "vitest";
+import { timeProfileToSpeedscope } from "../export/TimeExport.ts";
+import type { TimeProfile } from "../profiling/node/TimeSampler.ts";
+
+/** Build a minimal TimeProfile for testing */
+function mockProfile(): TimeProfile {
+  return {
+    nodes: [
+      {
+        id: 1,
+        callFrame: { functionName: "", url: "", lineNumber: -1 },
+        children: [2],
+      },
+      {
+        id: 2,
+        callFrame: {
+          functionName: "main",
+          url: "file:///app.ts",
+          lineNumber: 9,
+          columnNumber: 0,
+        },
+        children: [3],
+      },
+      {
+        id: 3,
+        callFrame: {
+          functionName: "compute",
+          url: "file:///app.ts",
+          lineNumber: 19,
+          columnNumber: 4,
+        },
+        hitCount: 5,
+      },
+    ],
+    startTime: 0,
+    endTime: 5000,
+    samples: [3, 3, 2, 3, 3],
+    timeDeltas: [1000, 1000, 1000, 1000, 1000],
+  };
+}
+
+test("converts TimeProfile to valid SpeedScope format", () => {
+  const profile = mockProfile();
+  const file = timeProfileToSpeedscope("test-bench", profile);
+
+  expect(file.$schema).toBe(
+    "https://www.speedscope.app/file-format-schema.json",
+  );
+  expect(file.exporter).toBe("benchforge");
+  expect(file.profiles).toHaveLength(1);
+
+  const p = file.profiles[0];
+  expect(p.type).toBe("sampled");
+  expect(p.name).toBe("test-bench");
+  expect(p.unit).toBe("microseconds");
+  expect(p.samples).toHaveLength(5);
+  expect(p.weights).toEqual([1000, 1000, 1000, 1000, 1000]);
+  expect(p.endValue).toBe(5000);
+});
+
+test("resolves stacks from leaf to root (root-first order)", () => {
+  const profile = mockProfile();
+  const file = timeProfileToSpeedscope("test", profile);
+
+  const p = file.profiles[0];
+  const frames = file.shared.frames;
+
+  // Sample at node 3 (compute) should have stack: [main, compute]
+  const deepStack = p.samples[0];
+  expect(deepStack).toHaveLength(2); // root is skipped
+  expect(frames[deepStack[0]].name).toBe("main");
+  expect(frames[deepStack[1]].name).toBe("compute");
+
+  // Sample at node 2 (main) should have stack: [main]
+  const shallowStack = p.samples[2];
+  expect(shallowStack).toHaveLength(1);
+  expect(frames[shallowStack[0]].name).toBe("main");
+});
+
+test("deduplicates shared frames", () => {
+  const profile = mockProfile();
+  const file = timeProfileToSpeedscope("test", profile);
+
+  // "main" and "compute" — only 2 unique frames
+  expect(file.shared.frames).toHaveLength(2);
+});
+
+test("handles empty samples gracefully", () => {
+  const profile: TimeProfile = {
+    nodes: [
+      { id: 1, callFrame: { functionName: "", url: "", lineNumber: -1 } },
+    ],
+    startTime: 0,
+    endTime: 0,
+    samples: [],
+    timeDeltas: [],
+  };
+  const file = timeProfileToSpeedscope("empty", profile);
+
+  expect(file.profiles[0].samples).toHaveLength(0);
+  expect(file.profiles[0].weights).toHaveLength(0);
+  expect(file.profiles[0].endValue).toBe(0);
+});
+
+test("converts 0-indexed V8 lines to 1-indexed", () => {
+  const profile = mockProfile();
+  const file = timeProfileToSpeedscope("test", profile);
+
+  const mainFrame = file.shared.frames.find(f => f.name === "main")!;
+  expect(mainFrame.line).toBe(10); // lineNumber 9 -> line 10
+  expect(mainFrame.col).toBe(1); // columnNumber 0 -> col 1
+});
+
+test("anonymous functions get location hint in name", () => {
+  const profile: TimeProfile = {
+    nodes: [
+      {
+        id: 1,
+        callFrame: { functionName: "", url: "", lineNumber: -1 },
+        children: [2],
+      },
+      {
+        id: 2,
+        callFrame: {
+          functionName: "",
+          url: "file:///lib/utils.ts",
+          lineNumber: 41,
+        },
+      },
+    ],
+    startTime: 0,
+    endTime: 1000,
+    samples: [2],
+    timeDeltas: [1000],
+  };
+  const file = timeProfileToSpeedscope("test", profile);
+
+  expect(file.shared.frames[0].name).toBe("(anonymous utils.ts:42)");
+});

package/src/test/TimeSampler.test.ts

@@ -0,0 +1,37 @@
+import { expect, test } from "vitest";
+import { withTimeProfiling } from "../profiling/node/TimeSampler.ts";
+
+test("withTimeProfiling returns valid V8 CPU profile", async () => {
+  // Burn some CPU to produce samples
+  const { result, profile } = await withTimeProfiling({}, () => {
+    let sum = 0;
+    for (let i = 0; i < 1e6; i++) sum += Math.sqrt(i);
+    return sum;
+  });
+
+  expect(result).toBeGreaterThan(0);
+  expect(profile.nodes.length).toBeGreaterThan(0);
+  expect(profile.startTime).toBeLessThan(profile.endTime);
+  expect(profile.samples).toBeDefined();
+  expect(profile.timeDeltas).toBeDefined();
+  expect(profile.samples!.length).toBe(profile.timeDeltas!.length);
+
+  // Verify node structure
+  const node = profile.nodes[0];
+  expect(node).toHaveProperty("id");
+  expect(node).toHaveProperty("callFrame");
+  expect(node.callFrame).toHaveProperty("functionName");
+  expect(node.callFrame).toHaveProperty("url");
+  expect(node.callFrame).toHaveProperty("lineNumber");
+});
+
+test("withTimeProfiling respects custom interval", async () => {
+  const { profile } = await withTimeProfiling({ interval: 100 }, () => {
+    let sum = 0;
+    for (let i = 0; i < 1e6; i++) sum += Math.sqrt(i);
+    return sum;
+  });
+
+  // Finer interval should produce more samples for the same work
+  expect(profile.samples!.length).toBeGreaterThan(0);
+});