benchforge 0.1.9 → 0.2.4

package/src/test/BrowserBench.test.ts

@@ -2,8 +2,8 @@ import { expect, test } from "vitest";
 import {
   browserGcStats,
   parseGcTraceEvents,
-  type TraceEvent,
-} from "../browser/BrowserGcStats.ts";
+} from "../profiling/browser/BrowserGcStats.ts";
+import type { TraceEvent } from "../profiling/browser/ChromeTraceEvent.ts";
 
 test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
   const events: TraceEvent[] = [
@@ -11,6 +11,7 @@ test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
       cat: "v8.gc",
       name: "MinorGC",
       ph: "X",
+      ts: 0,
       dur: 500,
       args: { usedHeapSizeBefore: 10000, usedHeapSizeAfter: 8000 },
     },
@@ -18,6 +19,7 @@ test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
       cat: "v8.gc",
       name: "MajorGC",
       ph: "X",
+      ts: 0,
       dur: 12000,
       args: { usedHeapSizeBefore: 50000, usedHeapSizeAfter: 30000 },
     },
@@ -38,9 +40,9 @@ test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
 
 test("parseGcTraceEvents ignores non-complete and non-GC events", () => {
   const events: TraceEvent[] = [
-    { cat: "v8.gc", name: "MinorGC", ph: "B", dur: 500 }, // not complete
-    { cat: "v8", name: "V8.Execute", ph: "X", dur: 100 }, // not GC
-    { cat: "v8.gc", name: "MinorGC", ph: "X" }, // valid, missing dur/args
+    { cat: "v8.gc", name: "MinorGC", ph: "B", ts: 0, dur: 500 }, // not complete
+    { cat: "v8", name: "V8.Execute", ph: "X", ts: 0, dur: 100 }, // not GC
+    { cat: "v8.gc", name: "MinorGC", ph: "X", ts: 0 }, // valid, missing dur/args
   ];
   const parsed = parseGcTraceEvents(events);
   expect(parsed).toHaveLength(1);
@@ -53,6 +55,7 @@ test("browserGcStats aggregates trace events into GcStats", () => {
       cat: "v8.gc",
       name: "MinorGC",
       ph: "X",
+      ts: 0,
       dur: 300,
       args: { usedHeapSizeBefore: 5000, usedHeapSizeAfter: 3000 },
     },
@@ -60,6 +63,7 @@ test("browserGcStats aggregates trace events into GcStats", () => {
       cat: "v8.gc",
       name: "MinorGC",
       ph: "X",
+      ts: 0,
       dur: 200,
       args: { usedHeapSizeBefore: 6000, usedHeapSizeAfter: 4000 },
     },
@@ -67,6 +71,7 @@ test("browserGcStats aggregates trace events into GcStats", () => {
       cat: "v8.gc",
       name: "MajorGC",
       ph: "X",
+      ts: 0,
       dur: 8000,
       args: { usedHeapSizeBefore: 40000, usedHeapSizeAfter: 20000 },
     },

package/src/test/BuildTimeSection.test.ts

@@ -0,0 +1,130 @@
+import { expect, test } from "vitest";
+import type { BenchmarkReport } from "../report/BenchmarkReport.ts";
+import { computeColumnValues } from "../report/BenchmarkReport.ts";
+import { buildTimeSection } from "../report/StandardSections.ts";
+import { reportResults, valuesForReports } from "../report/text/TextReport.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+
+/** @return minimal MeasuredResults with the given samples (time fields derived trivially). */
+function measured(samples: number[]): MeasuredResults {
+  const sorted = [...samples].sort((a, b) => a - b);
+  return {
+    name: "t",
+    samples,
+    time: {
+      min: sorted[0],
+      max: sorted[sorted.length - 1],
+      avg: samples.reduce((a, b) => a + b, 0) / samples.length,
+      p50: sorted[Math.floor(sorted.length * 0.5)],
+      p75: sorted[Math.floor(sorted.length * 0.75)],
+      p99: sorted[Math.floor(sorted.length * 0.99)],
+      p999: sorted[Math.floor(sorted.length * 0.999)],
+    },
+  };
+}
+
+function report(name: string, samples: number[]): BenchmarkReport {
+  return { name, measuredResults: measured(samples) };
+}
+
+function range(n: number): number[] {
+  return Array.from({ length: n }, (_, i) => i + 1);
+}
+
+test("default buildTimeSection produces mean, p50, p99 columns", () => {
+  const section = buildTimeSection();
+  expect(section.columns.map(c => c.key ?? c.title)).toEqual([
+    "mean",
+    "p50",
+    "p99",
+  ]);
+});
+
+test("computeColumnValues computes values from samples", () => {
+  const section = buildTimeSection("mean,p50,max,min");
+  const row = computeColumnValues(section, measured([10, 20, 30, 40, 50]));
+  expect(row.mean).toBe(30);
+  expect(row.min).toBe(10);
+  expect(row.max).toBe(50);
+  expect(row.p50).toBeGreaterThanOrEqual(20);
+  expect(row.p50).toBeLessThanOrEqual(40);
+});
+
+test("p70 returns value near 70th percentile of [1..100]", () => {
+  const section = buildTimeSection("p70");
+  const row = computeColumnValues(section, measured(range(100)));
+  expect(row.p70).toBeGreaterThanOrEqual(69);
+  expect(row.p70).toBeLessThanOrEqual(71);
+});
+
+test("p999 uses divide-by-1000 convention", () => {
+  const section = buildTimeSection("p999");
+  const row = computeColumnValues(section, measured(range(1000)));
+  expect(row.p999).toBeGreaterThanOrEqual(999);
+});
+
+test("p9999 uses divide-by-10000 convention", () => {
+  const section = buildTimeSection("p9999");
+  const row = computeColumnValues(section, measured(range(10000)));
+  expect(row.p9999).toBeGreaterThanOrEqual(9999);
+});
+
+test("median and p50 produce the same value", () => {
+  const a = computeColumnValues(
+    buildTimeSection("median"),
+    measured(range(100)),
+  );
+  const b = computeColumnValues(buildTimeSection("p50"), measured(range(100)));
+  expect(a.p50).toBe(b.p50);
+});
+
+test("mean and avg dedupe to a single column", () => {
+  const section = buildTimeSection("mean,avg");
+  expect(section.columns.length).toBe(1);
+});
+
+test("min and max return exact values", () => {
+  const section = buildTimeSection("min,max");
+  const row = computeColumnValues(section, measured([5, 1, 9, 3, 7]));
+  expect(row.min).toBe(1);
+  expect(row.max).toBe(9);
+});
+
+test("empty stats string throws", () => {
+  expect(() => buildTimeSection("")).toThrow(/at least one column/);
+  expect(() => buildTimeSection("  ,  ")).toThrow(/at least one column/);
+});
+
+test("unknown token throws with vocabulary hint", () => {
+  expect(() => buildTimeSection("wat")).toThrow(
+    /expected mean, median, min, max, or p<N>/,
+  );
+});
+
+test("single-digit percentile token is rejected", () => {
+  expect(() => buildTimeSection("p5")).toThrow(/at least 2 digits/);
+});
+
+test("3+ digit percentile tokens not starting with 9 are rejected", () => {
+  expect(() => buildTimeSection("p100")).toThrow(/must start with 9/);
+  expect(() => buildTimeSection("p500")).toThrow(/must start with 9/);
+  expect(() => buildTimeSection("p1000")).toThrow(/must start with 9/);
+});
+
+test("reportResults renders user-chosen columns as table headers", () => {
+  const groups = [{ name: "g", reports: [report("bench", range(100))] }];
+  const table = reportResults(groups, [buildTimeSection("p70,p95")]);
+  expect(table).toContain("p70");
+  expect(table).toContain("p95");
+});
+
+test("valuesForReports extracts user-chosen keys", () => {
+  const rows = valuesForReports(
+    [report("bench", range(100))],
+    [buildTimeSection("p70,p95")],
+  );
+  expect(rows[0].p70).toBeGreaterThanOrEqual(69);
+  expect(rows[0].p70).toBeLessThanOrEqual(71);
+  expect(rows[0].p95).toBeGreaterThanOrEqual(94);
+  expect(rows[0].p95).toBeLessThanOrEqual(96);
+});

package/src/test/CapSamples.test.ts

@@ -0,0 +1,82 @@
+import { expect, test } from "vitest";
+import { sampleDifferenceCI } from "../stats/BootstrapDifference.ts";
+import {
+  average,
+  maxBootstrapInput,
+  percentile,
+  sampleBootstrap,
+} from "../stats/StatisticalUtils.ts";
+
+test("sampleBootstrap uses full samples for point estimate", () => {
+  const samples = Array.from({ length: 5000 }, (_, i) => i);
+  const result = sampleBootstrap(samples, average, { resamples: 100 });
+  expect(result.estimate).toBe(average(samples));
+});
+
+test("sampleDifferenceCI preserves point estimate", () => {
+  const a = Array.from({ length: 5000 }, () => 50 + Math.random() * 10);
+  const b = a.map(v => v * 1.1);
+  const result = sampleDifferenceCI(a, b, average, { resamples: 100 });
+  const expected = ((average(b) - average(a)) / average(a)) * 100;
+  expect(result.percent).toBeCloseTo(expected, 10);
+});
+
+test("sampleBootstrap point estimate uses full array when capped", () => {
+  const n = maxBootstrapInput + 5000;
+  const samples = Array.from({ length: n }, (_, i) => i);
+  const result = sampleBootstrap(samples, average, { resamples: 50 });
+  expect(result.estimate).toBe(average(samples));
+  expect(result.subsampled).toBe(n);
+});
+
+test("sampleBootstrap does not set subsampled when under cap", () => {
+  const samples = Array.from({ length: 100 }, (_, i) => i);
+  const result = sampleBootstrap(samples, average, { resamples: 50 });
+  expect(result.subsampled).toBeUndefined();
+});
+
+test("sampleDifferenceCI sets subsampled when inputs exceed cap", () => {
+  const n = maxBootstrapInput + 1000;
+  const a = Array.from({ length: n }, () => 50 + Math.random() * 10);
+  const b = a.map(v => v * 1.1);
+  const result = sampleDifferenceCI(a, b, average, { resamples: 50 });
+  expect(result.percent).toBeCloseTo(10, 0);
+  expect(result.subsampled).toBe(n);
+});
+
+test("sampleDifferenceCI no subsampled flag when under cap", () => {
+  const a = Array.from({ length: 100 }, () => 50 + Math.random() * 10);
+  const b = a.map(v => v * 1.1);
+  const result = sampleDifferenceCI(a, b, average, { resamples: 50 });
+  expect(result.subsampled).toBeUndefined();
+});
+
+test("quickselect-based percentile matches sorted percentile", () => {
+  const data = Array.from({ length: 1000 }, () => Math.random() * 100);
+  const sorted = [...data].sort((a, b) => a - b);
+  for (const p of [0.25, 0.5, 0.75, 0.99]) {
+    const k = Math.max(0, Math.ceil(sorted.length * p) - 1);
+    expect(percentile(data, p)).toBe(sorted[k]);
+  }
+});
+
+test("quickselect handles small arrays", () => {
+  expect(percentile([42], 0.5)).toBe(42);
+  expect(percentile([1, 2], 0.5)).toBe(1);
+  expect(percentile([1, 2], 1.0)).toBe(2);
+});
+
+test("quickselect handles duplicate values", () => {
+  const data = [5, 5, 5, 5, 5, 10, 10, 10, 10, 10];
+  expect(percentile(data, 0.5)).toBe(5);
+  expect(percentile(data, 0.99)).toBe(10);
+});
+
+test("sampleBootstrap reuses buffer (no per-iteration allocation)", () => {
+  const samples = [10, 20, 30, 40, 50];
+  const result = sampleBootstrap(samples, average, { resamples: 50 });
+  expect(result.estimate).toBe(average(samples));
+  expect(result.samples).toHaveLength(50);
+  expect(result.ci[0]).toBeLessThanOrEqual(result.estimate);
+  expect(result.ci[1]).toBeGreaterThanOrEqual(result.estimate);
+});

package/src/test/CoverageExport.test.ts

@@ -0,0 +1,115 @@
+import { expect, test } from "vitest";
+import {
+  annotateFramesWithCounts,
+  buildCoverageMap,
+} from "../export/CoverageExport.ts";
+import type { CoverageData } from "../profiling/node/CoverageTypes.ts";
+
+const source = `function foo() {
+  return 1;
+}
+function bar() {
+  return 2;
+}
+const baz = () => 3;
+`;
+
+const coverage: CoverageData = {
+  scripts: [
+    {
+      url: "file:///test.js",
+      functions: [
+        {
+          functionName: "foo",
+          ranges: [{ startOffset: 0, endOffset: 30, count: 10 }],
+        },
+        {
+          functionName: "bar",
+          ranges: [{ startOffset: 31, endOffset: 60, count: 5 }],
+        },
+        {
+          functionName: "",
+          ranges: [{ startOffset: 61, endOffset: 80, count: 3 }],
+        },
+      ],
+    },
+  ],
+};
+
+test("buildCoverageMap resolves offsets to lines", () => {
+  const result = buildCoverageMap(coverage, { "file:///test.js": source });
+
+  expect(result.map.has("file:///test.js")).toBe(true);
+  const entries = result.map.get("file:///test.js")!;
+  expect(entries).toHaveLength(3);
+
+  const foo = entries.find(e => e.functionName === "foo");
+  expect(foo).toBeDefined();
+  expect(foo!.startLine).toBe(1);
+  expect(foo!.count).toBe(10);
+
+  const bar = entries.find(e => e.functionName === "bar");
+  expect(bar).toBeDefined();
+  expect(bar!.startLine).toBe(4);
+  expect(bar!.count).toBe(5);
+
+  // byName aggregates across all scripts
+  expect(result.byName.get("foo")).toBe(10);
+  expect(result.byName.get("bar")).toBe(5);
+});
+
+test("annotateFramesWithCounts appends [N] to matched frames", () => {
+  const result = buildCoverageMap(coverage, { "file:///test.js": source });
+
+  const frames = [
+    { name: "foo", file: "file:///test.js", line: 1 },
+    { name: "bar", file: "file:///test.js", line: 4 },
+    { name: "unmatched", file: "file:///other.js", line: 1 },
+  ];
+
+  annotateFramesWithCounts(frames, result);
+
+  expect(frames[0].name).toBe("foo [10]");
+  expect(frames[1].name).toBe("bar [5]");
+  expect(frames[2].name).toBe("unmatched"); // no coverage data for this file
+});
+
+test("annotateFramesWithCounts falls back to name-only for frames without file", () => {
+  const result = buildCoverageMap(coverage, { "file:///test.js": source });
+
+  const frames = [
+    { name: "foo" }, // no file — should match by name
+    { name: "bar" },
+    { name: "(anonymous)" }, // anonymous — should not match by name
+  ];
+
+  annotateFramesWithCounts(frames, result);
+
+  expect(frames[0].name).toBe("foo [10]");
+  expect(frames[1].name).toBe("bar [5]");
+  expect(frames[2].name).toBe("(anonymous)");
+});
+
+test("annotateFramesWithCounts formats large counts", () => {
+  const bigCoverage: CoverageData = {
+    scripts: [
+      {
+        url: "file:///big.js",
+        functions: [
+          {
+            functionName: "hot",
+            ranges: [{ startOffset: 0, endOffset: 10, count: 1_500_000 }],
+          },
+        ],
+      },
+    ],
+  };
+  const result = buildCoverageMap(bigCoverage, {
+    "file:///big.js": "function hot() {}",
+  });
+  const frames = [{ name: "hot", file: "file:///big.js", line: 1 }];
+
+  annotateFramesWithCounts(frames, result);
+
+  expect(frames[0].name).toBe("hot [1.5M]");
+});

package/src/test/CoverageSampler.test.ts

@@ -0,0 +1,33 @@
+import { expect, test } from "vitest";
+import { withCoverageProfiling } from "../profiling/node/CoverageSampler.ts";
+
+test("withCoverageProfiling returns function execution counts", async () => {
+  function hotFunction() {
+    let sum = 0;
+    for (let i = 0; i < 100; i++) sum += i;
+    return sum;
+  }
+
+  const { result, coverage } = await withCoverageProfiling(_session => {
+    for (let i = 0; i < 10; i++) hotFunction();
+    return 42;
+  });
+
+  expect(result).toBe(42);
+  expect(coverage.scripts.length).toBeGreaterThan(0);
+
+  // Find our test file in the coverage data
+  const thisScript = coverage.scripts.find(s =>
+    s.url.includes("CoverageSampler.test"),
+  );
+  expect(thisScript).toBeDefined();
+  expect(thisScript!.functions.length).toBeGreaterThan(0);
+
+  // Find hotFunction and verify its count
+  const hotFn = thisScript!.functions.find(
+    f => f.functionName === "hotFunction",
+  );
+  expect(hotFn).toBeDefined();
+  const count = hotFn!.ranges[0].count;
+  expect(count).toBe(10);
+});

package/src/test/HeapAttribution.test.ts

@@ -0,0 +1,51 @@
+import { expect, test } from "vitest";
+import {
+  aggregateSites,
+  type HeapSite,
+} from "../profiling/node/HeapSampleReport.ts";
+
+test("unknown column does not merge distinct functions on same line", () => {
+  const sites: HeapSite[] = [
+    { name: "Foo", url: "test.ts", line: 10, col: undefined, bytes: 100 },
+    { name: "Bar", url: "test.ts", line: 10, col: undefined, bytes: 200 },
+  ];
+  const aggregated = aggregateSites(sites);
+  expect(aggregated).toHaveLength(2);
+});
+
+test("same column merges regardless of function name", () => {
+  const sites: HeapSite[] = [
+    { name: "Foo", url: "test.ts", line: 10, col: 5, bytes: 100 },
+    { name: "Foo", url: "test.ts", line: 10, col: 5, bytes: 200 },
+  ];
+  const aggregated = aggregateSites(sites);
+  expect(aggregated).toHaveLength(1);
+  expect(aggregated[0].bytes).toBe(300);
+});
+
+test("aggregation preserves distinct caller stacks", () => {
+  const stackA = [
+    { name: "root", url: "a.ts", line: 1, col: 0 },
+    { name: "foo", url: "a.ts", line: 10, col: 0 },
+    { name: "alloc", url: "a.ts", line: 20, col: 5 },
+  ];
+  const stackB = [
+    { name: "root", url: "a.ts", line: 1, col: 0 },
+    { name: "bar", url: "a.ts", line: 15, col: 0 },
+    { name: "alloc", url: "a.ts", line: 20, col: 5 },
+  ];
+  const sites: HeapSite[] = [
+    { name: "alloc", url: "a.ts", line: 20, col: 5, bytes: 800, stack: stackA },
+    { name: "alloc", url: "a.ts", line: 20, col: 5, bytes: 200, stack: stackB },
+  ];
+  const aggregated = aggregateSites(sites);
+
+  expect(aggregated).toHaveLength(1);
+  expect(aggregated[0].bytes).toBe(1000);
+  expect(aggregated[0].callers).toHaveLength(2);
+  // Primary stack should be the highest-bytes path (foo)
+  expect(aggregated[0].stack![1].name).toBe("foo");
+  // Callers sorted by bytes descending
+  expect(aggregated[0].callers![0].bytes).toBe(800);
+  expect(aggregated[0].callers![1].bytes).toBe(200);
+});

package/src/{tests → test}/MatrixFilter.test.ts

@@ -1,7 +1,22 @@
 import { expect, test } from "vitest";
-import type { BenchMatrix } from "../BenchMatrix.ts";
+import type { BenchMatrix } from "../matrix/BenchMatrix.ts";
 import { filterMatrix, parseMatrixFilter } from "../matrix/MatrixFilter.ts";
 
+const inlineMatrix: BenchMatrix<string> = {
+  name: "Test",
+  variants: {
+    fast: (s: string) => s.toUpperCase(),
+    slow: (s: string) => s.toLowerCase(),
+    medium: (s: string) => s,
+  },
+  cases: ["small", "large", "bevy_env_map"],
+};
+
+const noExplicitCases: BenchMatrix = {
+  name: "NoCase",
+  variants: { fast: () => {} },
+};
+
 test("parseMatrixFilter: case/variant", () => {
   expect(parseMatrixFilter("bevy/link")).toEqual({
     case: "bevy",
@@ -34,16 +49,6 @@ test("parseMatrixFilter: empty parts", () => {
   });
 });
 
-const inlineMatrix: BenchMatrix<string> = {
-  name: "Test",
-  variants: {
-    fast: (s: string) => s.toUpperCase(),
-    slow: (s: string) => s.toLowerCase(),
-    medium: (s: string) => s,
-  },
-  cases: ["small", "large", "bevy_env_map"],
-};
-
 test("filterMatrix: no filter returns original", async () => {
   const result = await filterMatrix(inlineMatrix, undefined);
   expect(result).toBe(inlineMatrix);
@@ -106,11 +111,6 @@ test("filterMatrix: multiple matching variants", async () => {
   expect(result.filteredVariants?.sort()).toEqual(["fast", "slow"]);
 });
 
-const noExplicitCases: BenchMatrix = {
-  name: "NoCase",
-  variants: { fast: () => {} },
-};
-
 test("filterMatrix: implicit default case returns default", async () => {
   const result = await filterMatrix(noExplicitCases, { case: "default" });
   expect(result.filteredCases).toEqual(["default"]);

package/src/{tests → test}/MatrixReport.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import type { CaseResult, MatrixResults } from "../BenchMatrix.ts";
+import type { CaseResult, MatrixResults } from "../matrix/BenchMatrix.ts";
 import { reportMatrixResults } from "../matrix/MatrixReport.ts";
 
 /** Create simple measured results for testing */

package/src/test/PermutationTest.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import { compareWithBaseline } from "../PermutationTest.ts";
+import { compareWithBaseline } from "../stats/PermutationTest.ts";
 import { assertValid, getSampleData } from "./TestUtils.ts";
 
 test("detects 20% performance improvement", () => {

package/src/{tests → test}/RealDataValidation.test.ts

@@ -4,10 +4,10 @@ import {
   coefficientOfVariation,
   medianAbsoluteDeviation,
   percentile,
-} from "../StatisticalUtils.ts";
+} from "../stats/StatisticalUtils.ts";
 import { bevy30SamplesMs, bevy30SamplesNs } from "./fixtures/bevy30-samples.ts";
 
-test("bevy30 data characteristics", () => {
+test.skip("bevy30 data characteristics", () => {
   const sortedMs = [...bevy30SamplesMs].sort((a, b) => a - b);
 
   const stats = {
@@ -48,7 +48,7 @@ test("bevy30 data characteristics", () => {
   if (stats.cv > 0.5) console.warn("Very high variation - may be unstable");
 });
 
-test("convergence at different time points matches CLI behavior", () => {
+test.skip("convergence at different time points matches CLI behavior", () => {
   // Simulate 5-second run (approximately 100 samples at ~50ms each)
   const samples5s = bevy30SamplesNs.slice(0, 100);
   const result5s = checkConvergence(samples5s);
@@ -76,7 +76,7 @@ test("convergence at different time points matches CLI behavior", () => {
     );
 });
 
-test("warm-up detection in real data", () => {
+test.skip("warm-up detection in real data", () => {
   const windowSize = 20;
   const windows: Array<{ start: number; median: number }> = [];
 
@@ -104,7 +104,7 @@ test("warm-up detection in real data", () => {
   }
 });
 
-test("convergence stability over sliding windows", () => {
+test.skip("convergence stability over sliding windows", () => {
   const windowSize = 100;
   const step = 50;
   const history: Array<{ start: number; confidence: number }> = [];
@@ -132,7 +132,7 @@ test("convergence stability over sliding windows", () => {
   }
 });
 
-test("adaptive algorithm would stop at correct time", () => {
+test.skip("adaptive algorithm would stop at correct time", () => {
   const target = 95;
   const fallback = 80;
   const minSamples = 50;