benchforge 0.1.11 → 0.2.4

package/src/test/RunnerOrchestrator.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import type { BenchmarkSpec } from "../Benchmark.ts";
+import type { BenchmarkSpec } from "../runners/BenchmarkSpec.ts";
 import { runBenchmark } from "../runners/RunnerOrchestrator.ts";
 
 /** lightweight function for testing worker communication */
@@ -9,15 +9,15 @@ function simpleTestFunction(): number {
   return sum;
 }
 
-test("BasicRunner runs benchmark in worker mode", async () => {
+test("TimingRunner runs benchmark in worker mode", async () => {
   const spec: BenchmarkSpec = {
-    name: "basic-worker-test",
+    name: "timing-worker-test",
     fn: simpleTestFunction,
   };
 
   const results = await runBenchmark({
     spec,
-    runner: "basic",
+    runner: "timing",
     options: {
       maxTime: 5,
       maxIterations: 50,
@@ -28,7 +28,7 @@ test("BasicRunner runs benchmark in worker mode", async () => {
   expect(results).toHaveLength(1);
   const result = results[0];
 
-  expect(result.name).toBe("basic-worker-test");
+  expect(result.name).toBe("timing-worker-test");
   expect(result.samples.length).toBeGreaterThan(0);
   expect(result.samples.length).toBeLessThanOrEqual(500);
   expect(result.time.min).toBeGreaterThan(0);
@@ -38,15 +38,15 @@ test("BasicRunner runs benchmark in worker mode", async () => {
   expect(result.time.p99).toBeGreaterThan(0);
 });
 
-test("BasicRunner runs benchmark in non-worker mode", async () => {
+test("TimingRunner runs benchmark in non-worker mode", async () => {
   const spec: BenchmarkSpec = {
-    name: "basic-test",
+    name: "timing-test",
     fn: simpleTestFunction,
   };
 
   const results = await runBenchmark({
     spec,
-    runner: "basic",
+    runner: "timing",
     options: {
       maxTime: 5,
       maxIterations: 50,
@@ -57,12 +57,12 @@ test("BasicRunner runs benchmark in non-worker mode", async () => {
   expect(results).toHaveLength(1);
   const result = results[0];
 
-  expect(result.name).toBe("basic-test");
+  expect(result.name).toBe("timing-test");
   expect(result.samples.length).toBeGreaterThan(0);
   expect(result.time.p50).toBeGreaterThan(0);
 });
 
-test("BasicRunner with parameterized benchmark", async () => {
+test("TimingRunner with parameterized benchmark", async () => {
   const spec: BenchmarkSpec<number> = {
     name: "parameterized-test",
     fn: (n: number) => {
@@ -74,7 +74,7 @@ test("BasicRunner with parameterized benchmark", async () => {
 
   const results = await runBenchmark({
     spec,
-    runner: "basic",
+    runner: "timing",
     options: { maxTime: 5, maxIterations: 20 },
     useWorker: false,
     params: 100,
@@ -94,7 +94,7 @@ test("RunnerOrchestrator propagates errors from worker", async () => {
 
   const promise = runBenchmark({
     spec,
-    runner: "basic",
+    runner: "timing",
     options: { maxTime: 1, maxIterations: 1 },
     useWorker: true,
   });

package/src/test/StatisticalUtils.test.ts

@@ -1,14 +1,19 @@
 import { expect, test } from "vitest";
+import {
+  blockDifferenceCI,
+  sampleDifferenceCI,
+} from "../stats/BootstrapDifference.ts";
 import {
   average,
-  bootstrapDifferenceCI,
-  bootstrapMedian,
+  blockBootstrap,
   coefficientOfVariation,
   findOutliers,
+  median,
   medianAbsoluteDeviation,
   percentile,
+  sampleBootstrap,
   standardDeviation,
-} from "../StatisticalUtils.ts";
+} from "../stats/StatisticalUtils.ts";
 import { assertValid, getSampleData } from "./TestUtils.ts";
 
 test("calculates mean correctly", () => {
@@ -68,11 +73,21 @@ test("identifies outliers in mixed data", () => {
   expect(outliers.indices).toContain(51);
 });
 
-test("bootstrap estimates median with confidence intervals", () => {
+test("sampleBootstrap estimates median with CI", () => {
+  const stable = getSampleData(400, 450);
+  const result = sampleBootstrap(stable, median, { resamples: 1000 });
+  expect(result.ciLevel).toBe("sample");
+  expect(result.ci[0]).toBeLessThanOrEqual(result.estimate);
+  expect(result.ci[1]).toBeGreaterThanOrEqual(result.estimate);
+});
+
+test("blockBootstrap estimates median with confidence intervals", () => {
   const stable = getSampleData(400, 450);
   const actual = percentile(stable, 0.5);
-  const result = bootstrapMedian(stable, { resamples: 1000 });
+  const blocks = Array.from({ length: 5 }, (_, i) => i * 10);
+  const result = blockBootstrap(stable, blocks, median, { resamples: 1000 });
 
+  expect(result.ciLevel).toBe("block");
   expect(result.estimate).toBeCloseTo(actual, 1);
   expect(result.ci[0]).toBeLessThanOrEqual(result.estimate);
   expect(result.ci[1]).toBeGreaterThanOrEqual(result.estimate);
@@ -80,32 +95,53 @@ test("bootstrap estimates median with confidence intervals", () => {
   expect(result.samples).toHaveLength(1000);
 });
 
-test("bootstrapDifferenceCI detects improvement", () => {
+test("sampleDifferenceCI detects improvement", () => {
+  const baseline = getSampleData(0, 100);
+  const improved = baseline.map(v => v * 0.8);
+  const result = sampleDifferenceCI(baseline, improved, median, {
+    resamples: 1000,
+  });
+  expect(result.ciLevel).toBe("sample");
+  expect(result.percent).toBeCloseTo(-20, 0);
+});
+
+test("blockDifferenceCI detects improvement", () => {
   const baseline = getSampleData(0, 100);
   const improved = baseline.map(v => v * 0.8);
-  const result = bootstrapDifferenceCI(baseline, improved, { resamples: 1000 });
+  const blocks = Array.from({ length: 10 }, (_, i) => i * 10);
+  const result = blockDifferenceCI(baseline, blocks, improved, median, {
+    resamples: 1000,
+  });
 
+  expect(result.ciLevel).toBe("block");
   expect(result.percent).toBeCloseTo(-20, 0);
   expect(result.ci[1]).toBeLessThan(0);
   expect(result.direction).toBe("faster");
 });
 
-test("bootstrapDifferenceCI detects regression", () => {
+test("blockDifferenceCI detects regression", () => {
   const baseline = getSampleData(0, 100);
   const slower = baseline.map(v => v * 1.2);
-  const result = bootstrapDifferenceCI(baseline, slower, { resamples: 1000 });
+  const blocks = Array.from({ length: 10 }, (_, i) => i * 10);
+  const result = blockDifferenceCI(baseline, blocks, slower, median, {
+    resamples: 1000,
+  });
 
+  expect(result.ciLevel).toBe("block");
   expect(result.percent).toBeCloseTo(20, 0);
   expect(result.ci[0]).toBeGreaterThan(0);
   expect(result.direction).toBe("slower");
 });
 
-test("bootstrapDifferenceCI shows uncertainty for noise", () => {
+test("blockDifferenceCI shows uncertainty for noise", () => {
   const baseline = getSampleData(0, 100);
   const noisy = baseline.map(v => v + (Math.random() - 0.5) * 2);
-  const result = bootstrapDifferenceCI(baseline, noisy, { resamples: 1000 });
+  const blocks = Array.from({ length: 10 }, (_, i) => i * 10);
+  const result = blockDifferenceCI(baseline, blocks, noisy, median, {
+    resamples: 1000,
+  });
 
-  // CI should span zero for no real change
+  expect(result.ciLevel).toBe("block");
   expect(result.ci[0]).toBeLessThanOrEqual(0);
   expect(result.ci[1]).toBeGreaterThanOrEqual(0);
   expect(result.direction).toBe("uncertain");

package/src/{table-util/test → test}/TableReport.test.ts

@@ -1,10 +1,10 @@
 import { expect, test } from "vitest";
-import { integer } from "../Formatters.ts";
+import { integer } from "../report/Formatters.ts";
 import {
   buildTable,
   type ColumnGroup,
   type ResultGroup,
-} from "../TableReport.ts";
+} from "../report/text/TableReport.ts";
 
 interface TestRecord {
   name: string;

package/src/test/TestUtils.ts

@@ -1,14 +1,19 @@
-import type { BenchSuite } from "../Benchmark.ts";
-import type { BenchmarkReport } from "../BenchmarkReport.ts";
 import type { Configure, DefaultCliArgs } from "../cli/CliArgs.ts";
 import { parseCliArgs } from "../cli/CliArgs.ts";
-import { defaultReport, runBenchmarks } from "../cli/RunBenchCLI.ts";
-import type { MeasuredResults } from "../MeasuredResults.ts";
-import { average, percentile } from "../StatisticalUtils.ts";
-import { bevy30SamplesMs } from "../tests/fixtures/bevy30-samples.ts";
+import { defaultReport } from "../cli/CliReport.ts";
+import { runBenchmarks } from "../cli/SuiteRunner.ts";
+import type { BenchmarkReport } from "../report/BenchmarkReport.ts";
+import type { BenchSuite } from "../runners/BenchmarkSpec.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { average, percentile } from "../stats/StatisticalUtils.ts";
+import { bevy30SamplesMs } from "./fixtures/bevy30-samples.ts";
 
 /** Validation helpers for statistical tests */
-export const assertValid = {
+export const assertValid: {
+  pValue: (value: number) => void;
+  percentileOrder: (p25: number, p50: number, p75: number, p99: number) => void;
+  significance: (level: string) => void;
+} = {
   pValue: (value: number) => {
     if (value < 0 || value > 1) {
       throw new Error(`Expected p-value between 0 and 1, got ${value}`);

package/src/test/TimeExport.test.ts

@@ -0,0 +1,139 @@
+import { expect, test } from "vitest";
+import { timeProfileToSpeedscope } from "../export/TimeExport.ts";
+import type { TimeProfile } from "../profiling/node/TimeSampler.ts";
+
+/** Build a minimal TimeProfile for testing */
+function mockProfile(): TimeProfile {
+  return {
+    nodes: [
+      {
+        id: 1,
+        callFrame: { functionName: "", url: "", lineNumber: -1 },
+        children: [2],
+      },
+      {
+        id: 2,
+        callFrame: {
+          functionName: "main",
+          url: "file:///app.ts",
+          lineNumber: 9,
+          columnNumber: 0,
+        },
+        children: [3],
+      },
+      {
+        id: 3,
+        callFrame: {
+          functionName: "compute",
+          url: "file:///app.ts",
+          lineNumber: 19,
+          columnNumber: 4,
+        },
+        hitCount: 5,
+      },
+    ],
+    startTime: 0,
+    endTime: 5000,
+    samples: [3, 3, 2, 3, 3],
+    timeDeltas: [1000, 1000, 1000, 1000, 1000],
+  };
+}
+
+test("converts TimeProfile to valid SpeedScope format", () => {
+  const profile = mockProfile();
+  const file = timeProfileToSpeedscope("test-bench", profile);
+
+  expect(file.$schema).toBe(
+    "https://www.speedscope.app/file-format-schema.json",
+  );
+  expect(file.exporter).toBe("benchforge");
+  expect(file.profiles).toHaveLength(1);
+
+  const p = file.profiles[0];
+  expect(p.type).toBe("sampled");
+  expect(p.name).toBe("test-bench");
+  expect(p.unit).toBe("microseconds");
+  expect(p.samples).toHaveLength(5);
+  expect(p.weights).toEqual([1000, 1000, 1000, 1000, 1000]);
+  expect(p.endValue).toBe(5000);
+});
+
+test("resolves stacks from leaf to root (root-first order)", () => {
+  const profile = mockProfile();
+  const file = timeProfileToSpeedscope("test", profile);
+
+  const p = file.profiles[0];
+  const frames = file.shared.frames;
+
+  // Sample at node 3 (compute) should have stack: [main, compute]
+  const deepStack = p.samples[0];
+  expect(deepStack).toHaveLength(2); // root is skipped
+  expect(frames[deepStack[0]].name).toBe("main");
+  expect(frames[deepStack[1]].name).toBe("compute");
+
+  // Sample at node 2 (main) should have stack: [main]
+  const shallowStack = p.samples[2];
+  expect(shallowStack).toHaveLength(1);
+  expect(frames[shallowStack[0]].name).toBe("main");
+});
+
+test("deduplicates shared frames", () => {
+  const profile = mockProfile();
+  const file = timeProfileToSpeedscope("test", profile);
+
+  // "main" and "compute" — only 2 unique frames
+  expect(file.shared.frames).toHaveLength(2);
+});
+
+test("handles empty samples gracefully", () => {
+  const profile: TimeProfile = {
+    nodes: [
+      { id: 1, callFrame: { functionName: "", url: "", lineNumber: -1 } },
+    ],
+    startTime: 0,
+    endTime: 0,
+    samples: [],
+    timeDeltas: [],
+  };
+  const file = timeProfileToSpeedscope("empty", profile);
+
+  expect(file.profiles[0].samples).toHaveLength(0);
+  expect(file.profiles[0].weights).toHaveLength(0);
+  expect(file.profiles[0].endValue).toBe(0);
+});
+
+test("converts 0-indexed V8 lines to 1-indexed", () => {
+  const profile = mockProfile();
+  const file = timeProfileToSpeedscope("test", profile);
+
+  const mainFrame = file.shared.frames.find(f => f.name === "main")!;
+  expect(mainFrame.line).toBe(10); // lineNumber 9 -> line 10
+  expect(mainFrame.col).toBe(1); // columnNumber 0 -> col 1
+});
+
+test("anonymous functions get location hint in name", () => {
+  const profile: TimeProfile = {
+    nodes: [
+      {
+        id: 1,
+        callFrame: { functionName: "", url: "", lineNumber: -1 },
+        children: [2],
+      },
+      {
+        id: 2,
+        callFrame: {
+          functionName: "",
+          url: "file:///lib/utils.ts",
+          lineNumber: 41,
+        },
+      },
+    ],
+    startTime: 0,
+    endTime: 1000,
+    samples: [2],
+    timeDeltas: [1000],
+  };
+  const file = timeProfileToSpeedscope("test", profile);
+
+  expect(file.shared.frames[0].name).toBe("(anonymous utils.ts:42)");
+});

package/src/test/TimeSampler.test.ts

@@ -0,0 +1,37 @@
+import { expect, test } from "vitest";
+import { withTimeProfiling } from "../profiling/node/TimeSampler.ts";
+
+test("withTimeProfiling returns valid V8 CPU profile", async () => {
+  // Burn some CPU to produce samples
+  const { result, profile } = await withTimeProfiling({}, () => {
+    let sum = 0;
+    for (let i = 0; i < 1e6; i++) sum += Math.sqrt(i);
+    return sum;
+  });
+
+  expect(result).toBeGreaterThan(0);
+  expect(profile.nodes.length).toBeGreaterThan(0);
+  expect(profile.startTime).toBeLessThan(profile.endTime);
+  expect(profile.samples).toBeDefined();
+  expect(profile.timeDeltas).toBeDefined();
+  expect(profile.samples!.length).toBe(profile.timeDeltas!.length);
+
+  // Verify node structure
+  const node = profile.nodes[0];
+  expect(node).toHaveProperty("id");
+  expect(node).toHaveProperty("callFrame");
+  expect(node.callFrame).toHaveProperty("functionName");
+  expect(node.callFrame).toHaveProperty("url");
+  expect(node.callFrame).toHaveProperty("lineNumber");
+});
+
+test("withTimeProfiling respects custom interval", async () => {
+  const { profile } = await withTimeProfiling({ interval: 100 }, () => {
+    let sum = 0;
+    for (let i = 0; i < 1e6; i++) sum += Math.sqrt(i);
+    return sum;
+  });
+
+  // Finer interval should produce more samples for the same work
+  expect(profile.samples!.length).toBeGreaterThan(0);
+});

package/src/test/ViewerLive.e2e.test.ts

@@ -0,0 +1,159 @@
+import type { ChildProcess } from "node:child_process";
+import { spawn } from "node:child_process";
+import path from "node:path";
+import type { Browser } from "playwright";
+import { chromium } from "playwright";
+import { afterAll, beforeAll, expect, test } from "vitest";
+
+const binPath = path.resolve(import.meta.dirname!, "../../bin/benchforge");
+const examplePath = path.resolve(
+  import.meta.dirname!,
+  "../../examples/simple-cli.ts",
+);
+
+let proc: ChildProcess;
+let port: number;
+let browser: Browser;
+
+test("live viewer: summary tab shows stats", {
+  timeout: 30_000,
+}, async () => {
+  const consoleErrors: string[] = [];
+  const page = await browser.newPage();
+  try {
+    page.on("console", msg => {
+      if (msg.type() === "error" && !msg.text().includes("WebGL"))
+        consoleErrors.push(msg.text());
+    });
+    await page.goto(`http://localhost:${port}`, { waitUntil: "networkidle" });
+
+    const summaryPanel = page.locator("#summary-panel");
+    const panel = summaryPanel.locator(".section-panel").first();
+    await panel.waitFor({ state: "visible", timeout: 15_000 });
+    const statRows = await summaryPanel.locator(".stat-row").count();
+    expect(statRows).toBeGreaterThan(0);
+  } finally {
+    await page.close();
+  }
+  expect(consoleErrors).toEqual([]);
+});
+
+test("live viewer: samples tab shows chart SVG", {
+  timeout: 30_000,
+}, async () => {
+  const consoleErrors: string[] = [];
+  const page = await browser.newPage();
+  try {
+    page.on("console", msg => {
+      if (msg.type() === "error" && !msg.text().includes("WebGL"))
+        consoleErrors.push(msg.text());
+    });
+    await page.goto(`http://localhost:${port}`, { waitUntil: "networkidle" });
+
+    // Wait for summary to load (samples tab becomes enabled)
+    await page
+      .locator("#summary-panel .section-panel")
+      .first()
+      .waitFor({ state: "visible", timeout: 15_000 });
+
+    await page.locator("#tab-samples").click();
+
+    const samplesPanel = page.locator("#samples-panel");
+    const svg = samplesPanel.locator("svg").first();
+    await svg.waitFor({ state: "visible", timeout: 15_000 });
+    const childCount = await svg
+      .locator("path, rect, circle, line, text")
+      .count();
+    expect(childCount).toBeGreaterThan(0);
+  } finally {
+    await page.close();
+  }
+  expect(consoleErrors).toEqual([]);
+});
+
+test("live viewer: allocation tab has speedscope content", {
+  timeout: 30_000,
+}, async () => {
+  const page = await browser.newPage();
+  try {
+    await page.goto(`http://localhost:${port}`, { waitUntil: "networkidle" });
+
+    await page.locator("#tab-flamechart").click();
+    const frame = page.frameLocator("#speedscope-iframe");
+    await frame
+      .locator("body *")
+      .first()
+      .waitFor({ state: "visible", timeout: 15_000 });
+  } finally {
+    await page.close();
+  }
+});
+
+test("live viewer: timing tab has speedscope content", {
+  timeout: 30_000,
+}, async () => {
+  const page = await browser.newPage();
+  try {
+    await page.goto(`http://localhost:${port}`, { waitUntil: "networkidle" });
+
+    await page.locator("#tab-time-flamechart").click();
+    const frame = page.frameLocator("#time-speedscope-iframe");
+    await frame
+      .locator("body *")
+      .first()
+      .waitFor({ state: "visible", timeout: 15_000 });
+  } finally {
+    await page.close();
+  }
+});
+
+beforeAll(async () => {
+  const args = [
+    examplePath,
+    "--view-serve",
+    "--alloc",
+    "--profile",
+    "--iterations",
+    "3",
+    "--warmup",
+    "0",
+  ];
+
+  proc = spawn(binPath, args, {
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  // Parse port from stdout line like "Viewer: http://localhost:3939"
+  const portP = new Promise<number>((resolve, reject) => {
+    let stdout = "";
+    proc.stdout!.on("data", (chunk: Buffer) => {
+      stdout += chunk.toString();
+      const match = stdout.match(/Viewer: http:\/\/localhost:(\d+)/);
+      if (match) resolve(Number(match[1]));
+    });
+    proc.on("error", reject);
+    proc.on("exit", code => {
+      if (!port)
+        reject(
+          new Error(
+            `Process exited (${code}) before viewer started.\nstdout: ${stdout}`,
+          ),
+        );
+    });
+    setTimeout(
+      () =>
+        reject(new Error(`Timed out waiting for viewer.\nstdout: ${stdout}`)),
+      60_000,
+    );
+  });
+
+  [port, browser] = await Promise.all([
+    portP,
+    chromium.launch({ headless: true }),
+  ]);
+}, 90_000);
+
+afterAll(async () => {
+  await browser?.close();
+  proc?.kill();
+});