benchforge 0.1.11 → 0.2.4

package/src/cli/SuiteRunner.ts

@@ -0,0 +1,160 @@
+import type {
+  BenchmarkReport,
+  ReportGroup,
+} from "../report/BenchmarkReport.ts";
+import type {
+  BenchGroup,
+  BenchmarkSpec,
+  BenchSuite,
+} from "../runners/BenchmarkSpec.ts";
+import type { RunnerOptions } from "../runners/BenchRunner.ts";
+import type { KnownRunner } from "../runners/CreateRunner.ts";
+import { runBatched } from "../runners/MergeBatches.ts";
+import { runBenchmark } from "../runners/RunnerOrchestrator.ts";
+import type { DefaultCliArgs } from "./CliArgs.ts";
+import { cliToRunnerOptions, validateArgs } from "./CliOptions.ts";
+import { filterBenchmarks } from "./FilterBenchmarks.ts";
+
+type RunParams = {
+  runner: KnownRunner;
+  options: RunnerOptions;
+  useWorker: boolean;
+  params: unknown;
+  metadata?: Record<string, any>;
+};
+
+type SuiteParams = {
+  runner: KnownRunner;
+  options: RunnerOptions;
+  useWorker: boolean;
+  batches: number;
+  warmupBatch: boolean;
+};
+
+/** Run a benchmark suite with CLI arguments. */
+export async function runBenchmarks(
+  suite: BenchSuite,
+  args: DefaultCliArgs,
+): Promise<ReportGroup[]> {
+  validateArgs(args);
+  const { filter, worker: useWorker, batches = 1 } = args;
+  const warmupBatch = args["warmup-batch"] ?? false;
+  const options = cliToRunnerOptions(args);
+  const filtered = filterBenchmarks(suite, filter);
+
+  const runner = "timing";
+  const suiteParams: SuiteParams = {
+    runner,
+    options,
+    useWorker,
+    batches,
+    warmupBatch,
+  };
+  return serialMap(filtered.groups, g => runGroup(g, suiteParams));
+}
+
+/** Like Promise.all(arr.map(fn)) but runs one at a time. */
+async function serialMap<T, R>(
+  arr: T[],
+  fn: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = [];
+  for (const item of arr) {
+    results.push(await fn(item));
+  }
+  return results;
+}
+
+/** Execute group with shared setup, optionally batching to reduce ordering bias. */
+async function runGroup(
+  group: BenchGroup,
+  suiteParams: SuiteParams,
+): Promise<ReportGroup> {
+  const { batches, warmupBatch, ...rest } = suiteParams;
+  const { name, benchmarks, baseline, setup, metadata } = group;
+  const setupParams = await setup?.();
+  validateBenchmarkParameters(group);
+
+  const runParams: RunParams = { ...rest, params: setupParams, metadata };
+  if (batches === 1)
+    return runSingleBatch(name, benchmarks, baseline, runParams);
+  return runMultipleBatches(
+    name,
+    benchmarks,
+    baseline,
+    runParams,
+    batches,
+    warmupBatch,
+  );
+}
+
+/** Warn if parameterized benchmarks lack a setup function. */
+function validateBenchmarkParameters(group: BenchGroup): void {
+  if (group.setup) return;
+  const { benchmarks, baseline } = group;
+  const all = baseline ? [...benchmarks, baseline] : benchmarks;
+  for (const bench of all.filter(b => b.fn.length > 0)) {
+    console.warn(
+      `Benchmark "${bench.name}" in group "${group.name}" expects parameters but no setup() provided.`,
+    );
+  }
+}
+
+/** Run benchmarks in a single batch. */
+async function runSingleBatch(
+  name: string,
+  benchmarks: BenchmarkSpec[],
+  baseline: BenchmarkSpec | undefined,
+  runParams: RunParams,
+): Promise<ReportGroup> {
+  const baselineReport = baseline
+    ? await runSingleBenchmark(baseline, runParams)
+    : undefined;
+  const reports = await serialMap(benchmarks, b =>
+    runSingleBenchmark(b, runParams),
+  );
+  return { name, reports, baseline: baselineReport };
+}
+
+/** Run benchmarks in multiple batches, alternating order to reduce bias. */
+async function runMultipleBatches(
+  name: string,
+  benchmarks: BenchmarkSpec[],
+  baseline: BenchmarkSpec | undefined,
+  runParams: RunParams,
+  batches: number,
+  warmupBatch: boolean,
+): Promise<ReportGroup> {
+  const { metadata } = runParams;
+  const run = (spec: BenchmarkSpec) => async () =>
+    (await runSingleBenchmark(spec, runParams)).measuredResults;
+  const runners = benchmarks.map(run);
+  const baselineFn = baseline ? run(baseline) : undefined;
+
+  const batched = await runBatched(runners, baselineFn, batches, warmupBatch);
+  const reports = benchmarks.map((b, i) => ({
+    name: b.name,
+    measuredResults: batched.results[i],
+    metadata,
+  }));
+  const baselineReport =
+    batched.baseline && baseline
+      ? { name: baseline.name, measuredResults: batched.baseline, metadata }
+      : undefined;
+  return { name, reports, baseline: baselineReport };
+}
+
+/** Run single benchmark and create report. */
+async function runSingleBenchmark(
+  spec: BenchmarkSpec,
+  { runner, options, useWorker, params, metadata }: RunParams,
+): Promise<BenchmarkReport> {
+  const [result] = await runBenchmark({
+    spec,
+    runner,
+    options,
+    useWorker,
+    params,
+  });
+  return { name: spec.name, measuredResults: result, metadata };
+}

package/src/cli/ViewerServer.ts

@@ -0,0 +1,282 @@
+import { readFile } from "node:fs/promises";
+import {
+  createServer,
+  type IncomingMessage,
+  type Server,
+  type ServerResponse,
+} from "node:http";
+import { basename, dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import open from "open";
+import sirv from "sirv";
+import {
+  buildArchiveObject,
+  collectProfileFrames,
+  collectSources,
+  defaultArchiveName,
+  fetchSource,
+} from "../export/ArchiveExport.ts";
+import {
+  archiveSchemaVersion,
+  migrateArchive,
+} from "../export/ArchiveFormat.ts";
+
+export interface ViewerServerOptions {
+  /** Speedscope JSON profile data (allocation) */
+  profileData?: string;
+  /** Speedscope JSON profile data (time/CPU) */
+  timeProfileData?: string;
+  /** Per-function coverage data (JSON-serialized Record<url, LineCoverage[]>) */
+  coverageData?: string;
+  /** HTML report JSON data */
+  reportData?: string;
+  /** Editor URI prefix for Cmd+Shift+click (e.g. "vscode://file") */
+  editorUri?: string;
+  /** Port to listen on (default 3939) */
+  port?: number;
+  /** Open browser on start (default true) */
+  open?: boolean;
+  /** Pre-loaded sources (e.g. from an archive) to seed the source cache */
+  sources?: Record<string, string>;
+}
+
+type RouteHandler = (
+  res: ServerResponse,
+  query: string,
+  method: string,
+) => Promise<void> | void;
+
+/** Start the viewer HTTP server and open in browser. */
+export async function startViewerServer(
+  options: ViewerServerOptions,
+): Promise<{ server: Server; port: number; close: () => void }> {
+  const port = options.port ?? 3939;
+
+  const sourceCache = new Map(Object.entries(options.sources ?? {}));
+
+  const assets = sirv(join(packageRoot(), "dist/viewer"), { single: true });
+  const handler = createRequestHandler(options, sourceCache, assets);
+  const server = createServer(handler);
+
+  const bound = await tryListen(server, port);
+  const url = `http://localhost:${bound.port}`;
+  if (options.open !== false) await open(url);
+  console.log(`Viewer: ${url}`);
+
+  const close = () => {
+    bound.server.closeAllConnections();
+    bound.server.close();
+  };
+  return { server: bound.server, port: bound.port, close };
+}
+
+/** Open a .benchforge archive in the viewer. */
+export async function viewArchive(filePath: string): Promise<void> {
+  const absPath = resolve(filePath);
+  const content = await readFile(absPath, "utf-8");
+  const raw = JSON.parse(content);
+
+  const schema = raw.schema ?? 0;
+  if (schema > archiveSchemaVersion) {
+    const msg = `Archive schema version ${schema} is newer than supported (${archiveSchemaVersion}).`;
+    console.error(`${msg} Please update benchforge to view this archive.`);
+    process.exit(1);
+  }
+
+  const archive = migrateArchive(raw);
+  const sources = archive.sources as Record<string, string> | undefined;
+  const { close } = await startViewerServer({
+    profileData: optionalJson(archive.allocProfile),
+    timeProfileData: optionalJson(archive.timeProfile),
+    coverageData: optionalJson(archive.coverage),
+    reportData: optionalJson(archive.report),
+    sources,
+  });
+
+  await waitForCtrlC();
+  close();
+}
+
+/** Serialize a value to JSON if truthy, otherwise return undefined. */
+export function optionalJson(v: unknown): string | undefined {
+  return v ? JSON.stringify(v) : undefined;
+}
+
+/** Wait for Ctrl+C (SIGINT) before resolving. */
+export function waitForCtrlC(): Promise<void> {
+  return new Promise(resolve => {
+    console.log("\nPress Ctrl+C to exit");
+    process.once("SIGINT", () => {
+      console.log();
+      resolve();
+    });
+  });
+}
+
+/** Resolve the package root (dev: src/cli/ ==> up 2, dist: dist/ ==> up 1). */
+function packageRoot(): string {
+  const thisDir = dirname(fileURLToPath(import.meta.url));
+  if (basename(thisDir) === "cli") return join(thisDir, "../..");
+  return join(thisDir, "..");
+}
+
+/** Build HTTP request handler with API routes and static asset fallback. */
+function createRequestHandler(
+  ctx: ViewerServerOptions,
+  sourceCache: Map<string, string>,
+  assets: ReturnType<typeof sirv>,
+): (req: IncomingMessage, res: ServerResponse) => void {
+  const routes: Record<string, RouteHandler> = {
+    "/api/config": res => {
+      const config = {
+        editorUri: ctx.editorUri || null,
+        hasReport: !!ctx.reportData,
+        hasProfile: !!ctx.profileData,
+        hasTimeProfile: !!ctx.timeProfileData,
+        hasCoverage: !!ctx.coverageData,
+      };
+      res.setHeader("Content-Type", "application/json");
+      res.end(JSON.stringify(config));
+    },
+    "/api/report-data": res => sendJson(res, ctx.reportData, "report data"),
+    "/api/coverage": res => sendJson(res, ctx.coverageData, "coverage data"),
+    "/api/profile": res => sendJson(res, ctx.profileData, "profile data", true),
+    "/api/profile/alloc": res =>
+      sendJson(res, ctx.profileData, "profile data", true),
+    "/api/profile/time": res =>
+      sendJson(res, ctx.timeProfileData, "time profile data", true),
+    "/api/source": (res, query) => handleSourceRequest(res, query, sourceCache),
+    "/api/archive": (res, _q, method) => {
+      if (method !== "POST") {
+        res.statusCode = 405;
+        return void res.end("Method not allowed");
+      }
+      return handleArchiveRequest(res, ctx, sourceCache);
+    },
+  };
+
+  return async (req, res) => {
+    const url = req.url || "/";
+    const qIdx = url.indexOf("?");
+    const pathname = qIdx >= 0 ? url.slice(0, qIdx) : url;
+    const query = qIdx >= 0 ? url.slice(qIdx + 1) : "";
+
+    const handler = routes[pathname];
+    if (handler) {
+      await handler(res, query, req.method || "GET");
+      return;
+    }
+
+    assets(req, res, () => {
+      res.statusCode = 404;
+      res.end("Not found");
+    });
+  };
+}
+
+/** Listen on port, retrying on next port if EADDRINUSE. */
+function tryListen(
+  server: Server,
+  port: number,
+  maxRetries = 10,
+): Promise<{ server: Server; port: number }> {
+  return new Promise((resolve, reject) => {
+    let attempt = 0;
+    const listen = (p: number) => {
+      server.once("error", (err: NodeJS.ErrnoException) => {
+        if (err.code === "EADDRINUSE" && attempt < maxRetries) {
+          attempt++;
+          listen(p + 1);
+        } else {
+          reject(err);
+        }
+      });
+      server.listen(p, () => {
+        server.removeAllListeners("error");
+        const addr = server.address();
+        const listenPort = typeof addr === "object" && addr ? addr.port : p;
+        resolve({ server, port: listenPort });
+      });
+    };
+    listen(port);
+  });
+}
+
+/** Send pre-serialized JSON or 404 if data is absent. */
+function sendJson(
+  res: ServerResponse,
+  data: string | undefined,
+  label: string,
+  cors = false,
+): void {
+  if (!data) {
+    res.statusCode = 404;
+    res.end(`No ${label}`);
+    return;
+  }
+  res.setHeader("Content-Type", "application/json");
+  if (cors) res.setHeader("Access-Control-Allow-Origin", "*");
+  res.end(data);
+}
+
+/** Fetch source text by URL query param, caching for subsequent requests. */
+async function handleSourceRequest(
+  res: ServerResponse,
+  query: string,
+  cache: Map<string, string>,
+): Promise<void> {
+  const params = new URLSearchParams(query);
+  const sourceUrl = params.get("url");
+  if (!sourceUrl) {
+    res.statusCode = 400;
+    res.end("Missing url parameter");
+    return;
+  }
+  try {
+    let source = cache.get(sourceUrl);
+    if (source === undefined) {
+      source = await fetchSource(sourceUrl);
+      if (source === undefined) throw new Error("not found");
+      cache.set(sourceUrl, source);
+    }
+    res.setHeader("Content-Type", "text/plain; charset=utf-8");
+    res.end(source);
+  } catch {
+    res.statusCode = 404;
+    res.end("Source unavailable");
+  }
+}
+
+/** Build a .benchforge archive from current session data and send as download. */
+async function handleArchiveRequest(
+  res: ServerResponse,
+  ctx: ViewerServerOptions,
+  sourceCache: Map<string, string>,
+): Promise<void> {
+  try {
+    const parse = (s?: string) => (s ? JSON.parse(s) : undefined);
+    const profile = parse(ctx.profileData);
+    const timeProfile = parse(ctx.timeProfileData);
+    const coverage = parse(ctx.coverageData);
+    const report = parse(ctx.reportData);
+    const allFrames = collectProfileFrames(profile, timeProfile);
+    const sources = allFrames.length
+      ? await collectSources(allFrames, sourceCache)
+      : Object.fromEntries(sourceCache);
+    const { archive, timestamp } = buildArchiveObject({
+      allocProfile: profile,
+      timeProfile,
+      coverage,
+      report,
+      sources,
+    });
+    const body = JSON.stringify(archive);
+    const filename = defaultArchiveName(profile, timestamp);
+    res.setHeader("Content-Type", "application/json");
+    res.setHeader("Content-Disposition", `attachment; filename="${filename}"`);
+    res.end(body);
+  } catch {
+    res.statusCode = 500;
+    res.end("Archive failed");
+  }
+}

package/src/export/AllocExport.ts

@@ -0,0 +1,121 @@
+/** Heap profile export to Speedscope format. */
+
+import { writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import type { HeapProfile } from "../profiling/node/HeapSampler.ts";
+import {
+  type ResolvedProfile,
+  resolveProfile,
+} from "../profiling/node/ResolvedProfile.ts";
+import { groupReports, type ReportGroup } from "../report/BenchmarkReport.ts";
+import {
+  type FrameContext,
+  frameContext,
+  internFrame,
+  type SpeedscopeFile,
+  type SpeedscopeHeapProfile,
+  speedscopeFile,
+} from "./SpeedscopeTypes.ts";
+
+/** Export heap profiles to speedscope JSON. Returns output path, or undefined if no profiles. */
+export function exportSpeedscope(
+  groups: ReportGroup[],
+  outputPath: string,
+): string | undefined {
+  const file = buildSpeedscopeFile(groups);
+  if (!file) {
+    console.log("No heap profiles to export.");
+    return undefined;
+  }
+
+  const absPath = resolve(outputPath);
+  writeFileSync(absPath, JSON.stringify(file));
+  console.log(`Speedscope profile exported to: ${outputPath}`);
+  return absPath;
+}
+
+/** Convert a single HeapProfile to speedscope format. */
+export function heapProfileToSpeedscope(
+  name: string,
+  profile: HeapProfile,
+): SpeedscopeFile {
+  const ctx = frameContext();
+  const p = buildProfile(name, resolveProfile(profile), ctx);
+  return speedscopeFile(ctx, [p]);
+}
+
+/** Build SpeedscopeFile from report groups. Returns undefined if no profiles found. */
+export function buildSpeedscopeFile(
+  groups: ReportGroup[],
+): SpeedscopeFile | undefined {
+  const ctx = frameContext();
+  const profiles: SpeedscopeHeapProfile[] = [];
+
+  for (const group of groups) {
+    for (const report of groupReports(group)) {
+      const { heapProfile } = report.measuredResults;
+      if (!heapProfile) continue;
+      const resolved = resolveProfile(heapProfile);
+      profiles.push(buildProfile(report.name, resolved, ctx));
+    }
+  }
+
+  if (profiles.length === 0) return undefined;
+
+  return speedscopeFile(ctx, profiles);
+}
+
+/** Build a single speedscope profile from a resolved heap profile. */
+function buildProfile(
+  name: string,
+  resolved: ResolvedProfile,
+  ctx: FrameContext,
+): SpeedscopeHeapProfile {
+  type Frame = { name: string; url: string; line: number; col?: number | null };
+  const intern = (f: Frame) => internFrame(f.name, f.url, f.line, f.col, ctx);
+
+  const nodeStacks = new Map(
+    resolved.nodes.map(node => [node.nodeId, node.stack.map(intern)] as const),
+  );
+
+  if (!resolved.sortedSamples?.length) {
+    console.error(
+      `Speedscope export: no samples in heap profile for "${name}", skipping`,
+    );
+    return emptyProfile(name);
+  }
+
+  const samples: number[][] = [];
+  const weights: number[] = [];
+  for (const sample of resolved.sortedSamples) {
+    const stack = nodeStacks.get(sample.nodeId);
+    if (stack) {
+      samples.push(stack);
+      weights.push(sample.size);
+    }
+  }
+
+  const totalBytes = weights.reduce((sum, w) => sum + w, 0);
+  return {
+    type: "sampled",
+    name,
+    unit: "bytes",
+    startValue: 0,
+    endValue: totalBytes,
+    samples,
+    weights,
+  };
+}
+
+/** Placeholder profile with no samples (used when heap data is missing). */
+function emptyProfile(name: string): SpeedscopeHeapProfile {
+  return {
+    type: "sampled",
+    name,
+    unit: "bytes",
+    startValue: 0,
+    endValue: 0,
+    samples: [],
+    weights: [],
+  };
+}

package/src/export/ArchiveExport.ts

@@ -0,0 +1,146 @@
+/** .benchforge archive creation, source collection, and archive filename derivation. */
+
+import { writeFileSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { ReportGroup } from "../report/BenchmarkReport.ts";
+import type { ReportData } from "../viewer/ReportData.ts";
+import { buildSpeedscopeFile } from "./AllocExport.ts";
+import {
+  archiveSchemaVersion,
+  type BenchforgeArchive,
+} from "./ArchiveFormat.ts";
+import type { LineCoverage } from "./CoverageExport.ts";
+import type { SpeedscopeFile } from "./SpeedscopeTypes.ts";
+
+export interface ArchiveOptions {
+  groups: ReportGroup[];
+  reportData?: ReportData;
+  timeProfileData?: string;
+  coverageData?: string;
+  outputPath?: string;
+}
+
+export interface ArchiveInput {
+  allocProfile?: SpeedscopeFile;
+  timeProfile?: SpeedscopeFile;
+  coverage?: Record<string, LineCoverage[]>;
+  report?: ReportData;
+  sources: Record<string, string>;
+}
+
+/** Build a .benchforge archive file. Returns output path, or undefined if nothing to archive. */
+export async function archiveBenchmark(
+  options: ArchiveOptions,
+): Promise<string | undefined> {
+  const { groups, reportData, timeProfileData, coverageData, outputPath } =
+    options;
+  const allocProfile = buildSpeedscopeFile(groups) ?? undefined;
+  const timeProfile = timeProfileData ? JSON.parse(timeProfileData) : undefined;
+  if (!allocProfile && !timeProfile && !reportData) {
+    console.log("No data to archive.");
+    return undefined;
+  }
+
+  const allFrames = collectProfileFrames(allocProfile, timeProfile);
+  const sources = allFrames.length ? await collectSources(allFrames) : {};
+  const coverage = coverageData ? JSON.parse(coverageData) : undefined;
+  const input: ArchiveInput = {
+    allocProfile,
+    timeProfile,
+    coverage,
+    report: reportData,
+    sources,
+  };
+  const { archive, timestamp } = buildArchiveObject(input);
+  const filename = outputPath || defaultArchiveName(allocProfile, timestamp);
+  const absPath = resolve(filename);
+  writeFileSync(absPath, JSON.stringify(archive));
+  console.log(`Archive written to: ${filename}`);
+  return absPath;
+}
+
+export function buildArchiveObject(input: ArchiveInput): {
+  archive: BenchforgeArchive;
+  timestamp: string;
+} {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const archive = {
+    schema: archiveSchemaVersion,
+    allocProfile: input.allocProfile,
+    timeProfile: input.timeProfile,
+    coverage: input.coverage,
+    report: input.report,
+    sources: input.sources,
+    metadata: {
+      timestamp,
+      benchforgeVersion: process.env.npm_package_version || "unknown",
+    },
+  };
+  return { archive, timestamp };
+}
+
+export function collectProfileFrames(
+  allocProfile: SpeedscopeFile | null | undefined,
+  timeProfile: { shared?: { frames: { file?: string }[] } } | null | undefined,
+): { file?: string }[] {
+  const heapFrames = allocProfile?.shared?.frames ?? [];
+  const timeFrames = timeProfile?.shared?.frames ?? [];
+  return [...heapFrames, ...timeFrames];
+}
+
+/** Fetch source code for all unique file URLs in profile frames. */
+export async function collectSources(
+  frames: { file?: string }[],
+  cache?: Map<string, string>,
+): Promise<Record<string, string>> {
+  const urls = new Set(frames.map(f => f.file).filter((u): u is string => !!u));
+
+  const sources: Record<string, string> = {};
+  for (const url of urls) {
+    const cached = cache?.get(url);
+    const text = cached ?? (await fetchSource(url));
+    if (text === undefined) continue;
+    sources[url] = text;
+    if (!cached) cache?.set(url, text);
+  }
+
+  return sources;
+}
+
+/** Derive archive filename from profile (or generic fallback). */
+export function defaultArchiveName(
+  profile: SpeedscopeFile | null | undefined,
+  timestamp: string,
+): string {
+  return profile
+    ? archiveFileName(profile, timestamp)
+    : `benchforge-${timestamp}.benchforge`;
+}
+
+/** Fetch source text from a file:// or http(s):// URL. */
+export async function fetchSource(url: string): Promise<string | undefined> {
+  try {
+    if (url.startsWith("file://")) {
+      return await readFile(fileURLToPath(url), "utf-8");
+    }
+    const resp = await fetch(url, { signal: AbortSignal.timeout(5000) });
+    if (!resp.ok) return undefined;
+    return await resp.text();
+  } catch {
+    return undefined;
+  }
+}
+
+/** Derive an archive filename from the profile name (sanitizes URLs to safe filenames). */
+function archiveFileName(file: SpeedscopeFile, timestamp: string): string {
+  const raw = file.profiles[0]?.name || "profile";
+  const sanitized = raw
+    .replace(/^https?:\/\//, "")
+    .replace(/[^a-zA-Z0-9._-]+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "");
+  const base = sanitized || "profile";
+  return `${base}-${timestamp}.benchforge`;
+}