benchforge 0.1.9 → 0.2.4

package/src/viewer/plots/CIPlot.ts

@@ -0,0 +1,313 @@
+import type {
+  CILevel,
+  DifferenceCI,
+  HistogramBin,
+} from "../../stats/StatisticalUtils.ts";
+import { formatPct } from "./PlotTypes.ts";
+import {
+  createSvg,
+  ensureHatchPattern,
+  ensureSketchFilter,
+  line,
+  path,
+  rect,
+  text,
+} from "./SvgHelpers.ts";
+
+export interface DistributionPlotOptions {
+  width?: number;
+  height?: number;
+  title?: string;
+  smooth?: boolean;
+  direction?: "faster" | "slower" | "uncertain" | "equivalent";
+  /** Pre-formatted CI bound labels (overrides default formatPct) */
+  ciLabels?: [string, string];
+  /** Include zero in x scale (default true, set false for absolute-value plots) */
+  includeZero?: boolean;
+  /** Centered label above chart (e.g., the formatted point estimate) */
+  pointLabel?: string;
+  /** Equivalence margin in percent (draws shaded band at +/- margin) */
+  equivMargin?: number;
+  /** Block-level or sample-level CI */
+  ciLevel?: CILevel;
+  /** false ==> dashed border (insufficient batches for reliable CI) */
+  ciReliable?: boolean;
+}
+
+type Scales = { x: (v: number) => number; y: (v: number) => number };
+type Layout = {
+  width: number;
+  height: number;
+  margin: { top: number; right: number; bottom: number; left: number };
+  plot: { w: number; h: number };
+};
+const defaultMargin = { top: 22, right: 12, bottom: 22, left: 12 };
+
+const defaultOpts = {
+  width: 260,
+  height: 85,
+  title: "Δ%",
+  smooth: true,
+  direction: "uncertain" as const,
+  includeZero: true,
+};
+
+const colors = {
+  faster: { fill: "#bbf7d0", stroke: "#22c55e" },
+  slower: { fill: "#fee2e2", stroke: "#ef4444" },
+  uncertain: { fill: "#dbeafe", stroke: "#3b82f6" },
+  equivalent: { fill: "#dcfce7", stroke: "#86efac" },
+};
+
+/** Create a small distribution plot showing histogram with CI shading */
+export function createDistributionPlot(
+  histogram: HistogramBin[],
+  ci: [number, number],
+  pointEstimate: number,
+  options: DistributionPlotOptions = {},
+): SVGSVGElement {
+  const opts = { ...defaultOpts, ...options };
+  const layout = buildLayout(opts.width, opts.height, !!opts.pointLabel);
+  const svg = createSvg(layout.width, layout.height);
+  if (!histogram?.length) return svg;
+
+  const { fill, stroke } = colors[opts.direction];
+  const { includeZero, equivMargin } = opts;
+  const scales = buildScales(
+    histogram,
+    ci,
+    layout,
+    includeZero,
+    equivMargin,
+    pointEstimate,
+  );
+  const { margin, plot } = layout;
+  const ptX = scales.x(pointEstimate);
+
+  drawTitles(svg, opts, margin, ptX);
+
+  if (equivMargin && includeZero)
+    drawMarginZone(svg, equivMargin, scales, layout);
+
+  const ciX = scales.x(ci[0]);
+  const ciRect = rect(ciX, margin.top, scales.x(ci[1]) - ciX, plot.h, { fill });
+  const strength = includeZero ? "ci-region-strong" : "ci-region";
+  ciRect.classList.add(strength, `ci-${opts.direction}`);
+  if (opts.ciReliable === false) {
+    ciRect.classList.add("ci-unreliable");
+    ciRect.setAttribute("filter", `url(#${ensureSketchFilter(svg)})`);
+  }
+  svg.appendChild(ciRect);
+
+  if (opts.smooth) drawSmoothedDist(svg, histogram, scales, stroke);
+  else drawHistogramBars(svg, histogram, scales, layout, stroke);
+
+  drawReferenceLine(svg, scales, layout, includeZero);
+  svg.appendChild(
+    line(ptX, margin.top, ptX, margin.top + plot.h, {
+      stroke,
+      strokeWidth: "2",
+    }),
+  );
+
+  drawCILabels(svg, ci, scales, layout, opts);
+  return svg;
+}
+
+/** Convenience wrapper for DifferenceCI data */
+export function createCIPlot(
+  ci: DifferenceCI,
+  options: Partial<DistributionPlotOptions> = {},
+): SVGSVGElement {
+  if (!ci.histogram) return createSvg(0, 0);
+  return createDistributionPlot(ci.histogram, ci.ci, ci.percent, {
+    title: ci.label,
+    direction: ci.direction,
+    ciLevel: ci.ciLevel,
+    ciReliable: ci.ciReliable,
+    ...options,
+  });
+}
+
+/** Use minimal margins when the chart is too small for standard spacing. */
+function buildLayout(
+  width: number,
+  height: number,
+  hasPointLabel?: boolean,
+): Layout {
+  const compact = height < defaultMargin.top + defaultMargin.bottom + 10;
+  const margin = compact
+    ? { top: 4, right: 6, bottom: 4, left: 6 }
+    : { ...defaultMargin, top: hasPointLabel ? 30 : defaultMargin.top };
+  const plot = {
+    w: width - margin.left - margin.right,
+    h: height - margin.top - margin.bottom,
+  };
+  return { width, height, margin, plot };
+}
+
+function buildScales(
+  histogram: HistogramBin[],
+  ci: [number, number],
+  layout: Layout,
+  includeZero: boolean,
+  equivMargin?: number,
+  pointEstimate?: number,
+): Scales {
+  const { margin, plot } = layout;
+  const xs = histogram.map(b => b.x);
+  const extra = includeZero ? [0] : [];
+  const marginBounds = equivMargin ? [-equivMargin, equivMargin] : [];
+  const ptBounds = pointEstimate != null ? [pointEstimate] : [];
+  const xMin = Math.min(...xs, ci[0], ...extra, ...marginBounds, ...ptBounds);
+  const xMax = Math.max(...xs, ci[1], ...extra, ...marginBounds, ...ptBounds);
+  const yMax = Math.max(...histogram.map(b => b.count));
+  const xRange = xMax - xMin || 1;
+  return {
+    x: (v: number) => margin.left + ((v - xMin) / xRange) * plot.w,
+    y: (v: number) => margin.top + plot.h - (v / yMax) * plot.h,
+  };
+}
+
+function drawTitles(
+  svg: SVGSVGElement,
+  opts: DistributionPlotOptions,
+  margin: Layout["margin"],
+  pointX: number,
+): void {
+  if (opts.title)
+    svg.appendChild(
+      text(margin.left, 14, opts.title, "start", "13", "currentColor", "600"),
+    );
+  if (opts.pointLabel) {
+    const el = text(
+      pointX,
+      margin.top - 6,
+      opts.pointLabel,
+      "middle",
+      "15",
+      "currentColor",
+      "700",
+    );
+    svg.appendChild(el);
+  }
+}
+
+/** Draw equivalence margin zone: hatched band centered vertically */
+function drawMarginZone(
+  svg: SVGSVGElement,
+  equivMargin: number,
+  scales: Scales,
+  layout: Layout,
+): void {
+  const { margin, plot } = layout;
+  const x1 = scales.x(-equivMargin);
+  const x2 = scales.x(equivMargin);
+  const fill = `url(#${ensureHatchPattern(svg)})`;
+  const bandH = plot.h / 3;
+  const bandY = margin.top + (plot.h - bandH) / 2;
+  const zone = rect(x1, bandY, x2 - x1, bandH, { fill, strokeWidth: "1.5" });
+  zone.classList.add("margin-zone");
+  svg.appendChild(zone);
+}
+
+/** Draw a filled area + stroke path using gaussian-smoothed histogram data */
+function drawSmoothedDist(
+  svg: SVGSVGElement,
+  histogram: HistogramBin[],
+  scales: Scales,
+  stroke: string,
+): void {
+  const sorted = [...histogram].sort((a, b) => a.x - b.x);
+  const smoothed = gaussianSmooth(sorted, 2);
+  const pts = smoothed.map(b => `${scales.x(b.x)},${scales.y(b.count)}`);
+  const base = scales.y(0);
+  const startX = scales.x(smoothed[0].x);
+  const endX = scales.x(smoothed.at(-1)!.x);
+  const fillD = `M${startX},${base}L${pts.join("L")}L${endX},${base}Z`;
+  const fillPath = path(fillD, { fill: stroke });
+  fillPath.classList.add("dist-fill");
+  svg.appendChild(fillPath);
+  const strokePath = path(`M${pts.join("L")}`, {
+    stroke,
+    fill: "none",
+    strokeWidth: "1.5",
+  });
+  strokePath.classList.add("dist-stroke");
+  svg.appendChild(strokePath);
+}
+
+function drawHistogramBars(
+  svg: SVGSVGElement,
+  histogram: HistogramBin[],
+  scales: Scales,
+  layout: Layout,
+  stroke: string,
+): void {
+  const sorted = [...histogram].sort((a, b) => a.x - b.x);
+  const binW = sorted.length > 1 ? sorted[1].x - sorted[0].x : 1;
+  const xRange = scales.x(sorted.at(-1)!.x) - scales.x(sorted[0].x) + binW;
+  const barW = (binW / xRange) * layout.plot.w * 0.9;
+  const base = scales.y(0);
+  const attrs = { fill: stroke, opacity: "0.6" };
+  for (const bin of sorted) {
+    const top = scales.y(bin.count);
+    svg.appendChild(
+      rect(scales.x(bin.x) - barW / 2, top, barW, base - top, attrs),
+    );
+  }
+}
+
+/** Draw zero reference line extending past plot area (comparison CIs only) */
+function drawReferenceLine(
+  svg: SVGSVGElement,
+  scales: Scales,
+  layout: Layout,
+  includeZero: boolean,
+): void {
+  const { margin, plot } = layout;
+  const zeroX = scales.x(0);
+  const inBounds = zeroX >= margin.left && zeroX <= layout.width - margin.right;
+  if (!includeZero || !inBounds) return;
+
+  svg.appendChild(
+    line(zeroX, margin.top - 4, zeroX, margin.top + plot.h + 4, {
+      stroke: "#000",
+      strokeWidth: "1",
+    }),
+  );
+}
+
+function drawCILabels(
+  svg: SVGSVGElement,
+  ci: [number, number],
+  scales: Scales,
+  layout: Layout,
+  opts: DistributionPlotOptions & { includeZero: boolean },
+): void {
+  if (layout.margin.bottom < 15) return;
+  const labelY = layout.height - 4;
+  const loLabel = opts.ciLabels?.[0] ?? formatPct(ci[0], 0);
+  const hiLabel = opts.ciLabels?.[1] ?? formatPct(ci[1], 0);
+  const loX = scales.x(ci[0]);
+  const hiX = scales.x(ci[1]);
+  const minGap = Math.max(loLabel.length, hiLabel.length) * 6;
+  if (!opts.includeZero || hiX - loX >= minGap) {
+    svg.appendChild(text(loX, labelY, loLabel, "middle", "11"));
+    svg.appendChild(text(hiX, labelY, hiLabel, "middle", "11"));
+  }
+}
+
+/** Apply gaussian kernel smoothing to histogram bins */
+function gaussianSmooth(bins: HistogramBin[], sigma: number): HistogramBin[] {
+  return bins.map((bin, i) => {
+    let sum = 0;
+    let wt = 0;
+    for (let j = 0; j < bins.length; j++) {
+      const w = Math.exp(-((i - j) ** 2) / (2 * sigma ** 2));
+      sum += bins[j].count * w;
+      wt += w;
+    }
+    return { x: bin.x, count: sum / wt };
+  });
+}

package/src/{html/browser → viewer/plots}/HistogramKde.ts

@@ -1,34 +1,40 @@
 import * as Plot from "@observablehq/plot";
 import * as d3 from "d3";
 import { buildLegend, type LegendItem } from "./LegendUtils.ts";
-import type { Sample } from "./Types.ts";
+import { getTimeUnit, plotLayout, type Sample } from "./PlotTypes.ts";
+
+interface Bar {
+  benchmark: string;
+  count: number;
+  x1: number;
+  x2: number;
+}
 
 /** Create histogram + KDE plot for sample distribution */
 export function createHistogramKde(
   allSamples: Sample[],
   benchmarkNames: string[],
 ): SVGSVGElement | HTMLElement {
+  const values = allSamples.map(d => d.value);
+  const { unitSuffix, convertValue, formatValue } = getTimeUnit(values);
+  const converted = allSamples.map(d => ({
+    ...d,
+    value: convertValue(d.value),
+  }));
   const { barData, binMin, binMax, yMax } = buildBarData(
-    allSamples,
+    converted,
     benchmarkNames,
   );
   const { colorMap, legendItems } = buildColorData(benchmarkNames);
-  const xMax = binMax + (binMax - binMin) * 0.45; // extend for legend
 
   return Plot.plot({
-    marginTop: 24,
-    marginLeft: 70,
-    marginRight: 10,
-    marginBottom: 60,
-    width: 550,
-    height: 300,
-    style: { fontSize: "14px" },
+    ...plotLayout,
     x: {
-      label: "Time (ms)",
+      label: `Time (${unitSuffix})`,
       labelAnchor: "center",
-      domain: [binMin, xMax],
+      domain: [binMin, binMax],
       labelOffset: 45,
-      tickFormat: (d: number) => d.toFixed(1),
+      tickFormat: formatValue,
       ticks: 5,
     },
     y: {
@@ -43,58 +49,34 @@ export function createHistogramKde(
         x1: "x1",
         x2: "x2",
         y: "count",
-        fill: (d: (typeof barData)[0]) => colorMap.get(d.benchmark),
+        fill: (d: Bar) => colorMap.get(d.benchmark),
         fillOpacity: 0.6,
-        tip: true,
-        title: (d: (typeof barData)[0]) => `${d.benchmark}: ${d.count}`,
       }),
       Plot.ruleY([0]),
-      ...buildLegend({ xMin: binMin, xMax, yMax }, legendItems),
+      ...buildLegend({ xMin: binMin, xMax: binMax, yMax }, legendItems),
     ],
   });
 }
 
-function buildColorData(benchmarkNames: string[]) {
-  const scheme = (d3 as any).schemeObservable10;
-  const colorMap = new Map(
-    benchmarkNames.map((name, i) => [name, scheme[i % 10]]),
-  );
-  const legendItems: LegendItem[] = benchmarkNames.map((name, i) => ({
-    color: scheme[i % 10],
-    label: name,
-    style: "vertical-bar",
-  }));
-  return { colorMap, legendItems };
-}
-
 /** Bin samples into grouped histogram bars for each benchmark */
 function buildBarData(allSamples: Sample[], benchmarkNames: string[]) {
-  const values = allSamples.map(d => d.value);
-  const sorted = values.sort((a, b) => a - b);
-  const binMin = d3.quantile(sorted, 0.01)!;
-  const binMax = d3.quantile(sorted, 0.99)!;
+  const sortedValues = allSamples.map(d => d.value).sort((a, b) => a - b);
+  const binMin = d3.quantile(sortedValues, 0.01)!;
+  const binMax = d3.quantile(sortedValues, 0.99)!;
   const binCount = 25;
   const step = (binMax - binMin) / binCount;
   const thresholds = d3.range(1, binCount).map(i => binMin + i * step);
-  const plotWidth = 550;
-
   const bins = d3
     .bin<Sample, number>()
     .domain([binMin, binMax])
     .thresholds(thresholds)
     .value(d => d.value)(allSamples);
 
-  const barData: {
-    benchmark: string;
-    count: number;
-    x1: number;
-    x2: number;
-  }[] = [];
   const n = benchmarkNames.length;
-  const unitsPerPx = (binMax - binMin) / plotWidth;
+  const unitsPerPx = (binMax - binMin) / plotLayout.width;
   const groupGapPx = 8;
 
-  for (const bin of bins) {
+  const barData: Bar[] = bins.flatMap(bin => {
     const counts = new Map<string, number>();
     for (const d of bin)
       counts.set(d.benchmark, (counts.get(d.benchmark) || 0) + 1);
@@ -104,15 +86,28 @@ function buildBarData(allSamples: Sample[], benchmarkNames: string[]) {
     const start = bin.x0! + groupGap / 2;
     const w = (full - groupGap) / n;
 
-    benchmarkNames.forEach((benchmark, i) => {
+    return benchmarkNames.map((benchmark, i) => {
       const x1 = start + i * w;
       const x2 = start + (i + 1) * w;
-      barData.push({ benchmark, count: counts.get(benchmark) || 0, x1, x2 });
+      return { benchmark, count: counts.get(benchmark) || 0, x1, x2 };
     });
-  }
+  });
 
   const maxCount = d3.max(barData, d => d.count)! || 1;
   const yMax = maxCount * 1.15;
 
   return { barData, binMin, binMax, yMax };
 }
+
+/** Map benchmark names to colors and legend items using Observable 10 palette */
+function buildColorData(benchmarkNames: string[]) {
+  const scheme = (d3 as any).schemeObservable10;
+  const color = (i: number) => scheme[i % 10];
+  const colorMap = new Map(benchmarkNames.map((name, i) => [name, color(i)]));
+  const legendItems: LegendItem[] = benchmarkNames.map((name, i) => ({
+    color: color(i),
+    label: name,
+    style: "vertical-bar",
+  }));
+  return { colorMap, legendItems };
+}

package/src/viewer/plots/LegendUtils.ts

@@ -0,0 +1,134 @@
+import * as Plot from "@observablehq/plot";
+
+/** Plot data bounds used to position the legend overlay */
+export interface LegendBounds {
+  xMin: number;
+  xMax: number;
+  yMin?: number;
+  yMax: number;
+}
+
+/** A single entry in the plot legend with color, label, and symbol style */
+export interface LegendItem {
+  color: string;
+  label: string;
+  style:
+    | "filled-dot"
+    | "hollow-dot"
+    | "vertical-bar"
+    | "vertical-line"
+    | "rect";
+  strokeDash?: string;
+}
+
+interface LegendPos {
+  legendX: number;
+  y: number;
+  textX: number;
+  xRange: number;
+  yRange: number;
+}
+
+const rectFields = { x1: "x1", x2: "x2", y1: "y1", y2: "y2" } as const;
+
+/** Build complete legend marks array, positioned in the right margin */
+export function buildLegend(bounds: LegendBounds, items: LegendItem[]): any[] {
+  const xRange = Math.max(bounds.xMax - bounds.xMin, bounds.xMax * 0.1 || 1);
+  const yRange = bounds.yMax - (bounds.yMin ?? 0);
+  const legendX = bounds.xMax + xRange * 0.04;
+  const textX = legendX + xRange * 0.03;
+  const itemHeight = yRange * 0.07;
+  const topY = bounds.yMax - yRange * 0.02;
+
+  const pos = (i: number): LegendPos => ({
+    legendX,
+    y: topY - i * itemHeight,
+    textX,
+    xRange,
+    yRange,
+  });
+  return items.flatMap((item, i) => [
+    symbolMark(pos(i), item),
+    textMark(pos(i), item.label),
+  ]);
+}
+
+function symbolMark(pos: LegendPos, item: LegendItem): any {
+  switch (item.style) {
+    case "filled-dot":
+      return dotMark(pos.legendX, pos.y, item.color, true);
+    case "hollow-dot":
+      return dotMark(pos.legendX, pos.y, item.color, false);
+    case "vertical-bar":
+      return verticalBarMark(pos, item.color);
+    case "vertical-line":
+      return verticalLineMark(pos, item.color, item.strokeDash);
+    case "rect":
+      return rectMark(pos, item.color);
+  }
+}
+
+function textMark(pos: LegendPos, label: string): any {
+  return Plot.text([{ x: pos.textX, y: pos.y, text: label }], {
+    x: "x",
+    y: "y",
+    text: "text",
+    fontSize: 11,
+    textAnchor: "start",
+    fill: "#333",
+    clip: false,
+  });
+}
+
+function dotMark(x: number, y: number, color: string, filled: boolean): any {
+  const base = { x: "x", y: "y", r: 4, clip: false };
+  const style = filled
+    ? { ...base, fill: color }
+    : { ...base, stroke: color, fill: "none", strokeWidth: 1.5 };
+  return Plot.dot([{ x, y }], style);
+}
+
+function verticalBarMark(pos: LegendPos, color: string): any {
+  const { legendX, y, xRange, yRange } = pos;
+  const hw = xRange * 0.006;
+  const hh = yRange * 0.025;
+  const data = [{ x1: legendX - hw, x2: legendX + hw, y1: y - hh, y2: y + hh }];
+  return Plot.rect(data, {
+    ...rectFields,
+    fill: color,
+    fillOpacity: 0.6,
+    clip: false,
+  });
+}
+
+function verticalLineMark(
+  pos: LegendPos,
+  color: string,
+  strokeDash?: string,
+): any {
+  const { legendX, y, yRange } = pos;
+  const half = yRange * 0.025;
+  return Plot.ruleX([legendX], {
+    y1: y - half,
+    y2: y + half,
+    stroke: color,
+    strokeWidth: 2,
+    strokeDasharray: strokeDash,
+    clip: false,
+  });
+}
+
+function rectMark(pos: LegendPos, color: string): any {
+  const { legendX, y, xRange, yRange } = pos;
+  const hw = xRange * 0.015;
+  const hh = yRange * 0.02;
+  const data = [{ x1: legendX - hw, x2: legendX + hw, y1: y - hh, y2: y + hh }];
+  return Plot.rect(data, {
+    ...rectFields,
+    fill: color,
+    fillOpacity: 0.3,
+    stroke: color,
+    strokeWidth: 1,
+    clip: false,
+  });
+}

package/src/viewer/plots/PlotTypes.ts

@@ -0,0 +1,85 @@
+/** A single timing sample from a benchmark run */
+export interface Sample {
+  benchmark: string;
+  value: number;
+  iteration: number;
+}
+
+/** A sample with warmup/optimization metadata for time series plots */
+export interface TimeSeriesPoint {
+  benchmark: string;
+  iteration: number;
+  value: number;
+  isWarmup: boolean;
+  isBaseline?: boolean;
+  isRejected?: boolean;
+  /** V8 optimization status code (e.g. 17=turbofan, 33=maglev) */
+  optStatus?: number;
+}
+
+/** Heap usage sample (in bytes) at a given iteration */
+export interface HeapPoint {
+  benchmark: string;
+  iteration: number;
+  value: number;
+}
+
+/** GcEvent flattened with benchmark name for multi-series plots */
+export interface FlatGcEvent {
+  benchmark: string;
+  sampleIndex: number;
+  duration: number;
+}
+
+/** PausePoint flattened with benchmark name for multi-series plots */
+export interface FlatPausePoint {
+  benchmark: string;
+  sampleIndex: number;
+  durationMs: number;
+}
+
+/** Display unit (ns/us/ms) with conversion and formatting functions */
+export interface TimeUnit {
+  unitSuffix: string;
+  convertValue: (ms: number) => number;
+  formatValue: (d: number) => string;
+}
+
+/** Shared Observable Plot layout: margins, dimensions, font size */
+export const plotLayout = {
+  marginTop: 24,
+  marginLeft: 70,
+  marginRight: 110,
+  marginBottom: 60,
+  width: 550,
+  height: 300,
+  style: { fontSize: "14px" },
+} as const;
+
+/** Format a number as a signed percentage string (e.g. "+1.2%", "-3.4%") */
+export function formatPct(v: number, precision = 1): string {
+  const sign = v >= 0 ? "+" : "";
+  return `${sign}${v.toFixed(precision)}%`;
+}
+
+/** Pick display unit (ns/us/ms) based on average value magnitude (in ms) */
+export function getTimeUnit(values: number[]): TimeUnit {
+  const avg = values.reduce((s, v) => s + v, 0) / values.length;
+  const locale = (digits: number) => (d: number) =>
+    d.toLocaleString("en-US", { maximumFractionDigits: digits });
+  const fmt0 = locale(0);
+  const fmt1 = locale(1);
+  if (avg < 0.001)
+    return {
+      unitSuffix: "ns",
+      convertValue: ms => ms * 1e6,
+      formatValue: fmt0,
+    };
+  if (avg < 1)
+    return {
+      unitSuffix: "\u00b5s",
+      convertValue: ms => ms * 1e3,
+      formatValue: fmt1,
+    };
+  return { unitSuffix: "ms", convertValue: ms => ms, formatValue: fmt1 };
+}