benchforge 0.1.9 → 0.2.4

package/src/viewer/plots/RenderPlots.ts

@@ -0,0 +1,230 @@
+import {
+  average,
+  splitByOffsets,
+  tukeyKeep,
+} from "../../stats/StatisticalUtils.ts";
+import type { BenchmarkEntry, ReportData } from "../ReportData.ts";
+import type {
+  FlatGcEvent,
+  FlatPausePoint,
+  HeapPoint,
+  Sample,
+  TimeSeriesPoint,
+} from "./PlotTypes.ts";
+
+/** Benchmark entry tagged with whether it's the baseline for comparison */
+export interface PreparedBenchmark extends BenchmarkEntry {
+  isBaseline: boolean;
+}
+
+/** All sample data flattened across benchmarks into arrays for plotting */
+export interface FlattenedData {
+  allSamples: Sample[];
+  timeSeries: TimeSeriesPoint[];
+  heapSeries: HeapPoint[];
+  baselineHeapSeries: HeapPoint[];
+  allGcEvents: FlatGcEvent[];
+  allPausePoints: FlatPausePoint[];
+}
+
+/** Combine baseline and benchmarks into a single list with display names */
+export function prepareBenchmarks(
+  group: ReportData["groups"][0],
+): PreparedBenchmark[] {
+  const base = group.baseline;
+  const current = group.benchmarks.map(b => ({ ...b, isBaseline: false }));
+  if (!base) return current;
+
+  const baseName = base.name.endsWith("(baseline)")
+    ? base.name
+    : base.name + " (baseline)";
+  return [{ ...base, name: baseName, isBaseline: true }, ...current];
+}
+
+/** Collect all sample data across benchmarks into flat arrays for plotting */
+export function flattenSamples(benchmarks: PreparedBenchmark[]): FlattenedData {
+  const out: FlattenedData = {
+    allSamples: [],
+    timeSeries: [],
+    heapSeries: [],
+    baselineHeapSeries: [],
+    allGcEvents: [],
+    allPausePoints: [],
+  };
+  for (const b of benchmarks) {
+    if (b.samples?.length) flattenBenchmark(b, out);
+  }
+  return out;
+}
+
+/** @return batch count from the first benchmark with batchOffsets, or 0 */
+export function batchCount(benchmarks: PreparedBenchmark[]): number {
+  return (
+    benchmarks.find(b => b.batchOffsets?.length)?.batchOffsets?.length ?? 0
+  );
+}
+
+/** Filter flattened data to a single batch, re-indexing iterations from 0 */
+export function filterToBatch(
+  flat: FlattenedData,
+  benchmarks: PreparedBenchmark[],
+  batchIndex: number,
+): FlattenedData {
+  const ranges = new Map<string, [number, number]>();
+  for (const b of benchmarks) {
+    const offsets = b.batchOffsets;
+    if (!offsets?.length) continue;
+    const start = offsets[batchIndex];
+    const end =
+      batchIndex + 1 < offsets.length
+        ? offsets[batchIndex + 1]
+        : b.samples.length;
+    ranges.set(b.name, [start, end]);
+  }
+
+  const inBatch = (name: string, iter: number) => {
+    const r = ranges.get(name);
+    return r ? iter >= r[0] && iter < r[1] : true;
+  };
+  const reindex = (name: string, iter: number) => {
+    const r = ranges.get(name);
+    return r ? iter - r[0] : iter;
+  };
+  const sliceIter = <T extends { benchmark: string; iteration: number }>(
+    arr: T[],
+  ) =>
+    arr
+      .filter(d => inBatch(d.benchmark, d.iteration))
+      .map(d => ({ ...d, iteration: reindex(d.benchmark, d.iteration) }));
+
+  return {
+    allSamples: sliceIter(flat.allSamples),
+    timeSeries: sliceIter(flat.timeSeries.filter(d => !d.isWarmup)),
+    heapSeries: sliceIter(flat.heapSeries),
+    baselineHeapSeries: sliceIter(flat.baselineHeapSeries),
+    allGcEvents: flat.allGcEvents.filter(d =>
+      inBatch(d.benchmark, d.sampleIndex),
+    ),
+    allPausePoints: flat.allPausePoints.filter(d =>
+      inBatch(d.benchmark, d.sampleIndex),
+    ),
+  };
+}
+
+/** Extract time series, heap, GC, and pause data from one benchmark */
+function flattenBenchmark(b: PreparedBenchmark, out: FlattenedData): void {
+  flattenWarmup(b, b.name, out);
+  flattenSamplesAndHeap(b, b.name, out);
+  flattenGcEvents(b, b.name, out);
+  flattenPausePoints(b, b.name, out);
+}
+
+/** Warmup samples get negative iteration indices so they appear left of zero */
+function flattenWarmup(
+  b: PreparedBenchmark,
+  name: string,
+  out: FlattenedData,
+): void {
+  const warmupCount = b.warmupSamples?.length || 0;
+  b.warmupSamples?.forEach((value, i) => {
+    out.timeSeries.push({
+      benchmark: name,
+      iteration: i - warmupCount,
+      value,
+      isWarmup: true,
+    });
+  });
+}
+
+/** Populate timeSeries, allSamples (excluding rejected), and heap data */
+function flattenSamplesAndHeap(
+  b: PreparedBenchmark,
+  name: string,
+  out: FlattenedData,
+): void {
+  const rejected = rejectedIndices(b);
+  const isBase = b.isBaseline || undefined;
+  b.samples.forEach((value, i) => {
+    const isRejected = rejected?.has(i) || undefined;
+    if (!isRejected)
+      out.allSamples.push({ benchmark: name, value, iteration: i });
+    const optStatus = b.optSamples?.[i];
+    out.timeSeries.push({
+      benchmark: name,
+      iteration: i,
+      value,
+      isWarmup: false,
+      isBaseline: isBase,
+      isRejected,
+      optStatus,
+    });
+    if (b.heapSamples?.[i] !== undefined) {
+      const target = b.isBaseline ? out.baselineHeapSeries : out.heapSeries;
+      target.push({ benchmark: name, iteration: i, value: b.heapSamples[i] });
+    }
+  });
+}
+
+/** Map GC events to sample indices using cumulative sample durations */
+function flattenGcEvents(
+  b: PreparedBenchmark,
+  name: string,
+  out: FlattenedData,
+): void {
+  if (!b.gcEvents?.length) return;
+  const endTimes = cumulativeSum(b.samples);
+  for (const gc of b.gcEvents) {
+    const idx = endTimes.findIndex(t => t >= gc.offset);
+    const sampleIndex = idx >= 0 ? idx : b.samples.length - 1;
+    out.allGcEvents.push({
+      benchmark: name,
+      sampleIndex,
+      duration: gc.duration,
+    });
+  }
+}
+
+/** Flatten benchmark pause points into the shared output arrays */
+function flattenPausePoints(
+  b: PreparedBenchmark,
+  name: string,
+  out: FlattenedData,
+): void {
+  if (!b.pausePoints) return;
+  for (const p of b.pausePoints)
+    out.allPausePoints.push({
+      benchmark: name,
+      sampleIndex: p.sampleIndex,
+      durationMs: p.durationMs,
+    });
+}
+
+/** @return sample indices in Tukey-rejected batches, or undefined if none */
+function rejectedIndices(b: PreparedBenchmark): Set<number> | undefined {
+  const offsets = b.batchOffsets;
+  if (!offsets || offsets.length < 4) return undefined;
+
+  const means = splitByOffsets(b.samples, offsets).map(s => average(s));
+  const kept = new Set(tukeyKeep(means));
+
+  const rejected = new Set<number>();
+  for (let bi = 0; bi < means.length; bi++) {
+    if (!kept.has(bi)) {
+      const start = offsets[bi];
+      const end = bi + 1 < offsets.length ? offsets[bi + 1] : b.samples.length;
+      for (let j = start; j < end; j++) rejected.add(j);
+    }
+  }
+  return rejected.size > 0 ? rejected : undefined;
+}
+
+/** Running total of sample durations, used to map GC offsets to sample indices */
+function cumulativeSum(arr: number[]): number[] {
+  const result: number[] = [];
+  let sum = 0;
+  for (const v of arr) {
+    sum += v;
+    result.push(sum);
+  }
+  return result;
+}

package/src/viewer/plots/SampleTimeSeries.ts

@@ -0,0 +1,306 @@
+import * as Plot from "@observablehq/plot";
+import * as d3 from "d3";
+import { optStatusNames } from "../../runners/MeasuredResults.ts";
+import { buildLegend, type LegendItem } from "./LegendUtils.ts";
+import {
+  type FlatGcEvent,
+  type FlatPausePoint,
+  getTimeUnit,
+  type HeapPoint,
+  plotLayout,
+  type TimeSeriesPoint,
+} from "./PlotTypes.ts";
+import {
+  buildLegendItems,
+  gcMark,
+  type HeapScale,
+  heapAxisMarks,
+  heapMarks,
+  type PlotContext,
+  pauseMarks,
+  type SampleData,
+  sampleDotMarks,
+} from "./TimeSeriesMarks.ts";
+
+/** Controls which data series are visible in the time series plot */
+export interface SeriesVisibility {
+  baseline: boolean;
+  heap: boolean;
+  baselineHeap: boolean;
+  rejected: boolean;
+}
+
+type HeapPlotPoint = { sample: number; y: number };
+
+interface MarkParams {
+  ctx: PlotContext;
+  heapData: HeapPlotPoint[];
+  baselineHeapData: HeapPlotPoint[];
+  heapScale: HeapScale | undefined;
+  gcEvents: FlatGcEvent[];
+  pausePoints: FlatPausePoint[];
+  legendItems: LegendItem[];
+  showRejected: boolean;
+}
+
+const defaultVisibility: SeriesVisibility = {
+  baseline: true,
+  heap: true,
+  baselineHeap: false,
+  rejected: true,
+};
+
+/** Time series plot with samples, GC events, heap overlay, and opt tiers */
+export function createSampleTimeSeries(
+  timeSeries: TimeSeriesPoint[],
+  gcEvents: FlatGcEvent[] = [],
+  pausePoints: FlatPausePoint[] = [],
+  heapSeries: HeapPoint[] = [],
+  baselineHeapSeries: HeapPoint[] = [],
+  visibility: SeriesVisibility = defaultVisibility,
+): SVGSVGElement | HTMLElement {
+  const filtered = visibility.baseline
+    ? timeSeries
+    : timeSeries.filter(d => !d.isBaseline);
+  const ctx = buildPlotContext(filtered);
+  const series = prepareSeriesData(
+    ctx,
+    heapSeries,
+    baselineHeapSeries,
+    visibility,
+    gcEvents,
+    pausePoints,
+  );
+
+  return Plot.plot({
+    ...plotLayout,
+    x: {
+      label: "Iteration",
+      labelAnchor: "center",
+      labelOffset: 45,
+      grid: true,
+      domain: [ctx.xMin, ctx.xMax],
+    },
+    y: {
+      label: `Time (${ctx.unitSuffix})`,
+      labelAnchor: "top",
+      labelArrow: false,
+      grid: true,
+      domain: [ctx.yMin, ctx.yMax],
+      tickFormat: ctx.formatValue,
+    },
+    color: { legend: false, scheme: "observable10" },
+    marks: buildMarks({ ctx, ...series, gcEvents, pausePoints }),
+  });
+}
+
+/** Derive scales, units, and metadata from time series data */
+function buildPlotContext(timeSeries: TimeSeriesPoint[]): PlotContext {
+  const benchmarks = [...new Set(timeSeries.map(d => d.benchmark))];
+  const sampleData = buildSampleData(timeSeries);
+  const values = sampleData.map(d => d.value);
+  const { unitSuffix, convertValue, formatValue } = getTimeUnit(values);
+  const convertedData: SampleData[] = sampleData.map(d => ({
+    ...d,
+    displayValue: convertValue(d.value),
+  }));
+  const { yMin, yMax } = computeYRange(convertedData.map(d => d.displayValue));
+  let xMin = d3.min(convertedData, d => d.sample)!;
+  let xMax = d3.max(convertedData, d => d.sample)!;
+  if (xMin === xMax) {
+    xMin -= 0.5;
+    xMax += 0.5;
+  }
+  const hasWarmup = convertedData.some(d => d.isWarmup);
+  const hasRejected = convertedData.some(d => d.isRejected);
+  const baselineNames = new Set(
+    convertedData.filter(d => d.isBaseline).map(d => d.benchmark),
+  );
+  const optTiers = [
+    ...new Set(
+      convertedData.filter(d => d.optTier && !d.isWarmup).map(d => d.optTier!),
+    ),
+  ];
+  return {
+    convertedData,
+    xMin,
+    xMax,
+    yMin,
+    yMax,
+    unitSuffix,
+    formatValue,
+    convertValue,
+    hasWarmup,
+    hasRejected,
+    baselineNames,
+    optTiers,
+    benchmarks,
+  };
+}
+
+/** Prepare heap, legend, and visibility state for the time series plot */
+function prepareSeriesData(
+  ctx: PlotContext,
+  heapSeries: HeapPoint[],
+  baselineHeapSeries: HeapPoint[],
+  visibility: SeriesVisibility,
+  gcEvents: FlatGcEvent[],
+  pausePoints: FlatPausePoint[],
+) {
+  const visibleHeap = [
+    ...(visibility.heap ? heapSeries : []),
+    ...(visibility.baselineHeap ? baselineHeapSeries : []),
+  ];
+  const heapScale = computeHeapScale(visibleHeap, ctx.yMin, ctx.yMax);
+  const heapData =
+    heapScale && visibility.heap ? prepareHeapData(heapSeries, heapScale) : [];
+  const baselineHeapData =
+    heapScale && visibility.baselineHeap
+      ? prepareHeapData(baselineHeapSeries, heapScale)
+      : [];
+  const showRejected = visibility.rejected && ctx.hasRejected;
+  const legendItems = buildLegendItems({
+    hasWarmup: ctx.hasWarmup,
+    gcCount: gcEvents.length,
+    pauseCount: pausePoints.length,
+    hasHeap: heapData.length > 0,
+    hasBaselineHeap: baselineHeapData.length > 0,
+    hasRejected: showRejected,
+    optTiers: ctx.optTiers,
+    benchmarks: ctx.benchmarks,
+    baselineNames: ctx.baselineNames,
+  });
+  return { heapScale, heapData, baselineHeapData, showRejected, legendItems };
+}
+
+/** Assemble all Observable Plot marks for the time series chart */
+function buildMarks(p: MarkParams): Plot.Markish[] {
+  const { ctx, heapData, baselineHeapData, heapScale } = p;
+  const { gcEvents, pausePoints, legendItems, showRejected } = p;
+  const dashStyle = { stroke: "#999", strokeWidth: 1, strokeDasharray: "4,4" };
+  const warmupRule = ctx.hasWarmup ? [Plot.ruleX([0], dashStyle)] : [];
+  const { xMin, xMax, yMin, yMax } = ctx;
+  return [
+    ...heapMarks(baselineHeapData, yMin, "#fcd34d"),
+    ...heapMarks(heapData, yMin, "#93c5fd"),
+    ...heapAxisMarks(heapScale, xMax, xMin),
+    ...warmupRule,
+    gcMark(gcEvents, yMin, ctx.convertValue),
+    ...pauseMarks(pausePoints, yMin, yMax),
+    ...sampleDotMarks(ctx, showRejected, lttb),
+    Plot.ruleY([yMin], { stroke: "black", strokeWidth: 1 }),
+    ...buildLegend({ xMin, xMax, yMin, yMax }, legendItems),
+  ];
+}
+
+/** Convert TimeSeriesPoint data to SampleData with opt tier names */
+function buildSampleData(
+  timeSeries: TimeSeriesPoint[],
+): Omit<SampleData, "displayValue">[] {
+  return timeSeries.map(d => ({
+    benchmark: d.benchmark,
+    sample: d.iteration,
+    value: d.value,
+    isBaseline: d.isBaseline || false,
+    isWarmup: d.isWarmup || false,
+    isRejected: d.isRejected || false,
+    optTier:
+      d.optStatus !== undefined
+        ? optStatusNames[d.optStatus] || "unknown"
+        : null,
+  }));
+}
+
+/** Pad Y range and snap yMin to a round number for clean axis ticks */
+function computeYRange(values: number[]) {
+  const dataMin = d3.min(values)!;
+  const dataMax = d3.max(values)!;
+  const range = dataMax - dataMin;
+  let yMin = dataMin - range * 0.15;
+  const mag = 10 ** Math.floor(Math.log10(Math.abs(yMin)));
+  yMin = Math.floor(yMin / mag) * mag;
+  if (dataMin > 0 && yMin < 0) yMin = 0;
+  return { yMin, yMax: dataMax + range * 0.05 };
+}
+
+/** Compute scale to map heap byte values into the bottom 25% of the Y axis */
+function computeHeapScale(
+  allHeap: HeapPoint[],
+  yMin: number,
+  yMax: number,
+): HeapScale | undefined {
+  if (allHeap.length === 0) return undefined;
+  const heapMinBytes = d3.min(allHeap, d => d.value)!;
+  const heapRangeBytes = d3.max(allHeap, d => d.value)! - heapMinBytes || 1;
+  return {
+    heapMinBytes,
+    heapRangeBytes,
+    scale: ((yMax - yMin) * 0.25) / heapRangeBytes,
+    yMin,
+  };
+}
+
+/** Map heap byte values to the time-series Y scale and downsample via LTTB */
+function prepareHeapData(
+  heapSeries: HeapPoint[],
+  hs: HeapScale,
+): HeapPlotPoint[] {
+  if (heapSeries.length === 0) return [];
+  const mapped = heapSeries.map(d => ({
+    sample: d.iteration,
+    y: hs.yMin + (d.value - hs.heapMinBytes) * hs.scale,
+  }));
+  return lttb(
+    mapped,
+    500,
+    d => d.sample,
+    d => d.y,
+  );
+}
+
+/** LTTB downsampling: select n points that best preserve visual shape */
+function lttb<T>(
+  data: T[],
+  n: number,
+  getX: (d: T) => number,
+  getY: (d: T) => number,
+): T[] {
+  if (data.length <= n) return data;
+  const bucketSize = (data.length - 2) / (n - 2);
+  const result: T[] = [data[0]];
+  for (let i = 0; i < n - 2; i++) {
+    const bStart = Math.floor(i * bucketSize) + 1;
+    const bEnd = Math.floor((i + 1) * bucketSize) + 1;
+    const nStart = bEnd;
+    const nEnd = Math.min(
+      Math.floor((i + 2) * bucketSize) + 1,
+      data.length - 1,
+    );
+    let avgX = 0;
+    let avgY = 0;
+    for (let j = nStart; j < nEnd; j++) {
+      avgX += getX(data[j]);
+      avgY += getY(data[j]);
+    }
+    const cnt = nEnd - nStart || 1;
+    avgX /= cnt;
+    avgY /= cnt;
+    const prev = result[result.length - 1];
+    const px = getX(prev);
+    const py = getY(prev);
+    let maxArea = -1;
+    let maxIdx = bStart;
+    for (let j = bStart; j < bEnd; j++) {
+      const area = Math.abs(
+        (px - avgX) * (getY(data[j]) - py) - (px - getX(data[j])) * (avgY - py),
+      );
+      if (area > maxArea) {
+        maxArea = area;
+        maxIdx = j;
+      }
+    }
+    result.push(data[maxIdx]);
+  }
+  result.push(data[data.length - 1]);
+  return result;
+}

package/src/viewer/plots/SvgHelpers.ts

@@ -0,0 +1,136 @@
+export const svgNS = "http://www.w3.org/2000/svg";
+
+const toKebab = (k: string) => k.replace(/[A-Z]/g, c => "-" + c.toLowerCase());
+const svgEl = (tag: string) => document.createElementNS(svgNS, tag);
+
+/** Apply camelCase attributes to an SVG element, converting to kebab-case */
+export function setAttrs(el: SVGElement, attrs: Record<string, string>): void {
+  for (const [k, v] of Object.entries(attrs)) el.setAttribute(toKebab(k), v);
+}
+
+/** Create an SVG root element with viewBox. */
+export function createSvg(w: number, h: number): SVGSVGElement {
+  const svg = document.createElementNS(svgNS, "svg");
+  svg.setAttribute("width", String(w));
+  svg.setAttribute("height", String(h));
+  if (w && h) svg.setAttribute("viewBox", `0 0 ${w} ${h}`);
+  return svg;
+}
+
+export function rect(
+  x: number,
+  y: number,
+  w: number,
+  h: number,
+  attrs: Record<string, string>,
+): SVGRectElement {
+  const el = document.createElementNS(svgNS, "rect");
+  el.setAttribute("x", String(x));
+  el.setAttribute("y", String(y));
+  el.setAttribute("width", String(w));
+  el.setAttribute("height", String(h));
+  setAttrs(el, attrs);
+  return el;
+}
+
+export function line(
+  x1: number,
+  y1: number,
+  x2: number,
+  y2: number,
+  attrs: Record<string, string>,
+): SVGLineElement {
+  const el = document.createElementNS(svgNS, "line");
+  el.setAttribute("x1", String(x1));
+  el.setAttribute("y1", String(y1));
+  el.setAttribute("x2", String(x2));
+  el.setAttribute("y2", String(y2));
+  setAttrs(el, attrs);
+  return el;
+}
+
+export function text(
+  x: number,
+  y: number,
+  content: string,
+  anchor = "start",
+  size = "9",
+  fill = "#666",
+  weight = "400",
+): SVGTextElement {
+  const el = document.createElementNS(svgNS, "text");
+  el.setAttribute("x", String(x));
+  el.setAttribute("y", String(y));
+  el.setAttribute("text-anchor", anchor);
+  el.setAttribute("font-size", size);
+  el.setAttribute("font-weight", weight);
+  el.setAttribute("fill", fill);
+  el.textContent = content;
+  return el;
+}
+
+export function path(d: string, attrs: Record<string, string>): SVGPathElement {
+  const el = document.createElementNS(svgNS, "path");
+  el.setAttribute("d", d);
+  setAttrs(el, attrs);
+  return el;
+}
+
+/** Add a turbulence displacement filter for a sketchy/wobbly look */
+export function ensureSketchFilter(svg: SVGSVGElement): string {
+  const id = "ci-sketch";
+  if (svg.querySelector(`#${id}`)) return id;
+  const defs = ensureDefs(svg);
+  const filter = svgEl("filter");
+  setAttrs(filter, { id, x: "-5%", y: "-5%", width: "110%", height: "110%" });
+  const turb = svgEl("feTurbulence");
+  setAttrs(turb, {
+    type: "turbulence",
+    baseFrequency: "0.06",
+    numOctaves: "4",
+    seed: "1",
+    result: "noise",
+  });
+  const disp = svgEl("feDisplacementMap");
+  setAttrs(disp, {
+    in: "SourceGraphic",
+    in2: "noise",
+    scale: "10",
+    xChannelSelector: "R",
+    yChannelSelector: "G",
+  });
+  filter.appendChild(turb);
+  filter.appendChild(disp);
+  defs.appendChild(filter);
+  return id;
+}
+
+/** Add a diagonal hatch pattern to the SVG defs, reusing if already present */
+export function ensureHatchPattern(svg: SVGSVGElement): string {
+  const id = "margin-hatch";
+  if (svg.querySelector(`#${id}`)) return id;
+  const defs = ensureDefs(svg);
+  const pattern = svgEl("pattern");
+  setAttrs(pattern, {
+    id,
+    patternUnits: "userSpaceOnUse",
+    width: "5",
+    height: "5",
+    patternTransform: "rotate(45)",
+  });
+  const stripe = svgEl("line");
+  setAttrs(stripe, { x1: "0", y1: "0", x2: "0", y2: "5" });
+  stripe.classList.add("margin-hatch-stroke");
+  pattern.appendChild(stripe);
+  defs.appendChild(pattern);
+  return id;
+}
+
+function ensureDefs(svg: SVGSVGElement): SVGDefsElement {
+  let defs = svg.querySelector("defs") as SVGDefsElement | null;
+  if (!defs) {
+    defs = document.createElementNS(svgNS, "defs") as SVGDefsElement;
+    svg.insertBefore(defs, svg.firstChild);
+  }
+  return defs;
+}