@mui/internal-benchmark 0.0.3-canary.2 → 0.0.3-canary.21

package/reporter.mjs

@@ -1,46 +1,33 @@
 import * as path from 'node:path';
 import * as fs from 'node:fs/promises';
 import { benchmarkUploadSchema, getCiMetadata } from "./ciReport.mjs";
-import { calculateMean, calculateStdDev, quantile, isOutlier } from "./stats.mjs";
+import { calculateMean, aggregateSamples } from "./stats.mjs";
 import { dim, red, green, yellow, cyan, printTable, fileUrl } from "./format.mjs";
 import { uploadCiReport } from "./upload.mjs";
 import { syncPrComment } from "./syncPrComment.mjs";
 // Import for TaskMeta augmentation side effect
 import "./taskMetaAugmentation.mjs";
-const byNumeric = (a, b) => a - b;
 function getEventKey(event) {
   return `${event.id}:${event.phase}`;
 }
-function aggregateMetrics(iterations) {
-  // Collect all metric names across iterations
-  const metricValues = new Map();
-  for (const iteration of iterations) {
-    for (const metric of iteration.metrics) {
-      let values = metricValues.get(metric.name);
-      if (!values) {
-        values = [];
-        metricValues.set(metric.name, values);
-      }
-      values.push(metric.value);
-    }
+
+/** Order-insensitive deep equality, treating a missing key and an `undefined` value as equal. */
+function deepEqual(first, second) {
+  if (first === second) {
+    return true;
   }
-  const result = {};
-  for (const [name, values] of metricValues) {
-    // Apply IQR filtering
-    const sorted = [...values].sort(byNumeric);
-    const q1 = quantile(sorted, 0.25);
-    const q3 = quantile(sorted, 0.75);
-    const filtered = values.filter(d => !isOutlier(d, q1, q3));
-    const used = filtered.length > 0 ? filtered : values;
-    const mean = calculateMean(used);
-    const stdDev = calculateStdDev(used, mean);
-    result[name] = {
-      mean,
-      stdDev,
-      outliers: values.length - used.length
-    };
+  if (typeof first !== 'object' || first === null || typeof second !== 'object' || second === null) {
+    return false;
   }
-  return result;
+  const firstRecord = first;
+  const secondRecord = second;
+  const keys = new Set([...Object.keys(firstRecord), ...Object.keys(secondRecord)]);
+  for (const key of keys) {
+    if (!deepEqual(firstRecord[key], secondRecord[key])) {
+      return false;
+    }
+  }
+  return true;
 }
 function generateReportFromIterations(iterations) {
   if (iterations.length === 0) {
@@ -69,13 +56,11 @@ function generateReportFromIterations(iterations) {
   const renderStats = [];
   for (let index = 0; index < expectedLength; index += 1) {
     const durations = iterations.map(iteration => iteration.renders[index].actualDuration);
-    const sorted = [...durations].sort(byNumeric);
-    const q1 = quantile(sorted, 0.25);
-    const q3 = quantile(sorted, 0.75);
-    const filtered = durations.filter(d => !isOutlier(d, q1, q3));
-    const used = filtered.length > 0 ? filtered : durations;
-    const iqrMean = calculateMean(used);
-    const iqrStdDev = calculateStdDev(used, iqrMean);
+    const {
+      mean: iqrMean,
+      stdDev: iqrStdDev,
+      outliers
+    } = aggregateSamples(durations);
     const coefficientOfVariation = iqrMean > 0 ? iqrStdDev / iqrMean : 0;
     if (iqrMean > 1 && coefficientOfVariation > 0.1) {
       const event = firstIteration.renders[index];
@@ -85,7 +70,7 @@ function generateReportFromIterations(iterations) {
       event: firstIteration.renders[index],
       iqrMean,
       iqrStdDev,
-      outliers: durations.length - used.length
+      outliers
     });
   }
 
@@ -119,8 +104,8 @@ function generateReportFromIterations(iterations) {
     totalDuration += iqrMean;
   }
 
-  // Aggregate metrics
-  const metrics = aggregateMetrics(iterations);
+  // Custom + paint metrics are merged separately from `task.meta.benchmarkMetrics`.
+  const metrics = {};
   return {
     iterations: iterationCount,
     totalDuration,
@@ -167,21 +152,71 @@ function printDurationMatrix(name, report, footer) {
     width: 4
   }], rows, footer, name);
 }
-function printMetricsTable(name, metrics, iterationCount) {
+
+/** Strips a `#sub-series` suffix to recover the metric name used to look up its definition. */
+function baseMetricName(key) {
+  const hashIndex = key.indexOf('#');
+  return hashIndex === -1 ? key : key.slice(0, hashIndex);
+}
+function formatMetricValue(value, definition) {
+  if (definition?.format) {
+    return new Intl.NumberFormat(undefined, definition.format).format(value);
+  }
+  return value.toFixed(2);
+}
+
+/**
+ * Merges aggregated custom metrics (already stats, not raw samples) into a report entry, keyed
+ * `name` or `name#id` for sub-series, and collects each metric's config into the shared
+ * top-level definitions. For a metric-only test, derives the iteration count from the samples.
+ */
+function mergeCustomMetrics(report, customMetrics, definitions) {
+  let maxCount = 0;
+  for (const [metricName, metric] of Object.entries(customMetrics)) {
+    for (const [seriesId, stats] of Object.entries(metric.series)) {
+      const key = seriesId === '' ? metricName : `${metricName}#${seriesId}`;
+      report.metrics[key] = {
+        mean: stats.mean,
+        stdDev: stats.stdDev,
+        outliers: stats.outliers
+      };
+      maxCount = Math.max(maxCount, stats.count);
+    }
+    const definition = {
+      kind: metric.kind,
+      format: metric.config.format,
+      alarm: metric.config.alarm
+    };
+    // A metric name maps to one definition. Reusing it across benchmarks is fine when the config
+    // matches (e.g. the harness `bench:paint`), but conflicting config would silently apply
+    // last-write-wins to every entry — reject it instead.
+    const existing = definitions[metricName];
+    if (existing && !deepEqual(existing, definition)) {
+      throw new Error(`Benchmark metric "${metricName}" is defined with conflicting configuration across ` + `benchmarks. A metric name must map to a single kind, format, and alarm.`);
+    }
+    definitions[metricName] = definition;
+  }
+  if (report.iterations === 0) {
+    report.iterations = maxCount;
+  }
+}
+function printMetricsTable(name, metrics, iterationCount, definitions) {
   const entries = Object.entries(metrics);
   if (entries.length === 0) {
     return;
   }
   const rows = entries.map(([metricName, stats]) => {
-    const iqrStr = `${stats.mean.toFixed(2)}±${stats.stdDev.toFixed(2)}`;
+    const definition = definitions[baseMetricName(metricName)];
+    const iqrStr = `${formatMetricValue(stats.mean, definition)}±${formatMetricValue(stats.stdDev, definition)}`;
     const cv = stats.mean > 0 ? stats.stdDev / stats.mean * 100 : 0;
-    return [metricName.slice(0, LABEL_WIDTH).padStart(LABEL_WIDTH), cyan(iqrStr.padStart(STAT_WIDTH)), colorCV(cv), stats.outliers > 0 ? yellow(String(stats.outliers).padStart(4)) : dim('0'.padStart(4))];
+    const label = definition?.alarm ? `${metricName} ⚠` : metricName;
+    return [label.slice(0, LABEL_WIDTH).padStart(LABEL_WIDTH), cyan(iqrStr.padStart(STAT_WIDTH)), colorCV(cv), stats.outliers > 0 ? yellow(String(stats.outliers).padStart(4)) : dim('0'.padStart(4))];
   });
   printTable([{
     header: 'Metric',
     width: LABEL_WIDTH
   }, {
-    header: 'Mean±σ (ms)',
+    header: 'Mean±σ',
     width: STAT_WIDTH
   }, {
     header: 'Var%',
@@ -204,28 +239,47 @@ async function loadBaselineReport(baselinePath) {
 }
 class BenchmarkReporter {
   benchmarks = {};
+  metricDefinitions = {};
   hasFailures = false;
   constructor(options) {
     this.outputPath = options?.outputPath ?? process.env.BENCHMARK_OUTPUT_PATH ?? path.resolve(process.cwd(), 'benchmarks', 'results.json');
     this.upload = options?.upload ?? process.env.BENCHMARK_UPLOAD === 'true';
     this.baselinePath = options?.baselinePath ?? process.env.BENCHMARK_BASELINE_PATH;
   }
+
+  // Reset accumulated state at the start of every run so watch-mode re-runs start clean (the
+  // reporter instance is reused across runs). Otherwise stale benchmarks/definitions linger — and
+  // an edited metric config would conflict with its own previous-run definition.
+  onTestRunStart() {
+    this.benchmarks = {};
+    this.metricDefinitions = {};
+    this.hasFailures = false;
+  }
   onTestCaseResult(testCase) {
     if (testCase.result().state === 'failed') {
       this.hasFailures = true;
     }
     const meta = testCase.meta();
     const iterations = meta.benchmarkIterations;
-    if (!iterations) {
+    const customMetrics = meta.benchmarkMetrics;
+    if (!iterations && !customMetrics) {
       console.warn(yellow(`  No iterations recorded for: ${testCase.fullName}`));
       return;
     }
     const name = meta.benchmarkName ?? testCase.fullName;
-    const report = generateReportFromIterations(iterations);
+    const report = iterations ? generateReportFromIterations(iterations) : {
+      iterations: 0,
+      totalDuration: 0,
+      renders: [],
+      metrics: {}
+    };
+    if (customMetrics) {
+      mergeCustomMetrics(report, customMetrics, this.metricDefinitions);
+    }
     this.benchmarks[name] = report;
     const summary = dim('Total: ') + green(`${report.totalDuration.toFixed(2)}ms`) + dim(` (${report.renders.length} renders, ${report.iterations} iterations)`);
     printDurationMatrix(`${name} — React`, report, summary);
-    printMetricsTable(name, report.metrics, report.iterations);
+    printMetricsTable(name, report.metrics, report.iterations, this.metricDefinitions);
   }
   async onTestRunEnd() {
     const count = Object.keys(this.benchmarks).length;
@@ -237,11 +291,15 @@ class BenchmarkReporter {
       console.log(`  ${name}: ${result.totalDuration.toFixed(2)}ms ${dim(`(${result.renders.length} renders, ${result.iterations} iterations)`)}`);
     }
     const baseline = this.baselinePath ? await loadBaselineReport(this.baselinePath) : undefined;
+    const hasMetricDefinitions = Object.keys(this.metricDefinitions).length > 0;
     const results = {
       version: 1,
       reportType: 'benchmark',
       ...(await getCiMetadata()),
       report: this.benchmarks,
+      ...(hasMetricDefinitions ? {
+        metricDefinitions: this.metricDefinitions
+      } : {}),
       ...(baseline ? {
         base: baseline
       } : {})

package/stats.d.mts

@@ -7,4 +7,14 @@ export declare function quantile(sorted: number[], q: number): number;
  *
  * See https://en.wikipedia.org/wiki/Interquartile_range#Outliers
  */
-export declare function isOutlier(value: number, q1: number, q3: number): boolean;
+export declare function isOutlier(value: number, q1: number, q3: number): boolean;
+/**
+ * Aggregates a series of samples into a mean, standard deviation, and outlier count using
+ * IQR-based outlier removal. Falls back to the raw values when filtering would remove
+ * everything. This is the shared aggregation core for custom metrics.
+ */
+export declare function aggregateSamples(values: number[]): {
+  mean: number;
+  stdDev: number;
+  outliers: number;
+};

package/stats.mjs

@@ -24,4 +24,24 @@ export function quantile(sorted, q) {
 export function isOutlier(value, q1, q3) {
   const iqr = q3 - q1;
   return value < q1 - 1.5 * iqr || value > q3 + 1.5 * iqr;
+}
+
+/**
+ * Aggregates a series of samples into a mean, standard deviation, and outlier count using
+ * IQR-based outlier removal. Falls back to the raw values when filtering would remove
+ * everything. This is the shared aggregation core for custom metrics.
+ */
+export function aggregateSamples(values) {
+  const sorted = values.toSorted((first, second) => first - second);
+  const q1 = quantile(sorted, 0.25);
+  const q3 = quantile(sorted, 0.75);
+  const filtered = values.filter(value => !isOutlier(value, q1, q3));
+  const used = filtered.length > 0 ? filtered : values;
+  const mean = calculateMean(used);
+  const stdDev = calculateStdDev(used, mean);
+  return {
+    mean,
+    stdDev,
+    outliers: values.length - used.length
+  };
 }

package/taskMetaAugmentation.d.mts

@@ -1,7 +1,9 @@
-import type { IterationData } from "./types.mjs";
+import type { IterationData, MetricReport } from "./types.mjs";
 declare module 'vitest' {
   interface TaskMeta {
     benchmarkName?: string;
     benchmarkIterations?: IterationData[];
+    /** Custom metrics recorded via `ScalarMetric`/`DiscreteMetric`, keyed by metric name. */
+    benchmarkMetrics?: Record<string, MetricReport>;
   }
 }

package/types.d.mts

@@ -13,15 +13,8 @@ export interface RenderEvent {
   /** Start time in milliseconds (from performance.now()) */
   startTime: number;
 }
-export interface BenchmarkMetric {
-  /** Metric name, e.g. "paint:bench", "paint:grid-header" */
-  name: string;
-  /** Measured value in ms */
-  value: number;
-}
 export interface IterationData {
   renders: RenderEvent[];
-  metrics: BenchmarkMetric[];
 }
 export interface InteractionContext {
   /**
@@ -30,4 +23,79 @@ export interface InteractionContext {
    * @param timeout - Timeout in ms. Default: 5000. Pass 0 or Infinity to rely on the test timeout.
    */
   waitForElementTiming: (identifier: string, timeout?: number) => Promise<void>;
+  /**
+   * Pause recording of the harness's React render/paint measurements. Custom metrics keep
+   * recording. Throws if recording is already paused.
+   */
+  pauseReactRecording: () => void;
+  /**
+   * Resume recording of the harness's React render/paint measurements. Throws if recording is
+   * already active.
+   */
+  resumeReactRecording: () => void;
+}
+export interface BenchmarkCaseRuntime {
+  mount: () => void;
+  /**
+   * Runs the case's interaction with the harness context. Absent when the case has no interaction —
+   * the profiling panel uses its presence to decide whether to show the "Run interaction" button.
+   */
+  interact?: () => Promise<void>;
+  unmount: () => void;
+  isMounted: () => boolean;
+}
+/**
+ * Whether a custom metric measures a continuous value or a discrete count.
+ * - `scalar` — continuous measurements (timings, sizes); compared with a relative noise band.
+ * - `discrete` — counts/events; compared as exact integers.
+ */
+export type MetricKind = 'scalar' | 'discrete';
+/** Which direction of change counts as a regression for a metric in alarm mode. */
+export type MetricDirection = 'lowerIsBetter' | 'higherIsBetter';
+export interface MetricAlarm {
+  /** Defaults to `lowerIsBetter`. */
+  direction?: MetricDirection;
+  /**
+   * Softer band: a regression past `warn` (but within `error`) is flagged as a warning.
+   * Scalar metrics: a relative fraction (`0.1` = 10%). Discrete metrics: an absolute count delta.
+   */
+  warn?: number;
+  /**
+   * Harder band: a regression past `error` is flagged as an error (the alarm). When **both**
+   * `warn` and `error` are omitted, `error` defaults to the dashboard's global noise band; with
+   * only `warn` set there is no error band (warning-only).
+   * Scalar metrics: a relative fraction (`0.25` = 25%). Discrete metrics: an absolute count delta.
+   */
+  error?: number;
+}
+export interface MetricConfig {
+  name: string;
+  /** Display formatting applied by the reporter and dashboard via `Intl.NumberFormat`. */
+  format?: Intl.NumberFormatOptions;
+  /**
+   * Regression judgment. Its presence opts the metric into alarming; when omitted the metric
+   * is informational (the diff is shown but never flagged).
+   */
+  alarm?: MetricAlarm;
+}
+/** Aggregated stats for a single metric sub-series, as they cross the browser→runner boundary. */
+export interface MetricSampleStats {
+  mean: number;
+  stdDev: number;
+  outliers: number;
+  /** Number of recorded samples (used to derive iteration counts; stripped from the report). */
+  count: number;
+}
+/** A custom metric's aggregated data attached to `task.meta`, keyed by metric name. */
+export interface MetricReport {
+  kind: MetricKind;
+  config: MetricConfig;
+  /** Aggregated stats keyed by sub-series id (`''` is the base series). */
+  series: Record<string, MetricSampleStats>;
+}
+/** Per-metric configuration hoisted to the top level of the report (keyed by metric name). */
+export interface MetricDefinition {
+  kind: MetricKind;
+  format?: Intl.NumberFormatOptions;
+  alarm?: MetricAlarm;
 }

package/vitest.d.mts

@@ -1,4 +1,4 @@
-import { ViteUserConfig } from 'vitest/config';
+import type { ViteUserConfig } from 'vitest/config';
 export interface CreateBenchmarkVitestConfigOptions {
   /**
    * Path to save benchmark results JSON file. If not provided, results will not be saved to disk.
@@ -15,5 +15,22 @@ export interface CreateBenchmarkVitestConfigOptions {
    * Additional Chromium launch arguments.
    */
   launchArgs?: string[];
+  /**
+   * Run each `benchmark()` case as an interactive profiling session in a headed
+   * browser instead of the automated measurement loop. Each case renders a
+   * control panel with Render / Unmount / Finish buttons so you can start the
+   * DevTools profiler before the component mounts. Profiling auto-opens DevTools
+   * and runs headed (pass `viewport` to size the window). Also settable via
+   * `BENCHMARK_PROFILE=true`.
+   */
+  profile?: boolean;
+  /**
+   * Browser viewport — and, in profile mode, the matching window size. Defaults to 1920x1080 for
+   * both measurement and profiling. Also settable via `BENCHMARK_VIEWPORT` (e.g. `2560x1440`).
+   */
+  viewport?: {
+    width: number;
+    height: number;
+  };
 }
 export declare function createBenchmarkVitestConfig(options?: CreateBenchmarkVitestConfigOptions): ViteUserConfig;

package/vitest.mjs

@@ -1,15 +1,56 @@
 import react from '@vitejs/plugin-react';
 import { playwright } from '@vitest/browser-playwright';
+// Default viewport for all benchmark runs (measurement and profiling alike) — a desktop size is
+// more representative for component benchmarks than Vitest's phone-sized 414x896 browser default.
+const DEFAULT_VIEWPORT = {
+  width: 1920,
+  height: 1080
+};
+
+// Explicit viewport from the `viewport` option or the `BENCHMARK_VIEWPORT` env var
+// (`<width>x<height>`); undefined when neither is set, so the caller can apply the default.
+function resolveViewport(option) {
+  if (option) {
+    return option;
+  }
+  const env = process.env.BENCHMARK_VIEWPORT;
+  const match = env ? env.split('x') : null;
+  if (match && match.length === 2) {
+    return {
+      width: Number(match[0]),
+      height: Number(match[1])
+    };
+  }
+  return undefined;
+}
+
+// Chromium/V8 launch args shared by measurement and profiling, kept intentionally minimal.
+// `--expose-gc` is required: the harness forces GC between iterations for clean, comparable
+// timings. The backgrounding flags stop Chrome from throttling the (headless or occluded)
+// benchmark tab, which would otherwise add large variance. Heavier "determinism" flags
+// (`--no-opt`, `--predictable`, `--hash-seed`/`--random-seed`, `--disable-gpu`,
+// `--enable-benchmarking`) were measured to slow renders ~40% and distort paint timing without
+// reducing variance, so they are omitted — add them per project via `launchArgs` if a specific
+// workload needs them.
+const LAUNCH_ARGS = ['--js-flags=--expose-gc', '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-renderer-backgrounding'];
 export function createBenchmarkVitestConfig(options) {
   const {
     outputPath,
     baselinePath,
     launchArgs = []
   } = options ?? {};
+  const profile = options?.profile ?? process.env.BENCHMARK_PROFILE === 'true';
+  const viewport = resolveViewport(options?.viewport) ?? DEFAULT_VIEWPORT;
+
+  // Profiling adds DevTools on top of the shared args, plus a window sized to match the viewport —
+  // Vitest's `viewport` only sizes the iframe, so otherwise it's cropped/scrolled in the headed
+  // window instead of filling it. (Measurement is headless, so it has no window to size.)
+  const profileArgs = [...LAUNCH_ARGS, '--auto-open-devtools-for-tabs', `--window-size=${viewport.width},${viewport.height}`];
   return {
     plugins: [react()],
     define: {
-      'process.env.NODE_ENV': '"production"'
+      'process.env.NODE_ENV': '"production"',
+      'process.env.BENCHMARK_PROFILE': JSON.stringify(profile ? 'true' : '')
     },
     resolve: {
       dedupe: ['react', 'react-dom'],
@@ -21,29 +62,28 @@ export function createBenchmarkVitestConfig(options) {
     test: {
       browser: {
         enabled: true,
-        headless: true,
+        headless: !profile,
+        // Profiling renders into a clean page: hide Vitest's browser runner UI
+        // so the orchestrator chrome doesn't clutter what you're profiling.
+        ui: profile ? false : undefined,
+        // Same viewport for both modes (DEFAULT_VIEWPORT unless overridden).
+        viewport,
         screenshotFailures: false,
         instances: [{
           browser: 'chromium',
-          testTimeout: 120_000
+          // Profiling sessions are driven by hand, so give them effectively
+          // unlimited time instead of the measurement timeout.
+          testTimeout: profile ? 0 : 120_000
         }],
         provider: playwright({
           launchOptions: {
-            args: [
-            // V8 flags for deterministic JS execution
-            '--js-flags=--expose-gc,--predictable,--no-opt,--predictable-gc-schedule,--no-concurrent-sweeping,--hash-seed=1,--random-seed=1,--max-old-space-size=4096',
-            // Chromium flags to reduce renderer/compositor noise
-            '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-renderer-backgrounding', '--disable-background-networking',
-            // Reduces environmental noise by disabling field trials,
-            // for more consistent profiling results.
-            '--enable-benchmarking',
-            // Forces software rendering instead of GPU, which is more deterministic.
-            '--disable-gpu', ...launchArgs]
+            args: [...(profile ? profileArgs : LAUNCH_ARGS), ...launchArgs]
           }
         })
       },
       fileParallelism: false,
-      reporters: ['default', ['@mui/internal-benchmark/reporter', {
+      // Profiling sessions don't measure anything, so skip the results reporter.
+      reporters: profile ? ['default'] : ['default', ['@mui/internal-benchmark/reporter', {
         outputPath,
         baselinePath
       }]],