@mui/internal-benchmark 0.0.3-canary.16 → 0.0.3-canary.17

package/Metric.d.mts

@@ -1,5 +1,24 @@
+import type { RunnerTestCase } from 'vitest';
 import type { MetricConfig, MetricKind } from "./types.mjs";
 import "./taskMetaAugmentation.mjs";
+/** One alarmed scalar (sub-)series the adaptive stopping rule tracks, named for reporting. */
+export interface AdaptiveMetricSamples {
+  /** Metric name, with a `#sub-series` suffix for named sub-series. */
+  name: string;
+  samples: number[];
+}
+/**
+ * Snapshot of the raw samples collected so far for a test's alarmed scalar metrics — one entry per
+ * (metric, sub-series), each named. The adaptive stopping rule reads this so measurement keeps
+ * sampling while any metric that will be significance-tested downstream is still imprecise, not just
+ * the render duration, and so a non-convergence warning can name the specific metric that stayed
+ * noisy.
+ *
+ * Discrete metrics (compared as exact counts) and informational scalar metrics (shown but never
+ * flagged) are excluded: their precision never changes a verdict, and gating on a noisy one would
+ * needlessly block convergence.
+ */
+export declare function collectAdaptiveMetricSamples(test: RunnerTestCase): AdaptiveMetricSamples[];
 export interface MetricRecordOptions {
   /** Sub-series label. Recorded under `${name}#${id}` in the report; omit for the base series. */
   id?: string;

package/Metric.mjs

@@ -12,10 +12,9 @@ function flush(test, accumulator) {
   for (const [name, entry] of accumulator) {
     const series = {};
     for (const [seriesId, samples] of entry.series) {
-      series[seriesId] = {
-        ...aggregateSamples(samples),
-        count: samples.length
-      };
+      // `aggregateSamples` reports the effective (post-outlier-removal) count as `count`, which is
+      // the `n` behind mean/stdDev — exactly what a downstream Welch's t-test needs.
+      series[seriesId] = aggregateSamples(samples);
     }
     store[name] = {
       kind: entry.kind,
@@ -25,6 +24,39 @@ function flush(test, accumulator) {
   }
   test.meta.benchmarkMetrics = store;
 }
+
+/** One alarmed scalar (sub-)series the adaptive stopping rule tracks, named for reporting. */
+
+/**
+ * Snapshot of the raw samples collected so far for a test's alarmed scalar metrics — one entry per
+ * (metric, sub-series), each named. The adaptive stopping rule reads this so measurement keeps
+ * sampling while any metric that will be significance-tested downstream is still imprecise, not just
+ * the render duration, and so a non-convergence warning can name the specific metric that stayed
+ * noisy.
+ *
+ * Discrete metrics (compared as exact counts) and informational scalar metrics (shown but never
+ * flagged) are excluded: their precision never changes a verdict, and gating on a noisy one would
+ * needlessly block convergence.
+ */
+export function collectAdaptiveMetricSamples(test) {
+  const accumulator = accumulators.get(test);
+  if (!accumulator) {
+    return [];
+  }
+  const sampleSets = [];
+  for (const [metricName, entry] of accumulator) {
+    if (entry.kind !== 'scalar' || entry.config.alarm === undefined) {
+      continue;
+    }
+    for (const [seriesId, samples] of entry.series) {
+      sampleSets.push({
+        name: seriesId === '' ? metricName : `${metricName}#${seriesId}`,
+        samples
+      });
+    }
+  }
+  return sampleSets;
+}
 /**
  * Base class for custom benchmark metrics. Use `ScalarMetric` or `DiscreteMetric`.
  *

package/ciReport.d.mts

@@ -10,15 +10,19 @@ declare const renderStatsSchema: z.ZodObject<{
   actualDuration: z.ZodNumber;
   stdDev: z.ZodNumber;
   outliers: z.ZodNumber;
+  count: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 declare const metricStatsSchema: z.ZodObject<{
   mean: z.ZodNumber;
   stdDev: z.ZodNumber;
   outliers: z.ZodNumber;
+  count: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 declare const benchmarkReportEntrySchema: z.ZodObject<{
   iterations: z.ZodNumber;
   totalDuration: z.ZodNumber;
+  totalStdDev: z.ZodOptional<z.ZodNumber>;
+  totalCount: z.ZodOptional<z.ZodNumber>;
   renders: z.ZodArray<z.ZodObject<{
     id: z.ZodString;
     phase: z.ZodEnum<{
@@ -30,16 +34,20 @@ declare const benchmarkReportEntrySchema: z.ZodObject<{
     actualDuration: z.ZodNumber;
     stdDev: z.ZodNumber;
     outliers: z.ZodNumber;
+    count: z.ZodOptional<z.ZodNumber>;
   }, z.core.$strip>>;
   metrics: z.ZodRecord<z.ZodString, z.ZodObject<{
     mean: z.ZodNumber;
     stdDev: z.ZodNumber;
     outliers: z.ZodNumber;
+    count: z.ZodOptional<z.ZodNumber>;
   }, z.core.$strip>>;
 }, z.core.$strip>;
 declare const benchmarkReportSchema: z.ZodRecord<z.ZodString, z.ZodObject<{
   iterations: z.ZodNumber;
   totalDuration: z.ZodNumber;
+  totalStdDev: z.ZodOptional<z.ZodNumber>;
+  totalCount: z.ZodOptional<z.ZodNumber>;
   renders: z.ZodArray<z.ZodObject<{
     id: z.ZodString;
     phase: z.ZodEnum<{
@@ -51,11 +59,13 @@ declare const benchmarkReportSchema: z.ZodRecord<z.ZodString, z.ZodObject<{
     actualDuration: z.ZodNumber;
     stdDev: z.ZodNumber;
     outliers: z.ZodNumber;
+    count: z.ZodOptional<z.ZodNumber>;
   }, z.core.$strip>>;
   metrics: z.ZodRecord<z.ZodString, z.ZodObject<{
     mean: z.ZodNumber;
     stdDev: z.ZodNumber;
     outliers: z.ZodNumber;
+    count: z.ZodOptional<z.ZodNumber>;
   }, z.core.$strip>>;
 }, z.core.$strip>>;
 declare const metricDefinitionSchema: z.ZodObject<{

package/ciReport.mjs

@@ -31,16 +31,26 @@ const renderStatsSchema = z.object({
   startTime: z.number(),
   actualDuration: z.number(),
   stdDev: z.number(),
-  outliers: z.number()
+  outliers: z.number(),
+  // Effective sample count behind mean/stdDev, used as `n` for the comparison's Welch's t-test.
+  // Optional so baselines uploaded before it was added still parse.
+  count: z.number().optional()
 });
 const metricStatsSchema = z.object({
   mean: z.number(),
   stdDev: z.number(),
-  outliers: z.number()
+  outliers: z.number(),
+  // See `renderStatsSchema.count`. Optional for backward compatibility with older uploads.
+  count: z.number().optional()
 });
 const benchmarkReportEntrySchema = z.object({
   iterations: z.number(),
   totalDuration: z.number(),
+  // Spread and effective sample count of the per-iteration total duration, used as the stdDev/`n`
+  // for the comparison's Welch test on total duration. Optional for backward compatibility with
+  // uploads made before they were added.
+  totalStdDev: z.number().optional(),
+  totalCount: z.number().optional(),
   renders: z.array(renderStatsSchema),
   metrics: z.record(z.string(), metricStatsSchema)
 });

package/format.d.mts

@@ -4,9 +4,15 @@ export declare const green: (s: string) => string;
 export declare const yellow: (s: string) => string;
 export declare const cyan: (s: string) => string;
 export declare function fileUrl(filePath: string): string;
-interface Column {
+export interface Column {
   header: string;
-  width: number;
+  /** Width the column keeps when every cell is narrower. It grows to fit a wider cell. */
+  minWidth: number;
+  /**
+   * Width the column may not exceed; wider cells are truncated with an ellipsis. Omit to let the
+   * column grow freely, which is what a value the caller can't pre-measure (a metric formatted with
+   * an arbitrary unit) needs — truncating those would destroy the number the table exists to show.
+   */
+  maxWidth?: number;
 }
-export declare function printTable(columns: Column[], rows: string[][], footer?: string, title?: string): void;
-export {};
+export declare function printTable(columns: Column[], rows: string[][], footer?: string, title?: string): void;

package/format.mjs

@@ -8,19 +8,72 @@ export const cyan = s => styleText('cyan', s);
 export function fileUrl(filePath) {
   return pathToFileURL(filePath).href;
 }
-function truncate(str, maxLength) {
-  if (str.length <= maxLength) {
+
+/** Matches an ANSI colour code. Captured, so `split` keeps the codes alongside the text. */
+// eslint-disable-next-line no-control-regex
+const ANSI_PATTERN = /(\x1b\[[0-9;]*m)/g;
+function stripAnsi(str) {
+  return str.replace(ANSI_PATTERN, '');
+}
+/** Printable width, i.e. ignoring the zero-width ANSI escape codes that colour a string. */
+function visibleWidth(str) {
+  return stripAnsi(str).length;
+}
+
+/**
+ * Truncates to a visible width, marking the cut with an ellipsis. Escape codes are carried over
+ * rather than counted, so a styled string keeps the closing code that would otherwise be dropped
+ * along with the text it terminated (leaving the rest of the terminal painted).
+ */
+function truncate(str, maxWidth) {
+  if (visibleWidth(str) <= maxWidth) {
     return str;
   }
-  return `${str.slice(0, maxLength - 1)}…`;
+  let result = '';
+  let remaining = maxWidth - 1;
+  for (const segment of str.split(ANSI_PATTERN)) {
+    if (segment.startsWith('\x1b')) {
+      result += segment;
+    } else if (remaining > 0) {
+      result += segment.slice(0, remaining);
+      remaining -= Math.min(segment.length, remaining);
+    }
+  }
+  return `${result}…`;
+}
+
+/** Right-aligns to a visible width, so colour codes in the cell don't eat into the padding. */
+function padCell(cell, width) {
+  return ' '.repeat(Math.max(0, width - visibleWidth(cell))) + cell;
+}
+
+/** Fits a string to an exact visible width, truncating or padding as needed. */
+function fitCell(cell, width) {
+  return padCell(truncate(cell, width), width);
 }
 export function printTable(columns, rows, footer, title) {
-  const colWidths = columns.map(col => col.width);
-  const totalInner = colWidths.reduce((sum, w) => sum + w + 2, 0) + colWidths.length - 1;
+  // Cells are sized off content, because a caller can't know how wide a value will render (a metric
+  // carrying a unit — `0.456 ms±0.057 ms` — is far wider than a bare `0.46±0.06`). Padding a cell
+  // that already overflows its column is a no-op, which used to let one long value push that row's
+  // dividers out of line with every other row.
+  const colWidths = columns.map((col, index) => {
+    const content = Math.max(col.minWidth, visibleWidth(col.header), ...rows.map(row => visibleWidth(row[index] ?? '')));
+    return col.maxWidth === undefined ? content : Math.min(content, col.maxWidth);
+  });
+  const innerWidth = () => colWidths.reduce((sum, w) => sum + w + 2, 0) + colWidths.length - 1;
+  // The footer spans the whole table, so it too can be the widest thing in it. It reports counts
+  // that are worth nothing abbreviated, so the table stretches to fit it — the slack goes on the
+  // last column, keeping the columns left of it where the reader expects. A title, by contrast, is
+  // a name: arbitrarily long, and legible truncated, so it fits itself to the table below.
+  const footerWidth = footer ? visibleWidth(footer) + 2 : 0;
+  if (colWidths.length > 0 && footerWidth > innerWidth()) {
+    colWidths[colWidths.length - 1] += footerWidth - innerWidth();
+  }
+  const totalInner = innerWidth();
   if (title) {
     const titleTop = dim(`┌${'─'.repeat(totalInner)}┐`);
     const titleContent = ` ${truncate(title, totalInner - 2)}`;
-    const titlePadding = totalInner - titleContent.length;
+    const titlePadding = totalInner - visibleWidth(titleContent);
     const titleLine = dim('│') + titleContent + ' '.repeat(Math.max(0, titlePadding)) + dim('│');
     const titleSep = dim(`├${colWidths.map(w => '─'.repeat(w + 2)).join('┬')}┤`);
 
@@ -36,7 +89,7 @@ export function printTable(columns, rows, footer, title) {
     console.log(topBorder);
   }
   const headerSep = dim(`├${colWidths.map(w => '─'.repeat(w + 2)).join('┼')}┤`);
-  const headerCells = columns.map(col => ` ${col.header.padStart(col.width)} `);
+  const headerCells = columns.map((col, index) => ` ${fitCell(col.header, colWidths[index])} `);
   const headerLine = dim('│') + headerCells.join(dim('│')) + dim('│');
 
   // eslint-disable-next-line no-console
@@ -44,14 +97,15 @@ export function printTable(columns, rows, footer, title) {
   // eslint-disable-next-line no-console
   console.log(headerSep);
   for (const row of rows) {
-    const cells = row.map((cell, i) => ` ${cell.padStart(colWidths[i])} `);
+    // Driven by the columns, not the row, so a short row still emits every cell and divider.
+    const cells = colWidths.map((width, index) => ` ${fitCell(row[index] ?? '', width)} `);
     // eslint-disable-next-line no-console
     console.log(dim('│') + cells.join(dim('│')) + dim('│'));
   }
   if (footer) {
     const footerSep = dim(`├${colWidths.map(w => '─'.repeat(w + 2)).join('┴')}┤`);
     const footerContent = ` ${footer}`;
-    const padding = totalInner - stripAnsi(footerContent).length;
+    const padding = totalInner - visibleWidth(footerContent);
     const footerLine = dim('│') + footerContent + ' '.repeat(Math.max(0, padding)) + dim('│');
     const bottomBorder = dim(`└${'─'.repeat(totalInner)}┘`);
 
@@ -66,10 +120,4 @@ export function printTable(columns, rows, footer, title) {
     // eslint-disable-next-line no-console
     console.log(bottomBorder);
   }
-}
-
-// Strip ANSI escape codes to measure visible string length
-function stripAnsi(str) {
-  // eslint-disable-next-line no-control-regex
-  return str.replace(/\x1b\[[0-9;]*m/g, '');
 }

package/index.d.mts

@@ -14,8 +14,26 @@ declare global {
   }
 }
 interface BenchmarkOptions {
+  /**
+   * Fixed number of measured iterations. When set, disables adaptive sampling (equivalent to
+   * `minRuns === maxRuns === runs`). Prefer leaving this unset and letting the harness sample
+   * adaptively; use it only to pin a benchmark to an exact iteration count.
+   */
   runs?: number;
+  /** Warmup iterations run before measurement begins (not recorded). Defaults to `5`. */
   warmupRuns?: number;
+  /**
+   * Adaptive sampling: the harness keeps measuring until the mean render duration is estimated to
+   * within `targetRme`, then stops. `minRuns` is the floor before the stopping rule can trigger,
+   * `maxRuns` the ceiling. Ignored when `runs` is set. Defaults: `minRuns` 10, `maxRuns` 50.
+   */
+  minRuns?: number;
+  maxRuns?: number;
+  /**
+   * Target relative margin of error (half-width of the 95% confidence interval of the mean, as a
+   * fraction of the mean) at which adaptive sampling stops. Defaults to `0.02` (2%).
+   */
+  targetRme?: number;
   afterEach?: () => Promise<void> | void;
   /**
    * Start each iteration with React render/paint recording paused. The interaction callback then

package/index.mjs

@@ -12,6 +12,8 @@ import * as ReactDOMClient from 'react-dom/client'; // aliased to react-dom/prof
 import * as ReactDOM from 'react-dom';
 import { ElementTiming } from "./ElementTiming.mjs";
 import { ScalarMetric } from "./ScalarMetric.mjs";
+import { describeUnconvergedSignals, relativeMarginOfError } from "./stats.mjs";
+import { collectAdaptiveMetricSamples } from "./Metric.mjs";
 import { metricsGate } from "./metricsGate.mjs";
 import { createReactRecordingControls } from "./reactRecording.mjs";
 import { runProfileSession } from "./profileSession.mjs";
@@ -199,10 +201,30 @@ export function benchmark(name, renderFn, interactionOrOptions, maybeOptions) {
   it(name, async ({
     task
   }) => {
-    const runs = options?.runs ?? 20;
-    const warmupRuns = options?.warmupRuns ?? 10;
-    const totalRuns = warmupRuns + runs;
+    const warmupRuns = options?.warmupRuns ?? 5;
+    // A fixed `runs` pins both bounds; otherwise sample adaptively between min and max.
+    const isAdaptive = options?.runs === undefined;
+    const minRuns = options?.runs ?? options?.minRuns ?? 10;
+    const maxRuns = options?.runs ?? options?.maxRuns ?? 50;
+    const targetRme = options?.targetRme ?? 0.02;
+
+    // Fail fast on misconfigured sampling bounds rather than silently running a surprising loop.
+    if (minRuns < 1 || maxRuns < minRuns || targetRme <= 0) {
+      throw new Error(`Invalid benchmark sampling options for "${name}": require 1 <= minRuns (${minRuns}) <= ` + `maxRuns (${maxRuns}) and targetRme (${targetRme}) > 0.`);
+    }
+
+    // Upper bound on the loop; the adaptive stopping rule usually breaks out earlier.
+    const totalRuns = warmupRuns + maxRuns;
     const iterations = [];
+    // Per measured iteration: total render duration, the primary signal the stopping rule converges
+    // on (alongside each alarmed scalar metric).
+    const iterationDurations = [];
+    // Relative margin of error of the render duration at the last stopping check — reused for the
+    // non-convergence warning instead of recomputing the same trim/variance pass.
+    let durationRme = Infinity;
+    // Per-metric relative margin of error from the last stopping check (only populated once render
+    // duration has converged), named so the non-convergence warning can point at the noisy metric.
+    let metricRmes = [];
 
     // Paint timings are recorded as one harness-owned `bench:paint` metric: the default sentinel
     // is the base series (`bench:paint`) and named `elementtiming` markers are sub-series
@@ -297,11 +319,57 @@ export function benchmark(name, renderFn, interactionOrOptions, maybeOptions) {
         iterations.push({
           renders: captures
         });
+        // Total render duration of this iteration — the adaptive stopping rule's convergence signal.
+        iterationDurations.push(captures.reduce((sum, capture) => sum + capture.actualDuration, 0));
       }
       if (options?.afterEach) {
         // eslint-disable-next-line no-await-in-loop
         await options.afterEach();
       }
+
+      // Adaptive stopping: once past the floor, stop as soon as every measured signal — the mean
+      // render duration and each alarmed scalar metric — is estimated tightly enough. Custom metrics
+      // can only *delay* the stop (the `minRuns` floor and duration target still apply), so a noisy
+      // metric keeps sampling rather than being left underpowered for the dashboard's significance
+      // test. `maxRuns` caps the work for benchmarks too noisy to reach `targetRme`.
+      if (!isWarmup && iterationDurations.length >= minRuns) {
+        durationRme = relativeMarginOfError(iterationDurations);
+        // Duration is the primary (and usually last-to-tighten) signal, so check it first: the
+        // per-metric margin-of-error pass only runs once duration itself has converged. A
+        // zero-centered metric (mean <= 0) yields a margin of error of 0 — relative precision is
+        // undefined for it — so it never blocks; such a metric can't fire a relative alarm anyway.
+        if (durationRme <= targetRme) {
+          metricRmes = collectAdaptiveMetricSamples(task).map(({
+            name: metricName,
+            samples
+          }) => ({
+            name: metricName,
+            rme: relativeMarginOfError(samples)
+          }));
+          if (metricRmes.every(({
+            rme
+          }) => rme <= targetRme)) {
+            break;
+          }
+        }
+      }
+    }
+
+    // Warn only when adaptive sampling exhausted `maxRuns` without every signal reaching `targetRme`,
+    // naming each signal that stayed noisy so the author knows what to fix — the common (converged)
+    // case stays quiet so large suites don't flood CI logs. Skipped in fixed mode (`runs` set), where
+    // there is no convergence target to miss. Reuses the margins of error already computed above.
+    if (isAdaptive && iterationDurations.length >= maxRuns) {
+      const unconverged = describeUnconvergedSignals([{
+        label: 'duration',
+        rme: durationRme
+      }, ...metricRmes.map(metric => ({
+        label: `metric '${metric.name}'`,
+        rme: metric.rme
+      }))], targetRme);
+      if (unconverged.length > 0) {
+        console.warn(`Benchmark "${name}" reached maxRuns (${maxRuns} runs) without converging to the ` + `${(targetRme * 100).toFixed(2)}% target: ${unconverged.join(', ')}. Results may be ` + `noisier than intended — consider raising maxRuns or reducing variance.`);
+      }
     }
     task.meta.benchmarkIterations = iterations;
     task.meta.benchmarkName = name;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mui/internal-benchmark",
-  "version": "0.0.3-canary.16",
+  "version": "0.0.3-canary.17",
   "author": "MUI Team",
   "description": "Benchmark utilities for MUI projects. Internal package.",
   "repository": {
@@ -68,5 +68,5 @@
       }
     }
   },
-  "gitSha": "c11419b9b49a34230751911dc3bbd5fc2764b05b"
+  "gitSha": "69ae6a8458bc68be5bef922f72637180b2b160e5"
 }

package/reporter.mjs

@@ -7,9 +7,6 @@ import { uploadCiReport } from "./upload.mjs";
 import { syncPrComment } from "./syncPrComment.mjs";
 // Import for TaskMeta augmentation side effect
 import "./taskMetaAugmentation.mjs";
-function getEventKey(event) {
-  return `${event.id}:${event.phase}`;
-}
 
 /** Order-insensitive deep equality, treating a missing key and an `undefined` value as equal. */
 function deepEqual(first, second) {
@@ -56,21 +53,24 @@ function generateReportFromIterations(iterations) {
   const renderStats = [];
   for (let index = 0; index < expectedLength; index += 1) {
     const durations = iterations.map(iteration => iteration.renders[index].actualDuration);
+
+    // The per-render coefficient of variation is surfaced as the "Var%" column of the results
+    // table (informational). It no longer warrants a warning: the Welch comparison already accounts
+    // for a render's spread (a noisy render yields a high p-value rather than a false flag), and
+    // adaptive sampling drives the *mean's* precision to `targetRme`, warning separately when it
+    // can't converge. High spread alone no longer implies an unreliable result.
     const {
       mean: iqrMean,
       stdDev: iqrStdDev,
-      outliers
+      outliers,
+      count
     } = aggregateSamples(durations);
-    const coefficientOfVariation = iqrMean > 0 ? iqrStdDev / iqrMean : 0;
-    if (iqrMean > 1 && coefficientOfVariation > 0.1) {
-      const event = firstIteration.renders[index];
-      console.warn(`High coefficient of variation (${(coefficientOfVariation * 100).toFixed(1)}%) for render #${index} event "${getEventKey(event)}". ` + `Mean: ${iqrMean.toFixed(2)}ms, StdDev: ${iqrStdDev.toFixed(2)}ms. Results may be unreliable.`);
-    }
     renderStats.push({
       event: firstIteration.renders[index],
       iqrMean,
       iqrStdDev,
-      outliers
+      outliers,
+      count
     });
   }
 
@@ -84,13 +84,13 @@ function generateReportFromIterations(iterations) {
     meanGaps.push(calculateMean(gaps));
   }
   const renders = [];
-  let totalDuration = 0;
   for (let index = 0; index < expectedLength; index += 1) {
     const {
       event,
       iqrMean,
       iqrStdDev,
-      outliers
+      outliers,
+      count
     } = renderStats[index];
     const startTime = index === 0 ? 0 : renders[index - 1].startTime + renders[index - 1].actualDuration + meanGaps[index];
     renders.push({
@@ -99,16 +99,27 @@ function generateReportFromIterations(iterations) {
       startTime,
       actualDuration: iqrMean,
       stdDev: iqrStdDev,
-      outliers
+      outliers,
+      count
     });
-    totalDuration += iqrMean;
   }
 
+  // Aggregate the *actual* per-iteration total duration (sum of that iteration's render durations).
+  // `totalDuration`, `totalStdDev`, and `totalCount` are all taken from this one distribution so the
+  // comparison's Welch test gets a mutually consistent (mean, stdDev, n) triple — and the spread
+  // reflects real cross-render correlation, which a sum of per-render variances cannot. (Because
+  // outliers are filtered here on the totals, not per render, `totalDuration` may differ marginally
+  // from summing the per-render means shown in the render table.)
+  const perIterationTotals = iterations.map(iteration => iteration.renders.reduce((sum, render) => sum + render.actualDuration, 0));
+  const totalStats = aggregateSamples(perIterationTotals);
+
   // Custom + paint metrics are merged separately from `task.meta.benchmarkMetrics`.
   const metrics = {};
   return {
     iterations: iterationCount,
-    totalDuration,
+    totalDuration: totalStats.mean,
+    totalStdDev: totalStats.stdDev,
+    totalCount: totalStats.count,
     renders,
     metrics
   };
@@ -116,8 +127,9 @@ function generateReportFromIterations(iterations) {
 const LABEL_WIDTH = 28;
 const STAT_WIDTH = 16;
 const CV_WIDTH = 8;
+const OUT_WIDTH = 4;
 function colorCV(cv) {
-  const str = `${cv.toFixed(1)}%`.padStart(CV_WIDTH);
+  const str = `${cv.toFixed(1)}%`;
   if (cv > 10) {
     return red(str);
   }
@@ -126,6 +138,33 @@ function colorCV(cv) {
   }
   return dim(str);
 }
+
+/**
+ * Columns shared by the duration and metric tables: a label pinned to a fixed width, then stats
+ * free to grow (a metric can be formatted with any unit, so its width isn't knowable here).
+ */
+function statColumns(labelHeader, statHeader) {
+  return [{
+    header: labelHeader,
+    minWidth: LABEL_WIDTH,
+    maxWidth: LABEL_WIDTH
+  }, {
+    header: statHeader,
+    minWidth: STAT_WIDTH
+  }, {
+    header: 'Var%',
+    minWidth: CV_WIDTH
+  }, {
+    header: 'Out',
+    minWidth: OUT_WIDTH
+  }];
+}
+
+/** A `label | mean±σ | variation | outliers` row, as used by both tables. */
+function statRow(label, iqrStr, mean, stdDev, outliers) {
+  const cv = mean > 0 ? stdDev / mean * 100 : 0;
+  return [label, cyan(iqrStr), colorCV(cv), outliers > 0 ? yellow(String(outliers)) : dim('0')];
+}
 function printDurationMatrix(name, report, footer) {
   if (report.renders.length === 0) {
     return;
@@ -133,24 +172,10 @@ function printDurationMatrix(name, report, footer) {
   const rows = [];
   for (let r = 0; r < report.renders.length; r += 1) {
     const render = report.renders[r];
-    const label = `#${r} ${render.id}:${render.phase}`;
     const iqrStr = `${render.actualDuration.toFixed(2)}±${render.stdDev.toFixed(2)}`;
-    const cv = render.actualDuration > 0 ? render.stdDev / render.actualDuration * 100 : 0;
-    rows.push([label.slice(0, LABEL_WIDTH).padStart(LABEL_WIDTH), cyan(iqrStr.padStart(STAT_WIDTH)), colorCV(cv), render.outliers > 0 ? yellow(String(render.outliers).padStart(4)) : dim('0'.padStart(4))]);
+    rows.push(statRow(`#${r} ${render.id}:${render.phase}`, iqrStr, render.actualDuration, render.stdDev, render.outliers));
   }
-  printTable([{
-    header: 'Render',
-    width: LABEL_WIDTH
-  }, {
-    header: 'Mean±σ (ms)',
-    width: STAT_WIDTH
-  }, {
-    header: 'Var%',
-    width: CV_WIDTH
-  }, {
-    header: 'Out',
-    width: 4
-  }], rows, footer, name);
+  printTable(statColumns('Render', 'Mean±σ (ms)'), rows, footer, name);
 }
 
 /** Strips a `#sub-series` suffix to recover the metric name used to look up its definition. */
@@ -178,9 +203,13 @@ function mergeCustomMetrics(report, customMetrics, definitions) {
       report.metrics[key] = {
         mean: stats.mean,
         stdDev: stats.stdDev,
-        outliers: stats.outliers
+        outliers: stats.outliers,
+        count: stats.count
       };
-      maxCount = Math.max(maxCount, stats.count);
+      // `stats.count` is the post-outlier-removal count; add back the dropped outliers to recover
+      // the raw number of recorded samples, which is what a metric-only benchmark reports as its
+      // iteration count below.
+      maxCount = Math.max(maxCount, stats.count + stats.outliers);
     }
     const definition = {
       kind: metric.kind,
@@ -208,23 +237,10 @@ function printMetricsTable(name, metrics, iterationCount, definitions) {
   const rows = entries.map(([metricName, stats]) => {
     const definition = definitions[baseMetricName(metricName)];
     const iqrStr = `${formatMetricValue(stats.mean, definition)}±${formatMetricValue(stats.stdDev, definition)}`;
-    const cv = stats.mean > 0 ? stats.stdDev / stats.mean * 100 : 0;
     const label = definition?.alarm ? `${metricName} ⚠` : metricName;
-    return [label.slice(0, LABEL_WIDTH).padStart(LABEL_WIDTH), cyan(iqrStr.padStart(STAT_WIDTH)), colorCV(cv), stats.outliers > 0 ? yellow(String(stats.outliers).padStart(4)) : dim('0'.padStart(4))];
+    return statRow(label, iqrStr, stats.mean, stats.stdDev, stats.outliers);
   });
-  printTable([{
-    header: 'Metric',
-    width: LABEL_WIDTH
-  }, {
-    header: 'Mean±σ',
-    width: STAT_WIDTH
-  }, {
-    header: 'Var%',
-    width: CV_WIDTH
-  }, {
-    header: 'Out',
-    width: 4
-  }], rows, dim(`${iterationCount} iterations`), `${name} — Metrics`);
+  printTable(statColumns('Metric', 'Mean±σ'), rows, dim(`${iterationCount} iterations`), `${name} — Metrics`);
 }
 async function loadBaselineReport(baselinePath) {
   const raw = await fs.readFile(baselinePath, 'utf8');

package/stats.d.mts

@@ -9,12 +9,54 @@ export declare function quantile(sorted: number[], q: number): number;
  */
 export declare function isOutlier(value: number, q1: number, q3: number): boolean;
 /**
- * Aggregates a series of samples into a mean, standard deviation, and outlier count using
- * IQR-based outlier removal. Falls back to the raw values when filtering would remove
- * everything. This is the shared aggregation core for custom metrics.
+ * Drops IQR outliers (values outside the 1.5×IQR fences), returning the surviving samples. Falls
+ * back to the original values when filtering would remove everything. Shared by every consumer that
+ * wants to reason about typical performance rather than measurement artifacts (GC pauses, scheduling
+ * hiccups), so they all trim identically.
+ */
+export declare function removeOutliers(values: number[]): number[];
+/**
+ * Aggregates a series of samples into a mean, standard deviation, outlier count, and effective
+ * sample count using IQR-based outlier removal. This is the shared aggregation core for custom
+ * metrics.
+ *
+ * `count` is the number of samples that actually back `mean`/`stdDev` (post-outlier-removal), so a
+ * downstream Welch's t-test can use it directly as the `n` behind those stats.
  */
 export declare function aggregateSamples(values: number[]): {
   mean: number;
   stdDev: number;
   outliers: number;
-};
+  count: number;
+};
+/**
+ * Relative margin of error of the mean — half the 95% confidence interval width divided by the
+ * mean, using the normal approximation (`z = 1.96`). This is the adaptive-sampling stopping signal:
+ * measurement continues until this drops to a target. A precise t-quantile is unnecessary just to
+ * decide when enough samples have been collected.
+ *
+ * IQR outliers are removed first, so the margin of error is measured on the same trimmed
+ * distribution the reported stats and the Welch comparison use. Without this, a benchmark that is
+ * stable apart from recurring GC/scheduling spikes would keep a high raw margin of error and sample
+ * all the way to its maximum (and warn that it "did not converge") even though its typical estimate
+ * settled long ago.
+ *
+ * Returns `Infinity` below two (trimmed) samples (spread can't be estimated) and `0` when the mean
+ * is non-positive (e.g. a metric-only benchmark with no render duration to converge on), so such
+ * benchmarks stop at their minimum run count rather than sampling to the maximum.
+ */
+export declare function relativeMarginOfError(samples: number[]): number;
+/** A named adaptive-sampling signal (render duration or a metric) and its achieved margin of error. */
+export interface SignalConvergence {
+  /** Human label, e.g. `duration` or `metric 'input-latency'`. */
+  label: string;
+  /** Relative margin of error achieved; `Infinity` when there were too few samples to estimate it. */
+  rme: number;
+}
+/**
+ * Describes the signals that failed to reach `targetRme`, each as a short phrase naming the signal
+ * and the margin of error it settled at (or "too few samples" when it couldn't be estimated).
+ * Returns an empty array when every signal converged. Pure so the non-convergence warning can be
+ * unit-tested apart from the measured benchmark loop.
+ */
+export declare function describeUnconvergedSignals(signals: SignalConvergence[], targetRme: number): string[];

package/stats.mjs

@@ -27,21 +27,89 @@ export function isOutlier(value, q1, q3) {
 }
 
 /**
- * Aggregates a series of samples into a mean, standard deviation, and outlier count using
- * IQR-based outlier removal. Falls back to the raw values when filtering would remove
- * everything. This is the shared aggregation core for custom metrics.
+ * Drops IQR outliers (values outside the 1.5×IQR fences), returning the surviving samples. Falls
+ * back to the original values when filtering would remove everything. Shared by every consumer that
+ * wants to reason about typical performance rather than measurement artifacts (GC pauses, scheduling
+ * hiccups), so they all trim identically.
  */
-export function aggregateSamples(values) {
+export function removeOutliers(values) {
   const sorted = values.toSorted((first, second) => first - second);
   const q1 = quantile(sorted, 0.25);
   const q3 = quantile(sorted, 0.75);
   const filtered = values.filter(value => !isOutlier(value, q1, q3));
-  const used = filtered.length > 0 ? filtered : values;
+  return filtered.length > 0 ? filtered : values;
+}
+
+/**
+ * Aggregates a series of samples into a mean, standard deviation, outlier count, and effective
+ * sample count using IQR-based outlier removal. This is the shared aggregation core for custom
+ * metrics.
+ *
+ * `count` is the number of samples that actually back `mean`/`stdDev` (post-outlier-removal), so a
+ * downstream Welch's t-test can use it directly as the `n` behind those stats.
+ */
+export function aggregateSamples(values) {
+  const used = removeOutliers(values);
   const mean = calculateMean(used);
   const stdDev = calculateStdDev(used, mean);
   return {
     mean,
     stdDev,
-    outliers: values.length - used.length
+    outliers: values.length - used.length,
+    count: used.length
   };
+}
+
+/**
+ * Relative margin of error of the mean — half the 95% confidence interval width divided by the
+ * mean, using the normal approximation (`z = 1.96`). This is the adaptive-sampling stopping signal:
+ * measurement continues until this drops to a target. A precise t-quantile is unnecessary just to
+ * decide when enough samples have been collected.
+ *
+ * IQR outliers are removed first, so the margin of error is measured on the same trimmed
+ * distribution the reported stats and the Welch comparison use. Without this, a benchmark that is
+ * stable apart from recurring GC/scheduling spikes would keep a high raw margin of error and sample
+ * all the way to its maximum (and warn that it "did not converge") even though its typical estimate
+ * settled long ago.
+ *
+ * Returns `Infinity` below two (trimmed) samples (spread can't be estimated) and `0` when the mean
+ * is non-positive (e.g. a metric-only benchmark with no render duration to converge on), so such
+ * benchmarks stop at their minimum run count rather than sampling to the maximum.
+ */
+export function relativeMarginOfError(samples) {
+  const used = removeOutliers(samples);
+  const n = used.length;
+  if (n < 2) {
+    return Infinity;
+  }
+  const mean = calculateMean(used);
+  if (mean <= 0) {
+    return 0;
+  }
+  // Bessel-corrected (sample) variance: the samples estimate the spread of the population.
+  const variance = used.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (n - 1);
+  const standardError = Math.sqrt(variance / n);
+  return 1.96 * standardError / mean;
+}
+
+/** A named adaptive-sampling signal (render duration or a metric) and its achieved margin of error. */
+
+/**
+ * Describes the signals that failed to reach `targetRme`, each as a short phrase naming the signal
+ * and the margin of error it settled at (or "too few samples" when it couldn't be estimated).
+ * Returns an empty array when every signal converged. Pure so the non-convergence warning can be
+ * unit-tested apart from the measured benchmark loop.
+ */
+export function describeUnconvergedSignals(signals, targetRme) {
+  const unconverged = [];
+  for (const {
+    label,
+    rme
+  } of signals) {
+    if (rme > targetRme) {
+      const achieved = Number.isFinite(rme) ? `RME ${(rme * 100).toFixed(2)}%` : 'too few samples';
+      unconverged.push(`${label} (${achieved})`);
+    }
+  }
+  return unconverged;
 }

package/types.d.mts

@@ -83,7 +83,10 @@ export interface MetricSampleStats {
   mean: number;
   stdDev: number;
   outliers: number;
-  /** Number of recorded samples (used to derive iteration counts; stripped from the report). */
+  /**
+   * Effective sample count behind `mean`/`stdDev` (post-outlier-removal). Serialized into the
+   * report as the `n` a Welch's t-test uses to compare this series against a baseline.
+   */
   count: number;
 }
 /** A custom metric's aggregated data attached to `task.meta`, keyed by metric name. */