@mui/internal-benchmark 0.0.3-canary.17 → 0.0.3-canary.18

package/Metric.d.mts

@@ -1,24 +1,5 @@
-import type { RunnerTestCase } from 'vitest';
 import type { MetricConfig, MetricKind } from "./types.mjs";
 import "./taskMetaAugmentation.mjs";
-/** One alarmed scalar (sub-)series the adaptive stopping rule tracks, named for reporting. */
-export interface AdaptiveMetricSamples {
-  /** Metric name, with a `#sub-series` suffix for named sub-series. */
-  name: string;
-  samples: number[];
-}
-/**
- * Snapshot of the raw samples collected so far for a test's alarmed scalar metrics — one entry per
- * (metric, sub-series), each named. The adaptive stopping rule reads this so measurement keeps
- * sampling while any metric that will be significance-tested downstream is still imprecise, not just
- * the render duration, and so a non-convergence warning can name the specific metric that stayed
- * noisy.
- *
- * Discrete metrics (compared as exact counts) and informational scalar metrics (shown but never
- * flagged) are excluded: their precision never changes a verdict, and gating on a noisy one would
- * needlessly block convergence.
- */
-export declare function collectAdaptiveMetricSamples(test: RunnerTestCase): AdaptiveMetricSamples[];
 export interface MetricRecordOptions {
   /** Sub-series label. Recorded under `${name}#${id}` in the report; omit for the base series. */
   id?: string;

package/Metric.mjs

@@ -12,9 +12,10 @@ function flush(test, accumulator) {
   for (const [name, entry] of accumulator) {
     const series = {};
     for (const [seriesId, samples] of entry.series) {
-      // `aggregateSamples` reports the effective (post-outlier-removal) count as `count`, which is
-      // the `n` behind mean/stdDev — exactly what a downstream Welch's t-test needs.
-      series[seriesId] = aggregateSamples(samples);
+      series[seriesId] = {
+        ...aggregateSamples(samples),
+        count: samples.length
+      };
     }
     store[name] = {
       kind: entry.kind,
@@ -24,39 +25,6 @@ function flush(test, accumulator) {
   }
   test.meta.benchmarkMetrics = store;
 }
-
-/** One alarmed scalar (sub-)series the adaptive stopping rule tracks, named for reporting. */
-
-/**
- * Snapshot of the raw samples collected so far for a test's alarmed scalar metrics — one entry per
- * (metric, sub-series), each named. The adaptive stopping rule reads this so measurement keeps
- * sampling while any metric that will be significance-tested downstream is still imprecise, not just
- * the render duration, and so a non-convergence warning can name the specific metric that stayed
- * noisy.
- *
- * Discrete metrics (compared as exact counts) and informational scalar metrics (shown but never
- * flagged) are excluded: their precision never changes a verdict, and gating on a noisy one would
- * needlessly block convergence.
- */
-export function collectAdaptiveMetricSamples(test) {
-  const accumulator = accumulators.get(test);
-  if (!accumulator) {
-    return [];
-  }
-  const sampleSets = [];
-  for (const [metricName, entry] of accumulator) {
-    if (entry.kind !== 'scalar' || entry.config.alarm === undefined) {
-      continue;
-    }
-    for (const [seriesId, samples] of entry.series) {
-      sampleSets.push({
-        name: seriesId === '' ? metricName : `${metricName}#${seriesId}`,
-        samples
-      });
-    }
-  }
-  return sampleSets;
-}
 /**
  * Base class for custom benchmark metrics. Use `ScalarMetric` or `DiscreteMetric`.
  *

package/ciReport.d.mts

@@ -10,19 +10,15 @@ declare const renderStatsSchema: z.ZodObject<{
   actualDuration: z.ZodNumber;
   stdDev: z.ZodNumber;
   outliers: z.ZodNumber;
-  count: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 declare const metricStatsSchema: z.ZodObject<{
   mean: z.ZodNumber;
   stdDev: z.ZodNumber;
   outliers: z.ZodNumber;
-  count: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 declare const benchmarkReportEntrySchema: z.ZodObject<{
   iterations: z.ZodNumber;
   totalDuration: z.ZodNumber;
-  totalStdDev: z.ZodOptional<z.ZodNumber>;
-  totalCount: z.ZodOptional<z.ZodNumber>;
   renders: z.ZodArray<z.ZodObject<{
     id: z.ZodString;
     phase: z.ZodEnum<{
@@ -34,20 +30,16 @@ declare const benchmarkReportEntrySchema: z.ZodObject<{
     actualDuration: z.ZodNumber;
     stdDev: z.ZodNumber;
     outliers: z.ZodNumber;
-    count: z.ZodOptional<z.ZodNumber>;
   }, z.core.$strip>>;
   metrics: z.ZodRecord<z.ZodString, z.ZodObject<{
     mean: z.ZodNumber;
     stdDev: z.ZodNumber;
     outliers: z.ZodNumber;
-    count: z.ZodOptional<z.ZodNumber>;
   }, z.core.$strip>>;
 }, z.core.$strip>;
 declare const benchmarkReportSchema: z.ZodRecord<z.ZodString, z.ZodObject<{
   iterations: z.ZodNumber;
   totalDuration: z.ZodNumber;
-  totalStdDev: z.ZodOptional<z.ZodNumber>;
-  totalCount: z.ZodOptional<z.ZodNumber>;
   renders: z.ZodArray<z.ZodObject<{
     id: z.ZodString;
     phase: z.ZodEnum<{
@@ -59,13 +51,11 @@ declare const benchmarkReportSchema: z.ZodRecord<z.ZodString, z.ZodObject<{
     actualDuration: z.ZodNumber;
     stdDev: z.ZodNumber;
     outliers: z.ZodNumber;
-    count: z.ZodOptional<z.ZodNumber>;
   }, z.core.$strip>>;
   metrics: z.ZodRecord<z.ZodString, z.ZodObject<{
     mean: z.ZodNumber;
     stdDev: z.ZodNumber;
     outliers: z.ZodNumber;
-    count: z.ZodOptional<z.ZodNumber>;
   }, z.core.$strip>>;
 }, z.core.$strip>>;
 declare const metricDefinitionSchema: z.ZodObject<{

package/ciReport.mjs

@@ -31,26 +31,16 @@ const renderStatsSchema = z.object({
   startTime: z.number(),
   actualDuration: z.number(),
   stdDev: z.number(),
-  outliers: z.number(),
-  // Effective sample count behind mean/stdDev, used as `n` for the comparison's Welch's t-test.
-  // Optional so baselines uploaded before it was added still parse.
-  count: z.number().optional()
+  outliers: z.number()
 });
 const metricStatsSchema = z.object({
   mean: z.number(),
   stdDev: z.number(),
-  outliers: z.number(),
-  // See `renderStatsSchema.count`. Optional for backward compatibility with older uploads.
-  count: z.number().optional()
+  outliers: z.number()
 });
 const benchmarkReportEntrySchema = z.object({
   iterations: z.number(),
   totalDuration: z.number(),
-  // Spread and effective sample count of the per-iteration total duration, used as the stdDev/`n`
-  // for the comparison's Welch test on total duration. Optional for backward compatibility with
-  // uploads made before they were added.
-  totalStdDev: z.number().optional(),
-  totalCount: z.number().optional(),
   renders: z.array(renderStatsSchema),
   metrics: z.record(z.string(), metricStatsSchema)
 });

package/format.d.mts

@@ -4,15 +4,9 @@ export declare const green: (s: string) => string;
 export declare const yellow: (s: string) => string;
 export declare const cyan: (s: string) => string;
 export declare function fileUrl(filePath: string): string;
-export interface Column {
+interface Column {
   header: string;
-  /** Width the column keeps when every cell is narrower. It grows to fit a wider cell. */
-  minWidth: number;
-  /**
-   * Width the column may not exceed; wider cells are truncated with an ellipsis. Omit to let the
-   * column grow freely, which is what a value the caller can't pre-measure (a metric formatted with
-   * an arbitrary unit) needs — truncating those would destroy the number the table exists to show.
-   */
-  maxWidth?: number;
+  width: number;
 }
-export declare function printTable(columns: Column[], rows: string[][], footer?: string, title?: string): void;
+export declare function printTable(columns: Column[], rows: string[][], footer?: string, title?: string): void;
+export {};

package/format.mjs

@@ -8,72 +8,19 @@ export const cyan = s => styleText('cyan', s);
 export function fileUrl(filePath) {
   return pathToFileURL(filePath).href;
 }
-
-/** Matches an ANSI colour code. Captured, so `split` keeps the codes alongside the text. */
-// eslint-disable-next-line no-control-regex
-const ANSI_PATTERN = /(\x1b\[[0-9;]*m)/g;
-function stripAnsi(str) {
-  return str.replace(ANSI_PATTERN, '');
-}
-/** Printable width, i.e. ignoring the zero-width ANSI escape codes that colour a string. */
-function visibleWidth(str) {
-  return stripAnsi(str).length;
-}
-
-/**
- * Truncates to a visible width, marking the cut with an ellipsis. Escape codes are carried over
- * rather than counted, so a styled string keeps the closing code that would otherwise be dropped
- * along with the text it terminated (leaving the rest of the terminal painted).
- */
-function truncate(str, maxWidth) {
-  if (visibleWidth(str) <= maxWidth) {
+function truncate(str, maxLength) {
+  if (str.length <= maxLength) {
     return str;
   }
-  let result = '';
-  let remaining = maxWidth - 1;
-  for (const segment of str.split(ANSI_PATTERN)) {
-    if (segment.startsWith('\x1b')) {
-      result += segment;
-    } else if (remaining > 0) {
-      result += segment.slice(0, remaining);
-      remaining -= Math.min(segment.length, remaining);
-    }
-  }
-  return `${result}…`;
-}
-
-/** Right-aligns to a visible width, so colour codes in the cell don't eat into the padding. */
-function padCell(cell, width) {
-  return ' '.repeat(Math.max(0, width - visibleWidth(cell))) + cell;
-}
-
-/** Fits a string to an exact visible width, truncating or padding as needed. */
-function fitCell(cell, width) {
-  return padCell(truncate(cell, width), width);
+  return `${str.slice(0, maxLength - 1)}…`;
 }
 export function printTable(columns, rows, footer, title) {
-  // Cells are sized off content, because a caller can't know how wide a value will render (a metric
-  // carrying a unit — `0.456 ms±0.057 ms` — is far wider than a bare `0.46±0.06`). Padding a cell
-  // that already overflows its column is a no-op, which used to let one long value push that row's
-  // dividers out of line with every other row.
-  const colWidths = columns.map((col, index) => {
-    const content = Math.max(col.minWidth, visibleWidth(col.header), ...rows.map(row => visibleWidth(row[index] ?? '')));
-    return col.maxWidth === undefined ? content : Math.min(content, col.maxWidth);
-  });
-  const innerWidth = () => colWidths.reduce((sum, w) => sum + w + 2, 0) + colWidths.length - 1;
-  // The footer spans the whole table, so it too can be the widest thing in it. It reports counts
-  // that are worth nothing abbreviated, so the table stretches to fit it — the slack goes on the
-  // last column, keeping the columns left of it where the reader expects. A title, by contrast, is
-  // a name: arbitrarily long, and legible truncated, so it fits itself to the table below.
-  const footerWidth = footer ? visibleWidth(footer) + 2 : 0;
-  if (colWidths.length > 0 && footerWidth > innerWidth()) {
-    colWidths[colWidths.length - 1] += footerWidth - innerWidth();
-  }
-  const totalInner = innerWidth();
+  const colWidths = columns.map(col => col.width);
+  const totalInner = colWidths.reduce((sum, w) => sum + w + 2, 0) + colWidths.length - 1;
   if (title) {
     const titleTop = dim(`┌${'─'.repeat(totalInner)}┐`);
     const titleContent = ` ${truncate(title, totalInner - 2)}`;
-    const titlePadding = totalInner - visibleWidth(titleContent);
+    const titlePadding = totalInner - titleContent.length;
     const titleLine = dim('│') + titleContent + ' '.repeat(Math.max(0, titlePadding)) + dim('│');
     const titleSep = dim(`├${colWidths.map(w => '─'.repeat(w + 2)).join('┬')}┤`);
 
@@ -89,7 +36,7 @@ export function printTable(columns, rows, footer, title) {
     console.log(topBorder);
   }
   const headerSep = dim(`├${colWidths.map(w => '─'.repeat(w + 2)).join('┼')}┤`);
-  const headerCells = columns.map((col, index) => ` ${fitCell(col.header, colWidths[index])} `);
+  const headerCells = columns.map(col => ` ${col.header.padStart(col.width)} `);
   const headerLine = dim('│') + headerCells.join(dim('│')) + dim('│');
 
   // eslint-disable-next-line no-console
@@ -97,15 +44,14 @@ export function printTable(columns, rows, footer, title) {
   // eslint-disable-next-line no-console
   console.log(headerSep);
   for (const row of rows) {
-    // Driven by the columns, not the row, so a short row still emits every cell and divider.
-    const cells = colWidths.map((width, index) => ` ${fitCell(row[index] ?? '', width)} `);
+    const cells = row.map((cell, i) => ` ${cell.padStart(colWidths[i])} `);
     // eslint-disable-next-line no-console
     console.log(dim('│') + cells.join(dim('│')) + dim('│'));
   }
   if (footer) {
     const footerSep = dim(`├${colWidths.map(w => '─'.repeat(w + 2)).join('┴')}┤`);
     const footerContent = ` ${footer}`;
-    const padding = totalInner - visibleWidth(footerContent);
+    const padding = totalInner - stripAnsi(footerContent).length;
     const footerLine = dim('│') + footerContent + ' '.repeat(Math.max(0, padding)) + dim('│');
     const bottomBorder = dim(`└${'─'.repeat(totalInner)}┘`);
 
@@ -120,4 +66,10 @@ export function printTable(columns, rows, footer, title) {
     // eslint-disable-next-line no-console
     console.log(bottomBorder);
   }
+}
+
+// Strip ANSI escape codes to measure visible string length
+function stripAnsi(str) {
+  // eslint-disable-next-line no-control-regex
+  return str.replace(/\x1b\[[0-9;]*m/g, '');
 }

package/index.d.mts

@@ -14,26 +14,8 @@ declare global {
   }
 }
 interface BenchmarkOptions {
-  /**
-   * Fixed number of measured iterations. When set, disables adaptive sampling (equivalent to
-   * `minRuns === maxRuns === runs`). Prefer leaving this unset and letting the harness sample
-   * adaptively; use it only to pin a benchmark to an exact iteration count.
-   */
   runs?: number;
-  /** Warmup iterations run before measurement begins (not recorded). Defaults to `5`. */
   warmupRuns?: number;
-  /**
-   * Adaptive sampling: the harness keeps measuring until the mean render duration is estimated to
-   * within `targetRme`, then stops. `minRuns` is the floor before the stopping rule can trigger,
-   * `maxRuns` the ceiling. Ignored when `runs` is set. Defaults: `minRuns` 10, `maxRuns` 50.
-   */
-  minRuns?: number;
-  maxRuns?: number;
-  /**
-   * Target relative margin of error (half-width of the 95% confidence interval of the mean, as a
-   * fraction of the mean) at which adaptive sampling stops. Defaults to `0.02` (2%).
-   */
-  targetRme?: number;
   afterEach?: () => Promise<void> | void;
   /**
    * Start each iteration with React render/paint recording paused. The interaction callback then

package/index.mjs

@@ -12,8 +12,6 @@ import * as ReactDOMClient from 'react-dom/client'; // aliased to react-dom/prof
 import * as ReactDOM from 'react-dom';
 import { ElementTiming } from "./ElementTiming.mjs";
 import { ScalarMetric } from "./ScalarMetric.mjs";
-import { describeUnconvergedSignals, relativeMarginOfError } from "./stats.mjs";
-import { collectAdaptiveMetricSamples } from "./Metric.mjs";
 import { metricsGate } from "./metricsGate.mjs";
 import { createReactRecordingControls } from "./reactRecording.mjs";
 import { runProfileSession } from "./profileSession.mjs";
@@ -201,30 +199,10 @@ export function benchmark(name, renderFn, interactionOrOptions, maybeOptions) {
   it(name, async ({
     task
   }) => {
-    const warmupRuns = options?.warmupRuns ?? 5;
-    // A fixed `runs` pins both bounds; otherwise sample adaptively between min and max.
-    const isAdaptive = options?.runs === undefined;
-    const minRuns = options?.runs ?? options?.minRuns ?? 10;
-    const maxRuns = options?.runs ?? options?.maxRuns ?? 50;
-    const targetRme = options?.targetRme ?? 0.02;
-
-    // Fail fast on misconfigured sampling bounds rather than silently running a surprising loop.
-    if (minRuns < 1 || maxRuns < minRuns || targetRme <= 0) {
-      throw new Error(`Invalid benchmark sampling options for "${name}": require 1 <= minRuns (${minRuns}) <= ` + `maxRuns (${maxRuns}) and targetRme (${targetRme}) > 0.`);
-    }
-
-    // Upper bound on the loop; the adaptive stopping rule usually breaks out earlier.
-    const totalRuns = warmupRuns + maxRuns;
+    const runs = options?.runs ?? 20;
+    const warmupRuns = options?.warmupRuns ?? 10;
+    const totalRuns = warmupRuns + runs;
     const iterations = [];
-    // Per measured iteration: total render duration, the primary signal the stopping rule converges
-    // on (alongside each alarmed scalar metric).
-    const iterationDurations = [];
-    // Relative margin of error of the render duration at the last stopping check — reused for the
-    // non-convergence warning instead of recomputing the same trim/variance pass.
-    let durationRme = Infinity;
-    // Per-metric relative margin of error from the last stopping check (only populated once render
-    // duration has converged), named so the non-convergence warning can point at the noisy metric.
-    let metricRmes = [];
 
     // Paint timings are recorded as one harness-owned `bench:paint` metric: the default sentinel
     // is the base series (`bench:paint`) and named `elementtiming` markers are sub-series
@@ -319,57 +297,11 @@ export function benchmark(name, renderFn, interactionOrOptions, maybeOptions) {
         iterations.push({
           renders: captures
         });
-        // Total render duration of this iteration — the adaptive stopping rule's convergence signal.
-        iterationDurations.push(captures.reduce((sum, capture) => sum + capture.actualDuration, 0));
       }
       if (options?.afterEach) {
         // eslint-disable-next-line no-await-in-loop
         await options.afterEach();
       }
-
-      // Adaptive stopping: once past the floor, stop as soon as every measured signal — the mean
-      // render duration and each alarmed scalar metric — is estimated tightly enough. Custom metrics
-      // can only *delay* the stop (the `minRuns` floor and duration target still apply), so a noisy
-      // metric keeps sampling rather than being left underpowered for the dashboard's significance
-      // test. `maxRuns` caps the work for benchmarks too noisy to reach `targetRme`.
-      if (!isWarmup && iterationDurations.length >= minRuns) {
-        durationRme = relativeMarginOfError(iterationDurations);
-        // Duration is the primary (and usually last-to-tighten) signal, so check it first: the
-        // per-metric margin-of-error pass only runs once duration itself has converged. A
-        // zero-centered metric (mean <= 0) yields a margin of error of 0 — relative precision is
-        // undefined for it — so it never blocks; such a metric can't fire a relative alarm anyway.
-        if (durationRme <= targetRme) {
-          metricRmes = collectAdaptiveMetricSamples(task).map(({
-            name: metricName,
-            samples
-          }) => ({
-            name: metricName,
-            rme: relativeMarginOfError(samples)
-          }));
-          if (metricRmes.every(({
-            rme
-          }) => rme <= targetRme)) {
-            break;
-          }
-        }
-      }
-    }
-
-    // Warn only when adaptive sampling exhausted `maxRuns` without every signal reaching `targetRme`,
-    // naming each signal that stayed noisy so the author knows what to fix — the common (converged)
-    // case stays quiet so large suites don't flood CI logs. Skipped in fixed mode (`runs` set), where
-    // there is no convergence target to miss. Reuses the margins of error already computed above.
-    if (isAdaptive && iterationDurations.length >= maxRuns) {
-      const unconverged = describeUnconvergedSignals([{
-        label: 'duration',
-        rme: durationRme
-      }, ...metricRmes.map(metric => ({
-        label: `metric '${metric.name}'`,
-        rme: metric.rme
-      }))], targetRme);
-      if (unconverged.length > 0) {
-        console.warn(`Benchmark "${name}" reached maxRuns (${maxRuns} runs) without converging to the ` + `${(targetRme * 100).toFixed(2)}% target: ${unconverged.join(', ')}. Results may be ` + `noisier than intended — consider raising maxRuns or reducing variance.`);
-      }
     }
     task.meta.benchmarkIterations = iterations;
     task.meta.benchmarkName = name;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mui/internal-benchmark",
-  "version": "0.0.3-canary.17",
+  "version": "0.0.3-canary.18",
   "author": "MUI Team",
   "description": "Benchmark utilities for MUI projects. Internal package.",
   "repository": {
@@ -68,5 +68,5 @@
       }
     }
   },
-  "gitSha": "69ae6a8458bc68be5bef922f72637180b2b160e5"
+  "gitSha": "29a55c8557630acaee4f2b750c6a9813563a0bdf"
 }

package/reporter.mjs

@@ -7,6 +7,9 @@ import { uploadCiReport } from "./upload.mjs";
 import { syncPrComment } from "./syncPrComment.mjs";
 // Import for TaskMeta augmentation side effect
 import "./taskMetaAugmentation.mjs";
+function getEventKey(event) {
+  return `${event.id}:${event.phase}`;
+}
 
 /** Order-insensitive deep equality, treating a missing key and an `undefined` value as equal. */
 function deepEqual(first, second) {
@@ -53,24 +56,21 @@ function generateReportFromIterations(iterations) {
   const renderStats = [];
   for (let index = 0; index < expectedLength; index += 1) {
     const durations = iterations.map(iteration => iteration.renders[index].actualDuration);
-
-    // The per-render coefficient of variation is surfaced as the "Var%" column of the results
-    // table (informational). It no longer warrants a warning: the Welch comparison already accounts
-    // for a render's spread (a noisy render yields a high p-value rather than a false flag), and
-    // adaptive sampling drives the *mean's* precision to `targetRme`, warning separately when it
-    // can't converge. High spread alone no longer implies an unreliable result.
     const {
       mean: iqrMean,
       stdDev: iqrStdDev,
-      outliers,
-      count
+      outliers
     } = aggregateSamples(durations);
+    const coefficientOfVariation = iqrMean > 0 ? iqrStdDev / iqrMean : 0;
+    if (iqrMean > 1 && coefficientOfVariation > 0.1) {
+      const event = firstIteration.renders[index];
+      console.warn(`High coefficient of variation (${(coefficientOfVariation * 100).toFixed(1)}%) for render #${index} event "${getEventKey(event)}". ` + `Mean: ${iqrMean.toFixed(2)}ms, StdDev: ${iqrStdDev.toFixed(2)}ms. Results may be unreliable.`);
+    }
     renderStats.push({
       event: firstIteration.renders[index],
       iqrMean,
       iqrStdDev,
-      outliers,
-      count
+      outliers
     });
   }
 
@@ -84,13 +84,13 @@ function generateReportFromIterations(iterations) {
     meanGaps.push(calculateMean(gaps));
   }
   const renders = [];
+  let totalDuration = 0;
   for (let index = 0; index < expectedLength; index += 1) {
     const {
       event,
       iqrMean,
       iqrStdDev,
-      outliers,
-      count
+      outliers
     } = renderStats[index];
     const startTime = index === 0 ? 0 : renders[index - 1].startTime + renders[index - 1].actualDuration + meanGaps[index];
     renders.push({
@@ -99,27 +99,16 @@ function generateReportFromIterations(iterations) {
       startTime,
       actualDuration: iqrMean,
       stdDev: iqrStdDev,
-      outliers,
-      count
+      outliers
     });
+    totalDuration += iqrMean;
   }
 
-  // Aggregate the *actual* per-iteration total duration (sum of that iteration's render durations).
-  // `totalDuration`, `totalStdDev`, and `totalCount` are all taken from this one distribution so the
-  // comparison's Welch test gets a mutually consistent (mean, stdDev, n) triple — and the spread
-  // reflects real cross-render correlation, which a sum of per-render variances cannot. (Because
-  // outliers are filtered here on the totals, not per render, `totalDuration` may differ marginally
-  // from summing the per-render means shown in the render table.)
-  const perIterationTotals = iterations.map(iteration => iteration.renders.reduce((sum, render) => sum + render.actualDuration, 0));
-  const totalStats = aggregateSamples(perIterationTotals);
-
   // Custom + paint metrics are merged separately from `task.meta.benchmarkMetrics`.
   const metrics = {};
   return {
     iterations: iterationCount,
-    totalDuration: totalStats.mean,
-    totalStdDev: totalStats.stdDev,
-    totalCount: totalStats.count,
+    totalDuration,
     renders,
     metrics
   };
@@ -127,9 +116,8 @@ function generateReportFromIterations(iterations) {
 const LABEL_WIDTH = 28;
 const STAT_WIDTH = 16;
 const CV_WIDTH = 8;
-const OUT_WIDTH = 4;
 function colorCV(cv) {
-  const str = `${cv.toFixed(1)}%`;
+  const str = `${cv.toFixed(1)}%`.padStart(CV_WIDTH);
   if (cv > 10) {
     return red(str);
   }
@@ -138,33 +126,6 @@ function colorCV(cv) {
   }
   return dim(str);
 }
-
-/**
- * Columns shared by the duration and metric tables: a label pinned to a fixed width, then stats
- * free to grow (a metric can be formatted with any unit, so its width isn't knowable here).
- */
-function statColumns(labelHeader, statHeader) {
-  return [{
-    header: labelHeader,
-    minWidth: LABEL_WIDTH,
-    maxWidth: LABEL_WIDTH
-  }, {
-    header: statHeader,
-    minWidth: STAT_WIDTH
-  }, {
-    header: 'Var%',
-    minWidth: CV_WIDTH
-  }, {
-    header: 'Out',
-    minWidth: OUT_WIDTH
-  }];
-}
-
-/** A `label | mean±σ | variation | outliers` row, as used by both tables. */
-function statRow(label, iqrStr, mean, stdDev, outliers) {
-  const cv = mean > 0 ? stdDev / mean * 100 : 0;
-  return [label, cyan(iqrStr), colorCV(cv), outliers > 0 ? yellow(String(outliers)) : dim('0')];
-}
 function printDurationMatrix(name, report, footer) {
   if (report.renders.length === 0) {
     return;
@@ -172,10 +133,24 @@ function printDurationMatrix(name, report, footer) {
   const rows = [];
   for (let r = 0; r < report.renders.length; r += 1) {
     const render = report.renders[r];
+    const label = `#${r} ${render.id}:${render.phase}`;
     const iqrStr = `${render.actualDuration.toFixed(2)}±${render.stdDev.toFixed(2)}`;
-    rows.push(statRow(`#${r} ${render.id}:${render.phase}`, iqrStr, render.actualDuration, render.stdDev, render.outliers));
+    const cv = render.actualDuration > 0 ? render.stdDev / render.actualDuration * 100 : 0;
+    rows.push([label.slice(0, LABEL_WIDTH).padStart(LABEL_WIDTH), cyan(iqrStr.padStart(STAT_WIDTH)), colorCV(cv), render.outliers > 0 ? yellow(String(render.outliers).padStart(4)) : dim('0'.padStart(4))]);
   }
-  printTable(statColumns('Render', 'Mean±σ (ms)'), rows, footer, name);
+  printTable([{
+    header: 'Render',
+    width: LABEL_WIDTH
+  }, {
+    header: 'Mean±σ (ms)',
+    width: STAT_WIDTH
+  }, {
+    header: 'Var%',
+    width: CV_WIDTH
+  }, {
+    header: 'Out',
+    width: 4
+  }], rows, footer, name);
 }
 
 /** Strips a `#sub-series` suffix to recover the metric name used to look up its definition. */
@@ -203,13 +178,9 @@ function mergeCustomMetrics(report, customMetrics, definitions) {
       report.metrics[key] = {
         mean: stats.mean,
         stdDev: stats.stdDev,
-        outliers: stats.outliers,
-        count: stats.count
+        outliers: stats.outliers
       };
-      // `stats.count` is the post-outlier-removal count; add back the dropped outliers to recover
-      // the raw number of recorded samples, which is what a metric-only benchmark reports as its
-      // iteration count below.
-      maxCount = Math.max(maxCount, stats.count + stats.outliers);
+      maxCount = Math.max(maxCount, stats.count);
     }
     const definition = {
       kind: metric.kind,
@@ -237,10 +208,23 @@ function printMetricsTable(name, metrics, iterationCount, definitions) {
   const rows = entries.map(([metricName, stats]) => {
     const definition = definitions[baseMetricName(metricName)];
     const iqrStr = `${formatMetricValue(stats.mean, definition)}±${formatMetricValue(stats.stdDev, definition)}`;
+    const cv = stats.mean > 0 ? stats.stdDev / stats.mean * 100 : 0;
     const label = definition?.alarm ? `${metricName} ⚠` : metricName;
-    return statRow(label, iqrStr, stats.mean, stats.stdDev, stats.outliers);
+    return [label.slice(0, LABEL_WIDTH).padStart(LABEL_WIDTH), cyan(iqrStr.padStart(STAT_WIDTH)), colorCV(cv), stats.outliers > 0 ? yellow(String(stats.outliers).padStart(4)) : dim('0'.padStart(4))];
   });
-  printTable(statColumns('Metric', 'Mean±σ'), rows, dim(`${iterationCount} iterations`), `${name} — Metrics`);
+  printTable([{
+    header: 'Metric',
+    width: LABEL_WIDTH
+  }, {
+    header: 'Mean±σ',
+    width: STAT_WIDTH
+  }, {
+    header: 'Var%',
+    width: CV_WIDTH
+  }, {
+    header: 'Out',
+    width: 4
+  }], rows, dim(`${iterationCount} iterations`), `${name} — Metrics`);
 }
 async function loadBaselineReport(baselinePath) {
   const raw = await fs.readFile(baselinePath, 'utf8');

package/stats.d.mts

@@ -9,54 +9,12 @@ export declare function quantile(sorted: number[], q: number): number;
  */
 export declare function isOutlier(value: number, q1: number, q3: number): boolean;
 /**
- * Drops IQR outliers (values outside the 1.5×IQR fences), returning the surviving samples. Falls
- * back to the original values when filtering would remove everything. Shared by every consumer that
- * wants to reason about typical performance rather than measurement artifacts (GC pauses, scheduling
- * hiccups), so they all trim identically.
- */
-export declare function removeOutliers(values: number[]): number[];
-/**
- * Aggregates a series of samples into a mean, standard deviation, outlier count, and effective
- * sample count using IQR-based outlier removal. This is the shared aggregation core for custom
- * metrics.
- *
- * `count` is the number of samples that actually back `mean`/`stdDev` (post-outlier-removal), so a
- * downstream Welch's t-test can use it directly as the `n` behind those stats.
+ * Aggregates a series of samples into a mean, standard deviation, and outlier count using
+ * IQR-based outlier removal. Falls back to the raw values when filtering would remove
+ * everything. This is the shared aggregation core for custom metrics.
  */
 export declare function aggregateSamples(values: number[]): {
   mean: number;
   stdDev: number;
   outliers: number;
-  count: number;
-};
-/**
- * Relative margin of error of the mean — half the 95% confidence interval width divided by the
- * mean, using the normal approximation (`z = 1.96`). This is the adaptive-sampling stopping signal:
- * measurement continues until this drops to a target. A precise t-quantile is unnecessary just to
- * decide when enough samples have been collected.
- *
- * IQR outliers are removed first, so the margin of error is measured on the same trimmed
- * distribution the reported stats and the Welch comparison use. Without this, a benchmark that is
- * stable apart from recurring GC/scheduling spikes would keep a high raw margin of error and sample
- * all the way to its maximum (and warn that it "did not converge") even though its typical estimate
- * settled long ago.
- *
- * Returns `Infinity` below two (trimmed) samples (spread can't be estimated) and `0` when the mean
- * is non-positive (e.g. a metric-only benchmark with no render duration to converge on), so such
- * benchmarks stop at their minimum run count rather than sampling to the maximum.
- */
-export declare function relativeMarginOfError(samples: number[]): number;
-/** A named adaptive-sampling signal (render duration or a metric) and its achieved margin of error. */
-export interface SignalConvergence {
-  /** Human label, e.g. `duration` or `metric 'input-latency'`. */
-  label: string;
-  /** Relative margin of error achieved; `Infinity` when there were too few samples to estimate it. */
-  rme: number;
-}
-/**
- * Describes the signals that failed to reach `targetRme`, each as a short phrase naming the signal
- * and the margin of error it settled at (or "too few samples" when it couldn't be estimated).
- * Returns an empty array when every signal converged. Pure so the non-convergence warning can be
- * unit-tested apart from the measured benchmark loop.
- */
-export declare function describeUnconvergedSignals(signals: SignalConvergence[], targetRme: number): string[];
+};

package/stats.mjs

@@ -27,89 +27,21 @@ export function isOutlier(value, q1, q3) {
 }
 
 /**
- * Drops IQR outliers (values outside the 1.5×IQR fences), returning the surviving samples. Falls
- * back to the original values when filtering would remove everything. Shared by every consumer that
- * wants to reason about typical performance rather than measurement artifacts (GC pauses, scheduling
- * hiccups), so they all trim identically.
+ * Aggregates a series of samples into a mean, standard deviation, and outlier count using
+ * IQR-based outlier removal. Falls back to the raw values when filtering would remove
+ * everything. This is the shared aggregation core for custom metrics.
  */
-export function removeOutliers(values) {
+export function aggregateSamples(values) {
   const sorted = values.toSorted((first, second) => first - second);
   const q1 = quantile(sorted, 0.25);
   const q3 = quantile(sorted, 0.75);
   const filtered = values.filter(value => !isOutlier(value, q1, q3));
-  return filtered.length > 0 ? filtered : values;
-}
-
-/**
- * Aggregates a series of samples into a mean, standard deviation, outlier count, and effective
- * sample count using IQR-based outlier removal. This is the shared aggregation core for custom
- * metrics.
- *
- * `count` is the number of samples that actually back `mean`/`stdDev` (post-outlier-removal), so a
- * downstream Welch's t-test can use it directly as the `n` behind those stats.
- */
-export function aggregateSamples(values) {
-  const used = removeOutliers(values);
+  const used = filtered.length > 0 ? filtered : values;
   const mean = calculateMean(used);
   const stdDev = calculateStdDev(used, mean);
   return {
     mean,
     stdDev,
-    outliers: values.length - used.length,
-    count: used.length
+    outliers: values.length - used.length
   };
-}
-
-/**
- * Relative margin of error of the mean — half the 95% confidence interval width divided by the
- * mean, using the normal approximation (`z = 1.96`). This is the adaptive-sampling stopping signal:
- * measurement continues until this drops to a target. A precise t-quantile is unnecessary just to
- * decide when enough samples have been collected.
- *
- * IQR outliers are removed first, so the margin of error is measured on the same trimmed
- * distribution the reported stats and the Welch comparison use. Without this, a benchmark that is
- * stable apart from recurring GC/scheduling spikes would keep a high raw margin of error and sample
- * all the way to its maximum (and warn that it "did not converge") even though its typical estimate
- * settled long ago.
- *
- * Returns `Infinity` below two (trimmed) samples (spread can't be estimated) and `0` when the mean
- * is non-positive (e.g. a metric-only benchmark with no render duration to converge on), so such
- * benchmarks stop at their minimum run count rather than sampling to the maximum.
- */
-export function relativeMarginOfError(samples) {
-  const used = removeOutliers(samples);
-  const n = used.length;
-  if (n < 2) {
-    return Infinity;
-  }
-  const mean = calculateMean(used);
-  if (mean <= 0) {
-    return 0;
-  }
-  // Bessel-corrected (sample) variance: the samples estimate the spread of the population.
-  const variance = used.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (n - 1);
-  const standardError = Math.sqrt(variance / n);
-  return 1.96 * standardError / mean;
-}
-
-/** A named adaptive-sampling signal (render duration or a metric) and its achieved margin of error. */
-
-/**
- * Describes the signals that failed to reach `targetRme`, each as a short phrase naming the signal
- * and the margin of error it settled at (or "too few samples" when it couldn't be estimated).
- * Returns an empty array when every signal converged. Pure so the non-convergence warning can be
- * unit-tested apart from the measured benchmark loop.
- */
-export function describeUnconvergedSignals(signals, targetRme) {
-  const unconverged = [];
-  for (const {
-    label,
-    rme
-  } of signals) {
-    if (rme > targetRme) {
-      const achieved = Number.isFinite(rme) ? `RME ${(rme * 100).toFixed(2)}%` : 'too few samples';
-      unconverged.push(`${label} (${achieved})`);
-    }
-  }
-  return unconverged;
 }

package/types.d.mts

@@ -83,10 +83,7 @@ export interface MetricSampleStats {
   mean: number;
   stdDev: number;
   outliers: number;
-  /**
-   * Effective sample count behind `mean`/`stdDev` (post-outlier-removal). Serialized into the
-   * report as the `n` a Welch's t-test uses to compare this series against a baseline.
-   */
+  /** Number of recorded samples (used to derive iteration counts; stripped from the report). */
   count: number;
 }
 /** A custom metric's aggregated data attached to `task.meta`, keyed by metric name. */