overtake 1.3.2 → 2.0.0

package/src/index.ts

@@ -1,15 +1,14 @@
 import { cpus } from 'node:os';
 import Progress from 'progress';
-import { createExecutor, ExecutorOptions, ExecutorReport } from './executor.js';
-import { MaybePromise, StepFn, SetupFn, TeardownFn, FeedFn, ReportType, ReportTypeList, DEFAULT_CYCLES, ProgressInfo } from './types.js';
+import { createExecutor, type ExecutorOptions, type ExecutorReport } from './executor.ts';
+import { type MaybePromise, type StepFn, type SetupFn, type TeardownFn, type FeedFn, type ReportType, type ReportTypeList, DEFAULT_CYCLES, type ProgressInfo } from './types.ts';
 
 declare global {
   const benchmark: typeof Benchmark.create;
 }
 
-export const DEFAULT_WORKERS = cpus().length;
+export const DEFAULT_WORKERS = Math.max(1, Math.ceil(cpus().length / 4));
 
-export const AsyncFunction = (async () => {}).constructor;
 const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
 
 export interface TargetReport<R extends ReportTypeList> {
@@ -30,14 +29,24 @@ export interface FeedReport<R extends ReportTypeList> {
 export const DEFAULT_REPORT_TYPES = ['ops'] as const;
 export type DefaultReportTypes = (typeof DEFAULT_REPORT_TYPES)[number];
 
-export class MeasureContext<TContext, TInput> {
-  public pre?: StepFn<TContext, TInput>;
-  public post?: StepFn<TContext, TInput>;
+const createExecutorErrorReport = <R extends ReportTypeList>(error: unknown): ExecutorReport<R> =>
+  ({
+    count: 0,
+    heapUsedKB: 0,
+    dceWarning: false,
+    error: error instanceof Error ? error.message : String(error),
+  }) as ExecutorReport<R>;
 
-  constructor(
-    public title: string,
-    public run: StepFn<TContext, TInput>,
-  ) {}
+export class MeasureContext<TContext, TInput> {
+  pre?: StepFn<TContext, TInput>;
+  post?: StepFn<TContext, TInput>;
+  title: string;
+  run: StepFn<TContext, TInput>;
+
+  constructor(title: string, run: StepFn<TContext, TInput>) {
+    this.title = title;
+    this.run = run;
+  }
 }
 
 export class Measure<TContext, TInput> {
@@ -58,13 +67,15 @@ export class Measure<TContext, TInput> {
 }
 
 export class TargetContext<TContext, TInput> {
-  public teardown?: TeardownFn<TContext>;
-  public measures: MeasureContext<TContext, TInput>[] = [];
-
-  constructor(
-    readonly title: string,
-    readonly setup?: SetupFn<MaybePromise<TContext>>,
-  ) {}
+  teardown?: TeardownFn<TContext>;
+  measures: MeasureContext<TContext, TInput>[] = [];
+  readonly title: string;
+  readonly setup?: SetupFn<MaybePromise<TContext>>;
+
+  constructor(title: string, setup?: SetupFn<MaybePromise<TContext>>) {
+    this.title = title;
+    this.setup = setup;
+  }
 }
 
 export class Target<TContext, TInput> {
@@ -87,10 +98,13 @@ export class Target<TContext, TInput> {
 }
 
 export class FeedContext<TInput> {
-  constructor(
-    readonly title: string,
-    readonly fn?: FeedFn<TInput>,
-  ) {}
+  readonly title: string;
+  readonly fn?: FeedFn<TInput>;
+
+  constructor(title: string, fn?: FeedFn<TInput>) {
+    this.title = title;
+    this.fn = fn;
+  }
 }
 
 export class Benchmark<TInput> {
@@ -136,7 +150,7 @@ export class Benchmark<TInput> {
     return new Target<TContext, TInput>(target);
   }
 
-  async execute<R extends readonly ReportType[] = typeof DEFAULT_REPORT_TYPES>(options: ExecutorOptions<R> & { progress?: boolean }): Promise<TargetReport<R>[]> {
+  async execute<R extends readonly ReportType[] = typeof DEFAULT_REPORT_TYPES>(options: Partial<ExecutorOptions<R>> & { progress?: boolean } = {}): Promise<TargetReport<R>[]> {
     const {
       workers = DEFAULT_WORKERS,
       warmupCycles = 20,
@@ -193,45 +207,59 @@ export class Benchmark<TInput> {
     } as Required<ExecutorOptions<R>>);
 
     const reports: TargetReport<R>[] = [];
-    for (const target of this.#targets) {
-      const targetReport: TargetReport<R> = { target: target.title, measures: [] };
-      for (const measure of target.measures) {
-        const measureReport: MeasureReport<R> = { measure: measure.title, feeds: [] };
-        for (const feed of this.#feeds) {
-          const id = `${target.title}/${measure.title}/${feed.title}`;
-          const data = await feed.fn?.();
-          executor
-            .push<ExecutorReport<R>>({
-              id,
-              setup: target.setup,
-              teardown: target.teardown,
-              pre: measure.pre,
-              run: measure.run,
-              post: measure.post,
-              data,
-            })
-            .then((data) => {
-              progressMap.delete(id);
-              completedBenchmarks++;
-              measureReport.feeds.push({
-                feed: feed.title,
-                data,
-              });
-            });
+    const pendingReports: Promise<void>[] = [];
+
+    try {
+      const feedData = await Promise.all(this.#feeds.map(async (feed) => ({ title: feed.title, data: await feed.fn?.() })));
+      for (const target of this.#targets) {
+        const targetReport: TargetReport<R> = { target: target.title, measures: [] };
+        for (const measure of target.measures) {
+          const measureReport: MeasureReport<R> = { measure: measure.title, feeds: [] };
+          for (const feed of feedData) {
+            const id = `${target.title}/${measure.title}/${feed.title}`;
+            const feedReport: FeedReport<R> = {
+              feed: feed.title,
+              data: createExecutorErrorReport<R>('Benchmark did not produce a report'),
+            };
+
+            measureReport.feeds.push(feedReport);
+            pendingReports.push(
+              (async () => {
+                try {
+                  feedReport.data = await executor.pushAsync<ExecutorReport<R>>({
+                    id,
+                    setup: target.setup,
+                    teardown: target.teardown,
+                    pre: measure.pre,
+                    run: measure.run,
+                    post: measure.post,
+                    data: feed.data,
+                  });
+                } catch (error) {
+                  feedReport.data = createExecutorErrorReport<R>(error);
+                } finally {
+                  progressMap.delete(id);
+                  completedBenchmarks++;
+                }
+              })(),
+            );
+          }
+          targetReport.measures.push(measureReport);
         }
-        targetReport.measures.push(measureReport);
+        reports.push(targetReport);
       }
-      reports.push(targetReport);
-    }
-    await executor.drain();
-    executor.kill();
 
-    if (bar) {
-      bar.update(1, { label: 'done' });
-      bar.terminate();
-    }
+      await Promise.all(pendingReports);
 
-    return reports;
+      if (bar) {
+        bar.update(1, { label: 'done' });
+        bar.terminate();
+      }
+
+      return reports;
+    } finally {
+      executor.kill();
+    }
   }
 }
 
@@ -240,7 +268,11 @@ export const printSimpleReports = <R extends ReportTypeList>(reports: TargetRepo
     for (const { measure, feeds } of report.measures) {
       console.group('\n', report.target, measure);
       for (const { feed, data } of feeds) {
-        const { count, heapUsedKB, dceWarning, ...metrics } = data as Record<string, unknown>;
+        const { count, heapUsedKB, dceWarning, error: benchError, ...metrics } = data as Record<string, unknown>;
+        if (benchError) {
+          console.log(feed, `\x1b[31m[error: ${benchError}]\x1b[0m`);
+          continue;
+        }
         const output = Object.entries(metrics)
           .map(([key, report]) => `${key}: ${(report as { toString(): string }).toString()}`)
           .join('; ');
@@ -261,7 +293,12 @@ export const printTableReports = <R extends ReportTypeList>(reports: TargetRepor
       console.log('\n', report.target, measure);
       const table: Record<string, unknown> = {};
       for (const { feed, data } of feeds) {
-        table[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
+        const { error: benchError } = data as Record<string, unknown>;
+        if (benchError) {
+          table[feed] = { error: benchError };
+        } else {
+          table[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
+        }
       }
       console.table(table);
     }
@@ -274,7 +311,12 @@ export const printJSONReports = <R extends ReportTypeList>(reports: TargetReport
     for (const { measure, feeds } of report.measures) {
       const row = {} as Record<string, Record<string, string>>;
       for (const { feed, data } of feeds) {
-        row[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
+        const { error: benchError } = data as Record<string, unknown>;
+        if (benchError) {
+          row[feed] = { error: String(benchError) };
+        } else {
+          row[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
+        }
       }
       output[`${report.target} ${measure}`] = row;
     }
@@ -288,7 +330,14 @@ export const printMarkdownReports = <R extends ReportTypeList>(reports: TargetRe
       console.log(`\n## ${report.target} - ${measure}\n`);
       if (feeds.length === 0) continue;
 
-      const keys = Object.keys(feeds[0].data).filter((k) => k !== 'count');
+      const firstValid = feeds.find((f) => !(f.data as Record<string, unknown>).error);
+      if (!firstValid) {
+        for (const { feed, data } of feeds) {
+          console.log(`| ${feed} | error: ${(data as Record<string, unknown>).error} |`);
+        }
+        continue;
+      }
+      const keys = Object.keys(firstValid.data).filter((k) => k !== 'count' && k !== 'error');
       const header = ['Feed', ...keys].join(' | ');
       const separator = ['---', ...keys.map(() => '---')].join(' | ');
 
@@ -296,6 +345,10 @@ export const printMarkdownReports = <R extends ReportTypeList>(reports: TargetRe
       console.log(`| ${separator} |`);
 
       for (const { feed, data } of feeds) {
+        if ((data as Record<string, unknown>).error) {
+          console.log(`| ${feed} | error: ${(data as Record<string, unknown>).error} |`);
+          continue;
+        }
         const values = keys.map((k) => (data as Record<string, { toString(): string }>)[k]?.toString() ?? '-');
         console.log(`| ${[feed, ...values].join(' | ')} |`);
       }
@@ -309,18 +362,26 @@ export const printHistogramReports = <R extends ReportTypeList>(reports: TargetR
       console.log(`\n${report.target} - ${measure}\n`);
 
       const opsKey = 'ops';
-      const values = feeds.map((f) => ({
-        feed: f.feed,
-        value: (f.data as Record<string, { valueOf(): number }>)[opsKey]?.valueOf() ?? 0,
-      }));
+      const values = feeds.map((f) => {
+        const { error: benchError } = f.data as Record<string, unknown>;
+        return {
+          feed: f.feed,
+          value: benchError ? 0 : ((f.data as Record<string, { valueOf(): number }>)[opsKey]?.valueOf() ?? 0),
+          error: benchError as string | undefined,
+        };
+      });
 
       const maxValue = Math.max(...values.map((v) => v.value));
       const maxLabelLen = Math.max(...values.map((v) => v.feed.length));
 
-      for (const { feed, value } of values) {
+      for (const { feed, value, error } of values) {
+        const label = feed.padEnd(maxLabelLen);
+        if (error) {
+          console.log(`  ${label} | \x1b[31m[error: ${error}]\x1b[0m`);
+          continue;
+        }
         const barLen = maxValue > 0 ? Math.round((value / maxValue) * width) : 0;
         const bar = '\u2588'.repeat(barLen);
-        const label = feed.padEnd(maxLabelLen);
         const formatted = value.toLocaleString('en-US', { maximumFractionDigits: 2 });
         console.log(`  ${label} | ${bar} ${formatted} ops/s`);
       }
@@ -339,6 +400,7 @@ export const reportsToBaseline = <R extends ReportTypeList>(reports: TargetRepor
   for (const report of reports) {
     for (const { measure, feeds } of report.measures) {
       for (const { feed, data } of feeds) {
+        if ((data as Record<string, unknown>).error) continue;
         const key = `${report.target}/${measure}/${feed}`;
         results[key] = {};
         for (const [metric, value] of Object.entries(data)) {
@@ -367,6 +429,11 @@ export const printComparisonReports = <R extends ReportTypeList>(reports: Target
 
         console.log(`  ${feed}:`);
 
+        if ((data as Record<string, unknown>).error) {
+          console.log(`    \x1b[31m[error: ${(data as Record<string, unknown>).error}]\x1b[0m`);
+          continue;
+        }
+
         for (const [metric, value] of Object.entries(data)) {
           if (metric === 'count') continue;
           const current = (value as { valueOf(): number }).valueOf();

package/src/reporter.ts

@@ -1,5 +1,5 @@
-import { div, max, divs } from './utils.js';
-import { ReportType, DURATION_SCALE, Z95 } from './types.js';
+import { div, max, divs, isqrt } from './utils.ts';
+import { type ReportType, DURATION_SCALE } from './types.ts';
 
 const units = [
   { unit: 'ns', factor: 1 },
@@ -18,12 +18,17 @@ function smartFixed(n: number): string {
   });
 }
 export class Report {
-  constructor(
-    public readonly type: ReportType,
-    public readonly value: bigint,
-    public readonly uncertainty: number = 0,
-    public readonly scale: bigint = 1n,
-  ) {}
+  readonly type: ReportType;
+  readonly value: bigint;
+  readonly uncertainty: number;
+  readonly scale: bigint;
+
+  constructor(type: ReportType, value: bigint, uncertainty: number = 0, scale: bigint = 1n) {
+    this.type = type;
+    this.value = value;
+    this.uncertainty = uncertainty;
+    this.scale = scale;
+  }
   valueOf() {
     return Number(div(this.value, this.scale));
   }
@@ -70,11 +75,32 @@ export class Report {
   }
 }
 
-export const createReport = (durations: BigUint64Array, type: ReportType): Report => {
+const SQRT_SCALE = 1_000_000n;
+const SQRT_SCALE_SQ = SQRT_SCALE * SQRT_SCALE;
+const Z95_NUM = 196n;
+const Z95_DENOM = 100n;
+
+type Stats = { sum: bigint; mean: bigint; ssd: bigint; n: bigint };
+
+export const computeStats = (durations: BigUint64Array): Stats => {
+  let sum = 0n;
+  for (const d of durations) sum += d;
+  const n = BigInt(durations.length);
+  const mean = sum / n;
+  let ssd = 0n;
+  for (const d of durations) {
+    const diff = d - mean;
+    ssd += diff * diff;
+  }
+  return { sum, mean, ssd, n };
+};
+
+export const createReport = (durations: BigUint64Array, type: ReportType, stats?: Stats): Report => {
   const n = durations.length;
   if (n === 0) {
     return new Report(type, 0n);
   }
+  const st = stats ?? computeStats(durations);
   switch (type) {
     case 'min': {
       return new Report(type, durations[0], 0, DURATION_SCALE);
@@ -112,11 +138,7 @@ export const createReport = (durations: BigUint64Array, type: ReportType): Repor
     }
 
     case 'ops': {
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const avgScaled = sum / BigInt(n);
+      const { mean: avgScaled, ssd, n: nBig } = st;
       const nsPerSecScaled = 1_000_000_000n * DURATION_SCALE;
       const raw = Number(nsPerSecScaled) / Number(avgScaled);
       const extra = raw < 1 ? Math.ceil(-Math.log10(raw)) : 0;
@@ -126,111 +148,61 @@ export const createReport = (durations: BigUint64Array, type: ReportType): Repor
       const scale = 10n ** BigInt(exp);
 
       const value = avgScaled > 0n ? (nsPerSecScaled * scale) / avgScaled : 0n;
-      const deviation = durations[n - 1] - durations[0];
-      const uncertainty = avgScaled > 0 ? Number(div(deviation * scale, 2n * avgScaled)) : 0;
+      let uncertainty = 0;
+      if (n >= 2 && avgScaled > 0n) {
+        const RME_PRECISION = 1_000_000n;
+        const semOverMeanSqScaled = (ssd * RME_PRECISION * RME_PRECISION) / (BigInt(n - 1) * nBig * avgScaled * avgScaled);
+        const semOverMeanScaled = isqrt(semOverMeanSqScaled);
+        uncertainty = Number(Z95_NUM * semOverMeanScaled) / Number(RME_PRECISION);
+      }
       return new Report(type, value, uncertainty, scale);
     }
     case 'mean': {
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
+      const { sum } = st;
       const value = divs(sum, BigInt(n), 1n);
       return new Report(type, value, 0, DURATION_SCALE);
     }
 
     case 'variance': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE * DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = sum / BigInt(n);
-      let sumSquaredDiff = 0n;
-      for (const duration of durations) {
-        const diff = duration - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = sumSquaredDiff / BigInt(n - 1);
+      const { ssd } = st;
+      const variance = ssd / BigInt(n - 1);
       return new Report(type, variance, 0, DURATION_SCALE * DURATION_SCALE);
     }
 
     case 'sd': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = sum / BigInt(n);
-      let sumSquaredDiff = 0n;
-      for (const duration of durations) {
-        const diff = duration - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = Number(sumSquaredDiff) / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sdScaled = BigInt(Math.round(sd));
-      return new Report(type, sdScaled, 0, DURATION_SCALE);
+      const { ssd } = st;
+      const scaledVariance = (ssd * SQRT_SCALE_SQ) / BigInt(n - 1);
+      const sdScaled = isqrt(scaledVariance);
+      return new Report(type, sdScaled, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     case 'sem': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = sum / BigInt(n);
-      let sumSquaredDiff = 0n;
-      for (const duration of durations) {
-        const diff = duration - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = Number(sumSquaredDiff) / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const semScaled = BigInt(Math.round(sem));
-      return new Report(type, semScaled, 0, DURATION_SCALE);
+      const { ssd, n: nBig } = st;
+      const semSqScaled = (ssd * SQRT_SCALE_SQ) / (BigInt(n - 1) * nBig);
+      const semScaled = isqrt(semSqScaled);
+      return new Report(type, semScaled, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     case 'moe': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = sum / BigInt(n);
-      let sumSquaredDiff = 0n;
-      for (const duration of durations) {
-        const diff = duration - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = Number(sumSquaredDiff) / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const moe = Z95 * sem;
-      const moeScaled = BigInt(Math.round(moe));
-      return new Report(type, moeScaled, 0, DURATION_SCALE);
+      const { ssd, n: nBig } = st;
+      const semSqScaled = (ssd * SQRT_SCALE_SQ) / (BigInt(n - 1) * nBig);
+      const semScaled = isqrt(semSqScaled);
+      const moeScaled = (Z95_NUM * semScaled) / Z95_DENOM;
+      return new Report(type, moeScaled, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     case 'rme': {
       if (n < 2) return new Report(type, 0n);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = Number(sum) / n;
-      if (mean === 0) return new Report(type, 0n);
-      let sumSquaredDiff = 0;
-      for (const duration of durations) {
-        const diff = Number(duration) - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = sumSquaredDiff / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const moe = Z95 * sem;
-      const rme = (moe / mean) * 100;
-      const rmeScaled = BigInt(Math.round(rme * 100));
+      const { mean, ssd, n: nBig } = st;
+      if (mean === 0n) return new Report(type, 0n);
+      const RME_PRECISION = 1_000_000n;
+      const semOverMeanSqScaled = (ssd * RME_PRECISION * RME_PRECISION) / (BigInt(n - 1) * nBig * mean * mean);
+      const semOverMeanScaled = isqrt(semOverMeanSqScaled);
+      const rmeScaled = (Z95_NUM * semOverMeanScaled * 100n) / RME_PRECISION;
       return new Report(type, rmeScaled, 0, 100n);
     }
 
@@ -259,42 +231,22 @@ export const createReport = (durations: BigUint64Array, type: ReportType): Repor
 
     case 'ci_lower': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = Number(sum) / n;
-      let sumSquaredDiff = 0;
-      for (const duration of durations) {
-        const diff = Number(duration) - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = sumSquaredDiff / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const moe = Z95 * sem;
-      const ciLower = Math.max(0, mean - moe);
-      return new Report(type, BigInt(Math.round(ciLower)), 0, DURATION_SCALE);
+      const { mean, ssd, n: nBig } = st;
+      const semSqScaled = (ssd * SQRT_SCALE_SQ) / (BigInt(n - 1) * nBig);
+      const semScaled = isqrt(semSqScaled);
+      const moeScaled = (Z95_NUM * semScaled) / Z95_DENOM;
+      const ciLowerScaled = mean * SQRT_SCALE - moeScaled;
+      return new Report(type, ciLowerScaled > 0n ? ciLowerScaled : 0n, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     case 'ci_upper': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = Number(sum) / n;
-      let sumSquaredDiff = 0;
-      for (const duration of durations) {
-        const diff = Number(duration) - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = sumSquaredDiff / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const moe = Z95 * sem;
-      const ciUpper = mean + moe;
-      return new Report(type, BigInt(Math.round(ciUpper)), 0, DURATION_SCALE);
+      const { mean, ssd, n: nBig } = st;
+      const semSqScaled = (ssd * SQRT_SCALE_SQ) / (BigInt(n - 1) * nBig);
+      const semScaled = isqrt(semSqScaled);
+      const moeScaled = (Z95_NUM * semScaled) / Z95_DENOM;
+      const ciUpperScaled = mean * SQRT_SCALE + moeScaled;
+      return new Report(type, ciUpperScaled, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     default: {