overtake 1.3.2 → 1.4.0

package/src/reporter.ts

@@ -1,5 +1,5 @@
-import { div, max, divs } from './utils.js';
-import { ReportType, DURATION_SCALE, Z95 } from './types.js';
+import { div, max, divs, isqrt } from './utils.js';
+import { ReportType, DURATION_SCALE } from './types.js';
 
 const units = [
   { unit: 'ns', factor: 1 },
@@ -70,6 +70,24 @@ export class Report {
   }
 }
 
+const SQRT_SCALE = 1_000_000n;
+const SQRT_SCALE_SQ = SQRT_SCALE * SQRT_SCALE;
+const Z95_NUM = 196n;
+const Z95_DENOM = 100n;
+
+const computeStats = (durations: BigUint64Array) => {
+  let sum = 0n;
+  for (const d of durations) sum += d;
+  const n = BigInt(durations.length);
+  const mean = sum / n;
+  let ssd = 0n;
+  for (const d of durations) {
+    const diff = d - mean;
+    ssd += diff * diff;
+  }
+  return { sum, mean, ssd, n };
+};
+
 export const createReport = (durations: BigUint64Array, type: ReportType): Report => {
   const n = durations.length;
   if (n === 0) {
@@ -112,11 +130,7 @@ export const createReport = (durations: BigUint64Array, type: ReportType): Repor
     }
 
     case 'ops': {
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const avgScaled = sum / BigInt(n);
+      const { mean: avgScaled, ssd, n: nBig } = computeStats(durations);
       const nsPerSecScaled = 1_000_000_000n * DURATION_SCALE;
       const raw = Number(nsPerSecScaled) / Number(avgScaled);
       const extra = raw < 1 ? Math.ceil(-Math.log10(raw)) : 0;
@@ -126,111 +140,61 @@ export const createReport = (durations: BigUint64Array, type: ReportType): Repor
       const scale = 10n ** BigInt(exp);
 
       const value = avgScaled > 0n ? (nsPerSecScaled * scale) / avgScaled : 0n;
-      const deviation = durations[n - 1] - durations[0];
-      const uncertainty = avgScaled > 0 ? Number(div(deviation * scale, 2n * avgScaled)) : 0;
+      let uncertainty = 0;
+      if (n >= 2 && avgScaled > 0n) {
+        const RME_PRECISION = 1_000_000n;
+        const semOverMeanSqScaled = (ssd * RME_PRECISION * RME_PRECISION) / (BigInt(n - 1) * nBig * avgScaled * avgScaled);
+        const semOverMeanScaled = isqrt(semOverMeanSqScaled);
+        uncertainty = Number(Z95_NUM * semOverMeanScaled) / Number(RME_PRECISION);
+      }
       return new Report(type, value, uncertainty, scale);
     }
     case 'mean': {
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
+      const { sum } = computeStats(durations);
       const value = divs(sum, BigInt(n), 1n);
       return new Report(type, value, 0, DURATION_SCALE);
     }
 
     case 'variance': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE * DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = sum / BigInt(n);
-      let sumSquaredDiff = 0n;
-      for (const duration of durations) {
-        const diff = duration - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = sumSquaredDiff / BigInt(n - 1);
+      const { ssd } = computeStats(durations);
+      const variance = ssd / BigInt(n - 1);
       return new Report(type, variance, 0, DURATION_SCALE * DURATION_SCALE);
     }
 
     case 'sd': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = sum / BigInt(n);
-      let sumSquaredDiff = 0n;
-      for (const duration of durations) {
-        const diff = duration - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = Number(sumSquaredDiff) / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sdScaled = BigInt(Math.round(sd));
-      return new Report(type, sdScaled, 0, DURATION_SCALE);
+      const { ssd } = computeStats(durations);
+      const scaledVariance = (ssd * SQRT_SCALE_SQ) / BigInt(n - 1);
+      const sdScaled = isqrt(scaledVariance);
+      return new Report(type, sdScaled, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     case 'sem': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = sum / BigInt(n);
-      let sumSquaredDiff = 0n;
-      for (const duration of durations) {
-        const diff = duration - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = Number(sumSquaredDiff) / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const semScaled = BigInt(Math.round(sem));
-      return new Report(type, semScaled, 0, DURATION_SCALE);
+      const { ssd, n: nBig } = computeStats(durations);
+      const semSqScaled = (ssd * SQRT_SCALE_SQ) / (BigInt(n - 1) * nBig);
+      const semScaled = isqrt(semSqScaled);
+      return new Report(type, semScaled, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     case 'moe': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = sum / BigInt(n);
-      let sumSquaredDiff = 0n;
-      for (const duration of durations) {
-        const diff = duration - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = Number(sumSquaredDiff) / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const moe = Z95 * sem;
-      const moeScaled = BigInt(Math.round(moe));
-      return new Report(type, moeScaled, 0, DURATION_SCALE);
+      const { ssd, n: nBig } = computeStats(durations);
+      const semSqScaled = (ssd * SQRT_SCALE_SQ) / (BigInt(n - 1) * nBig);
+      const semScaled = isqrt(semSqScaled);
+      const moeScaled = (Z95_NUM * semScaled) / Z95_DENOM;
+      return new Report(type, moeScaled, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     case 'rme': {
       if (n < 2) return new Report(type, 0n);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = Number(sum) / n;
-      if (mean === 0) return new Report(type, 0n);
-      let sumSquaredDiff = 0;
-      for (const duration of durations) {
-        const diff = Number(duration) - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = sumSquaredDiff / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const moe = Z95 * sem;
-      const rme = (moe / mean) * 100;
-      const rmeScaled = BigInt(Math.round(rme * 100));
+      const { mean, ssd, n: nBig } = computeStats(durations);
+      if (mean === 0n) return new Report(type, 0n);
+      const RME_PRECISION = 1_000_000n;
+      const semOverMeanSqScaled = (ssd * RME_PRECISION * RME_PRECISION) / (BigInt(n - 1) * nBig * mean * mean);
+      const semOverMeanScaled = isqrt(semOverMeanSqScaled);
+      const rmeScaled = (Z95_NUM * semOverMeanScaled * 100n) / RME_PRECISION;
       return new Report(type, rmeScaled, 0, 100n);
     }
 
@@ -259,42 +223,22 @@ export const createReport = (durations: BigUint64Array, type: ReportType): Repor
 
     case 'ci_lower': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = Number(sum) / n;
-      let sumSquaredDiff = 0;
-      for (const duration of durations) {
-        const diff = Number(duration) - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = sumSquaredDiff / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const moe = Z95 * sem;
-      const ciLower = Math.max(0, mean - moe);
-      return new Report(type, BigInt(Math.round(ciLower)), 0, DURATION_SCALE);
+      const { mean, ssd, n: nBig } = computeStats(durations);
+      const semSqScaled = (ssd * SQRT_SCALE_SQ) / (BigInt(n - 1) * nBig);
+      const semScaled = isqrt(semSqScaled);
+      const moeScaled = (Z95_NUM * semScaled) / Z95_DENOM;
+      const ciLowerScaled = mean * SQRT_SCALE - moeScaled;
+      return new Report(type, ciLowerScaled > 0n ? ciLowerScaled : 0n, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     case 'ci_upper': {
       if (n < 2) return new Report(type, 0n, 0, DURATION_SCALE);
-      let sum = 0n;
-      for (const duration of durations) {
-        sum += duration;
-      }
-      const mean = Number(sum) / n;
-      let sumSquaredDiff = 0;
-      for (const duration of durations) {
-        const diff = Number(duration) - mean;
-        sumSquaredDiff += diff * diff;
-      }
-      const variance = sumSquaredDiff / (n - 1);
-      const sd = Math.sqrt(variance);
-      const sem = sd / Math.sqrt(n);
-      const moe = Z95 * sem;
-      const ciUpper = mean + moe;
-      return new Report(type, BigInt(Math.round(ciUpper)), 0, DURATION_SCALE);
+      const { mean, ssd, n: nBig } = computeStats(durations);
+      const semSqScaled = (ssd * SQRT_SCALE_SQ) / (BigInt(n - 1) * nBig);
+      const semScaled = isqrt(semSqScaled);
+      const moeScaled = (Z95_NUM * semScaled) / Z95_DENOM;
+      const ciUpperScaled = mean * SQRT_SCALE + moeScaled;
+      return new Report(type, ciUpperScaled, 0, DURATION_SCALE * SQRT_SCALE);
     }
 
     default: {

package/src/runner.ts

@@ -7,7 +7,7 @@ const COMPLETE_VALUE = 100_00;
 
 const hr = process.hrtime.bigint.bind(process.hrtime);
 
-const sink = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
+const sink = new Int32Array(1);
 const consume = (value: unknown) => {
   let payload = 0;
   switch (typeof value) {
@@ -32,7 +32,7 @@ const consume = (value: unknown) => {
     default:
       payload = -1;
   }
-  Atomics.xor(sink, 0, payload);
+  sink[0] ^= payload;
 };
 
 const runSync = (run: Function, overhead: bigint) => {
@@ -518,13 +518,23 @@ export const benchmark = async <TContext, TInput>({
     }
 
     let i = 0;
-    let mean = 0n;
-    let m2 = 0n;
+    const WELFORD_SCALE = 1_000_000n;
+    let meanS = 0n;
+    let m2S = 0n;
     const outlierWindow: number[] = [];
     let skipped = 0;
     const maxSkipped = maxCycles * 10;
     let disableFiltering = false;
 
+    const absThScaled = BigInt(Math.round(absThreshold)) * WELFORD_SCALE;
+    const absThSq = absThScaled * absThScaled;
+    const REL_PRECISION = 1_000_000n;
+    const relThBigint = BigInt(Math.round(relThreshold * Number(REL_PRECISION)));
+    const relThSq = relThBigint * relThBigint;
+    const relPrecSq = REL_PRECISION * REL_PRECISION;
+    const Z95_SQ_NUM = 38416n;
+    const Z95_SQ_DENOM = 10000n;
+
     while (true) {
       if (i >= maxCycles) break;
       if (!disableFiltering && skipped >= maxSkipped) {
@@ -583,26 +593,28 @@ export const benchmark = async <TContext, TInput>({
       }
 
       const durationNumber = Number(sampleDuration);
-      pushWindow(outlierWindow, durationNumber, OUTLIER_WINDOW);
       if (!disableFiltering) {
         const { median, iqr } = medianAndIqr(outlierWindow);
+        pushWindow(outlierWindow, durationNumber, OUTLIER_WINDOW);
         const maxAllowed = median + OUTLIER_IQR_MULTIPLIER * iqr || Number.POSITIVE_INFINITY;
         if (outlierWindow.length >= 8 && durationNumber > maxAllowed && durationNumber - median > OUTLIER_ABS_THRESHOLD) {
           skipped++;
           continue;
         }
 
-        const meanNumber = Number(mean);
+        const meanNumber = Number(meanS / WELFORD_SCALE);
         if (i >= 8 && meanNumber > 0 && durationNumber > OUTLIER_MULTIPLIER * meanNumber && durationNumber - meanNumber > OUTLIER_ABS_THRESHOLD) {
           skipped++;
           continue;
         }
+      } else {
+        pushWindow(outlierWindow, durationNumber, OUTLIER_WINDOW);
       }
 
       durations[i++] = sampleDuration;
-      const delta = sampleDuration - mean;
-      mean += delta / BigInt(i);
-      m2 += delta * (sampleDuration - mean);
+      const deltaS = sampleDuration * WELFORD_SCALE - meanS;
+      meanS += deltaS / BigInt(i);
+      m2S += deltaS * (sampleDuration * WELFORD_SCALE - meanS);
 
       const progress = (i / maxCycles) * COMPLETE_VALUE;
       if (i % PROGRESS_STRIDE === 0) {
@@ -610,17 +622,11 @@ export const benchmark = async <TContext, TInput>({
       }
 
       if (i >= minCycles) {
-        const variance = Number(m2) / (i - 1);
-        const stddev = Math.sqrt(variance);
-        if (stddev <= Number(absThreshold)) {
-          break;
-        }
-
-        const meanNum = Number(mean);
-        const cov = stddev / (meanNum || 1);
-        if (cov <= relThreshold) {
-          break;
-        }
+        if (m2S <= absThSq * BigInt(i - 1)) break;
+        // RME convergence: Z95 * sem/mean <= relThreshold
+        // Z95^2 * m2S / (n*(n-1)*meanS^2) <= relThreshold^2
+        const ni = BigInt(i);
+        if (meanS !== 0n && Z95_SQ_NUM * m2S * relPrecSq <= relThSq * ni * (ni - 1n) * meanS * meanS * Z95_SQ_DENOM) break;
       }
     }
 

package/src/types.ts

@@ -107,7 +107,7 @@ export enum Control {
 }
 
 export const CONTROL_SLOTS = Object.values(Control).length / 2;
-export const DEFAULT_CYCLES = 1_000;
+export const DEFAULT_CYCLES = 10_000;
 export const Z95 = 1.96;
 export const DURATION_SCALE = 1000n;
 export const COMPLETE_VALUE = 100_00;

package/src/utils.ts

@@ -1,4 +1,16 @@
-import { transform } from '@swc/core';
+import { transform, parseSync } from '@swc/core';
+
+export const isqrt = (n: bigint): bigint => {
+  if (n < 0n) throw new RangeError('Square root of negative');
+  if (n < 2n) return n;
+  let x = n;
+  let y = (x + 1n) >> 1n;
+  while (y < x) {
+    x = y;
+    y = (x + n / x) >> 1n;
+  }
+  return x;
+};
 
 export const abs = (value: bigint) => {
   if (value < 0n) {
@@ -66,6 +78,49 @@ export class ScaledBigInt {
   }
 }
 
+const KNOWN_GLOBALS = new Set(Object.getOwnPropertyNames(globalThis));
+KNOWN_GLOBALS.add('arguments');
+
+function collectUnresolved(node: unknown, result: Set<string>) {
+  if (!node || typeof node !== 'object') return;
+  if (Array.isArray(node)) {
+    for (const item of node) collectUnresolved(item, result);
+    return;
+  }
+  const obj = node as Record<string, unknown>;
+  if (obj.type === 'Identifier' && obj.ctxt === 1 && typeof obj.value === 'string') {
+    result.add(obj.value);
+  }
+  for (const key of Object.keys(obj)) {
+    if (key === 'span') continue;
+    collectUnresolved(obj[key], result);
+  }
+}
+
+export function assertNoClosure(code: string, name: string): void {
+  let ast;
+  try {
+    ast = parseSync(`var __fn = ${code}`, { syntax: 'ecmascript', target: 'esnext' });
+  } catch {
+    return;
+  }
+  const unresolved = new Set<string>();
+  collectUnresolved(ast, unresolved);
+  for (const g of KNOWN_GLOBALS) unresolved.delete(g);
+  if (unresolved.size === 0) return;
+
+  const vars = [...unresolved].join(', ');
+  throw new Error(
+    `Benchmark "${name}" function references outer-scope variables: ${vars}\n\n` +
+      `Benchmark functions are serialized with .toString() and executed in an isolated\n` +
+      `worker thread. Closed-over variables from the original module scope are not\n` +
+      `available in the worker and will cause a ReferenceError at runtime.\n\n` +
+      `To fix this, move the referenced values into:\n` +
+      `  - "setup" function (returned value becomes the first argument of run/pre/post)\n` +
+      `  - "data" option (passed as the second argument of run/pre/post)`,
+  );
+}
+
 export const transpile = async (code: string): Promise<string> => {
   const output = await transform(code, {
     filename: 'benchmark.ts',

package/src/worker.ts

@@ -1,5 +1,5 @@
 import { workerData } from 'node:worker_threads';
-import { SourceTextModule, SyntheticModule, createContext } from 'node:vm';
+import { SourceTextModule, SyntheticModule } from 'node:vm';
 import { createRequire } from 'node:module';
 import { isAbsolute } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -41,7 +41,11 @@ const resolveSpecifier = (specifier: string) => {
   if (isAbsolute(specifier)) {
     return pathToFileURL(specifier).href;
   }
-  return requireFrom.resolve(specifier);
+  try {
+    return requireFrom.resolve(specifier);
+  } catch {
+    return specifier;
+  }
 };
 
 const source = `
@@ -52,11 +56,6 @@ export const run = ${serialize(runCode)};
 export const post = ${serialize(postCode)};
   `;
 
-const globals = Object.create(null);
-for (const k of Object.getOwnPropertyNames(globalThis)) {
-  globals[k] = (globalThis as any)[k];
-}
-const context = createContext(globals);
 const imports = new Map<string, SyntheticModule>();
 
 const createSyntheticModule = (moduleExports: unknown, exportNames: string[], identifier: string) => {
@@ -71,7 +70,7 @@ const createSyntheticModule = (moduleExports: unknown, exportNames: string[], id
         mod.setExport(name, (moduleExports as Record<string, unknown>)[name]);
       }
     },
-    { identifier, context },
+    { identifier },
   );
   return mod;
 };
@@ -111,7 +110,6 @@ const loadDynamicModule = async (target: string) => {
 };
 const mod = new SourceTextModule(source, {
   identifier: resolvedBenchmarkUrl,
-  context,
   initializeImportMeta(meta) {
     meta.url = resolvedBenchmarkUrl;
   },