overtake 1.0.5 → 1.1.0

package/src/runner.ts

@@ -1,23 +1,177 @@
+import { performance, PerformanceObserver } from 'node:perf_hooks';
 import { Options, Control } from './types.js';
+import { GCWatcher } from './gc-watcher.js';
+import { StepFn, MaybePromise } from './types.js';
 
 const COMPLETE_VALUE = 100_00;
 
+const hr = process.hrtime.bigint.bind(process.hrtime);
+
 const runSync = (run: Function) => {
   return (...args: unknown[]) => {
-    const start = process.hrtime.bigint();
+    const start = hr();
     run(...args);
-    return process.hrtime.bigint() - start;
+    return hr() - start;
   };
 };
 
 const runAsync = (run: Function) => {
   return async (...args: unknown[]) => {
-    const start = process.hrtime.bigint();
+    const start = hr();
     await run(...args);
-    return process.hrtime.bigint() - start;
+    return hr() - start;
   };
 };
 
+const TARGET_SAMPLE_NS = 1_000_000n; // aim for ~1ms per measured sample
+const MAX_BATCH = 1_048_576;
+const PROGRESS_STRIDE = 16;
+const GC_STRIDE = 32;
+const OUTLIER_MULTIPLIER = 4;
+const OUTLIER_IQR_MULTIPLIER = 3;
+const OUTLIER_WINDOW = 64;
+
+type GCEvent = { start: number; end: number };
+
+const collectSample = async <TContext, TInput>(
+  batchSize: number,
+  run: (ctx: TContext, data: TInput) => MaybePromise<bigint>,
+  pre: StepFn<TContext, TInput> | undefined,
+  post: StepFn<TContext, TInput> | undefined,
+  context: TContext,
+  data: TInput,
+) => {
+  let sampleDuration = 0n;
+  for (let b = 0; b < batchSize; b++) {
+    await pre?.(context, data);
+    sampleDuration += await run(context, data);
+    await post?.(context, data);
+  }
+  return sampleDuration / BigInt(batchSize);
+};
+
+const tuneParameters = async <TContext, TInput>({
+  initialBatch,
+  run,
+  pre,
+  post,
+  context,
+  data,
+  minCycles,
+  relThreshold,
+  maxCycles,
+}: {
+  initialBatch: number;
+  run: (ctx: TContext, data: TInput) => MaybePromise<bigint>;
+  pre?: StepFn<TContext, TInput>;
+  post?: StepFn<TContext, TInput>;
+  context: TContext;
+  data: TInput;
+  minCycles: number;
+  relThreshold: number;
+  maxCycles: number;
+}) => {
+  let batchSize = initialBatch;
+  let bestCv = Number.POSITIVE_INFINITY;
+  let bestBatch = batchSize;
+
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const samples: number[] = [];
+    const sampleCount = Math.min(8, maxCycles);
+    for (let s = 0; s < sampleCount; s++) {
+      const duration = await collectSample(batchSize, run, pre, post, context, data);
+      samples.push(Number(duration));
+    }
+    const mean = samples.reduce((acc, v) => acc + v, 0) / samples.length;
+    const variance = samples.reduce((acc, v) => acc + (v - mean) * (v - mean), 0) / Math.max(1, samples.length - 1);
+    const stddev = Math.sqrt(variance);
+    const cv = mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
+
+    if (cv < bestCv) {
+      bestCv = cv;
+      bestBatch = batchSize;
+    }
+
+    if (cv <= relThreshold || batchSize >= MAX_BATCH) {
+      break;
+    }
+    batchSize = Math.min(MAX_BATCH, batchSize * 2);
+  }
+
+  const tunedRel = bestCv < relThreshold ? Math.max(bestCv * 1.5, relThreshold * 0.5) : relThreshold;
+  const tunedMin = Math.min(maxCycles, Math.max(minCycles, Math.ceil(minCycles * Math.max(1, bestCv / (relThreshold || 1e-6)))));
+
+  return { batchSize: bestBatch, relThreshold: tunedRel, minCycles: tunedMin };
+};
+
+const createGCTracker = () => {
+  if (process.env.OVERTAKE_GC_OBSERVER !== '1') {
+    return null;
+  }
+  if (typeof PerformanceObserver === 'undefined') {
+    return null;
+  }
+
+  const events: GCEvent[] = [];
+  const observer = new PerformanceObserver((list) => {
+    for (const entry of list.getEntries()) {
+      events.push({ start: entry.startTime, end: entry.startTime + entry.duration });
+    }
+  });
+
+  try {
+    observer.observe({ entryTypes: ['gc'] });
+  } catch {
+    return null;
+  }
+
+  const overlaps = (start: number, end: number) => {
+    let noisy = false;
+    for (let i = events.length - 1; i >= 0; i--) {
+      const event = events[i];
+      if (event.end < start - 5_000) {
+        events.splice(i, 1);
+        continue;
+      }
+      if (event.start <= end && event.end >= start) {
+        noisy = true;
+      }
+    }
+    return noisy;
+  };
+
+  const dispose = () => observer.disconnect();
+
+  return { overlaps, dispose };
+};
+
+const pushWindow = (arr: number[], value: number, cap: number) => {
+  if (arr.length === cap) {
+    arr.shift();
+  }
+  arr.push(value);
+};
+
+const medianAndIqr = (arr: number[]) => {
+  if (arr.length === 0) return { median: 0, iqr: 0 };
+  const sorted = [...arr].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+  const q1Idx = Math.floor(sorted.length * 0.25);
+  const q3Idx = Math.floor(sorted.length * 0.75);
+  const q1 = sorted[q1Idx];
+  const q3 = sorted[q3Idx];
+  return { median, iqr: q3 - q1 };
+};
+
+const windowCv = (arr: number[]) => {
+  if (arr.length < 2) return Number.POSITIVE_INFINITY;
+  const mean = arr.reduce((a, v) => a + v, 0) / arr.length;
+  const variance = arr.reduce((a, v) => a + (v - mean) * (v - mean), 0) / (arr.length - 1);
+  const stddev = Math.sqrt(variance);
+  return mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
+};
+
 export const benchmark = async <TContext, TInput>({
   setup,
   teardown,
@@ -30,6 +184,7 @@ export const benchmark = async <TContext, TInput>({
   minCycles,
   absThreshold,
   relThreshold,
+  gcObserver = false,
 
   durationsSAB,
   controlSAB,
@@ -43,47 +198,133 @@ export const benchmark = async <TContext, TInput>({
 
   const context = (await setup?.()) as TContext;
   const maxCycles = durations.length;
+  const gcWatcher = new GCWatcher();
+  const gcTracker = gcObserver ? createGCTracker() : null;
 
   try {
+    // classify sync/async and capture initial duration
     await pre?.(context, data!);
-    const result = runRaw(context, data!);
+    const probeStart = hr();
+    const probeResult = runRaw(context, data!);
+    const isAsync = probeResult instanceof Promise;
+    if (isAsync) {
+      await probeResult;
+    }
+    const durationProbe = hr() - probeStart;
     await post?.(context, data!);
-    global.gc?.();
-    global.gc?.();
 
-    const run = result instanceof Promise ? runAsync(runRaw) : runSync(runRaw);
-    const start = Date.now();
-    while (Date.now() - start < 1_000) {
-      Math.sqrt(Math.random());
+    const run = isAsync ? runAsync(runRaw) : runSync(runRaw);
+
+    // choose batch size to amortize timer overhead
+    const durationPerRun = durationProbe === 0n ? 1n : durationProbe;
+    const suggestedBatch = Number(TARGET_SAMPLE_NS / durationPerRun);
+    const initialBatchSize = Math.min(MAX_BATCH, Math.max(1, suggestedBatch));
+
+    // auto-tune based on warmup samples
+    const tuned = await tuneParameters({
+      initialBatch: initialBatchSize,
+      run,
+      pre,
+      post,
+      context,
+      data: data as TInput,
+      minCycles,
+      relThreshold,
+      maxCycles,
+    });
+    let batchSize = tuned.batchSize;
+    minCycles = tuned.minCycles;
+    relThreshold = tuned.relThreshold;
+
+    // warmup: run until requested cycles, adapt if unstable
+    const warmupStart = Date.now();
+    let warmupRemaining = warmupCycles;
+    const warmupWindow: number[] = [];
+    const warmupCap = Math.max(warmupCycles, Math.min(maxCycles, warmupCycles * 4 || 1000));
+
+    while (Date.now() - warmupStart < 1_000 && warmupRemaining > 0) {
+      const start = hr();
+      await pre?.(context, data!);
+      await run(context, data);
+      await post?.(context, data!);
+      pushWindow(warmupWindow, Number(hr() - start), warmupCap);
+      warmupRemaining--;
     }
-    for (let i = 0; i < warmupCycles; i++) {
+    let warmupDone = 0;
+    while (warmupDone < warmupRemaining) {
+      const start = hr();
       await pre?.(context, data!);
       await run(context, data);
       await post?.(context, data!);
-      global.gc?.();
-      global.gc?.();
+      pushWindow(warmupWindow, Number(hr() - start), warmupCap);
+      warmupDone++;
+      if (global.gc && warmupDone % GC_STRIDE === 0) {
+        global.gc();
+      }
+    }
+    while (warmupWindow.length >= 8 && warmupWindow.length < warmupCap) {
+      const cv = windowCv(warmupWindow);
+      if (cv <= relThreshold * 2) {
+        break;
+      }
+      const start = hr();
+      await pre?.(context, data!);
+      await run(context, data);
+      await post?.(context, data!);
+      pushWindow(warmupWindow, Number(hr() - start), warmupCap);
     }
 
     let i = 0;
     let mean = 0n;
     let m2 = 0n;
+    const outlierWindow: number[] = [];
 
     while (true) {
       if (i >= maxCycles) break;
 
-      await pre?.(context, data!);
-      const duration = await run(context, data);
-      await post?.(context, data!);
-      global.gc?.();
-      global.gc?.();
+      const gcMarker = gcWatcher.start();
+      const sampleStart = performance.now();
+      let sampleDuration = 0n;
+      for (let b = 0; b < batchSize; b++) {
+        await pre?.(context, data!);
+        sampleDuration += await run(context, data);
+        await post?.(context, data!);
+        if (global.gc && (i + b) % GC_STRIDE === 0) {
+          global.gc();
+        }
+      }
 
-      durations[i++] = duration;
-      const delta = duration - mean;
+      // normalize by batch size
+      sampleDuration /= BigInt(batchSize);
+
+      const sampleEnd = performance.now();
+      const gcNoise = gcWatcher.seen(gcMarker) || (gcTracker?.overlaps(sampleStart, sampleEnd) ?? false);
+      if (gcNoise) {
+        continue;
+      }
+
+      const durationNumber = Number(sampleDuration);
+      pushWindow(outlierWindow, durationNumber, OUTLIER_WINDOW);
+      const { median, iqr } = medianAndIqr(outlierWindow);
+      const maxAllowed = median + OUTLIER_IQR_MULTIPLIER * iqr || Number.POSITIVE_INFINITY;
+      if (outlierWindow.length >= 8 && durationNumber > maxAllowed) {
+        continue;
+      }
+
+      const meanNumber = Number(mean);
+      if (i >= 8 && meanNumber > 0 && durationNumber > OUTLIER_MULTIPLIER * meanNumber) {
+        continue;
+      }
+
+      durations[i++] = sampleDuration;
+      const delta = sampleDuration - mean;
       mean += delta / BigInt(i);
-      m2 += delta * (duration - mean);
+      m2 += delta * (sampleDuration - mean);
 
       const progress = Math.max(i / maxCycles) * COMPLETE_VALUE;
-      control[Control.PROGRESS] = progress;
+      if (i % PROGRESS_STRIDE === 0) {
+        control[Control.PROGRESS] = progress;
+      }
 
       if (i >= minCycles) {
         const variance = Number(m2) / (i - 1);
@@ -106,6 +347,7 @@ export const benchmark = async <TContext, TInput>({
     console.error(e && typeof e === 'object' && 'stack' in e ? e.stack : e);
     control[Control.COMPLETE] = 1;
   } finally {
+    gcTracker?.dispose?.();
     try {
       await teardown?.(context);
     } catch (e) {

package/src/types.ts

@@ -33,9 +33,12 @@ export interface BenchmarkOptions {
   minCycles?: number;
   absThreshold?: number; // ns
   relThreshold?: number; // %
+  gcObserver?: boolean;
+  baseUrl?: string;
 }
 
 export interface RunOptions<TContext, TInput> {
+  baseUrl?: string;
   setup?: SetupFn<TContext>;
   teardown?: TeardownFn<TContext>;
   pre?: StepFn<TContext, TInput>;
@@ -45,6 +48,7 @@ export interface RunOptions<TContext, TInput> {
 }
 
 export interface WorkerOptions extends Required<BenchmarkOptions> {
+  baseUrl: string;
   setupCode?: string;
   teardownCode?: string;
   preCode?: string;

package/src/utils.ts

@@ -1,3 +1,5 @@
+import { transform } from '@swc/core';
+
 export const abs = (value: bigint) => {
   if (value < 0n) {
     return -value;
@@ -63,3 +65,21 @@ export class ScaledBigInt {
     return Number(div(this.value, this.scale));
   }
 }
+
+export const transpile = async (code: string): Promise<string> => {
+  const output = await transform(code, {
+    filename: 'benchmark.ts',
+    jsc: {
+      parser: {
+        syntax: 'typescript',
+        tsx: false,
+        dynamicImport: true,
+      },
+      target: 'esnext',
+    },
+    module: {
+      type: 'es6',
+    },
+  });
+  return output.code;
+};

package/src/worker.ts

@@ -1,8 +1,12 @@
 import { workerData } from 'node:worker_threads';
+import { SourceTextModule, SyntheticModule, createContext } from 'node:vm';
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
 import { benchmark } from './runner.js';
-import { SetupFn, TeardownFn, StepFn, WorkerOptions } from './types.js';
+import { WorkerOptions } from './types.js';
 
 const {
+  baseUrl,
   setupCode,
   teardownCode,
   preCode,
@@ -14,19 +18,66 @@ const {
   minCycles,
   absThreshold,
   relThreshold,
+  gcObserver = true,
 
   durationsSAB,
   controlSAB,
 }: WorkerOptions = workerData;
 
-const setup: SetupFn<unknown> = setupCode && Function(`return ${setupCode};`)();
-const teardown: TeardownFn<unknown> = teardownCode && Function(`return ${teardownCode};`)();
+const serialize = (code?: string) => (code ? code : '() => {}');
 
-const pre: StepFn<unknown, unknown> = preCode && Function(`return ${preCode};`)();
-const run: StepFn<unknown, unknown> = runCode && Function(`return ${runCode};`)();
-const post: StepFn<unknown, unknown> = postCode && Function(`return ${postCode};`)();
+const source = `
+export const setup = ${serialize(setupCode)};
+export const teardown = ${serialize(teardownCode)};
+export const pre = ${serialize(preCode)};
+export const run = ${serialize(runCode)};
+export const post = ${serialize(postCode)};
+  `;
 
-export const exitCode = await benchmark({
+const context = createContext({ console, Buffer });
+const imports = new Map<string, SyntheticModule>();
+const mod = new SourceTextModule(source, {
+  identifier: baseUrl,
+  context,
+  initializeImportMeta(meta) {
+    meta.url = baseUrl;
+  },
+  importModuleDynamically(specifier, referencingModule) {
+    const base = referencingModule.identifier ?? baseUrl;
+    const resolveFrom = createRequire(fileURLToPath(base));
+    return import(resolveFrom.resolve(specifier));
+  },
+});
+
+await mod.link(async (specifier, referencingModule) => {
+  const base = referencingModule.identifier ?? baseUrl;
+  const resolveFrom = createRequire(fileURLToPath(base));
+  const target = resolveFrom.resolve(specifier);
+  const cached = imports.get(target);
+  if (cached) return cached;
+
+  const importedModule = await import(target);
+  const exportNames = Object.keys(importedModule);
+  const imported = new SyntheticModule(
+    exportNames,
+    () => {
+      exportNames.forEach((key) => imported.setExport(key, importedModule[key]));
+    },
+    { identifier: target, context: referencingModule.context },
+  );
+  imports.set(target, imported);
+  return imported;
+});
+
+await mod.evaluate();
+const { setup, teardown, pre, run, post } = mod.namespace as any;
+
+if (!run) {
+  throw new Error('Benchmark run function is required');
+}
+
+process.exitCode = await benchmark({
+  baseUrl,
   setup,
   teardown,
   pre,
@@ -38,9 +89,8 @@ export const exitCode = await benchmark({
   minCycles,
   absThreshold,
   relThreshold,
+  gcObserver,
 
   durationsSAB,
   controlSAB,
 });
-
-process.exit(exitCode);