overtake 0.1.1 → 1.0.0-rc.0

package/src/reporter.ts

@@ -0,0 +1,139 @@
+import { div, max, divs } from './utils.js';
+import { ReportType } from './types.js';
+
+const units = [
+  { unit: 'ns', factor: 1 },
+  { unit: 'µs', factor: 1e3 },
+  { unit: 'ms', factor: 1e6 },
+  { unit: 's', factor: 1e9 },
+  { unit: 'm', factor: 60 * 1e9 },
+  { unit: 'h', factor: 3600 * 1e9 },
+] as const;
+
+function smartFixed(n: number): string {
+  return n.toLocaleString(undefined, {
+    minimumFractionDigits: 0,
+    maximumFractionDigits: 2,
+    useGrouping: false,
+  });
+}
+export class Report {
+  constructor(
+    public readonly type: ReportType,
+    public readonly value: bigint,
+    public readonly uncertainty: number = 0,
+    public readonly scale: bigint = 1n,
+  ) {}
+  valueOf() {
+    return Number(div(this.value, this.scale));
+  }
+  toString() {
+    const uncertainty = this.uncertainty ? ` ± ${smartFixed(this.uncertainty)}%` : '';
+
+    const value = this.valueOf();
+    if (this.type === 'ops') {
+      return `${smartFixed(value)} ops/s${uncertainty}`;
+    }
+    let display = value;
+    let unit = 'ns';
+
+    for (const { unit: u, factor } of units) {
+      const candidate = value / factor;
+      if (candidate < 1000) {
+        display = candidate;
+        unit = u;
+        break;
+      }
+    }
+    return `${smartFixed(display)} ${unit}${uncertainty}`;
+  }
+}
+
+export const createReport = (durations: BigUint64Array, type: ReportType): Report => {
+  const n = durations.length;
+  if (n === 0) {
+    return new Report(type, 0n);
+  }
+  switch (type) {
+    case 'min': {
+      return new Report(type, durations[0]);
+    }
+    case 'max': {
+      return new Report(type, durations[n - 1]);
+    }
+    case 'median': {
+      const mid = Math.floor(n / 2);
+      const med = n % 2 === 0 ? (durations[mid - 1] + durations[mid]) / 2n : durations[mid];
+      return new Report(type, med);
+    }
+
+    case 'mode': {
+      const freq = new Map<bigint, bigint>();
+      let maxCount = 0n;
+      let modeVal = durations[0];
+      for (const d of durations) {
+        const count = (freq.get(d) || 0n) + 1n;
+        freq.set(d, count);
+        if (count > maxCount) {
+          maxCount = count;
+          modeVal = d;
+        }
+      }
+      let lower = modeVal;
+      let upper = modeVal;
+      const firstIdx = durations.indexOf(modeVal);
+      const lastIdx = durations.lastIndexOf(modeVal);
+      if (firstIdx > 0) lower = durations[firstIdx - 1];
+      if (lastIdx < n - 1) upper = durations[lastIdx + 1];
+      const gap = max(modeVal - lower, upper - modeVal);
+      const uncertainty = modeVal > 0 ? Number(((gap / 2n) * 100n) / modeVal) : 0;
+      return new Report(type, modeVal, uncertainty);
+    }
+
+    case 'ops': {
+      let sum = 0n;
+      for (const duration of durations) {
+        sum += duration;
+      }
+      const avgNs = sum / BigInt(n);
+      const nsPerSec = 1_000_000_000n;
+      const raw = Number(nsPerSec) / Number(avgNs);
+      const extra = raw < 1 ? Math.ceil(-Math.log10(raw)) : 0;
+
+      const exp = raw > 100 ? 0 : 2 + extra;
+
+      const scale = 10n ** BigInt(exp);
+
+      const value = avgNs > 0n ? (nsPerSec * scale) / avgNs : 0n;
+      const deviation = durations[n - 1] - durations[0];
+      const uncertainty = avgNs > 0 ? Number(div(deviation * scale, 2n * avgNs)) : 0;
+      return new Report(type, value, uncertainty, scale);
+    }
+    case 'mean': {
+      let sum = 0n;
+      for (const duration of durations) {
+        sum += duration;
+      }
+      const value = divs(sum, BigInt(n), 1n);
+      return new Report(type, value);
+    }
+
+    default: {
+      const p = Number(type.slice(1));
+      if (p === 0) {
+        return new Report(type, durations[0]);
+      }
+      if (p === 100) {
+        return new Report(type, durations[n - 1]);
+      }
+      const idx = Math.ceil((p / 100) * n) - 1;
+      const value = durations[Math.min(Math.max(idx, 0), n - 1)];
+      const prev = idx > 0 ? durations[idx - 1] : value;
+      const next = idx < n - 1 ? durations[idx + 1] : value;
+      const gap = max(value - prev, next - value);
+      const uncertainty = value > 0 ? Number(div(divs(gap, 2n, 100_00n), value)) / 100 : 0;
+
+      return new Report(type, value, uncertainty);
+    }
+  }
+};

package/src/runner.ts

@@ -0,0 +1,111 @@
+import { Options, Control } from './types.js';
+
+const COMPLETE_VALUE = 100_00;
+
+const runSync = (run: Function) => {
+  return (...args: unknown[]) => {
+    const start = process.hrtime.bigint();
+    run(...args);
+    return process.hrtime.bigint() - start;
+  };
+};
+
+const runAsync = (run: Function) => {
+  return async (...args: unknown[]) => {
+    const start = process.hrtime.bigint();
+    await run(...args);
+    return process.hrtime.bigint() - start;
+  };
+};
+
+export const benchmark = async <TContext, TInput>({
+  setup,
+  teardown,
+  pre,
+  run: runRaw,
+  post,
+  data,
+
+  warmupCycles,
+  minCycles,
+  absThreshold,
+  relThreshold,
+
+  durationsSAB,
+  controlSAB,
+}: Required<Options<TContext, TInput>>) => {
+  const durations = new BigUint64Array(durationsSAB);
+  const control = new Int32Array(controlSAB);
+
+  control[Control.INDEX] = 0;
+  control[Control.PROGRESS] = 0;
+  control[Control.COMPLETE] = 255;
+
+  const context = (await setup?.()) as TContext;
+  const maxCycles = durations.length;
+
+  try {
+    await pre?.(context, data!);
+    const result = runRaw(context, data!);
+    await post?.(context, data!);
+    const run = result instanceof Promise ? runAsync(runRaw) : runSync(runRaw);
+    const start = Date.now();
+    while (Date.now() - start < 1_000) {
+      Math.sqrt(Math.random());
+    }
+    for (let i = 0; i < warmupCycles; i++) {
+      await pre?.(context, data!);
+      await run(context, data);
+      await post?.(context, data!);
+    }
+
+    let i = 0;
+    let mean = 0n;
+    let m2 = 0n;
+
+    while (true) {
+      if (i >= maxCycles) break;
+
+      await pre?.(context, data!);
+      const duration = await run(context, data);
+      await post?.(context, data!);
+
+      durations[i++] = duration;
+      const delta = duration - mean;
+      mean += delta / BigInt(i);
+      m2 += delta * (duration - mean);
+
+      const progress = Math.max(i / maxCycles) * COMPLETE_VALUE;
+      control[Control.PROGRESS] = progress;
+
+      if (i >= minCycles) {
+        const variance = Number(m2) / (i - 1);
+        const stddev = Math.sqrt(variance);
+        if (stddev <= Number(absThreshold)) {
+          break;
+        }
+
+        const meanNum = Number(mean);
+        const cov = stddev / (meanNum || 1);
+        if (cov <= relThreshold) {
+          break;
+        }
+      }
+    }
+
+    control[Control.INDEX] = i;
+    control[Control.COMPLETE] = 0;
+  } catch (e) {
+    console.error(e && typeof e === 'object' && 'stack' in e ? e.stack : e);
+    control[Control.COMPLETE] = 1;
+  } finally {
+    try {
+      await teardown?.(context);
+    } catch (e) {
+      control[Control.COMPLETE] = 2;
+      console.error(e && typeof e === 'object' && 'stack' in e ? e.stack : e);
+    }
+  }
+
+  return control[Control.COMPLETE];
+};

package/src/types.ts

@@ -0,0 +1,72 @@
+export type MaybePromise<T> = Promise<T> | PromiseLike<T> | T;
+
+export interface SetupFn<TContext> {
+  (): MaybePromise<TContext>;
+}
+
+export interface TeardownFn<TContext> {
+  (ctx: TContext): MaybePromise<void>;
+}
+
+export interface StepFn<TContext, TInput> {
+  (ctx: TContext, input: TInput): MaybePromise<void>;
+}
+
+export interface FeedFn<TInput> {
+  (): MaybePromise<TInput>;
+}
+
+type _Sequence<To extends number, R extends unknown[]> = R['length'] extends To ? R[number] : _Sequence<To, [R['length'], ...R]>;
+export type Sequence<To extends number> = number extends To ? number : _Sequence<To, []>;
+export type Between<From extends number, To extends number> = Exclude<Sequence<To>, Sequence<From>>;
+
+export type ReportType = 'ops' | 'min' | 'max' | 'mean' | 'median' | 'mode' | `p${Between<1, 100>}`;
+export type ReportTypeList = readonly ReportType[];
+export const REPORT_TYPES: ReportTypeList = Array.from({ length: 99 }, (_, idx) => `p${idx + 1}` as ReportType).concat(['ops', 'mean', 'min', 'max', 'median', 'mode']);
+
+export interface ReportOptions<R extends ReportTypeList> {
+  reportTypes: R;
+}
+
+export interface BenchmarkOptions {
+  warmupCycles?: number;
+  minCycles?: number;
+  absThreshold?: number; // ns
+  relThreshold?: number; // %
+}
+
+export interface RunOptions<TContext, TInput> {
+  setup?: SetupFn<TContext>;
+  teardown?: TeardownFn<TContext>;
+  pre?: StepFn<TContext, TInput>;
+  run: StepFn<TContext, TInput>;
+  post?: StepFn<TContext, TInput>;
+  data?: TInput;
+}
+
+export interface WorkerOptions extends Required<BenchmarkOptions> {
+  setupCode?: string;
+  teardownCode?: string;
+  preCode?: string;
+  runCode: string;
+  postCode?: string;
+  data?: unknown;
+
+  durationsSAB: SharedArrayBuffer;
+  controlSAB: SharedArrayBuffer;
+}
+
+export interface Options<TContext, TInput> extends RunOptions<TContext, TInput>, BenchmarkOptions {
+  durationsSAB: SharedArrayBuffer;
+  controlSAB: SharedArrayBuffer;
+}
+
+export enum Control {
+  INDEX,
+  PROGRESS,
+  COMPLETE,
+}
+
+export const CONTROL_SLOTS = Object.values(Control).length / 2;
+export const DEFAULT_CYCLES = 1_000;
+export const Z95 = 1.96;

package/src/utils.ts

@@ -0,0 +1,65 @@
+export const abs = (value: bigint) => {
+  if (value < 0n) {
+    return -value;
+  }
+  return value;
+};
+export const cmp = (a: bigint | number, b: bigint | number): number => {
+  if (a > b) {
+    return 1;
+  }
+  if (a < b) {
+    return -1;
+  }
+  return 0;
+};
+
+export const max = (a: bigint, b: bigint) => {
+  if (a > b) {
+    return a;
+  }
+  return b;
+};
+
+export const divMod = (a: bigint, b: bigint) => {
+  return { quotient: a / b, remainder: a % b };
+};
+
+export function div(a: bigint, b: bigint, decimals: number = 2): string {
+  if (b === 0n) throw new RangeError('Division by zero');
+  const scale = 10n ** BigInt(decimals);
+  const scaled = (a * scale) / b;
+  const intPart = scaled / scale;
+  const fracPart = scaled % scale;
+  return `${intPart}.${fracPart.toString().padStart(decimals, '0')}`;
+}
+
+export function divs(a: bigint, b: bigint, scale: bigint): bigint {
+  if (b === 0n) throw new RangeError('Division by zero');
+  return (a * scale) / b;
+}
+
+export class ScaledBigInt {
+  constructor(
+    public value: bigint,
+    public scale: bigint,
+  ) {}
+  add(value: bigint) {
+    this.value += value * this.scale;
+  }
+  sub(value: bigint) {
+    this.value -= value * this.scale;
+  }
+  div(value: bigint) {
+    this.value /= value;
+  }
+  mul(value: bigint) {
+    this.value *= value;
+  }
+  unscale() {
+    return this.value / this.value;
+  }
+  number() {
+    return Number(div(this.value, this.scale));
+  }
+}

package/src/worker.ts

@@ -0,0 +1,46 @@
+import { workerData } from 'node:worker_threads';
+import { benchmark } from './runner.js';
+import { SetupFn, TeardownFn, StepFn, WorkerOptions } from './types.js';
+
+const {
+  setupCode,
+  teardownCode,
+  preCode,
+  runCode,
+  postCode,
+  data,
+
+  warmupCycles,
+  minCycles,
+  absThreshold,
+  relThreshold,
+
+  durationsSAB,
+  controlSAB,
+}: WorkerOptions = workerData;
+
+const setup: SetupFn<unknown> = setupCode && Function(`return ${setupCode};`)();
+const teardown: TeardownFn<unknown> = teardownCode && Function(`return ${teardownCode};`)();
+
+const pre: StepFn<unknown, unknown> = preCode && Function(`return ${preCode};`)();
+const run: StepFn<unknown, unknown> = runCode && Function(`return ${runCode};`)();
+const post: StepFn<unknown, unknown> = postCode && Function(`return ${postCode};`)();
+
+const exitCode = await benchmark({
+  setup,
+  teardown,
+  pre,
+  run,
+  post,
+  data,
+
+  warmupCycles,
+  minCycles,
+  absThreshold,
+  relThreshold,
+
+  durationsSAB,
+  controlSAB,
+});
+
+process.exit(exitCode);

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "target": "ESNext",
+    "module": "NodeNext",
+    "declaration": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "allowJs": true,
+    "moduleResolution": "NodeNext",
+    "lib": ["ESNext"],
+    "rootDir": "src",
+    "outDir": "build"
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "**/*.tmp.ts"]
+}

package/cli.js

@@ -1,70 +0,0 @@
-#!/usr/bin/env -S node --no-warnings
-
-import { Command, Option } from 'commander';
-import Path from 'path';
-import { glob } from 'glob';
-import { load, createScript, benchmark, setup, teardown, measure, perform, run, defaultReporter, allowedFields } from './index.js';
-import packageJson from './package.json' assert { type: 'json' };
-
-const commands = new Command();
-
-commands.name('overtake').description(packageJson.description).version(packageJson.version, '-v, --version');
-
-commands
-  .argument('[files...]', 'File paths or path patterns to search benchmark scripts')
-  .option('-i, --inline [inline]', 'Inline benchmark.', (value, previous) => previous.concat([value]), [])
-  .option('-c, --count [count]', 'Perform count for inline benchmark.', (v) => parseInt(v))
-  .addOption(
-    new Option('-f, --fields [fields]', `Comma separated list of fields to report. Allowed values are: ${allowedFields}.`)
-      .default(['med', 'p95', 'p99', 'sum:total', 'count'])
-      .argParser((fields) =>
-        fields.split(',').filter((field) => {
-          if (!allowedFields.includes(field)) {
-            console.error(`Invalid field name: ${field}. Allowed values are: ${allowedFields.join(', ')}.`);
-            process.exit(1);
-          }
-          return true;
-        }),
-      ),
-  )
-  .action(async (patterns, { count = 1, inline, fields }) => {
-    Object.assign(globalThis, { benchmark, setup, teardown, measure, perform });
-
-    const foundFiles = await glob(patterns);
-    if (!foundFiles.length) {
-      console.error(`No files found with patterns ${patterns.join(', ')}`);
-      process.exit(1);
-    }
-    const files = [...new Set(foundFiles.map((filename) => Path.resolve(filename)).filter(Boolean))];
-
-    const scripts = [];
-    if (inline.length) {
-      const inlineScript = await createScript('', () => {
-        benchmark('', () => {
-          inline.forEach((code) => {
-            measure(code, `() => () => { ${code} }`);
-          });
-          perform('', count);
-        });
-      });
-      scripts.push(inlineScript);
-    }
-
-    for (const file of files) {
-      const filename = Path.resolve(file);
-      const script = await load(filename);
-      scripts.push(script);
-    }
-
-    await run(scripts, defaultReporter, fields);
-  });
-
-commands.on('--help', () => {
-  console.log('');
-  console.log('Examples:');
-  console.log('  $ overtake **/__benchmarks__/*.js');
-  console.log('  $ overtake -i "class A{}" -i "function A(){}" -i "const A = () => {}" -c 1000000');
-  console.log('  $ overtake -v');
-});
-
-commands.parse(process.argv);

package/index.d.ts

@@ -1,56 +0,0 @@
-declare global {
-  type CanBePromise<T> = Promise<T> | T;
-
-  interface Report {
-    type: string;
-    success: boolean;
-    count: number;
-    min: number;
-    max: number;
-    sum: number;
-    avg: number;
-    mode: number;
-    p1: number;
-    p5: number;
-    p20: number;
-    p33: number;
-    p50: number;
-    med: number;
-    p66: number;
-    p80: number;
-    p90: number;
-    p95: number;
-    p99: number;
-    setup: number;
-    init: number;
-    cycles: number;
-    teardown: number;
-    total: number;
-  }
-
-  type MeasureInitResult = CanBePromise<() => void>;
-
-  function measure(title: string, init: () => MeasureInitResult): void;
-  function measure(title: string, init: (next: () => void) => MeasureInitResult): void;
-  function measure<C>(title: string, init: (context: C, next: () => void) => MeasureInitResult): void;
-  function measure<C, A>(title: string, init: (context: C, args: A, next: () => void) => MeasureInitResult): void;
-
-  function perform<A>(title: string, counter: number, args: A): void;
-
-  function setup<C>(init: () => CanBePromise<C>): void;
-
-  function teardown<C>(teardown: (context: C) => CanBePromise<void>): void;
-
-  interface Suite {
-    title: string;
-    setup: typeof setup;
-    teardown: typeof teardown;
-    measures: typeof measure[];
-    performs: typeof perform[];
-    init: () => void;
-  }
-
-  function benchmark(title: string, init: () => void): void;
-
-  function script(filename): Promise<Suite[]>;
-}