overtake 1.0.4 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +68 -25
  2. package/bin/overtake.js +1 -1
  3. package/build/cli.cjs +44 -33
  4. package/build/cli.cjs.map +1 -1
  5. package/build/cli.js +43 -32
  6. package/build/cli.js.map +1 -1
  7. package/build/executor.cjs +6 -3
  8. package/build/executor.cjs.map +1 -1
  9. package/build/executor.d.ts +3 -2
  10. package/build/executor.js +6 -3
  11. package/build/executor.js.map +1 -1
  12. package/build/gc-watcher.cjs +31 -0
  13. package/build/gc-watcher.cjs.map +1 -0
  14. package/build/gc-watcher.d.ts +9 -0
  15. package/build/gc-watcher.js +21 -0
  16. package/build/gc-watcher.js.map +1 -0
  17. package/build/index.cjs +9 -1
  18. package/build/index.cjs.map +1 -1
  19. package/build/index.d.ts +1 -1
  20. package/build/index.js +9 -1
  21. package/build/index.js.map +1 -1
  22. package/build/runner.cjs +226 -18
  23. package/build/runner.cjs.map +1 -1
  24. package/build/runner.d.ts +1 -1
  25. package/build/runner.js +226 -18
  26. package/build/runner.js.map +1 -1
  27. package/build/types.cjs.map +1 -1
  28. package/build/types.d.ts +4 -0
  29. package/build/types.js.map +1 -1
  30. package/build/utils.cjs +21 -0
  31. package/build/utils.cjs.map +1 -1
  32. package/build/utils.d.ts +1 -0
  33. package/build/utils.js +18 -0
  34. package/build/utils.js.map +1 -1
  35. package/build/worker.cjs +95 -8
  36. package/build/worker.cjs.map +1 -1
  37. package/build/worker.js +54 -8
  38. package/build/worker.js.map +1 -1
  39. package/examples/accuracy.ts +54 -0
  40. package/examples/complete.ts +3 -3
  41. package/examples/custom-reports.ts +21 -0
  42. package/examples/imports.ts +47 -0
  43. package/examples/quick-start.ts +10 -9
  44. package/package.json +10 -9
  45. package/src/cli.ts +46 -30
  46. package/src/executor.ts +8 -2
  47. package/src/gc-watcher.ts +23 -0
  48. package/src/index.ts +11 -0
  49. package/src/runner.ts +266 -17
  50. package/src/types.ts +4 -0
  51. package/src/utils.ts +20 -0
  52. package/src/worker.ts +59 -9
  53. package/CLAUDE.md +0 -145
  54. package/examples/array-copy.ts +0 -17
  55. package/examples/object-merge.ts +0 -41
  56. package/examples/serialization.ts +0 -22
@@ -1,16 +1,17 @@
1
- // run using the following command
2
- // npx overtake examples/quick-start.ts
1
+ // Minimal example - comparing array sum algorithms
2
+ // Run: npx overtake examples/quick-start.ts
3
+ // Import overtake to support global types
4
+ import 'overtake';
3
5
 
4
- const sumSuite = benchmark('1M array', () => Array.from({ length: 1_000_000 }, (_, idx) => idx));
6
+ const sumBenchmark = benchmark('1M numbers', () => Array.from({ length: 1_000_000 }, (_, index) => index));
5
7
 
6
- sumSuite.target('for loop').measure('sum', (_, input) => {
7
- const n = input.length;
8
+ sumBenchmark.target('for loop').measure('sum', (_, numbers) => {
8
9
  let sum = 0;
9
- for (let i = 0; i < n; i++) {
10
- sum += input[i];
10
+ for (let i = 0; i < numbers.length; i++) {
11
+ sum += numbers[i];
11
12
  }
12
13
  });
13
14
 
14
- sumSuite.target('reduce').measure('sum', (_, input) => {
15
- input.reduce((a, b) => a + b, 0);
15
+ sumBenchmark.target('reduce').measure('sum', (_, numbers) => {
16
+ numbers.reduce((accumulator, current) => accumulator + current, 0);
16
17
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "overtake",
3
- "version": "1.0.4",
3
+ "version": "1.1.0",
4
4
  "description": "NodeJS performance benchmark",
5
5
  "type": "module",
6
6
  "types": "build/index.d.ts",
@@ -41,23 +41,24 @@
41
41
  },
42
42
  "homepage": "https://github.com/3axap4eHko/overtake#readme",
43
43
  "devDependencies": {
44
- "@jest/globals": "^30.0.5",
44
+ "@jest/globals": "^30.2.0",
45
45
  "@swc/jest": "^0.2.39",
46
46
  "@types/async": "^3.2.25",
47
47
  "@types/jest": "^30.0.0",
48
- "@types/node": "^24.3.0",
48
+ "@types/node": "^24.10.1",
49
49
  "husky": "^9.1.7",
50
- "inop": "^0.7.9",
51
- "jest": "^30.0.5",
50
+ "inop": "^0.8.0",
51
+ "jest": "^30.2.0",
52
+ "overtake": "^1.0.5",
52
53
  "prettier": "^3.6.2",
53
54
  "pretty-quick": "^4.2.2",
54
- "typescript": "^5.9.2"
55
+ "typescript": "^5.9.3"
55
56
  },
56
57
  "dependencies": {
57
- "@swc/core": "^1.13.5",
58
+ "@swc/core": "^1.15.2",
58
59
  "async": "^3.2.6",
59
- "commander": "^14.0.0",
60
- "glob": "^11.0.3"
60
+ "commander": "^14.0.2",
61
+ "glob": "^13.0.0"
61
62
  },
62
63
  "scripts": {
63
64
  "build": "rm -rf build && inop src build -i __tests__ -i *.tmp.ts && tsc --declaration --emitDeclarationOnly",
package/src/cli.ts CHANGED
@@ -1,10 +1,11 @@
1
1
  import { createRequire, Module } from 'node:module';
2
+ import { fileURLToPath, pathToFileURL } from 'node:url';
2
3
  import { SyntheticModule, createContext, SourceTextModule } from 'node:vm';
3
4
  import { stat, readFile } from 'node:fs/promises';
4
- import { transform } from '@swc/core';
5
5
  import { Command, Option } from 'commander';
6
6
  import { glob } from 'glob';
7
7
  import { Benchmark, printTableReports, printJSONReports, printSimpleReports, DEFAULT_REPORT_TYPES, DEFAULT_WORKERS } from './index.js';
8
+ import { transpile } from './utils.js';
8
9
  import { REPORT_TYPES } from './types.js';
9
10
 
10
11
  const require = createRequire(import.meta.url);
@@ -12,29 +13,11 @@ const { name, description, version } = require('../package.json');
12
13
 
13
14
  const commander = new Command();
14
15
 
15
- const transpile = async (code: string): Promise<string> => {
16
- const output = await transform(code, {
17
- filename: 'benchmark.ts',
18
- jsc: {
19
- parser: {
20
- syntax: 'typescript',
21
- tsx: false,
22
- dynamicImport: true,
23
- },
24
- target: 'esnext',
25
- },
26
- module: {
27
- type: 'es6',
28
- },
29
- });
30
- return output.code;
31
- };
32
-
33
16
  commander
34
17
  .name(name)
35
18
  .description(description)
36
19
  .version(version)
37
- .argument('<path>', 'glob pattern to find benchmarks')
20
+ .argument('<paths...>', 'glob pattern to find benchmarks')
38
21
  .addOption(new Option('-r, --report-types [reportTypes...]', 'statistic types to include in the report').choices(REPORT_TYPES).default(DEFAULT_REPORT_TYPES))
39
22
  .addOption(new Option('-w, --workers [workers]', 'number of concurent workers').default(DEFAULT_WORKERS).argParser(parseInt))
40
23
  .addOption(new Option('-f, --format [format]', 'output format').default('simple').choices(['simple', 'json', 'pjson', 'table']))
@@ -43,12 +26,21 @@ commander
43
26
  .addOption(new Option('--warmup-cycles [warmupCycles]', 'number of warmup cycles before measuring').argParser(parseInt))
44
27
  .addOption(new Option('--max-cycles [maxCycles]', 'maximum measurement cycles per feed').argParser(parseInt))
45
28
  .addOption(new Option('--min-cycles [minCycles]', 'minimum measurement cycles per feed').argParser(parseInt))
46
- .action(async (path, executeOptions) => {
47
- const files = await glob(path, { absolute: true, cwd: process.cwd() }).catch(() => []);
29
+ .addOption(new Option('--no-gc-observer', 'disable GC overlap detection'))
30
+ .action(async (patterns: string[], executeOptions) => {
31
+ const files = new Set<string>();
32
+ await Promise.all(
33
+ patterns.map(async (pattern) => {
34
+ const matches = await glob(pattern, { absolute: true, cwd: process.cwd() }).catch(() => []);
35
+ matches.forEach((file) => files.add(file));
36
+ }),
37
+ );
38
+
48
39
  for (const file of files) {
49
40
  const stats = await stat(file).catch(() => false as const);
50
41
  if (stats && stats.isFile()) {
51
42
  const content = await readFile(file, 'utf8');
43
+ const identifier = pathToFileURL(file).href;
52
44
  const code = await transpile(content);
53
45
  let instance: Benchmark<unknown> | undefined;
54
46
  const benchmark = (...args: Parameters<(typeof Benchmark)['create']>) => {
@@ -59,30 +51,54 @@ commander
59
51
  return instance;
60
52
  };
61
53
  const script = new SourceTextModule(code, {
62
- context: createContext({ benchmark }),
54
+ identifier,
55
+ context: createContext({
56
+ benchmark,
57
+ Buffer,
58
+ console,
59
+ }),
60
+ initializeImportMeta(meta) {
61
+ meta.url = identifier;
62
+ },
63
+ async importModuleDynamically(specifier, referencingModule) {
64
+ if (Module.isBuiltin(specifier)) {
65
+ return import(specifier);
66
+ }
67
+ const baseIdentifier = referencingModule.identifier ?? identifier;
68
+ const resolveFrom = createRequire(fileURLToPath(baseIdentifier));
69
+ const resolved = resolveFrom.resolve(specifier);
70
+ return import(resolved);
71
+ },
63
72
  });
64
- const imports = new Map();
73
+ const imports = new Map<string, SyntheticModule>();
65
74
  await script.link(async (specifier: string, referencingModule) => {
66
- if (imports.has(specifier)) {
67
- return imports.get(specifier);
75
+ const baseIdentifier = referencingModule.identifier ?? identifier;
76
+ const resolveFrom = createRequire(fileURLToPath(baseIdentifier));
77
+ const target = Module.isBuiltin(specifier) ? specifier : resolveFrom.resolve(specifier);
78
+ const cached = imports.get(target);
79
+ if (cached) {
80
+ return cached;
68
81
  }
69
- const mod = await import(Module.isBuiltin(specifier) ? specifier : require.resolve(specifier));
82
+ const mod = await import(target);
70
83
  const exportNames = Object.keys(mod);
71
84
  const imported = new SyntheticModule(
72
85
  exportNames,
73
86
  () => {
74
87
  exportNames.forEach((key) => imported.setExport(key, mod[key]));
75
88
  },
76
- { identifier: specifier, context: referencingModule.context },
89
+ { identifier: target, context: referencingModule.context },
77
90
  );
78
91
 
79
- imports.set(specifier, imported);
92
+ imports.set(target, imported);
80
93
  return imported;
81
94
  });
82
95
  await script.evaluate();
83
96
 
84
97
  if (instance) {
85
- const reports = await instance.execute(executeOptions);
98
+ const reports = await instance.execute({
99
+ ...executeOptions,
100
+ baseUrl: identifier,
101
+ });
86
102
  switch (executeOptions.format) {
87
103
  case 'json':
88
104
  {
package/src/executor.ts CHANGED
@@ -1,27 +1,31 @@
1
1
  import { Worker } from 'node:worker_threads';
2
2
  import { once } from 'node:events';
3
3
  import { queue } from 'async';
4
- import { RunOptions, ReportOptions, WorkerOptions, BenchmarkOptions, Control, ReportType, ReportTypeList, CONTROL_SLOTS } from './types.js';
4
+ import { pathToFileURL } from 'node:url';
5
5
  import { createReport, Report } from './reporter.js';
6
6
  import { cmp } from './utils.js';
7
+ import { RunOptions, ReportOptions, WorkerOptions, BenchmarkOptions, Control, ReportType, ReportTypeList, CONTROL_SLOTS } from './types.js';
7
8
 
8
9
  export type ExecutorReport<R extends ReportTypeList> = Record<R[number], Report> & { count: number };
9
10
 
10
11
  export interface ExecutorOptions<R extends ReportTypeList> extends BenchmarkOptions, ReportOptions<R> {
12
+ baseUrl?: string;
11
13
  workers?: number;
12
14
  maxCycles?: number;
13
15
  }
14
16
 
15
17
  export const createExecutor = <TContext, TInput, R extends ReportTypeList>({
18
+ baseUrl = pathToFileURL(process.cwd()).href,
16
19
  workers,
17
20
  warmupCycles,
18
21
  maxCycles,
19
22
  minCycles,
20
23
  absThreshold,
21
24
  relThreshold,
25
+ gcObserver = true,
22
26
  reportTypes,
23
27
  }: Required<ExecutorOptions<R>>) => {
24
- const executor = queue<RunOptions<TContext, TInput>>(async ({ setup, teardown, pre, run, post, data }) => {
28
+ const executor = queue<RunOptions<TContext, TInput>>(async ({ baseUrl: runBaseUrl = baseUrl, setup, teardown, pre, run, post, data }) => {
25
29
  const setupCode = setup?.toString();
26
30
  const teardownCode = teardown?.toString();
27
31
  const preCode = pre?.toString();
@@ -33,6 +37,7 @@ export const createExecutor = <TContext, TInput, R extends ReportTypeList>({
33
37
 
34
38
  const workerFile = new URL('./worker.js', import.meta.url);
35
39
  const workerData: WorkerOptions = {
40
+ baseUrl: runBaseUrl,
36
41
  setupCode,
37
42
  teardownCode,
38
43
  preCode,
@@ -44,6 +49,7 @@ export const createExecutor = <TContext, TInput, R extends ReportTypeList>({
44
49
  minCycles,
45
50
  absThreshold,
46
51
  relThreshold,
52
+ gcObserver,
47
53
 
48
54
  controlSAB,
49
55
  durationsSAB,
@@ -0,0 +1,23 @@
1
+ export interface GCMarker {
2
+ ref: WeakRef<object>;
3
+ token: object;
4
+ }
5
+
6
+ export class GCWatcher {
7
+ #registry = new FinalizationRegistry(() => {});
8
+
9
+ start(): GCMarker {
10
+ const token = {};
11
+ const ref = new WeakRef(token);
12
+ this.#registry.register(token, null, token);
13
+ return { ref, token };
14
+ }
15
+
16
+ seen(marker: GCMarker): boolean {
17
+ const collected = marker.ref.deref() === undefined;
18
+ if (!collected) {
19
+ this.#registry.unregister(marker.token);
20
+ }
21
+ return collected;
22
+ }
23
+ }
package/src/index.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { cpus } from 'node:os';
2
+ import { pathToFileURL } from 'node:url';
2
3
  import { createExecutor, ExecutorOptions, ExecutorReport } from './executor.js';
3
4
  import { MaybePromise, StepFn, SetupFn, TeardownFn, FeedFn, ReportType, ReportTypeList, DEFAULT_CYCLES } from './types.js';
4
5
 
@@ -141,20 +142,29 @@ export class Benchmark<TInput> {
141
142
  minCycles = 50,
142
143
  absThreshold = 1_000,
143
144
  relThreshold = 0.02,
145
+ gcObserver = true,
144
146
  reportTypes = DEFAULT_REPORT_TYPES as unknown as R,
147
+ baseUrl,
145
148
  }: ExecutorOptions<R>): Promise<TargetReport<R>[]> {
146
149
  if (this.#executed) {
147
150
  throw new Error("Benchmark is executed and can't be reused");
148
151
  }
149
152
  this.#executed = true;
150
153
 
154
+ const resolvedBaseUrl = baseUrl ?? pathToFileURL(process.cwd()).href;
155
+ if (!baseUrl) {
156
+ console.warn("Overtake: baseUrl not provided; defaulting to process.cwd(). Pass the benchmark's import.meta.url so relative imports resolve correctly.");
157
+ }
158
+
151
159
  const executor = createExecutor<unknown, TInput, R>({
160
+ baseUrl: resolvedBaseUrl,
152
161
  workers,
153
162
  warmupCycles,
154
163
  maxCycles,
155
164
  minCycles,
156
165
  absThreshold,
157
166
  relThreshold,
167
+ gcObserver,
158
168
  reportTypes,
159
169
  });
160
170
 
@@ -167,6 +177,7 @@ export class Benchmark<TInput> {
167
177
  const data = await feed.fn?.();
168
178
  executor
169
179
  .push<ExecutorReport<R>>({
180
+ baseUrl: resolvedBaseUrl,
170
181
  setup: target.setup,
171
182
  teardown: target.teardown,
172
183
  pre: measure.pre,
package/src/runner.ts CHANGED
@@ -1,21 +1,175 @@
1
+ import { performance, PerformanceObserver } from 'node:perf_hooks';
1
2
  import { Options, Control } from './types.js';
3
+ import { GCWatcher } from './gc-watcher.js';
4
+ import { StepFn, MaybePromise } from './types.js';
2
5
 
3
6
  const COMPLETE_VALUE = 100_00;
4
7
 
8
+ const hr = process.hrtime.bigint.bind(process.hrtime);
9
+
5
10
  const runSync = (run: Function) => {
6
11
  return (...args: unknown[]) => {
7
- const start = process.hrtime.bigint();
12
+ const start = hr();
8
13
  run(...args);
9
- return process.hrtime.bigint() - start;
14
+ return hr() - start;
10
15
  };
11
16
  };
12
17
 
13
18
  const runAsync = (run: Function) => {
14
19
  return async (...args: unknown[]) => {
15
- const start = process.hrtime.bigint();
20
+ const start = hr();
16
21
  await run(...args);
17
- return process.hrtime.bigint() - start;
22
+ return hr() - start;
23
+ };
24
+ };
25
+
26
+ const TARGET_SAMPLE_NS = 1_000_000n; // aim for ~1ms per measured sample
27
+ const MAX_BATCH = 1_048_576;
28
+ const PROGRESS_STRIDE = 16;
29
+ const GC_STRIDE = 32;
30
+ const OUTLIER_MULTIPLIER = 4;
31
+ const OUTLIER_IQR_MULTIPLIER = 3;
32
+ const OUTLIER_WINDOW = 64;
33
+
34
+ type GCEvent = { start: number; end: number };
35
+
36
+ const collectSample = async <TContext, TInput>(
37
+ batchSize: number,
38
+ run: (ctx: TContext, data: TInput) => MaybePromise<bigint>,
39
+ pre: StepFn<TContext, TInput> | undefined,
40
+ post: StepFn<TContext, TInput> | undefined,
41
+ context: TContext,
42
+ data: TInput,
43
+ ) => {
44
+ let sampleDuration = 0n;
45
+ for (let b = 0; b < batchSize; b++) {
46
+ await pre?.(context, data);
47
+ sampleDuration += await run(context, data);
48
+ await post?.(context, data);
49
+ }
50
+ return sampleDuration / BigInt(batchSize);
51
+ };
52
+
53
+ const tuneParameters = async <TContext, TInput>({
54
+ initialBatch,
55
+ run,
56
+ pre,
57
+ post,
58
+ context,
59
+ data,
60
+ minCycles,
61
+ relThreshold,
62
+ maxCycles,
63
+ }: {
64
+ initialBatch: number;
65
+ run: (ctx: TContext, data: TInput) => MaybePromise<bigint>;
66
+ pre?: StepFn<TContext, TInput>;
67
+ post?: StepFn<TContext, TInput>;
68
+ context: TContext;
69
+ data: TInput;
70
+ minCycles: number;
71
+ relThreshold: number;
72
+ maxCycles: number;
73
+ }) => {
74
+ let batchSize = initialBatch;
75
+ let bestCv = Number.POSITIVE_INFINITY;
76
+ let bestBatch = batchSize;
77
+
78
+ for (let attempt = 0; attempt < 3; attempt++) {
79
+ const samples: number[] = [];
80
+ const sampleCount = Math.min(8, maxCycles);
81
+ for (let s = 0; s < sampleCount; s++) {
82
+ const duration = await collectSample(batchSize, run, pre, post, context, data);
83
+ samples.push(Number(duration));
84
+ }
85
+ const mean = samples.reduce((acc, v) => acc + v, 0) / samples.length;
86
+ const variance = samples.reduce((acc, v) => acc + (v - mean) * (v - mean), 0) / Math.max(1, samples.length - 1);
87
+ const stddev = Math.sqrt(variance);
88
+ const cv = mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
89
+
90
+ if (cv < bestCv) {
91
+ bestCv = cv;
92
+ bestBatch = batchSize;
93
+ }
94
+
95
+ if (cv <= relThreshold || batchSize >= MAX_BATCH) {
96
+ break;
97
+ }
98
+ batchSize = Math.min(MAX_BATCH, batchSize * 2);
99
+ }
100
+
101
+ const tunedRel = bestCv < relThreshold ? Math.max(bestCv * 1.5, relThreshold * 0.5) : relThreshold;
102
+ const tunedMin = Math.min(maxCycles, Math.max(minCycles, Math.ceil(minCycles * Math.max(1, bestCv / (relThreshold || 1e-6)))));
103
+
104
+ return { batchSize: bestBatch, relThreshold: tunedRel, minCycles: tunedMin };
105
+ };
106
+
107
+ const createGCTracker = () => {
108
+ if (process.env.OVERTAKE_GC_OBSERVER !== '1') {
109
+ return null;
110
+ }
111
+ if (typeof PerformanceObserver === 'undefined') {
112
+ return null;
113
+ }
114
+
115
+ const events: GCEvent[] = [];
116
+ const observer = new PerformanceObserver((list) => {
117
+ for (const entry of list.getEntries()) {
118
+ events.push({ start: entry.startTime, end: entry.startTime + entry.duration });
119
+ }
120
+ });
121
+
122
+ try {
123
+ observer.observe({ entryTypes: ['gc'] });
124
+ } catch {
125
+ return null;
126
+ }
127
+
128
+ const overlaps = (start: number, end: number) => {
129
+ let noisy = false;
130
+ for (let i = events.length - 1; i >= 0; i--) {
131
+ const event = events[i];
132
+ if (event.end < start - 5_000) {
133
+ events.splice(i, 1);
134
+ continue;
135
+ }
136
+ if (event.start <= end && event.end >= start) {
137
+ noisy = true;
138
+ }
139
+ }
140
+ return noisy;
18
141
  };
142
+
143
+ const dispose = () => observer.disconnect();
144
+
145
+ return { overlaps, dispose };
146
+ };
147
+
148
+ const pushWindow = (arr: number[], value: number, cap: number) => {
149
+ if (arr.length === cap) {
150
+ arr.shift();
151
+ }
152
+ arr.push(value);
153
+ };
154
+
155
+ const medianAndIqr = (arr: number[]) => {
156
+ if (arr.length === 0) return { median: 0, iqr: 0 };
157
+ const sorted = [...arr].sort((a, b) => a - b);
158
+ const mid = Math.floor(sorted.length / 2);
159
+ const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
160
+ const q1Idx = Math.floor(sorted.length * 0.25);
161
+ const q3Idx = Math.floor(sorted.length * 0.75);
162
+ const q1 = sorted[q1Idx];
163
+ const q3 = sorted[q3Idx];
164
+ return { median, iqr: q3 - q1 };
165
+ };
166
+
167
+ const windowCv = (arr: number[]) => {
168
+ if (arr.length < 2) return Number.POSITIVE_INFINITY;
169
+ const mean = arr.reduce((a, v) => a + v, 0) / arr.length;
170
+ const variance = arr.reduce((a, v) => a + (v - mean) * (v - mean), 0) / (arr.length - 1);
171
+ const stddev = Math.sqrt(variance);
172
+ return mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
19
173
  };
20
174
 
21
175
  export const benchmark = async <TContext, TInput>({
@@ -30,6 +184,7 @@ export const benchmark = async <TContext, TInput>({
30
184
  minCycles,
31
185
  absThreshold,
32
186
  relThreshold,
187
+ gcObserver = false,
33
188
 
34
189
  durationsSAB,
35
190
  controlSAB,
@@ -43,40 +198,133 @@ export const benchmark = async <TContext, TInput>({
43
198
 
44
199
  const context = (await setup?.()) as TContext;
45
200
  const maxCycles = durations.length;
201
+ const gcWatcher = new GCWatcher();
202
+ const gcTracker = gcObserver ? createGCTracker() : null;
46
203
 
47
204
  try {
205
+ // classify sync/async and capture initial duration
48
206
  await pre?.(context, data!);
49
- const result = runRaw(context, data!);
207
+ const probeStart = hr();
208
+ const probeResult = runRaw(context, data!);
209
+ const isAsync = probeResult instanceof Promise;
210
+ if (isAsync) {
211
+ await probeResult;
212
+ }
213
+ const durationProbe = hr() - probeStart;
50
214
  await post?.(context, data!);
51
- const run = result instanceof Promise ? runAsync(runRaw) : runSync(runRaw);
52
- const start = Date.now();
53
- while (Date.now() - start < 1_000) {
54
- Math.sqrt(Math.random());
215
+
216
+ const run = isAsync ? runAsync(runRaw) : runSync(runRaw);
217
+
218
+ // choose batch size to amortize timer overhead
219
+ const durationPerRun = durationProbe === 0n ? 1n : durationProbe;
220
+ const suggestedBatch = Number(TARGET_SAMPLE_NS / durationPerRun);
221
+ const initialBatchSize = Math.min(MAX_BATCH, Math.max(1, suggestedBatch));
222
+
223
+ // auto-tune based on warmup samples
224
+ const tuned = await tuneParameters({
225
+ initialBatch: initialBatchSize,
226
+ run,
227
+ pre,
228
+ post,
229
+ context,
230
+ data: data as TInput,
231
+ minCycles,
232
+ relThreshold,
233
+ maxCycles,
234
+ });
235
+ let batchSize = tuned.batchSize;
236
+ minCycles = tuned.minCycles;
237
+ relThreshold = tuned.relThreshold;
238
+
239
+ // warmup: run until requested cycles, adapt if unstable
240
+ const warmupStart = Date.now();
241
+ let warmupRemaining = warmupCycles;
242
+ const warmupWindow: number[] = [];
243
+ const warmupCap = Math.max(warmupCycles, Math.min(maxCycles, warmupCycles * 4 || 1000));
244
+
245
+ while (Date.now() - warmupStart < 1_000 && warmupRemaining > 0) {
246
+ const start = hr();
247
+ await pre?.(context, data!);
248
+ await run(context, data);
249
+ await post?.(context, data!);
250
+ pushWindow(warmupWindow, Number(hr() - start), warmupCap);
251
+ warmupRemaining--;
55
252
  }
56
- for (let i = 0; i < warmupCycles; i++) {
253
+ let warmupDone = 0;
254
+ while (warmupDone < warmupRemaining) {
255
+ const start = hr();
57
256
  await pre?.(context, data!);
58
257
  await run(context, data);
59
258
  await post?.(context, data!);
259
+ pushWindow(warmupWindow, Number(hr() - start), warmupCap);
260
+ warmupDone++;
261
+ if (global.gc && warmupDone % GC_STRIDE === 0) {
262
+ global.gc();
263
+ }
264
+ }
265
+ while (warmupWindow.length >= 8 && warmupWindow.length < warmupCap) {
266
+ const cv = windowCv(warmupWindow);
267
+ if (cv <= relThreshold * 2) {
268
+ break;
269
+ }
270
+ const start = hr();
271
+ await pre?.(context, data!);
272
+ await run(context, data);
273
+ await post?.(context, data!);
274
+ pushWindow(warmupWindow, Number(hr() - start), warmupCap);
60
275
  }
61
276
 
62
277
  let i = 0;
63
278
  let mean = 0n;
64
279
  let m2 = 0n;
280
+ const outlierWindow: number[] = [];
65
281
 
66
282
  while (true) {
67
283
  if (i >= maxCycles) break;
68
284
 
69
- await pre?.(context, data!);
70
- const duration = await run(context, data);
71
- await post?.(context, data!);
285
+ const gcMarker = gcWatcher.start();
286
+ const sampleStart = performance.now();
287
+ let sampleDuration = 0n;
288
+ for (let b = 0; b < batchSize; b++) {
289
+ await pre?.(context, data!);
290
+ sampleDuration += await run(context, data);
291
+ await post?.(context, data!);
292
+ if (global.gc && (i + b) % GC_STRIDE === 0) {
293
+ global.gc();
294
+ }
295
+ }
296
+
297
+ // normalize by batch size
298
+ sampleDuration /= BigInt(batchSize);
72
299
 
73
- durations[i++] = duration;
74
- const delta = duration - mean;
300
+ const sampleEnd = performance.now();
301
+ const gcNoise = gcWatcher.seen(gcMarker) || (gcTracker?.overlaps(sampleStart, sampleEnd) ?? false);
302
+ if (gcNoise) {
303
+ continue;
304
+ }
305
+
306
+ const durationNumber = Number(sampleDuration);
307
+ pushWindow(outlierWindow, durationNumber, OUTLIER_WINDOW);
308
+ const { median, iqr } = medianAndIqr(outlierWindow);
309
+ const maxAllowed = median + OUTLIER_IQR_MULTIPLIER * iqr || Number.POSITIVE_INFINITY;
310
+ if (outlierWindow.length >= 8 && durationNumber > maxAllowed) {
311
+ continue;
312
+ }
313
+
314
+ const meanNumber = Number(mean);
315
+ if (i >= 8 && meanNumber > 0 && durationNumber > OUTLIER_MULTIPLIER * meanNumber) {
316
+ continue;
317
+ }
318
+
319
+ durations[i++] = sampleDuration;
320
+ const delta = sampleDuration - mean;
75
321
  mean += delta / BigInt(i);
76
- m2 += delta * (duration - mean);
322
+ m2 += delta * (sampleDuration - mean);
77
323
 
78
324
  const progress = Math.max(i / maxCycles) * COMPLETE_VALUE;
79
- control[Control.PROGRESS] = progress;
325
+ if (i % PROGRESS_STRIDE === 0) {
326
+ control[Control.PROGRESS] = progress;
327
+ }
80
328
 
81
329
  if (i >= minCycles) {
82
330
  const variance = Number(m2) / (i - 1);
@@ -99,6 +347,7 @@ export const benchmark = async <TContext, TInput>({
99
347
  console.error(e && typeof e === 'object' && 'stack' in e ? e.stack : e);
100
348
  control[Control.COMPLETE] = 1;
101
349
  } finally {
350
+ gcTracker?.dispose?.();
102
351
  try {
103
352
  await teardown?.(context);
104
353
  } catch (e) {
package/src/types.ts CHANGED
@@ -33,9 +33,12 @@ export interface BenchmarkOptions {
33
33
  minCycles?: number;
34
34
  absThreshold?: number; // ns
35
35
  relThreshold?: number; // %
36
+ gcObserver?: boolean;
37
+ baseUrl?: string;
36
38
  }
37
39
 
38
40
  export interface RunOptions<TContext, TInput> {
41
+ baseUrl?: string;
39
42
  setup?: SetupFn<TContext>;
40
43
  teardown?: TeardownFn<TContext>;
41
44
  pre?: StepFn<TContext, TInput>;
@@ -45,6 +48,7 @@ export interface RunOptions<TContext, TInput> {
45
48
  }
46
49
 
47
50
  export interface WorkerOptions extends Required<BenchmarkOptions> {
51
+ baseUrl: string;
48
52
  setupCode?: string;
49
53
  teardownCode?: string;
50
54
  preCode?: string;