overtake 1.0.4 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -25
- package/bin/overtake.js +1 -1
- package/build/cli.cjs +44 -33
- package/build/cli.cjs.map +1 -1
- package/build/cli.js +43 -32
- package/build/cli.js.map +1 -1
- package/build/executor.cjs +6 -3
- package/build/executor.cjs.map +1 -1
- package/build/executor.d.ts +3 -2
- package/build/executor.js +6 -3
- package/build/executor.js.map +1 -1
- package/build/gc-watcher.cjs +31 -0
- package/build/gc-watcher.cjs.map +1 -0
- package/build/gc-watcher.d.ts +9 -0
- package/build/gc-watcher.js +21 -0
- package/build/gc-watcher.js.map +1 -0
- package/build/index.cjs +9 -1
- package/build/index.cjs.map +1 -1
- package/build/index.d.ts +1 -1
- package/build/index.js +9 -1
- package/build/index.js.map +1 -1
- package/build/runner.cjs +226 -18
- package/build/runner.cjs.map +1 -1
- package/build/runner.d.ts +1 -1
- package/build/runner.js +226 -18
- package/build/runner.js.map +1 -1
- package/build/types.cjs.map +1 -1
- package/build/types.d.ts +4 -0
- package/build/types.js.map +1 -1
- package/build/utils.cjs +21 -0
- package/build/utils.cjs.map +1 -1
- package/build/utils.d.ts +1 -0
- package/build/utils.js +18 -0
- package/build/utils.js.map +1 -1
- package/build/worker.cjs +95 -8
- package/build/worker.cjs.map +1 -1
- package/build/worker.js +54 -8
- package/build/worker.js.map +1 -1
- package/examples/accuracy.ts +54 -0
- package/examples/complete.ts +3 -3
- package/examples/custom-reports.ts +21 -0
- package/examples/imports.ts +47 -0
- package/examples/quick-start.ts +10 -9
- package/package.json +10 -9
- package/src/cli.ts +46 -30
- package/src/executor.ts +8 -2
- package/src/gc-watcher.ts +23 -0
- package/src/index.ts +11 -0
- package/src/runner.ts +266 -17
- package/src/types.ts +4 -0
- package/src/utils.ts +20 -0
- package/src/worker.ts +59 -9
- package/CLAUDE.md +0 -145
- package/examples/array-copy.ts +0 -17
- package/examples/object-merge.ts +0 -41
- package/examples/serialization.ts +0 -22
package/examples/quick-start.ts
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
|
-
//
|
|
2
|
-
// npx overtake examples/quick-start.ts
|
|
1
|
+
// Minimal example - comparing array sum algorithms
|
|
2
|
+
// Run: npx overtake examples/quick-start.ts
|
|
3
|
+
// Import overtake to support global types
|
|
4
|
+
import 'overtake';
|
|
3
5
|
|
|
4
|
-
const
|
|
6
|
+
const sumBenchmark = benchmark('1M numbers', () => Array.from({ length: 1_000_000 }, (_, index) => index));
|
|
5
7
|
|
|
6
|
-
|
|
7
|
-
const n = input.length;
|
|
8
|
+
sumBenchmark.target('for loop').measure('sum', (_, numbers) => {
|
|
8
9
|
let sum = 0;
|
|
9
|
-
for (let i = 0; i <
|
|
10
|
-
sum +=
|
|
10
|
+
for (let i = 0; i < numbers.length; i++) {
|
|
11
|
+
sum += numbers[i];
|
|
11
12
|
}
|
|
12
13
|
});
|
|
13
14
|
|
|
14
|
-
|
|
15
|
-
|
|
15
|
+
sumBenchmark.target('reduce').measure('sum', (_, numbers) => {
|
|
16
|
+
numbers.reduce((accumulator, current) => accumulator + current, 0);
|
|
16
17
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "overtake",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "NodeJS performance benchmark",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "build/index.d.ts",
|
|
@@ -41,23 +41,24 @@
|
|
|
41
41
|
},
|
|
42
42
|
"homepage": "https://github.com/3axap4eHko/overtake#readme",
|
|
43
43
|
"devDependencies": {
|
|
44
|
-
"@jest/globals": "^30.0
|
|
44
|
+
"@jest/globals": "^30.2.0",
|
|
45
45
|
"@swc/jest": "^0.2.39",
|
|
46
46
|
"@types/async": "^3.2.25",
|
|
47
47
|
"@types/jest": "^30.0.0",
|
|
48
|
-
"@types/node": "^24.
|
|
48
|
+
"@types/node": "^24.10.1",
|
|
49
49
|
"husky": "^9.1.7",
|
|
50
|
-
"inop": "^0.
|
|
51
|
-
"jest": "^30.0
|
|
50
|
+
"inop": "^0.8.0",
|
|
51
|
+
"jest": "^30.2.0",
|
|
52
|
+
"overtake": "^1.0.5",
|
|
52
53
|
"prettier": "^3.6.2",
|
|
53
54
|
"pretty-quick": "^4.2.2",
|
|
54
|
-
"typescript": "^5.9.
|
|
55
|
+
"typescript": "^5.9.3"
|
|
55
56
|
},
|
|
56
57
|
"dependencies": {
|
|
57
|
-
"@swc/core": "^1.
|
|
58
|
+
"@swc/core": "^1.15.2",
|
|
58
59
|
"async": "^3.2.6",
|
|
59
|
-
"commander": "^14.0.
|
|
60
|
-
"glob": "^
|
|
60
|
+
"commander": "^14.0.2",
|
|
61
|
+
"glob": "^13.0.0"
|
|
61
62
|
},
|
|
62
63
|
"scripts": {
|
|
63
64
|
"build": "rm -rf build && inop src build -i __tests__ -i *.tmp.ts && tsc --declaration --emitDeclarationOnly",
|
package/src/cli.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { createRequire, Module } from 'node:module';
|
|
2
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
2
3
|
import { SyntheticModule, createContext, SourceTextModule } from 'node:vm';
|
|
3
4
|
import { stat, readFile } from 'node:fs/promises';
|
|
4
|
-
import { transform } from '@swc/core';
|
|
5
5
|
import { Command, Option } from 'commander';
|
|
6
6
|
import { glob } from 'glob';
|
|
7
7
|
import { Benchmark, printTableReports, printJSONReports, printSimpleReports, DEFAULT_REPORT_TYPES, DEFAULT_WORKERS } from './index.js';
|
|
8
|
+
import { transpile } from './utils.js';
|
|
8
9
|
import { REPORT_TYPES } from './types.js';
|
|
9
10
|
|
|
10
11
|
const require = createRequire(import.meta.url);
|
|
@@ -12,29 +13,11 @@ const { name, description, version } = require('../package.json');
|
|
|
12
13
|
|
|
13
14
|
const commander = new Command();
|
|
14
15
|
|
|
15
|
-
const transpile = async (code: string): Promise<string> => {
|
|
16
|
-
const output = await transform(code, {
|
|
17
|
-
filename: 'benchmark.ts',
|
|
18
|
-
jsc: {
|
|
19
|
-
parser: {
|
|
20
|
-
syntax: 'typescript',
|
|
21
|
-
tsx: false,
|
|
22
|
-
dynamicImport: true,
|
|
23
|
-
},
|
|
24
|
-
target: 'esnext',
|
|
25
|
-
},
|
|
26
|
-
module: {
|
|
27
|
-
type: 'es6',
|
|
28
|
-
},
|
|
29
|
-
});
|
|
30
|
-
return output.code;
|
|
31
|
-
};
|
|
32
|
-
|
|
33
16
|
commander
|
|
34
17
|
.name(name)
|
|
35
18
|
.description(description)
|
|
36
19
|
.version(version)
|
|
37
|
-
.argument('<
|
|
20
|
+
.argument('<paths...>', 'glob pattern to find benchmarks')
|
|
38
21
|
.addOption(new Option('-r, --report-types [reportTypes...]', 'statistic types to include in the report').choices(REPORT_TYPES).default(DEFAULT_REPORT_TYPES))
|
|
39
22
|
.addOption(new Option('-w, --workers [workers]', 'number of concurent workers').default(DEFAULT_WORKERS).argParser(parseInt))
|
|
40
23
|
.addOption(new Option('-f, --format [format]', 'output format').default('simple').choices(['simple', 'json', 'pjson', 'table']))
|
|
@@ -43,12 +26,21 @@ commander
|
|
|
43
26
|
.addOption(new Option('--warmup-cycles [warmupCycles]', 'number of warmup cycles before measuring').argParser(parseInt))
|
|
44
27
|
.addOption(new Option('--max-cycles [maxCycles]', 'maximum measurement cycles per feed').argParser(parseInt))
|
|
45
28
|
.addOption(new Option('--min-cycles [minCycles]', 'minimum measurement cycles per feed').argParser(parseInt))
|
|
46
|
-
.
|
|
47
|
-
|
|
29
|
+
.addOption(new Option('--no-gc-observer', 'disable GC overlap detection'))
|
|
30
|
+
.action(async (patterns: string[], executeOptions) => {
|
|
31
|
+
const files = new Set<string>();
|
|
32
|
+
await Promise.all(
|
|
33
|
+
patterns.map(async (pattern) => {
|
|
34
|
+
const matches = await glob(pattern, { absolute: true, cwd: process.cwd() }).catch(() => []);
|
|
35
|
+
matches.forEach((file) => files.add(file));
|
|
36
|
+
}),
|
|
37
|
+
);
|
|
38
|
+
|
|
48
39
|
for (const file of files) {
|
|
49
40
|
const stats = await stat(file).catch(() => false as const);
|
|
50
41
|
if (stats && stats.isFile()) {
|
|
51
42
|
const content = await readFile(file, 'utf8');
|
|
43
|
+
const identifier = pathToFileURL(file).href;
|
|
52
44
|
const code = await transpile(content);
|
|
53
45
|
let instance: Benchmark<unknown> | undefined;
|
|
54
46
|
const benchmark = (...args: Parameters<(typeof Benchmark)['create']>) => {
|
|
@@ -59,30 +51,54 @@ commander
|
|
|
59
51
|
return instance;
|
|
60
52
|
};
|
|
61
53
|
const script = new SourceTextModule(code, {
|
|
62
|
-
|
|
54
|
+
identifier,
|
|
55
|
+
context: createContext({
|
|
56
|
+
benchmark,
|
|
57
|
+
Buffer,
|
|
58
|
+
console,
|
|
59
|
+
}),
|
|
60
|
+
initializeImportMeta(meta) {
|
|
61
|
+
meta.url = identifier;
|
|
62
|
+
},
|
|
63
|
+
async importModuleDynamically(specifier, referencingModule) {
|
|
64
|
+
if (Module.isBuiltin(specifier)) {
|
|
65
|
+
return import(specifier);
|
|
66
|
+
}
|
|
67
|
+
const baseIdentifier = referencingModule.identifier ?? identifier;
|
|
68
|
+
const resolveFrom = createRequire(fileURLToPath(baseIdentifier));
|
|
69
|
+
const resolved = resolveFrom.resolve(specifier);
|
|
70
|
+
return import(resolved);
|
|
71
|
+
},
|
|
63
72
|
});
|
|
64
|
-
const imports = new Map();
|
|
73
|
+
const imports = new Map<string, SyntheticModule>();
|
|
65
74
|
await script.link(async (specifier: string, referencingModule) => {
|
|
66
|
-
|
|
67
|
-
|
|
75
|
+
const baseIdentifier = referencingModule.identifier ?? identifier;
|
|
76
|
+
const resolveFrom = createRequire(fileURLToPath(baseIdentifier));
|
|
77
|
+
const target = Module.isBuiltin(specifier) ? specifier : resolveFrom.resolve(specifier);
|
|
78
|
+
const cached = imports.get(target);
|
|
79
|
+
if (cached) {
|
|
80
|
+
return cached;
|
|
68
81
|
}
|
|
69
|
-
const mod = await import(
|
|
82
|
+
const mod = await import(target);
|
|
70
83
|
const exportNames = Object.keys(mod);
|
|
71
84
|
const imported = new SyntheticModule(
|
|
72
85
|
exportNames,
|
|
73
86
|
() => {
|
|
74
87
|
exportNames.forEach((key) => imported.setExport(key, mod[key]));
|
|
75
88
|
},
|
|
76
|
-
{ identifier:
|
|
89
|
+
{ identifier: target, context: referencingModule.context },
|
|
77
90
|
);
|
|
78
91
|
|
|
79
|
-
imports.set(
|
|
92
|
+
imports.set(target, imported);
|
|
80
93
|
return imported;
|
|
81
94
|
});
|
|
82
95
|
await script.evaluate();
|
|
83
96
|
|
|
84
97
|
if (instance) {
|
|
85
|
-
const reports = await instance.execute(
|
|
98
|
+
const reports = await instance.execute({
|
|
99
|
+
...executeOptions,
|
|
100
|
+
baseUrl: identifier,
|
|
101
|
+
});
|
|
86
102
|
switch (executeOptions.format) {
|
|
87
103
|
case 'json':
|
|
88
104
|
{
|
package/src/executor.ts
CHANGED
|
@@ -1,27 +1,31 @@
|
|
|
1
1
|
import { Worker } from 'node:worker_threads';
|
|
2
2
|
import { once } from 'node:events';
|
|
3
3
|
import { queue } from 'async';
|
|
4
|
-
import {
|
|
4
|
+
import { pathToFileURL } from 'node:url';
|
|
5
5
|
import { createReport, Report } from './reporter.js';
|
|
6
6
|
import { cmp } from './utils.js';
|
|
7
|
+
import { RunOptions, ReportOptions, WorkerOptions, BenchmarkOptions, Control, ReportType, ReportTypeList, CONTROL_SLOTS } from './types.js';
|
|
7
8
|
|
|
8
9
|
export type ExecutorReport<R extends ReportTypeList> = Record<R[number], Report> & { count: number };
|
|
9
10
|
|
|
10
11
|
export interface ExecutorOptions<R extends ReportTypeList> extends BenchmarkOptions, ReportOptions<R> {
|
|
12
|
+
baseUrl?: string;
|
|
11
13
|
workers?: number;
|
|
12
14
|
maxCycles?: number;
|
|
13
15
|
}
|
|
14
16
|
|
|
15
17
|
export const createExecutor = <TContext, TInput, R extends ReportTypeList>({
|
|
18
|
+
baseUrl = pathToFileURL(process.cwd()).href,
|
|
16
19
|
workers,
|
|
17
20
|
warmupCycles,
|
|
18
21
|
maxCycles,
|
|
19
22
|
minCycles,
|
|
20
23
|
absThreshold,
|
|
21
24
|
relThreshold,
|
|
25
|
+
gcObserver = true,
|
|
22
26
|
reportTypes,
|
|
23
27
|
}: Required<ExecutorOptions<R>>) => {
|
|
24
|
-
const executor = queue<RunOptions<TContext, TInput>>(async ({ setup, teardown, pre, run, post, data }) => {
|
|
28
|
+
const executor = queue<RunOptions<TContext, TInput>>(async ({ baseUrl: runBaseUrl = baseUrl, setup, teardown, pre, run, post, data }) => {
|
|
25
29
|
const setupCode = setup?.toString();
|
|
26
30
|
const teardownCode = teardown?.toString();
|
|
27
31
|
const preCode = pre?.toString();
|
|
@@ -33,6 +37,7 @@ export const createExecutor = <TContext, TInput, R extends ReportTypeList>({
|
|
|
33
37
|
|
|
34
38
|
const workerFile = new URL('./worker.js', import.meta.url);
|
|
35
39
|
const workerData: WorkerOptions = {
|
|
40
|
+
baseUrl: runBaseUrl,
|
|
36
41
|
setupCode,
|
|
37
42
|
teardownCode,
|
|
38
43
|
preCode,
|
|
@@ -44,6 +49,7 @@ export const createExecutor = <TContext, TInput, R extends ReportTypeList>({
|
|
|
44
49
|
minCycles,
|
|
45
50
|
absThreshold,
|
|
46
51
|
relThreshold,
|
|
52
|
+
gcObserver,
|
|
47
53
|
|
|
48
54
|
controlSAB,
|
|
49
55
|
durationsSAB,
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export interface GCMarker {
|
|
2
|
+
ref: WeakRef<object>;
|
|
3
|
+
token: object;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export class GCWatcher {
|
|
7
|
+
#registry = new FinalizationRegistry(() => {});
|
|
8
|
+
|
|
9
|
+
start(): GCMarker {
|
|
10
|
+
const token = {};
|
|
11
|
+
const ref = new WeakRef(token);
|
|
12
|
+
this.#registry.register(token, null, token);
|
|
13
|
+
return { ref, token };
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
seen(marker: GCMarker): boolean {
|
|
17
|
+
const collected = marker.ref.deref() === undefined;
|
|
18
|
+
if (!collected) {
|
|
19
|
+
this.#registry.unregister(marker.token);
|
|
20
|
+
}
|
|
21
|
+
return collected;
|
|
22
|
+
}
|
|
23
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { cpus } from 'node:os';
|
|
2
|
+
import { pathToFileURL } from 'node:url';
|
|
2
3
|
import { createExecutor, ExecutorOptions, ExecutorReport } from './executor.js';
|
|
3
4
|
import { MaybePromise, StepFn, SetupFn, TeardownFn, FeedFn, ReportType, ReportTypeList, DEFAULT_CYCLES } from './types.js';
|
|
4
5
|
|
|
@@ -141,20 +142,29 @@ export class Benchmark<TInput> {
|
|
|
141
142
|
minCycles = 50,
|
|
142
143
|
absThreshold = 1_000,
|
|
143
144
|
relThreshold = 0.02,
|
|
145
|
+
gcObserver = true,
|
|
144
146
|
reportTypes = DEFAULT_REPORT_TYPES as unknown as R,
|
|
147
|
+
baseUrl,
|
|
145
148
|
}: ExecutorOptions<R>): Promise<TargetReport<R>[]> {
|
|
146
149
|
if (this.#executed) {
|
|
147
150
|
throw new Error("Benchmark is executed and can't be reused");
|
|
148
151
|
}
|
|
149
152
|
this.#executed = true;
|
|
150
153
|
|
|
154
|
+
const resolvedBaseUrl = baseUrl ?? pathToFileURL(process.cwd()).href;
|
|
155
|
+
if (!baseUrl) {
|
|
156
|
+
console.warn("Overtake: baseUrl not provided; defaulting to process.cwd(). Pass the benchmark's import.meta.url so relative imports resolve correctly.");
|
|
157
|
+
}
|
|
158
|
+
|
|
151
159
|
const executor = createExecutor<unknown, TInput, R>({
|
|
160
|
+
baseUrl: resolvedBaseUrl,
|
|
152
161
|
workers,
|
|
153
162
|
warmupCycles,
|
|
154
163
|
maxCycles,
|
|
155
164
|
minCycles,
|
|
156
165
|
absThreshold,
|
|
157
166
|
relThreshold,
|
|
167
|
+
gcObserver,
|
|
158
168
|
reportTypes,
|
|
159
169
|
});
|
|
160
170
|
|
|
@@ -167,6 +177,7 @@ export class Benchmark<TInput> {
|
|
|
167
177
|
const data = await feed.fn?.();
|
|
168
178
|
executor
|
|
169
179
|
.push<ExecutorReport<R>>({
|
|
180
|
+
baseUrl: resolvedBaseUrl,
|
|
170
181
|
setup: target.setup,
|
|
171
182
|
teardown: target.teardown,
|
|
172
183
|
pre: measure.pre,
|
package/src/runner.ts
CHANGED
|
@@ -1,21 +1,175 @@
|
|
|
1
|
+
import { performance, PerformanceObserver } from 'node:perf_hooks';
|
|
1
2
|
import { Options, Control } from './types.js';
|
|
3
|
+
import { GCWatcher } from './gc-watcher.js';
|
|
4
|
+
import { StepFn, MaybePromise } from './types.js';
|
|
2
5
|
|
|
3
6
|
const COMPLETE_VALUE = 100_00;
|
|
4
7
|
|
|
8
|
+
const hr = process.hrtime.bigint.bind(process.hrtime);
|
|
9
|
+
|
|
5
10
|
const runSync = (run: Function) => {
|
|
6
11
|
return (...args: unknown[]) => {
|
|
7
|
-
const start =
|
|
12
|
+
const start = hr();
|
|
8
13
|
run(...args);
|
|
9
|
-
return
|
|
14
|
+
return hr() - start;
|
|
10
15
|
};
|
|
11
16
|
};
|
|
12
17
|
|
|
13
18
|
const runAsync = (run: Function) => {
|
|
14
19
|
return async (...args: unknown[]) => {
|
|
15
|
-
const start =
|
|
20
|
+
const start = hr();
|
|
16
21
|
await run(...args);
|
|
17
|
-
return
|
|
22
|
+
return hr() - start;
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
const TARGET_SAMPLE_NS = 1_000_000n; // aim for ~1ms per measured sample
|
|
27
|
+
const MAX_BATCH = 1_048_576;
|
|
28
|
+
const PROGRESS_STRIDE = 16;
|
|
29
|
+
const GC_STRIDE = 32;
|
|
30
|
+
const OUTLIER_MULTIPLIER = 4;
|
|
31
|
+
const OUTLIER_IQR_MULTIPLIER = 3;
|
|
32
|
+
const OUTLIER_WINDOW = 64;
|
|
33
|
+
|
|
34
|
+
type GCEvent = { start: number; end: number };
|
|
35
|
+
|
|
36
|
+
const collectSample = async <TContext, TInput>(
|
|
37
|
+
batchSize: number,
|
|
38
|
+
run: (ctx: TContext, data: TInput) => MaybePromise<bigint>,
|
|
39
|
+
pre: StepFn<TContext, TInput> | undefined,
|
|
40
|
+
post: StepFn<TContext, TInput> | undefined,
|
|
41
|
+
context: TContext,
|
|
42
|
+
data: TInput,
|
|
43
|
+
) => {
|
|
44
|
+
let sampleDuration = 0n;
|
|
45
|
+
for (let b = 0; b < batchSize; b++) {
|
|
46
|
+
await pre?.(context, data);
|
|
47
|
+
sampleDuration += await run(context, data);
|
|
48
|
+
await post?.(context, data);
|
|
49
|
+
}
|
|
50
|
+
return sampleDuration / BigInt(batchSize);
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
const tuneParameters = async <TContext, TInput>({
|
|
54
|
+
initialBatch,
|
|
55
|
+
run,
|
|
56
|
+
pre,
|
|
57
|
+
post,
|
|
58
|
+
context,
|
|
59
|
+
data,
|
|
60
|
+
minCycles,
|
|
61
|
+
relThreshold,
|
|
62
|
+
maxCycles,
|
|
63
|
+
}: {
|
|
64
|
+
initialBatch: number;
|
|
65
|
+
run: (ctx: TContext, data: TInput) => MaybePromise<bigint>;
|
|
66
|
+
pre?: StepFn<TContext, TInput>;
|
|
67
|
+
post?: StepFn<TContext, TInput>;
|
|
68
|
+
context: TContext;
|
|
69
|
+
data: TInput;
|
|
70
|
+
minCycles: number;
|
|
71
|
+
relThreshold: number;
|
|
72
|
+
maxCycles: number;
|
|
73
|
+
}) => {
|
|
74
|
+
let batchSize = initialBatch;
|
|
75
|
+
let bestCv = Number.POSITIVE_INFINITY;
|
|
76
|
+
let bestBatch = batchSize;
|
|
77
|
+
|
|
78
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
79
|
+
const samples: number[] = [];
|
|
80
|
+
const sampleCount = Math.min(8, maxCycles);
|
|
81
|
+
for (let s = 0; s < sampleCount; s++) {
|
|
82
|
+
const duration = await collectSample(batchSize, run, pre, post, context, data);
|
|
83
|
+
samples.push(Number(duration));
|
|
84
|
+
}
|
|
85
|
+
const mean = samples.reduce((acc, v) => acc + v, 0) / samples.length;
|
|
86
|
+
const variance = samples.reduce((acc, v) => acc + (v - mean) * (v - mean), 0) / Math.max(1, samples.length - 1);
|
|
87
|
+
const stddev = Math.sqrt(variance);
|
|
88
|
+
const cv = mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
|
|
89
|
+
|
|
90
|
+
if (cv < bestCv) {
|
|
91
|
+
bestCv = cv;
|
|
92
|
+
bestBatch = batchSize;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (cv <= relThreshold || batchSize >= MAX_BATCH) {
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
batchSize = Math.min(MAX_BATCH, batchSize * 2);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const tunedRel = bestCv < relThreshold ? Math.max(bestCv * 1.5, relThreshold * 0.5) : relThreshold;
|
|
102
|
+
const tunedMin = Math.min(maxCycles, Math.max(minCycles, Math.ceil(minCycles * Math.max(1, bestCv / (relThreshold || 1e-6)))));
|
|
103
|
+
|
|
104
|
+
return { batchSize: bestBatch, relThreshold: tunedRel, minCycles: tunedMin };
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
const createGCTracker = () => {
|
|
108
|
+
if (process.env.OVERTAKE_GC_OBSERVER !== '1') {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
if (typeof PerformanceObserver === 'undefined') {
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const events: GCEvent[] = [];
|
|
116
|
+
const observer = new PerformanceObserver((list) => {
|
|
117
|
+
for (const entry of list.getEntries()) {
|
|
118
|
+
events.push({ start: entry.startTime, end: entry.startTime + entry.duration });
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
try {
|
|
123
|
+
observer.observe({ entryTypes: ['gc'] });
|
|
124
|
+
} catch {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const overlaps = (start: number, end: number) => {
|
|
129
|
+
let noisy = false;
|
|
130
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
131
|
+
const event = events[i];
|
|
132
|
+
if (event.end < start - 5_000) {
|
|
133
|
+
events.splice(i, 1);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
if (event.start <= end && event.end >= start) {
|
|
137
|
+
noisy = true;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return noisy;
|
|
18
141
|
};
|
|
142
|
+
|
|
143
|
+
const dispose = () => observer.disconnect();
|
|
144
|
+
|
|
145
|
+
return { overlaps, dispose };
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
const pushWindow = (arr: number[], value: number, cap: number) => {
|
|
149
|
+
if (arr.length === cap) {
|
|
150
|
+
arr.shift();
|
|
151
|
+
}
|
|
152
|
+
arr.push(value);
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
const medianAndIqr = (arr: number[]) => {
|
|
156
|
+
if (arr.length === 0) return { median: 0, iqr: 0 };
|
|
157
|
+
const sorted = [...arr].sort((a, b) => a - b);
|
|
158
|
+
const mid = Math.floor(sorted.length / 2);
|
|
159
|
+
const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
|
|
160
|
+
const q1Idx = Math.floor(sorted.length * 0.25);
|
|
161
|
+
const q3Idx = Math.floor(sorted.length * 0.75);
|
|
162
|
+
const q1 = sorted[q1Idx];
|
|
163
|
+
const q3 = sorted[q3Idx];
|
|
164
|
+
return { median, iqr: q3 - q1 };
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
const windowCv = (arr: number[]) => {
|
|
168
|
+
if (arr.length < 2) return Number.POSITIVE_INFINITY;
|
|
169
|
+
const mean = arr.reduce((a, v) => a + v, 0) / arr.length;
|
|
170
|
+
const variance = arr.reduce((a, v) => a + (v - mean) * (v - mean), 0) / (arr.length - 1);
|
|
171
|
+
const stddev = Math.sqrt(variance);
|
|
172
|
+
return mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
|
|
19
173
|
};
|
|
20
174
|
|
|
21
175
|
export const benchmark = async <TContext, TInput>({
|
|
@@ -30,6 +184,7 @@ export const benchmark = async <TContext, TInput>({
|
|
|
30
184
|
minCycles,
|
|
31
185
|
absThreshold,
|
|
32
186
|
relThreshold,
|
|
187
|
+
gcObserver = false,
|
|
33
188
|
|
|
34
189
|
durationsSAB,
|
|
35
190
|
controlSAB,
|
|
@@ -43,40 +198,133 @@ export const benchmark = async <TContext, TInput>({
|
|
|
43
198
|
|
|
44
199
|
const context = (await setup?.()) as TContext;
|
|
45
200
|
const maxCycles = durations.length;
|
|
201
|
+
const gcWatcher = new GCWatcher();
|
|
202
|
+
const gcTracker = gcObserver ? createGCTracker() : null;
|
|
46
203
|
|
|
47
204
|
try {
|
|
205
|
+
// classify sync/async and capture initial duration
|
|
48
206
|
await pre?.(context, data!);
|
|
49
|
-
const
|
|
207
|
+
const probeStart = hr();
|
|
208
|
+
const probeResult = runRaw(context, data!);
|
|
209
|
+
const isAsync = probeResult instanceof Promise;
|
|
210
|
+
if (isAsync) {
|
|
211
|
+
await probeResult;
|
|
212
|
+
}
|
|
213
|
+
const durationProbe = hr() - probeStart;
|
|
50
214
|
await post?.(context, data!);
|
|
51
|
-
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
|
|
215
|
+
|
|
216
|
+
const run = isAsync ? runAsync(runRaw) : runSync(runRaw);
|
|
217
|
+
|
|
218
|
+
// choose batch size to amortize timer overhead
|
|
219
|
+
const durationPerRun = durationProbe === 0n ? 1n : durationProbe;
|
|
220
|
+
const suggestedBatch = Number(TARGET_SAMPLE_NS / durationPerRun);
|
|
221
|
+
const initialBatchSize = Math.min(MAX_BATCH, Math.max(1, suggestedBatch));
|
|
222
|
+
|
|
223
|
+
// auto-tune based on warmup samples
|
|
224
|
+
const tuned = await tuneParameters({
|
|
225
|
+
initialBatch: initialBatchSize,
|
|
226
|
+
run,
|
|
227
|
+
pre,
|
|
228
|
+
post,
|
|
229
|
+
context,
|
|
230
|
+
data: data as TInput,
|
|
231
|
+
minCycles,
|
|
232
|
+
relThreshold,
|
|
233
|
+
maxCycles,
|
|
234
|
+
});
|
|
235
|
+
let batchSize = tuned.batchSize;
|
|
236
|
+
minCycles = tuned.minCycles;
|
|
237
|
+
relThreshold = tuned.relThreshold;
|
|
238
|
+
|
|
239
|
+
// warmup: run until requested cycles, adapt if unstable
|
|
240
|
+
const warmupStart = Date.now();
|
|
241
|
+
let warmupRemaining = warmupCycles;
|
|
242
|
+
const warmupWindow: number[] = [];
|
|
243
|
+
const warmupCap = Math.max(warmupCycles, Math.min(maxCycles, warmupCycles * 4 || 1000));
|
|
244
|
+
|
|
245
|
+
while (Date.now() - warmupStart < 1_000 && warmupRemaining > 0) {
|
|
246
|
+
const start = hr();
|
|
247
|
+
await pre?.(context, data!);
|
|
248
|
+
await run(context, data);
|
|
249
|
+
await post?.(context, data!);
|
|
250
|
+
pushWindow(warmupWindow, Number(hr() - start), warmupCap);
|
|
251
|
+
warmupRemaining--;
|
|
55
252
|
}
|
|
56
|
-
|
|
253
|
+
let warmupDone = 0;
|
|
254
|
+
while (warmupDone < warmupRemaining) {
|
|
255
|
+
const start = hr();
|
|
57
256
|
await pre?.(context, data!);
|
|
58
257
|
await run(context, data);
|
|
59
258
|
await post?.(context, data!);
|
|
259
|
+
pushWindow(warmupWindow, Number(hr() - start), warmupCap);
|
|
260
|
+
warmupDone++;
|
|
261
|
+
if (global.gc && warmupDone % GC_STRIDE === 0) {
|
|
262
|
+
global.gc();
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
while (warmupWindow.length >= 8 && warmupWindow.length < warmupCap) {
|
|
266
|
+
const cv = windowCv(warmupWindow);
|
|
267
|
+
if (cv <= relThreshold * 2) {
|
|
268
|
+
break;
|
|
269
|
+
}
|
|
270
|
+
const start = hr();
|
|
271
|
+
await pre?.(context, data!);
|
|
272
|
+
await run(context, data);
|
|
273
|
+
await post?.(context, data!);
|
|
274
|
+
pushWindow(warmupWindow, Number(hr() - start), warmupCap);
|
|
60
275
|
}
|
|
61
276
|
|
|
62
277
|
let i = 0;
|
|
63
278
|
let mean = 0n;
|
|
64
279
|
let m2 = 0n;
|
|
280
|
+
const outlierWindow: number[] = [];
|
|
65
281
|
|
|
66
282
|
while (true) {
|
|
67
283
|
if (i >= maxCycles) break;
|
|
68
284
|
|
|
69
|
-
|
|
70
|
-
const
|
|
71
|
-
|
|
285
|
+
const gcMarker = gcWatcher.start();
|
|
286
|
+
const sampleStart = performance.now();
|
|
287
|
+
let sampleDuration = 0n;
|
|
288
|
+
for (let b = 0; b < batchSize; b++) {
|
|
289
|
+
await pre?.(context, data!);
|
|
290
|
+
sampleDuration += await run(context, data);
|
|
291
|
+
await post?.(context, data!);
|
|
292
|
+
if (global.gc && (i + b) % GC_STRIDE === 0) {
|
|
293
|
+
global.gc();
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// normalize by batch size
|
|
298
|
+
sampleDuration /= BigInt(batchSize);
|
|
72
299
|
|
|
73
|
-
|
|
74
|
-
const
|
|
300
|
+
const sampleEnd = performance.now();
|
|
301
|
+
const gcNoise = gcWatcher.seen(gcMarker) || (gcTracker?.overlaps(sampleStart, sampleEnd) ?? false);
|
|
302
|
+
if (gcNoise) {
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const durationNumber = Number(sampleDuration);
|
|
307
|
+
pushWindow(outlierWindow, durationNumber, OUTLIER_WINDOW);
|
|
308
|
+
const { median, iqr } = medianAndIqr(outlierWindow);
|
|
309
|
+
const maxAllowed = median + OUTLIER_IQR_MULTIPLIER * iqr || Number.POSITIVE_INFINITY;
|
|
310
|
+
if (outlierWindow.length >= 8 && durationNumber > maxAllowed) {
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const meanNumber = Number(mean);
|
|
315
|
+
if (i >= 8 && meanNumber > 0 && durationNumber > OUTLIER_MULTIPLIER * meanNumber) {
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
durations[i++] = sampleDuration;
|
|
320
|
+
const delta = sampleDuration - mean;
|
|
75
321
|
mean += delta / BigInt(i);
|
|
76
|
-
m2 += delta * (
|
|
322
|
+
m2 += delta * (sampleDuration - mean);
|
|
77
323
|
|
|
78
324
|
const progress = Math.max(i / maxCycles) * COMPLETE_VALUE;
|
|
79
|
-
|
|
325
|
+
if (i % PROGRESS_STRIDE === 0) {
|
|
326
|
+
control[Control.PROGRESS] = progress;
|
|
327
|
+
}
|
|
80
328
|
|
|
81
329
|
if (i >= minCycles) {
|
|
82
330
|
const variance = Number(m2) / (i - 1);
|
|
@@ -99,6 +347,7 @@ export const benchmark = async <TContext, TInput>({
|
|
|
99
347
|
console.error(e && typeof e === 'object' && 'stack' in e ? e.stack : e);
|
|
100
348
|
control[Control.COMPLETE] = 1;
|
|
101
349
|
} finally {
|
|
350
|
+
gcTracker?.dispose?.();
|
|
102
351
|
try {
|
|
103
352
|
await teardown?.(context);
|
|
104
353
|
} catch (e) {
|
package/src/types.ts
CHANGED
|
@@ -33,9 +33,12 @@ export interface BenchmarkOptions {
|
|
|
33
33
|
minCycles?: number;
|
|
34
34
|
absThreshold?: number; // ns
|
|
35
35
|
relThreshold?: number; // %
|
|
36
|
+
gcObserver?: boolean;
|
|
37
|
+
baseUrl?: string;
|
|
36
38
|
}
|
|
37
39
|
|
|
38
40
|
export interface RunOptions<TContext, TInput> {
|
|
41
|
+
baseUrl?: string;
|
|
39
42
|
setup?: SetupFn<TContext>;
|
|
40
43
|
teardown?: TeardownFn<TContext>;
|
|
41
44
|
pre?: StepFn<TContext, TInput>;
|
|
@@ -45,6 +48,7 @@ export interface RunOptions<TContext, TInput> {
|
|
|
45
48
|
}
|
|
46
49
|
|
|
47
50
|
export interface WorkerOptions extends Required<BenchmarkOptions> {
|
|
51
|
+
baseUrl: string;
|
|
48
52
|
setupCode?: string;
|
|
49
53
|
teardownCode?: string;
|
|
50
54
|
preCode?: string;
|