overtake 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/overtake.js +1 -1
- package/build/cli.js +135 -0
- package/build/executor.d.ts +2 -2
- package/build/executor.js +126 -0
- package/build/gc-watcher.js +16 -0
- package/build/index.d.ts +2 -2
- package/build/index.js +375 -0
- package/build/register-hook.d.ts +1 -0
- package/build/register-hook.js +15 -0
- package/build/reporter.d.ts +1 -1
- package/build/reporter.js +255 -0
- package/build/runner.d.ts +1 -1
- package/build/runner.js +531 -0
- package/build/types.js +28 -0
- package/build/utils.js +100 -0
- package/build/worker.js +111 -0
- package/package.json +5 -5
- package/src/__tests__/assert-no-closure.ts +1 -1
- package/src/__tests__/benchmark-execute.ts +2 -2
- package/src/cli.ts +3 -3
- package/src/executor.ts +4 -4
- package/src/index.ts +2 -2
- package/src/register-hook.ts +15 -0
- package/src/reporter.ts +2 -2
- package/src/runner.ts +2 -2
- package/src/worker.ts +3 -3
- package/tsconfig.json +0 -1
package/bin/overtake.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env -S node --experimental-vm-modules --no-warnings --expose-gc
|
|
2
|
-
import '../
|
|
2
|
+
import '../build/cli.js';
|
package/build/cli.js
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { createRequire, register } from 'node:module';
|
|
2
|
+
import { parseArgs } from 'node:util';
|
|
3
|
+
import { pathToFileURL } from 'node:url';
|
|
4
|
+
import { stat, readFile, writeFile, glob } from 'node:fs/promises';
|
|
5
|
+
import { Benchmark, printTableReports, printJSONReports, printSimpleReports, printMarkdownReports, printHistogramReports, printComparisonReports, reportsToBaseline, DEFAULT_REPORT_TYPES, DEFAULT_WORKERS, } from './index.js';
|
|
6
|
+
import { REPORT_TYPES } from './types.js';
|
|
7
|
+
import { resolveHookUrl } from './utils.js';
|
|
8
|
+
register(resolveHookUrl);
|
|
9
|
+
const require = createRequire(import.meta.url);
|
|
10
|
+
const { name, version, description } = require('../package.json');
|
|
11
|
+
const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
|
|
12
|
+
const FORMATS = ['simple', 'json', 'pjson', 'table', 'markdown', 'histogram'];
|
|
13
|
+
const { values: opts, positionals: patterns } = parseArgs({
|
|
14
|
+
args: process.argv.slice(2),
|
|
15
|
+
allowPositionals: true,
|
|
16
|
+
options: {
|
|
17
|
+
'report-types': { type: 'string', short: 'r', multiple: true },
|
|
18
|
+
workers: { type: 'string', short: 'w' },
|
|
19
|
+
format: { type: 'string', short: 'f' },
|
|
20
|
+
'abs-threshold': { type: 'string' },
|
|
21
|
+
'rel-threshold': { type: 'string' },
|
|
22
|
+
'warmup-cycles': { type: 'string' },
|
|
23
|
+
'max-cycles': { type: 'string' },
|
|
24
|
+
'min-cycles': { type: 'string' },
|
|
25
|
+
'no-gc-observer': { type: 'boolean' },
|
|
26
|
+
progress: { type: 'boolean' },
|
|
27
|
+
'save-baseline': { type: 'string' },
|
|
28
|
+
'compare-baseline': { type: 'string' },
|
|
29
|
+
help: { type: 'boolean', short: 'h' },
|
|
30
|
+
version: { type: 'boolean', short: 'v' },
|
|
31
|
+
},
|
|
32
|
+
});
|
|
33
|
+
if (opts.version) {
|
|
34
|
+
console.log(version);
|
|
35
|
+
process.exit(0);
|
|
36
|
+
}
|
|
37
|
+
if (opts.help || patterns.length === 0) {
|
|
38
|
+
console.log(`${name} v${version} - ${description}
|
|
39
|
+
|
|
40
|
+
Usage: overtake [options] <paths...>
|
|
41
|
+
|
|
42
|
+
Options:
|
|
43
|
+
-r, --report-types <type> statistic type, repeat for multiple (-r ops -r p99)
|
|
44
|
+
-w, --workers <n> number of concurrent workers (default: ${DEFAULT_WORKERS})
|
|
45
|
+
-f, --format <format> output format: ${FORMATS.join(', ')} (default: simple)
|
|
46
|
+
--abs-threshold <ns> absolute error threshold in nanoseconds
|
|
47
|
+
--rel-threshold <frac> relative error threshold (0-1)
|
|
48
|
+
--warmup-cycles <n> warmup cycles before measuring
|
|
49
|
+
--max-cycles <n> maximum measurement cycles per feed
|
|
50
|
+
--min-cycles <n> minimum measurement cycles per feed
|
|
51
|
+
--no-gc-observer disable GC overlap detection
|
|
52
|
+
--progress show progress bar
|
|
53
|
+
--save-baseline <file> save results to baseline file
|
|
54
|
+
--compare-baseline <file> compare results against baseline file
|
|
55
|
+
-v, --version show version
|
|
56
|
+
-h, --help show this help`);
|
|
57
|
+
process.exit(0);
|
|
58
|
+
}
|
|
59
|
+
const reportTypes = opts['report-types']?.length
|
|
60
|
+
? opts['report-types'].filter((t) => REPORT_TYPES.includes(t))
|
|
61
|
+
: DEFAULT_REPORT_TYPES;
|
|
62
|
+
const format = opts.format && FORMATS.includes(opts.format) ? opts.format : 'simple';
|
|
63
|
+
const executeOptions = {
|
|
64
|
+
reportTypes,
|
|
65
|
+
workers: opts.workers ? parseInt(opts.workers) : DEFAULT_WORKERS,
|
|
66
|
+
absThreshold: opts['abs-threshold'] ? parseFloat(opts['abs-threshold']) : undefined,
|
|
67
|
+
relThreshold: opts['rel-threshold'] ? parseFloat(opts['rel-threshold']) : undefined,
|
|
68
|
+
warmupCycles: opts['warmup-cycles'] ? parseInt(opts['warmup-cycles']) : undefined,
|
|
69
|
+
maxCycles: opts['max-cycles'] ? parseInt(opts['max-cycles']) : undefined,
|
|
70
|
+
minCycles: opts['min-cycles'] ? parseInt(opts['min-cycles']) : undefined,
|
|
71
|
+
gcObserver: !opts['no-gc-observer'],
|
|
72
|
+
progress: opts.progress ?? false,
|
|
73
|
+
format,
|
|
74
|
+
};
|
|
75
|
+
let baseline = null;
|
|
76
|
+
if (opts['compare-baseline']) {
|
|
77
|
+
try {
|
|
78
|
+
const content = await readFile(opts['compare-baseline'], 'utf8');
|
|
79
|
+
baseline = JSON.parse(content);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
console.error(`Warning: Could not load baseline file: ${opts['compare-baseline']}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const files = new Set((await Promise.all(patterns.map((pattern) => Array.fromAsync(glob(pattern, { cwd: process.cwd() })).catch(() => [])))).flat());
|
|
86
|
+
for (const file of files) {
|
|
87
|
+
const stats = await stat(file).catch(() => false);
|
|
88
|
+
if (stats && stats.isFile()) {
|
|
89
|
+
const identifier = pathToFileURL(file).href;
|
|
90
|
+
let instance;
|
|
91
|
+
globalThis.benchmark = (...args) => {
|
|
92
|
+
if (instance) {
|
|
93
|
+
throw new Error('Only one benchmark per file is supported');
|
|
94
|
+
}
|
|
95
|
+
instance = Benchmark.create(...args);
|
|
96
|
+
return instance;
|
|
97
|
+
};
|
|
98
|
+
await import(identifier);
|
|
99
|
+
if (instance) {
|
|
100
|
+
const reports = await instance.execute({
|
|
101
|
+
...executeOptions,
|
|
102
|
+
[BENCHMARK_URL]: identifier,
|
|
103
|
+
});
|
|
104
|
+
if (opts['save-baseline']) {
|
|
105
|
+
const baselineData = reportsToBaseline(reports);
|
|
106
|
+
await writeFile(opts['save-baseline'], JSON.stringify(baselineData, null, 2));
|
|
107
|
+
console.log(`Baseline saved to: ${opts['save-baseline']}`);
|
|
108
|
+
}
|
|
109
|
+
if (baseline) {
|
|
110
|
+
printComparisonReports(reports, baseline);
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
switch (format) {
|
|
114
|
+
case 'json':
|
|
115
|
+
printJSONReports(reports);
|
|
116
|
+
break;
|
|
117
|
+
case 'pjson':
|
|
118
|
+
printJSONReports(reports, 2);
|
|
119
|
+
break;
|
|
120
|
+
case 'table':
|
|
121
|
+
printTableReports(reports);
|
|
122
|
+
break;
|
|
123
|
+
case 'markdown':
|
|
124
|
+
printMarkdownReports(reports);
|
|
125
|
+
break;
|
|
126
|
+
case 'histogram':
|
|
127
|
+
printHistogramReports(reports);
|
|
128
|
+
break;
|
|
129
|
+
default:
|
|
130
|
+
printSimpleReports(reports);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
package/build/executor.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Report } from './reporter.
|
|
2
|
-
import { type ExecutorRunOptions, type ReportOptions, type BenchmarkOptions, type ReportTypeList, type ProgressCallback } from './types.
|
|
1
|
+
import { Report } from './reporter.js';
|
|
2
|
+
import { type ExecutorRunOptions, type ReportOptions, type BenchmarkOptions, type ReportTypeList, type ProgressCallback } from './types.js';
|
|
3
3
|
export type ExecutorReport<R extends ReportTypeList> = Record<R[number], Report> & {
|
|
4
4
|
count: number;
|
|
5
5
|
heapUsedKB: number;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { Worker } from 'node:worker_threads';
|
|
2
|
+
import { once } from 'node:events';
|
|
3
|
+
import { pathToFileURL } from 'node:url';
|
|
4
|
+
import { createReport, computeStats, Report } from './reporter.js';
|
|
5
|
+
import { cmp, assertNoClosure } from './utils.js';
|
|
6
|
+
import { Control, CONTROL_SLOTS, COMPLETE_VALUE, } from './types.js';
|
|
7
|
+
const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
|
|
8
|
+
export const createExecutor = (options) => {
|
|
9
|
+
const { workers, warmupCycles, maxCycles, minCycles, absThreshold, relThreshold, gcObserver = true, reportTypes, onProgress, progressInterval = 100 } = options;
|
|
10
|
+
const benchmarkUrl = options[BENCHMARK_URL];
|
|
11
|
+
const resolvedBenchmarkUrl = typeof benchmarkUrl === 'string' ? benchmarkUrl : pathToFileURL(process.cwd()).href;
|
|
12
|
+
const pending = [];
|
|
13
|
+
let running = 0;
|
|
14
|
+
const schedule = async (task) => {
|
|
15
|
+
running++;
|
|
16
|
+
try {
|
|
17
|
+
return await runTask(task);
|
|
18
|
+
}
|
|
19
|
+
finally {
|
|
20
|
+
running--;
|
|
21
|
+
if (pending.length > 0) {
|
|
22
|
+
const next = pending.shift();
|
|
23
|
+
schedule(next.task).then(next.resolve, next.reject);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
const pushAsync = (task) => {
|
|
28
|
+
if (running < workers) {
|
|
29
|
+
return schedule(task);
|
|
30
|
+
}
|
|
31
|
+
return new Promise((resolve, reject) => {
|
|
32
|
+
pending.push({ task, resolve: resolve, reject });
|
|
33
|
+
});
|
|
34
|
+
};
|
|
35
|
+
const runTask = async ({ id, setup, teardown, pre, run, post, data }) => {
|
|
36
|
+
const setupCode = setup?.toString();
|
|
37
|
+
const teardownCode = teardown?.toString();
|
|
38
|
+
const preCode = pre?.toString();
|
|
39
|
+
const runCode = run.toString();
|
|
40
|
+
const postCode = post?.toString();
|
|
41
|
+
if (setupCode)
|
|
42
|
+
assertNoClosure(setupCode, 'setup');
|
|
43
|
+
if (teardownCode)
|
|
44
|
+
assertNoClosure(teardownCode, 'teardown');
|
|
45
|
+
if (preCode)
|
|
46
|
+
assertNoClosure(preCode, 'pre');
|
|
47
|
+
assertNoClosure(runCode, 'run');
|
|
48
|
+
if (postCode)
|
|
49
|
+
assertNoClosure(postCode, 'post');
|
|
50
|
+
const controlSAB = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT * CONTROL_SLOTS);
|
|
51
|
+
const durationsSAB = new SharedArrayBuffer(BigUint64Array.BYTES_PER_ELEMENT * maxCycles);
|
|
52
|
+
const workerFile = new URL('./worker.js', import.meta.url);
|
|
53
|
+
const workerData = {
|
|
54
|
+
benchmarkUrl: resolvedBenchmarkUrl,
|
|
55
|
+
setupCode,
|
|
56
|
+
teardownCode,
|
|
57
|
+
preCode,
|
|
58
|
+
runCode,
|
|
59
|
+
postCode,
|
|
60
|
+
data,
|
|
61
|
+
warmupCycles,
|
|
62
|
+
minCycles,
|
|
63
|
+
absThreshold,
|
|
64
|
+
relThreshold,
|
|
65
|
+
gcObserver,
|
|
66
|
+
controlSAB,
|
|
67
|
+
durationsSAB,
|
|
68
|
+
};
|
|
69
|
+
const worker = new Worker(workerFile, {
|
|
70
|
+
workerData,
|
|
71
|
+
});
|
|
72
|
+
const control = new Int32Array(controlSAB);
|
|
73
|
+
let progressIntervalId;
|
|
74
|
+
if (onProgress && id) {
|
|
75
|
+
progressIntervalId = setInterval(() => {
|
|
76
|
+
const progress = control[Control.PROGRESS] / COMPLETE_VALUE;
|
|
77
|
+
onProgress({ id, progress });
|
|
78
|
+
}, progressInterval);
|
|
79
|
+
}
|
|
80
|
+
const WORKER_TIMEOUT_MS = 300_000;
|
|
81
|
+
const exitPromise = once(worker, 'exit');
|
|
82
|
+
const timeoutId = setTimeout(() => worker.terminate(), WORKER_TIMEOUT_MS);
|
|
83
|
+
let workerError;
|
|
84
|
+
try {
|
|
85
|
+
const [exitCode] = await exitPromise;
|
|
86
|
+
clearTimeout(timeoutId);
|
|
87
|
+
if (progressIntervalId)
|
|
88
|
+
clearInterval(progressIntervalId);
|
|
89
|
+
if (exitCode !== 0) {
|
|
90
|
+
workerError = `worker exited with code ${exitCode}`;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch (err) {
|
|
94
|
+
clearTimeout(timeoutId);
|
|
95
|
+
if (progressIntervalId)
|
|
96
|
+
clearInterval(progressIntervalId);
|
|
97
|
+
workerError = err instanceof Error ? err.message : String(err);
|
|
98
|
+
}
|
|
99
|
+
const count = control[Control.INDEX];
|
|
100
|
+
const heapUsedKB = control[Control.HEAP_USED];
|
|
101
|
+
const durations = new BigUint64Array(durationsSAB).slice(0, count).sort(cmp);
|
|
102
|
+
const DCE_THRESHOLD_OPS = 5_000_000_000;
|
|
103
|
+
let dceWarning = false;
|
|
104
|
+
if (count > 0) {
|
|
105
|
+
let sum = 0n;
|
|
106
|
+
for (const d of durations)
|
|
107
|
+
sum += d;
|
|
108
|
+
const avgNs = Number(sum / BigInt(count)) / 1000;
|
|
109
|
+
const opsPerSec = avgNs > 0 ? 1_000_000_000 / avgNs : Infinity;
|
|
110
|
+
if (opsPerSec > DCE_THRESHOLD_OPS) {
|
|
111
|
+
dceWarning = true;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
const stats = count > 0 ? computeStats(durations) : undefined;
|
|
115
|
+
const report = reportTypes
|
|
116
|
+
.map((type) => [type, createReport(durations, type, stats)])
|
|
117
|
+
.concat([
|
|
118
|
+
['count', count],
|
|
119
|
+
['heapUsedKB', heapUsedKB],
|
|
120
|
+
['dceWarning', dceWarning],
|
|
121
|
+
['error', workerError],
|
|
122
|
+
]);
|
|
123
|
+
return Object.fromEntries(report);
|
|
124
|
+
};
|
|
125
|
+
return { pushAsync, kill() { } };
|
|
126
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export class GCWatcher {
|
|
2
|
+
#registry = new FinalizationRegistry(() => { });
|
|
3
|
+
start() {
|
|
4
|
+
const target = {};
|
|
5
|
+
const ref = new WeakRef(target);
|
|
6
|
+
this.#registry.register(target, null, ref);
|
|
7
|
+
return { ref };
|
|
8
|
+
}
|
|
9
|
+
seen(marker) {
|
|
10
|
+
const collected = marker.ref.deref() === undefined;
|
|
11
|
+
if (!collected) {
|
|
12
|
+
this.#registry.unregister(marker.ref);
|
|
13
|
+
}
|
|
14
|
+
return collected;
|
|
15
|
+
}
|
|
16
|
+
}
|
package/build/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { type ExecutorOptions, type ExecutorReport } from './executor.
|
|
2
|
-
import { type MaybePromise, type StepFn, type SetupFn, type TeardownFn, type FeedFn, type ReportType, type ReportTypeList } from './types.
|
|
1
|
+
import { type ExecutorOptions, type ExecutorReport } from './executor.js';
|
|
2
|
+
import { type MaybePromise, type StepFn, type SetupFn, type TeardownFn, type FeedFn, type ReportType, type ReportTypeList } from './types.js';
|
|
3
3
|
declare global {
|
|
4
4
|
const benchmark: typeof Benchmark.create;
|
|
5
5
|
}
|
package/build/index.js
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
import { cpus } from 'node:os';
|
|
2
|
+
import Progress from 'progress';
|
|
3
|
+
import { createExecutor } from './executor.js';
|
|
4
|
+
import { DEFAULT_CYCLES } from './types.js';
|
|
5
|
+
export const DEFAULT_WORKERS = Math.max(1, Math.ceil(cpus().length / 4));
|
|
6
|
+
const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
|
|
7
|
+
export const DEFAULT_REPORT_TYPES = ['ops'];
|
|
8
|
+
const createExecutorErrorReport = (error) => ({
|
|
9
|
+
count: 0,
|
|
10
|
+
heapUsedKB: 0,
|
|
11
|
+
dceWarning: false,
|
|
12
|
+
error: error instanceof Error ? error.message : String(error),
|
|
13
|
+
});
|
|
14
|
+
export class MeasureContext {
|
|
15
|
+
pre;
|
|
16
|
+
post;
|
|
17
|
+
title;
|
|
18
|
+
run;
|
|
19
|
+
constructor(title, run) {
|
|
20
|
+
this.title = title;
|
|
21
|
+
this.run = run;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export class Measure {
|
|
25
|
+
#ctx;
|
|
26
|
+
constructor(ctx) {
|
|
27
|
+
this.#ctx = ctx;
|
|
28
|
+
}
|
|
29
|
+
pre(fn) {
|
|
30
|
+
this.#ctx.pre = fn;
|
|
31
|
+
return this;
|
|
32
|
+
}
|
|
33
|
+
post(fn) {
|
|
34
|
+
this.#ctx.post = fn;
|
|
35
|
+
return this;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export class TargetContext {
|
|
39
|
+
teardown;
|
|
40
|
+
measures = [];
|
|
41
|
+
title;
|
|
42
|
+
setup;
|
|
43
|
+
constructor(title, setup) {
|
|
44
|
+
this.title = title;
|
|
45
|
+
this.setup = setup;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
export class Target {
|
|
49
|
+
#ctx;
|
|
50
|
+
constructor(ctx) {
|
|
51
|
+
this.#ctx = ctx;
|
|
52
|
+
}
|
|
53
|
+
teardown(fn) {
|
|
54
|
+
this.#ctx.teardown = fn;
|
|
55
|
+
return this;
|
|
56
|
+
}
|
|
57
|
+
measure(title, run) {
|
|
58
|
+
const measure = new MeasureContext(title, run);
|
|
59
|
+
this.#ctx.measures.push(measure);
|
|
60
|
+
return new Measure(measure);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
export class FeedContext {
|
|
64
|
+
title;
|
|
65
|
+
fn;
|
|
66
|
+
constructor(title, fn) {
|
|
67
|
+
this.title = title;
|
|
68
|
+
this.fn = fn;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export class Benchmark {
|
|
72
|
+
#targets = [];
|
|
73
|
+
#feeds = [];
|
|
74
|
+
#executed = false;
|
|
75
|
+
static create(title, fn) {
|
|
76
|
+
if (fn) {
|
|
77
|
+
return new Benchmark(title, fn);
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
return new Benchmark(title);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
constructor(title, fn) {
|
|
84
|
+
if (fn) {
|
|
85
|
+
this.feed(title, fn);
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
this.feed(title);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
feed(title, fn) {
|
|
92
|
+
const self = this;
|
|
93
|
+
self.#feeds.push(fn ? new FeedContext(title, fn) : new FeedContext(title));
|
|
94
|
+
return self;
|
|
95
|
+
}
|
|
96
|
+
target(title, setup) {
|
|
97
|
+
const target = new TargetContext(title, setup);
|
|
98
|
+
this.#targets.push(target);
|
|
99
|
+
return new Target(target);
|
|
100
|
+
}
|
|
101
|
+
async execute(options = {}) {
|
|
102
|
+
const { workers = DEFAULT_WORKERS, warmupCycles = 20, maxCycles = DEFAULT_CYCLES, minCycles = 50, absThreshold = 1_000, relThreshold = 0.02, gcObserver = true, reportTypes = DEFAULT_REPORT_TYPES, progress = false, progressInterval = 100, } = options;
|
|
103
|
+
if (this.#executed) {
|
|
104
|
+
throw new Error("Benchmark is executed and can't be reused");
|
|
105
|
+
}
|
|
106
|
+
this.#executed = true;
|
|
107
|
+
const benchmarkUrl = options[BENCHMARK_URL];
|
|
108
|
+
const totalBenchmarks = this.#targets.reduce((acc, t) => acc + t.measures.length * this.#feeds.length, 0);
|
|
109
|
+
const progressMap = new Map();
|
|
110
|
+
let completedBenchmarks = 0;
|
|
111
|
+
let bar = null;
|
|
112
|
+
if (progress && totalBenchmarks > 0) {
|
|
113
|
+
bar = new Progress(' [:bar] :percent :current/:total :label', {
|
|
114
|
+
total: totalBenchmarks * 100,
|
|
115
|
+
width: 30,
|
|
116
|
+
complete: '=',
|
|
117
|
+
incomplete: ' ',
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
const onProgress = progress
|
|
121
|
+
? (info) => {
|
|
122
|
+
progressMap.set(info.id, info.progress);
|
|
123
|
+
const totalProgress = (completedBenchmarks + [...progressMap.values()].reduce((a, b) => a + b, 0)) * 100;
|
|
124
|
+
const label = info.id.length > 30 ? info.id.slice(0, 27) + '...' : info.id;
|
|
125
|
+
bar?.update(totalProgress / (totalBenchmarks * 100), { label });
|
|
126
|
+
}
|
|
127
|
+
: undefined;
|
|
128
|
+
const executor = createExecutor({
|
|
129
|
+
workers,
|
|
130
|
+
warmupCycles,
|
|
131
|
+
maxCycles,
|
|
132
|
+
minCycles,
|
|
133
|
+
absThreshold,
|
|
134
|
+
relThreshold,
|
|
135
|
+
gcObserver,
|
|
136
|
+
reportTypes,
|
|
137
|
+
onProgress,
|
|
138
|
+
progressInterval,
|
|
139
|
+
[BENCHMARK_URL]: benchmarkUrl,
|
|
140
|
+
});
|
|
141
|
+
const reports = [];
|
|
142
|
+
const pendingReports = [];
|
|
143
|
+
try {
|
|
144
|
+
const feedData = await Promise.all(this.#feeds.map(async (feed) => ({ title: feed.title, data: await feed.fn?.() })));
|
|
145
|
+
for (const target of this.#targets) {
|
|
146
|
+
const targetReport = { target: target.title, measures: [] };
|
|
147
|
+
for (const measure of target.measures) {
|
|
148
|
+
const measureReport = { measure: measure.title, feeds: [] };
|
|
149
|
+
for (const feed of feedData) {
|
|
150
|
+
const id = `${target.title}/${measure.title}/${feed.title}`;
|
|
151
|
+
const feedReport = {
|
|
152
|
+
feed: feed.title,
|
|
153
|
+
data: createExecutorErrorReport('Benchmark did not produce a report'),
|
|
154
|
+
};
|
|
155
|
+
measureReport.feeds.push(feedReport);
|
|
156
|
+
pendingReports.push((async () => {
|
|
157
|
+
try {
|
|
158
|
+
feedReport.data = await executor.pushAsync({
|
|
159
|
+
id,
|
|
160
|
+
setup: target.setup,
|
|
161
|
+
teardown: target.teardown,
|
|
162
|
+
pre: measure.pre,
|
|
163
|
+
run: measure.run,
|
|
164
|
+
post: measure.post,
|
|
165
|
+
data: feed.data,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
feedReport.data = createExecutorErrorReport(error);
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
progressMap.delete(id);
|
|
173
|
+
completedBenchmarks++;
|
|
174
|
+
}
|
|
175
|
+
})());
|
|
176
|
+
}
|
|
177
|
+
targetReport.measures.push(measureReport);
|
|
178
|
+
}
|
|
179
|
+
reports.push(targetReport);
|
|
180
|
+
}
|
|
181
|
+
await Promise.all(pendingReports);
|
|
182
|
+
if (bar) {
|
|
183
|
+
bar.update(1, { label: 'done' });
|
|
184
|
+
bar.terminate();
|
|
185
|
+
}
|
|
186
|
+
return reports;
|
|
187
|
+
}
|
|
188
|
+
finally {
|
|
189
|
+
executor.kill();
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
export const printSimpleReports = (reports) => {
|
|
194
|
+
for (const report of reports) {
|
|
195
|
+
for (const { measure, feeds } of report.measures) {
|
|
196
|
+
console.group('\n', report.target, measure);
|
|
197
|
+
for (const { feed, data } of feeds) {
|
|
198
|
+
const { count, heapUsedKB, dceWarning, error: benchError, ...metrics } = data;
|
|
199
|
+
if (benchError) {
|
|
200
|
+
console.log(feed, `\x1b[31m[error: ${benchError}]\x1b[0m`);
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
const output = Object.entries(metrics)
|
|
204
|
+
.map(([key, report]) => `${key}: ${report.toString()}`)
|
|
205
|
+
.join('; ');
|
|
206
|
+
const extras = [];
|
|
207
|
+
if (heapUsedKB)
|
|
208
|
+
extras.push(`heap: ${heapUsedKB}KB`);
|
|
209
|
+
if (dceWarning)
|
|
210
|
+
extras.push('\x1b[33m[DCE warning]\x1b[0m');
|
|
211
|
+
const extrasStr = extras.length > 0 ? ` (${extras.join(', ')})` : '';
|
|
212
|
+
console.log(feed, output + extrasStr);
|
|
213
|
+
}
|
|
214
|
+
console.groupEnd();
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
};
|
|
218
|
+
export const printTableReports = (reports) => {
|
|
219
|
+
for (const report of reports) {
|
|
220
|
+
for (const { measure, feeds } of report.measures) {
|
|
221
|
+
console.log('\n', report.target, measure);
|
|
222
|
+
const table = {};
|
|
223
|
+
for (const { feed, data } of feeds) {
|
|
224
|
+
const { error: benchError } = data;
|
|
225
|
+
if (benchError) {
|
|
226
|
+
table[feed] = { error: benchError };
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
table[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
console.table(table);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
export const printJSONReports = (reports, padding) => {
|
|
237
|
+
const output = {};
|
|
238
|
+
for (const report of reports) {
|
|
239
|
+
for (const { measure, feeds } of report.measures) {
|
|
240
|
+
const row = {};
|
|
241
|
+
for (const { feed, data } of feeds) {
|
|
242
|
+
const { error: benchError } = data;
|
|
243
|
+
if (benchError) {
|
|
244
|
+
row[feed] = { error: String(benchError) };
|
|
245
|
+
}
|
|
246
|
+
else {
|
|
247
|
+
row[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
output[`${report.target} ${measure}`] = row;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
console.log(JSON.stringify(output, null, padding));
|
|
254
|
+
};
|
|
255
|
+
export const printMarkdownReports = (reports) => {
|
|
256
|
+
for (const report of reports) {
|
|
257
|
+
for (const { measure, feeds } of report.measures) {
|
|
258
|
+
console.log(`\n## ${report.target} - ${measure}\n`);
|
|
259
|
+
if (feeds.length === 0)
|
|
260
|
+
continue;
|
|
261
|
+
const firstValid = feeds.find((f) => !f.data.error);
|
|
262
|
+
if (!firstValid) {
|
|
263
|
+
for (const { feed, data } of feeds) {
|
|
264
|
+
console.log(`| ${feed} | error: ${data.error} |`);
|
|
265
|
+
}
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
const keys = Object.keys(firstValid.data).filter((k) => k !== 'count' && k !== 'error');
|
|
269
|
+
const header = ['Feed', ...keys].join(' | ');
|
|
270
|
+
const separator = ['---', ...keys.map(() => '---')].join(' | ');
|
|
271
|
+
console.log(`| ${header} |`);
|
|
272
|
+
console.log(`| ${separator} |`);
|
|
273
|
+
for (const { feed, data } of feeds) {
|
|
274
|
+
if (data.error) {
|
|
275
|
+
console.log(`| ${feed} | error: ${data.error} |`);
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
const values = keys.map((k) => data[k]?.toString() ?? '-');
|
|
279
|
+
console.log(`| ${[feed, ...values].join(' | ')} |`);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
export const printHistogramReports = (reports, width = 40) => {
|
|
285
|
+
for (const report of reports) {
|
|
286
|
+
for (const { measure, feeds } of report.measures) {
|
|
287
|
+
console.log(`\n${report.target} - ${measure}\n`);
|
|
288
|
+
const opsKey = 'ops';
|
|
289
|
+
const values = feeds.map((f) => {
|
|
290
|
+
const { error: benchError } = f.data;
|
|
291
|
+
return {
|
|
292
|
+
feed: f.feed,
|
|
293
|
+
value: benchError ? 0 : (f.data[opsKey]?.valueOf() ?? 0),
|
|
294
|
+
error: benchError,
|
|
295
|
+
};
|
|
296
|
+
});
|
|
297
|
+
const maxValue = Math.max(...values.map((v) => v.value));
|
|
298
|
+
const maxLabelLen = Math.max(...values.map((v) => v.feed.length));
|
|
299
|
+
for (const { feed, value, error } of values) {
|
|
300
|
+
const label = feed.padEnd(maxLabelLen);
|
|
301
|
+
if (error) {
|
|
302
|
+
console.log(` ${label} | \x1b[31m[error: ${error}]\x1b[0m`);
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
const barLen = maxValue > 0 ? Math.round((value / maxValue) * width) : 0;
|
|
306
|
+
const bar = '\u2588'.repeat(barLen);
|
|
307
|
+
const formatted = value.toLocaleString('en-US', { maximumFractionDigits: 2 });
|
|
308
|
+
console.log(` ${label} | ${bar} ${formatted} ops/s`);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
};
|
|
313
|
+
export const reportsToBaseline = (reports) => {
|
|
314
|
+
const results = {};
|
|
315
|
+
for (const report of reports) {
|
|
316
|
+
for (const { measure, feeds } of report.measures) {
|
|
317
|
+
for (const { feed, data } of feeds) {
|
|
318
|
+
if (data.error)
|
|
319
|
+
continue;
|
|
320
|
+
const key = `${report.target}/${measure}/${feed}`;
|
|
321
|
+
results[key] = {};
|
|
322
|
+
for (const [metric, value] of Object.entries(data)) {
|
|
323
|
+
if (metric !== 'count' && typeof value.valueOf === 'function') {
|
|
324
|
+
results[key][metric] = value.valueOf();
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return {
|
|
331
|
+
version: 1,
|
|
332
|
+
timestamp: new Date().toISOString(),
|
|
333
|
+
results,
|
|
334
|
+
};
|
|
335
|
+
};
|
|
336
|
+
export const printComparisonReports = (reports, baseline, threshold = 5) => {
|
|
337
|
+
for (const report of reports) {
|
|
338
|
+
for (const { measure, feeds } of report.measures) {
|
|
339
|
+
console.log(`\n${report.target} - ${measure}\n`);
|
|
340
|
+
for (const { feed, data } of feeds) {
|
|
341
|
+
const key = `${report.target}/${measure}/${feed}`;
|
|
342
|
+
const baselineData = baseline.results[key];
|
|
343
|
+
console.log(` ${feed}:`);
|
|
344
|
+
if (data.error) {
|
|
345
|
+
console.log(` \x1b[31m[error: ${data.error}]\x1b[0m`);
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
for (const [metric, value] of Object.entries(data)) {
|
|
349
|
+
if (metric === 'count')
|
|
350
|
+
continue;
|
|
351
|
+
const current = value.valueOf();
|
|
352
|
+
const baselineValue = baselineData?.[metric];
|
|
353
|
+
if (baselineValue !== undefined && baselineValue !== 0) {
|
|
354
|
+
const change = ((current - baselineValue) / baselineValue) * 100;
|
|
355
|
+
const isOps = metric === 'ops';
|
|
356
|
+
const improved = isOps ? change > threshold : change < -threshold;
|
|
357
|
+
const regressed = isOps ? change < -threshold : change > threshold;
|
|
358
|
+
let indicator = ' ';
|
|
359
|
+
if (improved)
|
|
360
|
+
indicator = '\x1b[32m+\x1b[0m';
|
|
361
|
+
else if (regressed)
|
|
362
|
+
indicator = '\x1b[31m!\x1b[0m';
|
|
363
|
+
const changeStr = change >= 0 ? `+${change.toFixed(1)}%` : `${change.toFixed(1)}%`;
|
|
364
|
+
const coloredChange = regressed ? `\x1b[31m${changeStr}\x1b[0m` : improved ? `\x1b[32m${changeStr}\x1b[0m` : changeStr;
|
|
365
|
+
console.log(` ${indicator} ${metric}: ${value.toString()} (${coloredChange})`);
|
|
366
|
+
}
|
|
367
|
+
else {
|
|
368
|
+
console.log(` * ${metric}: ${value.toString()} (new)`);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
console.log(`\nBaseline from: ${baseline.timestamp}`);
|
|
375
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|