benchforge 0.1.8 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -42
- package/dist/{BenchRunner-CSKN9zPy.d.mts → BenchRunner-BzyUfiyB.d.mts} +32 -8
- package/dist/{BrowserHeapSampler-DCeL42RE.mjs → BrowserHeapSampler-B6asLKWQ.mjs} +57 -57
- package/dist/BrowserHeapSampler-B6asLKWQ.mjs.map +1 -0
- package/dist/{GcStats-ByEovUi1.mjs → GcStats-wX7Xyblu.mjs} +15 -15
- package/dist/GcStats-wX7Xyblu.mjs.map +1 -0
- package/dist/HeapSampler-B8dtKHn1.mjs.map +1 -1
- package/dist/{TimingUtils-ClclVQ7E.mjs → TimingUtils-DwOwkc8G.mjs} +225 -225
- package/dist/TimingUtils-DwOwkc8G.mjs.map +1 -0
- package/dist/bin/benchforge.mjs +1 -1
- package/dist/browser/index.js +210 -210
- package/dist/index.d.mts +106 -48
- package/dist/index.mjs +3 -3
- package/dist/runners/WorkerScript.d.mts +1 -1
- package/dist/runners/WorkerScript.mjs +66 -66
- package/dist/runners/WorkerScript.mjs.map +1 -1
- package/dist/{src-HfimYuW_.mjs → src-B-DDaCa9.mjs} +1250 -991
- package/dist/src-B-DDaCa9.mjs.map +1 -0
- package/package.json +4 -3
- package/src/BenchMatrix.ts +125 -125
- package/src/BenchmarkReport.ts +50 -45
- package/src/HtmlDataPrep.ts +21 -21
- package/src/PermutationTest.ts +24 -24
- package/src/StandardSections.ts +45 -45
- package/src/StatisticalUtils.ts +60 -61
- package/src/browser/BrowserGcStats.ts +5 -5
- package/src/browser/BrowserHeapSampler.ts +63 -63
- package/src/cli/CliArgs.ts +20 -6
- package/src/cli/FilterBenchmarks.ts +5 -5
- package/src/cli/RunBenchCLI.ts +533 -476
- package/src/export/JsonExport.ts +10 -10
- package/src/export/PerfettoExport.ts +74 -74
- package/src/export/SpeedscopeExport.ts +202 -0
- package/src/heap-sample/HeapSampleReport.ts +143 -70
- package/src/heap-sample/HeapSampler.ts +55 -12
- package/src/heap-sample/ResolvedProfile.ts +89 -0
- package/src/html/HtmlReport.ts +33 -33
- package/src/html/HtmlTemplate.ts +67 -67
- package/src/html/browser/CIPlot.ts +50 -50
- package/src/html/browser/HistogramKde.ts +13 -13
- package/src/html/browser/LegendUtils.ts +48 -48
- package/src/html/browser/RenderPlots.ts +98 -98
- package/src/html/browser/SampleTimeSeries.ts +79 -79
- package/src/index.ts +6 -0
- package/src/matrix/MatrixFilter.ts +6 -6
- package/src/matrix/MatrixReport.ts +96 -96
- package/src/matrix/VariantLoader.ts +5 -5
- package/src/runners/AdaptiveWrapper.ts +151 -151
- package/src/runners/BasicRunner.ts +175 -175
- package/src/runners/BenchRunner.ts +8 -8
- package/src/runners/GcStats.ts +22 -22
- package/src/runners/RunnerOrchestrator.ts +168 -168
- package/src/runners/WorkerScript.ts +96 -96
- package/src/table-util/Formatters.ts +41 -36
- package/src/table-util/TableReport.ts +122 -122
- package/src/table-util/test/TableValueExtractor.ts +9 -9
- package/src/test/AdaptiveStatistics.integration.ts +7 -39
- package/src/test/HeapAttribution.test.ts +51 -0
- package/src/test/RunBenchCLI.test.ts +36 -11
- package/src/test/TestUtils.ts +24 -24
- package/src/test/fixtures/fn-export-bench.ts +3 -0
- package/src/test/fixtures/suite-export-bench.ts +16 -0
- package/src/tests/BenchMatrix.test.ts +12 -12
- package/src/tests/MatrixFilter.test.ts +15 -15
- package/dist/BrowserHeapSampler-DCeL42RE.mjs.map +0 -1
- package/dist/GcStats-ByEovUi1.mjs.map +0 -1
- package/dist/TimingUtils-ClclVQ7E.mjs.map +0 -1
- package/dist/src-HfimYuW_.mjs.map +0 -1
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
import { a as createAdaptiveWrapper, c as BasicRunner, i as createRunner, l as computeStats, n as getElapsed, o as average, r as getPerfNow, s as bootstrapDifferenceCI, t as debugWorkerTiming, u as discoverVariants } from "./TimingUtils-
|
|
2
|
-
import { n as parseGcLine, t as aggregateGcStats } from "./GcStats-
|
|
1
|
+
import { a as createAdaptiveWrapper, c as BasicRunner, i as createRunner, l as computeStats, n as getElapsed, o as average, r as getPerfNow, s as bootstrapDifferenceCI, t as debugWorkerTiming, u as discoverVariants } from "./TimingUtils-DwOwkc8G.mjs";
|
|
2
|
+
import { n as parseGcLine, t as aggregateGcStats } from "./GcStats-wX7Xyblu.mjs";
|
|
3
3
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
4
|
-
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
5
5
|
import { execSync, fork, spawn } from "node:child_process";
|
|
6
6
|
import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
7
|
-
import path, { dirname, extname, join, resolve } from "node:path";
|
|
7
|
+
import path, { basename, dirname, extname, join, resolve } from "node:path";
|
|
8
8
|
import pico from "picocolors";
|
|
9
9
|
import { table } from "table";
|
|
10
10
|
import yargs from "yargs";
|
|
11
11
|
import { hideBin } from "yargs/helpers";
|
|
12
|
+
import { tmpdir } from "node:os";
|
|
12
13
|
import { createServer } from "node:http";
|
|
13
14
|
import open from "open";
|
|
14
15
|
|
|
@@ -50,6 +51,25 @@ async function runBenchmark({ spec, runner, options, useWorker = false, params }
|
|
|
50
51
|
params
|
|
51
52
|
});
|
|
52
53
|
}
|
|
54
|
+
/** Run a matrix variant benchmark in isolated worker process */
|
|
55
|
+
async function runMatrixVariant(params) {
|
|
56
|
+
const { variantDir, variantId, caseId, caseData, casesModule, runner, options } = params;
|
|
57
|
+
const name = `${variantId}/${caseId}`;
|
|
58
|
+
return runWorkerWithMessage(name, options, {
|
|
59
|
+
type: "run",
|
|
60
|
+
spec: {
|
|
61
|
+
name,
|
|
62
|
+
fn: () => {}
|
|
63
|
+
},
|
|
64
|
+
runnerName: runner,
|
|
65
|
+
options,
|
|
66
|
+
variantDir,
|
|
67
|
+
variantId,
|
|
68
|
+
caseId,
|
|
69
|
+
caseData,
|
|
70
|
+
casesModule
|
|
71
|
+
});
|
|
72
|
+
}
|
|
53
73
|
/** Resolve modulePath/exportName to a real function for non-worker mode */
|
|
54
74
|
async function resolveModuleSpec(spec, params) {
|
|
55
75
|
const module = await import(spec.modulePath);
|
|
@@ -81,6 +101,34 @@ async function runInWorker(workerParams) {
|
|
|
81
101
|
const msg = createRunMessage(spec, runner, options, params);
|
|
82
102
|
return runWorkerWithMessage(spec.name, options, msg);
|
|
83
103
|
}
|
|
104
|
+
/** Spawn worker, wire handlers, send message, return results */
|
|
105
|
+
function runWorkerWithMessage(name, options, message) {
|
|
106
|
+
const startTime = getPerfNow();
|
|
107
|
+
const collectGcStats = options.gcStats ?? false;
|
|
108
|
+
logTiming(`Starting worker for ${name}`);
|
|
109
|
+
return new Promise((resolve, reject) => {
|
|
110
|
+
const { worker, createTime, gcEvents } = createWorkerWithTiming(collectGcStats);
|
|
111
|
+
setupWorkerHandlers(worker, name, createWorkerHandlers(name, startTime, gcEvents, resolve, reject));
|
|
112
|
+
sendWorkerMessage(worker, message, createTime);
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
/** Create message for worker execution */
|
|
116
|
+
function createRunMessage(spec, runnerName, options, params) {
|
|
117
|
+
const { fn, ...rest } = spec;
|
|
118
|
+
const message = {
|
|
119
|
+
type: "run",
|
|
120
|
+
spec: rest,
|
|
121
|
+
runnerName,
|
|
122
|
+
options,
|
|
123
|
+
params
|
|
124
|
+
};
|
|
125
|
+
if (spec.modulePath) {
|
|
126
|
+
message.modulePath = spec.modulePath;
|
|
127
|
+
message.exportName = spec.exportName;
|
|
128
|
+
if (spec.setupExportName) message.setupExportName = spec.setupExportName;
|
|
129
|
+
} else message.fnCode = fn.toString();
|
|
130
|
+
return message;
|
|
131
|
+
}
|
|
84
132
|
/** Create worker process with timing logs */
|
|
85
133
|
function createWorkerWithTiming(gcStats) {
|
|
86
134
|
const workerStart = getPerfNow();
|
|
@@ -95,6 +143,49 @@ function createWorkerWithTiming(gcStats) {
|
|
|
95
143
|
gcEvents
|
|
96
144
|
};
|
|
97
145
|
}
|
|
146
|
+
/** @return handlers that attach GC stats and heap profile to results */
|
|
147
|
+
function createWorkerHandlers(specName, startTime, gcEvents, resolve, reject) {
|
|
148
|
+
return {
|
|
149
|
+
resolve: (results, heapProfile) => {
|
|
150
|
+
logTiming(`Total worker time for ${specName}: ${getElapsed(startTime).toFixed(1)}ms`);
|
|
151
|
+
if (gcEvents?.length) {
|
|
152
|
+
const gcStats = aggregateGcStats(gcEvents);
|
|
153
|
+
for (const r of results) r.gcStats = gcStats;
|
|
154
|
+
}
|
|
155
|
+
if (heapProfile) for (const r of results) r.heapProfile = heapProfile;
|
|
156
|
+
resolve(results);
|
|
157
|
+
},
|
|
158
|
+
reject
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
/** Setup worker event handlers with cleanup */
|
|
162
|
+
function setupWorkerHandlers(worker, specName, handlers) {
|
|
163
|
+
const { resolve, reject } = handlers;
|
|
164
|
+
const cleanup = createCleanup(worker, specName, reject);
|
|
165
|
+
worker.on("message", createMessageHandler(specName, cleanup, resolve, reject));
|
|
166
|
+
worker.on("error", createErrorHandler(specName, cleanup, reject));
|
|
167
|
+
worker.on("exit", createExitHandler(specName, cleanup, reject));
|
|
168
|
+
}
|
|
169
|
+
/** Send message to worker with timing log */
|
|
170
|
+
function sendWorkerMessage(worker, message, createTime) {
|
|
171
|
+
const messageTime = getPerfNow();
|
|
172
|
+
worker.send(message);
|
|
173
|
+
logTiming(`Message sent to worker in ${getElapsed(createTime, messageTime).toFixed(1)}ms`);
|
|
174
|
+
}
|
|
175
|
+
/** Create worker process with configuration */
|
|
176
|
+
function createWorkerProcess(gcStats) {
|
|
177
|
+
const workerPath = resolveWorkerPath();
|
|
178
|
+
const execArgv = ["--expose-gc", "--allow-natives-syntax"];
|
|
179
|
+
if (gcStats) execArgv.push("--trace-gc-nvp");
|
|
180
|
+
return fork(workerPath, [], {
|
|
181
|
+
execArgv,
|
|
182
|
+
silent: gcStats,
|
|
183
|
+
env: {
|
|
184
|
+
...process.env,
|
|
185
|
+
NODE_OPTIONS: ""
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
}
|
|
98
189
|
/** Capture and parse GC lines from stdout (V8's --trace-gc-nvp outputs to stdout) */
|
|
99
190
|
function setupGcCapture(worker, gcEvents) {
|
|
100
191
|
let buffer = "";
|
|
@@ -109,30 +200,17 @@ function setupGcCapture(worker, gcEvents) {
|
|
|
109
200
|
}
|
|
110
201
|
});
|
|
111
202
|
}
|
|
112
|
-
/**
|
|
113
|
-
function
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/** Send message to worker with timing log */
|
|
124
|
-
function sendWorkerMessage(worker, message, createTime) {
|
|
125
|
-
const messageTime = getPerfNow();
|
|
126
|
-
worker.send(message);
|
|
127
|
-
logTiming(`Message sent to worker in ${getElapsed(createTime, messageTime).toFixed(1)}ms`);
|
|
128
|
-
}
|
|
129
|
-
/** Setup worker event handlers with cleanup */
|
|
130
|
-
function setupWorkerHandlers(worker, specName, handlers) {
|
|
131
|
-
const { resolve, reject } = handlers;
|
|
132
|
-
const cleanup = createCleanup(worker, specName, reject);
|
|
133
|
-
worker.on("message", createMessageHandler(specName, cleanup, resolve, reject));
|
|
134
|
-
worker.on("error", createErrorHandler(specName, cleanup, reject));
|
|
135
|
-
worker.on("exit", createExitHandler(specName, cleanup, reject));
|
|
203
|
+
/** Create cleanup for timeout and termination */
|
|
204
|
+
function createCleanup(worker, specName, reject) {
|
|
205
|
+
const timeoutId = setTimeout(() => {
|
|
206
|
+
cleanup();
|
|
207
|
+
reject(/* @__PURE__ */ new Error(`Benchmark "${specName}" timed out after 60 seconds`));
|
|
208
|
+
}, 6e4);
|
|
209
|
+
const cleanup = () => {
|
|
210
|
+
clearTimeout(timeoutId);
|
|
211
|
+
if (!worker.killed) worker.kill("SIGTERM");
|
|
212
|
+
};
|
|
213
|
+
return cleanup;
|
|
136
214
|
}
|
|
137
215
|
/** Handle worker messages (results or errors) */
|
|
138
216
|
function createMessageHandler(specName, cleanup, resolve, reject) {
|
|
@@ -163,32 +241,6 @@ function createExitHandler(specName, cleanup, reject) {
|
|
|
163
241
|
}
|
|
164
242
|
};
|
|
165
243
|
}
|
|
166
|
-
/** Create cleanup for timeout and termination */
|
|
167
|
-
function createCleanup(worker, specName, reject) {
|
|
168
|
-
const timeoutId = setTimeout(() => {
|
|
169
|
-
cleanup();
|
|
170
|
-
reject(/* @__PURE__ */ new Error(`Benchmark "${specName}" timed out after 60 seconds`));
|
|
171
|
-
}, 6e4);
|
|
172
|
-
const cleanup = () => {
|
|
173
|
-
clearTimeout(timeoutId);
|
|
174
|
-
if (!worker.killed) worker.kill("SIGTERM");
|
|
175
|
-
};
|
|
176
|
-
return cleanup;
|
|
177
|
-
}
|
|
178
|
-
/** Create worker process with configuration */
|
|
179
|
-
function createWorkerProcess(gcStats) {
|
|
180
|
-
const workerPath = resolveWorkerPath();
|
|
181
|
-
const execArgv = ["--expose-gc", "--allow-natives-syntax"];
|
|
182
|
-
if (gcStats) execArgv.push("--trace-gc-nvp");
|
|
183
|
-
return fork(workerPath, [], {
|
|
184
|
-
execArgv,
|
|
185
|
-
silent: gcStats,
|
|
186
|
-
env: {
|
|
187
|
-
...process.env,
|
|
188
|
-
NODE_OPTIONS: ""
|
|
189
|
-
}
|
|
190
|
-
});
|
|
191
|
-
}
|
|
192
244
|
/** Resolve WorkerScript path for dev (.ts) or dist (.mjs) */
|
|
193
245
|
function resolveWorkerPath() {
|
|
194
246
|
const dir = import.meta.dirname;
|
|
@@ -196,57 +248,6 @@ function resolveWorkerPath() {
|
|
|
196
248
|
if (existsSync(tsPath)) return tsPath;
|
|
197
249
|
return path.join(dir, "runners", "WorkerScript.mjs");
|
|
198
250
|
}
|
|
199
|
-
/** @return handlers that attach GC stats and heap profile to results */
|
|
200
|
-
function createWorkerHandlers(specName, startTime, gcEvents, resolve, reject) {
|
|
201
|
-
return {
|
|
202
|
-
resolve: (results, heapProfile) => {
|
|
203
|
-
logTiming(`Total worker time for ${specName}: ${getElapsed(startTime).toFixed(1)}ms`);
|
|
204
|
-
if (gcEvents?.length) {
|
|
205
|
-
const gcStats = aggregateGcStats(gcEvents);
|
|
206
|
-
for (const r of results) r.gcStats = gcStats;
|
|
207
|
-
}
|
|
208
|
-
if (heapProfile) for (const r of results) r.heapProfile = heapProfile;
|
|
209
|
-
resolve(results);
|
|
210
|
-
},
|
|
211
|
-
reject
|
|
212
|
-
};
|
|
213
|
-
}
|
|
214
|
-
/** Create message for worker execution */
|
|
215
|
-
function createRunMessage(spec, runnerName, options, params) {
|
|
216
|
-
const { fn, ...rest } = spec;
|
|
217
|
-
const message = {
|
|
218
|
-
type: "run",
|
|
219
|
-
spec: rest,
|
|
220
|
-
runnerName,
|
|
221
|
-
options,
|
|
222
|
-
params
|
|
223
|
-
};
|
|
224
|
-
if (spec.modulePath) {
|
|
225
|
-
message.modulePath = spec.modulePath;
|
|
226
|
-
message.exportName = spec.exportName;
|
|
227
|
-
if (spec.setupExportName) message.setupExportName = spec.setupExportName;
|
|
228
|
-
} else message.fnCode = fn.toString();
|
|
229
|
-
return message;
|
|
230
|
-
}
|
|
231
|
-
/** Run a matrix variant benchmark in isolated worker process */
|
|
232
|
-
async function runMatrixVariant(params) {
|
|
233
|
-
const { variantDir, variantId, caseId, caseData, casesModule, runner, options } = params;
|
|
234
|
-
const name = `${variantId}/${caseId}`;
|
|
235
|
-
return runWorkerWithMessage(name, options, {
|
|
236
|
-
type: "run",
|
|
237
|
-
spec: {
|
|
238
|
-
name,
|
|
239
|
-
fn: () => {}
|
|
240
|
-
},
|
|
241
|
-
runnerName: runner,
|
|
242
|
-
options,
|
|
243
|
-
variantDir,
|
|
244
|
-
variantId,
|
|
245
|
-
caseId,
|
|
246
|
-
caseData,
|
|
247
|
-
casesModule
|
|
248
|
-
});
|
|
249
|
-
}
|
|
250
251
|
|
|
251
252
|
//#endregion
|
|
252
253
|
//#region src/BenchMatrix.ts
|
|
@@ -270,31 +271,15 @@ function validateBaseline(matrix) {
|
|
|
270
271
|
const msg = "BenchMatrix cannot have both 'baselineDir' and 'baselineVariant'";
|
|
271
272
|
if (matrix.baselineDir && matrix.baselineVariant) throw new Error(msg);
|
|
272
273
|
}
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
cpuCounters: options.cpuCounters,
|
|
280
|
-
traceOpt: options.traceOpt,
|
|
281
|
-
noSettle: options.noSettle,
|
|
282
|
-
pauseFirst: options.pauseFirst,
|
|
283
|
-
pauseInterval: options.pauseInterval,
|
|
284
|
-
pauseDuration: options.pauseDuration,
|
|
285
|
-
gcStats: options.gcStats,
|
|
286
|
-
heapSample: options.heapSample,
|
|
287
|
-
heapInterval: options.heapInterval,
|
|
288
|
-
heapDepth: options.heapDepth
|
|
289
|
-
};
|
|
290
|
-
}
|
|
291
|
-
/** Load cases module and resolve filtered case IDs */
|
|
292
|
-
async function resolveCases(matrix, options) {
|
|
293
|
-
const casesModule = matrix.casesModule ? await loadCasesModule(matrix.casesModule) : void 0;
|
|
294
|
-
const allCaseIds = casesModule?.cases ?? matrix.cases ?? ["default"];
|
|
274
|
+
/** Run matrix with variantDir (worker mode for memory isolation) */
|
|
275
|
+
async function runMatrixWithDir(matrix, options) {
|
|
276
|
+
const allVariantIds = await discoverVariants(matrix.variantDir);
|
|
277
|
+
if (allVariantIds.length === 0) throw new Error(`No variants found in ${matrix.variantDir}`);
|
|
278
|
+
const variants = await runDirVariants(options.filteredVariants ?? allVariantIds, await createDirContext(matrix, options));
|
|
279
|
+
if (matrix.baselineVariant) applyBaselineVariant(variants, matrix.baselineVariant);
|
|
295
280
|
return {
|
|
296
|
-
|
|
297
|
-
|
|
281
|
+
name: matrix.name,
|
|
282
|
+
variants
|
|
298
283
|
};
|
|
299
284
|
}
|
|
300
285
|
/** Run matrix with inline variants (non-worker mode) */
|
|
@@ -328,17 +313,6 @@ async function runMatrixInline(matrix, options) {
|
|
|
328
313
|
variants
|
|
329
314
|
};
|
|
330
315
|
}
|
|
331
|
-
/** Run matrix with variantDir (worker mode for memory isolation) */
|
|
332
|
-
async function runMatrixWithDir(matrix, options) {
|
|
333
|
-
const allVariantIds = await discoverVariants(matrix.variantDir);
|
|
334
|
-
if (allVariantIds.length === 0) throw new Error(`No variants found in ${matrix.variantDir}`);
|
|
335
|
-
const variants = await runDirVariants(options.filteredVariants ?? allVariantIds, await createDirContext(matrix, options));
|
|
336
|
-
if (matrix.baselineVariant) applyBaselineVariant(variants, matrix.baselineVariant);
|
|
337
|
-
return {
|
|
338
|
-
name: matrix.name,
|
|
339
|
-
variants
|
|
340
|
-
};
|
|
341
|
-
}
|
|
342
316
|
/** Create context for directory-based matrix execution */
|
|
343
317
|
async function createDirContext(matrix, options) {
|
|
344
318
|
const baselineIds = matrix.baselineDir ? await discoverVariants(matrix.baselineDir) : [];
|
|
@@ -363,56 +337,6 @@ async function runDirVariants(variantIds, ctx) {
|
|
|
363
337
|
}
|
|
364
338
|
return variants;
|
|
365
339
|
}
|
|
366
|
-
/** Run all cases for a single variant */
|
|
367
|
-
async function runDirVariantCases(variantId, ctx) {
|
|
368
|
-
const { matrix, casesModule, caseIds, runnerOpts } = ctx;
|
|
369
|
-
const cases = [];
|
|
370
|
-
for (const caseId of caseIds) {
|
|
371
|
-
const caseData = !matrix.casesModule && matrix.cases ? caseId : void 0;
|
|
372
|
-
const [measured] = await runMatrixVariant({
|
|
373
|
-
variantDir: matrix.variantDir,
|
|
374
|
-
variantId,
|
|
375
|
-
caseId,
|
|
376
|
-
caseData,
|
|
377
|
-
casesModule: matrix.casesModule,
|
|
378
|
-
runner: "basic",
|
|
379
|
-
options: runnerOpts
|
|
380
|
-
});
|
|
381
|
-
const loaded = await loadCaseData(casesModule, caseId);
|
|
382
|
-
const baseline = await runBaselineIfExists(variantId, caseId, caseData, ctx);
|
|
383
|
-
const deltaPercent = baseline ? computeDeltaPercent(baseline, measured) : void 0;
|
|
384
|
-
const metadata = loaded.metadata;
|
|
385
|
-
cases.push({
|
|
386
|
-
caseId,
|
|
387
|
-
measured,
|
|
388
|
-
metadata,
|
|
389
|
-
baseline,
|
|
390
|
-
deltaPercent
|
|
391
|
-
});
|
|
392
|
-
}
|
|
393
|
-
return cases;
|
|
394
|
-
}
|
|
395
|
-
/** Run baseline variant if it exists in baselineDir */
|
|
396
|
-
async function runBaselineIfExists(variantId, caseId, caseData, ctx) {
|
|
397
|
-
const { matrix, baselineIds, runnerOpts } = ctx;
|
|
398
|
-
if (!matrix.baselineDir || !baselineIds.includes(variantId)) return void 0;
|
|
399
|
-
const [measured] = await runMatrixVariant({
|
|
400
|
-
variantDir: matrix.baselineDir,
|
|
401
|
-
variantId,
|
|
402
|
-
caseId,
|
|
403
|
-
caseData,
|
|
404
|
-
casesModule: matrix.casesModule,
|
|
405
|
-
runner: "basic",
|
|
406
|
-
options: runnerOpts
|
|
407
|
-
});
|
|
408
|
-
return measured;
|
|
409
|
-
}
|
|
410
|
-
/** Compute delta percentage: (current - baseline) / baseline * 100 */
|
|
411
|
-
function computeDeltaPercent(baseline, current) {
|
|
412
|
-
const baseAvg = average(baseline.samples);
|
|
413
|
-
if (baseAvg === 0) return 0;
|
|
414
|
-
return (average(current.samples) - baseAvg) / baseAvg * 100;
|
|
415
|
-
}
|
|
416
340
|
/** Apply baselineVariant comparison - one variant is the reference for all others */
|
|
417
341
|
function applyBaselineVariant(variants, baselineVariantId) {
|
|
418
342
|
const baselineVariant = variants.find((v) => v.id === baselineVariantId);
|
|
@@ -430,6 +354,33 @@ function applyBaselineVariant(variants, baselineVariantId) {
|
|
|
430
354
|
}
|
|
431
355
|
}
|
|
432
356
|
}
|
|
357
|
+
/** Load cases module and resolve filtered case IDs */
|
|
358
|
+
async function resolveCases(matrix, options) {
|
|
359
|
+
const casesModule = matrix.casesModule ? await loadCasesModule(matrix.casesModule) : void 0;
|
|
360
|
+
const allCaseIds = casesModule?.cases ?? matrix.cases ?? ["default"];
|
|
361
|
+
return {
|
|
362
|
+
casesModule,
|
|
363
|
+
caseIds: options.filteredCases ?? allCaseIds
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
function buildRunnerOptions(options) {
|
|
367
|
+
return {
|
|
368
|
+
maxIterations: options.iterations,
|
|
369
|
+
maxTime: options.maxTime ?? 1e3,
|
|
370
|
+
warmup: options.warmup ?? 0,
|
|
371
|
+
collect: options.collect,
|
|
372
|
+
cpuCounters: options.cpuCounters,
|
|
373
|
+
traceOpt: options.traceOpt,
|
|
374
|
+
noSettle: options.noSettle,
|
|
375
|
+
pauseFirst: options.pauseFirst,
|
|
376
|
+
pauseInterval: options.pauseInterval,
|
|
377
|
+
pauseDuration: options.pauseDuration,
|
|
378
|
+
gcStats: options.gcStats,
|
|
379
|
+
heapSample: options.heapSample,
|
|
380
|
+
heapInterval: options.heapInterval,
|
|
381
|
+
heapDepth: options.heapDepth
|
|
382
|
+
};
|
|
383
|
+
}
|
|
433
384
|
/** Run a single variant with case data */
|
|
434
385
|
async function runVariant(variant, caseData, name, runner, options) {
|
|
435
386
|
if (isStatefulVariant(variant)) {
|
|
@@ -446,6 +397,56 @@ async function runVariant(variant, caseData, name, runner, options) {
|
|
|
446
397
|
}, options);
|
|
447
398
|
return result;
|
|
448
399
|
}
|
|
400
|
+
/** Run all cases for a single variant */
|
|
401
|
+
async function runDirVariantCases(variantId, ctx) {
|
|
402
|
+
const { matrix, casesModule, caseIds, runnerOpts } = ctx;
|
|
403
|
+
const cases = [];
|
|
404
|
+
for (const caseId of caseIds) {
|
|
405
|
+
const caseData = !matrix.casesModule && matrix.cases ? caseId : void 0;
|
|
406
|
+
const [measured] = await runMatrixVariant({
|
|
407
|
+
variantDir: matrix.variantDir,
|
|
408
|
+
variantId,
|
|
409
|
+
caseId,
|
|
410
|
+
caseData,
|
|
411
|
+
casesModule: matrix.casesModule,
|
|
412
|
+
runner: "basic",
|
|
413
|
+
options: runnerOpts
|
|
414
|
+
});
|
|
415
|
+
const loaded = await loadCaseData(casesModule, caseId);
|
|
416
|
+
const baseline = await runBaselineIfExists(variantId, caseId, caseData, ctx);
|
|
417
|
+
const deltaPercent = baseline ? computeDeltaPercent(baseline, measured) : void 0;
|
|
418
|
+
const metadata = loaded.metadata;
|
|
419
|
+
cases.push({
|
|
420
|
+
caseId,
|
|
421
|
+
measured,
|
|
422
|
+
metadata,
|
|
423
|
+
baseline,
|
|
424
|
+
deltaPercent
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
return cases;
|
|
428
|
+
}
|
|
429
|
+
/** Compute delta percentage: (current - baseline) / baseline * 100 */
|
|
430
|
+
function computeDeltaPercent(baseline, current) {
|
|
431
|
+
const baseAvg = average(baseline.samples);
|
|
432
|
+
if (baseAvg === 0) return 0;
|
|
433
|
+
return (average(current.samples) - baseAvg) / baseAvg * 100;
|
|
434
|
+
}
|
|
435
|
+
/** Run baseline variant if it exists in baselineDir */
|
|
436
|
+
async function runBaselineIfExists(variantId, caseId, caseData, ctx) {
|
|
437
|
+
const { matrix, baselineIds, runnerOpts } = ctx;
|
|
438
|
+
if (!matrix.baselineDir || !baselineIds.includes(variantId)) return void 0;
|
|
439
|
+
const [measured] = await runMatrixVariant({
|
|
440
|
+
variantDir: matrix.baselineDir,
|
|
441
|
+
variantId,
|
|
442
|
+
caseId,
|
|
443
|
+
caseData,
|
|
444
|
+
casesModule: matrix.casesModule,
|
|
445
|
+
runner: "basic",
|
|
446
|
+
options: runnerOpts
|
|
447
|
+
});
|
|
448
|
+
return measured;
|
|
449
|
+
}
|
|
449
450
|
|
|
450
451
|
//#endregion
|
|
451
452
|
//#region src/table-util/Formatters.ts
|
|
@@ -491,21 +492,15 @@ function diffPercent(main, base) {
|
|
|
491
492
|
if (typeof main !== "number" || typeof base !== "number") return " ";
|
|
492
493
|
return coloredPercent(main - base, base);
|
|
493
494
|
}
|
|
494
|
-
/** Format
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
if (Number.isNaN(fraction) || !Number.isFinite(fraction)) return " ";
|
|
498
|
-
const positive = fraction >= 0;
|
|
499
|
-
const percentStr = `${positive ? "+" : "-"}${percent(fraction)}`;
|
|
500
|
-
return positive === positiveIsGreen ? green(percentStr) : red$1(percentStr);
|
|
501
|
-
}
|
|
502
|
-
/** Format bytes with appropriate units (B, KB, MB, GB) */
|
|
503
|
-
function formatBytes(bytes) {
|
|
495
|
+
/** Format bytes with appropriate units (B, KB, MB, GB).
|
|
496
|
+
* Use `space: true` for human-readable console output (`1.5 KB`). */
|
|
497
|
+
function formatBytes(bytes, opts) {
|
|
504
498
|
if (typeof bytes !== "number") return null;
|
|
505
|
-
|
|
506
|
-
if (bytes < 1024
|
|
507
|
-
if (bytes < 1024 * 1024
|
|
508
|
-
return `${(bytes / 1024 / 1024
|
|
499
|
+
const s = opts?.space ? " " : "";
|
|
500
|
+
if (bytes < 1024) return `${bytes.toFixed(0)}${s}B`;
|
|
501
|
+
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}${s}KB`;
|
|
502
|
+
if (bytes < 1024 * 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(1)}${s}MB`;
|
|
503
|
+
return `${(bytes / 1024 / 1024 / 1024).toFixed(1)}${s}GB`;
|
|
509
504
|
}
|
|
510
505
|
/** Format percentage difference with confidence interval */
|
|
511
506
|
function formatDiffWithCI(value) {
|
|
@@ -519,9 +514,21 @@ function formatDiffWithCIHigherIsBetter(value) {
|
|
|
519
514
|
const { percent, ci, direction } = value;
|
|
520
515
|
return colorByDirection(diffCIText(-percent, [-ci[1], -ci[0]]), direction);
|
|
521
516
|
}
|
|
522
|
-
/** @return
|
|
523
|
-
function
|
|
524
|
-
return
|
|
517
|
+
/** @return truncated string with ellipsis if over maxLen */
|
|
518
|
+
function truncate(str, maxLen = 30) {
|
|
519
|
+
return str.length > maxLen ? str.slice(0, maxLen - 3) + "..." : str;
|
|
520
|
+
}
|
|
521
|
+
/** Format fraction as colored +/- percentage */
|
|
522
|
+
function coloredPercent(numerator, denominator, positiveIsGreen = true) {
|
|
523
|
+
const fraction = numerator / denominator;
|
|
524
|
+
if (Number.isNaN(fraction) || !Number.isFinite(fraction)) return " ";
|
|
525
|
+
const positive = fraction >= 0;
|
|
526
|
+
const percentStr = `${positive ? "+" : "-"}${percent(fraction)}`;
|
|
527
|
+
return positive === positiveIsGreen ? green(percentStr) : red$1(percentStr);
|
|
528
|
+
}
|
|
529
|
+
/** @return true if value is a DifferenceCI object */
|
|
530
|
+
function isDifferenceCI(x) {
|
|
531
|
+
return typeof x === "object" && x !== null && "ci" in x && "direction" in x;
|
|
525
532
|
}
|
|
526
533
|
/** @return text colored green for faster, red for slower */
|
|
527
534
|
function colorByDirection(text, direction) {
|
|
@@ -529,78 +536,23 @@ function colorByDirection(text, direction) {
|
|
|
529
536
|
if (direction === "slower") return red$1(text);
|
|
530
537
|
return text;
|
|
531
538
|
}
|
|
539
|
+
/** @return formatted "pct [lo, hi]" text for a diff with CI */
|
|
540
|
+
function diffCIText(pct, ci) {
|
|
541
|
+
return `${formatBound(pct)} [${formatBound(ci[0])}, ${formatBound(ci[1])}]`;
|
|
542
|
+
}
|
|
532
543
|
/** @return signed percentage string (e.g. "+1.2%", "-3.4%") */
|
|
533
544
|
function formatBound(v) {
|
|
534
545
|
return `${v >= 0 ? "+" : ""}${v.toFixed(1)}%`;
|
|
535
546
|
}
|
|
536
|
-
/** @return true if value is a DifferenceCI object */
|
|
537
|
-
function isDifferenceCI(x) {
|
|
538
|
-
return typeof x === "object" && x !== null && "ci" in x && "direction" in x;
|
|
539
|
-
}
|
|
540
|
-
/** @return truncated string with ellipsis if over maxLen */
|
|
541
|
-
function truncate(str, maxLen = 30) {
|
|
542
|
-
return str.length > maxLen ? str.slice(0, maxLen - 3) + "..." : str;
|
|
543
|
-
}
|
|
544
547
|
|
|
545
548
|
//#endregion
|
|
546
549
|
//#region src/table-util/TableReport.ts
|
|
547
550
|
const { bold } = process.env.NODE_ENV === "test" || process.env.VITEST === "true" ? { bold: (str) => str } : pico;
|
|
551
|
+
const ansiEscapeRegex = new RegExp(String.fromCharCode(27) + "\\[[0-9;]*m", "g");
|
|
548
552
|
/** Build formatted table with column groups and baselines */
|
|
549
553
|
function buildTable(columnGroups, resultGroups, nameKey = "name") {
|
|
550
554
|
return createTable(columnGroups, flattenGroups(columnGroups, resultGroups, nameKey));
|
|
551
555
|
}
|
|
552
|
-
/** Convert columns and records to formatted table */
|
|
553
|
-
function createTable(groups, records) {
|
|
554
|
-
const dataRows = toRows(records, groups);
|
|
555
|
-
const { headerRows, config } = setup(groups, dataRows);
|
|
556
|
-
return table([...headerRows, ...dataRows], config);
|
|
557
|
-
}
|
|
558
|
-
/** Create header rows with group titles */
|
|
559
|
-
function createGroupHeaders(groups, numColumns) {
|
|
560
|
-
if (!groups.some((g) => g.groupTitle)) return [];
|
|
561
|
-
return [groups.flatMap((g) => {
|
|
562
|
-
return padWithBlanks(g.groupTitle ? [bold(g.groupTitle)] : [], g.columns.length);
|
|
563
|
-
}), padWithBlanks([], numColumns)];
|
|
564
|
-
}
|
|
565
|
-
/** @return draw functions for horizontal/vertical table borders */
|
|
566
|
-
function createLines(groups) {
|
|
567
|
-
const { sectionBorders, headerBottom } = calcBorders(groups);
|
|
568
|
-
function drawVerticalLine(index, size) {
|
|
569
|
-
return index === 0 || index === size || sectionBorders.includes(index);
|
|
570
|
-
}
|
|
571
|
-
function drawHorizontalLine(index, size) {
|
|
572
|
-
return index === 0 || index === size || index === headerBottom;
|
|
573
|
-
}
|
|
574
|
-
return {
|
|
575
|
-
drawHorizontalLine,
|
|
576
|
-
drawVerticalLine
|
|
577
|
-
};
|
|
578
|
-
}
|
|
579
|
-
/** @return spanning cell configs for group title headers */
|
|
580
|
-
function createSectionSpans(groups) {
|
|
581
|
-
let col = 0;
|
|
582
|
-
const alignment = "center";
|
|
583
|
-
return groups.map((g) => {
|
|
584
|
-
const colSpan = g.columns.length;
|
|
585
|
-
const span = {
|
|
586
|
-
row: 0,
|
|
587
|
-
col,
|
|
588
|
-
colSpan,
|
|
589
|
-
alignment
|
|
590
|
-
};
|
|
591
|
-
col += colSpan;
|
|
592
|
-
return span;
|
|
593
|
-
});
|
|
594
|
-
}
|
|
595
|
-
/** @return bolded column title strings */
|
|
596
|
-
function getTitles(groups) {
|
|
597
|
-
return groups.flatMap((g) => g.columns.map((c) => bold(c.title || " ")));
|
|
598
|
-
}
|
|
599
|
-
/** @return array padded with blank strings to the given length */
|
|
600
|
-
function padWithBlanks(arr, length) {
|
|
601
|
-
if (arr.length >= length) return arr;
|
|
602
|
-
return [...arr, ...Array(length - arr.length).fill(" ")];
|
|
603
|
-
}
|
|
604
556
|
/** Convert records to string arrays for table */
|
|
605
557
|
function toRows(records, groups) {
|
|
606
558
|
const allColumns = groups.flatMap((group) => group.columns);
|
|
@@ -609,20 +561,6 @@ function toRows(records, groups) {
|
|
|
609
561
|
return col.formatter ? col.formatter(value) : value;
|
|
610
562
|
})).map((row) => row.map((cell) => cell ?? " "));
|
|
611
563
|
}
|
|
612
|
-
/** Add comparison values for diff columns */
|
|
613
|
-
function addComparisons(groups, mainRecord, baselineRecord) {
|
|
614
|
-
const diffColumns = groups.flatMap((g) => g.columns).filter((col) => col.diffKey);
|
|
615
|
-
const updatedMain = { ...mainRecord };
|
|
616
|
-
for (const col of diffColumns) {
|
|
617
|
-
const dcol = col;
|
|
618
|
-
const diffKey = dcol.diffKey;
|
|
619
|
-
const mainValue = mainRecord[diffKey];
|
|
620
|
-
const baselineValue = baselineRecord[diffKey];
|
|
621
|
-
const diffStr = (dcol.diffFormatter ?? diffPercent)(mainValue, baselineValue);
|
|
622
|
-
updatedMain[col.key] = diffStr;
|
|
623
|
-
}
|
|
624
|
-
return updatedMain;
|
|
625
|
-
}
|
|
626
564
|
/** Flatten groups with spacing */
|
|
627
565
|
function flattenGroups(columnGroups, resultGroups, nameKey) {
|
|
628
566
|
return resultGroups.flatMap((group, i) => {
|
|
@@ -630,6 +568,12 @@ function flattenGroups(columnGroups, resultGroups, nameKey) {
|
|
|
630
568
|
return i === resultGroups.length - 1 ? groupRecords : [...groupRecords, {}];
|
|
631
569
|
});
|
|
632
570
|
}
|
|
571
|
+
/** Convert columns and records to formatted table */
|
|
572
|
+
function createTable(groups, records) {
|
|
573
|
+
const dataRows = toRows(records, groups);
|
|
574
|
+
const { headerRows, config } = setup(groups, dataRows);
|
|
575
|
+
return table([...headerRows, ...dataRows], config);
|
|
576
|
+
}
|
|
633
577
|
/** Process results with baseline comparisons */
|
|
634
578
|
function addBaseline(columnGroups, group, nameKey) {
|
|
635
579
|
const { results, baseline } = group;
|
|
@@ -641,23 +585,6 @@ function addBaseline(columnGroups, group, nameKey) {
|
|
|
641
585
|
};
|
|
642
586
|
return [...diffResults, markedBaseline];
|
|
643
587
|
}
|
|
644
|
-
/** Calculate vertical lines between sections and header bottom position */
|
|
645
|
-
function calcBorders(groups) {
|
|
646
|
-
if (groups.length === 0) return {
|
|
647
|
-
sectionBorders: [],
|
|
648
|
-
headerBottom: 1
|
|
649
|
-
};
|
|
650
|
-
const sectionBorders = [];
|
|
651
|
-
let border = 0;
|
|
652
|
-
for (const g of groups) {
|
|
653
|
-
border += g.columns.length;
|
|
654
|
-
sectionBorders.push(border);
|
|
655
|
-
}
|
|
656
|
-
return {
|
|
657
|
-
sectionBorders,
|
|
658
|
-
headerBottom: 3
|
|
659
|
-
};
|
|
660
|
-
}
|
|
661
588
|
/** Create headers and table configuration */
|
|
662
589
|
function setup(groups, dataRows) {
|
|
663
590
|
const titles = getTitles(groups);
|
|
@@ -671,18 +598,59 @@ function setup(groups, dataRows) {
|
|
|
671
598
|
}
|
|
672
599
|
};
|
|
673
600
|
}
|
|
674
|
-
/**
|
|
675
|
-
function
|
|
676
|
-
const
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
const
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
601
|
+
/** Add comparison values for diff columns */
|
|
602
|
+
function addComparisons(groups, mainRecord, baselineRecord) {
|
|
603
|
+
const diffColumns = groups.flatMap((g) => g.columns).filter((col) => col.diffKey);
|
|
604
|
+
const updatedMain = { ...mainRecord };
|
|
605
|
+
for (const col of diffColumns) {
|
|
606
|
+
const dcol = col;
|
|
607
|
+
const diffKey = dcol.diffKey;
|
|
608
|
+
const mainValue = mainRecord[diffKey];
|
|
609
|
+
const baselineValue = baselineRecord[diffKey];
|
|
610
|
+
const diffStr = (dcol.diffFormatter ?? diffPercent)(mainValue, baselineValue);
|
|
611
|
+
updatedMain[col.key] = diffStr;
|
|
612
|
+
}
|
|
613
|
+
return updatedMain;
|
|
614
|
+
}
|
|
615
|
+
/** @return bolded column title strings */
|
|
616
|
+
function getTitles(groups) {
|
|
617
|
+
return groups.flatMap((g) => g.columns.map((c) => bold(c.title || " ")));
|
|
618
|
+
}
|
|
619
|
+
/** Create header rows with group titles */
|
|
620
|
+
function createGroupHeaders(groups, numColumns) {
|
|
621
|
+
if (!groups.some((g) => g.groupTitle)) return [];
|
|
622
|
+
return [groups.flatMap((g) => {
|
|
623
|
+
return padWithBlanks(g.groupTitle ? [bold(g.groupTitle)] : [], g.columns.length);
|
|
624
|
+
}), padWithBlanks([], numColumns)];
|
|
625
|
+
}
|
|
626
|
+
/** @return spanning cell configs for group title headers */
|
|
627
|
+
function createSectionSpans(groups) {
|
|
628
|
+
let col = 0;
|
|
629
|
+
const alignment = "center";
|
|
630
|
+
return groups.map((g) => {
|
|
631
|
+
const colSpan = g.columns.length;
|
|
632
|
+
const span = {
|
|
633
|
+
row: 0,
|
|
634
|
+
col,
|
|
635
|
+
colSpan,
|
|
636
|
+
alignment
|
|
637
|
+
};
|
|
638
|
+
col += colSpan;
|
|
639
|
+
return span;
|
|
640
|
+
});
|
|
641
|
+
}
|
|
642
|
+
/** Calculate column widths based on content, including group titles */
|
|
643
|
+
function calcColumnWidths(groups, titles, dataRows) {
|
|
644
|
+
const widths = [];
|
|
645
|
+
for (let i = 0; i < titles.length; i++) {
|
|
646
|
+
const titleW = cellWidth(titles[i]);
|
|
647
|
+
const maxDataW = dataRows.reduce((max, row) => Math.max(max, cellWidth(row[i])), 0);
|
|
648
|
+
widths.push(Math.max(titleW, maxDataW));
|
|
649
|
+
}
|
|
650
|
+
let colIndex = 0;
|
|
651
|
+
for (const group of groups) {
|
|
652
|
+
const groupW = cellWidth(group.groupTitle);
|
|
653
|
+
if (groupW > 0) {
|
|
686
654
|
const numCols = group.columns.length;
|
|
687
655
|
const separatorWidth = (numCols - 1) * 3;
|
|
688
656
|
const needed = groupW - widths.slice(colIndex, colIndex + numCols).reduce((a, b) => a + b, 0) - separatorWidth;
|
|
@@ -695,20 +663,84 @@ function calcColumnWidths(groups, titles, dataRows) {
|
|
|
695
663
|
wrapWord: false
|
|
696
664
|
}]));
|
|
697
665
|
}
|
|
698
|
-
|
|
666
|
+
/** @return draw functions for horizontal/vertical table borders */
|
|
667
|
+
function createLines(groups) {
|
|
668
|
+
const { sectionBorders, headerBottom } = calcBorders(groups);
|
|
669
|
+
function drawVerticalLine(index, size) {
|
|
670
|
+
return index === 0 || index === size || sectionBorders.includes(index);
|
|
671
|
+
}
|
|
672
|
+
function drawHorizontalLine(index, size) {
|
|
673
|
+
return index === 0 || index === size || index === headerBottom;
|
|
674
|
+
}
|
|
675
|
+
return {
|
|
676
|
+
drawHorizontalLine,
|
|
677
|
+
drawVerticalLine
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
/** @return array padded with blank strings to the given length */
|
|
681
|
+
function padWithBlanks(arr, length) {
|
|
682
|
+
if (arr.length >= length) return arr;
|
|
683
|
+
return [...arr, ...Array(length - arr.length).fill(" ")];
|
|
684
|
+
}
|
|
699
685
|
/** Get visible length of a cell value (strips ANSI escape codes) */
|
|
700
686
|
function cellWidth(value) {
|
|
701
687
|
if (value == null) return 0;
|
|
702
688
|
return String(value).replace(ansiEscapeRegex, "").length;
|
|
703
689
|
}
|
|
690
|
+
/** Calculate vertical lines between sections and header bottom position */
|
|
691
|
+
function calcBorders(groups) {
|
|
692
|
+
if (groups.length === 0) return {
|
|
693
|
+
sectionBorders: [],
|
|
694
|
+
headerBottom: 1
|
|
695
|
+
};
|
|
696
|
+
const sectionBorders = [];
|
|
697
|
+
let border = 0;
|
|
698
|
+
for (const g of groups) {
|
|
699
|
+
border += g.columns.length;
|
|
700
|
+
sectionBorders.push(border);
|
|
701
|
+
}
|
|
702
|
+
return {
|
|
703
|
+
sectionBorders,
|
|
704
|
+
headerBottom: 3
|
|
705
|
+
};
|
|
706
|
+
}
|
|
704
707
|
|
|
705
708
|
//#endregion
|
|
706
709
|
//#region src/BenchmarkReport.ts
|
|
710
|
+
/** All reports in a group, including the baseline if present */
|
|
711
|
+
function groupReports(group) {
|
|
712
|
+
return group.baseline ? [...group.reports, group.baseline] : group.reports;
|
|
713
|
+
}
|
|
707
714
|
/** @return formatted table report with optional baseline comparisons */
|
|
708
715
|
function reportResults(groups, sections) {
|
|
709
716
|
const results = groups.map((group) => resultGroupValues(group, sections));
|
|
710
717
|
return buildTable(createColumnGroups(sections, results.some((g) => g.baseline)), results);
|
|
711
718
|
}
|
|
719
|
+
/** @return rows with stats from sections */
|
|
720
|
+
function valuesForReports(reports, sections) {
|
|
721
|
+
return reports.map((report) => ({
|
|
722
|
+
name: truncate(report.name),
|
|
723
|
+
...extractReportValues(report, sections)
|
|
724
|
+
}));
|
|
725
|
+
}
|
|
726
|
+
/** @return groups with single CI column after first comparable field */
|
|
727
|
+
function injectDiffColumns(reportGroups) {
|
|
728
|
+
let ciAdded = false;
|
|
729
|
+
return reportGroups.map((group) => ({
|
|
730
|
+
groupTitle: group.groupTitle,
|
|
731
|
+
columns: group.columns.flatMap((col) => {
|
|
732
|
+
if (col.comparable && !ciAdded) {
|
|
733
|
+
ciAdded = true;
|
|
734
|
+
return [col, {
|
|
735
|
+
title: "Δ% CI",
|
|
736
|
+
key: "diffCI",
|
|
737
|
+
formatter: col.higherIsBetter ? formatDiffWithCIHigherIsBetter : formatDiffWithCI
|
|
738
|
+
}];
|
|
739
|
+
}
|
|
740
|
+
return [col];
|
|
741
|
+
})
|
|
742
|
+
}));
|
|
743
|
+
}
|
|
712
744
|
/** @return values for report group */
|
|
713
745
|
function resultGroupValues(group, sections) {
|
|
714
746
|
const { reports, baseline } = group;
|
|
@@ -725,19 +757,6 @@ function resultGroupValues(group, sections) {
|
|
|
725
757
|
baseline: baseline && valuesForReports([baseline], sections)[0]
|
|
726
758
|
};
|
|
727
759
|
}
|
|
728
|
-
/** @return rows with stats from sections */
|
|
729
|
-
function valuesForReports(reports, sections) {
|
|
730
|
-
return reports.map((report) => ({
|
|
731
|
-
name: truncate(report.name),
|
|
732
|
-
...extractReportValues(report, sections)
|
|
733
|
-
}));
|
|
734
|
-
}
|
|
735
|
-
/** @return merged statistics from all sections */
|
|
736
|
-
function extractReportValues(report, sections) {
|
|
737
|
-
const { measuredResults, metadata } = report;
|
|
738
|
-
const entries = sections.flatMap((s) => Object.entries(s.extract(measuredResults, metadata)));
|
|
739
|
-
return Object.fromEntries(entries);
|
|
740
|
-
}
|
|
741
760
|
/** @return column groups with diff columns if baseline exists */
|
|
742
761
|
function createColumnGroups(sections, hasBaseline) {
|
|
743
762
|
const nameColumn = { columns: [{
|
|
@@ -747,23 +766,11 @@ function createColumnGroups(sections, hasBaseline) {
|
|
|
747
766
|
const groups = sections.flatMap((section) => section.columns());
|
|
748
767
|
return [nameColumn, ...hasBaseline ? injectDiffColumns(groups) : groups];
|
|
749
768
|
}
|
|
750
|
-
/** @return
|
|
751
|
-
function
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
columns: group.columns.flatMap((col) => {
|
|
756
|
-
if (col.comparable && !ciAdded) {
|
|
757
|
-
ciAdded = true;
|
|
758
|
-
return [col, {
|
|
759
|
-
title: "Δ% CI",
|
|
760
|
-
key: "diffCI",
|
|
761
|
-
formatter: col.higherIsBetter ? formatDiffWithCIHigherIsBetter : formatDiffWithCI
|
|
762
|
-
}];
|
|
763
|
-
}
|
|
764
|
-
return [col];
|
|
765
|
-
})
|
|
766
|
-
}));
|
|
769
|
+
/** @return merged statistics from all sections */
|
|
770
|
+
function extractReportValues(report, sections) {
|
|
771
|
+
const { measuredResults, metadata } = report;
|
|
772
|
+
const entries = sections.flatMap((s) => Object.entries(s.extract(measuredResults, metadata)));
|
|
773
|
+
return Object.fromEntries(entries);
|
|
767
774
|
}
|
|
768
775
|
|
|
769
776
|
//#endregion
|
|
@@ -846,11 +853,21 @@ const cliOptions = {
|
|
|
846
853
|
requiresArg: true,
|
|
847
854
|
describe: "export benchmark data to JSON file"
|
|
848
855
|
},
|
|
849
|
-
perfetto: {
|
|
856
|
+
"export-perfetto": {
|
|
850
857
|
type: "string",
|
|
851
858
|
requiresArg: true,
|
|
852
859
|
describe: "export Perfetto trace file (view at ui.perfetto.dev)"
|
|
853
860
|
},
|
|
861
|
+
speedscope: {
|
|
862
|
+
type: "boolean",
|
|
863
|
+
default: false,
|
|
864
|
+
describe: "open heap profile in speedscope (via npx)"
|
|
865
|
+
},
|
|
866
|
+
"export-speedscope": {
|
|
867
|
+
type: "string",
|
|
868
|
+
requiresArg: true,
|
|
869
|
+
describe: "export heap profile as speedscope JSON"
|
|
870
|
+
},
|
|
854
871
|
"trace-opt": {
|
|
855
872
|
type: "boolean",
|
|
856
873
|
default: false,
|
|
@@ -915,6 +932,11 @@ const cliOptions = {
|
|
|
915
932
|
default: false,
|
|
916
933
|
describe: "verbose output with file:// paths and line numbers"
|
|
917
934
|
},
|
|
935
|
+
"heap-raw": {
|
|
936
|
+
type: "boolean",
|
|
937
|
+
default: false,
|
|
938
|
+
describe: "dump every raw heap sample (ordinal, size, stack)"
|
|
939
|
+
},
|
|
918
940
|
"heap-user-only": {
|
|
919
941
|
type: "boolean",
|
|
920
942
|
default: false,
|
|
@@ -944,7 +966,12 @@ const cliOptions = {
|
|
|
944
966
|
};
|
|
945
967
|
/** @return yargs with standard benchmark options */
|
|
946
968
|
function defaultCliArgs(yargsInstance) {
|
|
947
|
-
return yargsInstance.
|
|
969
|
+
return yargsInstance.command("$0 [file]", "run benchmarks", (y) => {
|
|
970
|
+
y.positional("file", {
|
|
971
|
+
type: "string",
|
|
972
|
+
describe: "benchmark file to run"
|
|
973
|
+
});
|
|
974
|
+
}).options(cliOptions).help().strict();
|
|
948
975
|
}
|
|
949
976
|
/** @return parsed command line arguments */
|
|
950
977
|
function parseCliArgs(args, configure = defaultCliArgs) {
|
|
@@ -978,6 +1005,12 @@ function prepareJsonData(groups, args, suiteName) {
|
|
|
978
1005
|
}]
|
|
979
1006
|
};
|
|
980
1007
|
}
|
|
1008
|
+
/** Clean CLI args for JSON export (remove undefined values) */
|
|
1009
|
+
function cleanCliArgs(args) {
|
|
1010
|
+
const toCamel = (k) => k.replace(/-([a-z])/g, (_, l) => l.toUpperCase());
|
|
1011
|
+
const entries = Object.entries(args).filter(([, v]) => v !== void 0 && v !== null).map(([k, v]) => [toCamel(k), v]);
|
|
1012
|
+
return Object.fromEntries(entries);
|
|
1013
|
+
}
|
|
981
1014
|
/** Convert a report group, mapping each report to the JSON result format */
|
|
982
1015
|
function convertGroup(group) {
|
|
983
1016
|
return {
|
|
@@ -1021,12 +1054,6 @@ function convertReport(report) {
|
|
|
1021
1054
|
}
|
|
1022
1055
|
};
|
|
1023
1056
|
}
|
|
1024
|
-
/** Clean CLI args for JSON export (remove undefined values) */
|
|
1025
|
-
function cleanCliArgs(args) {
|
|
1026
|
-
const toCamel = (k) => k.replace(/-([a-z])/g, (_, l) => l.toUpperCase());
|
|
1027
|
-
const entries = Object.entries(args).filter(([, v]) => v !== void 0 && v !== null).map(([k, v]) => [toCamel(k), v]);
|
|
1028
|
-
return Object.fromEntries(entries);
|
|
1029
|
-
}
|
|
1030
1057
|
|
|
1031
1058
|
//#endregion
|
|
1032
1059
|
//#region src/export/PerfettoExport.ts
|
|
@@ -1060,60 +1087,6 @@ function buildTraceEvents(groups, args) {
|
|
|
1060
1087
|
}
|
|
1061
1088
|
return events;
|
|
1062
1089
|
}
|
|
1063
|
-
function instant(ts, name, args) {
|
|
1064
|
-
return {
|
|
1065
|
-
ph: "i",
|
|
1066
|
-
ts,
|
|
1067
|
-
pid,
|
|
1068
|
-
tid,
|
|
1069
|
-
cat: "bench",
|
|
1070
|
-
name,
|
|
1071
|
-
s: "t",
|
|
1072
|
-
args
|
|
1073
|
-
};
|
|
1074
|
-
}
|
|
1075
|
-
function counter(ts, name, args) {
|
|
1076
|
-
return {
|
|
1077
|
-
ph: "C",
|
|
1078
|
-
ts,
|
|
1079
|
-
pid,
|
|
1080
|
-
tid,
|
|
1081
|
-
cat: "bench",
|
|
1082
|
-
name,
|
|
1083
|
-
args
|
|
1084
|
-
};
|
|
1085
|
-
}
|
|
1086
|
-
/** Build events for a single benchmark run */
|
|
1087
|
-
function buildBenchmarkEvents(results) {
|
|
1088
|
-
const { samples, heapSamples, timestamps, pausePoints } = results;
|
|
1089
|
-
if (!timestamps?.length) return [];
|
|
1090
|
-
const events = [];
|
|
1091
|
-
for (let i = 0; i < samples.length; i++) {
|
|
1092
|
-
const ts = timestamps[i];
|
|
1093
|
-
const ms = Math.round(samples[i] * 100) / 100;
|
|
1094
|
-
events.push(instant(ts, results.name, {
|
|
1095
|
-
n: i,
|
|
1096
|
-
ms
|
|
1097
|
-
}));
|
|
1098
|
-
events.push(counter(ts, "duration", { ms }));
|
|
1099
|
-
if (heapSamples?.[i] !== void 0) {
|
|
1100
|
-
const MB = Math.round(heapSamples[i] / 1024 / 1024 * 10) / 10;
|
|
1101
|
-
events.push(counter(ts, "heap", { MB }));
|
|
1102
|
-
}
|
|
1103
|
-
}
|
|
1104
|
-
for (const pause of pausePoints ?? []) {
|
|
1105
|
-
const ts = timestamps[pause.sampleIndex];
|
|
1106
|
-
if (ts) events.push(instant(ts, "pause", { ms: pause.durationMs }));
|
|
1107
|
-
}
|
|
1108
|
-
return events;
|
|
1109
|
-
}
|
|
1110
|
-
/** Normalize timestamps so events start at 0 */
|
|
1111
|
-
function normalizeTimestamps(events) {
|
|
1112
|
-
const times = events.filter((e) => e.ts > 0).map((e) => e.ts);
|
|
1113
|
-
if (times.length === 0) return;
|
|
1114
|
-
const minTs = Math.min(...times);
|
|
1115
|
-
for (const e of events) if (e.ts > 0) e.ts -= minTs;
|
|
1116
|
-
}
|
|
1117
1090
|
/** Merge V8 trace events from a previous run, aligning timestamps */
|
|
1118
1091
|
function mergeV8Trace(customEvents) {
|
|
1119
1092
|
const v8Events = loadV8Events(readdirSync(".").filter((f) => f.startsWith("node_trace.") && f.endsWith(".log"))[0]);
|
|
@@ -1122,29 +1095,11 @@ function mergeV8Trace(customEvents) {
|
|
|
1122
1095
|
normalizeTimestamps(v8Events);
|
|
1123
1096
|
return [...v8Events, ...customEvents];
|
|
1124
1097
|
}
|
|
1125
|
-
/** Load V8 trace events from file, or undefined if unavailable */
|
|
1126
|
-
function loadV8Events(v8TracePath) {
|
|
1127
|
-
if (!v8TracePath) return void 0;
|
|
1128
|
-
try {
|
|
1129
|
-
const v8Data = JSON.parse(readFileSync(v8TracePath, "utf-8"));
|
|
1130
|
-
console.log(`Merged ${v8Data.traceEvents.length} V8 events from ${v8TracePath}`);
|
|
1131
|
-
return v8Data.traceEvents;
|
|
1132
|
-
} catch {
|
|
1133
|
-
console.warn(`Could not parse V8 trace file: ${v8TracePath}`);
|
|
1134
|
-
return;
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
1098
|
/** Write trace events to JSON file */
|
|
1138
1099
|
function writeTraceFile(outputPath, events) {
|
|
1139
1100
|
const traceFile = { traceEvents: events };
|
|
1140
1101
|
writeFileSync(outputPath, JSON.stringify(traceFile));
|
|
1141
1102
|
}
|
|
1142
|
-
/** Clean CLI args for metadata */
|
|
1143
|
-
function cleanArgs(args) {
|
|
1144
|
-
const skip = new Set(["_", "$0"]);
|
|
1145
|
-
const entries = Object.entries(args).filter(([k, v]) => v !== void 0 && !skip.has(k));
|
|
1146
|
-
return Object.fromEntries(entries);
|
|
1147
|
-
}
|
|
1148
1103
|
/** Spawn a detached child to merge V8 trace after process exit */
|
|
1149
1104
|
function scheduleDeferredMerge(outputPath) {
|
|
1150
1105
|
const cwd = process.cwd();
|
|
@@ -1177,25 +1132,234 @@ function scheduleDeferredMerge(outputPath) {
|
|
|
1177
1132
|
}).unref();
|
|
1178
1133
|
});
|
|
1179
1134
|
}
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
return (sections?.flatMap((s) => s.columns().flatMap((g) => g.columns)))?.find((c) => c.comparable)?.higherIsBetter ?? false;
|
|
1135
|
+
/** Clean CLI args for metadata */
|
|
1136
|
+
function cleanArgs(args) {
|
|
1137
|
+
const skip = new Set(["_", "$0"]);
|
|
1138
|
+
const entries = Object.entries(args).filter(([k, v]) => v !== void 0 && !skip.has(k));
|
|
1139
|
+
return Object.fromEntries(entries);
|
|
1186
1140
|
}
|
|
1187
|
-
/**
|
|
1188
|
-
function
|
|
1141
|
+
/** Build events for a single benchmark run */
|
|
1142
|
+
function buildBenchmarkEvents(results) {
|
|
1143
|
+
const { samples, heapSamples, timestamps, pausePoints } = results;
|
|
1144
|
+
if (!timestamps?.length) return [];
|
|
1145
|
+
const events = [];
|
|
1146
|
+
for (let i = 0; i < samples.length; i++) {
|
|
1147
|
+
const ts = timestamps[i];
|
|
1148
|
+
const ms = Math.round(samples[i] * 100) / 100;
|
|
1149
|
+
events.push(instant(ts, results.name, {
|
|
1150
|
+
n: i,
|
|
1151
|
+
ms
|
|
1152
|
+
}));
|
|
1153
|
+
events.push(counter(ts, "duration", { ms }));
|
|
1154
|
+
if (heapSamples?.[i] !== void 0) {
|
|
1155
|
+
const MB = Math.round(heapSamples[i] / 1024 / 1024 * 10) / 10;
|
|
1156
|
+
events.push(counter(ts, "heap", { MB }));
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
for (const pause of pausePoints ?? []) {
|
|
1160
|
+
const ts = timestamps[pause.sampleIndex];
|
|
1161
|
+
if (ts) events.push(instant(ts, "pause", { ms: pause.durationMs }));
|
|
1162
|
+
}
|
|
1163
|
+
return events;
|
|
1164
|
+
}
|
|
1165
|
+
/** Load V8 trace events from file, or undefined if unavailable */
|
|
1166
|
+
function loadV8Events(v8TracePath) {
|
|
1167
|
+
if (!v8TracePath) return void 0;
|
|
1168
|
+
try {
|
|
1169
|
+
const v8Data = JSON.parse(readFileSync(v8TracePath, "utf-8"));
|
|
1170
|
+
console.log(`Merged ${v8Data.traceEvents.length} V8 events from ${v8TracePath}`);
|
|
1171
|
+
return v8Data.traceEvents;
|
|
1172
|
+
} catch {
|
|
1173
|
+
console.warn(`Could not parse V8 trace file: ${v8TracePath}`);
|
|
1174
|
+
return;
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
/** Normalize timestamps so events start at 0 */
|
|
1178
|
+
function normalizeTimestamps(events) {
|
|
1179
|
+
const times = events.filter((e) => e.ts > 0).map((e) => e.ts);
|
|
1180
|
+
if (times.length === 0) return;
|
|
1181
|
+
const minTs = Math.min(...times);
|
|
1182
|
+
for (const e of events) if (e.ts > 0) e.ts -= minTs;
|
|
1183
|
+
}
|
|
1184
|
+
function instant(ts, name, args) {
|
|
1189
1185
|
return {
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1186
|
+
ph: "i",
|
|
1187
|
+
ts,
|
|
1188
|
+
pid,
|
|
1189
|
+
tid,
|
|
1190
|
+
cat: "bench",
|
|
1191
|
+
name,
|
|
1192
|
+
s: "t",
|
|
1193
|
+
args
|
|
1194
|
+
};
|
|
1195
|
+
}
|
|
1196
|
+
function counter(ts, name, args) {
|
|
1197
|
+
return {
|
|
1198
|
+
ph: "C",
|
|
1199
|
+
ts,
|
|
1200
|
+
pid,
|
|
1201
|
+
tid,
|
|
1202
|
+
cat: "bench",
|
|
1203
|
+
name,
|
|
1204
|
+
args
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
//#endregion
|
|
1209
|
+
//#region src/heap-sample/ResolvedProfile.ts
|
|
1210
|
+
/** Walk a HeapProfile tree once, producing a fully resolved intermediate form */
|
|
1211
|
+
function resolveProfile(profile) {
|
|
1212
|
+
const nodes = [];
|
|
1213
|
+
const nodeMap = /* @__PURE__ */ new Map();
|
|
1214
|
+
let totalBytes = 0;
|
|
1215
|
+
function walk(node, parentStack) {
|
|
1216
|
+
const { functionName, url, lineNumber, columnNumber } = node.callFrame;
|
|
1217
|
+
const frame = {
|
|
1218
|
+
name: functionName || "(anonymous)",
|
|
1219
|
+
url: url || "",
|
|
1220
|
+
line: lineNumber + 1,
|
|
1221
|
+
col: columnNumber != null ? columnNumber + 1 : void 0
|
|
1222
|
+
};
|
|
1223
|
+
const stack = [...parentStack, frame];
|
|
1224
|
+
const resolved = {
|
|
1225
|
+
frame,
|
|
1226
|
+
stack,
|
|
1227
|
+
selfSize: node.selfSize,
|
|
1228
|
+
nodeId: node.id
|
|
1229
|
+
};
|
|
1230
|
+
nodes.push(resolved);
|
|
1231
|
+
nodeMap.set(node.id, resolved);
|
|
1232
|
+
totalBytes += node.selfSize;
|
|
1233
|
+
for (const child of node.children || []) walk(child, stack);
|
|
1234
|
+
}
|
|
1235
|
+
walk(profile.head, []);
|
|
1236
|
+
return {
|
|
1237
|
+
nodes,
|
|
1238
|
+
nodeMap,
|
|
1239
|
+
allocationNodes: nodes.filter((n) => n.selfSize > 0).sort((a, b) => b.selfSize - a.selfSize),
|
|
1240
|
+
sortedSamples: profile.samples ? [...profile.samples].sort((a, b) => a.ordinal - b.ordinal) : void 0,
|
|
1241
|
+
totalBytes
|
|
1242
|
+
};
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
//#endregion
|
|
1246
|
+
//#region src/export/SpeedscopeExport.ts
|
|
1247
|
+
/** Export heap profiles from benchmark results to speedscope JSON format.
|
|
1248
|
+
* Creates one speedscope profile per benchmark that has a heapProfile.
|
|
1249
|
+
* @returns resolved output path, or undefined if no profiles were found */
|
|
1250
|
+
function exportSpeedscope(groups, outputPath) {
|
|
1251
|
+
const frames = [];
|
|
1252
|
+
const frameIndex = /* @__PURE__ */ new Map();
|
|
1253
|
+
const profiles = [];
|
|
1254
|
+
for (const group of groups) for (const report of groupReports(group)) {
|
|
1255
|
+
const { heapProfile } = report.measuredResults;
|
|
1256
|
+
if (!heapProfile) continue;
|
|
1257
|
+
const resolved = resolveProfile(heapProfile);
|
|
1258
|
+
profiles.push(buildProfile(report.name, resolved, frames, frameIndex));
|
|
1259
|
+
}
|
|
1260
|
+
if (profiles.length === 0) {
|
|
1261
|
+
console.log("No heap profiles to export.");
|
|
1262
|
+
return;
|
|
1263
|
+
}
|
|
1264
|
+
const file = {
|
|
1265
|
+
$schema: "https://www.speedscope.app/file-format-schema.json",
|
|
1266
|
+
shared: { frames },
|
|
1267
|
+
profiles,
|
|
1268
|
+
exporter: "benchforge"
|
|
1269
|
+
};
|
|
1270
|
+
const absPath = resolve(outputPath);
|
|
1271
|
+
writeFileSync(absPath, JSON.stringify(file));
|
|
1272
|
+
console.log(`Speedscope profile exported to: ${outputPath}`);
|
|
1273
|
+
return absPath;
|
|
1274
|
+
}
|
|
1275
|
+
/** Export to a temp file and open in speedscope via npx */
|
|
1276
|
+
function exportAndLaunchSpeedscope(groups) {
|
|
1277
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
1278
|
+
const absPath = exportSpeedscope(groups, join(tmpdir(), `benchforge-${timestamp}.speedscope.json`));
|
|
1279
|
+
if (absPath) launchSpeedscope(absPath);
|
|
1280
|
+
}
|
|
1281
|
+
/** Launch speedscope viewer on a file via npx */
|
|
1282
|
+
function launchSpeedscope(filePath) {
|
|
1283
|
+
console.log("Opening speedscope...");
|
|
1284
|
+
const child = spawn("npx", ["speedscope", filePath], {
|
|
1285
|
+
detached: true,
|
|
1286
|
+
stdio: "ignore"
|
|
1287
|
+
});
|
|
1288
|
+
child.unref();
|
|
1289
|
+
child.on("error", () => {
|
|
1290
|
+
console.error(`Failed to launch speedscope. Run manually:\n npx speedscope ${filePath}`);
|
|
1291
|
+
});
|
|
1292
|
+
}
|
|
1293
|
+
/** Convert a single HeapProfile to speedscope format (for standalone use) */
|
|
1294
|
+
function heapProfileToSpeedscope(name, profile) {
|
|
1295
|
+
const frames = [];
|
|
1296
|
+
const frameIndex = /* @__PURE__ */ new Map();
|
|
1297
|
+
const p = buildProfile(name, resolveProfile(profile), frames, frameIndex);
|
|
1298
|
+
return {
|
|
1299
|
+
$schema: "https://www.speedscope.app/file-format-schema.json",
|
|
1300
|
+
shared: { frames },
|
|
1301
|
+
profiles: [p],
|
|
1302
|
+
exporter: "benchforge"
|
|
1303
|
+
};
|
|
1304
|
+
}
|
|
1305
|
+
/** Build a single speedscope profile from a resolved heap profile */
|
|
1306
|
+
function buildProfile(name, resolved, sharedFrames, frameIndex) {
|
|
1307
|
+
const nodeStacks = /* @__PURE__ */ new Map();
|
|
1308
|
+
for (const node of resolved.nodes) {
|
|
1309
|
+
const stack = node.stack.map((f) => internFrame(f, sharedFrames, frameIndex));
|
|
1310
|
+
nodeStacks.set(node.nodeId, stack);
|
|
1311
|
+
}
|
|
1312
|
+
const samples = [];
|
|
1313
|
+
const weights = [];
|
|
1314
|
+
if (!resolved.sortedSamples || resolved.sortedSamples.length === 0) {
|
|
1315
|
+
console.error(`Speedscope export: no samples in heap profile for "${name}", skipping`);
|
|
1316
|
+
return {
|
|
1317
|
+
type: "sampled",
|
|
1318
|
+
name,
|
|
1319
|
+
unit: "bytes",
|
|
1320
|
+
startValue: 0,
|
|
1321
|
+
endValue: 0,
|
|
1322
|
+
samples,
|
|
1323
|
+
weights
|
|
1324
|
+
};
|
|
1325
|
+
}
|
|
1326
|
+
for (const sample of resolved.sortedSamples) {
|
|
1327
|
+
const stack = nodeStacks.get(sample.nodeId);
|
|
1328
|
+
if (stack) {
|
|
1329
|
+
samples.push(stack);
|
|
1330
|
+
weights.push(sample.size);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
return {
|
|
1334
|
+
type: "sampled",
|
|
1335
|
+
name,
|
|
1336
|
+
unit: "bytes",
|
|
1337
|
+
startValue: 0,
|
|
1338
|
+
endValue: weights.reduce((sum, w) => sum + w, 0),
|
|
1339
|
+
samples,
|
|
1340
|
+
weights
|
|
1197
1341
|
};
|
|
1198
1342
|
}
|
|
1343
|
+
/** Intern a call frame, returning its index in the shared frames array */
|
|
1344
|
+
function internFrame(frame, sharedFrames, frameIndex) {
|
|
1345
|
+
const { name, url, line, col } = frame;
|
|
1346
|
+
const key = `${name}\0${url}\0${line}\0${col}`;
|
|
1347
|
+
let idx = frameIndex.get(key);
|
|
1348
|
+
if (idx === void 0) {
|
|
1349
|
+
idx = sharedFrames.length;
|
|
1350
|
+
const shortFile = url ? url.split("/").pop() : void 0;
|
|
1351
|
+
const entry = { name: name !== "(anonymous)" ? name : shortFile ? `(anonymous ${shortFile}:${line})` : "(anonymous)" };
|
|
1352
|
+
if (url) entry.file = url;
|
|
1353
|
+
if (line > 0) entry.line = line;
|
|
1354
|
+
if (col != null) entry.col = col;
|
|
1355
|
+
sharedFrames.push(entry);
|
|
1356
|
+
frameIndex.set(key, idx);
|
|
1357
|
+
}
|
|
1358
|
+
return idx;
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
//#endregion
|
|
1362
|
+
//#region src/HtmlDataPrep.ts
|
|
1199
1363
|
/** Prepare ReportData from benchmark results for HTML rendering */
|
|
1200
1364
|
function prepareHtmlData(groups, options) {
|
|
1201
1365
|
const { cliArgs, sections, currentVersion, baselineVersion } = options;
|
|
@@ -1212,6 +1376,10 @@ function prepareHtmlData(groups, options) {
|
|
|
1212
1376
|
}
|
|
1213
1377
|
};
|
|
1214
1378
|
}
|
|
1379
|
+
/** Find higherIsBetter from first comparable column in sections */
|
|
1380
|
+
function findHigherIsBetter(sections) {
|
|
1381
|
+
return (sections?.flatMap((s) => s.columns().flatMap((g) => g.columns)))?.find((c) => c.comparable)?.higherIsBetter ?? false;
|
|
1382
|
+
}
|
|
1215
1383
|
/** @return group data with bootstrap CI comparisons against baseline */
|
|
1216
1384
|
function prepareGroupData(group, sections, higherIsBetter) {
|
|
1217
1385
|
const baselineSamples = group.baseline?.measuredResults.samples;
|
|
@@ -1246,6 +1414,18 @@ function prepareBenchmarkData(report, sections) {
|
|
|
1246
1414
|
sectionStats: sections ? extractSectionStats(report, sections) : void 0
|
|
1247
1415
|
};
|
|
1248
1416
|
}
|
|
1417
|
+
/** Flip CI percent for metrics where higher is better (e.g., lines/sec) */
|
|
1418
|
+
function flipCI(ci) {
|
|
1419
|
+
return {
|
|
1420
|
+
percent: -ci.percent,
|
|
1421
|
+
ci: [-ci.ci[1], -ci.ci[0]],
|
|
1422
|
+
direction: ci.direction,
|
|
1423
|
+
histogram: ci.histogram?.map((bin) => ({
|
|
1424
|
+
x: -bin.x,
|
|
1425
|
+
count: bin.count
|
|
1426
|
+
}))
|
|
1427
|
+
};
|
|
1428
|
+
}
|
|
1249
1429
|
/** @return formatted stats from all sections for tooltip display */
|
|
1250
1430
|
function extractSectionStats(report, sections) {
|
|
1251
1431
|
return sections.flatMap((section) => {
|
|
@@ -1274,36 +1454,34 @@ function formatColumnStat(values, col, groupTitle) {
|
|
|
1274
1454
|
//#region src/heap-sample/HeapSampleReport.ts
|
|
1275
1455
|
/** Sum selfSize across all nodes in profile (before any filtering) */
|
|
1276
1456
|
function totalProfileBytes(profile) {
|
|
1277
|
-
|
|
1278
|
-
function walk(node) {
|
|
1279
|
-
total += node.selfSize;
|
|
1280
|
-
for (const child of node.children || []) walk(child);
|
|
1281
|
-
}
|
|
1282
|
-
walk(profile.head);
|
|
1283
|
-
return total;
|
|
1457
|
+
return resolveProfile(profile).totalBytes;
|
|
1284
1458
|
}
|
|
1285
|
-
/** Flatten profile
|
|
1286
|
-
|
|
1459
|
+
/** Flatten resolved profile into sorted list of allocation sites with call stacks.
|
|
1460
|
+
* When raw samples are available, attaches them to corresponding sites. */
|
|
1461
|
+
function flattenProfile(resolved) {
|
|
1287
1462
|
const sites = [];
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
const
|
|
1291
|
-
const
|
|
1292
|
-
const
|
|
1293
|
-
fn,
|
|
1294
|
-
url: url || "",
|
|
1295
|
-
line: lineNumber + 1,
|
|
1296
|
-
col
|
|
1297
|
-
};
|
|
1298
|
-
const newStack = [...stack, frame];
|
|
1299
|
-
if (node.selfSize > 0) sites.push({
|
|
1463
|
+
const nodeIdToSites = /* @__PURE__ */ new Map();
|
|
1464
|
+
for (const node of resolved.allocationNodes) {
|
|
1465
|
+
const frame = toCallFrame(node.frame);
|
|
1466
|
+
const stack = node.stack.map(toCallFrame);
|
|
1467
|
+
const site = {
|
|
1300
1468
|
...frame,
|
|
1301
1469
|
bytes: node.selfSize,
|
|
1302
|
-
stack
|
|
1303
|
-
}
|
|
1304
|
-
|
|
1470
|
+
stack
|
|
1471
|
+
};
|
|
1472
|
+
sites.push(site);
|
|
1473
|
+
const existing = nodeIdToSites.get(node.nodeId);
|
|
1474
|
+
if (existing) existing.push(site);
|
|
1475
|
+
else nodeIdToSites.set(node.nodeId, [site]);
|
|
1476
|
+
}
|
|
1477
|
+
for (const sample of resolved.sortedSamples ?? []) {
|
|
1478
|
+
const matchingSites = nodeIdToSites.get(sample.nodeId);
|
|
1479
|
+
if (!matchingSites) continue;
|
|
1480
|
+
for (const site of matchingSites) {
|
|
1481
|
+
if (!site.samples) site.samples = [];
|
|
1482
|
+
site.samples.push(sample);
|
|
1483
|
+
}
|
|
1305
1484
|
}
|
|
1306
|
-
walk(profile.head, []);
|
|
1307
1485
|
return sites.sort((a, b) => b.bytes - a.bytes);
|
|
1308
1486
|
}
|
|
1309
1487
|
/** Check if site is user code (not node internals) */
|
|
@@ -1326,22 +1504,31 @@ function isBrowserUserCode(site) {
|
|
|
1326
1504
|
function filterSites(sites, isUser = isNodeUserCode) {
|
|
1327
1505
|
return sites.filter(isUser);
|
|
1328
1506
|
}
|
|
1329
|
-
/** Aggregate sites by location (combine same file:line:col)
|
|
1507
|
+
/** Aggregate sites by location (combine same file:line:col).
|
|
1508
|
+
* Tracks distinct caller stacks with byte weights when merging. */
|
|
1330
1509
|
function aggregateSites(sites) {
|
|
1331
1510
|
const byLocation = /* @__PURE__ */ new Map();
|
|
1332
1511
|
for (const site of sites) {
|
|
1333
|
-
const key = `${site.url}:${site.line}:${site.col}`;
|
|
1512
|
+
const key = site.col != null ? `${site.url}:${site.line}:${site.col}` : `${site.url}:${site.line}:?:${site.fn}`;
|
|
1334
1513
|
const existing = byLocation.get(key);
|
|
1335
|
-
if (existing)
|
|
1336
|
-
|
|
1514
|
+
if (existing) {
|
|
1515
|
+
existing.bytes += site.bytes;
|
|
1516
|
+
addCaller(existing, site);
|
|
1517
|
+
} else {
|
|
1518
|
+
const entry = { ...site };
|
|
1519
|
+
if (site.stack) entry.callers = [{
|
|
1520
|
+
stack: site.stack,
|
|
1521
|
+
bytes: site.bytes
|
|
1522
|
+
}];
|
|
1523
|
+
byLocation.set(key, entry);
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
for (const site of byLocation.values()) if (site.callers && site.callers.length > 1) {
|
|
1527
|
+
site.callers.sort((a, b) => b.bytes - a.bytes);
|
|
1528
|
+
site.stack = site.callers[0].stack;
|
|
1337
1529
|
}
|
|
1338
1530
|
return [...byLocation.values()].sort((a, b) => b.bytes - a.bytes);
|
|
1339
1531
|
}
|
|
1340
|
-
function fmtBytes(bytes) {
|
|
1341
|
-
if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(2)} MB`;
|
|
1342
|
-
if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
1343
|
-
return `${bytes} B`;
|
|
1344
|
-
}
|
|
1345
1532
|
/** Format heap report for console output */
|
|
1346
1533
|
function formatHeapReport(sites, options) {
|
|
1347
1534
|
const { topN, stackDepth = 3, verbose = false } = options;
|
|
@@ -1357,34 +1544,79 @@ function formatHeapReport(sites, options) {
|
|
|
1357
1544
|
if (sampleCount !== void 0) lines.push(`Samples: ${sampleCount.toLocaleString()}`);
|
|
1358
1545
|
return lines.join("\n");
|
|
1359
1546
|
}
|
|
1360
|
-
/**
|
|
1361
|
-
|
|
1547
|
+
/** Format every raw sample as one line, ordered by ordinal (time).
|
|
1548
|
+
* Output is tab-separated for easy piping/grep/diff. */
|
|
1549
|
+
function formatRawSamples(resolved) {
|
|
1550
|
+
if (!resolved.sortedSamples || resolved.sortedSamples.length === 0) return "No raw samples available.";
|
|
1551
|
+
const lines = ["ordinal size function location"];
|
|
1552
|
+
for (const s of resolved.sortedSamples) {
|
|
1553
|
+
const node = resolved.nodeMap.get(s.nodeId);
|
|
1554
|
+
const fn = node?.frame.name || "(unknown)";
|
|
1555
|
+
const url = node?.frame.url || "";
|
|
1556
|
+
const loc = url ? fmtLoc(url, node.frame.line, node.frame.col) : "(unknown)";
|
|
1557
|
+
lines.push(`${s.ordinal}\t${s.size}\t${fn}\t${loc}`);
|
|
1558
|
+
}
|
|
1559
|
+
return lines.join("\n");
|
|
1560
|
+
}
|
|
1561
|
+
function toCallFrame(f) {
|
|
1562
|
+
return {
|
|
1563
|
+
fn: f.name,
|
|
1564
|
+
url: f.url,
|
|
1565
|
+
line: f.line,
|
|
1566
|
+
col: f.col
|
|
1567
|
+
};
|
|
1568
|
+
}
|
|
1569
|
+
/** Add a caller stack to an aggregated site, merging if the same path exists */
|
|
1570
|
+
function addCaller(existing, site) {
|
|
1571
|
+
if (!site.stack) return;
|
|
1572
|
+
if (!existing.callers) existing.callers = [];
|
|
1573
|
+
const key = callerKey(site.stack);
|
|
1574
|
+
const match = existing.callers.find((c) => callerKey(c.stack) === key);
|
|
1575
|
+
if (match) match.bytes += site.bytes;
|
|
1576
|
+
else existing.callers.push({
|
|
1577
|
+
stack: site.stack,
|
|
1578
|
+
bytes: site.bytes
|
|
1579
|
+
});
|
|
1580
|
+
}
|
|
1581
|
+
/** Verbose multi-line format with file:// paths and line numbers */
|
|
1582
|
+
function formatVerboseSite(lines, site, stackDepth, isUser) {
|
|
1362
1583
|
const bytes = fmtBytes(site.bytes).padStart(10);
|
|
1363
|
-
const
|
|
1584
|
+
const loc = site.url ? fmtLoc(site.url, site.line, site.col) : "(unknown)";
|
|
1585
|
+
const dimFn = isUser(site) ? (s) => s : pico.dim;
|
|
1586
|
+
lines.push(dimFn(`${bytes} ${site.fn} ${loc}`));
|
|
1364
1587
|
if (site.stack && site.stack.length > 1) {
|
|
1365
1588
|
const callers = site.stack.slice(0, -1).reverse().slice(0, stackDepth);
|
|
1366
1589
|
for (const frame of callers) {
|
|
1367
1590
|
if (!frame.url || !isUser(frame)) continue;
|
|
1368
|
-
|
|
1591
|
+
const callerLoc = fmtLoc(frame.url, frame.line, frame.col);
|
|
1592
|
+
lines.push(dimFn(` <- ${frame.fn} ${callerLoc}`));
|
|
1369
1593
|
}
|
|
1370
1594
|
}
|
|
1371
|
-
const line = `${bytes} ${fns.join(" <- ")}`;
|
|
1372
|
-
lines.push(isUser(site) ? line : pico.dim(line));
|
|
1373
1595
|
}
|
|
1374
|
-
/**
|
|
1375
|
-
function
|
|
1596
|
+
/** Compact single-line format: `49 MB fn1 <- fn2 <- fn3` */
|
|
1597
|
+
function formatCompactSite(lines, site, stackDepth, isUser) {
|
|
1376
1598
|
const bytes = fmtBytes(site.bytes).padStart(10);
|
|
1377
|
-
const
|
|
1378
|
-
const dimFn = isUser(site) ? (s) => s : pico.dim;
|
|
1379
|
-
lines.push(dimFn(`${bytes} ${site.fn} ${loc}`));
|
|
1599
|
+
const fns = [site.fn];
|
|
1380
1600
|
if (site.stack && site.stack.length > 1) {
|
|
1381
1601
|
const callers = site.stack.slice(0, -1).reverse().slice(0, stackDepth);
|
|
1382
1602
|
for (const frame of callers) {
|
|
1383
1603
|
if (!frame.url || !isUser(frame)) continue;
|
|
1384
|
-
|
|
1385
|
-
lines.push(dimFn(` <- ${frame.fn} ${callerLoc}`));
|
|
1604
|
+
fns.push(frame.fn);
|
|
1386
1605
|
}
|
|
1387
1606
|
}
|
|
1607
|
+
const line = `${bytes} ${fns.join(" <- ")}`;
|
|
1608
|
+
lines.push(isUser(site) ? line : pico.dim(line));
|
|
1609
|
+
}
|
|
1610
|
+
function fmtBytes(bytes) {
|
|
1611
|
+
return formatBytes(bytes, { space: true }) ?? `${bytes} B`;
|
|
1612
|
+
}
|
|
1613
|
+
/** Format location, omitting column when unknown */
|
|
1614
|
+
function fmtLoc(url, line, col) {
|
|
1615
|
+
return col != null ? `${url}:${line}:${col}` : `${url}:${line}`;
|
|
1616
|
+
}
|
|
1617
|
+
/** Serialize a call stack for dedup comparison */
|
|
1618
|
+
function callerKey(stack) {
|
|
1619
|
+
return stack.map((f) => `${f.url}:${f.line}:${f.col}`).join("|");
|
|
1388
1620
|
}
|
|
1389
1621
|
|
|
1390
1622
|
//#endregion
|
|
@@ -1395,6 +1627,18 @@ const skipArgs = new Set([
|
|
|
1395
1627
|
"html",
|
|
1396
1628
|
"export-html"
|
|
1397
1629
|
]);
|
|
1630
|
+
const badgeLabels = {
|
|
1631
|
+
faster: "Faster",
|
|
1632
|
+
slower: "Slower",
|
|
1633
|
+
uncertain: "Inconclusive"
|
|
1634
|
+
};
|
|
1635
|
+
const defaultArgs = {
|
|
1636
|
+
worker: true,
|
|
1637
|
+
time: 5,
|
|
1638
|
+
warmup: 500,
|
|
1639
|
+
"pause-interval": 0,
|
|
1640
|
+
"pause-duration": 100
|
|
1641
|
+
};
|
|
1398
1642
|
/** Format ISO date as local time with UTC: "Jan 9, 2026, 3:45 PM (2026-01-09T23:45:00Z)" */
|
|
1399
1643
|
function formatDateWithTimezone(isoDate) {
|
|
1400
1644
|
const date = new Date(isoDate);
|
|
@@ -1423,62 +1667,6 @@ function formatRelativeTime(isoDate) {
|
|
|
1423
1667
|
day: "numeric"
|
|
1424
1668
|
});
|
|
1425
1669
|
}
|
|
1426
|
-
/** Format git version for display: "abc1234* (5m ago)" */
|
|
1427
|
-
function formatVersion(version) {
|
|
1428
|
-
if (!version || version.hash === "unknown") return "unknown";
|
|
1429
|
-
const hashDisplay = version.dirty ? `${version.hash}*` : version.hash;
|
|
1430
|
-
const timeDisplay = version.date ? formatRelativeTime(version.date) : "";
|
|
1431
|
-
return timeDisplay ? `${hashDisplay} (${timeDisplay})` : hashDisplay;
|
|
1432
|
-
}
|
|
1433
|
-
/** Render current/baseline version info as an HTML div */
|
|
1434
|
-
function versionInfoHtml(data) {
|
|
1435
|
-
const { currentVersion, baselineVersion } = data.metadata;
|
|
1436
|
-
if (!currentVersion && !baselineVersion) return "";
|
|
1437
|
-
const parts = [];
|
|
1438
|
-
if (currentVersion) parts.push(`Current: ${formatVersion(currentVersion)}`);
|
|
1439
|
-
if (baselineVersion) parts.push(`Baseline: ${formatVersion(baselineVersion)}`);
|
|
1440
|
-
return `<div class="version-info">${parts.join(" | ")}</div>`;
|
|
1441
|
-
}
|
|
1442
|
-
const badgeLabels = {
|
|
1443
|
-
faster: "Faster",
|
|
1444
|
-
slower: "Slower",
|
|
1445
|
-
uncertain: "Inconclusive"
|
|
1446
|
-
};
|
|
1447
|
-
/** Render faster/slower/uncertain badge with CI plot container */
|
|
1448
|
-
function comparisonBadge(group, groupIndex) {
|
|
1449
|
-
const ci = group.benchmarks[0]?.comparisonCI;
|
|
1450
|
-
if (!ci) return "";
|
|
1451
|
-
const label = badgeLabels[ci.direction];
|
|
1452
|
-
return `
|
|
1453
|
-
<span class="badge badge-${ci.direction}">${label}</span>
|
|
1454
|
-
<div id="ci-plot-${groupIndex}" class="ci-plot-container"></div>
|
|
1455
|
-
`;
|
|
1456
|
-
}
|
|
1457
|
-
const defaultArgs = {
|
|
1458
|
-
worker: true,
|
|
1459
|
-
time: 5,
|
|
1460
|
-
warmup: 500,
|
|
1461
|
-
"pause-interval": 0,
|
|
1462
|
-
"pause-duration": 100
|
|
1463
|
-
};
|
|
1464
|
-
/** @return true if this CLI arg should be hidden from the report header */
|
|
1465
|
-
function shouldSkipArg(key, value, adaptive) {
|
|
1466
|
-
if (skipArgs.has(key) || value === void 0 || value === false) return true;
|
|
1467
|
-
if (defaultArgs[key] === value) return true;
|
|
1468
|
-
if (!key.includes("-") && key !== key.toLowerCase()) return true;
|
|
1469
|
-
if (key === "convergence" && !adaptive) return true;
|
|
1470
|
-
return false;
|
|
1471
|
-
}
|
|
1472
|
-
/** Reconstruct the CLI invocation string, omitting default/internal args */
|
|
1473
|
-
function formatCliArgs(args) {
|
|
1474
|
-
if (!args) return "bb bench";
|
|
1475
|
-
const parts = ["bb bench"];
|
|
1476
|
-
for (const [key, value] of Object.entries(args)) {
|
|
1477
|
-
if (shouldSkipArg(key, value, args.adaptive)) continue;
|
|
1478
|
-
parts.push(value === true ? `--${key}` : `--${key} ${value}`);
|
|
1479
|
-
}
|
|
1480
|
-
return parts.join(" ");
|
|
1481
|
-
}
|
|
1482
1670
|
/** Generate complete HTML document with embedded data and visualizations */
|
|
1483
1671
|
function generateHtmlDocument(data) {
|
|
1484
1672
|
return `<!DOCTYPE html>
|
|
@@ -1653,14 +1841,58 @@ function generateHtmlDocument(data) {
|
|
|
1653
1841
|
</body>
|
|
1654
1842
|
</html>`;
|
|
1655
1843
|
}
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1844
|
+
/** Reconstruct the CLI invocation string, omitting default/internal args */
|
|
1845
|
+
function formatCliArgs(args) {
|
|
1846
|
+
if (!args) return "bb bench";
|
|
1847
|
+
const parts = ["bb bench"];
|
|
1848
|
+
for (const [key, value] of Object.entries(args)) {
|
|
1849
|
+
if (shouldSkipArg(key, value, args.adaptive)) continue;
|
|
1850
|
+
parts.push(value === true ? `--${key}` : `--${key} ${value}`);
|
|
1851
|
+
}
|
|
1852
|
+
return parts.join(" ");
|
|
1853
|
+
}
|
|
1854
|
+
/** Render current/baseline version info as an HTML div */
|
|
1855
|
+
function versionInfoHtml(data) {
|
|
1856
|
+
const { currentVersion, baselineVersion } = data.metadata;
|
|
1857
|
+
if (!currentVersion && !baselineVersion) return "";
|
|
1858
|
+
const parts = [];
|
|
1859
|
+
if (currentVersion) parts.push(`Current: ${formatVersion(currentVersion)}`);
|
|
1860
|
+
if (baselineVersion) parts.push(`Baseline: ${formatVersion(baselineVersion)}`);
|
|
1861
|
+
return `<div class="version-info">${parts.join(" | ")}</div>`;
|
|
1862
|
+
}
|
|
1863
|
+
/** Render faster/slower/uncertain badge with CI plot container */
|
|
1864
|
+
function comparisonBadge(group, groupIndex) {
|
|
1865
|
+
const ci = group.benchmarks[0]?.comparisonCI;
|
|
1866
|
+
if (!ci) return "";
|
|
1867
|
+
const label = badgeLabels[ci.direction];
|
|
1868
|
+
return `
|
|
1869
|
+
<span class="badge badge-${ci.direction}">${label}</span>
|
|
1870
|
+
<div id="ci-plot-${groupIndex}" class="ci-plot-container"></div>
|
|
1871
|
+
`;
|
|
1872
|
+
}
|
|
1873
|
+
/** @return true if this CLI arg should be hidden from the report header */
|
|
1874
|
+
function shouldSkipArg(key, value, adaptive) {
|
|
1875
|
+
if (skipArgs.has(key) || value === void 0 || value === false) return true;
|
|
1876
|
+
if (defaultArgs[key] === value) return true;
|
|
1877
|
+
if (!key.includes("-") && key !== key.toLowerCase()) return true;
|
|
1878
|
+
if (key === "convergence" && !adaptive) return true;
|
|
1879
|
+
return false;
|
|
1880
|
+
}
|
|
1881
|
+
/** Format git version for display: "abc1234* (5m ago)" */
|
|
1882
|
+
function formatVersion(version) {
|
|
1883
|
+
if (!version || version.hash === "unknown") return "unknown";
|
|
1884
|
+
const hashDisplay = version.dirty ? `${version.hash}*` : version.hash;
|
|
1885
|
+
const timeDisplay = version.date ? formatRelativeTime(version.date) : "";
|
|
1886
|
+
return timeDisplay ? `${hashDisplay} (${timeDisplay})` : hashDisplay;
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
//#endregion
|
|
1890
|
+
//#region src/html/HtmlReport.ts
|
|
1891
|
+
/** Generate HTML report from prepared data and optionally open in browser */
|
|
1892
|
+
async function generateHtmlReport(data, options) {
|
|
1893
|
+
const html = generateHtmlDocument(data);
|
|
1894
|
+
const reportDir = options.outputPath || await createReportDir();
|
|
1895
|
+
await mkdir(reportDir, { recursive: true });
|
|
1664
1896
|
await writeFile(join(reportDir, "index.html"), html, "utf-8");
|
|
1665
1897
|
const plots = await loadPlotsBundle();
|
|
1666
1898
|
await writeFile(join(reportDir, "plots.js"), plots, "utf-8");
|
|
@@ -1683,6 +1915,35 @@ async function generateHtmlReport(data, options) {
|
|
|
1683
1915
|
closeServer
|
|
1684
1916
|
};
|
|
1685
1917
|
}
|
|
1918
|
+
/** Create a timestamped report directory under ./bench-report/ */
|
|
1919
|
+
async function createReportDir() {
|
|
1920
|
+
const base = "./bench-report";
|
|
1921
|
+
await mkdir(base, { recursive: true });
|
|
1922
|
+
return join(base, `report-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19)}`);
|
|
1923
|
+
}
|
|
1924
|
+
/** Read the pre-built browser plots bundle from dist/ */
|
|
1925
|
+
async function loadPlotsBundle() {
|
|
1926
|
+
const thisDir = dirname(fileURLToPath(import.meta.url));
|
|
1927
|
+
const builtPath = join(thisDir, "browser/index.js");
|
|
1928
|
+
const devPath = join(thisDir, "../../dist/browser/index.js");
|
|
1929
|
+
try {
|
|
1930
|
+
return await readFile(builtPath, "utf-8");
|
|
1931
|
+
} catch {}
|
|
1932
|
+
return readFile(devPath, "utf-8");
|
|
1933
|
+
}
|
|
1934
|
+
/** Write an index.html in the parent dir that redirects to this report */
|
|
1935
|
+
async function writeLatestRedirect(reportDir) {
|
|
1936
|
+
const baseDir = dirname(reportDir);
|
|
1937
|
+
const reportName = reportDir.split("/").pop();
|
|
1938
|
+
const html = `<!DOCTYPE html>
|
|
1939
|
+
<html><head>
|
|
1940
|
+
<meta http-equiv="refresh" content="0; url=./${reportName}/">
|
|
1941
|
+
<script>location.href = "./${reportName}/";<\/script>
|
|
1942
|
+
</head><body>
|
|
1943
|
+
<a href="./${reportName}/">Latest report</a>
|
|
1944
|
+
</body></html>`;
|
|
1945
|
+
await writeFile(join(baseDir, "index.html"), html, "utf-8");
|
|
1946
|
+
}
|
|
1686
1947
|
/** Start HTTP server for report directory, trying fallback ports if needed */
|
|
1687
1948
|
async function startReportServer(baseDir, ...ports) {
|
|
1688
1949
|
const mimeTypes = {
|
|
@@ -1723,35 +1984,6 @@ function tryListen(server, port) {
|
|
|
1723
1984
|
});
|
|
1724
1985
|
});
|
|
1725
1986
|
}
|
|
1726
|
-
/** Create a timestamped report directory under ./bench-report/ */
|
|
1727
|
-
async function createReportDir() {
|
|
1728
|
-
const base = "./bench-report";
|
|
1729
|
-
await mkdir(base, { recursive: true });
|
|
1730
|
-
return join(base, `report-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19)}`);
|
|
1731
|
-
}
|
|
1732
|
-
/** Read the pre-built browser plots bundle from dist/ */
|
|
1733
|
-
async function loadPlotsBundle() {
|
|
1734
|
-
const thisDir = dirname(fileURLToPath(import.meta.url));
|
|
1735
|
-
const builtPath = join(thisDir, "browser/index.js");
|
|
1736
|
-
const devPath = join(thisDir, "../../dist/browser/index.js");
|
|
1737
|
-
try {
|
|
1738
|
-
return await readFile(builtPath, "utf-8");
|
|
1739
|
-
} catch {}
|
|
1740
|
-
return readFile(devPath, "utf-8");
|
|
1741
|
-
}
|
|
1742
|
-
/** Write an index.html in the parent dir that redirects to this report */
|
|
1743
|
-
async function writeLatestRedirect(reportDir) {
|
|
1744
|
-
const baseDir = dirname(reportDir);
|
|
1745
|
-
const reportName = reportDir.split("/").pop();
|
|
1746
|
-
const html = `<!DOCTYPE html>
|
|
1747
|
-
<html><head>
|
|
1748
|
-
<meta http-equiv="refresh" content="0; url=./${reportName}/">
|
|
1749
|
-
<script>location.href = "./${reportName}/";<\/script>
|
|
1750
|
-
</head><body>
|
|
1751
|
-
<a href="./${reportName}/">Latest report</a>
|
|
1752
|
-
</body></html>`;
|
|
1753
|
-
await writeFile(join(baseDir, "index.html"), html, "utf-8");
|
|
1754
|
-
}
|
|
1755
1987
|
|
|
1756
1988
|
//#endregion
|
|
1757
1989
|
//#region src/matrix/MatrixFilter.ts
|
|
@@ -2029,13 +2261,6 @@ const adaptiveSection = {
|
|
|
2029
2261
|
formatter: formatConvergence
|
|
2030
2262
|
}] }]
|
|
2031
2263
|
};
|
|
2032
|
-
/** Build generic sections based on CLI flags */
|
|
2033
|
-
function buildGenericSections(args) {
|
|
2034
|
-
const sections = [];
|
|
2035
|
-
if (args["gc-stats"]) sections.push(gcStatsSection);
|
|
2036
|
-
sections.push(runsSection);
|
|
2037
|
-
return sections;
|
|
2038
|
-
}
|
|
2039
2264
|
/** Section: V8 optimization tier distribution and deopt count */
|
|
2040
2265
|
const optSection = {
|
|
2041
2266
|
extract: (results) => {
|
|
@@ -2060,14 +2285,52 @@ const optSection = {
|
|
|
2060
2285
|
}]
|
|
2061
2286
|
}]
|
|
2062
2287
|
};
|
|
2288
|
+
/** Build generic sections based on CLI flags */
|
|
2289
|
+
function buildGenericSections(args) {
|
|
2290
|
+
const sections = [];
|
|
2291
|
+
if (args["gc-stats"]) sections.push(gcStatsSection);
|
|
2292
|
+
sections.push(runsSection);
|
|
2293
|
+
return sections;
|
|
2294
|
+
}
|
|
2063
2295
|
|
|
2064
2296
|
//#endregion
|
|
2065
2297
|
//#region src/matrix/MatrixReport.ts
|
|
2298
|
+
/** GC statistics columns - derived from gcStatsSection for consistency */
|
|
2299
|
+
const gcStatsColumns = gcStatsSection.columns()[0].columns.map((col) => ({
|
|
2300
|
+
key: col.key,
|
|
2301
|
+
title: col.title,
|
|
2302
|
+
groupTitle: "GC",
|
|
2303
|
+
extract: (r) => gcStatsSection.extract(r.measured)[col.key],
|
|
2304
|
+
formatter: (v) => col.formatter?.(v) ?? "-"
|
|
2305
|
+
}));
|
|
2306
|
+
/** GC pause time column */
|
|
2307
|
+
const gcPauseColumn = {
|
|
2308
|
+
key: "gcPause",
|
|
2309
|
+
title: "pause",
|
|
2310
|
+
groupTitle: "GC",
|
|
2311
|
+
extract: (r) => r.measured.gcStats?.gcPauseTime,
|
|
2312
|
+
formatter: (v) => v != null ? `${v.toFixed(1)}ms` : "-"
|
|
2313
|
+
};
|
|
2314
|
+
/** Heap sampling total bytes column */
|
|
2315
|
+
const heapTotalColumn = {
|
|
2316
|
+
key: "heapTotal",
|
|
2317
|
+
title: "heap",
|
|
2318
|
+
extract: (r) => {
|
|
2319
|
+
const profile = r.measured.heapProfile;
|
|
2320
|
+
if (!profile?.head) return void 0;
|
|
2321
|
+
return totalProfileBytes(profile);
|
|
2322
|
+
},
|
|
2323
|
+
formatter: formatBytesOrDash
|
|
2324
|
+
};
|
|
2066
2325
|
/** Format matrix results as one table per case */
|
|
2067
2326
|
function reportMatrixResults(results, options) {
|
|
2068
2327
|
const tables = buildCaseTables(results, options);
|
|
2069
2328
|
return [`Matrix: ${results.name}`, ...tables].join("\n\n");
|
|
2070
2329
|
}
|
|
2330
|
+
/** Format bytes with fallback to "-" for missing values */
|
|
2331
|
+
function formatBytesOrDash(value) {
|
|
2332
|
+
return formatBytes(value) ?? "-";
|
|
2333
|
+
}
|
|
2071
2334
|
/** Build one table for each case showing all variants */
|
|
2072
2335
|
function buildCaseTables(results, options) {
|
|
2073
2336
|
if (results.variants.length === 0) return [];
|
|
@@ -2080,6 +2343,12 @@ function buildCaseTable(results, caseId, options) {
|
|
|
2080
2343
|
const rows = buildCaseRows(results, caseId, options?.extraColumns);
|
|
2081
2344
|
return `${caseTitle}\n${buildTable(buildColumns(rows.some((r) => r.diffCI), options), [{ results: rows }])}`;
|
|
2082
2345
|
}
|
|
2346
|
+
/** Format case title with metadata if available */
|
|
2347
|
+
function formatCaseTitle(results, caseId) {
|
|
2348
|
+
const metadata = (results.variants[0]?.cases.find((c) => c.caseId === caseId))?.metadata;
|
|
2349
|
+
if (metadata && Object.keys(metadata).length > 0) return `${caseId} (${Object.entries(metadata).map(([k, v]) => `${v} ${k}`).join(", ")})`;
|
|
2350
|
+
return caseId;
|
|
2351
|
+
}
|
|
2083
2352
|
/** Build table using ResultsMapper sections */
|
|
2084
2353
|
function buildSectionTable(results, caseId, options, caseTitle) {
|
|
2085
2354
|
const sections = options.sections;
|
|
@@ -2100,15 +2369,6 @@ function buildSectionTable(results, caseId, options, caseTitle) {
|
|
|
2100
2369
|
}
|
|
2101
2370
|
return `${caseTitle}\n${buildTable(buildSectionColumns(sections, variantTitle, hasBaseline), [{ results: rows }])}`;
|
|
2102
2371
|
}
|
|
2103
|
-
/** Build column groups from ResultsMapper sections */
|
|
2104
|
-
function buildSectionColumns(sections, variantTitle, hasBaseline) {
|
|
2105
|
-
const nameCol = { columns: [{
|
|
2106
|
-
key: "name",
|
|
2107
|
-
title: variantTitle
|
|
2108
|
-
}] };
|
|
2109
|
-
const sectionColumns = sections.flatMap((s) => s.columns());
|
|
2110
|
-
return [nameCol, ...hasBaseline ? injectDiffColumns(sectionColumns) : sectionColumns];
|
|
2111
|
-
}
|
|
2112
2372
|
/** Build rows for all variants for a given case */
|
|
2113
2373
|
function buildCaseRows(results, caseId, extraColumns) {
|
|
2114
2374
|
return results.variants.flatMap((variant) => {
|
|
@@ -2116,20 +2376,6 @@ function buildCaseRows(results, caseId, extraColumns) {
|
|
|
2116
2376
|
return caseResult ? [buildRow(variant.id, caseResult, extraColumns)] : [];
|
|
2117
2377
|
});
|
|
2118
2378
|
}
|
|
2119
|
-
/** Build a single row from case result */
|
|
2120
|
-
function buildRow(variantId, caseResult, extraColumns) {
|
|
2121
|
-
const { measured, baseline } = caseResult;
|
|
2122
|
-
const samples = measured.samples;
|
|
2123
|
-
const time = measured.time?.avg ?? average(samples);
|
|
2124
|
-
const row = {
|
|
2125
|
-
name: truncate(variantId, 25),
|
|
2126
|
-
time,
|
|
2127
|
-
samples: samples.length
|
|
2128
|
-
};
|
|
2129
|
-
if (baseline) row.diffCI = bootstrapDifferenceCI(baseline.samples, samples);
|
|
2130
|
-
if (extraColumns) for (const col of extraColumns) row[col.key] = col.extract(caseResult);
|
|
2131
|
-
return row;
|
|
2132
|
-
}
|
|
2133
2379
|
/** Build column configuration */
|
|
2134
2380
|
function buildColumns(hasBaseline, options) {
|
|
2135
2381
|
const groups = [{ columns: [{
|
|
@@ -2163,48 +2409,34 @@ function buildColumns(hasBaseline, options) {
|
|
|
2163
2409
|
}
|
|
2164
2410
|
return groups;
|
|
2165
2411
|
}
|
|
2412
|
+
/** Build column groups from ResultsMapper sections */
|
|
2413
|
+
function buildSectionColumns(sections, variantTitle, hasBaseline) {
|
|
2414
|
+
const nameCol = { columns: [{
|
|
2415
|
+
key: "name",
|
|
2416
|
+
title: variantTitle
|
|
2417
|
+
}] };
|
|
2418
|
+
const sectionColumns = sections.flatMap((s) => s.columns());
|
|
2419
|
+
return [nameCol, ...hasBaseline ? injectDiffColumns(sectionColumns) : sectionColumns];
|
|
2420
|
+
}
|
|
2421
|
+
/** Build a single row from case result */
|
|
2422
|
+
function buildRow(variantId, caseResult, extraColumns) {
|
|
2423
|
+
const { measured, baseline } = caseResult;
|
|
2424
|
+
const samples = measured.samples;
|
|
2425
|
+
const time = measured.time?.avg ?? average(samples);
|
|
2426
|
+
const row = {
|
|
2427
|
+
name: truncate(variantId, 25),
|
|
2428
|
+
time,
|
|
2429
|
+
samples: samples.length
|
|
2430
|
+
};
|
|
2431
|
+
if (baseline) row.diffCI = bootstrapDifferenceCI(baseline.samples, samples);
|
|
2432
|
+
if (extraColumns) for (const col of extraColumns) row[col.key] = col.extract(caseResult);
|
|
2433
|
+
return row;
|
|
2434
|
+
}
|
|
2166
2435
|
/** Format diff with CI, or "baseline" marker */
|
|
2167
2436
|
function formatDiff(value) {
|
|
2168
2437
|
if (!value) return null;
|
|
2169
2438
|
return formatDiffWithCI(value);
|
|
2170
2439
|
}
|
|
2171
|
-
/** Format case title with metadata if available */
|
|
2172
|
-
function formatCaseTitle(results, caseId) {
|
|
2173
|
-
const metadata = (results.variants[0]?.cases.find((c) => c.caseId === caseId))?.metadata;
|
|
2174
|
-
if (metadata && Object.keys(metadata).length > 0) return `${caseId} (${Object.entries(metadata).map(([k, v]) => `${v} ${k}`).join(", ")})`;
|
|
2175
|
-
return caseId;
|
|
2176
|
-
}
|
|
2177
|
-
/** GC statistics columns - derived from gcStatsSection for consistency */
|
|
2178
|
-
const gcStatsColumns = gcStatsSection.columns()[0].columns.map((col) => ({
|
|
2179
|
-
key: col.key,
|
|
2180
|
-
title: col.title,
|
|
2181
|
-
groupTitle: "GC",
|
|
2182
|
-
extract: (r) => gcStatsSection.extract(r.measured)[col.key],
|
|
2183
|
-
formatter: (v) => col.formatter?.(v) ?? "-"
|
|
2184
|
-
}));
|
|
2185
|
-
/** Format bytes with fallback to "-" for missing values */
|
|
2186
|
-
function formatBytesOrDash(value) {
|
|
2187
|
-
return formatBytes(value) ?? "-";
|
|
2188
|
-
}
|
|
2189
|
-
/** GC pause time column */
|
|
2190
|
-
const gcPauseColumn = {
|
|
2191
|
-
key: "gcPause",
|
|
2192
|
-
title: "pause",
|
|
2193
|
-
groupTitle: "GC",
|
|
2194
|
-
extract: (r) => r.measured.gcStats?.gcPauseTime,
|
|
2195
|
-
formatter: (v) => v != null ? `${v.toFixed(1)}ms` : "-"
|
|
2196
|
-
};
|
|
2197
|
-
/** Heap sampling total bytes column */
|
|
2198
|
-
const heapTotalColumn = {
|
|
2199
|
-
key: "heapTotal",
|
|
2200
|
-
title: "heap",
|
|
2201
|
-
extract: (r) => {
|
|
2202
|
-
const profile = r.measured.heapProfile;
|
|
2203
|
-
if (!profile?.head) return void 0;
|
|
2204
|
-
return totalProfileBytes(profile);
|
|
2205
|
-
},
|
|
2206
|
-
formatter: formatBytesOrDash
|
|
2207
|
-
};
|
|
2208
2440
|
|
|
2209
2441
|
//#endregion
|
|
2210
2442
|
//#region src/cli/FilterBenchmarks.ts
|
|
@@ -2239,32 +2471,21 @@ function createFilterRegex(filter) {
|
|
|
2239
2471
|
function stripCaseSuffix(name) {
|
|
2240
2472
|
return name.replace(/ \[.*?\]$/, "");
|
|
2241
2473
|
}
|
|
2242
|
-
/** Escape regex special characters */
|
|
2243
|
-
function escapeRegex(str) {
|
|
2244
|
-
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2245
|
-
}
|
|
2246
2474
|
/** Ensure at least one benchmark matches filter */
|
|
2247
2475
|
function validateFilteredSuite(groups, filter) {
|
|
2248
2476
|
if (groups.every((g) => g.benchmarks.length === 0)) throw new Error(`No benchmarks match filter: "${filter}"`);
|
|
2249
2477
|
}
|
|
2478
|
+
/** Escape regex special characters */
|
|
2479
|
+
function escapeRegex(str) {
|
|
2480
|
+
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2481
|
+
}
|
|
2250
2482
|
|
|
2251
2483
|
//#endregion
|
|
2252
2484
|
//#region src/cli/RunBenchCLI.ts
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
}
|
|
2257
|
-
/** Warn about Node-only flags that are ignored in browser mode. */
|
|
2258
|
-
function warnBrowserFlags(args) {
|
|
2259
|
-
const ignored = [];
|
|
2260
|
-
if (!args.worker) ignored.push("--no-worker");
|
|
2261
|
-
if (args.cpu) ignored.push("--cpu");
|
|
2262
|
-
if (args["trace-opt"]) ignored.push("--trace-opt");
|
|
2263
|
-
if (args.collect) ignored.push("--collect");
|
|
2264
|
-
if (args.adaptive) ignored.push("--adaptive");
|
|
2265
|
-
if (args.batches > 1) ignored.push("--batches");
|
|
2266
|
-
if (ignored.length) console.warn(yellow(`Ignored in browser mode: ${ignored.join(", ")}`));
|
|
2267
|
-
}
|
|
2485
|
+
const { yellow, dim } = process.env.NODE_ENV === "test" || process.env.VITEST === "true" ? {
|
|
2486
|
+
yellow: (s) => s,
|
|
2487
|
+
dim: (s) => s
|
|
2488
|
+
} : pico;
|
|
2268
2489
|
/** Parse CLI with custom configuration */
|
|
2269
2490
|
function parseBenchArgs(configureArgs) {
|
|
2270
2491
|
return parseCliArgs(hideBin(process.argv), configureArgs);
|
|
@@ -2282,144 +2503,6 @@ async function runBenchmarks(suite, args) {
|
|
|
2282
2503
|
batches
|
|
2283
2504
|
});
|
|
2284
2505
|
}
|
|
2285
|
-
/** Execute all groups in suite */
|
|
2286
|
-
async function runSuite(params) {
|
|
2287
|
-
const { suite, runner, options, useWorker, batches } = params;
|
|
2288
|
-
const results = [];
|
|
2289
|
-
for (const group of suite.groups) results.push(await runGroup(group, runner, options, useWorker, batches));
|
|
2290
|
-
return results;
|
|
2291
|
-
}
|
|
2292
|
-
/** Execute group with shared setup, optionally batching to reduce ordering bias */
|
|
2293
|
-
async function runGroup(group, runner, options, useWorker, batches = 1) {
|
|
2294
|
-
const { name, benchmarks, baseline, setup, metadata } = group;
|
|
2295
|
-
const setupParams = await setup?.();
|
|
2296
|
-
validateBenchmarkParameters(group);
|
|
2297
|
-
const runParams = {
|
|
2298
|
-
runner,
|
|
2299
|
-
options,
|
|
2300
|
-
useWorker,
|
|
2301
|
-
params: setupParams,
|
|
2302
|
-
metadata
|
|
2303
|
-
};
|
|
2304
|
-
if (batches === 1) return runSingleBatch(name, benchmarks, baseline, runParams);
|
|
2305
|
-
return runMultipleBatches(name, benchmarks, baseline, runParams, batches);
|
|
2306
|
-
}
|
|
2307
|
-
/** Run benchmarks in a single batch */
|
|
2308
|
-
async function runSingleBatch(name, benchmarks, baseline, runParams) {
|
|
2309
|
-
const baselineReport = baseline ? await runSingleBenchmark(baseline, runParams) : void 0;
|
|
2310
|
-
return {
|
|
2311
|
-
name,
|
|
2312
|
-
reports: await serialMap(benchmarks, (b) => runSingleBenchmark(b, runParams)),
|
|
2313
|
-
baseline: baselineReport
|
|
2314
|
-
};
|
|
2315
|
-
}
|
|
2316
|
-
/** Run benchmarks in multiple batches, alternating order to reduce bias */
|
|
2317
|
-
async function runMultipleBatches(name, benchmarks, baseline, runParams, batches) {
|
|
2318
|
-
const timePerBatch = (runParams.options.maxTime || 5e3) / batches;
|
|
2319
|
-
const batchParams = {
|
|
2320
|
-
...runParams,
|
|
2321
|
-
options: {
|
|
2322
|
-
...runParams.options,
|
|
2323
|
-
maxTime: timePerBatch
|
|
2324
|
-
}
|
|
2325
|
-
};
|
|
2326
|
-
const baselineBatches = [];
|
|
2327
|
-
const benchmarkBatches = /* @__PURE__ */ new Map();
|
|
2328
|
-
for (let i = 0; i < batches; i++) await runBatchIteration(benchmarks, baseline, batchParams, i % 2 === 1, baselineBatches, benchmarkBatches);
|
|
2329
|
-
const meta = runParams.metadata;
|
|
2330
|
-
return mergeBatchResults(name, benchmarks, baseline, baselineBatches, benchmarkBatches, meta);
|
|
2331
|
-
}
|
|
2332
|
-
/** Run one batch iteration in either order */
|
|
2333
|
-
async function runBatchIteration(benchmarks, baseline, runParams, reverseOrder, baselineBatches, benchmarkBatches) {
|
|
2334
|
-
const runBaseline = async () => {
|
|
2335
|
-
if (baseline) {
|
|
2336
|
-
const r = await runSingleBenchmark(baseline, runParams);
|
|
2337
|
-
baselineBatches.push(r.measuredResults);
|
|
2338
|
-
}
|
|
2339
|
-
};
|
|
2340
|
-
const runBenches = async () => {
|
|
2341
|
-
for (const b of benchmarks) {
|
|
2342
|
-
const r = await runSingleBenchmark(b, runParams);
|
|
2343
|
-
appendToMap(benchmarkBatches, b.name, r.measuredResults);
|
|
2344
|
-
}
|
|
2345
|
-
};
|
|
2346
|
-
if (reverseOrder) {
|
|
2347
|
-
await runBenches();
|
|
2348
|
-
await runBaseline();
|
|
2349
|
-
} else {
|
|
2350
|
-
await runBaseline();
|
|
2351
|
-
await runBenches();
|
|
2352
|
-
}
|
|
2353
|
-
}
|
|
2354
|
-
/** Merge batch results into final ReportGroup */
|
|
2355
|
-
function mergeBatchResults(name, benchmarks, baseline, baselineBatches, benchmarkBatches, metadata) {
|
|
2356
|
-
const mergedBaseline = baseline ? {
|
|
2357
|
-
name: baseline.name,
|
|
2358
|
-
measuredResults: mergeResults(baselineBatches),
|
|
2359
|
-
metadata
|
|
2360
|
-
} : void 0;
|
|
2361
|
-
return {
|
|
2362
|
-
name,
|
|
2363
|
-
reports: benchmarks.map((b) => ({
|
|
2364
|
-
name: b.name,
|
|
2365
|
-
measuredResults: mergeResults(benchmarkBatches.get(b.name) || []),
|
|
2366
|
-
metadata
|
|
2367
|
-
})),
|
|
2368
|
-
baseline: mergedBaseline
|
|
2369
|
-
};
|
|
2370
|
-
}
|
|
2371
|
-
/** Run single benchmark and create report */
|
|
2372
|
-
async function runSingleBenchmark(spec, runParams) {
|
|
2373
|
-
const { runner, options, useWorker, params, metadata } = runParams;
|
|
2374
|
-
const [result] = await runBenchmark({
|
|
2375
|
-
spec,
|
|
2376
|
-
runner,
|
|
2377
|
-
options,
|
|
2378
|
-
useWorker,
|
|
2379
|
-
params
|
|
2380
|
-
});
|
|
2381
|
-
return {
|
|
2382
|
-
name: spec.name,
|
|
2383
|
-
measuredResults: result,
|
|
2384
|
-
metadata
|
|
2385
|
-
};
|
|
2386
|
-
}
|
|
2387
|
-
/** Warn if parameterized benchmarks lack setup */
|
|
2388
|
-
function validateBenchmarkParameters(group) {
|
|
2389
|
-
const { name, setup, benchmarks, baseline } = group;
|
|
2390
|
-
if (setup) return;
|
|
2391
|
-
const allBenchmarks = baseline ? [...benchmarks, baseline] : benchmarks;
|
|
2392
|
-
for (const benchmark of allBenchmarks) if (benchmark.fn.length > 0) console.warn(`Benchmark "${benchmark.name}" in group "${name}" expects parameters but no setup() provided.`);
|
|
2393
|
-
}
|
|
2394
|
-
/** Merge multiple batch results into a single MeasuredResults */
|
|
2395
|
-
function mergeResults(results) {
|
|
2396
|
-
if (results.length === 0) throw new Error("Cannot merge empty results array");
|
|
2397
|
-
if (results.length === 1) return results[0];
|
|
2398
|
-
const allSamples = results.flatMap((r) => r.samples);
|
|
2399
|
-
const allWarmup = results.flatMap((r) => r.warmupSamples || []);
|
|
2400
|
-
const time = computeStats(allSamples);
|
|
2401
|
-
let offset = 0;
|
|
2402
|
-
const allPausePoints = results.flatMap((r) => {
|
|
2403
|
-
const pts = (r.pausePoints ?? []).map((p) => ({
|
|
2404
|
-
sampleIndex: p.sampleIndex + offset,
|
|
2405
|
-
durationMs: p.durationMs
|
|
2406
|
-
}));
|
|
2407
|
-
offset += r.samples.length;
|
|
2408
|
-
return pts;
|
|
2409
|
-
});
|
|
2410
|
-
return {
|
|
2411
|
-
name: results[0].name,
|
|
2412
|
-
samples: allSamples,
|
|
2413
|
-
warmupSamples: allWarmup.length ? allWarmup : void 0,
|
|
2414
|
-
time,
|
|
2415
|
-
totalTime: results.reduce((sum, r) => sum + (r.totalTime || 0), 0),
|
|
2416
|
-
pausePoints: allPausePoints.length ? allPausePoints : void 0
|
|
2417
|
-
};
|
|
2418
|
-
}
|
|
2419
|
-
function appendToMap(map, key, value) {
|
|
2420
|
-
if (!map.has(key)) map.set(key, []);
|
|
2421
|
-
map.get(key).push(value);
|
|
2422
|
-
}
|
|
2423
2506
|
/** Generate table with standard sections */
|
|
2424
2507
|
function defaultReport(groups, args) {
|
|
2425
2508
|
const { adaptive, "gc-stats": gcStats, "trace-opt": traceOpt } = args;
|
|
@@ -2427,15 +2510,6 @@ function defaultReport(groups, args) {
|
|
|
2427
2510
|
const hasOpt = hasField(groups, "optStatus");
|
|
2428
2511
|
return reportResults(groups, buildReportSections(adaptive, gcStats, hasCpu, traceOpt && hasOpt));
|
|
2429
2512
|
}
|
|
2430
|
-
/** Build report sections based on CLI options */
|
|
2431
|
-
function buildReportSections(adaptive, gcStats, hasCpuData, hasOptData) {
|
|
2432
|
-
const sections = adaptive ? [adaptiveSection, totalTimeSection] : [timeSection];
|
|
2433
|
-
if (gcStats) sections.push(gcStatsSection);
|
|
2434
|
-
if (hasCpuData) sections.push(cpuSection);
|
|
2435
|
-
if (hasOptData) sections.push(optSection);
|
|
2436
|
-
sections.push(runsSection);
|
|
2437
|
-
return sections;
|
|
2438
|
-
}
|
|
2439
2513
|
/** Run benchmarks, display table, and optionally generate HTML report */
|
|
2440
2514
|
async function benchExports(suite, args) {
|
|
2441
2515
|
const results = await runBenchmarks(suite, args);
|
|
@@ -2448,7 +2522,7 @@ async function browserBenchExports(args) {
|
|
|
2448
2522
|
warnBrowserFlags(args);
|
|
2449
2523
|
let profileBrowser;
|
|
2450
2524
|
try {
|
|
2451
|
-
({profileBrowser} = await import("./BrowserHeapSampler-
|
|
2525
|
+
({profileBrowser} = await import("./BrowserHeapSampler-B6asLKWQ.mjs"));
|
|
2452
2526
|
} catch {
|
|
2453
2527
|
throw new Error("playwright is required for browser benchmarking (--url).\n\nQuick start: npx benchforge-browser --url <your-url>\n\nOr install manually:\n npm install playwright\n npx playwright install chromium");
|
|
2454
2528
|
}
|
|
@@ -2456,7 +2530,7 @@ async function browserBenchExports(args) {
|
|
|
2456
2530
|
const { iterations, time } = args;
|
|
2457
2531
|
const result = await profileBrowser({
|
|
2458
2532
|
url,
|
|
2459
|
-
heapSample: args
|
|
2533
|
+
heapSample: needsHeapSample(args),
|
|
2460
2534
|
heapOptions: {
|
|
2461
2535
|
samplingInterval: args["heap-interval"],
|
|
2462
2536
|
stackDepth: args["heap-depth"]
|
|
@@ -2475,82 +2549,29 @@ async function browserBenchExports(args) {
|
|
|
2475
2549
|
args
|
|
2476
2550
|
});
|
|
2477
2551
|
}
|
|
2478
|
-
/** Print browser benchmark tables and heap reports */
|
|
2479
|
-
function printBrowserReport(result, results, args) {
|
|
2480
|
-
const hasSamples = result.samples && result.samples.length > 0;
|
|
2481
|
-
const sections = [];
|
|
2482
|
-
if (hasSamples || result.wallTimeMs != null) sections.push(timeSection);
|
|
2483
|
-
if (result.gcStats) sections.push(browserGcStatsSection);
|
|
2484
|
-
if (hasSamples || result.wallTimeMs != null) sections.push(runsSection);
|
|
2485
|
-
if (sections.length > 0) console.log(reportResults(results, sections));
|
|
2486
|
-
if (result.heapProfile) printHeapReports(results, {
|
|
2487
|
-
...cliHeapReportOptions(args),
|
|
2488
|
-
isUserCode: isBrowserUserCode
|
|
2489
|
-
});
|
|
2490
|
-
}
|
|
2491
|
-
/** Wrap browser profile result as ReportGroup[] for the standard pipeline */
|
|
2492
|
-
function browserResultGroups(name, result) {
|
|
2493
|
-
const { gcStats, heapProfile } = result;
|
|
2494
|
-
let measured;
|
|
2495
|
-
if (result.samples && result.samples.length > 0) {
|
|
2496
|
-
const { samples } = result;
|
|
2497
|
-
const totalTime = result.wallTimeMs ? result.wallTimeMs / 1e3 : void 0;
|
|
2498
|
-
measured = {
|
|
2499
|
-
name,
|
|
2500
|
-
samples,
|
|
2501
|
-
time: computeStats(samples),
|
|
2502
|
-
totalTime,
|
|
2503
|
-
gcStats,
|
|
2504
|
-
heapProfile
|
|
2505
|
-
};
|
|
2506
|
-
} else {
|
|
2507
|
-
const wallMs = result.wallTimeMs ?? 0;
|
|
2508
|
-
measured = {
|
|
2509
|
-
name,
|
|
2510
|
-
samples: [wallMs],
|
|
2511
|
-
time: {
|
|
2512
|
-
min: wallMs,
|
|
2513
|
-
max: wallMs,
|
|
2514
|
-
avg: wallMs,
|
|
2515
|
-
p50: wallMs,
|
|
2516
|
-
p75: wallMs,
|
|
2517
|
-
p99: wallMs,
|
|
2518
|
-
p999: wallMs
|
|
2519
|
-
},
|
|
2520
|
-
gcStats,
|
|
2521
|
-
heapProfile
|
|
2522
|
-
};
|
|
2523
|
-
}
|
|
2524
|
-
return [{
|
|
2525
|
-
name,
|
|
2526
|
-
reports: [{
|
|
2527
|
-
name,
|
|
2528
|
-
measuredResults: measured
|
|
2529
|
-
}]
|
|
2530
|
-
}];
|
|
2531
|
-
}
|
|
2532
2552
|
/** Print heap allocation reports for benchmarks with heap profiles */
|
|
2533
2553
|
function printHeapReports(groups, options) {
|
|
2534
|
-
for (const group of groups) {
|
|
2535
|
-
const
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
}));
|
|
2554
|
+
for (const group of groups) for (const report of groupReports(group)) {
|
|
2555
|
+
const { heapProfile } = report.measuredResults;
|
|
2556
|
+
if (!heapProfile) continue;
|
|
2557
|
+
console.log(dim(`\n─── Heap profile: ${report.name} ───`));
|
|
2558
|
+
const resolved = resolveProfile(heapProfile);
|
|
2559
|
+
const sites = flattenProfile(resolved);
|
|
2560
|
+
const userSites = filterSites(sites, options.isUserCode);
|
|
2561
|
+
const totalUserCode = userSites.reduce((sum, s) => sum + s.bytes, 0);
|
|
2562
|
+
const aggregated = aggregateSites(options.userOnly ? userSites : sites);
|
|
2563
|
+
const extra = {
|
|
2564
|
+
totalAll: resolved.totalBytes,
|
|
2565
|
+
totalUserCode,
|
|
2566
|
+
sampleCount: resolved.sortedSamples?.length
|
|
2567
|
+
};
|
|
2568
|
+
console.log(formatHeapReport(aggregated, {
|
|
2569
|
+
...options,
|
|
2570
|
+
...extra
|
|
2571
|
+
}));
|
|
2572
|
+
if (options.raw) {
|
|
2573
|
+
console.log(dim(`\n─── Raw samples: ${report.name} ───`));
|
|
2574
|
+
console.log(formatRawSamples(resolved));
|
|
2554
2575
|
}
|
|
2555
2576
|
}
|
|
2556
2577
|
}
|
|
@@ -2559,7 +2580,8 @@ async function runDefaultBench(suite, configureArgs) {
|
|
|
2559
2580
|
const args = parseBenchArgs(configureArgs);
|
|
2560
2581
|
if (args.url) await browserBenchExports(args);
|
|
2561
2582
|
else if (suite) await benchExports(suite, args);
|
|
2562
|
-
else
|
|
2583
|
+
else if (args.file) await fileBenchExports(args.file, args);
|
|
2584
|
+
else throw new Error("Provide a benchmark file, --url for browser mode, or pass a BenchSuite directly.");
|
|
2563
2585
|
}
|
|
2564
2586
|
/** Convert CLI args to runner options */
|
|
2565
2587
|
function cliToRunnerOptions(args) {
|
|
@@ -2576,77 +2598,28 @@ function cliToRunnerOptions(args) {
|
|
|
2576
2598
|
...cliCommonOptions(args)
|
|
2577
2599
|
};
|
|
2578
2600
|
}
|
|
2579
|
-
/**
|
|
2580
|
-
function
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
};
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
const
|
|
2596
|
-
|
|
2597
|
-
return {
|
|
2598
|
-
collect,
|
|
2599
|
-
cpuCounters: cpu,
|
|
2600
|
-
warmup,
|
|
2601
|
-
traceOpt,
|
|
2602
|
-
noSettle,
|
|
2603
|
-
pauseFirst,
|
|
2604
|
-
pauseInterval,
|
|
2605
|
-
pauseDuration,
|
|
2606
|
-
gcStats,
|
|
2607
|
-
heapSample,
|
|
2608
|
-
heapInterval,
|
|
2609
|
-
heapDepth
|
|
2610
|
-
};
|
|
2611
|
-
}
|
|
2612
|
-
const { yellow, dim } = process.env.NODE_ENV === "test" || process.env.VITEST === "true" ? {
|
|
2613
|
-
yellow: (s) => s,
|
|
2614
|
-
dim: (s) => s
|
|
2615
|
-
} : pico;
|
|
2616
|
-
/** Log V8 optimization tier distribution and deoptimizations */
|
|
2617
|
-
function reportOptStatus(groups) {
|
|
2618
|
-
const optData = groups.flatMap(({ reports, baseline }) => {
|
|
2619
|
-
return (baseline ? [...reports, baseline] : reports).filter((r) => r.measuredResults.optStatus).map((r) => ({
|
|
2620
|
-
name: r.name,
|
|
2621
|
-
opt: r.measuredResults.optStatus,
|
|
2622
|
-
samples: r.measuredResults.samples.length
|
|
2623
|
-
}));
|
|
2624
|
-
});
|
|
2625
|
-
if (optData.length === 0) return;
|
|
2626
|
-
console.log(dim("\nV8 optimization:"));
|
|
2627
|
-
for (const { name, opt, samples } of optData) {
|
|
2628
|
-
const total = Object.values(opt.byTier).reduce((s, t) => s + t.count, 0);
|
|
2629
|
-
const tierParts = Object.entries(opt.byTier).sort((a, b) => b[1].count - a[1].count).map(([tier, info]) => `${tier} ${(info.count / total * 100).toFixed(0)}%`).join(", ");
|
|
2630
|
-
console.log(` ${name}: ${tierParts} ${dim(`(${samples} samples)`)}`);
|
|
2631
|
-
}
|
|
2632
|
-
const totalDeopts = optData.reduce((s, d) => s + d.opt.deoptCount, 0);
|
|
2633
|
-
if (totalDeopts > 0) console.log(yellow(` ⚠ ${totalDeopts} deoptimization${totalDeopts > 1 ? "s" : ""} detected`));
|
|
2601
|
+
/** Log V8 optimization tier distribution and deoptimizations */
|
|
2602
|
+
function reportOptStatus(groups) {
|
|
2603
|
+
const optData = groups.flatMap((group) => {
|
|
2604
|
+
return groupReports(group).filter((r) => r.measuredResults.optStatus).map((r) => ({
|
|
2605
|
+
name: r.name,
|
|
2606
|
+
opt: r.measuredResults.optStatus,
|
|
2607
|
+
samples: r.measuredResults.samples.length
|
|
2608
|
+
}));
|
|
2609
|
+
});
|
|
2610
|
+
if (optData.length === 0) return;
|
|
2611
|
+
console.log(dim("\nV8 optimization:"));
|
|
2612
|
+
for (const { name, opt, samples } of optData) {
|
|
2613
|
+
const total = Object.values(opt.byTier).reduce((s, t) => s + t.count, 0);
|
|
2614
|
+
const tierParts = Object.entries(opt.byTier).sort((a, b) => b[1].count - a[1].count).map(([tier, info]) => `${tier} ${(info.count / total * 100).toFixed(0)}%`).join(", ");
|
|
2615
|
+
console.log(` ${name}: ${tierParts} ${dim(`(${samples} samples)`)}`);
|
|
2616
|
+
}
|
|
2617
|
+
const totalDeopts = optData.reduce((s, d) => s + d.opt.deoptCount, 0);
|
|
2618
|
+
if (totalDeopts > 0) console.log(yellow(` ⚠ ${totalDeopts} deoptimization${totalDeopts > 1 ? "s" : ""} detected`));
|
|
2634
2619
|
}
|
|
2635
2620
|
/** @return true if any result has the specified field with a defined value */
|
|
2636
2621
|
function hasField(results, field) {
|
|
2637
|
-
return results.some(({
|
|
2638
|
-
return (baseline ? [...reports, baseline] : reports).some(({ measuredResults }) => measuredResults[field] !== void 0);
|
|
2639
|
-
});
|
|
2640
|
-
}
|
|
2641
|
-
/** Print heap reports (if enabled) and export results */
|
|
2642
|
-
async function finishReports(results, args, suiteName, exportOptions) {
|
|
2643
|
-
if (args["heap-sample"]) printHeapReports(results, cliHeapReportOptions(args));
|
|
2644
|
-
await exportReports({
|
|
2645
|
-
results,
|
|
2646
|
-
args,
|
|
2647
|
-
suiteName,
|
|
2648
|
-
...exportOptions
|
|
2649
|
-
});
|
|
2622
|
+
return results.some((group) => groupReports(group).some(({ measuredResults }) => measuredResults[field] !== void 0));
|
|
2650
2623
|
}
|
|
2651
2624
|
/** Export reports (HTML, JSON, Perfetto) based on CLI args */
|
|
2652
2625
|
async function exportReports(options) {
|
|
@@ -2664,22 +2637,14 @@ async function exportReports(options) {
|
|
|
2664
2637
|
outputPath: args["export-html"]
|
|
2665
2638
|
})).closeServer;
|
|
2666
2639
|
if (args.json) await exportBenchmarkJson(results, args.json, args, suiteName);
|
|
2667
|
-
if (args
|
|
2640
|
+
if (args["export-perfetto"]) exportPerfettoTrace(results, args["export-perfetto"], args);
|
|
2641
|
+
if (args["export-speedscope"]) exportSpeedscope(results, args["export-speedscope"]);
|
|
2642
|
+
if (args.speedscope) exportAndLaunchSpeedscope(results);
|
|
2668
2643
|
if (openInBrowser) {
|
|
2669
2644
|
await waitForCtrlC();
|
|
2670
2645
|
closeServer?.();
|
|
2671
2646
|
}
|
|
2672
2647
|
}
|
|
2673
|
-
/** Wait for Ctrl+C before exiting */
|
|
2674
|
-
function waitForCtrlC() {
|
|
2675
|
-
return new Promise((resolve) => {
|
|
2676
|
-
console.log(dim("\nPress Ctrl+C to exit"));
|
|
2677
|
-
process.on("SIGINT", () => {
|
|
2678
|
-
console.log();
|
|
2679
|
-
resolve();
|
|
2680
|
-
});
|
|
2681
|
-
});
|
|
2682
|
-
}
|
|
2683
2648
|
/** Run matrix suite with CLI arguments.
|
|
2684
2649
|
* no options ==> defaultCases/defaultVariants, --filter ==> subset of defaults,
|
|
2685
2650
|
* --all --filter ==> subset of all, --all ==> all cases/variants */
|
|
@@ -2721,24 +2686,6 @@ function defaultMatrixReport(results, reportOptions, args) {
|
|
|
2721
2686
|
const options = args ? mergeMatrixDefaults(reportOptions, args, results) : reportOptions;
|
|
2722
2687
|
return results.map((r) => reportMatrixResults(r, options)).join("\n\n");
|
|
2723
2688
|
}
|
|
2724
|
-
/** @return HeapReportOptions from CLI args */
|
|
2725
|
-
function cliHeapReportOptions(args) {
|
|
2726
|
-
return {
|
|
2727
|
-
topN: args["heap-rows"],
|
|
2728
|
-
stackDepth: args["heap-stack"],
|
|
2729
|
-
verbose: args["heap-verbose"],
|
|
2730
|
-
userOnly: args["heap-user-only"]
|
|
2731
|
-
};
|
|
2732
|
-
}
|
|
2733
|
-
/** Apply default sections and extra columns for matrix reports */
|
|
2734
|
-
function mergeMatrixDefaults(reportOptions, args, results) {
|
|
2735
|
-
const result = { ...reportOptions };
|
|
2736
|
-
if (!result.sections?.length) {
|
|
2737
|
-
const groups = matrixToReportGroups(results);
|
|
2738
|
-
result.sections = buildReportSections(args.adaptive, args["gc-stats"], hasField(groups, "cpu"), args["trace-opt"] && hasField(groups, "optStatus"));
|
|
2739
|
-
}
|
|
2740
|
-
return result;
|
|
2741
|
-
}
|
|
2742
2689
|
/** Run matrix suite with full CLI handling (parse, run, report, export) */
|
|
2743
2690
|
async function runDefaultMatrixBench(suite, configureArgs, reportOptions) {
|
|
2744
2691
|
await matrixBenchExports(suite, parseBenchArgs(configureArgs), reportOptions);
|
|
@@ -2764,6 +2711,58 @@ function matrixToReportGroups(results) {
|
|
|
2764
2711
|
};
|
|
2765
2712
|
})));
|
|
2766
2713
|
}
|
|
2714
|
+
/** Run matrix benchmarks, display table, and generate exports */
|
|
2715
|
+
async function matrixBenchExports(suite, args, reportOptions, exportOptions) {
|
|
2716
|
+
const results = await runMatrixSuite(suite, args);
|
|
2717
|
+
const report = defaultMatrixReport(results, reportOptions, args);
|
|
2718
|
+
console.log(report);
|
|
2719
|
+
await finishReports(matrixToReportGroups(results), args, suite.name, exportOptions);
|
|
2720
|
+
}
|
|
2721
|
+
/** Validate CLI argument combinations */
|
|
2722
|
+
function validateArgs(args) {
|
|
2723
|
+
if (args["gc-stats"] && !args.worker && !args.url) throw new Error("--gc-stats requires worker mode (the default). Remove --no-worker flag.");
|
|
2724
|
+
}
|
|
2725
|
+
/** Execute all groups in suite */
|
|
2726
|
+
async function runSuite(params) {
|
|
2727
|
+
const { suite, runner, options, useWorker, batches } = params;
|
|
2728
|
+
const results = [];
|
|
2729
|
+
for (const group of suite.groups) results.push(await runGroup(group, runner, options, useWorker, batches));
|
|
2730
|
+
return results;
|
|
2731
|
+
}
|
|
2732
|
+
/** Build report sections based on CLI options */
|
|
2733
|
+
function buildReportSections(adaptive, gcStats, hasCpuData, hasOptData) {
|
|
2734
|
+
const sections = adaptive ? [adaptiveSection, totalTimeSection] : [timeSection];
|
|
2735
|
+
if (gcStats) sections.push(gcStatsSection);
|
|
2736
|
+
if (hasCpuData) sections.push(cpuSection);
|
|
2737
|
+
if (hasOptData) sections.push(optSection);
|
|
2738
|
+
sections.push(runsSection);
|
|
2739
|
+
return sections;
|
|
2740
|
+
}
|
|
2741
|
+
/** Print heap reports (if enabled) and export results */
|
|
2742
|
+
async function finishReports(results, args, suiteName, exportOptions) {
|
|
2743
|
+
if (needsHeapSample(args)) printHeapReports(results, cliHeapReportOptions(args));
|
|
2744
|
+
await exportReports({
|
|
2745
|
+
results,
|
|
2746
|
+
args,
|
|
2747
|
+
suiteName,
|
|
2748
|
+
...exportOptions
|
|
2749
|
+
});
|
|
2750
|
+
}
|
|
2751
|
+
/** Warn about Node-only flags that are ignored in browser mode. */
|
|
2752
|
+
function warnBrowserFlags(args) {
|
|
2753
|
+
const ignored = [];
|
|
2754
|
+
if (!args.worker) ignored.push("--no-worker");
|
|
2755
|
+
if (args.cpu) ignored.push("--cpu");
|
|
2756
|
+
if (args["trace-opt"]) ignored.push("--trace-opt");
|
|
2757
|
+
if (args.collect) ignored.push("--collect");
|
|
2758
|
+
if (args.adaptive) ignored.push("--adaptive");
|
|
2759
|
+
if (args.batches > 1) ignored.push("--batches");
|
|
2760
|
+
if (ignored.length) console.warn(yellow(`Ignored in browser mode: ${ignored.join(", ")}`));
|
|
2761
|
+
}
|
|
2762
|
+
/** @return true if any heap-related flag implies heap sampling */
|
|
2763
|
+
function needsHeapSample(args) {
|
|
2764
|
+
return args["heap-sample"] || args.speedscope || !!args["export-speedscope"] || args["heap-raw"] || args["heap-verbose"] || args["heap-user-only"];
|
|
2765
|
+
}
|
|
2767
2766
|
/** Strip surrounding quotes from a chrome arg token.
|
|
2768
2767
|
*
|
|
2769
2768
|
* (Needed because --chrome-args values pass through yargs and spawn() without
|
|
@@ -2772,18 +2771,278 @@ function matrixToReportGroups(results) {
|
|
|
2772
2771
|
function stripQuotes(s) {
|
|
2773
2772
|
return s.replace(/^(['"])(.*)\1$/s, "$2").replace(/^(-[^=]+=)(['"])(.*)\2$/s, "$1$3");
|
|
2774
2773
|
}
|
|
2774
|
+
/** Wrap browser profile result as ReportGroup[] for the standard pipeline */
|
|
2775
|
+
function browserResultGroups(name, result) {
|
|
2776
|
+
const { gcStats, heapProfile } = result;
|
|
2777
|
+
let measured;
|
|
2778
|
+
if (result.samples && result.samples.length > 0) {
|
|
2779
|
+
const { samples } = result;
|
|
2780
|
+
const totalTime = result.wallTimeMs ? result.wallTimeMs / 1e3 : void 0;
|
|
2781
|
+
measured = {
|
|
2782
|
+
name,
|
|
2783
|
+
samples,
|
|
2784
|
+
time: computeStats(samples),
|
|
2785
|
+
totalTime,
|
|
2786
|
+
gcStats,
|
|
2787
|
+
heapProfile
|
|
2788
|
+
};
|
|
2789
|
+
} else {
|
|
2790
|
+
const wallMs = result.wallTimeMs ?? 0;
|
|
2791
|
+
measured = {
|
|
2792
|
+
name,
|
|
2793
|
+
samples: [wallMs],
|
|
2794
|
+
time: {
|
|
2795
|
+
min: wallMs,
|
|
2796
|
+
max: wallMs,
|
|
2797
|
+
avg: wallMs,
|
|
2798
|
+
p50: wallMs,
|
|
2799
|
+
p75: wallMs,
|
|
2800
|
+
p99: wallMs,
|
|
2801
|
+
p999: wallMs
|
|
2802
|
+
},
|
|
2803
|
+
gcStats,
|
|
2804
|
+
heapProfile
|
|
2805
|
+
};
|
|
2806
|
+
}
|
|
2807
|
+
return [{
|
|
2808
|
+
name,
|
|
2809
|
+
reports: [{
|
|
2810
|
+
name,
|
|
2811
|
+
measuredResults: measured
|
|
2812
|
+
}]
|
|
2813
|
+
}];
|
|
2814
|
+
}
|
|
2815
|
+
/** Print browser benchmark tables and heap reports */
|
|
2816
|
+
function printBrowserReport(result, results, args) {
|
|
2817
|
+
const hasSamples = result.samples && result.samples.length > 0;
|
|
2818
|
+
const sections = [];
|
|
2819
|
+
if (hasSamples || result.wallTimeMs != null) sections.push(timeSection);
|
|
2820
|
+
if (result.gcStats) sections.push(browserGcStatsSection);
|
|
2821
|
+
if (hasSamples || result.wallTimeMs != null) sections.push(runsSection);
|
|
2822
|
+
if (sections.length > 0) console.log(reportResults(results, sections));
|
|
2823
|
+
if (result.heapProfile) printHeapReports(results, {
|
|
2824
|
+
...cliHeapReportOptions(args),
|
|
2825
|
+
isUserCode: isBrowserUserCode
|
|
2826
|
+
});
|
|
2827
|
+
}
|
|
2828
|
+
/** Import a file and run it as a benchmark based on what it exports */
|
|
2829
|
+
async function fileBenchExports(filePath, args) {
|
|
2830
|
+
const candidate = (await import(pathToFileURL(resolve(filePath)).href)).default;
|
|
2831
|
+
if (candidate && Array.isArray(candidate.matrices)) await matrixBenchExports(candidate, args);
|
|
2832
|
+
else if (candidate && Array.isArray(candidate.groups)) await benchExports(candidate, args);
|
|
2833
|
+
else if (typeof candidate === "function") {
|
|
2834
|
+
const name = basename(filePath).replace(/\.[^.]+$/, "");
|
|
2835
|
+
await benchExports({
|
|
2836
|
+
name,
|
|
2837
|
+
groups: [{
|
|
2838
|
+
name,
|
|
2839
|
+
benchmarks: [{
|
|
2840
|
+
name,
|
|
2841
|
+
fn: candidate
|
|
2842
|
+
}]
|
|
2843
|
+
}]
|
|
2844
|
+
}, args);
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
/** Create options for adaptive mode */
|
|
2848
|
+
function createAdaptiveOptions(args) {
|
|
2849
|
+
return {
|
|
2850
|
+
minTime: (args["min-time"] ?? 1) * 1e3,
|
|
2851
|
+
maxTime: defaultAdaptiveMaxTime * 1e3,
|
|
2852
|
+
targetConfidence: args.convergence,
|
|
2853
|
+
adaptive: true,
|
|
2854
|
+
...cliCommonOptions(args)
|
|
2855
|
+
};
|
|
2856
|
+
}
|
|
2857
|
+
/** Runner/matrix options shared across all CLI modes */
|
|
2858
|
+
function cliCommonOptions(args) {
|
|
2859
|
+
const { collect, cpu, warmup } = args;
|
|
2860
|
+
const { "trace-opt": traceOpt, "skip-settle": noSettle } = args;
|
|
2861
|
+
const { "pause-first": pauseFirst, "pause-interval": pauseInterval } = args;
|
|
2862
|
+
const { "pause-duration": pauseDuration, "gc-stats": gcStats } = args;
|
|
2863
|
+
const heapSample = needsHeapSample(args);
|
|
2864
|
+
const { "heap-interval": heapInterval } = args;
|
|
2865
|
+
const { "heap-depth": heapDepth } = args;
|
|
2866
|
+
return {
|
|
2867
|
+
collect,
|
|
2868
|
+
cpuCounters: cpu,
|
|
2869
|
+
warmup,
|
|
2870
|
+
traceOpt,
|
|
2871
|
+
noSettle,
|
|
2872
|
+
pauseFirst,
|
|
2873
|
+
pauseInterval,
|
|
2874
|
+
pauseDuration,
|
|
2875
|
+
gcStats,
|
|
2876
|
+
heapSample,
|
|
2877
|
+
heapInterval,
|
|
2878
|
+
heapDepth
|
|
2879
|
+
};
|
|
2880
|
+
}
|
|
2881
|
+
/** Wait for Ctrl+C before exiting */
|
|
2882
|
+
function waitForCtrlC() {
|
|
2883
|
+
return new Promise((resolve) => {
|
|
2884
|
+
console.log(dim("\nPress Ctrl+C to exit"));
|
|
2885
|
+
process.on("SIGINT", () => {
|
|
2886
|
+
console.log();
|
|
2887
|
+
resolve();
|
|
2888
|
+
});
|
|
2889
|
+
});
|
|
2890
|
+
}
|
|
2891
|
+
/** Apply default sections and extra columns for matrix reports */
|
|
2892
|
+
function mergeMatrixDefaults(reportOptions, args, results) {
|
|
2893
|
+
const result = { ...reportOptions };
|
|
2894
|
+
if (!result.sections?.length) {
|
|
2895
|
+
const groups = matrixToReportGroups(results);
|
|
2896
|
+
result.sections = buildReportSections(args.adaptive, args["gc-stats"], hasField(groups, "cpu"), args["trace-opt"] && hasField(groups, "optStatus"));
|
|
2897
|
+
}
|
|
2898
|
+
return result;
|
|
2899
|
+
}
|
|
2900
|
+
/** Execute group with shared setup, optionally batching to reduce ordering bias */
|
|
2901
|
+
async function runGroup(group, runner, options, useWorker, batches = 1) {
|
|
2902
|
+
const { name, benchmarks, baseline, setup, metadata } = group;
|
|
2903
|
+
const setupParams = await setup?.();
|
|
2904
|
+
validateBenchmarkParameters(group);
|
|
2905
|
+
const runParams = {
|
|
2906
|
+
runner,
|
|
2907
|
+
options,
|
|
2908
|
+
useWorker,
|
|
2909
|
+
params: setupParams,
|
|
2910
|
+
metadata
|
|
2911
|
+
};
|
|
2912
|
+
if (batches === 1) return runSingleBatch(name, benchmarks, baseline, runParams);
|
|
2913
|
+
return runMultipleBatches(name, benchmarks, baseline, runParams, batches);
|
|
2914
|
+
}
|
|
2915
|
+
/** @return HeapReportOptions from CLI args */
|
|
2916
|
+
function cliHeapReportOptions(args) {
|
|
2917
|
+
return {
|
|
2918
|
+
topN: args["heap-rows"],
|
|
2919
|
+
stackDepth: args["heap-stack"],
|
|
2920
|
+
verbose: args["heap-verbose"],
|
|
2921
|
+
raw: args["heap-raw"],
|
|
2922
|
+
userOnly: args["heap-user-only"]
|
|
2923
|
+
};
|
|
2924
|
+
}
|
|
2925
|
+
/** Warn if parameterized benchmarks lack setup */
|
|
2926
|
+
function validateBenchmarkParameters(group) {
|
|
2927
|
+
const { name, setup, benchmarks, baseline } = group;
|
|
2928
|
+
if (setup) return;
|
|
2929
|
+
const allBenchmarks = baseline ? [...benchmarks, baseline] : benchmarks;
|
|
2930
|
+
for (const benchmark of allBenchmarks) if (benchmark.fn.length > 0) console.warn(`Benchmark "${benchmark.name}" in group "${name}" expects parameters but no setup() provided.`);
|
|
2931
|
+
}
|
|
2932
|
+
/** Run benchmarks in a single batch */
|
|
2933
|
+
async function runSingleBatch(name, benchmarks, baseline, runParams) {
|
|
2934
|
+
const baselineReport = baseline ? await runSingleBenchmark(baseline, runParams) : void 0;
|
|
2935
|
+
return {
|
|
2936
|
+
name,
|
|
2937
|
+
reports: await serialMap(benchmarks, (b) => runSingleBenchmark(b, runParams)),
|
|
2938
|
+
baseline: baselineReport
|
|
2939
|
+
};
|
|
2940
|
+
}
|
|
2941
|
+
/** Run benchmarks in multiple batches, alternating order to reduce bias */
|
|
2942
|
+
async function runMultipleBatches(name, benchmarks, baseline, runParams, batches) {
|
|
2943
|
+
const timePerBatch = (runParams.options.maxTime || 5e3) / batches;
|
|
2944
|
+
const batchParams = {
|
|
2945
|
+
...runParams,
|
|
2946
|
+
options: {
|
|
2947
|
+
...runParams.options,
|
|
2948
|
+
maxTime: timePerBatch
|
|
2949
|
+
}
|
|
2950
|
+
};
|
|
2951
|
+
const baselineBatches = [];
|
|
2952
|
+
const benchmarkBatches = /* @__PURE__ */ new Map();
|
|
2953
|
+
for (let i = 0; i < batches; i++) await runBatchIteration(benchmarks, baseline, batchParams, i % 2 === 1, baselineBatches, benchmarkBatches);
|
|
2954
|
+
const meta = runParams.metadata;
|
|
2955
|
+
return mergeBatchResults(name, benchmarks, baseline, baselineBatches, benchmarkBatches, meta);
|
|
2956
|
+
}
|
|
2957
|
+
/** Run single benchmark and create report */
|
|
2958
|
+
async function runSingleBenchmark(spec, runParams) {
|
|
2959
|
+
const { runner, options, useWorker, params, metadata } = runParams;
|
|
2960
|
+
const [result] = await runBenchmark({
|
|
2961
|
+
spec,
|
|
2962
|
+
runner,
|
|
2963
|
+
options,
|
|
2964
|
+
useWorker,
|
|
2965
|
+
params
|
|
2966
|
+
});
|
|
2967
|
+
return {
|
|
2968
|
+
name: spec.name,
|
|
2969
|
+
measuredResults: result,
|
|
2970
|
+
metadata
|
|
2971
|
+
};
|
|
2972
|
+
}
|
|
2775
2973
|
/** Sequential map - like Promise.all(arr.map(fn)) but runs one at a time */
|
|
2776
2974
|
async function serialMap(arr, fn) {
|
|
2777
2975
|
const results = [];
|
|
2778
2976
|
for (const item of arr) results.push(await fn(item));
|
|
2779
2977
|
return results;
|
|
2780
2978
|
}
|
|
2781
|
-
/** Run
|
|
2782
|
-
async function
|
|
2783
|
-
const
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2979
|
+
/** Run one batch iteration in either order */
|
|
2980
|
+
async function runBatchIteration(benchmarks, baseline, runParams, reverseOrder, baselineBatches, benchmarkBatches) {
|
|
2981
|
+
const runBaseline = async () => {
|
|
2982
|
+
if (baseline) {
|
|
2983
|
+
const r = await runSingleBenchmark(baseline, runParams);
|
|
2984
|
+
baselineBatches.push(r.measuredResults);
|
|
2985
|
+
}
|
|
2986
|
+
};
|
|
2987
|
+
const runBenches = async () => {
|
|
2988
|
+
for (const b of benchmarks) {
|
|
2989
|
+
const r = await runSingleBenchmark(b, runParams);
|
|
2990
|
+
appendToMap(benchmarkBatches, b.name, r.measuredResults);
|
|
2991
|
+
}
|
|
2992
|
+
};
|
|
2993
|
+
if (reverseOrder) {
|
|
2994
|
+
await runBenches();
|
|
2995
|
+
await runBaseline();
|
|
2996
|
+
} else {
|
|
2997
|
+
await runBaseline();
|
|
2998
|
+
await runBenches();
|
|
2999
|
+
}
|
|
3000
|
+
}
|
|
3001
|
+
/** Merge batch results into final ReportGroup */
|
|
3002
|
+
function mergeBatchResults(name, benchmarks, baseline, baselineBatches, benchmarkBatches, metadata) {
|
|
3003
|
+
const mergedBaseline = baseline ? {
|
|
3004
|
+
name: baseline.name,
|
|
3005
|
+
measuredResults: mergeResults(baselineBatches),
|
|
3006
|
+
metadata
|
|
3007
|
+
} : void 0;
|
|
3008
|
+
return {
|
|
3009
|
+
name,
|
|
3010
|
+
reports: benchmarks.map((b) => ({
|
|
3011
|
+
name: b.name,
|
|
3012
|
+
measuredResults: mergeResults(benchmarkBatches.get(b.name) || []),
|
|
3013
|
+
metadata
|
|
3014
|
+
})),
|
|
3015
|
+
baseline: mergedBaseline
|
|
3016
|
+
};
|
|
3017
|
+
}
|
|
3018
|
+
function appendToMap(map, key, value) {
|
|
3019
|
+
if (!map.has(key)) map.set(key, []);
|
|
3020
|
+
map.get(key).push(value);
|
|
3021
|
+
}
|
|
3022
|
+
/** Merge multiple batch results into a single MeasuredResults */
|
|
3023
|
+
function mergeResults(results) {
|
|
3024
|
+
if (results.length === 0) throw new Error("Cannot merge empty results array");
|
|
3025
|
+
if (results.length === 1) return results[0];
|
|
3026
|
+
const allSamples = results.flatMap((r) => r.samples);
|
|
3027
|
+
const allWarmup = results.flatMap((r) => r.warmupSamples || []);
|
|
3028
|
+
const time = computeStats(allSamples);
|
|
3029
|
+
let offset = 0;
|
|
3030
|
+
const allPausePoints = results.flatMap((r) => {
|
|
3031
|
+
const pts = (r.pausePoints ?? []).map((p) => ({
|
|
3032
|
+
sampleIndex: p.sampleIndex + offset,
|
|
3033
|
+
durationMs: p.durationMs
|
|
3034
|
+
}));
|
|
3035
|
+
offset += r.samples.length;
|
|
3036
|
+
return pts;
|
|
3037
|
+
});
|
|
3038
|
+
return {
|
|
3039
|
+
name: results[0].name,
|
|
3040
|
+
samples: allSamples,
|
|
3041
|
+
warmupSamples: allWarmup.length ? allWarmup : void 0,
|
|
3042
|
+
time,
|
|
3043
|
+
totalTime: results.reduce((sum, r) => sum + (r.totalTime || 0), 0),
|
|
3044
|
+
pausePoints: allPausePoints.length ? allPausePoints : void 0
|
|
3045
|
+
};
|
|
2787
3046
|
}
|
|
2788
3047
|
|
|
2789
3048
|
//#endregion
|
|
@@ -2845,5 +3104,5 @@ function getMostRecentModifiedDate(dir) {
|
|
|
2845
3104
|
}
|
|
2846
3105
|
|
|
2847
3106
|
//#endregion
|
|
2848
|
-
export { timeSection as A,
|
|
2849
|
-
//# sourceMappingURL=src-
|
|
3107
|
+
export { loadCasesModule as $, timeSection as A, heapProfileToSpeedscope as B, adaptiveSection as C, gcStatsSection as D, gcSection as E, generateHtmlReport as F, reportResults as G, exportPerfettoTrace as H, formatDateWithTimezone as I, timeMs as J, formatBytes as K, prepareHtmlData as L, formatConvergence as M, filterMatrix as N, optSection as O, parseMatrixFilter as P, loadCaseData as Q, exportAndLaunchSpeedscope as R, reportMatrixResults as S, cpuSection as T, defaultCliArgs as U, launchSpeedscope as V, parseCliArgs as W, isStatefulVariant as X, truncate as Y, runMatrix as Z, runDefaultMatrixBench as _, cliToMatrixOptions as a, gcStatsColumns as b, exportReports as c, matrixToReportGroups as d, parseBenchArgs as f, runDefaultBench as g, runBenchmarks as h, benchExports as i, totalTimeSection as j, runsSection as k, hasField as l, reportOptStatus as m, getBaselineVersion as n, defaultMatrixReport as o, printHeapReports as p, integer as q, getCurrentGitVersion as r, defaultReport as s, formatGitVersion as t, matrixBenchExports as u, runMatrixSuite as v, buildGenericSections as w, heapTotalColumn as x, gcPauseColumn as y, exportSpeedscope as z };
|
|
3108
|
+
//# sourceMappingURL=src-B-DDaCa9.mjs.map
|