benchforge 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -129
- package/dist/{BenchRunner-BLfGX2wQ.d.mts → BenchRunner-CSKN9zPy.d.mts} +1 -1
- package/dist/BrowserHeapSampler-DQwmmuDu.mjs +187 -0
- package/dist/BrowserHeapSampler-DQwmmuDu.mjs.map +1 -0
- package/dist/GcStats-ByEovUi1.mjs +77 -0
- package/dist/GcStats-ByEovUi1.mjs.map +1 -0
- package/dist/{HeapSampler-BX3de22o.mjs → HeapSampler-B8dtKHn1.mjs} +1 -1
- package/dist/{HeapSampler-BX3de22o.mjs.map → HeapSampler-B8dtKHn1.mjs.map} +1 -1
- package/dist/{TimingUtils-D4z1jpp2.mjs → TimingUtils-ClclVQ7E.mjs} +276 -278
- package/dist/TimingUtils-ClclVQ7E.mjs.map +1 -0
- package/dist/bin/benchforge.mjs +1 -1
- package/dist/index.d.mts +10 -6
- package/dist/index.mjs +2 -2
- package/dist/runners/WorkerScript.d.mts +1 -1
- package/dist/runners/WorkerScript.mjs +2 -2
- package/dist/{src-D7zxOFGA.mjs → src-B06_i1RD.mjs} +21 -270
- package/dist/src-B06_i1RD.mjs.map +1 -0
- package/package.json +10 -2
- package/src/StandardSections.ts +1 -8
- package/src/browser/BrowserHeapSampler.ts +3 -2
- package/src/cli/CliArgs.ts +4 -3
- package/src/cli/RunBenchCLI.ts +16 -8
- package/src/runners/BasicRunner.ts +0 -4
- package/dist/TimingUtils-D4z1jpp2.mjs.map +0 -1
- package/dist/src-D7zxOFGA.mjs.map +0 -1
|
@@ -13,6 +13,261 @@ function variantModuleUrl(dirUrl, variantId) {
|
|
|
13
13
|
return new URL(`${variantId}.ts`, dirUrl).href;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
+
//#endregion
|
|
17
|
+
//#region src/runners/BenchRunner.ts
|
|
18
|
+
/** Execute benchmark with optional parameters */
|
|
19
|
+
function executeBenchmark(benchmark, params) {
|
|
20
|
+
benchmark.fn(params);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
//#endregion
|
|
24
|
+
//#region src/runners/BasicRunner.ts
|
|
25
|
+
/**
|
|
26
|
+
* Wait time after gc() for V8 to stabilize (ms).
|
|
27
|
+
*
|
|
28
|
+
* V8 has 4 compilation tiers: Ignition (interpreter) -> Sparkplug (baseline) ->
|
|
29
|
+
* Maglev (mid-tier optimizer) -> TurboFan (full optimizer). Tiering thresholds:
|
|
30
|
+
* - Ignition -> Sparkplug: 8 invocations
|
|
31
|
+
* - Sparkplug -> Maglev: 500 invocations
|
|
32
|
+
* - Maglev -> TurboFan: 6000 invocations
|
|
33
|
+
*
|
|
34
|
+
* Optimization compilation happens on background threads and requires idle time
|
|
35
|
+
* on the main thread to complete. Without sufficient warmup + settle time,
|
|
36
|
+
* benchmarks exhibit bimodal timing: slow Sparkplug samples (~30% slower) mixed
|
|
37
|
+
* with fast optimized samples.
|
|
38
|
+
*
|
|
39
|
+
* The warmup iterations trigger the optimization decision, then gcSettleTime
|
|
40
|
+
* provides idle time for background compilation to finish before measurement.
|
|
41
|
+
*
|
|
42
|
+
* @see https://v8.dev/blog/sparkplug
|
|
43
|
+
* @see https://v8.dev/blog/maglev
|
|
44
|
+
* @see https://v8.dev/blog/background-compilation
|
|
45
|
+
*/
|
|
46
|
+
const gcSettleTime = 1e3;
|
|
47
|
+
/** @return runner with time and iteration limits */
|
|
48
|
+
var BasicRunner = class {
|
|
49
|
+
async runBench(benchmark, options, params) {
|
|
50
|
+
const collected = await collectSamples({
|
|
51
|
+
benchmark,
|
|
52
|
+
params,
|
|
53
|
+
...defaultCollectOptions,
|
|
54
|
+
...options
|
|
55
|
+
});
|
|
56
|
+
return [buildMeasuredResults(benchmark.name, collected)];
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
const defaultCollectOptions = {
|
|
60
|
+
maxTime: 5e3,
|
|
61
|
+
maxIterations: 1e6,
|
|
62
|
+
warmup: 0,
|
|
63
|
+
traceOpt: false,
|
|
64
|
+
noSettle: false
|
|
65
|
+
};
|
|
66
|
+
function buildMeasuredResults(name, c) {
|
|
67
|
+
const time = computeStats(c.samples);
|
|
68
|
+
return {
|
|
69
|
+
name,
|
|
70
|
+
samples: c.samples,
|
|
71
|
+
warmupSamples: c.warmupSamples,
|
|
72
|
+
heapSamples: c.heapSamples,
|
|
73
|
+
timestamps: c.timestamps,
|
|
74
|
+
time,
|
|
75
|
+
heapSize: {
|
|
76
|
+
avg: c.heapGrowth,
|
|
77
|
+
min: c.heapGrowth,
|
|
78
|
+
max: c.heapGrowth
|
|
79
|
+
},
|
|
80
|
+
optStatus: c.optStatus,
|
|
81
|
+
optSamples: c.optSamples,
|
|
82
|
+
pausePoints: c.pausePoints
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
/** @return timing samples and amortized allocation from benchmark execution */
|
|
86
|
+
async function collectSamples(p) {
|
|
87
|
+
if (!p.maxIterations && !p.maxTime) throw new Error(`At least one of maxIterations or maxTime must be set`);
|
|
88
|
+
const warmupSamples = p.skipWarmup ? [] : await runWarmup(p);
|
|
89
|
+
const heapBefore = process.memoryUsage().heapUsed;
|
|
90
|
+
const { samples, heapSamples, timestamps, optStatuses, pausePoints } = await runSampleLoop(p);
|
|
91
|
+
const heapGrowth = Math.max(0, process.memoryUsage().heapUsed - heapBefore) / 1024 / samples.length;
|
|
92
|
+
if (samples.length === 0) throw new Error(`No samples collected for benchmark: ${p.benchmark.name}`);
|
|
93
|
+
return {
|
|
94
|
+
samples,
|
|
95
|
+
warmupSamples,
|
|
96
|
+
heapGrowth,
|
|
97
|
+
heapSamples,
|
|
98
|
+
timestamps,
|
|
99
|
+
optStatus: p.traceOpt ? analyzeOptStatus(samples, optStatuses) : void 0,
|
|
100
|
+
optSamples: p.traceOpt && optStatuses.length > 0 ? optStatuses : void 0,
|
|
101
|
+
pausePoints
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/** Run warmup iterations with gc + settle time for V8 optimization */
|
|
105
|
+
async function runWarmup(p) {
|
|
106
|
+
const gc = gcFunction();
|
|
107
|
+
const samples = new Array(p.warmup);
|
|
108
|
+
for (let i = 0; i < p.warmup; i++) {
|
|
109
|
+
const start = performance.now();
|
|
110
|
+
executeBenchmark(p.benchmark, p.params);
|
|
111
|
+
samples[i] = performance.now() - start;
|
|
112
|
+
}
|
|
113
|
+
gc();
|
|
114
|
+
if (!p.noSettle) {
|
|
115
|
+
await new Promise((r) => setTimeout(r, gcSettleTime));
|
|
116
|
+
gc();
|
|
117
|
+
}
|
|
118
|
+
return samples;
|
|
119
|
+
}
|
|
120
|
+
/** Estimate sample count for pre-allocation */
|
|
121
|
+
function estimateSampleCount(maxTime, maxIterations) {
|
|
122
|
+
return maxIterations || Math.ceil(maxTime / .1);
|
|
123
|
+
}
|
|
124
|
+
/** Pre-allocate arrays to reduce GC pressure during measurement */
|
|
125
|
+
function createSampleArrays(n, trackHeap, trackOpt) {
|
|
126
|
+
const arr = (track) => track ? new Array(n) : [];
|
|
127
|
+
return {
|
|
128
|
+
samples: new Array(n),
|
|
129
|
+
timestamps: new Array(n),
|
|
130
|
+
heapSamples: arr(trackHeap),
|
|
131
|
+
optStatuses: arr(trackOpt),
|
|
132
|
+
pausePoints: []
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
/** Trim arrays to actual sample count */
|
|
136
|
+
function trimArrays(a, count, trackHeap, trackOpt) {
|
|
137
|
+
a.samples.length = a.timestamps.length = count;
|
|
138
|
+
if (trackHeap) a.heapSamples.length = count;
|
|
139
|
+
if (trackOpt) a.optStatuses.length = count;
|
|
140
|
+
}
|
|
141
|
+
/** Collect timing samples with periodic pauses for V8 optimization */
|
|
142
|
+
async function runSampleLoop(p) {
|
|
143
|
+
const { maxTime, maxIterations, pauseFirst, pauseInterval = 0, pauseDuration = 100 } = p;
|
|
144
|
+
const trackHeap = true;
|
|
145
|
+
const getOptStatus = p.traceOpt ? createOptStatusGetter() : void 0;
|
|
146
|
+
const a = createSampleArrays(estimateSampleCount(maxTime, maxIterations), trackHeap, !!getOptStatus);
|
|
147
|
+
let count = 0;
|
|
148
|
+
let elapsed = 0;
|
|
149
|
+
let totalPauseTime = 0;
|
|
150
|
+
const loopStart = performance.now();
|
|
151
|
+
while ((!maxIterations || count < maxIterations) && (!maxTime || elapsed < maxTime)) {
|
|
152
|
+
const start = performance.now();
|
|
153
|
+
executeBenchmark(p.benchmark, p.params);
|
|
154
|
+
const end = performance.now();
|
|
155
|
+
a.samples[count] = end - start;
|
|
156
|
+
a.timestamps[count] = Number(process.hrtime.bigint() / 1000n);
|
|
157
|
+
a.heapSamples[count] = getHeapStatistics().used_heap_size;
|
|
158
|
+
if (getOptStatus) a.optStatuses[count] = getOptStatus(p.benchmark.fn);
|
|
159
|
+
count++;
|
|
160
|
+
if (shouldPause(count, pauseFirst, pauseInterval)) {
|
|
161
|
+
a.pausePoints.push({
|
|
162
|
+
sampleIndex: count - 1,
|
|
163
|
+
durationMs: pauseDuration
|
|
164
|
+
});
|
|
165
|
+
const pauseStart = performance.now();
|
|
166
|
+
await new Promise((r) => setTimeout(r, pauseDuration));
|
|
167
|
+
totalPauseTime += performance.now() - pauseStart;
|
|
168
|
+
}
|
|
169
|
+
elapsed = performance.now() - loopStart - totalPauseTime;
|
|
170
|
+
}
|
|
171
|
+
trimArrays(a, count, trackHeap, !!getOptStatus);
|
|
172
|
+
return {
|
|
173
|
+
samples: a.samples,
|
|
174
|
+
heapSamples: a.heapSamples,
|
|
175
|
+
timestamps: a.timestamps,
|
|
176
|
+
optStatuses: a.optStatuses,
|
|
177
|
+
pausePoints: a.pausePoints
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
/** Check if we should pause at this iteration for V8 optimization */
|
|
181
|
+
function shouldPause(iter, first, interval) {
|
|
182
|
+
if (first !== void 0 && iter === first) return true;
|
|
183
|
+
if (interval <= 0) return false;
|
|
184
|
+
if (first === void 0) return iter % interval === 0;
|
|
185
|
+
return (iter - first) % interval === 0;
|
|
186
|
+
}
|
|
187
|
+
/** @return percentiles and basic statistics */
|
|
188
|
+
function computeStats(samples) {
|
|
189
|
+
const sorted = [...samples].sort((a, b) => a - b);
|
|
190
|
+
const avg = samples.reduce((sum, s) => sum + s, 0) / samples.length;
|
|
191
|
+
return {
|
|
192
|
+
min: sorted[0],
|
|
193
|
+
max: sorted[sorted.length - 1],
|
|
194
|
+
avg,
|
|
195
|
+
p50: percentile$1(sorted, .5),
|
|
196
|
+
p75: percentile$1(sorted, .75),
|
|
197
|
+
p99: percentile$1(sorted, .99),
|
|
198
|
+
p999: percentile$1(sorted, .999)
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
/** @return percentile value with linear interpolation */
|
|
202
|
+
function percentile$1(sortedArray, p) {
|
|
203
|
+
const index = (sortedArray.length - 1) * p;
|
|
204
|
+
const lower = Math.floor(index);
|
|
205
|
+
const upper = Math.ceil(index);
|
|
206
|
+
const weight = index % 1;
|
|
207
|
+
if (upper >= sortedArray.length) return sortedArray[sortedArray.length - 1];
|
|
208
|
+
return sortedArray[lower] * (1 - weight) + sortedArray[upper] * weight;
|
|
209
|
+
}
|
|
210
|
+
/** @return runtime gc() function, or no-op if unavailable */
|
|
211
|
+
function gcFunction() {
|
|
212
|
+
const gc = globalThis.gc || globalThis.__gc;
|
|
213
|
+
if (gc) return gc;
|
|
214
|
+
console.warn("gc() not available, run node/bun with --expose-gc");
|
|
215
|
+
return () => {};
|
|
216
|
+
}
|
|
217
|
+
/** @return function to get V8 optimization status (requires --allow-natives-syntax) */
|
|
218
|
+
function createOptStatusGetter() {
|
|
219
|
+
try {
|
|
220
|
+
const getter = new Function("f", "return %GetOptimizationStatus(f)");
|
|
221
|
+
getter(() => {});
|
|
222
|
+
return getter;
|
|
223
|
+
} catch {
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* V8 optimization status bit meanings:
|
|
229
|
+
* Bit 0 (1): is_function
|
|
230
|
+
* Bit 4 (16): is_optimized (TurboFan)
|
|
231
|
+
* Bit 5 (32): is_optimized (Maglev)
|
|
232
|
+
* Bit 7 (128): is_baseline (Sparkplug)
|
|
233
|
+
* Bit 3 (8): maybe_deoptimized
|
|
234
|
+
*/
|
|
235
|
+
const statusNames = {
|
|
236
|
+
1: "interpreted",
|
|
237
|
+
129: "sparkplug",
|
|
238
|
+
17: "turbofan",
|
|
239
|
+
33: "maglev",
|
|
240
|
+
49: "turbofan+maglev",
|
|
241
|
+
32769: "optimized"
|
|
242
|
+
};
|
|
243
|
+
/** @return analysis of V8 optimization status per sample */
|
|
244
|
+
function analyzeOptStatus(samples, statuses) {
|
|
245
|
+
if (statuses.length === 0 || statuses[0] === void 0) return void 0;
|
|
246
|
+
const byStatusCode = /* @__PURE__ */ new Map();
|
|
247
|
+
let deoptCount = 0;
|
|
248
|
+
for (let i = 0; i < samples.length; i++) {
|
|
249
|
+
const status = statuses[i];
|
|
250
|
+
if (status === void 0) continue;
|
|
251
|
+
if (status & 8) deoptCount++;
|
|
252
|
+
if (!byStatusCode.has(status)) byStatusCode.set(status, []);
|
|
253
|
+
byStatusCode.get(status).push(samples[i]);
|
|
254
|
+
}
|
|
255
|
+
const byTier = {};
|
|
256
|
+
for (const [status, times] of byStatusCode) {
|
|
257
|
+
const name = statusNames[status] || `status=${status}`;
|
|
258
|
+
const sorted = [...times].sort((a, b) => a - b);
|
|
259
|
+
const median = sorted[Math.floor(sorted.length / 2)];
|
|
260
|
+
byTier[name] = {
|
|
261
|
+
count: times.length,
|
|
262
|
+
medianMs: median
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
return {
|
|
266
|
+
byTier,
|
|
267
|
+
deoptCount
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
|
|
16
271
|
//#endregion
|
|
17
272
|
//#region src/StatisticalUtils.ts
|
|
18
273
|
const bootstrapSamples = 1e4;
|
|
@@ -25,8 +280,8 @@ function coefficientOfVariation(samples) {
|
|
|
25
280
|
}
|
|
26
281
|
/** @return median absolute deviation for robust variability measure */
|
|
27
282
|
function medianAbsoluteDeviation(samples) {
|
|
28
|
-
const median = percentile
|
|
29
|
-
return percentile
|
|
283
|
+
const median = percentile(samples, .5);
|
|
284
|
+
return percentile(samples.map((x) => Math.abs(x - median)), .5);
|
|
30
285
|
}
|
|
31
286
|
/** @return mean of values */
|
|
32
287
|
function average(values) {
|
|
@@ -40,7 +295,7 @@ function standardDeviation(samples) {
|
|
|
40
295
|
return Math.sqrt(variance);
|
|
41
296
|
}
|
|
42
297
|
/** @return value at percentile p (0-1) */
|
|
43
|
-
function percentile
|
|
298
|
+
function percentile(values, p) {
|
|
44
299
|
const sorted = [...values].sort((a, b) => a - b);
|
|
45
300
|
const index = Math.ceil(sorted.length * p) - 1;
|
|
46
301
|
return sorted[Math.max(0, index)];
|
|
@@ -54,7 +309,7 @@ function createResample(samples) {
|
|
|
54
309
|
/** @return confidence interval [lower, upper] */
|
|
55
310
|
function computeInterval(medians, confidence) {
|
|
56
311
|
const alpha = (1 - confidence) / 2;
|
|
57
|
-
return [percentile
|
|
312
|
+
return [percentile(medians, alpha), percentile(medians, 1 - alpha)];
|
|
58
313
|
}
|
|
59
314
|
/** Bin values into histogram for compact visualization */
|
|
60
315
|
function binValues(values, binCount = 30) {
|
|
@@ -79,14 +334,14 @@ function binValues(values, binCount = 30) {
|
|
|
79
334
|
/** @return bootstrap CI for percentage difference between baseline and current medians */
|
|
80
335
|
function bootstrapDifferenceCI(baseline, current, options = {}) {
|
|
81
336
|
const { resamples = bootstrapSamples, confidence: conf = confidence } = options;
|
|
82
|
-
const baselineMedian = percentile
|
|
83
|
-
const observedPercent = (percentile
|
|
337
|
+
const baselineMedian = percentile(baseline, .5);
|
|
338
|
+
const observedPercent = (percentile(current, .5) - baselineMedian) / baselineMedian * 100;
|
|
84
339
|
const diffs = [];
|
|
85
340
|
for (let i = 0; i < resamples; i++) {
|
|
86
341
|
const resB = createResample(baseline);
|
|
87
342
|
const resC = createResample(current);
|
|
88
|
-
const medB = percentile
|
|
89
|
-
const medC = percentile
|
|
343
|
+
const medB = percentile(resB, .5);
|
|
344
|
+
const medC = percentile(resC, .5);
|
|
90
345
|
diffs.push((medC - medB) / medB * 100);
|
|
91
346
|
}
|
|
92
347
|
const ci = computeInterval(diffs, conf);
|
|
@@ -220,12 +475,12 @@ function getMinMaxSum(samples) {
|
|
|
220
475
|
/** @return percentiles in ms */
|
|
221
476
|
function getPercentiles(samples) {
|
|
222
477
|
return {
|
|
223
|
-
p25: percentile
|
|
224
|
-
p50: percentile
|
|
225
|
-
p75: percentile
|
|
226
|
-
p95: percentile
|
|
227
|
-
p99: percentile
|
|
228
|
-
p999: percentile
|
|
478
|
+
p25: percentile(samples, .25) / msToNs,
|
|
479
|
+
p50: percentile(samples, .5) / msToNs,
|
|
480
|
+
p75: percentile(samples, .75) / msToNs,
|
|
481
|
+
p95: percentile(samples, .95) / msToNs,
|
|
482
|
+
p99: percentile(samples, .99) / msToNs,
|
|
483
|
+
p999: percentile(samples, .999) / msToNs
|
|
229
484
|
};
|
|
230
485
|
}
|
|
231
486
|
/** @return robust variability metrics */
|
|
@@ -243,8 +498,8 @@ function getOutlierImpact(samples) {
|
|
|
243
498
|
ratio: 0,
|
|
244
499
|
count: 0
|
|
245
500
|
};
|
|
246
|
-
const median = percentile
|
|
247
|
-
const threshold = median + 1.5 * (percentile
|
|
501
|
+
const median = percentile(samples, .5);
|
|
502
|
+
const threshold = median + 1.5 * (percentile(samples, .75) - median);
|
|
248
503
|
let excessTime = 0;
|
|
249
504
|
let count = 0;
|
|
250
505
|
for (const sample of samples) if (sample > threshold) {
|
|
@@ -278,8 +533,8 @@ function getStability(samples, windowSize) {
|
|
|
278
533
|
const previous = samples.slice(-windowSize * 2, -windowSize);
|
|
279
534
|
const recentMs = recent.map((s) => s / msToNs);
|
|
280
535
|
const previousMs = previous.map((s) => s / msToNs);
|
|
281
|
-
const medianRecent = percentile
|
|
282
|
-
const medianPrevious = percentile
|
|
536
|
+
const medianRecent = percentile(recentMs, .5);
|
|
537
|
+
const medianPrevious = percentile(previousMs, .5);
|
|
283
538
|
const medianDrift = Math.abs(medianRecent - medianPrevious) / medianPrevious;
|
|
284
539
|
const impactRecent = getOutlierImpact(recentMs);
|
|
285
540
|
const impactPrevious = getOutlierImpact(previousMs);
|
|
@@ -310,7 +565,7 @@ function buildConvergence(metrics) {
|
|
|
310
565
|
/** @return window size scaled to execution time */
|
|
311
566
|
function getWindowSize(samples) {
|
|
312
567
|
if (samples.length < 20) return windowSize;
|
|
313
|
-
const recentMedian = percentile
|
|
568
|
+
const recentMedian = percentile(samples.slice(-20).map((s) => s / msToNs), .5);
|
|
314
569
|
if (recentMedian < .01) return 200;
|
|
315
570
|
if (recentMedian < .1) return 100;
|
|
316
571
|
if (recentMedian < 1) return 50;
|
|
@@ -318,263 +573,6 @@ function getWindowSize(samples) {
|
|
|
318
573
|
return 20;
|
|
319
574
|
}
|
|
320
575
|
|
|
321
|
-
//#endregion
|
|
322
|
-
//#region src/runners/BenchRunner.ts
|
|
323
|
-
/** Execute benchmark with optional parameters */
|
|
324
|
-
function executeBenchmark(benchmark, params) {
|
|
325
|
-
benchmark.fn(params);
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
//#endregion
|
|
329
|
-
//#region src/runners/BasicRunner.ts
|
|
330
|
-
/**
|
|
331
|
-
* Wait time after gc() for V8 to stabilize (ms).
|
|
332
|
-
*
|
|
333
|
-
* V8 has 4 compilation tiers: Ignition (interpreter) -> Sparkplug (baseline) ->
|
|
334
|
-
* Maglev (mid-tier optimizer) -> TurboFan (full optimizer). Tiering thresholds:
|
|
335
|
-
* - Ignition -> Sparkplug: 8 invocations
|
|
336
|
-
* - Sparkplug -> Maglev: 500 invocations
|
|
337
|
-
* - Maglev -> TurboFan: 6000 invocations
|
|
338
|
-
*
|
|
339
|
-
* Optimization compilation happens on background threads and requires idle time
|
|
340
|
-
* on the main thread to complete. Without sufficient warmup + settle time,
|
|
341
|
-
* benchmarks exhibit bimodal timing: slow Sparkplug samples (~30% slower) mixed
|
|
342
|
-
* with fast optimized samples.
|
|
343
|
-
*
|
|
344
|
-
* The warmup iterations trigger the optimization decision, then gcSettleTime
|
|
345
|
-
* provides idle time for background compilation to finish before measurement.
|
|
346
|
-
*
|
|
347
|
-
* @see https://v8.dev/blog/sparkplug
|
|
348
|
-
* @see https://v8.dev/blog/maglev
|
|
349
|
-
* @see https://v8.dev/blog/background-compilation
|
|
350
|
-
*/
|
|
351
|
-
const gcSettleTime = 1e3;
|
|
352
|
-
/** @return runner with time and iteration limits */
|
|
353
|
-
var BasicRunner = class {
|
|
354
|
-
async runBench(benchmark, options, params) {
|
|
355
|
-
const collected = await collectSamples({
|
|
356
|
-
benchmark,
|
|
357
|
-
params,
|
|
358
|
-
...defaultCollectOptions,
|
|
359
|
-
...options
|
|
360
|
-
});
|
|
361
|
-
return [buildMeasuredResults(benchmark.name, collected)];
|
|
362
|
-
}
|
|
363
|
-
};
|
|
364
|
-
const defaultCollectOptions = {
|
|
365
|
-
maxTime: 5e3,
|
|
366
|
-
maxIterations: 1e6,
|
|
367
|
-
warmup: 0,
|
|
368
|
-
traceOpt: false,
|
|
369
|
-
noSettle: false
|
|
370
|
-
};
|
|
371
|
-
function buildMeasuredResults(name, c) {
|
|
372
|
-
const time = computeStats(c.samples);
|
|
373
|
-
const convergence = checkConvergence(c.samples.map((s) => s * msToNs));
|
|
374
|
-
return {
|
|
375
|
-
name,
|
|
376
|
-
samples: c.samples,
|
|
377
|
-
warmupSamples: c.warmupSamples,
|
|
378
|
-
heapSamples: c.heapSamples,
|
|
379
|
-
timestamps: c.timestamps,
|
|
380
|
-
time,
|
|
381
|
-
heapSize: {
|
|
382
|
-
avg: c.heapGrowth,
|
|
383
|
-
min: c.heapGrowth,
|
|
384
|
-
max: c.heapGrowth
|
|
385
|
-
},
|
|
386
|
-
convergence,
|
|
387
|
-
optStatus: c.optStatus,
|
|
388
|
-
optSamples: c.optSamples,
|
|
389
|
-
pausePoints: c.pausePoints
|
|
390
|
-
};
|
|
391
|
-
}
|
|
392
|
-
/** @return timing samples and amortized allocation from benchmark execution */
|
|
393
|
-
async function collectSamples(p) {
|
|
394
|
-
if (!p.maxIterations && !p.maxTime) throw new Error(`At least one of maxIterations or maxTime must be set`);
|
|
395
|
-
const warmupSamples = p.skipWarmup ? [] : await runWarmup(p);
|
|
396
|
-
const heapBefore = process.memoryUsage().heapUsed;
|
|
397
|
-
const { samples, heapSamples, timestamps, optStatuses, pausePoints } = await runSampleLoop(p);
|
|
398
|
-
const heapGrowth = Math.max(0, process.memoryUsage().heapUsed - heapBefore) / 1024 / samples.length;
|
|
399
|
-
if (samples.length === 0) throw new Error(`No samples collected for benchmark: ${p.benchmark.name}`);
|
|
400
|
-
return {
|
|
401
|
-
samples,
|
|
402
|
-
warmupSamples,
|
|
403
|
-
heapGrowth,
|
|
404
|
-
heapSamples,
|
|
405
|
-
timestamps,
|
|
406
|
-
optStatus: p.traceOpt ? analyzeOptStatus(samples, optStatuses) : void 0,
|
|
407
|
-
optSamples: p.traceOpt && optStatuses.length > 0 ? optStatuses : void 0,
|
|
408
|
-
pausePoints
|
|
409
|
-
};
|
|
410
|
-
}
|
|
411
|
-
/** Run warmup iterations with gc + settle time for V8 optimization */
|
|
412
|
-
async function runWarmup(p) {
|
|
413
|
-
const gc = gcFunction();
|
|
414
|
-
const samples = new Array(p.warmup);
|
|
415
|
-
for (let i = 0; i < p.warmup; i++) {
|
|
416
|
-
const start = performance.now();
|
|
417
|
-
executeBenchmark(p.benchmark, p.params);
|
|
418
|
-
samples[i] = performance.now() - start;
|
|
419
|
-
}
|
|
420
|
-
gc();
|
|
421
|
-
if (!p.noSettle) {
|
|
422
|
-
await new Promise((r) => setTimeout(r, gcSettleTime));
|
|
423
|
-
gc();
|
|
424
|
-
}
|
|
425
|
-
return samples;
|
|
426
|
-
}
|
|
427
|
-
/** Estimate sample count for pre-allocation */
|
|
428
|
-
function estimateSampleCount(maxTime, maxIterations) {
|
|
429
|
-
return maxIterations || Math.ceil(maxTime / .1);
|
|
430
|
-
}
|
|
431
|
-
/** Pre-allocate arrays to reduce GC pressure during measurement */
|
|
432
|
-
function createSampleArrays(n, trackHeap, trackOpt) {
|
|
433
|
-
const arr = (track) => track ? new Array(n) : [];
|
|
434
|
-
return {
|
|
435
|
-
samples: new Array(n),
|
|
436
|
-
timestamps: new Array(n),
|
|
437
|
-
heapSamples: arr(trackHeap),
|
|
438
|
-
optStatuses: arr(trackOpt),
|
|
439
|
-
pausePoints: []
|
|
440
|
-
};
|
|
441
|
-
}
|
|
442
|
-
/** Trim arrays to actual sample count */
|
|
443
|
-
function trimArrays(a, count, trackHeap, trackOpt) {
|
|
444
|
-
a.samples.length = a.timestamps.length = count;
|
|
445
|
-
if (trackHeap) a.heapSamples.length = count;
|
|
446
|
-
if (trackOpt) a.optStatuses.length = count;
|
|
447
|
-
}
|
|
448
|
-
/** Collect timing samples with periodic pauses for V8 optimization */
|
|
449
|
-
async function runSampleLoop(p) {
|
|
450
|
-
const { maxTime, maxIterations, pauseFirst, pauseInterval = 0, pauseDuration = 100 } = p;
|
|
451
|
-
const trackHeap = true;
|
|
452
|
-
const getOptStatus = p.traceOpt ? createOptStatusGetter() : void 0;
|
|
453
|
-
const a = createSampleArrays(estimateSampleCount(maxTime, maxIterations), trackHeap, !!getOptStatus);
|
|
454
|
-
let count = 0;
|
|
455
|
-
let elapsed = 0;
|
|
456
|
-
let totalPauseTime = 0;
|
|
457
|
-
const loopStart = performance.now();
|
|
458
|
-
while ((!maxIterations || count < maxIterations) && (!maxTime || elapsed < maxTime)) {
|
|
459
|
-
const start = performance.now();
|
|
460
|
-
executeBenchmark(p.benchmark, p.params);
|
|
461
|
-
const end = performance.now();
|
|
462
|
-
a.samples[count] = end - start;
|
|
463
|
-
a.timestamps[count] = Number(process.hrtime.bigint() / 1000n);
|
|
464
|
-
a.heapSamples[count] = getHeapStatistics().used_heap_size;
|
|
465
|
-
if (getOptStatus) a.optStatuses[count] = getOptStatus(p.benchmark.fn);
|
|
466
|
-
count++;
|
|
467
|
-
if (shouldPause(count, pauseFirst, pauseInterval)) {
|
|
468
|
-
a.pausePoints.push({
|
|
469
|
-
sampleIndex: count - 1,
|
|
470
|
-
durationMs: pauseDuration
|
|
471
|
-
});
|
|
472
|
-
const pauseStart = performance.now();
|
|
473
|
-
await new Promise((r) => setTimeout(r, pauseDuration));
|
|
474
|
-
totalPauseTime += performance.now() - pauseStart;
|
|
475
|
-
}
|
|
476
|
-
elapsed = performance.now() - loopStart - totalPauseTime;
|
|
477
|
-
}
|
|
478
|
-
trimArrays(a, count, trackHeap, !!getOptStatus);
|
|
479
|
-
return {
|
|
480
|
-
samples: a.samples,
|
|
481
|
-
heapSamples: a.heapSamples,
|
|
482
|
-
timestamps: a.timestamps,
|
|
483
|
-
optStatuses: a.optStatuses,
|
|
484
|
-
pausePoints: a.pausePoints
|
|
485
|
-
};
|
|
486
|
-
}
|
|
487
|
-
/** Check if we should pause at this iteration for V8 optimization */
|
|
488
|
-
function shouldPause(iter, first, interval) {
|
|
489
|
-
if (first !== void 0 && iter === first) return true;
|
|
490
|
-
if (interval <= 0) return false;
|
|
491
|
-
if (first === void 0) return iter % interval === 0;
|
|
492
|
-
return (iter - first) % interval === 0;
|
|
493
|
-
}
|
|
494
|
-
/** @return percentiles and basic statistics */
|
|
495
|
-
function computeStats(samples) {
|
|
496
|
-
const sorted = [...samples].sort((a, b) => a - b);
|
|
497
|
-
const avg = samples.reduce((sum, s) => sum + s, 0) / samples.length;
|
|
498
|
-
return {
|
|
499
|
-
min: sorted[0],
|
|
500
|
-
max: sorted[sorted.length - 1],
|
|
501
|
-
avg,
|
|
502
|
-
p50: percentile(sorted, .5),
|
|
503
|
-
p75: percentile(sorted, .75),
|
|
504
|
-
p99: percentile(sorted, .99),
|
|
505
|
-
p999: percentile(sorted, .999)
|
|
506
|
-
};
|
|
507
|
-
}
|
|
508
|
-
/** @return percentile value with linear interpolation */
|
|
509
|
-
function percentile(sortedArray, p) {
|
|
510
|
-
const index = (sortedArray.length - 1) * p;
|
|
511
|
-
const lower = Math.floor(index);
|
|
512
|
-
const upper = Math.ceil(index);
|
|
513
|
-
const weight = index % 1;
|
|
514
|
-
if (upper >= sortedArray.length) return sortedArray[sortedArray.length - 1];
|
|
515
|
-
return sortedArray[lower] * (1 - weight) + sortedArray[upper] * weight;
|
|
516
|
-
}
|
|
517
|
-
/** @return runtime gc() function, or no-op if unavailable */
|
|
518
|
-
function gcFunction() {
|
|
519
|
-
const gc = globalThis.gc || globalThis.__gc;
|
|
520
|
-
if (gc) return gc;
|
|
521
|
-
console.warn("gc() not available, run node/bun with --expose-gc");
|
|
522
|
-
return () => {};
|
|
523
|
-
}
|
|
524
|
-
/** @return function to get V8 optimization status (requires --allow-natives-syntax) */
|
|
525
|
-
function createOptStatusGetter() {
|
|
526
|
-
try {
|
|
527
|
-
const getter = new Function("f", "return %GetOptimizationStatus(f)");
|
|
528
|
-
getter(() => {});
|
|
529
|
-
return getter;
|
|
530
|
-
} catch {
|
|
531
|
-
return;
|
|
532
|
-
}
|
|
533
|
-
}
|
|
534
|
-
/**
|
|
535
|
-
* V8 optimization status bit meanings:
|
|
536
|
-
* Bit 0 (1): is_function
|
|
537
|
-
* Bit 4 (16): is_optimized (TurboFan)
|
|
538
|
-
* Bit 5 (32): is_optimized (Maglev)
|
|
539
|
-
* Bit 7 (128): is_baseline (Sparkplug)
|
|
540
|
-
* Bit 3 (8): maybe_deoptimized
|
|
541
|
-
*/
|
|
542
|
-
const statusNames = {
|
|
543
|
-
1: "interpreted",
|
|
544
|
-
129: "sparkplug",
|
|
545
|
-
17: "turbofan",
|
|
546
|
-
33: "maglev",
|
|
547
|
-
49: "turbofan+maglev",
|
|
548
|
-
32769: "optimized"
|
|
549
|
-
};
|
|
550
|
-
/** @return analysis of V8 optimization status per sample */
|
|
551
|
-
function analyzeOptStatus(samples, statuses) {
|
|
552
|
-
if (statuses.length === 0 || statuses[0] === void 0) return void 0;
|
|
553
|
-
const byStatusCode = /* @__PURE__ */ new Map();
|
|
554
|
-
let deoptCount = 0;
|
|
555
|
-
for (let i = 0; i < samples.length; i++) {
|
|
556
|
-
const status = statuses[i];
|
|
557
|
-
if (status === void 0) continue;
|
|
558
|
-
if (status & 8) deoptCount++;
|
|
559
|
-
if (!byStatusCode.has(status)) byStatusCode.set(status, []);
|
|
560
|
-
byStatusCode.get(status).push(samples[i]);
|
|
561
|
-
}
|
|
562
|
-
const byTier = {};
|
|
563
|
-
for (const [status, times] of byStatusCode) {
|
|
564
|
-
const name = statusNames[status] || `status=${status}`;
|
|
565
|
-
const sorted = [...times].sort((a, b) => a - b);
|
|
566
|
-
const median = sorted[Math.floor(sorted.length / 2)];
|
|
567
|
-
byTier[name] = {
|
|
568
|
-
count: times.length,
|
|
569
|
-
medianMs: median
|
|
570
|
-
};
|
|
571
|
-
}
|
|
572
|
-
return {
|
|
573
|
-
byTier,
|
|
574
|
-
deoptCount
|
|
575
|
-
};
|
|
576
|
-
}
|
|
577
|
-
|
|
578
576
|
//#endregion
|
|
579
577
|
//#region src/runners/CreateRunner.ts
|
|
580
578
|
/** @return benchmark runner */
|
|
@@ -595,5 +593,5 @@ function getElapsed(startMark, endMark) {
|
|
|
595
593
|
}
|
|
596
594
|
|
|
597
595
|
//#endregion
|
|
598
|
-
export {
|
|
599
|
-
//# sourceMappingURL=TimingUtils-
|
|
596
|
+
export { createAdaptiveWrapper as a, BasicRunner as c, variantModuleUrl as d, createRunner as i, computeStats as l, getElapsed as n, average as o, getPerfNow as r, bootstrapDifferenceCI as s, debugWorkerTiming as t, discoverVariants as u };
|
|
597
|
+
//# sourceMappingURL=TimingUtils-ClclVQ7E.mjs.map
|