overtake 1.4.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -15
- package/build/cli.js +126 -125
- package/build/executor.d.ts +6 -2
- package/build/executor.js +59 -46
- package/build/gc-watcher.js +2 -6
- package/build/index.d.ts +10 -11
- package/build/index.js +153 -155
- package/build/register-hook.d.ts +1 -0
- package/build/register-hook.js +15 -0
- package/build/reporter.d.ts +10 -2
- package/build/reporter.js +176 -214
- package/build/runner.d.ts +1 -1
- package/build/runner.js +128 -119
- package/build/types.d.ts +6 -6
- package/build/types.js +9 -14
- package/build/utils.d.ts +1 -17
- package/build/utils.js +53 -85
- package/build/worker.js +25 -24
- package/package.json +7 -25
- package/src/__tests__/assert-no-closure.ts +135 -0
- package/src/__tests__/benchmark-execute.ts +48 -0
- package/src/cli.ts +137 -142
- package/src/executor.ts +45 -15
- package/src/index.ts +85 -57
- package/src/register-hook.ts +15 -0
- package/src/reporter.ts +26 -18
- package/src/runner.ts +1 -4
- package/src/types.ts +8 -8
- package/src/utils.ts +15 -54
- package/src/worker.ts +5 -2
- package/tsconfig.json +2 -1
- package/build/cli.cjs +0 -179
- package/build/cli.cjs.map +0 -1
- package/build/cli.js.map +0 -1
- package/build/executor.cjs +0 -123
- package/build/executor.cjs.map +0 -1
- package/build/executor.js.map +0 -1
- package/build/gc-watcher.cjs +0 -30
- package/build/gc-watcher.cjs.map +0 -1
- package/build/gc-watcher.js.map +0 -1
- package/build/index.cjs +0 -442
- package/build/index.cjs.map +0 -1
- package/build/index.js.map +0 -1
- package/build/reporter.cjs +0 -311
- package/build/reporter.cjs.map +0 -1
- package/build/reporter.js.map +0 -1
- package/build/runner.cjs +0 -532
- package/build/runner.cjs.map +0 -1
- package/build/runner.js.map +0 -1
- package/build/types.cjs +0 -66
- package/build/types.cjs.map +0 -1
- package/build/types.js.map +0 -1
- package/build/utils.cjs +0 -174
- package/build/utils.cjs.map +0 -1
- package/build/utils.js.map +0 -1
- package/build/worker.cjs +0 -155
- package/build/worker.cjs.map +0 -1
- package/build/worker.js.map +0 -1
- package/src/__tests__/assert-no-closure.js +0 -134
package/build/runner.js
CHANGED
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import { performance, PerformanceObserver } from 'node:perf_hooks';
|
|
2
|
-
import { Control, DURATION_SCALE } from
|
|
3
|
-
import { GCWatcher } from
|
|
4
|
-
const COMPLETE_VALUE = 100_00;
|
|
2
|
+
import { Control, DURATION_SCALE, COMPLETE_VALUE } from './types.js';
|
|
3
|
+
import { GCWatcher } from './gc-watcher.js';
|
|
5
4
|
const hr = process.hrtime.bigint.bind(process.hrtime);
|
|
6
5
|
const sink = new Int32Array(1);
|
|
7
|
-
const consume = (value)=>{
|
|
6
|
+
const consume = (value) => {
|
|
8
7
|
let payload = 0;
|
|
9
|
-
switch(typeof value){
|
|
8
|
+
switch (typeof value) {
|
|
10
9
|
case 'number':
|
|
11
10
|
payload = Number.isFinite(value) ? Math.trunc(value) : 0;
|
|
12
11
|
break;
|
|
13
12
|
case 'bigint':
|
|
14
|
-
payload = Number(value &
|
|
13
|
+
payload = Number(value & 0xffffffffn);
|
|
15
14
|
break;
|
|
16
15
|
case 'string':
|
|
17
16
|
payload = value.length;
|
|
@@ -30,8 +29,8 @@ const consume = (value)=>{
|
|
|
30
29
|
}
|
|
31
30
|
sink[0] ^= payload;
|
|
32
31
|
};
|
|
33
|
-
const runSync = (run, overhead)=>{
|
|
34
|
-
return (...args)=>{
|
|
32
|
+
const runSync = (run, overhead) => {
|
|
33
|
+
return (...args) => {
|
|
35
34
|
const start = hr();
|
|
36
35
|
const result = run(...args);
|
|
37
36
|
consume(result);
|
|
@@ -39,18 +38,18 @@ const runSync = (run, overhead)=>{
|
|
|
39
38
|
return duration > overhead ? duration - overhead : 0n;
|
|
40
39
|
};
|
|
41
40
|
};
|
|
42
|
-
const runAsync = (run)=>{
|
|
43
|
-
return async (...args)=>{
|
|
41
|
+
const runAsync = (run) => {
|
|
42
|
+
return async (...args) => {
|
|
44
43
|
const start = hr();
|
|
45
44
|
const result = await run(...args);
|
|
46
45
|
consume(result);
|
|
47
46
|
return hr() - start;
|
|
48
47
|
};
|
|
49
48
|
};
|
|
50
|
-
const isThenable = (value)=>{
|
|
49
|
+
const isThenable = (value) => {
|
|
51
50
|
return value !== null && (typeof value === 'object' || typeof value === 'function') && typeof value.then === 'function';
|
|
52
51
|
};
|
|
53
|
-
const TARGET_SAMPLE_NS =
|
|
52
|
+
const TARGET_SAMPLE_NS = 1000000n; // aim for ~1ms per measured sample
|
|
54
53
|
const MAX_BATCH = 1_048_576;
|
|
55
54
|
const PROGRESS_STRIDE = 16;
|
|
56
55
|
const GC_STRIDE = 32;
|
|
@@ -60,36 +59,38 @@ const OUTLIER_WINDOW = 64;
|
|
|
60
59
|
const OUTLIER_ABS_THRESHOLD = 10_000_000;
|
|
61
60
|
const BASELINE_SAMPLES = 16;
|
|
62
61
|
const OUTLIER_SCRATCH = new Float64Array(OUTLIER_WINDOW);
|
|
63
|
-
const measureTimerOverhead = ()=>{
|
|
62
|
+
const measureTimerOverhead = () => {
|
|
64
63
|
let total = 0n;
|
|
65
|
-
for(let i = 0; i < BASELINE_SAMPLES; i++){
|
|
64
|
+
for (let i = 0; i < BASELINE_SAMPLES; i++) {
|
|
66
65
|
const start = hr();
|
|
67
66
|
consume(0);
|
|
68
67
|
total += hr() - start;
|
|
69
68
|
}
|
|
70
69
|
return total / BigInt(BASELINE_SAMPLES);
|
|
71
70
|
};
|
|
72
|
-
const collectSample = async ({ batchSize, run, runRaw, runIsAsync, pre, preIsAsync, post, postIsAsync, context, data, nextNonce })=>{
|
|
71
|
+
const collectSample = async ({ batchSize, run, runRaw, runIsAsync, pre, preIsAsync, post, postIsAsync, context, data, nextNonce, }) => {
|
|
73
72
|
const canBatchTime = !runIsAsync && !pre && !post;
|
|
74
73
|
if (canBatchTime) {
|
|
75
74
|
const batchStart = hr();
|
|
76
75
|
if (nextNonce) {
|
|
77
|
-
for(let b = 0; b < batchSize; b++){
|
|
76
|
+
for (let b = 0; b < batchSize; b++) {
|
|
78
77
|
consume(runRaw(context, data, nextNonce()));
|
|
79
78
|
}
|
|
80
|
-
}
|
|
81
|
-
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
for (let b = 0; b < batchSize; b++) {
|
|
82
82
|
consume(runRaw(context, data));
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
|
-
return (hr() - batchStart) * DURATION_SCALE / BigInt(batchSize);
|
|
85
|
+
return ((hr() - batchStart) * DURATION_SCALE) / BigInt(batchSize);
|
|
86
86
|
}
|
|
87
87
|
let sampleDuration = 0n;
|
|
88
|
-
for(let b = 0; b < batchSize; b++){
|
|
88
|
+
for (let b = 0; b < batchSize; b++) {
|
|
89
89
|
if (pre) {
|
|
90
90
|
if (preIsAsync) {
|
|
91
91
|
await pre(context, data);
|
|
92
|
-
}
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
93
94
|
pre(context, data);
|
|
94
95
|
}
|
|
95
96
|
}
|
|
@@ -97,7 +98,8 @@ const collectSample = async ({ batchSize, run, runRaw, runIsAsync, pre, preIsAsy
|
|
|
97
98
|
const runAsyncFn = run;
|
|
98
99
|
const duration = nextNonce ? await runAsyncFn(context, data, nextNonce()) : await runAsyncFn(context, data);
|
|
99
100
|
sampleDuration += duration;
|
|
100
|
-
}
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
101
103
|
const runSyncFn = run;
|
|
102
104
|
const duration = nextNonce ? runSyncFn(context, data, nextNonce()) : runSyncFn(context, data);
|
|
103
105
|
sampleDuration += duration;
|
|
@@ -105,21 +107,22 @@ const collectSample = async ({ batchSize, run, runRaw, runIsAsync, pre, preIsAsy
|
|
|
105
107
|
if (post) {
|
|
106
108
|
if (postIsAsync) {
|
|
107
109
|
await post(context, data);
|
|
108
|
-
}
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
109
112
|
post(context, data);
|
|
110
113
|
}
|
|
111
114
|
}
|
|
112
115
|
}
|
|
113
|
-
return sampleDuration * DURATION_SCALE / BigInt(batchSize);
|
|
116
|
+
return (sampleDuration * DURATION_SCALE) / BigInt(batchSize);
|
|
114
117
|
};
|
|
115
|
-
const tuneParameters = async ({ initialBatch, run, runRaw, runIsAsync, pre, preIsAsync, post, postIsAsync, context, data, minCycles, relThreshold, maxCycles, nextNonce })=>{
|
|
118
|
+
const tuneParameters = async ({ initialBatch, run, runRaw, runIsAsync, pre, preIsAsync, post, postIsAsync, context, data, minCycles, relThreshold, maxCycles, nextNonce, }) => {
|
|
116
119
|
let batchSize = initialBatch;
|
|
117
120
|
let bestCv = Number.POSITIVE_INFINITY;
|
|
118
121
|
let bestBatch = batchSize;
|
|
119
|
-
for(let attempt = 0; attempt < 3; attempt++){
|
|
122
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
120
123
|
const samples = [];
|
|
121
124
|
const sampleCount = Math.min(8, maxCycles);
|
|
122
|
-
for(let s = 0; s < sampleCount; s++){
|
|
125
|
+
for (let s = 0; s < sampleCount; s++) {
|
|
123
126
|
const duration = await collectSample({
|
|
124
127
|
batchSize,
|
|
125
128
|
run,
|
|
@@ -131,12 +134,12 @@ const tuneParameters = async ({ initialBatch, run, runRaw, runIsAsync, pre, preI
|
|
|
131
134
|
postIsAsync,
|
|
132
135
|
context,
|
|
133
136
|
data,
|
|
134
|
-
nextNonce
|
|
137
|
+
nextNonce,
|
|
135
138
|
});
|
|
136
139
|
samples.push(Number(duration));
|
|
137
140
|
}
|
|
138
|
-
const mean = samples.reduce((acc, v)=>acc + v, 0) / samples.length;
|
|
139
|
-
const variance = samples.reduce((acc, v)=>acc + (v - mean) * (v - mean), 0) / Math.max(1, samples.length - 1);
|
|
141
|
+
const mean = samples.reduce((acc, v) => acc + v, 0) / samples.length;
|
|
142
|
+
const variance = samples.reduce((acc, v) => acc + (v - mean) * (v - mean), 0) / Math.max(1, samples.length - 1);
|
|
140
143
|
const stddev = Math.sqrt(variance);
|
|
141
144
|
const cv = mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
|
|
142
145
|
if (cv < bestCv) {
|
|
@@ -150,13 +153,9 @@ const tuneParameters = async ({ initialBatch, run, runRaw, runIsAsync, pre, preI
|
|
|
150
153
|
}
|
|
151
154
|
const tunedRel = bestCv < relThreshold ? Math.max(bestCv * 1.5, relThreshold * 0.5) : relThreshold;
|
|
152
155
|
const tunedMin = Math.min(maxCycles, Math.max(minCycles, Math.ceil(minCycles * Math.max(1, bestCv / (relThreshold || 1e-6)))));
|
|
153
|
-
return {
|
|
154
|
-
batchSize: bestBatch,
|
|
155
|
-
relThreshold: tunedRel,
|
|
156
|
-
minCycles: tunedMin
|
|
157
|
-
};
|
|
156
|
+
return { batchSize: bestBatch, relThreshold: tunedRel, minCycles: tunedMin };
|
|
158
157
|
};
|
|
159
|
-
const createGCTracker = ()=>{
|
|
158
|
+
const createGCTracker = () => {
|
|
160
159
|
if (process.env.OVERTAKE_GC_OBSERVER !== '1') {
|
|
161
160
|
return null;
|
|
162
161
|
}
|
|
@@ -164,26 +163,20 @@ const createGCTracker = ()=>{
|
|
|
164
163
|
return null;
|
|
165
164
|
}
|
|
166
165
|
const events = [];
|
|
167
|
-
const observer = new PerformanceObserver((list)=>{
|
|
168
|
-
for (const entry of list.getEntries()){
|
|
169
|
-
events.push({
|
|
170
|
-
start: entry.startTime,
|
|
171
|
-
end: entry.startTime + entry.duration
|
|
172
|
-
});
|
|
166
|
+
const observer = new PerformanceObserver((list) => {
|
|
167
|
+
for (const entry of list.getEntries()) {
|
|
168
|
+
events.push({ start: entry.startTime, end: entry.startTime + entry.duration });
|
|
173
169
|
}
|
|
174
170
|
});
|
|
175
171
|
try {
|
|
176
|
-
observer.observe({
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
]
|
|
180
|
-
});
|
|
181
|
-
} catch {
|
|
172
|
+
observer.observe({ entryTypes: ['gc'] });
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
182
175
|
return null;
|
|
183
176
|
}
|
|
184
|
-
const overlaps = (start, end)=>{
|
|
177
|
+
const overlaps = (start, end) => {
|
|
185
178
|
let noisy = false;
|
|
186
|
-
for(let i = events.length - 1; i >= 0; i--){
|
|
179
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
187
180
|
const event = events[i];
|
|
188
181
|
if (event.end < start - 5_000) {
|
|
189
182
|
events.splice(i, 1);
|
|
@@ -195,24 +188,19 @@ const createGCTracker = ()=>{
|
|
|
195
188
|
}
|
|
196
189
|
return noisy;
|
|
197
190
|
};
|
|
198
|
-
const dispose = ()=>observer.disconnect();
|
|
199
|
-
return {
|
|
200
|
-
overlaps,
|
|
201
|
-
dispose
|
|
202
|
-
};
|
|
191
|
+
const dispose = () => observer.disconnect();
|
|
192
|
+
return { overlaps, dispose };
|
|
203
193
|
};
|
|
204
|
-
const pushWindow = (arr, value, cap)=>{
|
|
194
|
+
const pushWindow = (arr, value, cap) => {
|
|
205
195
|
if (arr.length === cap) {
|
|
206
196
|
arr.shift();
|
|
207
197
|
}
|
|
208
198
|
arr.push(value);
|
|
209
199
|
};
|
|
210
|
-
const medianAndIqr = (arr)=>{
|
|
211
|
-
if (arr.length === 0)
|
|
212
|
-
median: 0,
|
|
213
|
-
|
|
214
|
-
};
|
|
215
|
-
for(let i = 0; i < arr.length; i++){
|
|
200
|
+
const medianAndIqr = (arr) => {
|
|
201
|
+
if (arr.length === 0)
|
|
202
|
+
return { median: 0, iqr: 0 };
|
|
203
|
+
for (let i = 0; i < arr.length; i++) {
|
|
216
204
|
OUTLIER_SCRATCH[i] = arr[i];
|
|
217
205
|
}
|
|
218
206
|
const view = OUTLIER_SCRATCH.subarray(0, arr.length);
|
|
@@ -223,32 +211,31 @@ const medianAndIqr = (arr)=>{
|
|
|
223
211
|
const q3Idx = Math.floor(view.length * 0.75);
|
|
224
212
|
const q1 = view[q1Idx];
|
|
225
213
|
const q3 = view[q3Idx];
|
|
226
|
-
return {
|
|
227
|
-
median,
|
|
228
|
-
iqr: q3 - q1
|
|
229
|
-
};
|
|
214
|
+
return { median, iqr: q3 - q1 };
|
|
230
215
|
};
|
|
231
|
-
const windowCv = (arr)=>{
|
|
232
|
-
if (arr.length < 2)
|
|
233
|
-
|
|
234
|
-
const
|
|
216
|
+
const windowCv = (arr) => {
|
|
217
|
+
if (arr.length < 2)
|
|
218
|
+
return Number.POSITIVE_INFINITY;
|
|
219
|
+
const mean = arr.reduce((a, v) => a + v, 0) / arr.length;
|
|
220
|
+
const variance = arr.reduce((a, v) => a + (v - mean) * (v - mean), 0) / (arr.length - 1);
|
|
235
221
|
const stddev = Math.sqrt(variance);
|
|
236
222
|
return mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
|
|
237
223
|
};
|
|
238
|
-
export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data, warmupCycles, minCycles, absThreshold, relThreshold, gcObserver = false, durationsSAB, controlSAB })=>{
|
|
224
|
+
export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data, warmupCycles, minCycles, absThreshold, relThreshold, gcObserver = false, durationsSAB, controlSAB, }) => {
|
|
239
225
|
const durations = new BigUint64Array(durationsSAB);
|
|
240
226
|
const control = new Int32Array(controlSAB);
|
|
241
227
|
control[Control.INDEX] = 0;
|
|
242
228
|
control[Control.PROGRESS] = 0;
|
|
243
229
|
control[Control.COMPLETE] = 255;
|
|
244
230
|
control[Control.HEAP_USED] = 0;
|
|
245
|
-
const context = await setup?.();
|
|
231
|
+
const context = (await setup?.());
|
|
246
232
|
const heapBefore = process.memoryUsage().heapUsed;
|
|
247
233
|
const input = data;
|
|
248
234
|
const maxCycles = durations.length;
|
|
249
235
|
const gcWatcher = gcObserver ? new GCWatcher() : null;
|
|
250
236
|
const gcTracker = gcObserver ? createGCTracker() : null;
|
|
251
237
|
try {
|
|
238
|
+
// classify sync/async and capture initial duration
|
|
252
239
|
let preIsAsync = false;
|
|
253
240
|
if (pre) {
|
|
254
241
|
const preResult = pre(context, input);
|
|
@@ -263,7 +250,8 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
263
250
|
if (runIsAsync) {
|
|
264
251
|
const resolved = await probeResult;
|
|
265
252
|
consume(resolved);
|
|
266
|
-
}
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
267
255
|
consume(probeResult);
|
|
268
256
|
}
|
|
269
257
|
const durationProbeRaw = hr() - probeStart;
|
|
@@ -279,21 +267,24 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
279
267
|
let durationProbe = runIsAsync ? durationProbeRaw : durationProbeRaw > timerOverhead ? durationProbeRaw - timerOverhead : 0n;
|
|
280
268
|
const shouldPerturbInput = process.env.OVERTAKE_PERTURB_INPUT === '1';
|
|
281
269
|
let nonce = 0;
|
|
282
|
-
const nextNonce = shouldPerturbInput
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
270
|
+
const nextNonce = shouldPerturbInput
|
|
271
|
+
? () => {
|
|
272
|
+
nonce = (nonce + 1) | 0;
|
|
273
|
+
return nonce;
|
|
274
|
+
}
|
|
275
|
+
: null;
|
|
286
276
|
if (!runIsAsync && !pre && !post) {
|
|
287
|
-
const PROBE_TIME_LIMIT_NS =
|
|
277
|
+
const PROBE_TIME_LIMIT_NS = 1000000000n;
|
|
288
278
|
const INITIAL_PROBE_SIZE = 10;
|
|
289
279
|
const MAX_PROBE_SIZE = 10_000;
|
|
290
280
|
const initialStart = hr();
|
|
291
281
|
if (nextNonce) {
|
|
292
|
-
for(let i = 0; i < INITIAL_PROBE_SIZE; i++){
|
|
282
|
+
for (let i = 0; i < INITIAL_PROBE_SIZE; i++) {
|
|
293
283
|
consume(runRaw(context, input, nextNonce()));
|
|
294
284
|
}
|
|
295
|
-
}
|
|
296
|
-
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
for (let i = 0; i < INITIAL_PROBE_SIZE; i++) {
|
|
297
288
|
consume(runRaw(context, input));
|
|
298
289
|
}
|
|
299
290
|
}
|
|
@@ -305,11 +296,12 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
305
296
|
let totalIterations = INITIAL_PROBE_SIZE;
|
|
306
297
|
if (cappedAdditional > 0) {
|
|
307
298
|
if (nextNonce) {
|
|
308
|
-
for(let i = 0; i < cappedAdditional; i++){
|
|
299
|
+
for (let i = 0; i < cappedAdditional; i++) {
|
|
309
300
|
consume(runRaw(context, input, nextNonce()));
|
|
310
301
|
}
|
|
311
|
-
}
|
|
312
|
-
|
|
302
|
+
}
|
|
303
|
+
else {
|
|
304
|
+
for (let i = 0; i < cappedAdditional; i++) {
|
|
313
305
|
consume(runRaw(context, input));
|
|
314
306
|
}
|
|
315
307
|
}
|
|
@@ -320,16 +312,18 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
320
312
|
const runTimedSync = runIsAsync ? null : runSync(runRaw, timerOverhead);
|
|
321
313
|
const runTimedAsync = runIsAsync ? runAsync(runRaw) : null;
|
|
322
314
|
const run = runIsAsync ? runTimedAsync : runTimedSync;
|
|
323
|
-
const runOnceSync = runIsAsync ? null : nextNonce ? (ctx, dataValue)=>runTimedSync(ctx, dataValue, nextNonce()) : runTimedSync;
|
|
324
|
-
const runOnceAsync = runIsAsync ? nextNonce ? (ctx, dataValue)=>runTimedAsync(ctx, dataValue, nextNonce()) : runTimedAsync : null;
|
|
315
|
+
const runOnceSync = runIsAsync ? null : nextNonce ? (ctx, dataValue) => runTimedSync(ctx, dataValue, nextNonce()) : runTimedSync;
|
|
316
|
+
const runOnceAsync = runIsAsync ? (nextNonce ? (ctx, dataValue) => runTimedAsync(ctx, dataValue, nextNonce()) : runTimedAsync) : null;
|
|
325
317
|
const preSync = preIsAsync ? null : pre;
|
|
326
318
|
const preAsync = preIsAsync ? pre : null;
|
|
327
319
|
const postSync = postIsAsync ? null : post;
|
|
328
320
|
const postAsync = postIsAsync ? post : null;
|
|
321
|
+
// choose batch size to amortize timer overhead
|
|
329
322
|
const durationPerRun = durationProbe === 0n ? 1n : durationProbe;
|
|
330
323
|
const suggestedBatch = Number(TARGET_SAMPLE_NS / durationPerRun);
|
|
331
324
|
const minBatchForFastOps = durationProbe < 100n ? 100_000 : 1;
|
|
332
325
|
const initialBatchSize = Math.min(MAX_BATCH, Math.max(minBatchForFastOps, suggestedBatch));
|
|
326
|
+
// auto-tune based on warmup samples
|
|
333
327
|
const tuned = await tuneParameters({
|
|
334
328
|
initialBatch: initialBatchSize,
|
|
335
329
|
run,
|
|
@@ -344,50 +338,54 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
344
338
|
minCycles,
|
|
345
339
|
relThreshold,
|
|
346
340
|
maxCycles,
|
|
347
|
-
nextNonce
|
|
341
|
+
nextNonce,
|
|
348
342
|
});
|
|
349
343
|
let batchSize = tuned.batchSize;
|
|
350
344
|
minCycles = tuned.minCycles;
|
|
351
345
|
relThreshold = tuned.relThreshold;
|
|
346
|
+
// warmup: run until requested cycles, adapt if unstable
|
|
352
347
|
const warmupStart = performance.now();
|
|
353
348
|
let warmupRemaining = warmupCycles;
|
|
354
349
|
const warmupWindow = [];
|
|
355
350
|
const warmupCap = Math.max(warmupCycles, Math.min(maxCycles, warmupCycles * 4 || 1000));
|
|
356
351
|
const canBatchTime = !runIsAsync && !preSync && !preAsync && !postSync && !postAsync;
|
|
357
|
-
const runWarmup = async ()=>{
|
|
352
|
+
const runWarmup = async () => {
|
|
358
353
|
if (canBatchTime) {
|
|
359
354
|
const batchStart = hr();
|
|
360
355
|
if (nextNonce) {
|
|
361
|
-
for(let b = 0; b < batchSize; b++){
|
|
356
|
+
for (let b = 0; b < batchSize; b++) {
|
|
362
357
|
consume(runRaw(context, input, nextNonce()));
|
|
363
358
|
}
|
|
364
|
-
}
|
|
365
|
-
|
|
359
|
+
}
|
|
360
|
+
else {
|
|
361
|
+
for (let b = 0; b < batchSize; b++) {
|
|
366
362
|
consume(runRaw(context, input));
|
|
367
363
|
}
|
|
368
364
|
}
|
|
369
|
-
return (hr() - batchStart) * DURATION_SCALE / BigInt(batchSize);
|
|
365
|
+
return ((hr() - batchStart) * DURATION_SCALE) / BigInt(batchSize);
|
|
370
366
|
}
|
|
371
367
|
if (preSync) {
|
|
372
368
|
preSync(context, input);
|
|
373
|
-
}
|
|
369
|
+
}
|
|
370
|
+
else if (preAsync) {
|
|
374
371
|
await preAsync(context, input);
|
|
375
372
|
}
|
|
376
373
|
const duration = runIsAsync ? await runOnceAsync(context, input) : runOnceSync(context, input);
|
|
377
374
|
if (postSync) {
|
|
378
375
|
postSync(context, input);
|
|
379
|
-
}
|
|
376
|
+
}
|
|
377
|
+
else if (postAsync) {
|
|
380
378
|
await postAsync(context, input);
|
|
381
379
|
}
|
|
382
380
|
return duration * DURATION_SCALE;
|
|
383
381
|
};
|
|
384
|
-
while(performance.now() - warmupStart < 1_000 && warmupRemaining > 0){
|
|
382
|
+
while (performance.now() - warmupStart < 1_000 && warmupRemaining > 0) {
|
|
385
383
|
const duration = await runWarmup();
|
|
386
384
|
pushWindow(warmupWindow, Number(duration), warmupCap);
|
|
387
385
|
warmupRemaining--;
|
|
388
386
|
}
|
|
389
387
|
let warmupDone = 0;
|
|
390
|
-
while(warmupDone < warmupRemaining){
|
|
388
|
+
while (warmupDone < warmupRemaining) {
|
|
391
389
|
const duration = await runWarmup();
|
|
392
390
|
pushWindow(warmupWindow, Number(duration), warmupCap);
|
|
393
391
|
warmupDone++;
|
|
@@ -395,7 +393,7 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
395
393
|
global.gc();
|
|
396
394
|
}
|
|
397
395
|
}
|
|
398
|
-
while(warmupWindow.length >= 8 && warmupWindow.length < warmupCap){
|
|
396
|
+
while (warmupWindow.length >= 8 && warmupWindow.length < warmupCap) {
|
|
399
397
|
const cv = windowCv(warmupWindow);
|
|
400
398
|
if (cv <= relThreshold * 2) {
|
|
401
399
|
break;
|
|
@@ -404,7 +402,7 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
404
402
|
pushWindow(warmupWindow, Number(duration), warmupCap);
|
|
405
403
|
}
|
|
406
404
|
let i = 0;
|
|
407
|
-
const WELFORD_SCALE =
|
|
405
|
+
const WELFORD_SCALE = 1000000n;
|
|
408
406
|
let meanS = 0n;
|
|
409
407
|
let m2S = 0n;
|
|
410
408
|
const outlierWindow = [];
|
|
@@ -413,14 +411,15 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
413
411
|
let disableFiltering = false;
|
|
414
412
|
const absThScaled = BigInt(Math.round(absThreshold)) * WELFORD_SCALE;
|
|
415
413
|
const absThSq = absThScaled * absThScaled;
|
|
416
|
-
const REL_PRECISION =
|
|
414
|
+
const REL_PRECISION = 1000000n;
|
|
417
415
|
const relThBigint = BigInt(Math.round(relThreshold * Number(REL_PRECISION)));
|
|
418
416
|
const relThSq = relThBigint * relThBigint;
|
|
419
417
|
const relPrecSq = REL_PRECISION * REL_PRECISION;
|
|
420
418
|
const Z95_SQ_NUM = 38416n;
|
|
421
419
|
const Z95_SQ_DENOM = 10000n;
|
|
422
|
-
while(true){
|
|
423
|
-
if (i >= maxCycles)
|
|
420
|
+
while (true) {
|
|
421
|
+
if (i >= maxCycles)
|
|
422
|
+
break;
|
|
424
423
|
if (!disableFiltering && skipped >= maxSkipped) {
|
|
425
424
|
console.error(`Warning: ${skipped} samples skipped due to noise/outlier detection. ` + `Disabling filtering for remaining samples. Results may have higher variance.`);
|
|
426
425
|
disableFiltering = true;
|
|
@@ -434,32 +433,36 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
434
433
|
if (canBatchTime) {
|
|
435
434
|
const batchStart = hr();
|
|
436
435
|
if (nextNonce) {
|
|
437
|
-
for(let b = 0; b < batchSize; b++){
|
|
436
|
+
for (let b = 0; b < batchSize; b++) {
|
|
438
437
|
consume(runRaw(context, input, nextNonce()));
|
|
439
438
|
}
|
|
440
|
-
}
|
|
441
|
-
|
|
439
|
+
}
|
|
440
|
+
else {
|
|
441
|
+
for (let b = 0; b < batchSize; b++) {
|
|
442
442
|
consume(runRaw(context, input));
|
|
443
443
|
}
|
|
444
444
|
}
|
|
445
445
|
const batchDuration = hr() - batchStart;
|
|
446
|
-
sampleDuration = batchDuration * DURATION_SCALE / BigInt(batchSize);
|
|
447
|
-
}
|
|
448
|
-
|
|
446
|
+
sampleDuration = (batchDuration * DURATION_SCALE) / BigInt(batchSize);
|
|
447
|
+
}
|
|
448
|
+
else {
|
|
449
|
+
for (let b = 0; b < batchSize; b++) {
|
|
449
450
|
if (preSync) {
|
|
450
451
|
preSync(context, input);
|
|
451
|
-
}
|
|
452
|
+
}
|
|
453
|
+
else if (preAsync) {
|
|
452
454
|
await preAsync(context, input);
|
|
453
455
|
}
|
|
454
456
|
const duration = runIsAsync ? await runOnceAsync(context, input) : runOnceSync(context, input);
|
|
455
457
|
sampleDuration += duration;
|
|
456
458
|
if (postSync) {
|
|
457
459
|
postSync(context, input);
|
|
458
|
-
}
|
|
460
|
+
}
|
|
461
|
+
else if (postAsync) {
|
|
459
462
|
await postAsync(context, input);
|
|
460
463
|
}
|
|
461
464
|
}
|
|
462
|
-
sampleDuration = sampleDuration * DURATION_SCALE / BigInt(batchSize);
|
|
465
|
+
sampleDuration = (sampleDuration * DURATION_SCALE) / BigInt(batchSize);
|
|
463
466
|
}
|
|
464
467
|
const sampleEnd = performance.now();
|
|
465
468
|
if (!disableFiltering) {
|
|
@@ -483,40 +486,46 @@ export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data,
|
|
|
483
486
|
skipped++;
|
|
484
487
|
continue;
|
|
485
488
|
}
|
|
486
|
-
}
|
|
489
|
+
}
|
|
490
|
+
else {
|
|
487
491
|
pushWindow(outlierWindow, durationNumber, OUTLIER_WINDOW);
|
|
488
492
|
}
|
|
489
493
|
durations[i++] = sampleDuration;
|
|
490
494
|
const deltaS = sampleDuration * WELFORD_SCALE - meanS;
|
|
491
495
|
meanS += deltaS / BigInt(i);
|
|
492
496
|
m2S += deltaS * (sampleDuration * WELFORD_SCALE - meanS);
|
|
493
|
-
const progress = i / maxCycles * COMPLETE_VALUE;
|
|
497
|
+
const progress = (i / maxCycles) * COMPLETE_VALUE;
|
|
494
498
|
if (i % PROGRESS_STRIDE === 0) {
|
|
495
499
|
control[Control.PROGRESS] = progress;
|
|
496
500
|
}
|
|
497
501
|
if (i >= minCycles) {
|
|
498
|
-
if (m2S <= absThSq * BigInt(i - 1))
|
|
502
|
+
if (m2S <= absThSq * BigInt(i - 1))
|
|
503
|
+
break;
|
|
504
|
+
// RME convergence: Z95 * sem/mean <= relThreshold
|
|
505
|
+
// Z95^2 * m2S / (n*(n-1)*meanS^2) <= relThreshold^2
|
|
499
506
|
const ni = BigInt(i);
|
|
500
|
-
if (meanS !== 0n && Z95_SQ_NUM * m2S * relPrecSq <= relThSq * ni * (ni - 1n) * meanS * meanS * Z95_SQ_DENOM)
|
|
507
|
+
if (meanS !== 0n && Z95_SQ_NUM * m2S * relPrecSq <= relThSq * ni * (ni - 1n) * meanS * meanS * Z95_SQ_DENOM)
|
|
508
|
+
break;
|
|
501
509
|
}
|
|
502
510
|
}
|
|
503
511
|
control[Control.INDEX] = i;
|
|
504
512
|
control[Control.COMPLETE] = 0;
|
|
505
513
|
const heapAfter = process.memoryUsage().heapUsed;
|
|
506
514
|
control[Control.HEAP_USED] = Math.max(0, Math.round((heapAfter - heapBefore) / 1024));
|
|
507
|
-
}
|
|
515
|
+
}
|
|
516
|
+
catch (e) {
|
|
508
517
|
console.error(e && typeof e === 'object' && 'stack' in e ? e.stack : e);
|
|
509
518
|
control[Control.COMPLETE] = 1;
|
|
510
|
-
}
|
|
519
|
+
}
|
|
520
|
+
finally {
|
|
511
521
|
gcTracker?.dispose?.();
|
|
512
522
|
try {
|
|
513
523
|
await teardown?.(context);
|
|
514
|
-
}
|
|
524
|
+
}
|
|
525
|
+
catch (e) {
|
|
515
526
|
control[Control.COMPLETE] = 2;
|
|
516
527
|
console.error(e && typeof e === 'object' && 'stack' in e ? e.stack : e);
|
|
517
528
|
}
|
|
518
529
|
}
|
|
519
530
|
return control[Control.COMPLETE];
|
|
520
531
|
};
|
|
521
|
-
|
|
522
|
-
//# sourceMappingURL=runner.js.map
|
package/build/types.d.ts
CHANGED
|
@@ -53,12 +53,12 @@ export interface Options<TContext, TInput> extends RunOptions<TContext, TInput>,
|
|
|
53
53
|
durationsSAB: SharedArrayBuffer;
|
|
54
54
|
controlSAB: SharedArrayBuffer;
|
|
55
55
|
}
|
|
56
|
-
export declare
|
|
57
|
-
INDEX
|
|
58
|
-
PROGRESS
|
|
59
|
-
COMPLETE
|
|
60
|
-
HEAP_USED
|
|
61
|
-
}
|
|
56
|
+
export declare const Control: {
|
|
57
|
+
readonly INDEX: 0;
|
|
58
|
+
readonly PROGRESS: 1;
|
|
59
|
+
readonly COMPLETE: 2;
|
|
60
|
+
readonly HEAP_USED: 3;
|
|
61
|
+
};
|
|
62
62
|
export declare const CONTROL_SLOTS: number;
|
|
63
63
|
export declare const DEFAULT_CYCLES = 10000;
|
|
64
64
|
export declare const Z95 = 1.96;
|
package/build/types.js
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
export const REPORT_TYPES = Array.from({
|
|
2
|
-
length: 99
|
|
3
|
-
}, (_, idx)=>`p${idx + 1}`).concat([
|
|
1
|
+
export const REPORT_TYPES = Array.from({ length: 99 }, (_, idx) => `p${idx + 1}`).concat([
|
|
4
2
|
'ops',
|
|
5
3
|
'mean',
|
|
6
4
|
'min',
|
|
@@ -15,19 +13,16 @@ export const REPORT_TYPES = Array.from({
|
|
|
15
13
|
'mad',
|
|
16
14
|
'iqr',
|
|
17
15
|
'ci_lower',
|
|
18
|
-
'ci_upper'
|
|
16
|
+
'ci_upper',
|
|
19
17
|
]);
|
|
20
|
-
export
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
export const CONTROL_SLOTS = Object.values(Control).length / 2;
|
|
18
|
+
export const Control = {
|
|
19
|
+
INDEX: 0,
|
|
20
|
+
PROGRESS: 1,
|
|
21
|
+
COMPLETE: 2,
|
|
22
|
+
HEAP_USED: 3,
|
|
23
|
+
};
|
|
24
|
+
export const CONTROL_SLOTS = Object.keys(Control).length;
|
|
28
25
|
export const DEFAULT_CYCLES = 10_000;
|
|
29
26
|
export const Z95 = 1.96;
|
|
30
27
|
export const DURATION_SCALE = 1000n;
|
|
31
28
|
export const COMPLETE_VALUE = 100_00;
|
|
32
|
-
|
|
33
|
-
//# sourceMappingURL=types.js.map
|
package/build/utils.d.ts
CHANGED
|
@@ -1,23 +1,7 @@
|
|
|
1
|
+
export declare const resolveHookUrl: string;
|
|
1
2
|
export declare const isqrt: (n: bigint) => bigint;
|
|
2
|
-
export declare const abs: (value: bigint) => bigint;
|
|
3
3
|
export declare const cmp: (a: bigint | number, b: bigint | number) => number;
|
|
4
4
|
export declare const max: (a: bigint, b: bigint) => bigint;
|
|
5
|
-
export declare const divMod: (a: bigint, b: bigint) => {
|
|
6
|
-
quotient: bigint;
|
|
7
|
-
remainder: bigint;
|
|
8
|
-
};
|
|
9
5
|
export declare function div(a: bigint, b: bigint, decimals?: number): string;
|
|
10
6
|
export declare function divs(a: bigint, b: bigint, scale: bigint): bigint;
|
|
11
|
-
export declare class ScaledBigInt {
|
|
12
|
-
value: bigint;
|
|
13
|
-
scale: bigint;
|
|
14
|
-
constructor(value: bigint, scale: bigint);
|
|
15
|
-
add(value: bigint): void;
|
|
16
|
-
sub(value: bigint): void;
|
|
17
|
-
div(value: bigint): void;
|
|
18
|
-
mul(value: bigint): void;
|
|
19
|
-
unscale(): bigint;
|
|
20
|
-
number(): number;
|
|
21
|
-
}
|
|
22
7
|
export declare function assertNoClosure(code: string, name: string): void;
|
|
23
|
-
export declare const transpile: (code: string) => Promise<string>;
|