overtake 2.0.0 → 2.0.2

package/build/runner.js

@@ -0,0 +1,528 @@
+import { performance, PerformanceObserver } from 'node:perf_hooks';
+import { Control, DURATION_SCALE, COMPLETE_VALUE } from './types.js';
+import { GCWatcher } from './gc-watcher.js';
+const hr = process.hrtime.bigint.bind(process.hrtime);
+const sink = new Int32Array(1);
+const consume = (value) => {
+    let payload = 0;
+    switch (typeof value) {
+        case 'number':
+            payload = Number.isFinite(value) ? Math.trunc(value) : 0;
+            break;
+        case 'bigint':
+            payload = Number(value & 0xffffffffn);
+            break;
+        case 'string':
+            payload = value.length;
+            break;
+        case 'boolean':
+            payload = value ? 1 : 0;
+            break;
+        case 'object':
+            payload = value === null ? 0 : 1;
+            break;
+        case 'function':
+            payload = 1;
+            break;
+        default:
+            payload = -1;
+    }
+    sink[0] ^= payload;
+};
+const runSync = (run, overhead) => {
+    return (...args) => {
+        const start = hr();
+        const result = run(...args);
+        consume(result);
+        const duration = hr() - start;
+        return duration > overhead ? duration - overhead : 0n;
+    };
+};
+const runAsync = (run) => {
+    return async (...args) => {
+        const start = hr();
+        const result = await run(...args);
+        consume(result);
+        return hr() - start;
+    };
+};
+const isThenable = (value) => {
+    return value !== null && (typeof value === 'object' || typeof value === 'function') && typeof value.then === 'function';
+};
+const TARGET_SAMPLE_NS = 1000000n; // aim for ~1ms per measured sample
+const MAX_BATCH = 1_048_576;
+const PROGRESS_STRIDE = 16;
+const GC_STRIDE = 32;
+const OUTLIER_MULTIPLIER = 4;
+const OUTLIER_IQR_MULTIPLIER = 3;
+const OUTLIER_WINDOW = 64;
+const OUTLIER_ABS_THRESHOLD = 10_000_000;
+const BASELINE_SAMPLES = 16;
+const OUTLIER_SCRATCH = new Float64Array(OUTLIER_WINDOW);
+const measureTimerOverhead = () => {
+    let total = 0n;
+    for (let i = 0; i < BASELINE_SAMPLES; i++) {
+        const start = hr();
+        consume(0);
+        total += hr() - start;
+    }
+    return total / BigInt(BASELINE_SAMPLES);
+};
+const collectSample = async ({ batchSize, run, runRaw, runIsAsync, pre, preIsAsync, post, postIsAsync, context, data, nextNonce, }) => {
+    const canBatchTime = !runIsAsync && !pre && !post;
+    if (canBatchTime) {
+        const batchStart = hr();
+        if (nextNonce) {
+            for (let b = 0; b < batchSize; b++) {
+                consume(runRaw(context, data, nextNonce()));
+            }
+        }
+        else {
+            for (let b = 0; b < batchSize; b++) {
+                consume(runRaw(context, data));
+            }
+        }
+        return ((hr() - batchStart) * DURATION_SCALE) / BigInt(batchSize);
+    }
+    let sampleDuration = 0n;
+    for (let b = 0; b < batchSize; b++) {
+        if (pre) {
+            if (preIsAsync) {
+                await pre(context, data);
+            }
+            else {
+                pre(context, data);
+            }
+        }
+        if (runIsAsync) {
+            const runAsyncFn = run;
+            const duration = nextNonce ? await runAsyncFn(context, data, nextNonce()) : await runAsyncFn(context, data);
+            sampleDuration += duration;
+        }
+        else {
+            const runSyncFn = run;
+            const duration = nextNonce ? runSyncFn(context, data, nextNonce()) : runSyncFn(context, data);
+            sampleDuration += duration;
+        }
+        if (post) {
+            if (postIsAsync) {
+                await post(context, data);
+            }
+            else {
+                post(context, data);
+            }
+        }
+    }
+    return (sampleDuration * DURATION_SCALE) / BigInt(batchSize);
+};
+const tuneParameters = async ({ initialBatch, run, runRaw, runIsAsync, pre, preIsAsync, post, postIsAsync, context, data, minCycles, relThreshold, maxCycles, nextNonce, }) => {
+    let batchSize = initialBatch;
+    let bestCv = Number.POSITIVE_INFINITY;
+    let bestBatch = batchSize;
+    for (let attempt = 0; attempt < 3; attempt++) {
+        const samples = [];
+        const sampleCount = Math.min(8, maxCycles);
+        for (let s = 0; s < sampleCount; s++) {
+            const duration = await collectSample({
+                batchSize,
+                run,
+                runRaw,
+                runIsAsync,
+                pre,
+                preIsAsync,
+                post,
+                postIsAsync,
+                context,
+                data,
+                nextNonce,
+            });
+            samples.push(Number(duration));
+        }
+        const mean = samples.reduce((acc, v) => acc + v, 0) / samples.length;
+        const variance = samples.reduce((acc, v) => acc + (v - mean) * (v - mean), 0) / Math.max(1, samples.length - 1);
+        const stddev = Math.sqrt(variance);
+        const cv = mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
+        if (cv < bestCv) {
+            bestCv = cv;
+            bestBatch = batchSize;
+        }
+        if (cv <= relThreshold || batchSize >= MAX_BATCH) {
+            break;
+        }
+        batchSize = Math.min(MAX_BATCH, batchSize * 2);
+    }
+    const tunedRel = bestCv < relThreshold ? Math.max(bestCv * 1.5, relThreshold * 0.5) : relThreshold;
+    const tunedMin = Math.min(maxCycles, Math.max(minCycles, Math.ceil(minCycles * Math.max(1, bestCv / (relThreshold || 1e-6)))));
+    return { batchSize: bestBatch, relThreshold: tunedRel, minCycles: tunedMin };
+};
+const createGCTracker = () => {
+    if (process.env.OVERTAKE_GC_OBSERVER !== '1') {
+        return null;
+    }
+    if (typeof PerformanceObserver === 'undefined') {
+        return null;
+    }
+    const events = [];
+    const observer = new PerformanceObserver((list) => {
+        for (const entry of list.getEntries()) {
+            events.push({ start: entry.startTime, end: entry.startTime + entry.duration });
+        }
+    });
+    try {
+        observer.observe({ entryTypes: ['gc'] });
+    }
+    catch {
+        return null;
+    }
+    const overlaps = (start, end) => {
+        let noisy = false;
+        for (let i = events.length - 1; i >= 0; i--) {
+            const event = events[i];
+            if (event.end < start - 5_000) {
+                events.splice(i, 1);
+                continue;
+            }
+            if (event.start <= end && event.end >= start) {
+                noisy = true;
+            }
+        }
+        return noisy;
+    };
+    const dispose = () => observer.disconnect();
+    return { overlaps, dispose };
+};
+const pushWindow = (arr, value, cap) => {
+    if (arr.length === cap) {
+        arr.shift();
+    }
+    arr.push(value);
+};
+const medianAndIqr = (arr) => {
+    if (arr.length === 0)
+        return { median: 0, iqr: 0 };
+    for (let i = 0; i < arr.length; i++) {
+        OUTLIER_SCRATCH[i] = arr[i];
+    }
+    const view = OUTLIER_SCRATCH.subarray(0, arr.length);
+    view.sort();
+    const mid = Math.floor(view.length / 2);
+    const median = view.length % 2 === 0 ? (view[mid - 1] + view[mid]) / 2 : view[mid];
+    const q1Idx = Math.floor(view.length * 0.25);
+    const q3Idx = Math.floor(view.length * 0.75);
+    const q1 = view[q1Idx];
+    const q3 = view[q3Idx];
+    return { median, iqr: q3 - q1 };
+};
+const windowCv = (arr) => {
+    if (arr.length < 2)
+        return Number.POSITIVE_INFINITY;
+    const mean = arr.reduce((a, v) => a + v, 0) / arr.length;
+    const variance = arr.reduce((a, v) => a + (v - mean) * (v - mean), 0) / (arr.length - 1);
+    const stddev = Math.sqrt(variance);
+    return mean === 0 ? Number.POSITIVE_INFINITY : stddev / mean;
+};
+export const benchmark = async ({ setup, teardown, pre, run: runRaw, post, data, warmupCycles, minCycles, absThreshold, relThreshold, gcObserver = false, durationsSAB, controlSAB, }) => {
+    const durations = new BigUint64Array(durationsSAB);
+    const control = new Int32Array(controlSAB);
+    control[Control.INDEX] = 0;
+    control[Control.PROGRESS] = 0;
+    control[Control.COMPLETE] = 255;
+    control[Control.HEAP_USED] = 0;
+    const context = (await setup?.());
+    const heapBefore = process.memoryUsage().heapUsed;
+    const input = data;
+    const maxCycles = durations.length;
+    const gcWatcher = gcObserver ? new GCWatcher() : null;
+    const gcTracker = gcObserver ? createGCTracker() : null;
+    try {
+        // classify sync/async and capture initial duration
+        let preIsAsync = false;
+        if (pre) {
+            const preResult = pre(context, input);
+            preIsAsync = isThenable(preResult);
+            if (preIsAsync) {
+                await preResult;
+            }
+        }
+        const probeStart = hr();
+        const probeResult = runRaw(context, input);
+        const runIsAsync = isThenable(probeResult);
+        if (runIsAsync) {
+            const resolved = await probeResult;
+            consume(resolved);
+        }
+        else {
+            consume(probeResult);
+        }
+        const durationProbeRaw = hr() - probeStart;
+        let postIsAsync = false;
+        if (post) {
+            const postResult = post(context, input);
+            postIsAsync = isThenable(postResult);
+            if (postIsAsync) {
+                await postResult;
+            }
+        }
+        const timerOverhead = runIsAsync ? 0n : measureTimerOverhead();
+        let durationProbe = runIsAsync ? durationProbeRaw : durationProbeRaw > timerOverhead ? durationProbeRaw - timerOverhead : 0n;
+        const shouldPerturbInput = process.env.OVERTAKE_PERTURB_INPUT === '1';
+        let nonce = 0;
+        const nextNonce = shouldPerturbInput
+            ? () => {
+                nonce = (nonce + 1) | 0;
+                return nonce;
+            }
+            : null;
+        if (!runIsAsync && !pre && !post) {
+            const PROBE_TIME_LIMIT_NS = 1000000000n;
+            const INITIAL_PROBE_SIZE = 10;
+            const MAX_PROBE_SIZE = 10_000;
+            const initialStart = hr();
+            if (nextNonce) {
+                for (let i = 0; i < INITIAL_PROBE_SIZE; i++) {
+                    consume(runRaw(context, input, nextNonce()));
+                }
+            }
+            else {
+                for (let i = 0; i < INITIAL_PROBE_SIZE; i++) {
+                    consume(runRaw(context, input));
+                }
+            }
+            const initialDuration = hr() - initialStart;
+            const estimatedPerOp = initialDuration / BigInt(INITIAL_PROBE_SIZE);
+            const remainingBudget = PROBE_TIME_LIMIT_NS - initialDuration;
+            const additionalIterations = estimatedPerOp > 0n ? Number(remainingBudget / estimatedPerOp) : MAX_PROBE_SIZE - INITIAL_PROBE_SIZE;
+            const cappedAdditional = Math.min(Math.max(0, additionalIterations), MAX_PROBE_SIZE - INITIAL_PROBE_SIZE);
+            let totalIterations = INITIAL_PROBE_SIZE;
+            if (cappedAdditional > 0) {
+                if (nextNonce) {
+                    for (let i = 0; i < cappedAdditional; i++) {
+                        consume(runRaw(context, input, nextNonce()));
+                    }
+                }
+                else {
+                    for (let i = 0; i < cappedAdditional; i++) {
+                        consume(runRaw(context, input));
+                    }
+                }
+                totalIterations += cappedAdditional;
+            }
+            durationProbe = (hr() - initialStart) / BigInt(totalIterations);
+        }
+        const runTimedSync = runIsAsync ? null : runSync(runRaw, timerOverhead);
+        const runTimedAsync = runIsAsync ? runAsync(runRaw) : null;
+        const run = runIsAsync ? runTimedAsync : runTimedSync;
+        const runOnceSync = runIsAsync ? null : nextNonce ? (ctx, dataValue) => runTimedSync(ctx, dataValue, nextNonce()) : runTimedSync;
+        const runOnceAsync = runIsAsync ? (nextNonce ? (ctx, dataValue) => runTimedAsync(ctx, dataValue, nextNonce()) : runTimedAsync) : null;
+        const preSync = preIsAsync ? null : pre;
+        const preAsync = preIsAsync ? pre : null;
+        const postSync = postIsAsync ? null : post;
+        const postAsync = postIsAsync ? post : null;
+        // choose batch size to amortize timer overhead
+        const durationPerRun = durationProbe === 0n ? 1n : durationProbe;
+        const suggestedBatch = Number(TARGET_SAMPLE_NS / durationPerRun);
+        const minBatchForFastOps = durationProbe < 100n ? 100_000 : 1;
+        const initialBatchSize = Math.min(MAX_BATCH, Math.max(minBatchForFastOps, suggestedBatch));
+        // auto-tune based on warmup samples
+        const tuned = await tuneParameters({
+            initialBatch: initialBatchSize,
+            run,
+            runRaw,
+            runIsAsync,
+            pre,
+            preIsAsync,
+            post,
+            postIsAsync,
+            context,
+            data: input,
+            minCycles,
+            relThreshold,
+            maxCycles,
+            nextNonce,
+        });
+        let batchSize = tuned.batchSize;
+        minCycles = tuned.minCycles;
+        relThreshold = tuned.relThreshold;
+        // warmup: run until requested cycles, adapt if unstable
+        const warmupStart = performance.now();
+        let warmupRemaining = warmupCycles;
+        const warmupWindow = [];
+        const warmupCap = Math.max(warmupCycles, Math.min(maxCycles, warmupCycles * 4 || 1000));
+        const canBatchTime = !runIsAsync && !preSync && !preAsync && !postSync && !postAsync;
+        const runWarmup = async () => {
+            if (canBatchTime) {
+                const batchStart = hr();
+                if (nextNonce) {
+                    for (let b = 0; b < batchSize; b++) {
+                        consume(runRaw(context, input, nextNonce()));
+                    }
+                }
+                else {
+                    for (let b = 0; b < batchSize; b++) {
+                        consume(runRaw(context, input));
+                    }
+                }
+                return ((hr() - batchStart) * DURATION_SCALE) / BigInt(batchSize);
+            }
+            if (preSync) {
+                preSync(context, input);
+            }
+            else if (preAsync) {
+                await preAsync(context, input);
+            }
+            const duration = runIsAsync ? await runOnceAsync(context, input) : runOnceSync(context, input);
+            if (postSync) {
+                postSync(context, input);
+            }
+            else if (postAsync) {
+                await postAsync(context, input);
+            }
+            return duration * DURATION_SCALE;
+        };
+        while (performance.now() - warmupStart < 1_000 && warmupRemaining > 0) {
+            const duration = await runWarmup();
+            pushWindow(warmupWindow, Number(duration), warmupCap);
+            warmupRemaining--;
+        }
+        let warmupDone = 0;
+        while (warmupDone < warmupRemaining) {
+            const duration = await runWarmup();
+            pushWindow(warmupWindow, Number(duration), warmupCap);
+            warmupDone++;
+            if (global.gc && warmupDone % GC_STRIDE === 0) {
+                global.gc();
+            }
+        }
+        while (warmupWindow.length >= 8 && warmupWindow.length < warmupCap) {
+            const cv = windowCv(warmupWindow);
+            if (cv <= relThreshold * 2) {
+                break;
+            }
+            const duration = await runWarmup();
+            pushWindow(warmupWindow, Number(duration), warmupCap);
+        }
+        let i = 0;
+        const WELFORD_SCALE = 1000000n;
+        let meanS = 0n;
+        let m2S = 0n;
+        const outlierWindow = [];
+        let skipped = 0;
+        const maxSkipped = maxCycles * 10;
+        let disableFiltering = false;
+        const absThScaled = BigInt(Math.round(absThreshold)) * WELFORD_SCALE;
+        const absThSq = absThScaled * absThScaled;
+        const REL_PRECISION = 1000000n;
+        const relThBigint = BigInt(Math.round(relThreshold * Number(REL_PRECISION)));
+        const relThSq = relThBigint * relThBigint;
+        const relPrecSq = REL_PRECISION * REL_PRECISION;
+        const Z95_SQ_NUM = 38416n;
+        const Z95_SQ_DENOM = 10000n;
+        while (true) {
+            if (i >= maxCycles)
+                break;
+            if (!disableFiltering && skipped >= maxSkipped) {
+                console.error(`Warning: ${skipped} samples skipped due to noise/outlier detection. ` + `Disabling filtering for remaining samples. Results may have higher variance.`);
+                disableFiltering = true;
+            }
+            if (global.gc && i > 0 && i % GC_STRIDE === 0) {
+                global.gc();
+            }
+            const gcMarker = gcWatcher?.start();
+            const sampleStart = performance.now();
+            let sampleDuration = 0n;
+            if (canBatchTime) {
+                const batchStart = hr();
+                if (nextNonce) {
+                    for (let b = 0; b < batchSize; b++) {
+                        consume(runRaw(context, input, nextNonce()));
+                    }
+                }
+                else {
+                    for (let b = 0; b < batchSize; b++) {
+                        consume(runRaw(context, input));
+                    }
+                }
+                const batchDuration = hr() - batchStart;
+                sampleDuration = (batchDuration * DURATION_SCALE) / BigInt(batchSize);
+            }
+            else {
+                for (let b = 0; b < batchSize; b++) {
+                    if (preSync) {
+                        preSync(context, input);
+                    }
+                    else if (preAsync) {
+                        await preAsync(context, input);
+                    }
+                    const duration = runIsAsync ? await runOnceAsync(context, input) : runOnceSync(context, input);
+                    sampleDuration += duration;
+                    if (postSync) {
+                        postSync(context, input);
+                    }
+                    else if (postAsync) {
+                        await postAsync(context, input);
+                    }
+                }
+                sampleDuration = (sampleDuration * DURATION_SCALE) / BigInt(batchSize);
+            }
+            const sampleEnd = performance.now();
+            if (!disableFiltering) {
+                const gcNoise = (gcMarker ? gcWatcher.seen(gcMarker) : false) || (gcTracker?.overlaps(sampleStart, sampleEnd) ?? false);
+                if (gcNoise) {
+                    skipped++;
+                    continue;
+                }
+            }
+            const durationNumber = Number(sampleDuration);
+            if (!disableFiltering) {
+                const { median, iqr } = medianAndIqr(outlierWindow);
+                const maxAllowed = median + OUTLIER_IQR_MULTIPLIER * iqr || Number.POSITIVE_INFINITY;
+                if (outlierWindow.length >= 8 && durationNumber > maxAllowed && durationNumber - median > OUTLIER_ABS_THRESHOLD) {
+                    skipped++;
+                    continue;
+                }
+                const meanNumber = Number(meanS / WELFORD_SCALE);
+                if (i >= 8 && meanNumber > 0 && durationNumber > OUTLIER_MULTIPLIER * meanNumber && durationNumber - meanNumber > OUTLIER_ABS_THRESHOLD) {
+                    skipped++;
+                    continue;
+                }
+            }
+            pushWindow(outlierWindow, durationNumber, OUTLIER_WINDOW);
+            durations[i++] = sampleDuration;
+            const deltaS = sampleDuration * WELFORD_SCALE - meanS;
+            meanS += deltaS / BigInt(i);
+            m2S += deltaS * (sampleDuration * WELFORD_SCALE - meanS);
+            const progress = (i / maxCycles) * COMPLETE_VALUE;
+            if (i % PROGRESS_STRIDE === 0) {
+                control[Control.PROGRESS] = progress;
+            }
+            if (i >= minCycles) {
+                if (m2S <= absThSq * BigInt(i - 1))
+                    break;
+                // RME convergence: Z95 * sem/mean <= relThreshold
+                // Z95^2 * m2S / (n*(n-1)*meanS^2) <= relThreshold^2
+                const ni = BigInt(i);
+                if (meanS !== 0n && Z95_SQ_NUM * m2S * relPrecSq <= relThSq * ni * (ni - 1n) * meanS * meanS * Z95_SQ_DENOM)
+                    break;
+            }
+        }
+        control[Control.INDEX] = i;
+        control[Control.COMPLETE] = 0;
+        const heapAfter = process.memoryUsage().heapUsed;
+        control[Control.HEAP_USED] = Math.max(0, Math.round((heapAfter - heapBefore) / 1024));
+    }
+    catch (e) {
+        console.error(e && typeof e === 'object' && 'stack' in e ? e.stack : e);
+        control[Control.COMPLETE] = 1;
+    }
+    finally {
+        gcTracker?.dispose?.();
+        try {
+            await teardown?.(context);
+        }
+        catch (e) {
+            control[Control.COMPLETE] = 2;
+            console.error(e && typeof e === 'object' && 'stack' in e ? e.stack : e);
+        }
+    }
+    return control[Control.COMPLETE];
+};

package/build/types.js

@@ -0,0 +1,28 @@
+export const REPORT_TYPES = Array.from({ length: 99 }, (_, idx) => `p${idx + 1}`).concat([
+    'ops',
+    'mean',
+    'min',
+    'max',
+    'median',
+    'mode',
+    'variance',
+    'sd',
+    'sem',
+    'moe',
+    'rme',
+    'mad',
+    'iqr',
+    'ci_lower',
+    'ci_upper',
+]);
+export const Control = {
+    INDEX: 0,
+    PROGRESS: 1,
+    COMPLETE: 2,
+    HEAP_USED: 3,
+};
+export const CONTROL_SLOTS = Object.keys(Control).length;
+export const DEFAULT_CYCLES = 10_000;
+export const Z95 = 1.96;
+export const DURATION_SCALE = 1000n;
+export const COMPLETE_VALUE = 100_00;

package/build/utils.d.ts

@@ -4,4 +4,5 @@ export declare const cmp: (a: bigint | number, b: bigint | number) => number;
 export declare const max: (a: bigint, b: bigint) => bigint;
 export declare function div(a: bigint, b: bigint, decimals?: number): string;
 export declare function divs(a: bigint, b: bigint, scale: bigint): bigint;
+export declare function normalizeFunction(code: string): string;
 export declare function assertNoClosure(code: string, name: string): void;