overtake 1.4.0 → 2.0.1

package/build/index.js

@@ -1,26 +1,29 @@
 import { cpus } from 'node:os';
 import Progress from 'progress';
-import { createExecutor } from "./executor.js";
-import { DEFAULT_CYCLES } from "./types.js";
+import { createExecutor } from './executor.js';
+import { DEFAULT_CYCLES } from './types.js';
 export const DEFAULT_WORKERS = Math.max(1, Math.ceil(cpus().length / 4));
-export const AsyncFunction = (async ()=>{}).constructor;
 const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
-export const DEFAULT_REPORT_TYPES = [
-    'ops'
-];
+export const DEFAULT_REPORT_TYPES = ['ops'];
+const createExecutorErrorReport = (error) => ({
+    count: 0,
+    heapUsedKB: 0,
+    dceWarning: false,
+    error: error instanceof Error ? error.message : String(error),
+});
 export class MeasureContext {
-    title;
-    run;
     pre;
     post;
-    constructor(title, run){
+    title;
+    run;
+    constructor(title, run) {
         this.title = title;
         this.run = run;
     }
 }
 export class Measure {
     #ctx;
-    constructor(ctx){
+    constructor(ctx) {
         this.#ctx = ctx;
     }
     pre(fn) {
@@ -33,18 +36,18 @@ export class Measure {
     }
 }
 export class TargetContext {
-    title;
-    setup;
     teardown;
     measures = [];
-    constructor(title, setup){
+    title;
+    setup;
+    constructor(title, setup) {
         this.title = title;
         this.setup = setup;
     }
 }
 export class Target {
     #ctx;
-    constructor(ctx){
+    constructor(ctx) {
         this.#ctx = ctx;
     }
     teardown(fn) {
@@ -60,7 +63,7 @@ export class Target {
 export class FeedContext {
     title;
     fn;
-    constructor(title, fn){
+    constructor(title, fn) {
         this.title = title;
         this.fn = fn;
     }
@@ -72,14 +75,16 @@ export class Benchmark {
     static create(title, fn) {
         if (fn) {
             return new Benchmark(title, fn);
-        } else {
+        }
+        else {
             return new Benchmark(title);
         }
     }
-    constructor(title, fn){
+    constructor(title, fn) {
         if (fn) {
             this.feed(title, fn);
-        } else {
+        }
+        else {
             this.feed(title);
         }
     }
@@ -93,14 +98,14 @@ export class Benchmark {
         this.#targets.push(target);
         return new Target(target);
     }
-    async execute(options) {
-        const { workers = DEFAULT_WORKERS, warmupCycles = 20, maxCycles = DEFAULT_CYCLES, minCycles = 50, absThreshold = 1_000, relThreshold = 0.02, gcObserver = true, reportTypes = DEFAULT_REPORT_TYPES, progress = false, progressInterval = 100 } = options;
+    async execute(options = {}) {
+        const { workers = DEFAULT_WORKERS, warmupCycles = 20, maxCycles = DEFAULT_CYCLES, minCycles = 50, absThreshold = 1_000, relThreshold = 0.02, gcObserver = true, reportTypes = DEFAULT_REPORT_TYPES, progress = false, progressInterval = 100, } = options;
         if (this.#executed) {
             throw new Error("Benchmark is executed and can't be reused");
         }
         this.#executed = true;
         const benchmarkUrl = options[BENCHMARK_URL];
-        const totalBenchmarks = this.#targets.reduce((acc, t)=>acc + t.measures.length * this.#feeds.length, 0);
+        const totalBenchmarks = this.#targets.reduce((acc, t) => acc + t.measures.length * this.#feeds.length, 0);
         const progressMap = new Map();
         let completedBenchmarks = 0;
         let bar = null;
@@ -109,19 +114,17 @@ export class Benchmark {
                 total: totalBenchmarks * 100,
                 width: 30,
                 complete: '=',
-                incomplete: ' '
+                incomplete: ' ',
             });
         }
-        const onProgress = progress ? (info)=>{
-            progressMap.set(info.id, info.progress);
-            const totalProgress = (completedBenchmarks + [
-                ...progressMap.values()
-            ].reduce((a, b)=>a + b, 0)) * 100;
-            const label = info.id.length > 30 ? info.id.slice(0, 27) + '...' : info.id;
-            bar?.update(totalProgress / (totalBenchmarks * 100), {
-                label
-            });
-        } : undefined;
+        const onProgress = progress
+            ? (info) => {
+                progressMap.set(info.id, info.progress);
+                const totalProgress = (completedBenchmarks + [...progressMap.values()].reduce((a, b) => a + b, 0)) * 100;
+                const label = info.id.length > 30 ? info.id.slice(0, 27) + '...' : info.id;
+                bar?.update(totalProgress / (totalBenchmarks * 100), { label });
+            }
+            : undefined;
         const executor = createExecutor({
             workers,
             warmupCycles,
@@ -133,68 +136,78 @@ export class Benchmark {
             reportTypes,
             onProgress,
             progressInterval,
-            [BENCHMARK_URL]: benchmarkUrl
+            [BENCHMARK_URL]: benchmarkUrl,
         });
         const reports = [];
-        for (const target of this.#targets){
-            const targetReport = {
-                target: target.title,
-                measures: []
-            };
-            for (const measure of target.measures){
-                const measureReport = {
-                    measure: measure.title,
-                    feeds: []
-                };
-                for (const feed of this.#feeds){
-                    const id = `${target.title}/${measure.title}/${feed.title}`;
-                    const data = await feed.fn?.();
-                    executor.push({
-                        id,
-                        setup: target.setup,
-                        teardown: target.teardown,
-                        pre: measure.pre,
-                        run: measure.run,
-                        post: measure.post,
-                        data
-                    }).then((data)=>{
-                        progressMap.delete(id);
-                        completedBenchmarks++;
-                        measureReport.feeds.push({
+        const pendingReports = [];
+        try {
+            const feedData = await Promise.all(this.#feeds.map(async (feed) => ({ title: feed.title, data: await feed.fn?.() })));
+            for (const target of this.#targets) {
+                const targetReport = { target: target.title, measures: [] };
+                for (const measure of target.measures) {
+                    const measureReport = { measure: measure.title, feeds: [] };
+                    for (const feed of feedData) {
+                        const id = `${target.title}/${measure.title}/${feed.title}`;
+                        const feedReport = {
                             feed: feed.title,
-                            data
-                        });
-                    });
+                            data: createExecutorErrorReport('Benchmark did not produce a report'),
+                        };
+                        measureReport.feeds.push(feedReport);
+                        pendingReports.push((async () => {
+                            try {
+                                feedReport.data = await executor.pushAsync({
+                                    id,
+                                    setup: target.setup,
+                                    teardown: target.teardown,
+                                    pre: measure.pre,
+                                    run: measure.run,
+                                    post: measure.post,
+                                    data: feed.data,
+                                });
+                            }
+                            catch (error) {
+                                feedReport.data = createExecutorErrorReport(error);
+                            }
+                            finally {
+                                progressMap.delete(id);
+                                completedBenchmarks++;
+                            }
+                        })());
+                    }
+                    targetReport.measures.push(measureReport);
                 }
-                targetReport.measures.push(measureReport);
+                reports.push(targetReport);
             }
-            reports.push(targetReport);
+            await Promise.all(pendingReports);
+            if (bar) {
+                bar.update(1, { label: 'done' });
+                bar.terminate();
+            }
+            return reports;
         }
-        await executor.drain();
-        executor.kill();
-        if (bar) {
-            bar.update(1, {
-                label: 'done'
-            });
-            bar.terminate();
+        finally {
+            executor.kill();
         }
-        return reports;
     }
 }
-export const printSimpleReports = (reports)=>{
-    for (const report of reports){
-        for (const { measure, feeds } of report.measures){
+export const printSimpleReports = (reports) => {
+    for (const report of reports) {
+        for (const { measure, feeds } of report.measures) {
             console.group('\n', report.target, measure);
-            for (const { feed, data } of feeds){
+            for (const { feed, data } of feeds) {
                 const { count, heapUsedKB, dceWarning, error: benchError, ...metrics } = data;
                 if (benchError) {
                     console.log(feed, `\x1b[31m[error: ${benchError}]\x1b[0m`);
                     continue;
                 }
-                const output = Object.entries(metrics).map(([key, report])=>`${key}: ${report.toString()}`).join('; ');
+                const output = Object.entries(metrics)
+                    .map(([key, report]) => `${key}: ${report.toString()}`)
+                    .join('; ');
                 const extras = [];
-                if (heapUsedKB) extras.push(`heap: ${heapUsedKB}KB`);
-                if (dceWarning) extras.push('\x1b[33m[DCE warning]\x1b[0m');
+                if (heapUsedKB)
+                    extras.push(`heap: ${heapUsedKB}KB`);
+                if (dceWarning)
+                    extras.push('\x1b[33m[DCE warning]\x1b[0m');
                 const extrasStr = extras.length > 0 ? ` (${extras.join(', ')})` : '';
                 console.log(feed, output + extrasStr);
             }
@@ -202,44 +215,36 @@ export const printSimpleReports = (reports)=>{
         }
     }
 };
-export const printTableReports = (reports)=>{
-    for (const report of reports){
-        for (const { measure, feeds } of report.measures){
+export const printTableReports = (reports) => {
+    for (const report of reports) {
+        for (const { measure, feeds } of report.measures) {
             console.log('\n', report.target, measure);
             const table = {};
-            for (const { feed, data } of feeds){
+            for (const { feed, data } of feeds) {
                 const { error: benchError } = data;
                 if (benchError) {
-                    table[feed] = {
-                        error: benchError
-                    };
-                } else {
-                    table[feed] = Object.fromEntries(Object.entries(data).map(([key, report])=>[
-                            key,
-                            report.toString()
-                        ]));
+                    table[feed] = { error: benchError };
+                }
+                else {
+                    table[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
                 }
             }
             console.table(table);
         }
     }
 };
-export const printJSONReports = (reports, padding)=>{
+export const printJSONReports = (reports, padding) => {
     const output = {};
-    for (const report of reports){
-        for (const { measure, feeds } of report.measures){
+    for (const report of reports) {
+        for (const { measure, feeds } of report.measures) {
             const row = {};
-            for (const { feed, data } of feeds){
+            for (const { feed, data } of feeds) {
                 const { error: benchError } = data;
                 if (benchError) {
-                    row[feed] = {
-                        error: String(benchError)
-                    };
-                } else {
-                    row[feed] = Object.fromEntries(Object.entries(data).map(([key, report])=>[
-                            key,
-                            report.toString()
-                        ]));
+                    row[feed] = { error: String(benchError) };
+                }
+                else {
+                    row[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
                 }
             }
             output[`${report.target} ${measure}`] = row;
@@ -247,83 +252,74 @@ export const printJSONReports = (reports, padding)=>{
     }
     console.log(JSON.stringify(output, null, padding));
 };
-export const printMarkdownReports = (reports)=>{
-    for (const report of reports){
-        for (const { measure, feeds } of report.measures){
+export const printMarkdownReports = (reports) => {
+    for (const report of reports) {
+        for (const { measure, feeds } of report.measures) {
             console.log(`\n## ${report.target} - ${measure}\n`);
-            if (feeds.length === 0) continue;
-            const firstValid = feeds.find((f)=>!f.data.error);
+            if (feeds.length === 0)
+                continue;
+            const firstValid = feeds.find((f) => !f.data.error);
             if (!firstValid) {
-                for (const { feed, data } of feeds){
+                for (const { feed, data } of feeds) {
                     console.log(`| ${feed} | error: ${data.error} |`);
                 }
                 continue;
             }
-            const keys = Object.keys(firstValid.data).filter((k)=>k !== 'count' && k !== 'error');
-            const header = [
-                'Feed',
-                ...keys
-            ].join(' | ');
-            const separator = [
-                '---',
-                ...keys.map(()=>'---')
-            ].join(' | ');
+            const keys = Object.keys(firstValid.data).filter((k) => k !== 'count' && k !== 'error');
+            const header = ['Feed', ...keys].join(' | ');
+            const separator = ['---', ...keys.map(() => '---')].join(' | ');
             console.log(`| ${header} |`);
             console.log(`| ${separator} |`);
-            for (const { feed, data } of feeds){
+            for (const { feed, data } of feeds) {
                 if (data.error) {
                     console.log(`| ${feed} | error: ${data.error} |`);
                     continue;
                 }
-                const values = keys.map((k)=>data[k]?.toString() ?? '-');
-                console.log(`| ${[
-                    feed,
-                    ...values
-                ].join(' | ')} |`);
+                const values = keys.map((k) => data[k]?.toString() ?? '-');
+                console.log(`| ${[feed, ...values].join(' | ')} |`);
             }
         }
     }
 };
-export const printHistogramReports = (reports, width = 40)=>{
-    for (const report of reports){
-        for (const { measure, feeds } of report.measures){
+export const printHistogramReports = (reports, width = 40) => {
+    for (const report of reports) {
+        for (const { measure, feeds } of report.measures) {
             console.log(`\n${report.target} - ${measure}\n`);
             const opsKey = 'ops';
-            const values = feeds.map((f)=>{
+            const values = feeds.map((f) => {
                 const { error: benchError } = f.data;
                 return {
                     feed: f.feed,
-                    value: benchError ? 0 : f.data[opsKey]?.valueOf() ?? 0,
-                    error: benchError
+                    value: benchError ? 0 : (f.data[opsKey]?.valueOf() ?? 0),
+                    error: benchError,
                 };
             });
-            const maxValue = Math.max(...values.map((v)=>v.value));
-            const maxLabelLen = Math.max(...values.map((v)=>v.feed.length));
-            for (const { feed, value, error } of values){
+            const maxValue = Math.max(...values.map((v) => v.value));
+            const maxLabelLen = Math.max(...values.map((v) => v.feed.length));
+            for (const { feed, value, error } of values) {
                 const label = feed.padEnd(maxLabelLen);
                 if (error) {
                     console.log(`  ${label} | \x1b[31m[error: ${error}]\x1b[0m`);
                     continue;
                 }
-                const barLen = maxValue > 0 ? Math.round(value / maxValue * width) : 0;
+                const barLen = maxValue > 0 ? Math.round((value / maxValue) * width) : 0;
                 const bar = '\u2588'.repeat(barLen);
-                const formatted = value.toLocaleString('en-US', {
-                    maximumFractionDigits: 2
-                });
+                const formatted = value.toLocaleString('en-US', { maximumFractionDigits: 2 });
                 console.log(`  ${label} | ${bar} ${formatted} ops/s`);
             }
         }
     }
 };
-export const reportsToBaseline = (reports)=>{
+export const reportsToBaseline = (reports) => {
     const results = {};
-    for (const report of reports){
-        for (const { measure, feeds } of report.measures){
-            for (const { feed, data } of feeds){
-                if (data.error) continue;
+    for (const report of reports) {
+        for (const { measure, feeds } of report.measures) {
+            for (const { feed, data } of feeds) {
+                if (data.error)
+                    continue;
                 const key = `${report.target}/${measure}/${feed}`;
                 results[key] = {};
-                for (const [metric, value] of Object.entries(data)){
+                for (const [metric, value] of Object.entries(data)) {
                     if (metric !== 'count' && typeof value.valueOf === 'function') {
                         results[key][metric] = value.valueOf();
                     }
@@ -334,14 +330,14 @@ export const reportsToBaseline = (reports)=>{
     return {
         version: 1,
         timestamp: new Date().toISOString(),
-        results
+        results,
     };
 };
-export const printComparisonReports = (reports, baseline, threshold = 5)=>{
-    for (const report of reports){
-        for (const { measure, feeds } of report.measures){
+export const printComparisonReports = (reports, baseline, threshold = 5) => {
+    for (const report of reports) {
+        for (const { measure, feeds } of report.measures) {
             console.log(`\n${report.target} - ${measure}\n`);
-            for (const { feed, data } of feeds){
+            for (const { feed, data } of feeds) {
                 const key = `${report.target}/${measure}/${feed}`;
                 const baselineData = baseline.results[key];
                 console.log(`  ${feed}:`);
@@ -349,22 +345,26 @@ export const printComparisonReports = (reports, baseline, threshold = 5)=>{
                     console.log(`    \x1b[31m[error: ${data.error}]\x1b[0m`);
                     continue;
                 }
-                for (const [metric, value] of Object.entries(data)){
-                    if (metric === 'count') continue;
+                for (const [metric, value] of Object.entries(data)) {
+                    if (metric === 'count')
+                        continue;
                     const current = value.valueOf();
                     const baselineValue = baselineData?.[metric];
                     if (baselineValue !== undefined && baselineValue !== 0) {
-                        const change = (current - baselineValue) / baselineValue * 100;
+                        const change = ((current - baselineValue) / baselineValue) * 100;
                         const isOps = metric === 'ops';
                         const improved = isOps ? change > threshold : change < -threshold;
                         const regressed = isOps ? change < -threshold : change > threshold;
                         let indicator = ' ';
-                        if (improved) indicator = '\x1b[32m+\x1b[0m';
-                        else if (regressed) indicator = '\x1b[31m!\x1b[0m';
+                        if (improved)
+                            indicator = '\x1b[32m+\x1b[0m';
+                        else if (regressed)
+                            indicator = '\x1b[31m!\x1b[0m';
                         const changeStr = change >= 0 ? `+${change.toFixed(1)}%` : `${change.toFixed(1)}%`;
                         const coloredChange = regressed ? `\x1b[31m${changeStr}\x1b[0m` : improved ? `\x1b[32m${changeStr}\x1b[0m` : changeStr;
                         console.log(`    ${indicator} ${metric}: ${value.toString()} (${coloredChange})`);
-                    } else {
+                    }
+                    else {
                         console.log(`    * ${metric}: ${value.toString()} (new)`);
                     }
                 }
@@ -373,5 +373,3 @@ export const printComparisonReports = (reports, baseline, threshold = 5)=>{
     }
     console.log(`\nBaseline from: ${baseline.timestamp}`);
 };
-
-//# sourceMappingURL=index.js.map

package/build/register-hook.d.ts

@@ -0,0 +1 @@
+export {};

package/build/register-hook.js

@@ -0,0 +1,15 @@
+import { register } from 'node:module';
+async function resolve(s, c, n) {
+    try {
+        return await n(s, c);
+    }
+    catch (e) {
+        if (s.endsWith('.js'))
+            try {
+                return await n(s.slice(0, -3) + '.ts', c);
+            }
+            catch { }
+        throw e;
+    }
+}
+register('data:text/javascript,' + encodeURIComponent(`export ${resolve.toString()}`));

package/build/reporter.d.ts

@@ -1,4 +1,4 @@
-import { ReportType } from './types.js';
+import { type ReportType } from './types.js';
 export declare class Report {
     readonly type: ReportType;
     readonly value: bigint;
@@ -8,4 +8,12 @@ export declare class Report {
     valueOf(): number;
     toString(): string;
 }
-export declare const createReport: (durations: BigUint64Array, type: ReportType) => Report;
+type Stats = {
+    sum: bigint;
+    mean: bigint;
+    ssd: bigint;
+    n: bigint;
+};
+export declare const computeStats: (durations: BigUint64Array) => Stats;
+export declare const createReport: (durations: BigUint64Array, type: ReportType, stats?: Stats) => Report;
+export {};