benchforge 0.1.9 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -0
- package/README.md +99 -260
- package/bin/benchforge +1 -2
- package/dist/AnalyzeArchive-8NCJhmhS.mjs +145 -0
- package/dist/AnalyzeArchive-8NCJhmhS.mjs.map +1 -0
- package/dist/BenchMatrix-BZVrBB_h.mjs +1050 -0
- package/dist/BenchMatrix-BZVrBB_h.mjs.map +1 -0
- package/dist/BenchRunner-DglX1NOn.d.mts +302 -0
- package/dist/CoverageSampler-D5T9DRqe.mjs +27 -0
- package/dist/CoverageSampler-D5T9DRqe.mjs.map +1 -0
- package/dist/Formatters-BWj3d4sv.mjs +95 -0
- package/dist/Formatters-BWj3d4sv.mjs.map +1 -0
- package/dist/{HeapSampler-B8dtKHn1.mjs → HeapSampler-Dq-hpXem.mjs} +4 -4
- package/dist/HeapSampler-Dq-hpXem.mjs.map +1 -0
- package/dist/RunBenchCLI-C17DrJz8.mjs +3075 -0
- package/dist/RunBenchCLI-C17DrJz8.mjs.map +1 -0
- package/dist/StatisticalUtils-BD92crgM.mjs +255 -0
- package/dist/StatisticalUtils-BD92crgM.mjs.map +1 -0
- package/dist/TimeSampler-Ds8n7l2B.mjs +29 -0
- package/dist/TimeSampler-Ds8n7l2B.mjs.map +1 -0
- package/dist/ViewerServer-BJhdnxlN.mjs +639 -0
- package/dist/ViewerServer-BJhdnxlN.mjs.map +1 -0
- package/dist/ViewerServer-CuMNdNBz.mjs +2 -0
- package/dist/bin/benchforge.mjs +4 -5
- package/dist/bin/benchforge.mjs.map +1 -1
- package/dist/index.d.mts +731 -522
- package/dist/index.mjs +98 -3
- package/dist/index.mjs.map +1 -0
- package/dist/runners/WorkerScript.d.mts +12 -4
- package/dist/runners/WorkerScript.mjs +92 -120
- package/dist/runners/WorkerScript.mjs.map +1 -1
- package/dist/viewer/assets/CIPlot-BkOvMoMa.js +1 -0
- package/dist/viewer/assets/HistogramKde-CmSyUFY0.js +1 -0
- package/dist/viewer/assets/LegendUtils-BJpbn_jr.js +55 -0
- package/dist/viewer/assets/SampleTimeSeries-C4VBhXr3.js +1 -0
- package/dist/viewer/assets/index-Br9bp_cX.js +153 -0
- package/dist/viewer/assets/index-NzXXe_CC.css +1 -0
- package/dist/viewer/index.html +19 -0
- package/dist/viewer/speedscope/LICENSE +21 -0
- package/dist/viewer/speedscope/SourceCodePro-Regular.ttf-ILST5JV6.woff2 +0 -0
- package/dist/viewer/speedscope/favicon-16x16-V2DMIAZS.js +2 -0
- package/dist/viewer/speedscope/favicon-16x16-V2DMIAZS.js.map +7 -0
- package/dist/viewer/speedscope/favicon-16x16-VSI62OPJ.png +0 -0
- package/dist/viewer/speedscope/favicon-32x32-3EB2YCUY.png +0 -0
- package/dist/viewer/speedscope/favicon-32x32-THY3JDJL.js +2 -0
- package/dist/viewer/speedscope/favicon-32x32-THY3JDJL.js.map +7 -0
- package/dist/viewer/speedscope/favicon-FOKUP5Y5.ico +0 -0
- package/dist/viewer/speedscope/favicon-M34RF7BI.js +2 -0
- package/dist/viewer/speedscope/favicon-M34RF7BI.js.map +7 -0
- package/dist/viewer/speedscope/file-format-schema.json +274 -0
- package/dist/viewer/speedscope/index.html +19 -0
- package/dist/viewer/speedscope/jfrview_bg-BLJXNNQB.wasm +0 -0
- package/dist/viewer/speedscope/perf-vertx-stacks-01-collapsed-all-ZNUIGAJL.txt +199 -0
- package/dist/viewer/speedscope/release.txt +3 -0
- package/dist/viewer/speedscope/source-code-pro.LICENSE.md +93 -0
- package/dist/viewer/speedscope/speedscope-GHPHNKXC.css +2 -0
- package/dist/viewer/speedscope/speedscope-GHPHNKXC.css.map +7 -0
- package/dist/viewer/speedscope/speedscope-QZFMJ7VP.js +212 -0
- package/dist/viewer/speedscope/speedscope-QZFMJ7VP.js.map +7 -0
- package/package.json +52 -26
- package/src/bin/benchforge.ts +2 -2
- package/src/cli/AnalyzeArchive.ts +232 -0
- package/src/cli/BrowserBench.ts +322 -0
- package/src/cli/CliArgs.ts +164 -48
- package/src/cli/CliExport.ts +179 -0
- package/src/cli/CliOptions.ts +147 -0
- package/src/cli/CliReport.ts +197 -0
- package/src/cli/FilterBenchmarks.ts +18 -30
- package/src/cli/RunBenchCLI.ts +138 -844
- package/src/cli/SuiteRunner.ts +160 -0
- package/src/cli/ViewerServer.ts +282 -0
- package/src/export/AllocExport.ts +121 -0
- package/src/export/ArchiveExport.ts +146 -0
- package/src/export/ArchiveFormat.ts +50 -0
- package/src/export/CoverageExport.ts +148 -0
- package/src/export/EditorUri.ts +10 -0
- package/src/export/PerfettoExport.ts +91 -126
- package/src/export/SpeedscopeTypes.ts +98 -0
- package/src/export/TimeExport.ts +115 -0
- package/src/index.ts +87 -62
- package/src/matrix/BenchMatrix.ts +230 -0
- package/src/matrix/CaseLoader.ts +8 -6
- package/src/matrix/MatrixDirRunner.ts +153 -0
- package/src/matrix/MatrixFilter.ts +55 -53
- package/src/matrix/MatrixInlineRunner.ts +50 -0
- package/src/matrix/MatrixReport.ts +94 -254
- package/src/matrix/VariantLoader.ts +9 -9
- package/src/profiling/browser/BenchLoop.ts +51 -0
- package/src/profiling/browser/BrowserCDP.ts +133 -0
- package/src/profiling/browser/BrowserGcStats.ts +33 -0
- package/src/profiling/browser/BrowserProfiler.ts +160 -0
- package/src/profiling/browser/CdpClient.ts +82 -0
- package/src/profiling/browser/CdpPage.ts +138 -0
- package/src/profiling/browser/ChromeLauncher.ts +158 -0
- package/src/profiling/browser/ChromeTraceEvent.ts +28 -0
- package/src/profiling/browser/PageLoadMode.ts +61 -0
- package/src/profiling/node/CoverageSampler.ts +27 -0
- package/src/profiling/node/CoverageTypes.ts +23 -0
- package/src/profiling/node/HeapSampleReport.ts +261 -0
- package/src/{heap-sample → profiling/node}/HeapSampler.ts +55 -13
- package/src/profiling/node/ResolvedProfile.ts +98 -0
- package/src/profiling/node/TimeSampler.ts +57 -0
- package/src/report/BenchmarkReport.ts +146 -0
- package/src/report/Colors.ts +9 -0
- package/src/report/Formatters.ts +110 -0
- package/src/report/GcSections.ts +151 -0
- package/src/{GitUtils.ts → report/GitUtils.ts} +18 -19
- package/src/report/HtmlReport.ts +223 -0
- package/src/report/ParseStats.ts +73 -0
- package/src/report/StandardSections.ts +147 -0
- package/src/report/ViewerSections.ts +286 -0
- package/src/report/text/TableReport.ts +253 -0
- package/src/report/text/TextReport.ts +123 -0
- package/src/runners/AdaptiveWrapper.ts +167 -287
- package/src/runners/BenchRunner.ts +27 -22
- package/src/{Benchmark.ts → runners/BenchmarkSpec.ts} +5 -6
- package/src/runners/CreateRunner.ts +5 -7
- package/src/runners/GcStats.ts +58 -61
- package/src/{MeasuredResults.ts → runners/MeasuredResults.ts} +43 -37
- package/src/runners/MergeBatches.ts +123 -0
- package/src/{NodeGC.ts → runners/NodeGC.ts} +2 -3
- package/src/runners/RunnerOrchestrator.ts +180 -296
- package/src/runners/RunnerUtils.ts +75 -1
- package/src/runners/SampleStats.ts +100 -0
- package/src/runners/TimingRunner.ts +244 -0
- package/src/runners/TimingUtils.ts +3 -2
- package/src/runners/WorkerScript.ts +162 -178
- package/src/stats/BootstrapDifference.ts +282 -0
- package/src/{PermutationTest.ts → stats/PermutationTest.ts} +31 -40
- package/src/stats/StatisticalUtils.ts +445 -0
- package/src/{tests → test}/AdaptiveConvergence.test.ts +10 -10
- package/src/test/AdaptiveRunner.test.ts +39 -41
- package/src/{tests → test}/AdaptiveSampling.test.ts +9 -9
- package/src/test/AdaptiveStatistics.integration.ts +9 -41
- package/src/{tests → test}/BenchMatrix.test.ts +31 -28
- package/src/test/BenchmarkReport.test.ts +63 -13
- package/src/test/BrowserBench.e2e.test.ts +186 -17
- package/src/test/BrowserBench.test.ts +10 -5
- package/src/test/BuildTimeSection.test.ts +130 -0
- package/src/test/CapSamples.test.ts +82 -0
- package/src/test/CoverageExport.test.ts +115 -0
- package/src/test/CoverageSampler.test.ts +33 -0
- package/src/test/HeapAttribution.test.ts +51 -0
- package/src/{tests → test}/MatrixFilter.test.ts +16 -16
- package/src/{tests → test}/MatrixReport.test.ts +1 -1
- package/src/test/PermutationTest.test.ts +1 -1
- package/src/{tests → test}/RealDataValidation.test.ts +6 -6
- package/src/test/RunBenchCLI.test.ts +57 -56
- package/src/test/RunnerOrchestrator.test.ts +12 -12
- package/src/test/StatisticalUtils.test.ts +48 -12
- package/src/{table-util/test → test}/TableReport.test.ts +2 -2
- package/src/test/TestUtils.ts +35 -30
- package/src/test/TimeExport.test.ts +139 -0
- package/src/test/TimeSampler.test.ts +37 -0
- package/src/test/ViewerLive.e2e.test.ts +159 -0
- package/src/test/ViewerStatic.static.e2e.test.ts +137 -0
- package/src/{tests → test}/fixtures/baseline/impl.ts +1 -1
- package/src/{tests → test}/fixtures/bevy30-samples.ts +3 -1
- package/src/test/fixtures/cases/asyncCases.ts +9 -0
- package/src/{tests → test}/fixtures/cases/cases.ts +5 -2
- package/src/test/fixtures/cases/variants/product.ts +2 -0
- package/src/test/fixtures/cases/variants/sum.ts +2 -0
- package/src/test/fixtures/discover/fast.ts +1 -0
- package/src/{tests → test}/fixtures/discover/slow.ts +1 -1
- package/src/test/fixtures/invalid/bad.ts +1 -0
- package/src/test/fixtures/loader/fast.ts +1 -0
- package/src/{tests → test}/fixtures/loader/slow.ts +1 -1
- package/src/test/fixtures/loader/stateful.ts +2 -0
- package/src/test/fixtures/stateful/stateful.ts +2 -0
- package/src/test/fixtures/variants/extra.ts +1 -0
- package/src/test/fixtures/variants/impl.ts +1 -0
- package/src/test/fixtures/worker/fast.ts +1 -0
- package/src/{tests → test}/fixtures/worker/slow.ts +1 -1
- package/src/viewer/DateFormat.ts +30 -0
- package/src/viewer/Helpers.ts +23 -0
- package/src/viewer/LineData.ts +120 -0
- package/src/viewer/Providers.ts +191 -0
- package/src/viewer/ReportData.ts +123 -0
- package/src/viewer/State.ts +49 -0
- package/src/viewer/Theme.ts +15 -0
- package/src/viewer/components/App.tsx +73 -0
- package/src/viewer/components/DropZone.tsx +71 -0
- package/src/viewer/components/LazyPlot.ts +33 -0
- package/src/viewer/components/SamplesPanel.tsx +214 -0
- package/src/viewer/components/Shell.tsx +26 -0
- package/src/viewer/components/SourcePanel.tsx +216 -0
- package/src/viewer/components/SummaryPanel.tsx +332 -0
- package/src/viewer/components/TabBar.tsx +131 -0
- package/src/viewer/components/TabContent.tsx +46 -0
- package/src/viewer/components/ThemeToggle.tsx +50 -0
- package/src/viewer/index.html +20 -0
- package/src/viewer/main.tsx +4 -0
- package/src/viewer/plots/CIPlot.ts +313 -0
- package/src/{html/browser → viewer/plots}/HistogramKde.ts +42 -47
- package/src/viewer/plots/LegendUtils.ts +134 -0
- package/src/viewer/plots/PlotTypes.ts +85 -0
- package/src/viewer/plots/RenderPlots.ts +230 -0
- package/src/viewer/plots/SampleTimeSeries.ts +306 -0
- package/src/viewer/plots/SvgHelpers.ts +136 -0
- package/src/viewer/plots/TimeSeriesMarks.ts +319 -0
- package/src/viewer/report.css +427 -0
- package/src/viewer/shell.css +357 -0
- package/src/viewer/tsconfig.json +11 -0
- package/dist/BenchRunner-CSKN9zPy.d.mts +0 -225
- package/dist/BrowserHeapSampler-DCeL42RE.mjs +0 -202
- package/dist/BrowserHeapSampler-DCeL42RE.mjs.map +0 -1
- package/dist/GcStats-ByEovUi1.mjs +0 -77
- package/dist/GcStats-ByEovUi1.mjs.map +0 -1
- package/dist/HeapSampler-B8dtKHn1.mjs.map +0 -1
- package/dist/TimingUtils-ClclVQ7E.mjs +0 -597
- package/dist/TimingUtils-ClclVQ7E.mjs.map +0 -1
- package/dist/browser/index.js +0 -914
- package/dist/src-Cf_LXwlp.mjs +0 -2873
- package/dist/src-Cf_LXwlp.mjs.map +0 -1
- package/src/BenchMatrix.ts +0 -380
- package/src/BenchmarkReport.ts +0 -156
- package/src/HtmlDataPrep.ts +0 -148
- package/src/StandardSections.ts +0 -261
- package/src/StatisticalUtils.ts +0 -176
- package/src/TypeUtil.ts +0 -8
- package/src/browser/BrowserGcStats.ts +0 -44
- package/src/browser/BrowserHeapSampler.ts +0 -271
- package/src/export/JsonExport.ts +0 -103
- package/src/export/JsonFormat.ts +0 -91
- package/src/heap-sample/HeapSampleReport.ts +0 -196
- package/src/html/HtmlReport.ts +0 -131
- package/src/html/HtmlTemplate.ts +0 -284
- package/src/html/Types.ts +0 -88
- package/src/html/browser/CIPlot.ts +0 -287
- package/src/html/browser/LegendUtils.ts +0 -163
- package/src/html/browser/RenderPlots.ts +0 -263
- package/src/html/browser/SampleTimeSeries.ts +0 -389
- package/src/html/browser/Types.ts +0 -96
- package/src/html/browser/index.ts +0 -1
- package/src/html/index.ts +0 -17
- package/src/runners/BasicRunner.ts +0 -364
- package/src/table-util/ConvergenceFormatters.ts +0 -19
- package/src/table-util/Formatters.ts +0 -152
- package/src/table-util/README.md +0 -70
- package/src/table-util/TableReport.ts +0 -293
- package/src/tests/fixtures/cases/asyncCases.ts +0 -7
- package/src/tests/fixtures/cases/variants/product.ts +0 -2
- package/src/tests/fixtures/cases/variants/sum.ts +0 -2
- package/src/tests/fixtures/discover/fast.ts +0 -1
- package/src/tests/fixtures/invalid/bad.ts +0 -1
- package/src/tests/fixtures/loader/fast.ts +0 -1
- package/src/tests/fixtures/loader/stateful.ts +0 -2
- package/src/tests/fixtures/stateful/stateful.ts +0 -2
- package/src/tests/fixtures/variants/extra.ts +0 -1
- package/src/tests/fixtures/variants/impl.ts +0 -1
- package/src/tests/fixtures/worker/fast.ts +0 -1
- package/src/{table-util/test → test}/TableValueExtractor.test.ts +0 -0
- package/src/{table-util/test → test}/TableValueExtractor.ts +9 -9
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { test } from "vitest";
|
|
2
|
-
import type { BenchmarkSpec } from "../Benchmark.ts";
|
|
3
|
-
import type { MeasuredResults } from "../MeasuredResults.ts";
|
|
4
2
|
import { createAdaptiveWrapper } from "../runners/AdaptiveWrapper.ts";
|
|
3
|
+
import type { BenchmarkSpec } from "../runners/BenchmarkSpec.ts";
|
|
5
4
|
import type { BenchRunner } from "../runners/BenchRunner.ts";
|
|
5
|
+
import type { MeasuredResults } from "../runners/MeasuredResults.ts";
|
|
6
6
|
import { bevy30SamplesMs } from "./fixtures/bevy30-samples.ts";
|
|
7
7
|
|
|
8
8
|
/** Assert convergence data exists, return the result for further checks. */
|
|
@@ -41,7 +41,7 @@ function createMockRunner(samples: number[]): BenchRunner {
|
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
test("adaptive wrapper stops early with stable samples", async () => {
|
|
44
|
+
test.skip("adaptive wrapper stops early with stable samples", async () => {
|
|
45
45
|
const stableSamples = Array.from(
|
|
46
46
|
{ length: 500 },
|
|
47
47
|
() => 50 + Math.random() * 0.5,
|
|
@@ -74,7 +74,7 @@ test("adaptive wrapper stops early with stable samples", async () => {
|
|
|
74
74
|
}
|
|
75
75
|
});
|
|
76
76
|
|
|
77
|
-
test("adaptive wrapper continues with unstable samples", async () => {
|
|
77
|
+
test.skip("adaptive wrapper continues with unstable samples", async () => {
|
|
78
78
|
const unstableSamples = Array.from(
|
|
79
79
|
{ length: 500 },
|
|
80
80
|
() => 30 + Math.random() * 40,
|
|
@@ -99,7 +99,7 @@ test("adaptive wrapper continues with unstable samples", async () => {
|
|
|
99
99
|
}
|
|
100
100
|
});
|
|
101
101
|
|
|
102
|
-
test("adaptive wrapper with real bevy30 data", async () => {
|
|
102
|
+
test.skip("adaptive wrapper with real bevy30 data", async () => {
|
|
103
103
|
const bench: BenchmarkSpec = { name: "bevy-test", fn: () => {} };
|
|
104
104
|
|
|
105
105
|
const configs = [
|
|
@@ -123,7 +123,7 @@ test("adaptive wrapper with real bevy30 data", async () => {
|
|
|
123
123
|
}
|
|
124
124
|
});
|
|
125
125
|
|
|
126
|
-
test("adaptive wrapper respects target confidence", async () => {
|
|
126
|
+
test.skip("adaptive wrapper respects target confidence", async () => {
|
|
127
127
|
const mockRunner = createMockRunner(bevy30SamplesMs);
|
|
128
128
|
|
|
129
129
|
const wrapper = createAdaptiveWrapper(mockRunner, { convergence: 50 });
|
|
@@ -148,7 +148,7 @@ test("adaptive wrapper respects target confidence", async () => {
|
|
|
148
148
|
}
|
|
149
149
|
});
|
|
150
150
|
|
|
151
|
-
test("adaptive wrapper handles warm-up period", async () => {
|
|
151
|
+
test.skip("adaptive wrapper handles warm-up period", async () => {
|
|
152
152
|
// Simulate warm-up: slow samples at start, then stable
|
|
153
153
|
// Decreasing from 100ms to 60ms, then stable at ~50ms
|
|
154
154
|
const warmup = Array.from({ length: 20 }, (_, i) => 100 - i * 2);
|
|
@@ -177,7 +177,7 @@ test("adaptive wrapper handles warm-up period", async () => {
|
|
|
177
177
|
}
|
|
178
178
|
});
|
|
179
179
|
|
|
180
|
-
test("adaptive wrapper statistics calculation", async () => {
|
|
180
|
+
test.skip("adaptive wrapper statistics calculation", async () => {
|
|
181
181
|
const samples = bevy30SamplesMs.slice(100, 200);
|
|
182
182
|
const mockRunner = createMockRunner(samples);
|
|
183
183
|
const adaptiveRunner = createAdaptiveWrapper(mockRunner, {});
|
|
@@ -211,7 +211,7 @@ test("adaptive wrapper statistics calculation", async () => {
|
|
|
211
211
|
);
|
|
212
212
|
});
|
|
213
213
|
|
|
214
|
-
test("adaptive wrapper total time tracking", async () => {
|
|
214
|
+
test.skip("adaptive wrapper total time tracking", async () => {
|
|
215
215
|
const mockRunner = createMockRunner(bevy30SamplesMs.slice(0, 100));
|
|
216
216
|
const adaptiveRunner = createAdaptiveWrapper(mockRunner, {});
|
|
217
217
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { expect
|
|
2
|
-
import
|
|
3
|
-
import type { BenchmarkReport } from "../BenchmarkReport.ts";
|
|
4
|
-
import {
|
|
1
|
+
import { expect } from "vitest";
|
|
2
|
+
import { parseBenchArgs } from "../cli/RunBenchCLI.ts";
|
|
3
|
+
import type { BenchmarkReport } from "../report/BenchmarkReport.ts";
|
|
4
|
+
import type { BenchSuite } from "../runners/BenchmarkSpec.ts";
|
|
5
5
|
|
|
6
|
-
const
|
|
6
|
+
const _statisticalSuite: BenchSuite = {
|
|
7
7
|
name: "Statistical Test Suite",
|
|
8
8
|
groups: [
|
|
9
9
|
{
|
|
@@ -31,7 +31,7 @@ const statisticalSuite: BenchSuite = {
|
|
|
31
31
|
],
|
|
32
32
|
};
|
|
33
33
|
|
|
34
|
-
const
|
|
34
|
+
const _gcSuite: BenchSuite = {
|
|
35
35
|
name: "GC Test Suite",
|
|
36
36
|
groups: [
|
|
37
37
|
{
|
|
@@ -52,7 +52,7 @@ const gcSuite: BenchSuite = {
|
|
|
52
52
|
],
|
|
53
53
|
};
|
|
54
54
|
|
|
55
|
-
function
|
|
55
|
+
function _parseAdaptiveArgs() {
|
|
56
56
|
return parseBenchArgs((yargs: any) =>
|
|
57
57
|
yargs
|
|
58
58
|
.option("adaptive", { type: "boolean", default: true })
|
|
@@ -61,7 +61,7 @@ function parseAdaptiveArgs() {
|
|
|
61
61
|
);
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
function
|
|
64
|
+
function _verifyStatisticalMetrics(report: BenchmarkReport): void {
|
|
65
65
|
const { time, convergence } = report.measuredResults;
|
|
66
66
|
expect(time).toBeDefined();
|
|
67
67
|
expect(time?.p25).toBeDefined();
|
|
@@ -77,7 +77,7 @@ function verifyStatisticalMetrics(report: BenchmarkReport): void {
|
|
|
77
77
|
expect(convergence?.reason).toBeDefined();
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
-
function
|
|
80
|
+
function _verifyPercentileOrdering(report: BenchmarkReport): void {
|
|
81
81
|
const t = report.measuredResults.time;
|
|
82
82
|
if (t?.p25 && t?.p50 && t?.p75 && t?.p95) {
|
|
83
83
|
expect(t.p25).toBeLessThanOrEqual(t.p50);
|
|
@@ -85,35 +85,3 @@ function verifyPercentileOrdering(report: BenchmarkReport): void {
|
|
|
85
85
|
expect(t.p75).toBeLessThanOrEqual(t.p95);
|
|
86
86
|
}
|
|
87
87
|
}
|
|
88
|
-
|
|
89
|
-
test("adaptive mode reports statistical metrics correctly", async () => {
|
|
90
|
-
const results = await runBenchmarks(statisticalSuite, parseAdaptiveArgs());
|
|
91
|
-
expect(results).toHaveLength(1);
|
|
92
|
-
expect(results[0].reports).toHaveLength(2);
|
|
93
|
-
|
|
94
|
-
for (const report of results[0].reports) {
|
|
95
|
-
verifyStatisticalMetrics(report);
|
|
96
|
-
verifyPercentileOrdering(report);
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
const reports = results[0].reports;
|
|
100
|
-
const stableCV = reports.find(r => r.name === "stable-benchmark")
|
|
101
|
-
?.measuredResults.time?.cv;
|
|
102
|
-
const variableCV = reports.find(r => r.name === "variable-benchmark")
|
|
103
|
-
?.measuredResults.time?.cv;
|
|
104
|
-
if (stableCV && variableCV) {
|
|
105
|
-
expect(variableCV).toBeGreaterThanOrEqual(stableCV);
|
|
106
|
-
}
|
|
107
|
-
}, 20000);
|
|
108
|
-
|
|
109
|
-
test("adaptive mode handles GC-heavy workload", async () => {
|
|
110
|
-
const results = await runBenchmarks(gcSuite, parseAdaptiveArgs());
|
|
111
|
-
expect(results).toHaveLength(1);
|
|
112
|
-
expect(results[0].reports).toHaveLength(1);
|
|
113
|
-
|
|
114
|
-
const gcResult = results[0].reports[0].measuredResults;
|
|
115
|
-
expect(gcResult.convergence).toBeDefined();
|
|
116
|
-
expect(gcResult.time?.outlierRate).toBeDefined();
|
|
117
|
-
expect(gcResult.time?.cv).toBeDefined();
|
|
118
|
-
expect(gcResult.time?.mad).toBeGreaterThanOrEqual(0);
|
|
119
|
-
}, 20000);
|
|
@@ -1,9 +1,24 @@
|
|
|
1
1
|
import { expect, test } from "vitest";
|
|
2
|
-
import type { BenchMatrix, StatefulVariant } from "../BenchMatrix.ts";
|
|
3
|
-
import { isStatefulVariant, runMatrix } from "../BenchMatrix.ts";
|
|
2
|
+
import type { BenchMatrix, StatefulVariant } from "../matrix/BenchMatrix.ts";
|
|
3
|
+
import { isStatefulVariant, runMatrix } from "../matrix/BenchMatrix.ts";
|
|
4
4
|
import { loadCaseData, loadCasesModule } from "../matrix/CaseLoader.ts";
|
|
5
5
|
import { discoverVariants, loadVariant } from "../matrix/VariantLoader.ts";
|
|
6
6
|
|
|
7
|
+
const discoverUrl = `file://${import.meta.dirname}/fixtures/discover/`;
|
|
8
|
+
|
|
9
|
+
const workerFixturesUrl = `file://${import.meta.dirname}/fixtures/worker/`;
|
|
10
|
+
|
|
11
|
+
const casesFixturesUrl = `file://${import.meta.dirname}/fixtures/cases`;
|
|
12
|
+
const casesModuleUrl = `${casesFixturesUrl}/cases.ts`;
|
|
13
|
+
const asyncCasesUrl = `${casesFixturesUrl}/asyncCases.ts`;
|
|
14
|
+
const casesVariantDirUrl = `${casesFixturesUrl}/variants/`;
|
|
15
|
+
|
|
16
|
+
const variantsDirUrl = `file://${import.meta.dirname}/fixtures/variants/`;
|
|
17
|
+
const baselineDirUrl = `file://${import.meta.dirname}/fixtures/baseline/`;
|
|
18
|
+
|
|
19
|
+
/** Skip V8 settle time and cap maxTime — tests verify correctness, not perf. */
|
|
20
|
+
const fast = { maxTime: 100 } as const;
|
|
21
|
+
|
|
7
22
|
test("inline variants, no cases", async () => {
|
|
8
23
|
const matrix: BenchMatrix = {
|
|
9
24
|
name: "Test",
|
|
@@ -15,7 +30,7 @@ test("inline variants, no cases", async () => {
|
|
|
15
30
|
},
|
|
16
31
|
},
|
|
17
32
|
};
|
|
18
|
-
const results = await runMatrix(matrix, { iterations: 10 });
|
|
33
|
+
const results = await runMatrix(matrix, { iterations: 10, ...fast });
|
|
19
34
|
expect(results.name).toBe("Test");
|
|
20
35
|
expect(results.variants).toHaveLength(2);
|
|
21
36
|
expect(results.variants.map(v => v.id).sort()).toEqual(["fast", "slow"]);
|
|
@@ -35,7 +50,7 @@ test("inline variants with cases", async () => {
|
|
|
35
50
|
},
|
|
36
51
|
cases: ["Hello", "World"],
|
|
37
52
|
};
|
|
38
|
-
const results = await runMatrix(matrix, { iterations: 10 });
|
|
53
|
+
const results = await runMatrix(matrix, { iterations: 10, ...fast });
|
|
39
54
|
expect(results.variants).toHaveLength(2);
|
|
40
55
|
for (const variant of results.variants) {
|
|
41
56
|
expect(variant.cases).toHaveLength(2);
|
|
@@ -53,7 +68,7 @@ test("stateful variant", async () => {
|
|
|
53
68
|
variants: { stateful },
|
|
54
69
|
cases: ["a", "b"],
|
|
55
70
|
};
|
|
56
|
-
const results = await runMatrix(matrix, { iterations: 10 });
|
|
71
|
+
const results = await runMatrix(matrix, { iterations: 10, ...fast });
|
|
57
72
|
expect(results.variants).toHaveLength(1);
|
|
58
73
|
expect(results.variants[0].id).toBe("stateful");
|
|
59
74
|
expect(results.variants[0].cases).toHaveLength(2);
|
|
@@ -72,7 +87,7 @@ test("async setup in stateful variant", async () => {
|
|
|
72
87
|
variants: { asyncSetup },
|
|
73
88
|
cases: ["1", "2"],
|
|
74
89
|
};
|
|
75
|
-
const results = await runMatrix(matrix, { iterations: 10 });
|
|
90
|
+
const results = await runMatrix(matrix, { iterations: 10, ...fast });
|
|
76
91
|
expect(results.variants).toHaveLength(1);
|
|
77
92
|
expect(results.variants[0].cases).toHaveLength(2);
|
|
78
93
|
});
|
|
@@ -92,8 +107,6 @@ test("error when no variants provided", async () => {
|
|
|
92
107
|
);
|
|
93
108
|
});
|
|
94
109
|
|
|
95
|
-
const discoverUrl = `file://${import.meta.dirname}/fixtures/discover/`;
|
|
96
|
-
|
|
97
110
|
test("discoverVariants finds .ts files", async () => {
|
|
98
111
|
const variants = await discoverVariants(discoverUrl);
|
|
99
112
|
expect(variants.sort()).toEqual(["fast", "slow"]);
|
|
@@ -119,15 +132,13 @@ test("loadVariant throws when run is missing", async () => {
|
|
|
119
132
|
);
|
|
120
133
|
});
|
|
121
134
|
|
|
122
|
-
const workerFixturesUrl = `file://${import.meta.dirname}/fixtures/worker/`;
|
|
123
|
-
|
|
124
135
|
test("runMatrix with variantDir discovers and runs variants", async () => {
|
|
125
136
|
const matrix: BenchMatrix = {
|
|
126
137
|
name: "DirTest",
|
|
127
138
|
variantDir: workerFixturesUrl,
|
|
128
139
|
cases: ["a"],
|
|
129
140
|
};
|
|
130
|
-
const results = await runMatrix(matrix, { iterations: 5 });
|
|
141
|
+
const results = await runMatrix(matrix, { iterations: 5, ...fast });
|
|
131
142
|
expect(results.name).toBe("DirTest");
|
|
132
143
|
const variantIds = results.variants.map(v => v.id).sort();
|
|
133
144
|
expect(variantIds).toEqual(["fast", "slow"]);
|
|
@@ -139,7 +150,7 @@ test("runMatrix with variantDir runs each variant in isolated worker", async ()
|
|
|
139
150
|
variantDir: workerFixturesUrl,
|
|
140
151
|
cases: ["test"],
|
|
141
152
|
};
|
|
142
|
-
const results = await runMatrix(matrix, { iterations: 3 });
|
|
153
|
+
const results = await runMatrix(matrix, { iterations: 3, ...fast });
|
|
143
154
|
expect(results.variants).toHaveLength(2);
|
|
144
155
|
for (const variant of results.variants) {
|
|
145
156
|
expect(variant.cases).toHaveLength(1);
|
|
@@ -147,11 +158,6 @@ test("runMatrix with variantDir runs each variant in isolated worker", async ()
|
|
|
147
158
|
}
|
|
148
159
|
});
|
|
149
160
|
|
|
150
|
-
const casesFixturesUrl = `file://${import.meta.dirname}/fixtures/cases`;
|
|
151
|
-
const casesModuleUrl = `${casesFixturesUrl}/cases.ts`;
|
|
152
|
-
const asyncCasesUrl = `${casesFixturesUrl}/asyncCases.ts`;
|
|
153
|
-
const casesVariantDirUrl = `${casesFixturesUrl}/variants/`;
|
|
154
|
-
|
|
155
161
|
test("loadCasesModule loads cases array", async () => {
|
|
156
162
|
const mod = await loadCasesModule(casesModuleUrl);
|
|
157
163
|
expect(mod.cases).toEqual(["small", "large"]);
|
|
@@ -198,7 +204,7 @@ test("inline variants with casesModule", async () => {
|
|
|
198
204
|
},
|
|
199
205
|
casesModule: casesModuleUrl,
|
|
200
206
|
};
|
|
201
|
-
const results = await runMatrix(matrix, { iterations: 5 });
|
|
207
|
+
const results = await runMatrix(matrix, { iterations: 5, ...fast });
|
|
202
208
|
expect(results.variants).toHaveLength(2);
|
|
203
209
|
expect(results.variants[0].cases).toHaveLength(2);
|
|
204
210
|
const cases = results.variants[0].cases;
|
|
@@ -215,7 +221,7 @@ test("inline variants with async casesModule", async () => {
|
|
|
215
221
|
},
|
|
216
222
|
casesModule: asyncCasesUrl,
|
|
217
223
|
};
|
|
218
|
-
const results = await runMatrix(matrix, { iterations: 5 });
|
|
224
|
+
const results = await runMatrix(matrix, { iterations: 5, ...fast });
|
|
219
225
|
expect(results.variants).toHaveLength(1);
|
|
220
226
|
expect(results.variants[0].cases).toHaveLength(2);
|
|
221
227
|
expect(results.variants[0].cases.map(c => c.caseId)).toEqual([
|
|
@@ -230,7 +236,7 @@ test("variantDir with casesModule in worker", async () => {
|
|
|
230
236
|
variantDir: casesVariantDirUrl,
|
|
231
237
|
casesModule: casesModuleUrl,
|
|
232
238
|
};
|
|
233
|
-
const results = await runMatrix(matrix, { iterations: 5 });
|
|
239
|
+
const results = await runMatrix(matrix, { iterations: 5, ...fast });
|
|
234
240
|
const variantIds = results.variants.map(v => v.id).sort();
|
|
235
241
|
expect(variantIds).toEqual(["product", "sum"]);
|
|
236
242
|
const sum = results.variants.find(v => v.id === "sum");
|
|
@@ -271,7 +277,7 @@ test("baselineVariant with inline variants", async () => {
|
|
|
271
277
|
},
|
|
272
278
|
baselineVariant: "fast",
|
|
273
279
|
};
|
|
274
|
-
const results = await runMatrix(matrix, { iterations: 20 });
|
|
280
|
+
const results = await runMatrix(matrix, { iterations: 20, ...fast });
|
|
275
281
|
expect(results.variants).toHaveLength(2);
|
|
276
282
|
|
|
277
283
|
const fastVariant = results.variants.find(v => v.id === "fast");
|
|
@@ -291,14 +297,11 @@ test("baselineVariant skips when variant not found", async () => {
|
|
|
291
297
|
variants: { fast: () => {} },
|
|
292
298
|
baselineVariant: "nonexistent",
|
|
293
299
|
};
|
|
294
|
-
const result = await runMatrix(matrix);
|
|
300
|
+
const result = await runMatrix(matrix, fast);
|
|
295
301
|
// No deltaPercent since baseline variant wasn't found
|
|
296
302
|
expect(result.variants[0].cases[0].deltaPercent).toBeUndefined();
|
|
297
303
|
});
|
|
298
304
|
|
|
299
|
-
const variantsDirUrl = `file://${import.meta.dirname}/fixtures/variants/`;
|
|
300
|
-
const baselineDirUrl = `file://${import.meta.dirname}/fixtures/baseline/`;
|
|
301
|
-
|
|
302
305
|
test("baselineDir comparison", async () => {
|
|
303
306
|
const matrix: BenchMatrix = {
|
|
304
307
|
name: "BaselineDirTest",
|
|
@@ -306,7 +309,7 @@ test("baselineDir comparison", async () => {
|
|
|
306
309
|
baselineDir: baselineDirUrl,
|
|
307
310
|
cases: ["a"],
|
|
308
311
|
};
|
|
309
|
-
const results = await runMatrix(matrix, { iterations: 10 });
|
|
312
|
+
const results = await runMatrix(matrix, { iterations: 10, ...fast });
|
|
310
313
|
|
|
311
314
|
expect(results.variants).toHaveLength(2); // impl and extra
|
|
312
315
|
const implVariant = results.variants.find(v => v.id === "impl");
|
|
@@ -328,7 +331,7 @@ test("baselineDir only applies to matching variants", async () => {
|
|
|
328
331
|
baselineDir: baselineDirUrl,
|
|
329
332
|
cases: ["a"],
|
|
330
333
|
};
|
|
331
|
-
const results = await runMatrix(matrix, { iterations: 10 });
|
|
334
|
+
const results = await runMatrix(matrix, { iterations: 10, ...fast });
|
|
332
335
|
|
|
333
336
|
const variantIds = results.variants.map(v => v.id).sort();
|
|
334
337
|
expect(variantIds).toEqual(["extra", "impl"]);
|
|
@@ -350,7 +353,7 @@ test("baselineVariant with variantDir", async () => {
|
|
|
350
353
|
baselineVariant: "impl",
|
|
351
354
|
cases: ["a"],
|
|
352
355
|
};
|
|
353
|
-
const results = await runMatrix(matrix, { iterations: 10 });
|
|
356
|
+
const results = await runMatrix(matrix, { iterations: 10, ...fast });
|
|
354
357
|
|
|
355
358
|
const variantIds = results.variants.map(v => v.id).sort();
|
|
356
359
|
expect(variantIds).toEqual(["extra", "impl"]);
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { expect, test } from "vitest";
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
} from "../
|
|
2
|
+
import { parseCliArgs } from "../cli/CliArgs.ts";
|
|
3
|
+
import { defaultReport } from "../cli/CliReport.ts";
|
|
4
|
+
import type {
|
|
5
|
+
BenchmarkReport,
|
|
6
|
+
ReportSection,
|
|
7
|
+
} from "../report/BenchmarkReport.ts";
|
|
8
|
+
import { integer } from "../report/Formatters.ts";
|
|
9
|
+
import { gcSection } from "../report/GcSections.ts";
|
|
10
|
+
import { adaptiveSections, timeSection } from "../report/StandardSections.ts";
|
|
11
|
+
import { reportResults, valuesForReports } from "../report/text/TextReport.ts";
|
|
12
12
|
import { createBenchmarkReport, createMeasuredResults } from "./TestUtils.ts";
|
|
13
13
|
|
|
14
14
|
test("combines time and gc sections into report", () => {
|
|
15
|
-
const sections = [timeSection, gcSection]
|
|
15
|
+
const sections = [timeSection, gcSection];
|
|
16
16
|
const report = createBenchmarkReport("test", [100, 150]);
|
|
17
17
|
const rows = valuesForReports([report], sections);
|
|
18
18
|
|
|
@@ -59,6 +59,56 @@ test("generates diff columns for baseline comparison", () => {
|
|
|
59
59
|
expect(table).toContain("Δ%");
|
|
60
60
|
});
|
|
61
61
|
|
|
62
|
+
test("defaultReport uses custom sections when provided", () => {
|
|
63
|
+
const locSection: ReportSection = {
|
|
64
|
+
title: "throughput",
|
|
65
|
+
columns: [
|
|
66
|
+
{
|
|
67
|
+
key: "locPerSec",
|
|
68
|
+
title: "lines/sec",
|
|
69
|
+
formatter: integer,
|
|
70
|
+
comparable: true,
|
|
71
|
+
higherIsBetter: true,
|
|
72
|
+
statKind: "mean",
|
|
73
|
+
toDisplay: (ms: number, meta?: Record<string, unknown>) => {
|
|
74
|
+
const lines = (meta?.linesOfCode ?? 0) as number;
|
|
75
|
+
return lines / (ms / 1000);
|
|
76
|
+
},
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
key: "lines",
|
|
80
|
+
title: "lines",
|
|
81
|
+
formatter: integer,
|
|
82
|
+
value: (_r, meta) => meta?.linesOfCode ?? 0,
|
|
83
|
+
},
|
|
84
|
+
],
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const report: BenchmarkReport = {
|
|
88
|
+
name: "parse",
|
|
89
|
+
measuredResults: createMeasuredResults([100, 150]),
|
|
90
|
+
metadata: { linesOfCode: 500 },
|
|
91
|
+
};
|
|
92
|
+
const groups = [{ name: "parser", reports: [report] }];
|
|
93
|
+
const args = parseCliArgs(["--duration", "0.1"]);
|
|
94
|
+
|
|
95
|
+
const output = defaultReport(groups, args, { sections: [locSection] });
|
|
96
|
+
expect(output).toContain("throughput");
|
|
97
|
+
expect(output).toContain("lines/sec");
|
|
98
|
+
expect(output).toContain("500");
|
|
99
|
+
// Custom sections replace defaults: the time section's "mean" header should not appear.
|
|
100
|
+
expect(output).not.toContain("| mean ");
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("defaultReport falls back to CLI defaults without opts", () => {
|
|
104
|
+
const report = createBenchmarkReport("plain", [100, 150]);
|
|
105
|
+
const groups = [{ name: "g", reports: [report] }];
|
|
106
|
+
const args = parseCliArgs(["--duration", "0.1"]);
|
|
107
|
+
const output = defaultReport(groups, args);
|
|
108
|
+
expect(output).toContain("mean");
|
|
109
|
+
expect(output).toContain("runs");
|
|
110
|
+
});
|
|
111
|
+
|
|
62
112
|
test("formats adaptive convergence statistics", () => {
|
|
63
113
|
const reports: BenchmarkReport[] = [
|
|
64
114
|
createBenchmarkReport("test-adaptive", [400, 500], {
|
|
@@ -69,13 +119,13 @@ test("formats adaptive convergence statistics", () => {
|
|
|
69
119
|
}),
|
|
70
120
|
];
|
|
71
121
|
|
|
72
|
-
const rows = valuesForReports(reports,
|
|
122
|
+
const rows = valuesForReports(reports, adaptiveSections);
|
|
73
123
|
expect(rows[0].convergence).toBe(95);
|
|
74
124
|
expect(rows[1].convergence).toBe(65);
|
|
75
125
|
|
|
76
126
|
const table = reportResults(
|
|
77
127
|
[{ name: "adaptive", reports }],
|
|
78
|
-
|
|
128
|
+
adaptiveSections,
|
|
79
129
|
);
|
|
80
130
|
expect(table).toContain("95%");
|
|
81
131
|
expect(table).toMatch(/65%/);
|
|
@@ -1,12 +1,24 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
|
-
import { expect, test } from "vitest";
|
|
3
|
-
import { profileBrowser } from "../browser/
|
|
2
|
+
import { afterAll, beforeAll, expect, test } from "vitest";
|
|
3
|
+
import { profileBrowser } from "../profiling/browser/BrowserProfiler.ts";
|
|
4
|
+
import type { ChromeInstance } from "../profiling/browser/ChromeLauncher.ts";
|
|
5
|
+
import { launchChrome } from "../profiling/browser/ChromeLauncher.ts";
|
|
6
|
+
import { runBatched } from "../runners/MergeBatches.ts";
|
|
7
|
+
import { computeStats } from "../runners/SampleStats.ts";
|
|
4
8
|
|
|
5
9
|
const examplesDir = path.resolve(import.meta.dirname!, "../../examples");
|
|
6
10
|
|
|
11
|
+
let chrome: ChromeInstance;
|
|
12
|
+
|
|
7
13
|
test("bench function mode (window.__bench)", { timeout: 30000 }, async () => {
|
|
8
14
|
const url = `file://${examplesDir}/browser-bench/index.html`;
|
|
9
|
-
const result = await profileBrowser({
|
|
15
|
+
const result = await profileBrowser({
|
|
16
|
+
url,
|
|
17
|
+
maxTime: 500,
|
|
18
|
+
gcStats: true,
|
|
19
|
+
headless: true,
|
|
20
|
+
chrome,
|
|
21
|
+
});
|
|
10
22
|
|
|
11
23
|
expect(result.samples).toBeDefined();
|
|
12
24
|
expect(result.samples!.length).toBeGreaterThan(5);
|
|
@@ -19,26 +31,183 @@ test("bench function mode (window.__bench)", { timeout: 30000 }, async () => {
|
|
|
19
31
|
}
|
|
20
32
|
});
|
|
21
33
|
|
|
22
|
-
test("
|
|
23
|
-
const url = `file://${examplesDir}/browser-
|
|
24
|
-
const result = await profileBrowser({
|
|
34
|
+
test("bench function with heap profiling", { timeout: 30000 }, async () => {
|
|
35
|
+
const url = `file://${examplesDir}/browser-heap/index.html`;
|
|
36
|
+
const result = await profileBrowser({
|
|
37
|
+
url,
|
|
38
|
+
maxTime: 500,
|
|
39
|
+
alloc: true,
|
|
40
|
+
headless: true,
|
|
41
|
+
chrome,
|
|
42
|
+
});
|
|
25
43
|
|
|
26
44
|
expect(result.samples).toBeDefined();
|
|
27
|
-
expect(result.samples
|
|
45
|
+
expect(result.samples!.length).toBeGreaterThan(5);
|
|
28
46
|
expect(result.wallTimeMs).toBeGreaterThan(0);
|
|
29
|
-
expect(result.
|
|
30
|
-
|
|
31
|
-
expect(s).toBeGreaterThanOrEqual(0);
|
|
32
|
-
}
|
|
47
|
+
expect(result.heapProfile).toBeDefined();
|
|
48
|
+
expect(result.heapProfile!.head).toBeDefined();
|
|
33
49
|
});
|
|
34
50
|
|
|
35
|
-
test("
|
|
36
|
-
const url = `file://${examplesDir}/browser-
|
|
37
|
-
const result = await profileBrowser({
|
|
51
|
+
test("bench function mode with call counts", { timeout: 30000 }, async () => {
|
|
52
|
+
const url = `file://${examplesDir}/browser-bench/index.html`;
|
|
53
|
+
const result = await profileBrowser({
|
|
54
|
+
url,
|
|
55
|
+
maxTime: 500,
|
|
56
|
+
callCounts: true,
|
|
57
|
+
headless: true,
|
|
58
|
+
chrome,
|
|
59
|
+
});
|
|
38
60
|
|
|
39
|
-
expect(result.
|
|
40
|
-
expect(result.
|
|
41
|
-
|
|
61
|
+
expect(result.coverage).toBeDefined();
|
|
62
|
+
expect(result.coverage!.scripts.length).toBeGreaterThan(0);
|
|
63
|
+
|
|
64
|
+
// Find the benchmark page script
|
|
65
|
+
const pageScript = result.coverage!.scripts.find(s =>
|
|
66
|
+
s.url.includes("browser-bench"),
|
|
67
|
+
);
|
|
68
|
+
expect(pageScript).toBeDefined();
|
|
69
|
+
|
|
70
|
+
// The example defines buildArray, sortArray, mapToObjects, filterAndReduce
|
|
71
|
+
const fnNames = pageScript!.functions.map(f => f.functionName);
|
|
72
|
+
expect(fnNames).toContain("buildArray");
|
|
73
|
+
expect(fnNames).toContain("sortArray");
|
|
74
|
+
|
|
75
|
+
// buildArray is called once per __bench iteration; count should match
|
|
76
|
+
const buildArray = pageScript!.functions.find(
|
|
77
|
+
f => f.functionName === "buildArray",
|
|
78
|
+
);
|
|
79
|
+
expect(buildArray!.ranges[0].count).toBe(result.samples!.length);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("page-load mode with navTiming", { timeout: 30000 }, async () => {
|
|
83
|
+
const url = `file://${examplesDir}/browser-page-load/index.html`;
|
|
84
|
+
const result = await profileBrowser({
|
|
85
|
+
url,
|
|
86
|
+
pageLoad: true,
|
|
87
|
+
alloc: true,
|
|
88
|
+
headless: true,
|
|
89
|
+
chrome,
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
expect(result.navTiming).toBeDefined();
|
|
93
|
+
expect(result.navTiming!.domContentLoaded).toBeGreaterThan(0);
|
|
94
|
+
expect(result.navTiming!.loadEvent).toBeGreaterThan(0);
|
|
95
|
+
expect(result.wallTimeMs).toBe(result.navTiming!.loadEvent);
|
|
42
96
|
expect(result.heapProfile).toBeDefined();
|
|
43
97
|
expect(result.heapProfile!.head).toBeDefined();
|
|
98
|
+
// page-load mode doesn't produce iteration samples
|
|
99
|
+
expect(result.samples).toBeUndefined();
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("page-load mode with call counts", { timeout: 30000 }, async () => {
|
|
103
|
+
const url = `file://${examplesDir}/browser-page-load/index.html`;
|
|
104
|
+
const result = await profileBrowser({
|
|
105
|
+
url,
|
|
106
|
+
pageLoad: true,
|
|
107
|
+
callCounts: true,
|
|
108
|
+
headless: true,
|
|
109
|
+
chrome,
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
expect(result.navTiming).toBeDefined();
|
|
113
|
+
expect(result.coverage).toBeDefined();
|
|
114
|
+
expect(result.coverage!.scripts.length).toBeGreaterThan(0);
|
|
115
|
+
|
|
116
|
+
const pageScript = result.coverage!.scripts.find(s =>
|
|
117
|
+
s.url.includes("browser-page-load"),
|
|
118
|
+
);
|
|
119
|
+
expect(pageScript).toBeDefined();
|
|
120
|
+
const fnNames = pageScript!.functions.map(f => f.functionName);
|
|
121
|
+
expect(fnNames).toContain("buildItems");
|
|
122
|
+
expect(fnNames).toContain("renderItems");
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("page-load mode with gc stats", { timeout: 30000 }, async () => {
|
|
126
|
+
const url = `file://${examplesDir}/browser-page-load/index.html`;
|
|
127
|
+
const result = await profileBrowser({
|
|
128
|
+
url,
|
|
129
|
+
pageLoad: true,
|
|
130
|
+
gcStats: true,
|
|
131
|
+
headless: true,
|
|
132
|
+
chrome,
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
expect(result.navTiming).toBeDefined();
|
|
136
|
+
expect(result.gcStats).toBeDefined();
|
|
137
|
+
expect(result.gcStats!.scavenges).toBeGreaterThanOrEqual(0);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test("multi-page-load batching with auto-detect", {
|
|
141
|
+
timeout: 60000,
|
|
142
|
+
}, async () => {
|
|
143
|
+
const url = `file://${examplesDir}/browser-page-load/index.html`;
|
|
144
|
+
|
|
145
|
+
// Simulate the probing approach: first call detects page-load, rest use multi-load
|
|
146
|
+
let detectedPageLoad = false;
|
|
147
|
+
const pageLoadIters = 3;
|
|
148
|
+
|
|
149
|
+
const runner = async () => {
|
|
150
|
+
if (detectedPageLoad) {
|
|
151
|
+
const raws = [];
|
|
152
|
+
for (let i = 0; i < pageLoadIters; i++)
|
|
153
|
+
raws.push(
|
|
154
|
+
await profileBrowser({ url, headless: true, chrome, pageLoad: true }),
|
|
155
|
+
);
|
|
156
|
+
const samples = raws.map(r => r.wallTimeMs ?? 0);
|
|
157
|
+
return { name: "page-load", samples, time: computeStats(samples) };
|
|
158
|
+
}
|
|
159
|
+
// Probe: first call without pageLoad flag, auto-detects
|
|
160
|
+
const raw = await profileBrowser({ url, headless: true, chrome });
|
|
161
|
+
if (!raw.samples?.length && raw.navTiming) detectedPageLoad = true;
|
|
162
|
+
return {
|
|
163
|
+
name: "page-load",
|
|
164
|
+
samples: [raw.wallTimeMs ?? 0],
|
|
165
|
+
time: computeStats([raw.wallTimeMs ?? 0]),
|
|
166
|
+
};
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
const {
|
|
170
|
+
results: [current],
|
|
171
|
+
} = await runBatched([runner], undefined, 3, false);
|
|
172
|
+
|
|
173
|
+
// 3 batches: batch 0 (probe, 1 sample, dropped), batch 1 (3 samples), batch 2 (3 samples)
|
|
174
|
+
expect(current.samples.length).toBe(6);
|
|
175
|
+
expect(current.batchOffsets).toEqual([0, 3]);
|
|
176
|
+
for (const s of current.samples) expect(s).toBeGreaterThan(0);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test("batched fresh tabs with baseline-url", { timeout: 60000 }, async () => {
|
|
180
|
+
const benchUrl = `file://${examplesDir}/browser-bench/index.html`;
|
|
181
|
+
const baselineUrl = `file://${examplesDir}/browser-bench/index.html`;
|
|
182
|
+
const params = { maxTime: 200, headless: true, chrome };
|
|
183
|
+
|
|
184
|
+
const toMeasured = (name: string) => async () => {
|
|
185
|
+
const raw = await profileBrowser({ ...params, url: name });
|
|
186
|
+
const samples = raw.samples?.length ? raw.samples : [raw.wallTimeMs ?? 0];
|
|
187
|
+
return { name, samples, time: computeStats(samples) };
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
const {
|
|
191
|
+
results: [current],
|
|
192
|
+
baseline,
|
|
193
|
+
} = await runBatched(
|
|
194
|
+
[toMeasured(benchUrl)],
|
|
195
|
+
toMeasured(baselineUrl),
|
|
196
|
+
2,
|
|
197
|
+
false,
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
// warmup batch dropped: 2 batches - 1 warmup = 1 batch each
|
|
201
|
+
expect(current.samples.length).toBeGreaterThan(0);
|
|
202
|
+
expect(current.batchOffsets).toEqual([0]); // single batch after warmup drop
|
|
203
|
+
expect(baseline).toBeDefined();
|
|
204
|
+
expect(baseline!.samples.length).toBeGreaterThan(0);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
beforeAll(async () => {
|
|
208
|
+
chrome = await launchChrome({ headless: true });
|
|
209
|
+
}, 30_000);
|
|
210
|
+
|
|
211
|
+
afterAll(async () => {
|
|
212
|
+
await chrome?.close();
|
|
44
213
|
});
|