benchforge 0.1.11 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/LICENSE +20 -0
  2. package/README.md +99 -294
  3. package/bin/benchforge +1 -2
  4. package/dist/AnalyzeArchive-8NCJhmhS.mjs +145 -0
  5. package/dist/AnalyzeArchive-8NCJhmhS.mjs.map +1 -0
  6. package/dist/BenchMatrix-BZVrBB_h.mjs +1050 -0
  7. package/dist/BenchMatrix-BZVrBB_h.mjs.map +1 -0
  8. package/dist/{BenchRunner-BzyUfiyB.d.mts → BenchRunner-DglX1NOn.d.mts} +119 -66
  9. package/dist/CoverageSampler-D5T9DRqe.mjs +27 -0
  10. package/dist/CoverageSampler-D5T9DRqe.mjs.map +1 -0
  11. package/dist/Formatters-BWj3d4sv.mjs +95 -0
  12. package/dist/Formatters-BWj3d4sv.mjs.map +1 -0
  13. package/dist/{HeapSampler-B8dtKHn1.mjs → HeapSampler-Dq-hpXem.mjs} +4 -4
  14. package/dist/HeapSampler-Dq-hpXem.mjs.map +1 -0
  15. package/dist/RunBenchCLI-C17DrJz8.mjs +3075 -0
  16. package/dist/RunBenchCLI-C17DrJz8.mjs.map +1 -0
  17. package/dist/StatisticalUtils-BD92crgM.mjs +255 -0
  18. package/dist/StatisticalUtils-BD92crgM.mjs.map +1 -0
  19. package/dist/TimeSampler-Ds8n7l2B.mjs +29 -0
  20. package/dist/TimeSampler-Ds8n7l2B.mjs.map +1 -0
  21. package/dist/ViewerServer-BJhdnxlN.mjs +639 -0
  22. package/dist/ViewerServer-BJhdnxlN.mjs.map +1 -0
  23. package/dist/ViewerServer-CuMNdNBz.mjs +2 -0
  24. package/dist/bin/benchforge.mjs +4 -5
  25. package/dist/bin/benchforge.mjs.map +1 -1
  26. package/dist/index.d.mts +711 -558
  27. package/dist/index.mjs +98 -3
  28. package/dist/index.mjs.map +1 -0
  29. package/dist/runners/WorkerScript.d.mts +12 -4
  30. package/dist/runners/WorkerScript.mjs +77 -105
  31. package/dist/runners/WorkerScript.mjs.map +1 -1
  32. package/dist/viewer/assets/CIPlot-BkOvMoMa.js +1 -0
  33. package/dist/viewer/assets/HistogramKde-CmSyUFY0.js +1 -0
  34. package/dist/viewer/assets/LegendUtils-BJpbn_jr.js +55 -0
  35. package/dist/viewer/assets/SampleTimeSeries-C4VBhXr3.js +1 -0
  36. package/dist/viewer/assets/index-Br9bp_cX.js +153 -0
  37. package/dist/viewer/assets/index-NzXXe_CC.css +1 -0
  38. package/dist/viewer/index.html +19 -0
  39. package/dist/viewer/speedscope/LICENSE +21 -0
  40. package/dist/viewer/speedscope/SourceCodePro-Regular.ttf-ILST5JV6.woff2 +0 -0
  41. package/dist/viewer/speedscope/favicon-16x16-V2DMIAZS.js +2 -0
  42. package/dist/viewer/speedscope/favicon-16x16-V2DMIAZS.js.map +7 -0
  43. package/dist/viewer/speedscope/favicon-16x16-VSI62OPJ.png +0 -0
  44. package/dist/viewer/speedscope/favicon-32x32-3EB2YCUY.png +0 -0
  45. package/dist/viewer/speedscope/favicon-32x32-THY3JDJL.js +2 -0
  46. package/dist/viewer/speedscope/favicon-32x32-THY3JDJL.js.map +7 -0
  47. package/dist/viewer/speedscope/favicon-FOKUP5Y5.ico +0 -0
  48. package/dist/viewer/speedscope/favicon-M34RF7BI.js +2 -0
  49. package/dist/viewer/speedscope/favicon-M34RF7BI.js.map +7 -0
  50. package/dist/viewer/speedscope/file-format-schema.json +274 -0
  51. package/dist/viewer/speedscope/index.html +19 -0
  52. package/dist/viewer/speedscope/jfrview_bg-BLJXNNQB.wasm +0 -0
  53. package/dist/viewer/speedscope/perf-vertx-stacks-01-collapsed-all-ZNUIGAJL.txt +199 -0
  54. package/dist/viewer/speedscope/release.txt +3 -0
  55. package/dist/viewer/speedscope/source-code-pro.LICENSE.md +93 -0
  56. package/dist/viewer/speedscope/speedscope-GHPHNKXC.css +2 -0
  57. package/dist/viewer/speedscope/speedscope-GHPHNKXC.css.map +7 -0
  58. package/dist/viewer/speedscope/speedscope-QZFMJ7VP.js +212 -0
  59. package/dist/viewer/speedscope/speedscope-QZFMJ7VP.js.map +7 -0
  60. package/package.json +52 -27
  61. package/src/bin/benchforge.ts +2 -2
  62. package/src/cli/AnalyzeArchive.ts +232 -0
  63. package/src/cli/BrowserBench.ts +322 -0
  64. package/src/cli/CliArgs.ts +164 -51
  65. package/src/cli/CliExport.ts +179 -0
  66. package/src/cli/CliOptions.ts +147 -0
  67. package/src/cli/CliReport.ts +197 -0
  68. package/src/cli/FilterBenchmarks.ts +18 -30
  69. package/src/cli/RunBenchCLI.ts +132 -866
  70. package/src/cli/SuiteRunner.ts +160 -0
  71. package/src/cli/ViewerServer.ts +282 -0
  72. package/src/export/AllocExport.ts +121 -0
  73. package/src/export/ArchiveExport.ts +146 -0
  74. package/src/export/ArchiveFormat.ts +50 -0
  75. package/src/export/CoverageExport.ts +148 -0
  76. package/src/export/EditorUri.ts +10 -0
  77. package/src/export/PerfettoExport.ts +64 -99
  78. package/src/export/SpeedscopeTypes.ts +98 -0
  79. package/src/export/TimeExport.ts +115 -0
  80. package/src/index.ts +86 -67
  81. package/src/matrix/BenchMatrix.ts +230 -0
  82. package/src/matrix/CaseLoader.ts +8 -6
  83. package/src/matrix/MatrixDirRunner.ts +153 -0
  84. package/src/matrix/MatrixFilter.ts +49 -47
  85. package/src/matrix/MatrixInlineRunner.ts +50 -0
  86. package/src/matrix/MatrixReport.ts +90 -250
  87. package/src/matrix/VariantLoader.ts +5 -5
  88. package/src/profiling/browser/BenchLoop.ts +51 -0
  89. package/src/profiling/browser/BrowserCDP.ts +133 -0
  90. package/src/profiling/browser/BrowserGcStats.ts +33 -0
  91. package/src/profiling/browser/BrowserProfiler.ts +160 -0
  92. package/src/profiling/browser/CdpClient.ts +82 -0
  93. package/src/profiling/browser/CdpPage.ts +138 -0
  94. package/src/profiling/browser/ChromeLauncher.ts +158 -0
  95. package/src/profiling/browser/ChromeTraceEvent.ts +28 -0
  96. package/src/profiling/browser/PageLoadMode.ts +61 -0
  97. package/src/profiling/node/CoverageSampler.ts +27 -0
  98. package/src/profiling/node/CoverageTypes.ts +23 -0
  99. package/src/profiling/node/HeapSampleReport.ts +261 -0
  100. package/src/{heap-sample → profiling/node}/HeapSampler.ts +1 -2
  101. package/src/{heap-sample → profiling/node}/ResolvedProfile.ts +18 -9
  102. package/src/profiling/node/TimeSampler.ts +57 -0
  103. package/src/report/BenchmarkReport.ts +146 -0
  104. package/src/report/Colors.ts +9 -0
  105. package/src/report/Formatters.ts +110 -0
  106. package/src/report/GcSections.ts +151 -0
  107. package/src/{GitUtils.ts → report/GitUtils.ts} +18 -19
  108. package/src/report/HtmlReport.ts +223 -0
  109. package/src/report/ParseStats.ts +73 -0
  110. package/src/report/StandardSections.ts +147 -0
  111. package/src/report/ViewerSections.ts +286 -0
  112. package/src/report/text/TableReport.ts +253 -0
  113. package/src/report/text/TextReport.ts +123 -0
  114. package/src/runners/AdaptiveWrapper.ts +116 -236
  115. package/src/runners/BenchRunner.ts +20 -15
  116. package/src/{Benchmark.ts → runners/BenchmarkSpec.ts} +5 -6
  117. package/src/runners/CreateRunner.ts +5 -7
  118. package/src/runners/GcStats.ts +47 -50
  119. package/src/{MeasuredResults.ts → runners/MeasuredResults.ts} +43 -37
  120. package/src/runners/MergeBatches.ts +123 -0
  121. package/src/{NodeGC.ts → runners/NodeGC.ts} +2 -3
  122. package/src/runners/RunnerOrchestrator.ts +127 -243
  123. package/src/runners/RunnerUtils.ts +75 -1
  124. package/src/runners/SampleStats.ts +100 -0
  125. package/src/runners/TimingRunner.ts +244 -0
  126. package/src/runners/TimingUtils.ts +3 -2
  127. package/src/runners/WorkerScript.ts +135 -151
  128. package/src/stats/BootstrapDifference.ts +282 -0
  129. package/src/{PermutationTest.ts → stats/PermutationTest.ts} +8 -17
  130. package/src/stats/StatisticalUtils.ts +445 -0
  131. package/src/{tests → test}/AdaptiveConvergence.test.ts +10 -10
  132. package/src/test/AdaptiveRunner.test.ts +39 -41
  133. package/src/{tests → test}/AdaptiveSampling.test.ts +9 -9
  134. package/src/test/AdaptiveStatistics.integration.ts +2 -2
  135. package/src/{tests → test}/BenchMatrix.test.ts +19 -16
  136. package/src/test/BenchmarkReport.test.ts +63 -13
  137. package/src/test/BrowserBench.e2e.test.ts +186 -17
  138. package/src/test/BrowserBench.test.ts +10 -5
  139. package/src/test/BuildTimeSection.test.ts +130 -0
  140. package/src/test/CapSamples.test.ts +82 -0
  141. package/src/test/CoverageExport.test.ts +115 -0
  142. package/src/test/CoverageSampler.test.ts +33 -0
  143. package/src/test/HeapAttribution.test.ts +14 -14
  144. package/src/{tests → test}/MatrixFilter.test.ts +1 -1
  145. package/src/{tests → test}/MatrixReport.test.ts +1 -1
  146. package/src/test/PermutationTest.test.ts +1 -1
  147. package/src/{tests → test}/RealDataValidation.test.ts +6 -6
  148. package/src/test/RunBenchCLI.test.ts +39 -38
  149. package/src/test/RunnerOrchestrator.test.ts +12 -12
  150. package/src/test/StatisticalUtils.test.ts +48 -12
  151. package/src/{table-util/test → test}/TableReport.test.ts +2 -2
  152. package/src/test/TestUtils.ts +12 -7
  153. package/src/test/TimeExport.test.ts +139 -0
  154. package/src/test/TimeSampler.test.ts +37 -0
  155. package/src/test/ViewerLive.e2e.test.ts +159 -0
  156. package/src/test/ViewerStatic.static.e2e.test.ts +137 -0
  157. package/src/{tests → test}/fixtures/baseline/impl.ts +1 -1
  158. package/src/{tests → test}/fixtures/bevy30-samples.ts +3 -1
  159. package/src/test/fixtures/cases/asyncCases.ts +9 -0
  160. package/src/{tests → test}/fixtures/cases/cases.ts +5 -2
  161. package/src/test/fixtures/cases/variants/product.ts +2 -0
  162. package/src/test/fixtures/cases/variants/sum.ts +2 -0
  163. package/src/test/fixtures/discover/fast.ts +1 -0
  164. package/src/{tests → test}/fixtures/discover/slow.ts +1 -1
  165. package/src/test/fixtures/invalid/bad.ts +1 -0
  166. package/src/test/fixtures/loader/fast.ts +1 -0
  167. package/src/{tests → test}/fixtures/loader/slow.ts +1 -1
  168. package/src/test/fixtures/loader/stateful.ts +2 -0
  169. package/src/test/fixtures/stateful/stateful.ts +2 -0
  170. package/src/test/fixtures/variants/extra.ts +1 -0
  171. package/src/test/fixtures/variants/impl.ts +1 -0
  172. package/src/test/fixtures/worker/fast.ts +1 -0
  173. package/src/{tests → test}/fixtures/worker/slow.ts +1 -1
  174. package/src/viewer/DateFormat.ts +30 -0
  175. package/src/viewer/Helpers.ts +23 -0
  176. package/src/viewer/LineData.ts +120 -0
  177. package/src/viewer/Providers.ts +191 -0
  178. package/src/viewer/ReportData.ts +123 -0
  179. package/src/viewer/State.ts +49 -0
  180. package/src/viewer/Theme.ts +15 -0
  181. package/src/viewer/components/App.tsx +73 -0
  182. package/src/viewer/components/DropZone.tsx +71 -0
  183. package/src/viewer/components/LazyPlot.ts +33 -0
  184. package/src/viewer/components/SamplesPanel.tsx +214 -0
  185. package/src/viewer/components/Shell.tsx +26 -0
  186. package/src/viewer/components/SourcePanel.tsx +216 -0
  187. package/src/viewer/components/SummaryPanel.tsx +332 -0
  188. package/src/viewer/components/TabBar.tsx +131 -0
  189. package/src/viewer/components/TabContent.tsx +46 -0
  190. package/src/viewer/components/ThemeToggle.tsx +50 -0
  191. package/src/viewer/index.html +20 -0
  192. package/src/viewer/main.tsx +4 -0
  193. package/src/viewer/plots/CIPlot.ts +313 -0
  194. package/src/{html/browser → viewer/plots}/HistogramKde.ts +33 -38
  195. package/src/viewer/plots/LegendUtils.ts +134 -0
  196. package/src/viewer/plots/PlotTypes.ts +85 -0
  197. package/src/viewer/plots/RenderPlots.ts +230 -0
  198. package/src/viewer/plots/SampleTimeSeries.ts +306 -0
  199. package/src/viewer/plots/SvgHelpers.ts +136 -0
  200. package/src/viewer/plots/TimeSeriesMarks.ts +319 -0
  201. package/src/viewer/report.css +427 -0
  202. package/src/viewer/shell.css +357 -0
  203. package/src/viewer/tsconfig.json +11 -0
  204. package/dist/BrowserHeapSampler-B6asLKWQ.mjs +0 -202
  205. package/dist/BrowserHeapSampler-B6asLKWQ.mjs.map +0 -1
  206. package/dist/GcStats-wX7Xyblu.mjs +0 -77
  207. package/dist/GcStats-wX7Xyblu.mjs.map +0 -1
  208. package/dist/HeapSampler-B8dtKHn1.mjs.map +0 -1
  209. package/dist/TimingUtils-DwOwkc8G.mjs +0 -597
  210. package/dist/TimingUtils-DwOwkc8G.mjs.map +0 -1
  211. package/dist/browser/index.js +0 -914
  212. package/dist/src-B-DDaCa9.mjs +0 -3108
  213. package/dist/src-B-DDaCa9.mjs.map +0 -1
  214. package/src/BenchMatrix.ts +0 -380
  215. package/src/BenchmarkReport.ts +0 -161
  216. package/src/HtmlDataPrep.ts +0 -148
  217. package/src/StandardSections.ts +0 -261
  218. package/src/StatisticalUtils.ts +0 -175
  219. package/src/TypeUtil.ts +0 -8
  220. package/src/browser/BrowserGcStats.ts +0 -44
  221. package/src/browser/BrowserHeapSampler.ts +0 -271
  222. package/src/export/JsonExport.ts +0 -103
  223. package/src/export/JsonFormat.ts +0 -91
  224. package/src/export/SpeedscopeExport.ts +0 -202
  225. package/src/heap-sample/HeapSampleReport.ts +0 -269
  226. package/src/html/HtmlReport.ts +0 -131
  227. package/src/html/HtmlTemplate.ts +0 -284
  228. package/src/html/Types.ts +0 -88
  229. package/src/html/browser/CIPlot.ts +0 -287
  230. package/src/html/browser/LegendUtils.ts +0 -163
  231. package/src/html/browser/RenderPlots.ts +0 -263
  232. package/src/html/browser/SampleTimeSeries.ts +0 -389
  233. package/src/html/browser/Types.ts +0 -96
  234. package/src/html/browser/index.ts +0 -1
  235. package/src/html/index.ts +0 -17
  236. package/src/runners/BasicRunner.ts +0 -364
  237. package/src/table-util/ConvergenceFormatters.ts +0 -19
  238. package/src/table-util/Formatters.ts +0 -157
  239. package/src/table-util/README.md +0 -70
  240. package/src/table-util/TableReport.ts +0 -293
  241. package/src/tests/fixtures/cases/asyncCases.ts +0 -7
  242. package/src/tests/fixtures/cases/variants/product.ts +0 -2
  243. package/src/tests/fixtures/cases/variants/sum.ts +0 -2
  244. package/src/tests/fixtures/discover/fast.ts +0 -1
  245. package/src/tests/fixtures/invalid/bad.ts +0 -1
  246. package/src/tests/fixtures/loader/fast.ts +0 -1
  247. package/src/tests/fixtures/loader/stateful.ts +0 -2
  248. package/src/tests/fixtures/stateful/stateful.ts +0 -2
  249. package/src/tests/fixtures/variants/extra.ts +0 -1
  250. package/src/tests/fixtures/variants/impl.ts +0 -1
  251. package/src/tests/fixtures/worker/fast.ts +0 -1
  252. /package/src/{table-util/test → test}/TableValueExtractor.test.ts +0 -0
  253. /package/src/{table-util/test → test}/TableValueExtractor.ts +0 -0
@@ -1,8 +1,8 @@
1
1
  import { test } from "vitest";
2
- import type { BenchmarkSpec } from "../Benchmark.ts";
3
- import type { MeasuredResults } from "../MeasuredResults.ts";
4
2
  import { createAdaptiveWrapper } from "../runners/AdaptiveWrapper.ts";
3
+ import type { BenchmarkSpec } from "../runners/BenchmarkSpec.ts";
5
4
  import type { BenchRunner } from "../runners/BenchRunner.ts";
5
+ import type { MeasuredResults } from "../runners/MeasuredResults.ts";
6
6
  import { bevy30SamplesMs } from "./fixtures/bevy30-samples.ts";
7
7
 
8
8
  /** Assert convergence data exists, return the result for further checks. */
@@ -41,7 +41,7 @@ function createMockRunner(samples: number[]): BenchRunner {
41
41
  };
42
42
  }
43
43
 
44
- test("adaptive wrapper stops early with stable samples", async () => {
44
+ test.skip("adaptive wrapper stops early with stable samples", async () => {
45
45
  const stableSamples = Array.from(
46
46
  { length: 500 },
47
47
  () => 50 + Math.random() * 0.5,
@@ -74,7 +74,7 @@ test("adaptive wrapper stops early with stable samples", async () => {
74
74
  }
75
75
  });
76
76
 
77
- test("adaptive wrapper continues with unstable samples", async () => {
77
+ test.skip("adaptive wrapper continues with unstable samples", async () => {
78
78
  const unstableSamples = Array.from(
79
79
  { length: 500 },
80
80
  () => 30 + Math.random() * 40,
@@ -99,7 +99,7 @@ test("adaptive wrapper continues with unstable samples", async () => {
99
99
  }
100
100
  });
101
101
 
102
- test("adaptive wrapper with real bevy30 data", async () => {
102
+ test.skip("adaptive wrapper with real bevy30 data", async () => {
103
103
  const bench: BenchmarkSpec = { name: "bevy-test", fn: () => {} };
104
104
 
105
105
  const configs = [
@@ -123,7 +123,7 @@ test("adaptive wrapper with real bevy30 data", async () => {
123
123
  }
124
124
  });
125
125
 
126
- test("adaptive wrapper respects target confidence", async () => {
126
+ test.skip("adaptive wrapper respects target confidence", async () => {
127
127
  const mockRunner = createMockRunner(bevy30SamplesMs);
128
128
 
129
129
  const wrapper = createAdaptiveWrapper(mockRunner, { convergence: 50 });
@@ -148,7 +148,7 @@ test("adaptive wrapper respects target confidence", async () => {
148
148
  }
149
149
  });
150
150
 
151
- test("adaptive wrapper handles warm-up period", async () => {
151
+ test.skip("adaptive wrapper handles warm-up period", async () => {
152
152
  // Simulate warm-up: slow samples at start, then stable
153
153
  // Decreasing from 100ms to 60ms, then stable at ~50ms
154
154
  const warmup = Array.from({ length: 20 }, (_, i) => 100 - i * 2);
@@ -177,7 +177,7 @@ test("adaptive wrapper handles warm-up period", async () => {
177
177
  }
178
178
  });
179
179
 
180
- test("adaptive wrapper statistics calculation", async () => {
180
+ test.skip("adaptive wrapper statistics calculation", async () => {
181
181
  const samples = bevy30SamplesMs.slice(100, 200);
182
182
  const mockRunner = createMockRunner(samples);
183
183
  const adaptiveRunner = createAdaptiveWrapper(mockRunner, {});
@@ -211,7 +211,7 @@ test("adaptive wrapper statistics calculation", async () => {
211
211
  );
212
212
  });
213
213
 
214
- test("adaptive wrapper total time tracking", async () => {
214
+ test.skip("adaptive wrapper total time tracking", async () => {
215
215
  const mockRunner = createMockRunner(bevy30SamplesMs.slice(0, 100));
216
216
  const adaptiveRunner = createAdaptiveWrapper(mockRunner, {});
217
217
 
@@ -1,7 +1,7 @@
1
1
  import { expect } from "vitest";
2
- import type { BenchSuite } from "../Benchmark.ts";
3
- import type { BenchmarkReport } from "../BenchmarkReport.ts";
4
2
  import { parseBenchArgs } from "../cli/RunBenchCLI.ts";
3
+ import type { BenchmarkReport } from "../report/BenchmarkReport.ts";
4
+ import type { BenchSuite } from "../runners/BenchmarkSpec.ts";
5
5
 
6
6
  const _statisticalSuite: BenchSuite = {
7
7
  name: "Statistical Test Suite",
@@ -1,6 +1,6 @@
1
1
  import { expect, test } from "vitest";
2
- import type { BenchMatrix, StatefulVariant } from "../BenchMatrix.ts";
3
- import { isStatefulVariant, runMatrix } from "../BenchMatrix.ts";
2
+ import type { BenchMatrix, StatefulVariant } from "../matrix/BenchMatrix.ts";
3
+ import { isStatefulVariant, runMatrix } from "../matrix/BenchMatrix.ts";
4
4
  import { loadCaseData, loadCasesModule } from "../matrix/CaseLoader.ts";
5
5
  import { discoverVariants, loadVariant } from "../matrix/VariantLoader.ts";
6
6
 
@@ -16,6 +16,9 @@ const casesVariantDirUrl = `${casesFixturesUrl}/variants/`;
16
16
  const variantsDirUrl = `file://${import.meta.dirname}/fixtures/variants/`;
17
17
  const baselineDirUrl = `file://${import.meta.dirname}/fixtures/baseline/`;
18
18
 
19
+ /** Skip V8 settle time and cap maxTime — tests verify correctness, not perf. */
20
+ const fast = { maxTime: 100 } as const;
21
+
19
22
  test("inline variants, no cases", async () => {
20
23
  const matrix: BenchMatrix = {
21
24
  name: "Test",
@@ -27,7 +30,7 @@ test("inline variants, no cases", async () => {
27
30
  },
28
31
  },
29
32
  };
30
- const results = await runMatrix(matrix, { iterations: 10 });
33
+ const results = await runMatrix(matrix, { iterations: 10, ...fast });
31
34
  expect(results.name).toBe("Test");
32
35
  expect(results.variants).toHaveLength(2);
33
36
  expect(results.variants.map(v => v.id).sort()).toEqual(["fast", "slow"]);
@@ -47,7 +50,7 @@ test("inline variants with cases", async () => {
47
50
  },
48
51
  cases: ["Hello", "World"],
49
52
  };
50
- const results = await runMatrix(matrix, { iterations: 10 });
53
+ const results = await runMatrix(matrix, { iterations: 10, ...fast });
51
54
  expect(results.variants).toHaveLength(2);
52
55
  for (const variant of results.variants) {
53
56
  expect(variant.cases).toHaveLength(2);
@@ -65,7 +68,7 @@ test("stateful variant", async () => {
65
68
  variants: { stateful },
66
69
  cases: ["a", "b"],
67
70
  };
68
- const results = await runMatrix(matrix, { iterations: 10 });
71
+ const results = await runMatrix(matrix, { iterations: 10, ...fast });
69
72
  expect(results.variants).toHaveLength(1);
70
73
  expect(results.variants[0].id).toBe("stateful");
71
74
  expect(results.variants[0].cases).toHaveLength(2);
@@ -84,7 +87,7 @@ test("async setup in stateful variant", async () => {
84
87
  variants: { asyncSetup },
85
88
  cases: ["1", "2"],
86
89
  };
87
- const results = await runMatrix(matrix, { iterations: 10 });
90
+ const results = await runMatrix(matrix, { iterations: 10, ...fast });
88
91
  expect(results.variants).toHaveLength(1);
89
92
  expect(results.variants[0].cases).toHaveLength(2);
90
93
  });
@@ -135,7 +138,7 @@ test("runMatrix with variantDir discovers and runs variants", async () => {
135
138
  variantDir: workerFixturesUrl,
136
139
  cases: ["a"],
137
140
  };
138
- const results = await runMatrix(matrix, { iterations: 5 });
141
+ const results = await runMatrix(matrix, { iterations: 5, ...fast });
139
142
  expect(results.name).toBe("DirTest");
140
143
  const variantIds = results.variants.map(v => v.id).sort();
141
144
  expect(variantIds).toEqual(["fast", "slow"]);
@@ -147,7 +150,7 @@ test("runMatrix with variantDir runs each variant in isolated worker", async ()
147
150
  variantDir: workerFixturesUrl,
148
151
  cases: ["test"],
149
152
  };
150
- const results = await runMatrix(matrix, { iterations: 3 });
153
+ const results = await runMatrix(matrix, { iterations: 3, ...fast });
151
154
  expect(results.variants).toHaveLength(2);
152
155
  for (const variant of results.variants) {
153
156
  expect(variant.cases).toHaveLength(1);
@@ -201,7 +204,7 @@ test("inline variants with casesModule", async () => {
201
204
  },
202
205
  casesModule: casesModuleUrl,
203
206
  };
204
- const results = await runMatrix(matrix, { iterations: 5 });
207
+ const results = await runMatrix(matrix, { iterations: 5, ...fast });
205
208
  expect(results.variants).toHaveLength(2);
206
209
  expect(results.variants[0].cases).toHaveLength(2);
207
210
  const cases = results.variants[0].cases;
@@ -218,7 +221,7 @@ test("inline variants with async casesModule", async () => {
218
221
  },
219
222
  casesModule: asyncCasesUrl,
220
223
  };
221
- const results = await runMatrix(matrix, { iterations: 5 });
224
+ const results = await runMatrix(matrix, { iterations: 5, ...fast });
222
225
  expect(results.variants).toHaveLength(1);
223
226
  expect(results.variants[0].cases).toHaveLength(2);
224
227
  expect(results.variants[0].cases.map(c => c.caseId)).toEqual([
@@ -233,7 +236,7 @@ test("variantDir with casesModule in worker", async () => {
233
236
  variantDir: casesVariantDirUrl,
234
237
  casesModule: casesModuleUrl,
235
238
  };
236
- const results = await runMatrix(matrix, { iterations: 5 });
239
+ const results = await runMatrix(matrix, { iterations: 5, ...fast });
237
240
  const variantIds = results.variants.map(v => v.id).sort();
238
241
  expect(variantIds).toEqual(["product", "sum"]);
239
242
  const sum = results.variants.find(v => v.id === "sum");
@@ -274,7 +277,7 @@ test("baselineVariant with inline variants", async () => {
274
277
  },
275
278
  baselineVariant: "fast",
276
279
  };
277
- const results = await runMatrix(matrix, { iterations: 20 });
280
+ const results = await runMatrix(matrix, { iterations: 20, ...fast });
278
281
  expect(results.variants).toHaveLength(2);
279
282
 
280
283
  const fastVariant = results.variants.find(v => v.id === "fast");
@@ -294,7 +297,7 @@ test("baselineVariant skips when variant not found", async () => {
294
297
  variants: { fast: () => {} },
295
298
  baselineVariant: "nonexistent",
296
299
  };
297
- const result = await runMatrix(matrix);
300
+ const result = await runMatrix(matrix, fast);
298
301
  // No deltaPercent since baseline variant wasn't found
299
302
  expect(result.variants[0].cases[0].deltaPercent).toBeUndefined();
300
303
  });
@@ -306,7 +309,7 @@ test("baselineDir comparison", async () => {
306
309
  baselineDir: baselineDirUrl,
307
310
  cases: ["a"],
308
311
  };
309
- const results = await runMatrix(matrix, { iterations: 10 });
312
+ const results = await runMatrix(matrix, { iterations: 10, ...fast });
310
313
 
311
314
  expect(results.variants).toHaveLength(2); // impl and extra
312
315
  const implVariant = results.variants.find(v => v.id === "impl");
@@ -328,7 +331,7 @@ test("baselineDir only applies to matching variants", async () => {
328
331
  baselineDir: baselineDirUrl,
329
332
  cases: ["a"],
330
333
  };
331
- const results = await runMatrix(matrix, { iterations: 10 });
334
+ const results = await runMatrix(matrix, { iterations: 10, ...fast });
332
335
 
333
336
  const variantIds = results.variants.map(v => v.id).sort();
334
337
  expect(variantIds).toEqual(["extra", "impl"]);
@@ -350,7 +353,7 @@ test("baselineVariant with variantDir", async () => {
350
353
  baselineVariant: "impl",
351
354
  cases: ["a"],
352
355
  };
353
- const results = await runMatrix(matrix, { iterations: 10 });
356
+ const results = await runMatrix(matrix, { iterations: 10, ...fast });
354
357
 
355
358
  const variantIds = results.variants.map(v => v.id).sort();
356
359
  expect(variantIds).toEqual(["extra", "impl"]);
@@ -1,18 +1,18 @@
1
1
  import { expect, test } from "vitest";
2
- import {
3
- type BenchmarkReport,
4
- reportResults,
5
- valuesForReports,
6
- } from "../BenchmarkReport.ts";
7
- import {
8
- adaptiveSection,
9
- gcSection,
10
- timeSection,
11
- } from "../StandardSections.ts";
2
+ import { parseCliArgs } from "../cli/CliArgs.ts";
3
+ import { defaultReport } from "../cli/CliReport.ts";
4
+ import type {
5
+ BenchmarkReport,
6
+ ReportSection,
7
+ } from "../report/BenchmarkReport.ts";
8
+ import { integer } from "../report/Formatters.ts";
9
+ import { gcSection } from "../report/GcSections.ts";
10
+ import { adaptiveSections, timeSection } from "../report/StandardSections.ts";
11
+ import { reportResults, valuesForReports } from "../report/text/TextReport.ts";
12
12
  import { createBenchmarkReport, createMeasuredResults } from "./TestUtils.ts";
13
13
 
14
14
  test("combines time and gc sections into report", () => {
15
- const sections = [timeSection, gcSection] as const;
15
+ const sections = [timeSection, gcSection];
16
16
  const report = createBenchmarkReport("test", [100, 150]);
17
17
  const rows = valuesForReports([report], sections);
18
18
 
@@ -59,6 +59,56 @@ test("generates diff columns for baseline comparison", () => {
59
59
  expect(table).toContain("Δ%");
60
60
  });
61
61
 
62
+ test("defaultReport uses custom sections when provided", () => {
63
+ const locSection: ReportSection = {
64
+ title: "throughput",
65
+ columns: [
66
+ {
67
+ key: "locPerSec",
68
+ title: "lines/sec",
69
+ formatter: integer,
70
+ comparable: true,
71
+ higherIsBetter: true,
72
+ statKind: "mean",
73
+ toDisplay: (ms: number, meta?: Record<string, unknown>) => {
74
+ const lines = (meta?.linesOfCode ?? 0) as number;
75
+ return lines / (ms / 1000);
76
+ },
77
+ },
78
+ {
79
+ key: "lines",
80
+ title: "lines",
81
+ formatter: integer,
82
+ value: (_r, meta) => meta?.linesOfCode ?? 0,
83
+ },
84
+ ],
85
+ };
86
+
87
+ const report: BenchmarkReport = {
88
+ name: "parse",
89
+ measuredResults: createMeasuredResults([100, 150]),
90
+ metadata: { linesOfCode: 500 },
91
+ };
92
+ const groups = [{ name: "parser", reports: [report] }];
93
+ const args = parseCliArgs(["--duration", "0.1"]);
94
+
95
+ const output = defaultReport(groups, args, { sections: [locSection] });
96
+ expect(output).toContain("throughput");
97
+ expect(output).toContain("lines/sec");
98
+ expect(output).toContain("500");
99
+ // Custom sections replace defaults: the time section's "mean" header should not appear.
100
+ expect(output).not.toContain("| mean ");
101
+ });
102
+
103
+ test("defaultReport falls back to CLI defaults without opts", () => {
104
+ const report = createBenchmarkReport("plain", [100, 150]);
105
+ const groups = [{ name: "g", reports: [report] }];
106
+ const args = parseCliArgs(["--duration", "0.1"]);
107
+ const output = defaultReport(groups, args);
108
+ expect(output).toContain("mean");
109
+ expect(output).toContain("runs");
110
+ });
111
+
62
112
  test("formats adaptive convergence statistics", () => {
63
113
  const reports: BenchmarkReport[] = [
64
114
  createBenchmarkReport("test-adaptive", [400, 500], {
@@ -69,13 +119,13 @@ test("formats adaptive convergence statistics", () => {
69
119
  }),
70
120
  ];
71
121
 
72
- const rows = valuesForReports(reports, [adaptiveSection]);
122
+ const rows = valuesForReports(reports, adaptiveSections);
73
123
  expect(rows[0].convergence).toBe(95);
74
124
  expect(rows[1].convergence).toBe(65);
75
125
 
76
126
  const table = reportResults(
77
127
  [{ name: "adaptive", reports }],
78
- [adaptiveSection],
128
+ adaptiveSections,
79
129
  );
80
130
  expect(table).toContain("95%");
81
131
  expect(table).toMatch(/65%/);
@@ -1,12 +1,24 @@
1
1
  import path from "node:path";
2
- import { expect, test } from "vitest";
3
- import { profileBrowser } from "../browser/BrowserHeapSampler.ts";
2
+ import { afterAll, beforeAll, expect, test } from "vitest";
3
+ import { profileBrowser } from "../profiling/browser/BrowserProfiler.ts";
4
+ import type { ChromeInstance } from "../profiling/browser/ChromeLauncher.ts";
5
+ import { launchChrome } from "../profiling/browser/ChromeLauncher.ts";
6
+ import { runBatched } from "../runners/MergeBatches.ts";
7
+ import { computeStats } from "../runners/SampleStats.ts";
4
8
 
5
9
  const examplesDir = path.resolve(import.meta.dirname!, "../../examples");
6
10
 
11
+ let chrome: ChromeInstance;
12
+
7
13
  test("bench function mode (window.__bench)", { timeout: 30000 }, async () => {
8
14
  const url = `file://${examplesDir}/browser-bench/index.html`;
9
- const result = await profileBrowser({ url, maxTime: 500, gcStats: true });
15
+ const result = await profileBrowser({
16
+ url,
17
+ maxTime: 500,
18
+ gcStats: true,
19
+ headless: true,
20
+ chrome,
21
+ });
10
22
 
11
23
  expect(result.samples).toBeDefined();
12
24
  expect(result.samples!.length).toBeGreaterThan(5);
@@ -19,26 +31,183 @@ test("bench function mode (window.__bench)", { timeout: 30000 }, async () => {
19
31
  }
20
32
  });
21
33
 
22
- test("lap mode with N laps", { timeout: 30000 }, async () => {
23
- const url = `file://${examplesDir}/browser-lap/index.html`;
24
- const result = await profileBrowser({ url, gcStats: true });
34
+ test("bench function with heap profiling", { timeout: 30000 }, async () => {
35
+ const url = `file://${examplesDir}/browser-heap/index.html`;
36
+ const result = await profileBrowser({
37
+ url,
38
+ maxTime: 500,
39
+ alloc: true,
40
+ headless: true,
41
+ chrome,
42
+ });
25
43
 
26
44
  expect(result.samples).toBeDefined();
27
- expect(result.samples!).toHaveLength(100);
45
+ expect(result.samples!.length).toBeGreaterThan(5);
28
46
  expect(result.wallTimeMs).toBeGreaterThan(0);
29
- expect(result.gcStats).toBeDefined();
30
- for (const s of result.samples!) {
31
- expect(s).toBeGreaterThanOrEqual(0);
32
- }
47
+ expect(result.heapProfile).toBeDefined();
48
+ expect(result.heapProfile!.head).toBeDefined();
33
49
  });
34
50
 
35
- test("lap mode 0 laps with heap profiling", { timeout: 30000 }, async () => {
36
- const url = `file://${examplesDir}/browser-heap/index.html`;
37
- const result = await profileBrowser({ url, heapSample: true });
51
+ test("bench function mode with call counts", { timeout: 30000 }, async () => {
52
+ const url = `file://${examplesDir}/browser-bench/index.html`;
53
+ const result = await profileBrowser({
54
+ url,
55
+ maxTime: 500,
56
+ callCounts: true,
57
+ headless: true,
58
+ chrome,
59
+ });
38
60
 
39
- expect(result.samples).toBeDefined();
40
- expect(result.samples!).toHaveLength(0);
41
- expect(result.wallTimeMs).toBeGreaterThan(0);
61
+ expect(result.coverage).toBeDefined();
62
+ expect(result.coverage!.scripts.length).toBeGreaterThan(0);
63
+
64
+ // Find the benchmark page script
65
+ const pageScript = result.coverage!.scripts.find(s =>
66
+ s.url.includes("browser-bench"),
67
+ );
68
+ expect(pageScript).toBeDefined();
69
+
70
+ // The example defines buildArray, sortArray, mapToObjects, filterAndReduce
71
+ const fnNames = pageScript!.functions.map(f => f.functionName);
72
+ expect(fnNames).toContain("buildArray");
73
+ expect(fnNames).toContain("sortArray");
74
+
75
+ // buildArray is called once per __bench iteration; count should match
76
+ const buildArray = pageScript!.functions.find(
77
+ f => f.functionName === "buildArray",
78
+ );
79
+ expect(buildArray!.ranges[0].count).toBe(result.samples!.length);
80
+ });
81
+
82
+ test("page-load mode with navTiming", { timeout: 30000 }, async () => {
83
+ const url = `file://${examplesDir}/browser-page-load/index.html`;
84
+ const result = await profileBrowser({
85
+ url,
86
+ pageLoad: true,
87
+ alloc: true,
88
+ headless: true,
89
+ chrome,
90
+ });
91
+
92
+ expect(result.navTiming).toBeDefined();
93
+ expect(result.navTiming!.domContentLoaded).toBeGreaterThan(0);
94
+ expect(result.navTiming!.loadEvent).toBeGreaterThan(0);
95
+ expect(result.wallTimeMs).toBe(result.navTiming!.loadEvent);
42
96
  expect(result.heapProfile).toBeDefined();
43
97
  expect(result.heapProfile!.head).toBeDefined();
98
+ // page-load mode doesn't produce iteration samples
99
+ expect(result.samples).toBeUndefined();
100
+ });
101
+
102
+ test("page-load mode with call counts", { timeout: 30000 }, async () => {
103
+ const url = `file://${examplesDir}/browser-page-load/index.html`;
104
+ const result = await profileBrowser({
105
+ url,
106
+ pageLoad: true,
107
+ callCounts: true,
108
+ headless: true,
109
+ chrome,
110
+ });
111
+
112
+ expect(result.navTiming).toBeDefined();
113
+ expect(result.coverage).toBeDefined();
114
+ expect(result.coverage!.scripts.length).toBeGreaterThan(0);
115
+
116
+ const pageScript = result.coverage!.scripts.find(s =>
117
+ s.url.includes("browser-page-load"),
118
+ );
119
+ expect(pageScript).toBeDefined();
120
+ const fnNames = pageScript!.functions.map(f => f.functionName);
121
+ expect(fnNames).toContain("buildItems");
122
+ expect(fnNames).toContain("renderItems");
123
+ });
124
+
125
+ test("page-load mode with gc stats", { timeout: 30000 }, async () => {
126
+ const url = `file://${examplesDir}/browser-page-load/index.html`;
127
+ const result = await profileBrowser({
128
+ url,
129
+ pageLoad: true,
130
+ gcStats: true,
131
+ headless: true,
132
+ chrome,
133
+ });
134
+
135
+ expect(result.navTiming).toBeDefined();
136
+ expect(result.gcStats).toBeDefined();
137
+ expect(result.gcStats!.scavenges).toBeGreaterThanOrEqual(0);
138
+ });
139
+
140
+ test("multi-page-load batching with auto-detect", {
141
+ timeout: 60000,
142
+ }, async () => {
143
+ const url = `file://${examplesDir}/browser-page-load/index.html`;
144
+
145
+ // Simulate the probing approach: first call detects page-load, rest use multi-load
146
+ let detectedPageLoad = false;
147
+ const pageLoadIters = 3;
148
+
149
+ const runner = async () => {
150
+ if (detectedPageLoad) {
151
+ const raws = [];
152
+ for (let i = 0; i < pageLoadIters; i++)
153
+ raws.push(
154
+ await profileBrowser({ url, headless: true, chrome, pageLoad: true }),
155
+ );
156
+ const samples = raws.map(r => r.wallTimeMs ?? 0);
157
+ return { name: "page-load", samples, time: computeStats(samples) };
158
+ }
159
+ // Probe: first call without pageLoad flag, auto-detects
160
+ const raw = await profileBrowser({ url, headless: true, chrome });
161
+ if (!raw.samples?.length && raw.navTiming) detectedPageLoad = true;
162
+ return {
163
+ name: "page-load",
164
+ samples: [raw.wallTimeMs ?? 0],
165
+ time: computeStats([raw.wallTimeMs ?? 0]),
166
+ };
167
+ };
168
+
169
+ const {
170
+ results: [current],
171
+ } = await runBatched([runner], undefined, 3, false);
172
+
173
+ // 3 batches: batch 0 (probe, 1 sample, dropped), batch 1 (3 samples), batch 2 (3 samples)
174
+ expect(current.samples.length).toBe(6);
175
+ expect(current.batchOffsets).toEqual([0, 3]);
176
+ for (const s of current.samples) expect(s).toBeGreaterThan(0);
177
+ });
178
+
179
+ test("batched fresh tabs with baseline-url", { timeout: 60000 }, async () => {
180
+ const benchUrl = `file://${examplesDir}/browser-bench/index.html`;
181
+ const baselineUrl = `file://${examplesDir}/browser-bench/index.html`;
182
+ const params = { maxTime: 200, headless: true, chrome };
183
+
184
+ const toMeasured = (name: string) => async () => {
185
+ const raw = await profileBrowser({ ...params, url: name });
186
+ const samples = raw.samples?.length ? raw.samples : [raw.wallTimeMs ?? 0];
187
+ return { name, samples, time: computeStats(samples) };
188
+ };
189
+
190
+ const {
191
+ results: [current],
192
+ baseline,
193
+ } = await runBatched(
194
+ [toMeasured(benchUrl)],
195
+ toMeasured(baselineUrl),
196
+ 2,
197
+ false,
198
+ );
199
+
200
+ // warmup batch dropped: 2 batches - 1 warmup = 1 batch each
201
+ expect(current.samples.length).toBeGreaterThan(0);
202
+ expect(current.batchOffsets).toEqual([0]); // single batch after warmup drop
203
+ expect(baseline).toBeDefined();
204
+ expect(baseline!.samples.length).toBeGreaterThan(0);
205
+ });
206
+
207
+ beforeAll(async () => {
208
+ chrome = await launchChrome({ headless: true });
209
+ }, 30_000);
210
+
211
+ afterAll(async () => {
212
+ await chrome?.close();
44
213
  });
@@ -2,8 +2,8 @@ import { expect, test } from "vitest";
2
2
  import {
3
3
  browserGcStats,
4
4
  parseGcTraceEvents,
5
- type TraceEvent,
6
- } from "../browser/BrowserGcStats.ts";
5
+ } from "../profiling/browser/BrowserGcStats.ts";
6
+ import type { TraceEvent } from "../profiling/browser/ChromeTraceEvent.ts";
7
7
 
8
8
  test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
9
9
  const events: TraceEvent[] = [
@@ -11,6 +11,7 @@ test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
11
11
  cat: "v8.gc",
12
12
  name: "MinorGC",
13
13
  ph: "X",
14
+ ts: 0,
14
15
  dur: 500,
15
16
  args: { usedHeapSizeBefore: 10000, usedHeapSizeAfter: 8000 },
16
17
  },
@@ -18,6 +19,7 @@ test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
18
19
  cat: "v8.gc",
19
20
  name: "MajorGC",
20
21
  ph: "X",
22
+ ts: 0,
21
23
  dur: 12000,
22
24
  args: { usedHeapSizeBefore: 50000, usedHeapSizeAfter: 30000 },
23
25
  },
@@ -38,9 +40,9 @@ test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
38
40
 
39
41
  test("parseGcTraceEvents ignores non-complete and non-GC events", () => {
40
42
  const events: TraceEvent[] = [
41
- { cat: "v8.gc", name: "MinorGC", ph: "B", dur: 500 }, // not complete
42
- { cat: "v8", name: "V8.Execute", ph: "X", dur: 100 }, // not GC
43
- { cat: "v8.gc", name: "MinorGC", ph: "X" }, // valid, missing dur/args
43
+ { cat: "v8.gc", name: "MinorGC", ph: "B", ts: 0, dur: 500 }, // not complete
44
+ { cat: "v8", name: "V8.Execute", ph: "X", ts: 0, dur: 100 }, // not GC
45
+ { cat: "v8.gc", name: "MinorGC", ph: "X", ts: 0 }, // valid, missing dur/args
44
46
  ];
45
47
  const parsed = parseGcTraceEvents(events);
46
48
  expect(parsed).toHaveLength(1);
@@ -53,6 +55,7 @@ test("browserGcStats aggregates trace events into GcStats", () => {
53
55
  cat: "v8.gc",
54
56
  name: "MinorGC",
55
57
  ph: "X",
58
+ ts: 0,
56
59
  dur: 300,
57
60
  args: { usedHeapSizeBefore: 5000, usedHeapSizeAfter: 3000 },
58
61
  },
@@ -60,6 +63,7 @@ test("browserGcStats aggregates trace events into GcStats", () => {
60
63
  cat: "v8.gc",
61
64
  name: "MinorGC",
62
65
  ph: "X",
66
+ ts: 0,
63
67
  dur: 200,
64
68
  args: { usedHeapSizeBefore: 6000, usedHeapSizeAfter: 4000 },
65
69
  },
@@ -67,6 +71,7 @@ test("browserGcStats aggregates trace events into GcStats", () => {
67
71
  cat: "v8.gc",
68
72
  name: "MajorGC",
69
73
  ph: "X",
74
+ ts: 0,
70
75
  dur: 8000,
71
76
  args: { usedHeapSizeBefore: 40000, usedHeapSizeAfter: 20000 },
72
77
  },