benchforge 0.1.11 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/LICENSE +20 -0
  2. package/README.md +99 -294
  3. package/bin/benchforge +1 -2
  4. package/dist/AnalyzeArchive-8NCJhmhS.mjs +145 -0
  5. package/dist/AnalyzeArchive-8NCJhmhS.mjs.map +1 -0
  6. package/dist/BenchMatrix-BZVrBB_h.mjs +1050 -0
  7. package/dist/BenchMatrix-BZVrBB_h.mjs.map +1 -0
  8. package/dist/{BenchRunner-BzyUfiyB.d.mts → BenchRunner-DglX1NOn.d.mts} +119 -66
  9. package/dist/CoverageSampler-D5T9DRqe.mjs +27 -0
  10. package/dist/CoverageSampler-D5T9DRqe.mjs.map +1 -0
  11. package/dist/Formatters-BWj3d4sv.mjs +95 -0
  12. package/dist/Formatters-BWj3d4sv.mjs.map +1 -0
  13. package/dist/{HeapSampler-B8dtKHn1.mjs → HeapSampler-Dq-hpXem.mjs} +4 -4
  14. package/dist/HeapSampler-Dq-hpXem.mjs.map +1 -0
  15. package/dist/RunBenchCLI-C17DrJz8.mjs +3075 -0
  16. package/dist/RunBenchCLI-C17DrJz8.mjs.map +1 -0
  17. package/dist/StatisticalUtils-BD92crgM.mjs +255 -0
  18. package/dist/StatisticalUtils-BD92crgM.mjs.map +1 -0
  19. package/dist/TimeSampler-Ds8n7l2B.mjs +29 -0
  20. package/dist/TimeSampler-Ds8n7l2B.mjs.map +1 -0
  21. package/dist/ViewerServer-BJhdnxlN.mjs +639 -0
  22. package/dist/ViewerServer-BJhdnxlN.mjs.map +1 -0
  23. package/dist/ViewerServer-CuMNdNBz.mjs +2 -0
  24. package/dist/bin/benchforge.mjs +4 -5
  25. package/dist/bin/benchforge.mjs.map +1 -1
  26. package/dist/index.d.mts +711 -558
  27. package/dist/index.mjs +98 -3
  28. package/dist/index.mjs.map +1 -0
  29. package/dist/runners/WorkerScript.d.mts +12 -4
  30. package/dist/runners/WorkerScript.mjs +77 -105
  31. package/dist/runners/WorkerScript.mjs.map +1 -1
  32. package/dist/viewer/assets/CIPlot-BkOvMoMa.js +1 -0
  33. package/dist/viewer/assets/HistogramKde-CmSyUFY0.js +1 -0
  34. package/dist/viewer/assets/LegendUtils-BJpbn_jr.js +55 -0
  35. package/dist/viewer/assets/SampleTimeSeries-C4VBhXr3.js +1 -0
  36. package/dist/viewer/assets/index-Br9bp_cX.js +153 -0
  37. package/dist/viewer/assets/index-NzXXe_CC.css +1 -0
  38. package/dist/viewer/index.html +19 -0
  39. package/dist/viewer/speedscope/LICENSE +21 -0
  40. package/dist/viewer/speedscope/SourceCodePro-Regular.ttf-ILST5JV6.woff2 +0 -0
  41. package/dist/viewer/speedscope/favicon-16x16-V2DMIAZS.js +2 -0
  42. package/dist/viewer/speedscope/favicon-16x16-V2DMIAZS.js.map +7 -0
  43. package/dist/viewer/speedscope/favicon-16x16-VSI62OPJ.png +0 -0
  44. package/dist/viewer/speedscope/favicon-32x32-3EB2YCUY.png +0 -0
  45. package/dist/viewer/speedscope/favicon-32x32-THY3JDJL.js +2 -0
  46. package/dist/viewer/speedscope/favicon-32x32-THY3JDJL.js.map +7 -0
  47. package/dist/viewer/speedscope/favicon-FOKUP5Y5.ico +0 -0
  48. package/dist/viewer/speedscope/favicon-M34RF7BI.js +2 -0
  49. package/dist/viewer/speedscope/favicon-M34RF7BI.js.map +7 -0
  50. package/dist/viewer/speedscope/file-format-schema.json +274 -0
  51. package/dist/viewer/speedscope/index.html +19 -0
  52. package/dist/viewer/speedscope/jfrview_bg-BLJXNNQB.wasm +0 -0
  53. package/dist/viewer/speedscope/perf-vertx-stacks-01-collapsed-all-ZNUIGAJL.txt +199 -0
  54. package/dist/viewer/speedscope/release.txt +3 -0
  55. package/dist/viewer/speedscope/source-code-pro.LICENSE.md +93 -0
  56. package/dist/viewer/speedscope/speedscope-GHPHNKXC.css +2 -0
  57. package/dist/viewer/speedscope/speedscope-GHPHNKXC.css.map +7 -0
  58. package/dist/viewer/speedscope/speedscope-QZFMJ7VP.js +212 -0
  59. package/dist/viewer/speedscope/speedscope-QZFMJ7VP.js.map +7 -0
  60. package/package.json +52 -27
  61. package/src/bin/benchforge.ts +2 -2
  62. package/src/cli/AnalyzeArchive.ts +232 -0
  63. package/src/cli/BrowserBench.ts +322 -0
  64. package/src/cli/CliArgs.ts +164 -51
  65. package/src/cli/CliExport.ts +179 -0
  66. package/src/cli/CliOptions.ts +147 -0
  67. package/src/cli/CliReport.ts +197 -0
  68. package/src/cli/FilterBenchmarks.ts +18 -30
  69. package/src/cli/RunBenchCLI.ts +132 -866
  70. package/src/cli/SuiteRunner.ts +160 -0
  71. package/src/cli/ViewerServer.ts +282 -0
  72. package/src/export/AllocExport.ts +121 -0
  73. package/src/export/ArchiveExport.ts +146 -0
  74. package/src/export/ArchiveFormat.ts +50 -0
  75. package/src/export/CoverageExport.ts +148 -0
  76. package/src/export/EditorUri.ts +10 -0
  77. package/src/export/PerfettoExport.ts +64 -99
  78. package/src/export/SpeedscopeTypes.ts +98 -0
  79. package/src/export/TimeExport.ts +115 -0
  80. package/src/index.ts +86 -67
  81. package/src/matrix/BenchMatrix.ts +230 -0
  82. package/src/matrix/CaseLoader.ts +8 -6
  83. package/src/matrix/MatrixDirRunner.ts +153 -0
  84. package/src/matrix/MatrixFilter.ts +49 -47
  85. package/src/matrix/MatrixInlineRunner.ts +50 -0
  86. package/src/matrix/MatrixReport.ts +90 -250
  87. package/src/matrix/VariantLoader.ts +5 -5
  88. package/src/profiling/browser/BenchLoop.ts +51 -0
  89. package/src/profiling/browser/BrowserCDP.ts +133 -0
  90. package/src/profiling/browser/BrowserGcStats.ts +33 -0
  91. package/src/profiling/browser/BrowserProfiler.ts +160 -0
  92. package/src/profiling/browser/CdpClient.ts +82 -0
  93. package/src/profiling/browser/CdpPage.ts +138 -0
  94. package/src/profiling/browser/ChromeLauncher.ts +158 -0
  95. package/src/profiling/browser/ChromeTraceEvent.ts +28 -0
  96. package/src/profiling/browser/PageLoadMode.ts +61 -0
  97. package/src/profiling/node/CoverageSampler.ts +27 -0
  98. package/src/profiling/node/CoverageTypes.ts +23 -0
  99. package/src/profiling/node/HeapSampleReport.ts +261 -0
  100. package/src/{heap-sample → profiling/node}/HeapSampler.ts +1 -2
  101. package/src/{heap-sample → profiling/node}/ResolvedProfile.ts +18 -9
  102. package/src/profiling/node/TimeSampler.ts +57 -0
  103. package/src/report/BenchmarkReport.ts +146 -0
  104. package/src/report/Colors.ts +9 -0
  105. package/src/report/Formatters.ts +110 -0
  106. package/src/report/GcSections.ts +151 -0
  107. package/src/{GitUtils.ts → report/GitUtils.ts} +18 -19
  108. package/src/report/HtmlReport.ts +223 -0
  109. package/src/report/ParseStats.ts +73 -0
  110. package/src/report/StandardSections.ts +147 -0
  111. package/src/report/ViewerSections.ts +286 -0
  112. package/src/report/text/TableReport.ts +253 -0
  113. package/src/report/text/TextReport.ts +123 -0
  114. package/src/runners/AdaptiveWrapper.ts +116 -236
  115. package/src/runners/BenchRunner.ts +20 -15
  116. package/src/{Benchmark.ts → runners/BenchmarkSpec.ts} +5 -6
  117. package/src/runners/CreateRunner.ts +5 -7
  118. package/src/runners/GcStats.ts +47 -50
  119. package/src/{MeasuredResults.ts → runners/MeasuredResults.ts} +43 -37
  120. package/src/runners/MergeBatches.ts +123 -0
  121. package/src/{NodeGC.ts → runners/NodeGC.ts} +2 -3
  122. package/src/runners/RunnerOrchestrator.ts +127 -243
  123. package/src/runners/RunnerUtils.ts +75 -1
  124. package/src/runners/SampleStats.ts +100 -0
  125. package/src/runners/TimingRunner.ts +244 -0
  126. package/src/runners/TimingUtils.ts +3 -2
  127. package/src/runners/WorkerScript.ts +135 -151
  128. package/src/stats/BootstrapDifference.ts +282 -0
  129. package/src/{PermutationTest.ts → stats/PermutationTest.ts} +8 -17
  130. package/src/stats/StatisticalUtils.ts +445 -0
  131. package/src/{tests → test}/AdaptiveConvergence.test.ts +10 -10
  132. package/src/test/AdaptiveRunner.test.ts +39 -41
  133. package/src/{tests → test}/AdaptiveSampling.test.ts +9 -9
  134. package/src/test/AdaptiveStatistics.integration.ts +2 -2
  135. package/src/{tests → test}/BenchMatrix.test.ts +19 -16
  136. package/src/test/BenchmarkReport.test.ts +63 -13
  137. package/src/test/BrowserBench.e2e.test.ts +186 -17
  138. package/src/test/BrowserBench.test.ts +10 -5
  139. package/src/test/BuildTimeSection.test.ts +130 -0
  140. package/src/test/CapSamples.test.ts +82 -0
  141. package/src/test/CoverageExport.test.ts +115 -0
  142. package/src/test/CoverageSampler.test.ts +33 -0
  143. package/src/test/HeapAttribution.test.ts +14 -14
  144. package/src/{tests → test}/MatrixFilter.test.ts +1 -1
  145. package/src/{tests → test}/MatrixReport.test.ts +1 -1
  146. package/src/test/PermutationTest.test.ts +1 -1
  147. package/src/{tests → test}/RealDataValidation.test.ts +6 -6
  148. package/src/test/RunBenchCLI.test.ts +39 -38
  149. package/src/test/RunnerOrchestrator.test.ts +12 -12
  150. package/src/test/StatisticalUtils.test.ts +48 -12
  151. package/src/{table-util/test → test}/TableReport.test.ts +2 -2
  152. package/src/test/TestUtils.ts +12 -7
  153. package/src/test/TimeExport.test.ts +139 -0
  154. package/src/test/TimeSampler.test.ts +37 -0
  155. package/src/test/ViewerLive.e2e.test.ts +159 -0
  156. package/src/test/ViewerStatic.static.e2e.test.ts +137 -0
  157. package/src/{tests → test}/fixtures/baseline/impl.ts +1 -1
  158. package/src/{tests → test}/fixtures/bevy30-samples.ts +3 -1
  159. package/src/test/fixtures/cases/asyncCases.ts +9 -0
  160. package/src/{tests → test}/fixtures/cases/cases.ts +5 -2
  161. package/src/test/fixtures/cases/variants/product.ts +2 -0
  162. package/src/test/fixtures/cases/variants/sum.ts +2 -0
  163. package/src/test/fixtures/discover/fast.ts +1 -0
  164. package/src/{tests → test}/fixtures/discover/slow.ts +1 -1
  165. package/src/test/fixtures/invalid/bad.ts +1 -0
  166. package/src/test/fixtures/loader/fast.ts +1 -0
  167. package/src/{tests → test}/fixtures/loader/slow.ts +1 -1
  168. package/src/test/fixtures/loader/stateful.ts +2 -0
  169. package/src/test/fixtures/stateful/stateful.ts +2 -0
  170. package/src/test/fixtures/variants/extra.ts +1 -0
  171. package/src/test/fixtures/variants/impl.ts +1 -0
  172. package/src/test/fixtures/worker/fast.ts +1 -0
  173. package/src/{tests → test}/fixtures/worker/slow.ts +1 -1
  174. package/src/viewer/DateFormat.ts +30 -0
  175. package/src/viewer/Helpers.ts +23 -0
  176. package/src/viewer/LineData.ts +120 -0
  177. package/src/viewer/Providers.ts +191 -0
  178. package/src/viewer/ReportData.ts +123 -0
  179. package/src/viewer/State.ts +49 -0
  180. package/src/viewer/Theme.ts +15 -0
  181. package/src/viewer/components/App.tsx +73 -0
  182. package/src/viewer/components/DropZone.tsx +71 -0
  183. package/src/viewer/components/LazyPlot.ts +33 -0
  184. package/src/viewer/components/SamplesPanel.tsx +214 -0
  185. package/src/viewer/components/Shell.tsx +26 -0
  186. package/src/viewer/components/SourcePanel.tsx +216 -0
  187. package/src/viewer/components/SummaryPanel.tsx +332 -0
  188. package/src/viewer/components/TabBar.tsx +131 -0
  189. package/src/viewer/components/TabContent.tsx +46 -0
  190. package/src/viewer/components/ThemeToggle.tsx +50 -0
  191. package/src/viewer/index.html +20 -0
  192. package/src/viewer/main.tsx +4 -0
  193. package/src/viewer/plots/CIPlot.ts +313 -0
  194. package/src/{html/browser → viewer/plots}/HistogramKde.ts +33 -38
  195. package/src/viewer/plots/LegendUtils.ts +134 -0
  196. package/src/viewer/plots/PlotTypes.ts +85 -0
  197. package/src/viewer/plots/RenderPlots.ts +230 -0
  198. package/src/viewer/plots/SampleTimeSeries.ts +306 -0
  199. package/src/viewer/plots/SvgHelpers.ts +136 -0
  200. package/src/viewer/plots/TimeSeriesMarks.ts +319 -0
  201. package/src/viewer/report.css +427 -0
  202. package/src/viewer/shell.css +357 -0
  203. package/src/viewer/tsconfig.json +11 -0
  204. package/dist/BrowserHeapSampler-B6asLKWQ.mjs +0 -202
  205. package/dist/BrowserHeapSampler-B6asLKWQ.mjs.map +0 -1
  206. package/dist/GcStats-wX7Xyblu.mjs +0 -77
  207. package/dist/GcStats-wX7Xyblu.mjs.map +0 -1
  208. package/dist/HeapSampler-B8dtKHn1.mjs.map +0 -1
  209. package/dist/TimingUtils-DwOwkc8G.mjs +0 -597
  210. package/dist/TimingUtils-DwOwkc8G.mjs.map +0 -1
  211. package/dist/browser/index.js +0 -914
  212. package/dist/src-B-DDaCa9.mjs +0 -3108
  213. package/dist/src-B-DDaCa9.mjs.map +0 -1
  214. package/src/BenchMatrix.ts +0 -380
  215. package/src/BenchmarkReport.ts +0 -161
  216. package/src/HtmlDataPrep.ts +0 -148
  217. package/src/StandardSections.ts +0 -261
  218. package/src/StatisticalUtils.ts +0 -175
  219. package/src/TypeUtil.ts +0 -8
  220. package/src/browser/BrowserGcStats.ts +0 -44
  221. package/src/browser/BrowserHeapSampler.ts +0 -271
  222. package/src/export/JsonExport.ts +0 -103
  223. package/src/export/JsonFormat.ts +0 -91
  224. package/src/export/SpeedscopeExport.ts +0 -202
  225. package/src/heap-sample/HeapSampleReport.ts +0 -269
  226. package/src/html/HtmlReport.ts +0 -131
  227. package/src/html/HtmlTemplate.ts +0 -284
  228. package/src/html/Types.ts +0 -88
  229. package/src/html/browser/CIPlot.ts +0 -287
  230. package/src/html/browser/LegendUtils.ts +0 -163
  231. package/src/html/browser/RenderPlots.ts +0 -263
  232. package/src/html/browser/SampleTimeSeries.ts +0 -389
  233. package/src/html/browser/Types.ts +0 -96
  234. package/src/html/browser/index.ts +0 -1
  235. package/src/html/index.ts +0 -17
  236. package/src/runners/BasicRunner.ts +0 -364
  237. package/src/table-util/ConvergenceFormatters.ts +0 -19
  238. package/src/table-util/Formatters.ts +0 -157
  239. package/src/table-util/README.md +0 -70
  240. package/src/table-util/TableReport.ts +0 -293
  241. package/src/tests/fixtures/cases/asyncCases.ts +0 -7
  242. package/src/tests/fixtures/cases/variants/product.ts +0 -2
  243. package/src/tests/fixtures/cases/variants/sum.ts +0 -2
  244. package/src/tests/fixtures/discover/fast.ts +0 -1
  245. package/src/tests/fixtures/invalid/bad.ts +0 -1
  246. package/src/tests/fixtures/loader/fast.ts +0 -1
  247. package/src/tests/fixtures/loader/stateful.ts +0 -2
  248. package/src/tests/fixtures/stateful/stateful.ts +0 -2
  249. package/src/tests/fixtures/variants/extra.ts +0 -1
  250. package/src/tests/fixtures/variants/impl.ts +0 -1
  251. package/src/tests/fixtures/worker/fast.ts +0 -1
  252. /package/src/{table-util/test → test}/TableValueExtractor.test.ts +0 -0
  253. /package/src/{table-util/test → test}/TableValueExtractor.ts +0 -0
package/package.json CHANGED
@@ -1,7 +1,25 @@
1
1
  {
2
2
  "name": "benchforge",
3
- "version": "0.1.11",
3
+ "version": "0.2.4",
4
+ "description": "GC aware benchmarking/profiling, with an interactive viewer. For Node and browser.",
5
+ "license": "MIT",
6
+ "keywords": [
7
+ "benchmark",
8
+ "performance",
9
+ "profiling",
10
+ "bootstrap",
11
+ "gc",
12
+ "v8"
13
+ ],
4
14
  "type": "module",
15
+ "repository": {
16
+ "type": "git",
17
+ "url": "git+https://github.com/mighdoll/benchforge.git"
18
+ },
19
+ "homepage": "https://github.com/mighdoll/benchforge#readme",
20
+ "bugs": {
21
+ "url": "https://github.com/mighdoll/benchforge/issues"
22
+ },
5
23
  "bin": {
6
24
  "benchforge": "dist/bin/benchforge.mjs"
7
25
  },
@@ -16,45 +34,52 @@
16
34
  }
17
35
  },
18
36
  "dependencies": {
19
- "esbuild": "^0.27.3",
37
+ "esbuild": "^0.27.4",
20
38
  "open": "^11.0.0",
21
39
  "picocolors": "^1.1.1",
40
+ "sirv": "^3.0.2",
22
41
  "table": "^6.9.0",
23
42
  "yargs": "^18.0.0"
24
43
  },
25
- "peerDependencies": {
26
- "playwright": ">=1.40.0"
27
- },
28
- "peerDependenciesMeta": {
29
- "playwright": {
30
- "optional": true
31
- }
32
- },
33
44
  "devDependencies": {
34
- "@biomejs/biome": "^2.4.4",
35
- "@types/node": "^25.3.0",
45
+ "@biomejs/biome": "^2.4.9",
46
+ "@preact/signals": "^2.9.0",
47
+ "@observablehq/plot": "^0.6.17",
48
+ "@types/d3": "^7.4.3",
49
+ "@types/node": "^25.5.0",
36
50
  "@types/yargs": "^17.0.35",
37
- "@typescript/native-preview": "7.0.0-dev.20260220.1",
51
+ "@typescript/native-preview": "7.0.0-dev.20260329.1",
52
+ "d3": "^7.9.0",
38
53
  "npm-run-all": "^4.1.5",
39
- "playwright": "^1.58.0",
40
- "tsdown": "0.20.3",
41
- "vite": "^7.3.1",
42
- "vitest": "^4.0.18"
54
+ "preact": "^10.29.0",
55
+ "playwright": "^1.58.2",
56
+ "shiki": "^4.0.2",
57
+ "tsdown": "0.21.7",
58
+ "vite": "^8.0.3",
59
+ "vitest": "^4.1.2"
43
60
  },
44
61
  "scripts": {
45
- "build": "tsdown",
46
- "build:plots": "tsdown --config tsdown.plots.config.ts",
47
- "example:node": "bin/benchforge examples/simple-cli.ts --gc-stats --heap-sample --html",
48
- "example:bench": "bin/benchforge --url file://$PWD/examples/browser-bench/index.html --gc-stats --heap-sample --html",
49
- "example:heap": "bin/benchforge --url file://$PWD/examples/browser-heap/index.html --heap-sample --gc-stats",
50
- "example:lap": "bin/benchforge --url file://$PWD/examples/browser-lap/index.html --gc-stats --heap-sample --html",
62
+ "build": "tsdown && vite build --config vite.viewer.config.ts",
63
+ "build:lib": "tsdown",
64
+ "build:viewer": "vite build --config vite.viewer.config.ts",
65
+ "deploy:viewer": "run-s build:viewer pages:viewer:deploy",
66
+ "dev:viewer": "vite --config vite.viewer.config.ts",
67
+ "example:node": "bin/benchforge examples/simple-cli.ts --gc-stats --alloc --batches 5 --iterations 100 --view",
68
+ "example:stats": "bin/benchforge examples/simple-cli.ts --stats mean,p50,p95,p99,max --batches 5 --iterations 200 --view",
69
+ "example:time": "bin/benchforge examples/simple-cli.ts --alloc --time-sample --batches 5 --iterations 100 --view --filter quicksort",
70
+ "example:bench": "bin/benchforge --url file://$PWD/examples/browser-bench/index.html --gc-stats --headless --alloc --view",
71
+ "example:counts": "bin/benchforge examples/simple-cli.ts --alloc --time-sample --call-counts --view --iterations 5",
72
+ "example:heap": "bin/benchforge --url file://$PWD/examples/browser-heap/index.html --gc-stats --headless --view",
73
+ "example:page-load": "bin/benchforge --url file://$PWD/examples/browser-page-load/index.html --headless --page-load --alloc --time-sample --call-counts --gc-stats --batches 2 --iterations 2 --view",
74
+ "pages:viewer:deploy": "pnpx wrangler pages deploy --project-name benchforge-viewer dist/viewer",
51
75
  "fix": "biome check --fix --unsafe --error-on-warnings",
52
76
  "lint": "biome check --error-on-warnings",
53
77
  "test": "pnpm --node-options=--expose-gc vitest --hideSkippedTests --exclude '**/*.e2e.test.ts'",
54
- "test:e2e": "pnpm --node-options=--expose-gc vitest run '.e2e.'",
78
+ "test:e2e": "pnpm --node-options=--expose-gc vitest run '.e2e.' --exclude '**/*.static.e2e.*'",
79
+ "test:static:e2e": "pnpm --node-options=--expose-gc vitest run '.static.e2e.'",
55
80
  "test:once": "pnpm --node-options=--expose-gc vitest run --exclude '**/*.e2e.test.ts'",
56
- "typecheck": "tsgo",
57
- "prepush": "run-s fix typecheck test:once build build:plots",
58
- "publish:all": "pnpm publish && pnpm --filter benchforge-browser publish"
81
+ "typecheck": "tsgo && tsgo -p src/viewer",
82
+ "prepush": "run-s fix typecheck test:once build test:e2e test:static:e2e",
83
+ "publish:all": "pnpm publish"
59
84
  }
60
85
  }
@@ -1,4 +1,4 @@
1
1
  #!/usr/bin/env node
2
- import { runDefaultBench } from "../index.ts";
2
+ import { dispatchCli } from "../cli/RunBenchCLI.ts";
3
3
 
4
- await runDefaultBench();
4
+ await dispatchCli();
@@ -0,0 +1,232 @@
1
+ /** Diagnostic analysis of a .benchforge archive's per-batch statistics. */
2
+ import { readFile } from "node:fs/promises";
3
+ import { resolve } from "node:path";
4
+ import colors from "../report/Colors.ts";
5
+ import { formatSignedPercent, timeMs } from "../report/Formatters.ts";
6
+ import {
7
+ average,
8
+ median,
9
+ percentile,
10
+ splitByOffsets,
11
+ tukeyFences,
12
+ } from "../stats/StatisticalUtils.ts";
13
+ import type { BenchmarkEntry, BenchmarkGroup } from "../viewer/ReportData.ts";
14
+
15
+ const { bold, dim, red, green, yellow } = colors;
16
+
17
+ const blockFenceMultiplier = 3;
18
+
19
+ /** Read an archive and print per-batch diagnostic analysis.
20
+ * (for benchforge debugging/development purposes, not a general user tool)
21
+ */
22
+ export async function analyzeArchive(filePath: string): Promise<void> {
23
+ const absPath = resolve(filePath);
24
+ const content = await readFile(absPath, "utf-8");
25
+ const { report } = JSON.parse(content);
26
+ if (!report?.groups?.length) {
27
+ console.error("No report data found in archive.");
28
+ return;
29
+ }
30
+ const batchCount = report.metadata?.cliArgs?.batches as number | undefined;
31
+ for (const group of report.groups) {
32
+ analyzeGroup(group, batchCount);
33
+ }
34
+ }
35
+
36
+ /** Print analysis for all benchmarks in a group. */
37
+ function analyzeGroup(group: BenchmarkGroup, batchCount?: number): void {
38
+ console.log(bold(`\n=== ${group.name} ===\n`));
39
+
40
+ const baseline = group.baseline;
41
+ for (const bench of group.benchmarks) {
42
+ analyzeBenchmark(bench, baseline, batchCount);
43
+ }
44
+ }
45
+
46
+ /** Print per-batch analysis for one benchmark entry. */
47
+ function analyzeBenchmark(
48
+ bench: BenchmarkEntry,
49
+ baseline: BenchmarkEntry | undefined,
50
+ batchCount?: number,
51
+ ): void {
52
+ const bOffsets =
53
+ bench.batchOffsets ?? inferOffsets(bench.samples, batchCount);
54
+ const baseOffsets =
55
+ baseline?.batchOffsets ?? inferOffsets(baseline?.samples, batchCount);
56
+ if (!bOffsets?.length) {
57
+ console.log(dim(" No batch data (single batch run)"));
58
+ return;
59
+ }
60
+
61
+ const batches = splitByOffsets(bench.samples, bOffsets);
62
+ const baseBatches =
63
+ baseOffsets && baseline
64
+ ? splitByOffsets(baseline.samples, baseOffsets)
65
+ : undefined;
66
+
67
+ printBatchHeader(bench, baseline, batches.length);
68
+ printBatchTable(batches, baseBatches);
69
+
70
+ if (baseBatches && baseBatches.length === batches.length) {
71
+ printOrderEffect(batches, baseBatches);
72
+ printPairedDeltas(batches, baseBatches);
73
+ printTrimmedBlocks(batches, baseBatches, bench.name);
74
+ }
75
+ console.log();
76
+ }
77
+
78
+ /** Infer equal-sized batch offsets when batchOffsets isn't in the archive. */
79
+ function inferOffsets(
80
+ samples: number[] | undefined,
81
+ batchCount?: number,
82
+ ): number[] | undefined {
83
+ if (!samples?.length || !batchCount || batchCount <= 1) return undefined;
84
+ const size = Math.floor(samples.length / batchCount);
85
+ return Array.from({ length: batchCount }, (_, i) => i * size);
86
+ }
87
+
88
+ /** Print benchmark name with batch/run summary. */
89
+ function printBatchHeader(
90
+ bench: BenchmarkEntry,
91
+ baseline: BenchmarkEntry | undefined,
92
+ nBatches: number,
93
+ ): void {
94
+ const baseRuns = baseline?.samples?.length;
95
+ const dur = bench.totalTime
96
+ ? (bench.totalTime / nBatches).toFixed(1) + "s"
97
+ : "?";
98
+ const runs = baseRuns
99
+ ? `${bench.samples.length}+${baseRuns} runs`
100
+ : `${bench.samples.length} runs`;
101
+ const info = dim(` (${nBatches} batches, ${runs}, ~${dur}/batch)`);
102
+ console.log(bold(` ${bench.name}`) + info);
103
+ }
104
+
105
+ /** Print per-batch median table for current and baseline. */
106
+ function printBatchTable(
107
+ benches: number[][],
108
+ baselines: number[][] | undefined,
109
+ ): void {
110
+ const header = baselines
111
+ ? ` ${"batch".padEnd(7)} ${"n".padStart(4)} ${"current".padStart(10)} ${"baseline".padStart(10)} ${"delta".padStart(8)}`
112
+ : ` ${"batch".padEnd(7)} ${"n".padStart(4)} ${"median".padStart(10)}`;
113
+ console.log(dim(header));
114
+
115
+ for (let i = 0; i < benches.length; i++) {
116
+ const n = String(benches[i].length).padStart(4);
117
+ const med = (timeMs(median(benches[i])) ?? "").padStart(10);
118
+ const idx = String(i).padEnd(7);
119
+ if (!baselines?.[i]) {
120
+ console.log(` ${idx} ${n} ${med}`);
121
+ continue;
122
+ }
123
+ const baseMed = (timeMs(median(baselines[i])) ?? "").padStart(10);
124
+ const delta = formatDelta(medianDelta(benches[i], baselines[i])).padStart(
125
+ 8,
126
+ );
127
+ const order = i % 2 === 0 ? dim(" B>C") : dim(" C>B");
128
+ console.log(` ${idx} ${n} ${med} ${baseMed} ${delta}${order}`);
129
+ }
130
+ }
131
+
132
+ /** Analyze order effect: does running second make a difference? */
133
+ function printOrderEffect(benches: number[][], baselines: number[][]): void {
134
+ // Even batches: baseline runs first (B>C), odd: current runs first (C>B)
135
+ const deltas = benches.map((b, i) => medianDelta(b, baselines[i]));
136
+ const baseFirstDeltas = deltas.filter((_, i) => i % 2 === 0);
137
+ const currFirstDeltas = deltas.filter((_, i) => i % 2 === 1);
138
+ const baseFirstAvg = baseFirstDeltas.length ? average(baseFirstDeltas) : 0;
139
+ const currFirstAvg = currFirstDeltas.length ? average(currFirstDeltas) : 0;
140
+
141
+ console.log();
142
+ console.log(bold(" Order effect:"));
143
+ console.log(
144
+ ` baseline first (B>C): avg delta ${formatDelta(baseFirstAvg)}` +
145
+ dim(` (${baseFirstDeltas.length} batches)`),
146
+ );
147
+ console.log(
148
+ ` current first (C>B): avg delta ${formatDelta(currFirstAvg)}` +
149
+ dim(` (${currFirstDeltas.length} batches)`),
150
+ );
151
+
152
+ const diff = Math.abs(baseFirstAvg - currFirstAvg);
153
+ if (diff > 2) {
154
+ console.log(yellow(` ==> ${diff.toFixed(1)}% order effect detected`));
155
+ } else {
156
+ console.log(dim(` order effect: ${diff.toFixed(1)}% (small)`));
157
+ }
158
+ }
159
+
160
+ /** Print paired batch deltas and their consistency. */
161
+ function printPairedDeltas(benches: number[][], baselines: number[][]): void {
162
+ const deltas = benches.map((b, i) => medianDelta(b, baselines[i]));
163
+
164
+ const positive = deltas.filter(d => d > 0).length;
165
+ const negative = deltas.filter(d => d < 0).length;
166
+ const avgDelta = average(deltas);
167
+ const med = median(deltas);
168
+ const spread = percentile(deltas, 0.75) - percentile(deltas, 0.25);
169
+
170
+ console.log();
171
+ console.log(bold(" Paired deltas:"));
172
+ console.log(
173
+ ` mean: ${formatDelta(avgDelta)} median: ${formatDelta(med)} IQR: ${spread.toFixed(1)}%`,
174
+ );
175
+ console.log(
176
+ ` direction: ${positive} slower, ${negative} faster` +
177
+ dim(` (${deltas.length} batches)`),
178
+ );
179
+
180
+ if (positive > 0 && negative > 0) {
181
+ console.log(green(" ==> batches disagree on direction"));
182
+ } else {
183
+ console.log(
184
+ red(" ==> all batches agree on direction (systematic bias?)"),
185
+ );
186
+ }
187
+ }
188
+
189
+ /** Show which blocks would be Tukey-trimmed per side. */
190
+ function printTrimmedBlocks(
191
+ benches: number[][],
192
+ baselines: number[][],
193
+ name: string,
194
+ ): void {
195
+ console.log();
196
+ console.log(bold(" Trimmed blocks:"));
197
+ const baseMeans = baselines.map(b => average(b));
198
+ const benchMeans = benches.map(b => average(b));
199
+ printSideTrim("baseline", baseMeans);
200
+ printSideTrim(name, benchMeans);
201
+ }
202
+
203
+ /** Color a percent delta: red if >1%, green if <-1%. */
204
+ function formatDelta(pct: number): string {
205
+ const str = formatSignedPercent(pct);
206
+ if (pct > 1) return red(str);
207
+ if (pct < -1) return green(str);
208
+ return str;
209
+ }
210
+
211
+ /** Percent delta between two medians. */
212
+ function medianDelta(samples: number[], baseSamples: number[]): number {
213
+ const med = median(samples);
214
+ const baseMed = median(baseSamples);
215
+ return ((med - baseMed) / baseMed) * 100;
216
+ }
217
+
218
+ /** Print trimming info for one side using 3x IQR fences. */
219
+ function printSideTrim(label: string, means: number[]): void {
220
+ const [, hi] = tukeyFences(means, blockFenceMultiplier);
221
+ const indices = means.map((v, i) => (v > hi ? i : -1)).filter(i => i >= 0);
222
+ if (indices.length === 0) {
223
+ console.log(dim(` ${label}: 0 trimmed`));
224
+ return;
225
+ }
226
+ const vals = indices.map(i => timeMs(means[i]) ?? "?").join(", ");
227
+ const fence = `hi: ${timeMs(hi)}`;
228
+ console.log(
229
+ ` ${label}: ${yellow(`${indices.length} trimmed`)} (${vals})` +
230
+ dim(` fence: ${fence}`),
231
+ );
232
+ }
@@ -0,0 +1,322 @@
1
+ import {
2
+ type BrowserProfileResult,
3
+ type NavTiming,
4
+ profileBrowser,
5
+ } from "../profiling/browser/BrowserProfiler.ts";
6
+ import { launchChrome } from "../profiling/browser/ChromeLauncher.ts";
7
+ import { isBrowserUserCode } from "../profiling/node/HeapSampleReport.ts";
8
+ import type { ReportGroup, ReportSection } from "../report/BenchmarkReport.ts";
9
+ import colors from "../report/Colors.ts";
10
+ import {
11
+ browserGcStatsSection,
12
+ pageLoadStatsSections,
13
+ } from "../report/GcSections.ts";
14
+ import { buildTimeSection, runsSection } from "../report/StandardSections.ts";
15
+ import { reportResults } from "../report/text/TextReport.ts";
16
+ import type { MeasuredResults } from "../runners/MeasuredResults.ts";
17
+ import {
18
+ type BatchProgress,
19
+ mergeGcStats,
20
+ runBatched,
21
+ } from "../runners/MergeBatches.ts";
22
+ import { computeStats } from "../runners/SampleStats.ts";
23
+ import type { DefaultCliArgs } from "./CliArgs.ts";
24
+ import { exportReports } from "./CliExport.ts";
25
+ import {
26
+ cliHeapReportOptions,
27
+ needsAlloc,
28
+ needsProfile,
29
+ resolveLimits,
30
+ } from "./CliOptions.ts";
31
+ import { printHeapReports, withStatus } from "./CliReport.ts";
32
+
33
+ /** State shared between makeTabRunner closures and runBatchedTabs. */
34
+ type TabRunnerState = {
35
+ lastRaw?: BrowserProfileResult;
36
+ detectedPageLoad: boolean;
37
+ };
38
+
39
+ const { yellow } = colors;
40
+
41
+ /** Run browser profiling via CDP and report with standard pipeline. */
42
+ export async function browserBenchExports(args: DefaultCliArgs): Promise<void> {
43
+ warnBrowserFlags(args);
44
+ const params = buildBrowserParams(args);
45
+ const name = nameFromUrl(args.url!);
46
+ const baselineUrl = args["baseline-url"];
47
+
48
+ const needsBatching =
49
+ args.batches > 1 ||
50
+ !!baselineUrl ||
51
+ (args.iterations ?? 0) > 1 ||
52
+ params.pageLoad;
53
+ if (!needsBatching) {
54
+ const result = await profileBrowser(params);
55
+ const results = browserResultGroups(name, result);
56
+
57
+ printBrowserReport(result, results, args);
58
+ await exportReports({ results, args });
59
+ return;
60
+ }
61
+
62
+ const { lastRaw, results } = await runBrowserBatches(params, name, args);
63
+ printBrowserReport(lastRaw, results, args);
64
+ await exportReports({ results, args });
65
+ }
66
+
67
+ /** Warn about Node-only flags ignored in browser mode. */
68
+ function warnBrowserFlags(args: DefaultCliArgs): void {
69
+ const checks: [boolean, string][] = [
70
+ [!args.worker, "--no-worker"],
71
+ [!!args["trace-opt"], "--trace-opt"],
72
+ [!!args["gc-force"], "--gc-force"],
73
+ [!!args.adaptive, "--adaptive"],
74
+ ];
75
+ const ignored = checks.filter(([active]) => active).map(([, flag]) => flag);
76
+ if (ignored.length > 0)
77
+ console.warn(yellow(`Ignored in browser mode: ${ignored.join(", ")}`));
78
+ }
79
+
80
+ /** Convert CLI args to browser profiler parameters. */
81
+ function buildBrowserParams(args: DefaultCliArgs) {
82
+ const { maxTime, maxIterations } = resolveLimits(args);
83
+ const chromeArgs = args["chrome-args"]
84
+ ?.flatMap(a => a.split(/\s+/))
85
+ .map(stripQuotes)
86
+ .filter(Boolean);
87
+ return {
88
+ url: args.url!,
89
+ pageLoad: args["page-load"] || !!args["wait-for"],
90
+ maxTime,
91
+ maxIterations,
92
+ chromeArgs,
93
+ allocOptions: {
94
+ samplingInterval: args["alloc-interval"],
95
+ stackDepth: args["alloc-depth"],
96
+ },
97
+ alloc: needsAlloc(args),
98
+ profile: needsProfile(args),
99
+ profileInterval: args["profile-interval"],
100
+ headless: args.headless,
101
+ chromePath: args.chrome,
102
+ chromeProfile: args["chrome-profile"],
103
+ timeout: args.timeout,
104
+ gcStats: args["gc-stats"],
105
+ callCounts: args["call-counts"],
106
+ waitFor: args["wait-for"],
107
+ };
108
+ }
109
+
110
+ /** Extract a short name from a URL for report labels. */
111
+ function nameFromUrl(url: string): string {
112
+ return new URL(url).pathname.split("/").pop() || "browser";
113
+ }
114
+
115
+ /** Wrap browser profile result as ReportGroup[] for the standard export pipeline. */
116
+ function browserResultGroups(
117
+ name: string,
118
+ result: BrowserProfileResult,
119
+ ): ReportGroup[] {
120
+ const measuredResults = toBrowserMeasured(name, result);
121
+ return [{ name, reports: [{ name, measuredResults }] }];
122
+ }
123
+
124
+ /** Print text report and optional heap profile for browser results. */
125
+ function printBrowserReport(
126
+ result: BrowserProfileResult,
127
+ results: ReportGroup[],
128
+ args: DefaultCliArgs,
129
+ ): void {
130
+ const mr = results[0]?.reports[0]?.measuredResults;
131
+ const hasPageLoad = (mr?.navTimings?.length ?? 0) > 0 || !!result.navTiming;
132
+ const hasIterSamples = !!result.samples?.length;
133
+ const sections: ReportSection[] = [
134
+ ...(hasPageLoad ? pageLoadStatsSections : []),
135
+ ...(hasIterSamples ? [buildTimeSection(args.stats)] : []),
136
+ ...(result.gcStats ? [browserGcStatsSection] : []),
137
+ ...(hasPageLoad || hasIterSamples ? [runsSection] : []),
138
+ ];
139
+ if (sections.length > 0) {
140
+ console.log(
141
+ withStatus("computing report", () => reportResults(results, sections)),
142
+ );
143
+ }
144
+ if (!result.heapProfile) return;
145
+ printHeapReports(results, {
146
+ ...cliHeapReportOptions(args),
147
+ isUserCode: isBrowserUserCode,
148
+ });
149
+ }
150
+
151
+ /** Launch Chrome, run batched fresh tabs, merge results. */
152
+ async function runBrowserBatches(
153
+ params: ReturnType<typeof buildBrowserParams>,
154
+ name: string,
155
+ args: DefaultCliArgs,
156
+ ): Promise<{ lastRaw: BrowserProfileResult; results: ReportGroup[] }> {
157
+ const { headless, chrome: chromePath } = args;
158
+ const chromeProfile = args["chrome-profile"];
159
+ const chrome = await launchChrome({
160
+ headless,
161
+ chromePath,
162
+ chromeProfile,
163
+ args: params.chromeArgs,
164
+ });
165
+ try {
166
+ return await runBatchedTabs(params, name, args, chrome);
167
+ } finally {
168
+ await chrome.close();
169
+ }
170
+ }
171
+
172
+ /** Strip surrounding quotes from a chrome-args token. */
173
+ function stripQuotes(s: string): string {
174
+ const bare = s.replace(/^(['"])(.*)\1$/s, "$2");
175
+ return bare.replace(/^(-[^=]+=)(['"])(.*)\2$/s, "$1$3");
176
+ }
177
+
178
+ /** Convert a browser profile result into a MeasuredResults for the report pipeline. */
179
+ function toBrowserMeasured(
180
+ name: string,
181
+ result: BrowserProfileResult,
182
+ ): MeasuredResults {
183
+ const { gcStats, heapProfile, timeProfile, coverage, navTiming, samples } =
184
+ result;
185
+ const navTimings = navTiming ? [navTiming] : undefined;
186
+ const base = {
187
+ name,
188
+ gcStats,
189
+ heapProfile,
190
+ timeProfile,
191
+ coverage,
192
+ navTimings,
193
+ };
194
+
195
+ if (samples?.length) {
196
+ const totalTime = result.wallTimeMs ? result.wallTimeMs / 1000 : undefined;
197
+ return { ...base, samples, time: computeStats(samples), totalTime };
198
+ }
199
+ const wallTime = result.wallTimeMs ?? 0;
200
+ return { ...base, samples: [wallTime], time: computeStats([wallTime]) };
201
+ }
202
+
203
+ /** Execute batched browser tabs within an already-launched Chrome instance. */
204
+ async function runBatchedTabs(
205
+ params: ReturnType<typeof buildBrowserParams>,
206
+ name: string,
207
+ args: DefaultCliArgs,
208
+ chrome: any,
209
+ ): Promise<{ lastRaw: BrowserProfileResult; results: ReportGroup[] }> {
210
+ const baselineUrl = args["baseline-url"];
211
+ const { maxTime, maxIterations } = params;
212
+ const limits = { maxTime, maxIterations };
213
+ const state: TabRunnerState = { detectedPageLoad: params.pageLoad };
214
+
215
+ const warmup = !(args["warmup-batch"] ?? false) && args.batches > 1;
216
+ const mk = (url: string, label: string) =>
217
+ makeTabRunner(params, chrome, limits, warmup, state, url, label);
218
+ const runCurrent = mk(params.url, name);
219
+ const runBaseline = baselineUrl
220
+ ? mk(baselineUrl, nameFromUrl(baselineUrl))
221
+ : undefined;
222
+
223
+ const progress = (p: BatchProgress) => {
224
+ const sec = (p.elapsed / 1000).toFixed(0);
225
+ const msg = `\r◊ batch ${p.batch + 1}/${p.batches} ${p.label} (${sec}s) `;
226
+ process.stderr.write(msg);
227
+ };
228
+
229
+ const {
230
+ results: [current],
231
+ baseline,
232
+ } = await runBatched(
233
+ [runCurrent],
234
+ runBaseline,
235
+ Math.max(args.batches, 2),
236
+ args["warmup-batch"] ?? false,
237
+ progress,
238
+ );
239
+ process.stderr.write("\r" + " ".repeat(50) + "\r");
240
+
241
+ const baseName = baselineUrl ? nameFromUrl(baselineUrl) : undefined;
242
+ const baselineEntry =
243
+ baseline && baseName
244
+ ? { name: baseName, measuredResults: baseline }
245
+ : undefined;
246
+ const reports = [{ name, measuredResults: current }];
247
+ return {
248
+ lastRaw: state.lastRaw!,
249
+ results: [{ name, reports, baseline: baselineEntry }],
250
+ };
251
+ }
252
+
253
+ /** Create a batch runner closure for a single URL (current or baseline). */
254
+ function makeTabRunner(
255
+ params: ReturnType<typeof buildBrowserParams>,
256
+ chrome: any,
257
+ limits: { maxTime?: number; maxIterations?: number },
258
+ warmup: boolean,
259
+ state: TabRunnerState,
260
+ url: string,
261
+ label: string,
262
+ ): () => Promise<MeasuredResults> {
263
+ let firstCall = warmup;
264
+ return async () => {
265
+ const isWarmup = firstCall;
266
+ firstCall = false;
267
+ const p = { ...params, chrome, url };
268
+ if (state.detectedPageLoad) {
269
+ const batchLimits = isWarmup ? { maxIterations: 1 } : limits;
270
+ const result = await runMultiPageLoad(
271
+ { ...p, pageLoad: true },
272
+ label,
273
+ batchLimits,
274
+ );
275
+ state.lastRaw ??= {
276
+ navTiming: result.navTimings?.[0],
277
+ wallTimeMs: result.time.p50,
278
+ };
279
+ return result;
280
+ }
281
+ const raw = await profileBrowser(p);
282
+ state.lastRaw = raw;
283
+ // Probe: if no iteration samples and navTiming present, it's page-load mode
284
+ if (!raw.samples?.length && raw.navTiming) state.detectedPageLoad = true;
285
+ return toBrowserMeasured(label, raw);
286
+ };
287
+ }
288
+
289
+ /** Run page loads until duration or iteration limit, collecting wallTimeMs as samples. */
290
+ async function runMultiPageLoad(
291
+ params: any, // BrowserProfileParams with chrome attached by caller
292
+ name: string,
293
+ limits: { maxTime?: number; maxIterations?: number },
294
+ ): Promise<MeasuredResults> {
295
+ const { maxTime, maxIterations } = limits;
296
+ const raws: BrowserProfileResult[] = [];
297
+ let accumulated = 0;
298
+ for (let i = 0; ; i++) {
299
+ if (maxIterations != null && i >= maxIterations) break;
300
+ const raw = await profileBrowser(params);
301
+ raws.push(raw);
302
+ accumulated += raw.wallTimeMs ?? 0;
303
+ if (maxTime != null && accumulated >= maxTime) break;
304
+ }
305
+
306
+ const samples = raws.map(r => r.wallTimeMs ?? 0);
307
+ const navTimings = raws.map(r => r.navTiming).filter(Boolean) as NavTiming[];
308
+ const { heapProfile, timeProfile, coverage } = raws[raws.length - 1];
309
+ const totalTime = accumulated / 1000;
310
+ const gcStats = mergeGcStats(raws);
311
+ return {
312
+ name,
313
+ samples,
314
+ time: computeStats(samples),
315
+ totalTime,
316
+ navTimings: navTimings.length ? navTimings : undefined,
317
+ gcStats,
318
+ heapProfile,
319
+ timeProfile,
320
+ coverage,
321
+ };
322
+ }