benchforge 0.1.11 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/LICENSE +20 -0
  2. package/README.md +99 -294
  3. package/bin/benchforge +1 -2
  4. package/dist/AnalyzeArchive-8NCJhmhS.mjs +145 -0
  5. package/dist/AnalyzeArchive-8NCJhmhS.mjs.map +1 -0
  6. package/dist/BenchMatrix-BZVrBB_h.mjs +1050 -0
  7. package/dist/BenchMatrix-BZVrBB_h.mjs.map +1 -0
  8. package/dist/{BenchRunner-BzyUfiyB.d.mts → BenchRunner-DglX1NOn.d.mts} +119 -66
  9. package/dist/CoverageSampler-D5T9DRqe.mjs +27 -0
  10. package/dist/CoverageSampler-D5T9DRqe.mjs.map +1 -0
  11. package/dist/Formatters-BWj3d4sv.mjs +95 -0
  12. package/dist/Formatters-BWj3d4sv.mjs.map +1 -0
  13. package/dist/{HeapSampler-B8dtKHn1.mjs → HeapSampler-Dq-hpXem.mjs} +4 -4
  14. package/dist/HeapSampler-Dq-hpXem.mjs.map +1 -0
  15. package/dist/RunBenchCLI-C17DrJz8.mjs +3075 -0
  16. package/dist/RunBenchCLI-C17DrJz8.mjs.map +1 -0
  17. package/dist/StatisticalUtils-BD92crgM.mjs +255 -0
  18. package/dist/StatisticalUtils-BD92crgM.mjs.map +1 -0
  19. package/dist/TimeSampler-Ds8n7l2B.mjs +29 -0
  20. package/dist/TimeSampler-Ds8n7l2B.mjs.map +1 -0
  21. package/dist/ViewerServer-BJhdnxlN.mjs +639 -0
  22. package/dist/ViewerServer-BJhdnxlN.mjs.map +1 -0
  23. package/dist/ViewerServer-CuMNdNBz.mjs +2 -0
  24. package/dist/bin/benchforge.mjs +4 -5
  25. package/dist/bin/benchforge.mjs.map +1 -1
  26. package/dist/index.d.mts +711 -558
  27. package/dist/index.mjs +98 -3
  28. package/dist/index.mjs.map +1 -0
  29. package/dist/runners/WorkerScript.d.mts +12 -4
  30. package/dist/runners/WorkerScript.mjs +77 -105
  31. package/dist/runners/WorkerScript.mjs.map +1 -1
  32. package/dist/viewer/assets/CIPlot-BkOvMoMa.js +1 -0
  33. package/dist/viewer/assets/HistogramKde-CmSyUFY0.js +1 -0
  34. package/dist/viewer/assets/LegendUtils-BJpbn_jr.js +55 -0
  35. package/dist/viewer/assets/SampleTimeSeries-C4VBhXr3.js +1 -0
  36. package/dist/viewer/assets/index-Br9bp_cX.js +153 -0
  37. package/dist/viewer/assets/index-NzXXe_CC.css +1 -0
  38. package/dist/viewer/index.html +19 -0
  39. package/dist/viewer/speedscope/LICENSE +21 -0
  40. package/dist/viewer/speedscope/SourceCodePro-Regular.ttf-ILST5JV6.woff2 +0 -0
  41. package/dist/viewer/speedscope/favicon-16x16-V2DMIAZS.js +2 -0
  42. package/dist/viewer/speedscope/favicon-16x16-V2DMIAZS.js.map +7 -0
  43. package/dist/viewer/speedscope/favicon-16x16-VSI62OPJ.png +0 -0
  44. package/dist/viewer/speedscope/favicon-32x32-3EB2YCUY.png +0 -0
  45. package/dist/viewer/speedscope/favicon-32x32-THY3JDJL.js +2 -0
  46. package/dist/viewer/speedscope/favicon-32x32-THY3JDJL.js.map +7 -0
  47. package/dist/viewer/speedscope/favicon-FOKUP5Y5.ico +0 -0
  48. package/dist/viewer/speedscope/favicon-M34RF7BI.js +2 -0
  49. package/dist/viewer/speedscope/favicon-M34RF7BI.js.map +7 -0
  50. package/dist/viewer/speedscope/file-format-schema.json +274 -0
  51. package/dist/viewer/speedscope/index.html +19 -0
  52. package/dist/viewer/speedscope/jfrview_bg-BLJXNNQB.wasm +0 -0
  53. package/dist/viewer/speedscope/perf-vertx-stacks-01-collapsed-all-ZNUIGAJL.txt +199 -0
  54. package/dist/viewer/speedscope/release.txt +3 -0
  55. package/dist/viewer/speedscope/source-code-pro.LICENSE.md +93 -0
  56. package/dist/viewer/speedscope/speedscope-GHPHNKXC.css +2 -0
  57. package/dist/viewer/speedscope/speedscope-GHPHNKXC.css.map +7 -0
  58. package/dist/viewer/speedscope/speedscope-QZFMJ7VP.js +212 -0
  59. package/dist/viewer/speedscope/speedscope-QZFMJ7VP.js.map +7 -0
  60. package/package.json +52 -27
  61. package/src/bin/benchforge.ts +2 -2
  62. package/src/cli/AnalyzeArchive.ts +232 -0
  63. package/src/cli/BrowserBench.ts +322 -0
  64. package/src/cli/CliArgs.ts +164 -51
  65. package/src/cli/CliExport.ts +179 -0
  66. package/src/cli/CliOptions.ts +147 -0
  67. package/src/cli/CliReport.ts +197 -0
  68. package/src/cli/FilterBenchmarks.ts +18 -30
  69. package/src/cli/RunBenchCLI.ts +132 -866
  70. package/src/cli/SuiteRunner.ts +160 -0
  71. package/src/cli/ViewerServer.ts +282 -0
  72. package/src/export/AllocExport.ts +121 -0
  73. package/src/export/ArchiveExport.ts +146 -0
  74. package/src/export/ArchiveFormat.ts +50 -0
  75. package/src/export/CoverageExport.ts +148 -0
  76. package/src/export/EditorUri.ts +10 -0
  77. package/src/export/PerfettoExport.ts +64 -99
  78. package/src/export/SpeedscopeTypes.ts +98 -0
  79. package/src/export/TimeExport.ts +115 -0
  80. package/src/index.ts +86 -67
  81. package/src/matrix/BenchMatrix.ts +230 -0
  82. package/src/matrix/CaseLoader.ts +8 -6
  83. package/src/matrix/MatrixDirRunner.ts +153 -0
  84. package/src/matrix/MatrixFilter.ts +49 -47
  85. package/src/matrix/MatrixInlineRunner.ts +50 -0
  86. package/src/matrix/MatrixReport.ts +90 -250
  87. package/src/matrix/VariantLoader.ts +5 -5
  88. package/src/profiling/browser/BenchLoop.ts +51 -0
  89. package/src/profiling/browser/BrowserCDP.ts +133 -0
  90. package/src/profiling/browser/BrowserGcStats.ts +33 -0
  91. package/src/profiling/browser/BrowserProfiler.ts +160 -0
  92. package/src/profiling/browser/CdpClient.ts +82 -0
  93. package/src/profiling/browser/CdpPage.ts +138 -0
  94. package/src/profiling/browser/ChromeLauncher.ts +158 -0
  95. package/src/profiling/browser/ChromeTraceEvent.ts +28 -0
  96. package/src/profiling/browser/PageLoadMode.ts +61 -0
  97. package/src/profiling/node/CoverageSampler.ts +27 -0
  98. package/src/profiling/node/CoverageTypes.ts +23 -0
  99. package/src/profiling/node/HeapSampleReport.ts +261 -0
  100. package/src/{heap-sample → profiling/node}/HeapSampler.ts +1 -2
  101. package/src/{heap-sample → profiling/node}/ResolvedProfile.ts +18 -9
  102. package/src/profiling/node/TimeSampler.ts +57 -0
  103. package/src/report/BenchmarkReport.ts +146 -0
  104. package/src/report/Colors.ts +9 -0
  105. package/src/report/Formatters.ts +110 -0
  106. package/src/report/GcSections.ts +151 -0
  107. package/src/{GitUtils.ts → report/GitUtils.ts} +18 -19
  108. package/src/report/HtmlReport.ts +223 -0
  109. package/src/report/ParseStats.ts +73 -0
  110. package/src/report/StandardSections.ts +147 -0
  111. package/src/report/ViewerSections.ts +286 -0
  112. package/src/report/text/TableReport.ts +253 -0
  113. package/src/report/text/TextReport.ts +123 -0
  114. package/src/runners/AdaptiveWrapper.ts +116 -236
  115. package/src/runners/BenchRunner.ts +20 -15
  116. package/src/{Benchmark.ts → runners/BenchmarkSpec.ts} +5 -6
  117. package/src/runners/CreateRunner.ts +5 -7
  118. package/src/runners/GcStats.ts +47 -50
  119. package/src/{MeasuredResults.ts → runners/MeasuredResults.ts} +43 -37
  120. package/src/runners/MergeBatches.ts +123 -0
  121. package/src/{NodeGC.ts → runners/NodeGC.ts} +2 -3
  122. package/src/runners/RunnerOrchestrator.ts +127 -243
  123. package/src/runners/RunnerUtils.ts +75 -1
  124. package/src/runners/SampleStats.ts +100 -0
  125. package/src/runners/TimingRunner.ts +244 -0
  126. package/src/runners/TimingUtils.ts +3 -2
  127. package/src/runners/WorkerScript.ts +135 -151
  128. package/src/stats/BootstrapDifference.ts +282 -0
  129. package/src/{PermutationTest.ts → stats/PermutationTest.ts} +8 -17
  130. package/src/stats/StatisticalUtils.ts +445 -0
  131. package/src/{tests → test}/AdaptiveConvergence.test.ts +10 -10
  132. package/src/test/AdaptiveRunner.test.ts +39 -41
  133. package/src/{tests → test}/AdaptiveSampling.test.ts +9 -9
  134. package/src/test/AdaptiveStatistics.integration.ts +2 -2
  135. package/src/{tests → test}/BenchMatrix.test.ts +19 -16
  136. package/src/test/BenchmarkReport.test.ts +63 -13
  137. package/src/test/BrowserBench.e2e.test.ts +186 -17
  138. package/src/test/BrowserBench.test.ts +10 -5
  139. package/src/test/BuildTimeSection.test.ts +130 -0
  140. package/src/test/CapSamples.test.ts +82 -0
  141. package/src/test/CoverageExport.test.ts +115 -0
  142. package/src/test/CoverageSampler.test.ts +33 -0
  143. package/src/test/HeapAttribution.test.ts +14 -14
  144. package/src/{tests → test}/MatrixFilter.test.ts +1 -1
  145. package/src/{tests → test}/MatrixReport.test.ts +1 -1
  146. package/src/test/PermutationTest.test.ts +1 -1
  147. package/src/{tests → test}/RealDataValidation.test.ts +6 -6
  148. package/src/test/RunBenchCLI.test.ts +39 -38
  149. package/src/test/RunnerOrchestrator.test.ts +12 -12
  150. package/src/test/StatisticalUtils.test.ts +48 -12
  151. package/src/{table-util/test → test}/TableReport.test.ts +2 -2
  152. package/src/test/TestUtils.ts +12 -7
  153. package/src/test/TimeExport.test.ts +139 -0
  154. package/src/test/TimeSampler.test.ts +37 -0
  155. package/src/test/ViewerLive.e2e.test.ts +159 -0
  156. package/src/test/ViewerStatic.static.e2e.test.ts +137 -0
  157. package/src/{tests → test}/fixtures/baseline/impl.ts +1 -1
  158. package/src/{tests → test}/fixtures/bevy30-samples.ts +3 -1
  159. package/src/test/fixtures/cases/asyncCases.ts +9 -0
  160. package/src/{tests → test}/fixtures/cases/cases.ts +5 -2
  161. package/src/test/fixtures/cases/variants/product.ts +2 -0
  162. package/src/test/fixtures/cases/variants/sum.ts +2 -0
  163. package/src/test/fixtures/discover/fast.ts +1 -0
  164. package/src/{tests → test}/fixtures/discover/slow.ts +1 -1
  165. package/src/test/fixtures/invalid/bad.ts +1 -0
  166. package/src/test/fixtures/loader/fast.ts +1 -0
  167. package/src/{tests → test}/fixtures/loader/slow.ts +1 -1
  168. package/src/test/fixtures/loader/stateful.ts +2 -0
  169. package/src/test/fixtures/stateful/stateful.ts +2 -0
  170. package/src/test/fixtures/variants/extra.ts +1 -0
  171. package/src/test/fixtures/variants/impl.ts +1 -0
  172. package/src/test/fixtures/worker/fast.ts +1 -0
  173. package/src/{tests → test}/fixtures/worker/slow.ts +1 -1
  174. package/src/viewer/DateFormat.ts +30 -0
  175. package/src/viewer/Helpers.ts +23 -0
  176. package/src/viewer/LineData.ts +120 -0
  177. package/src/viewer/Providers.ts +191 -0
  178. package/src/viewer/ReportData.ts +123 -0
  179. package/src/viewer/State.ts +49 -0
  180. package/src/viewer/Theme.ts +15 -0
  181. package/src/viewer/components/App.tsx +73 -0
  182. package/src/viewer/components/DropZone.tsx +71 -0
  183. package/src/viewer/components/LazyPlot.ts +33 -0
  184. package/src/viewer/components/SamplesPanel.tsx +214 -0
  185. package/src/viewer/components/Shell.tsx +26 -0
  186. package/src/viewer/components/SourcePanel.tsx +216 -0
  187. package/src/viewer/components/SummaryPanel.tsx +332 -0
  188. package/src/viewer/components/TabBar.tsx +131 -0
  189. package/src/viewer/components/TabContent.tsx +46 -0
  190. package/src/viewer/components/ThemeToggle.tsx +50 -0
  191. package/src/viewer/index.html +20 -0
  192. package/src/viewer/main.tsx +4 -0
  193. package/src/viewer/plots/CIPlot.ts +313 -0
  194. package/src/{html/browser → viewer/plots}/HistogramKde.ts +33 -38
  195. package/src/viewer/plots/LegendUtils.ts +134 -0
  196. package/src/viewer/plots/PlotTypes.ts +85 -0
  197. package/src/viewer/plots/RenderPlots.ts +230 -0
  198. package/src/viewer/plots/SampleTimeSeries.ts +306 -0
  199. package/src/viewer/plots/SvgHelpers.ts +136 -0
  200. package/src/viewer/plots/TimeSeriesMarks.ts +319 -0
  201. package/src/viewer/report.css +427 -0
  202. package/src/viewer/shell.css +357 -0
  203. package/src/viewer/tsconfig.json +11 -0
  204. package/dist/BrowserHeapSampler-B6asLKWQ.mjs +0 -202
  205. package/dist/BrowserHeapSampler-B6asLKWQ.mjs.map +0 -1
  206. package/dist/GcStats-wX7Xyblu.mjs +0 -77
  207. package/dist/GcStats-wX7Xyblu.mjs.map +0 -1
  208. package/dist/HeapSampler-B8dtKHn1.mjs.map +0 -1
  209. package/dist/TimingUtils-DwOwkc8G.mjs +0 -597
  210. package/dist/TimingUtils-DwOwkc8G.mjs.map +0 -1
  211. package/dist/browser/index.js +0 -914
  212. package/dist/src-B-DDaCa9.mjs +0 -3108
  213. package/dist/src-B-DDaCa9.mjs.map +0 -1
  214. package/src/BenchMatrix.ts +0 -380
  215. package/src/BenchmarkReport.ts +0 -161
  216. package/src/HtmlDataPrep.ts +0 -148
  217. package/src/StandardSections.ts +0 -261
  218. package/src/StatisticalUtils.ts +0 -175
  219. package/src/TypeUtil.ts +0 -8
  220. package/src/browser/BrowserGcStats.ts +0 -44
  221. package/src/browser/BrowserHeapSampler.ts +0 -271
  222. package/src/export/JsonExport.ts +0 -103
  223. package/src/export/JsonFormat.ts +0 -91
  224. package/src/export/SpeedscopeExport.ts +0 -202
  225. package/src/heap-sample/HeapSampleReport.ts +0 -269
  226. package/src/html/HtmlReport.ts +0 -131
  227. package/src/html/HtmlTemplate.ts +0 -284
  228. package/src/html/Types.ts +0 -88
  229. package/src/html/browser/CIPlot.ts +0 -287
  230. package/src/html/browser/LegendUtils.ts +0 -163
  231. package/src/html/browser/RenderPlots.ts +0 -263
  232. package/src/html/browser/SampleTimeSeries.ts +0 -389
  233. package/src/html/browser/Types.ts +0 -96
  234. package/src/html/browser/index.ts +0 -1
  235. package/src/html/index.ts +0 -17
  236. package/src/runners/BasicRunner.ts +0 -364
  237. package/src/table-util/ConvergenceFormatters.ts +0 -19
  238. package/src/table-util/Formatters.ts +0 -157
  239. package/src/table-util/README.md +0 -70
  240. package/src/table-util/TableReport.ts +0 -293
  241. package/src/tests/fixtures/cases/asyncCases.ts +0 -7
  242. package/src/tests/fixtures/cases/variants/product.ts +0 -2
  243. package/src/tests/fixtures/cases/variants/sum.ts +0 -2
  244. package/src/tests/fixtures/discover/fast.ts +0 -1
  245. package/src/tests/fixtures/invalid/bad.ts +0 -1
  246. package/src/tests/fixtures/loader/fast.ts +0 -1
  247. package/src/tests/fixtures/loader/stateful.ts +0 -2
  248. package/src/tests/fixtures/stateful/stateful.ts +0 -2
  249. package/src/tests/fixtures/variants/extra.ts +0 -1
  250. package/src/tests/fixtures/variants/impl.ts +0 -1
  251. package/src/tests/fixtures/worker/fast.ts +0 -1
  252. /package/src/{table-util/test → test}/TableValueExtractor.test.ts +0 -0
  253. /package/src/{table-util/test → test}/TableValueExtractor.ts +0 -0
@@ -1 +0,0 @@
1
- {"version":3,"file":"HeapSampler-B8dtKHn1.mjs","names":[],"sources":["../src/heap-sample/HeapSampler.ts"],"sourcesContent":["import { Session } from \"node:inspector/promises\";\n\nexport interface HeapSampleOptions {\n /** Bytes between samples (default 32768) */\n samplingInterval?: number;\n\n /** Max stack frames (default 64) */\n stackDepth?: number;\n\n /** Keep objects collected by minor GC (default true) */\n includeMinorGC?: boolean;\n\n /** Keep objects collected by major GC (default true) */\n includeMajorGC?: boolean;\n}\n\n/** V8 call frame location within a profiled script */\nexport interface CallFrame {\n /** Function name (empty string for anonymous) */\n functionName: string;\n\n /** Script URL or file path */\n url: string;\n\n /** Zero-based line number */\n lineNumber: number;\n\n /** Zero-based column number */\n columnNumber?: number;\n}\n\n/** Node in the V8 sampling heap profile tree */\nexport interface ProfileNode {\n /** Call site for this allocation node */\n callFrame: CallFrame;\n\n /** Bytes allocated directly at this node (not children) */\n selfSize: number;\n\n /** Unique node ID, links to {@link HeapSample.nodeId} */\n id: number;\n\n /** Child nodes in the call tree */\n children?: ProfileNode[];\n}\n\n/** Individual heap allocation sample from V8's SamplingHeapProfiler */\nexport interface HeapSample {\n /** Links to {@link ProfileNode.id} for stack lookup */\n nodeId: number;\n\n /** Allocation size in bytes */\n size: number;\n\n /** Monotonically increasing, gives temporal ordering */\n ordinal: number;\n}\n\n/** V8 sampling heap profile tree with optional per-allocation samples */\nexport interface HeapProfile {\n /** Root of the profile call tree */\n head: ProfileNode;\n\n /** Per-allocation samples, if collected */\n samples?: HeapSample[];\n}\n\nconst defaultOptions: Required<HeapSampleOptions> = {\n samplingInterval: 32768,\n stackDepth: 64,\n includeMinorGC: true,\n includeMajorGC: true,\n};\n\n/** Run a function while sampling heap allocations, return profile */\nexport async function withHeapSampling<T>(\n options: HeapSampleOptions,\n fn: () => Promise<T> | T,\n): Promise<{ result: T; profile: HeapProfile }> {\n const opts = { ...defaultOptions, ...options };\n const session = new Session();\n session.connect();\n\n try {\n await startSampling(session, opts);\n const result = await fn();\n const profile = await stopSampling(session);\n return { result, profile };\n } finally {\n session.disconnect();\n }\n}\n\n/** Start heap sampling, falling back if include-collected params aren't supported */\nasync function startSampling(\n session: Session,\n opts: Required<HeapSampleOptions>,\n): Promise<void> {\n const { samplingInterval, stackDepth } = opts;\n const base = { samplingInterval, stackDepth };\n const params = {\n ...base,\n includeObjectsCollectedByMinorGC: opts.includeMinorGC,\n includeObjectsCollectedByMajorGC: opts.includeMajorGC,\n };\n\n try {\n await session.post(\"HeapProfiler.startSampling\", params);\n } catch {\n console.warn(\n \"HeapProfiler: include-collected params not supported, falling back\",\n );\n await session.post(\"HeapProfiler.startSampling\", base);\n }\n}\n\nasync function stopSampling(session: Session): Promise<HeapProfile> {\n const { profile } = await session.post(\"HeapProfiler.stopSampling\");\n // V8 returns id/samples fields not in @types/node's incomplete SamplingHeapProfile\n return profile as unknown as HeapProfile;\n}\n"],"mappings":";;;AAmEA,MAAM,iBAA8C;CAClD,kBAAkB;CAClB,YAAY;CACZ,gBAAgB;CAChB,gBAAgB;CACjB;;AAGD,eAAsB,iBACpB,SACA,IAC8C;CAC9C,MAAM,OAAO;EAAE,GAAG;EAAgB,GAAG;EAAS;CAC9C,MAAM,UAAU,IAAI,SAAS;AAC7B,SAAQ,SAAS;AAEjB,KAAI;AACF,QAAM,cAAc,SAAS,KAAK;AAGlC,SAAO;GAAE,QAFM,MAAM,IAAI;GAER,SADD,MAAM,aAAa,QAAQ;GACjB;WAClB;AACR,UAAQ,YAAY;;;;AAKxB,eAAe,cACb,SACA,MACe;CACf,MAAM,EAAE,kBAAkB,eAAe;CACzC,MAAM,OAAO;EAAE;EAAkB;EAAY;CAC7C,MAAM,SAAS;EACb,GAAG;EACH,kCAAkC,KAAK;EACvC,kCAAkC,KAAK;EACxC;AAED,KAAI;AACF,QAAM,QAAQ,KAAK,8BAA8B,OAAO;SAClD;AACN,UAAQ,KACN,qEACD;AACD,QAAM,QAAQ,KAAK,8BAA8B,KAAK;;;AAI1D,eAAe,aAAa,SAAwC;CAClE,MAAM,EAAE,YAAY,MAAM,QAAQ,KAAK,4BAA4B;AAEnE,QAAO"}
@@ -1,597 +0,0 @@
1
- import fs from "node:fs/promises";
2
- import { fileURLToPath } from "node:url";
3
- import { getHeapStatistics } from "node:v8";
4
-
5
- //#region src/matrix/VariantLoader.ts
6
- /** Discover variant ids from a directory of .ts files */
7
- async function discoverVariants(dirUrl) {
8
- const dirPath = fileURLToPath(dirUrl);
9
- return (await fs.readdir(dirPath, { withFileTypes: true })).filter((e) => e.isFile() && e.name.endsWith(".ts")).map((e) => e.name.slice(0, -3)).sort();
10
- }
11
- /** Get module URL for a variant in a directory */
12
- function variantModuleUrl(dirUrl, variantId) {
13
- return new URL(`${variantId}.ts`, dirUrl).href;
14
- }
15
-
16
- //#endregion
17
- //#region src/runners/BenchRunner.ts
18
- /** Execute benchmark with optional parameters */
19
- function executeBenchmark(benchmark, params) {
20
- benchmark.fn(params);
21
- }
22
-
23
- //#endregion
24
- //#region src/runners/BasicRunner.ts
25
- /**
26
- * Wait time after gc() for V8 to stabilize (ms).
27
- *
28
- * V8 has 4 compilation tiers: Ignition (interpreter) -> Sparkplug (baseline) ->
29
- * Maglev (mid-tier optimizer) -> TurboFan (full optimizer). Tiering thresholds:
30
- * - Ignition -> Sparkplug: 8 invocations
31
- * - Sparkplug -> Maglev: 500 invocations
32
- * - Maglev -> TurboFan: 6000 invocations
33
- *
34
- * Optimization compilation happens on background threads and requires idle time
35
- * on the main thread to complete. Without sufficient warmup + settle time,
36
- * benchmarks exhibit bimodal timing: slow Sparkplug samples (~30% slower) mixed
37
- * with fast optimized samples.
38
- *
39
- * The warmup iterations trigger the optimization decision, then gcSettleTime
40
- * provides idle time for background compilation to finish before measurement.
41
- *
42
- * @see https://v8.dev/blog/sparkplug
43
- * @see https://v8.dev/blog/maglev
44
- * @see https://v8.dev/blog/background-compilation
45
- */
46
- const gcSettleTime = 1e3;
47
- const defaultCollectOptions = {
48
- maxTime: 5e3,
49
- maxIterations: 1e6,
50
- warmup: 0,
51
- traceOpt: false,
52
- noSettle: false
53
- };
54
- /**
55
- * V8 optimization status bit meanings:
56
- * Bit 0 (1): is_function
57
- * Bit 4 (16): is_optimized (TurboFan)
58
- * Bit 5 (32): is_optimized (Maglev)
59
- * Bit 7 (128): is_baseline (Sparkplug)
60
- * Bit 3 (8): maybe_deoptimized
61
- */
62
- const statusNames = {
63
- 1: "interpreted",
64
- 129: "sparkplug",
65
- 17: "turbofan",
66
- 33: "maglev",
67
- 49: "turbofan+maglev",
68
- 32769: "optimized"
69
- };
70
- /** @return runner with time and iteration limits */
71
- var BasicRunner = class {
72
- async runBench(benchmark, options, params) {
73
- const collected = await collectSamples({
74
- benchmark,
75
- params,
76
- ...defaultCollectOptions,
77
- ...options
78
- });
79
- return [buildMeasuredResults(benchmark.name, collected)];
80
- }
81
- };
82
- /** @return percentiles and basic statistics */
83
- function computeStats(samples) {
84
- const sorted = [...samples].sort((a, b) => a - b);
85
- const avg = samples.reduce((sum, s) => sum + s, 0) / samples.length;
86
- return {
87
- min: sorted[0],
88
- max: sorted[sorted.length - 1],
89
- avg,
90
- p50: percentile$1(sorted, .5),
91
- p75: percentile$1(sorted, .75),
92
- p99: percentile$1(sorted, .99),
93
- p999: percentile$1(sorted, .999)
94
- };
95
- }
96
- /** @return timing samples and amortized allocation from benchmark execution */
97
- async function collectSamples(p) {
98
- if (!p.maxIterations && !p.maxTime) throw new Error(`At least one of maxIterations or maxTime must be set`);
99
- const warmupSamples = p.skipWarmup ? [] : await runWarmup(p);
100
- const heapBefore = process.memoryUsage().heapUsed;
101
- const { samples, heapSamples, timestamps, optStatuses, pausePoints } = await runSampleLoop(p);
102
- const heapGrowth = Math.max(0, process.memoryUsage().heapUsed - heapBefore) / 1024 / samples.length;
103
- if (samples.length === 0) throw new Error(`No samples collected for benchmark: ${p.benchmark.name}`);
104
- return {
105
- samples,
106
- warmupSamples,
107
- heapGrowth,
108
- heapSamples,
109
- timestamps,
110
- optStatus: p.traceOpt ? analyzeOptStatus(samples, optStatuses) : void 0,
111
- optSamples: p.traceOpt && optStatuses.length > 0 ? optStatuses : void 0,
112
- pausePoints
113
- };
114
- }
115
- function buildMeasuredResults(name, c) {
116
- const time = computeStats(c.samples);
117
- return {
118
- name,
119
- samples: c.samples,
120
- warmupSamples: c.warmupSamples,
121
- heapSamples: c.heapSamples,
122
- timestamps: c.timestamps,
123
- time,
124
- heapSize: {
125
- avg: c.heapGrowth,
126
- min: c.heapGrowth,
127
- max: c.heapGrowth
128
- },
129
- optStatus: c.optStatus,
130
- optSamples: c.optSamples,
131
- pausePoints: c.pausePoints
132
- };
133
- }
134
- /** @return percentile value with linear interpolation */
135
- function percentile$1(sortedArray, p) {
136
- const index = (sortedArray.length - 1) * p;
137
- const lower = Math.floor(index);
138
- const upper = Math.ceil(index);
139
- const weight = index % 1;
140
- if (upper >= sortedArray.length) return sortedArray[sortedArray.length - 1];
141
- return sortedArray[lower] * (1 - weight) + sortedArray[upper] * weight;
142
- }
143
- /** Run warmup iterations with gc + settle time for V8 optimization */
144
- async function runWarmup(p) {
145
- const gc = gcFunction();
146
- const samples = new Array(p.warmup);
147
- for (let i = 0; i < p.warmup; i++) {
148
- const start = performance.now();
149
- executeBenchmark(p.benchmark, p.params);
150
- samples[i] = performance.now() - start;
151
- }
152
- gc();
153
- if (!p.noSettle) {
154
- await new Promise((r) => setTimeout(r, gcSettleTime));
155
- gc();
156
- }
157
- return samples;
158
- }
159
- /** Collect timing samples with periodic pauses for V8 optimization */
160
- async function runSampleLoop(p) {
161
- const { maxTime, maxIterations, pauseFirst, pauseInterval = 0, pauseDuration = 100 } = p;
162
- const trackHeap = true;
163
- const getOptStatus = p.traceOpt ? createOptStatusGetter() : void 0;
164
- const a = createSampleArrays(estimateSampleCount(maxTime, maxIterations), trackHeap, !!getOptStatus);
165
- let count = 0;
166
- let elapsed = 0;
167
- let totalPauseTime = 0;
168
- const loopStart = performance.now();
169
- while ((!maxIterations || count < maxIterations) && (!maxTime || elapsed < maxTime)) {
170
- const start = performance.now();
171
- executeBenchmark(p.benchmark, p.params);
172
- const end = performance.now();
173
- a.samples[count] = end - start;
174
- a.timestamps[count] = Number(process.hrtime.bigint() / 1000n);
175
- a.heapSamples[count] = getHeapStatistics().used_heap_size;
176
- if (getOptStatus) a.optStatuses[count] = getOptStatus(p.benchmark.fn);
177
- count++;
178
- if (shouldPause(count, pauseFirst, pauseInterval)) {
179
- a.pausePoints.push({
180
- sampleIndex: count - 1,
181
- durationMs: pauseDuration
182
- });
183
- const pauseStart = performance.now();
184
- await new Promise((r) => setTimeout(r, pauseDuration));
185
- totalPauseTime += performance.now() - pauseStart;
186
- }
187
- elapsed = performance.now() - loopStart - totalPauseTime;
188
- }
189
- trimArrays(a, count, trackHeap, !!getOptStatus);
190
- return {
191
- samples: a.samples,
192
- heapSamples: a.heapSamples,
193
- timestamps: a.timestamps,
194
- optStatuses: a.optStatuses,
195
- pausePoints: a.pausePoints
196
- };
197
- }
198
- /** @return analysis of V8 optimization status per sample */
199
- function analyzeOptStatus(samples, statuses) {
200
- if (statuses.length === 0 || statuses[0] === void 0) return void 0;
201
- const byStatusCode = /* @__PURE__ */ new Map();
202
- let deoptCount = 0;
203
- for (let i = 0; i < samples.length; i++) {
204
- const status = statuses[i];
205
- if (status === void 0) continue;
206
- if (status & 8) deoptCount++;
207
- if (!byStatusCode.has(status)) byStatusCode.set(status, []);
208
- byStatusCode.get(status).push(samples[i]);
209
- }
210
- const byTier = {};
211
- for (const [status, times] of byStatusCode) {
212
- const name = statusNames[status] || `status=${status}`;
213
- const sorted = [...times].sort((a, b) => a - b);
214
- const median = sorted[Math.floor(sorted.length / 2)];
215
- byTier[name] = {
216
- count: times.length,
217
- medianMs: median
218
- };
219
- }
220
- return {
221
- byTier,
222
- deoptCount
223
- };
224
- }
225
- /** @return runtime gc() function, or no-op if unavailable */
226
- function gcFunction() {
227
- const gc = globalThis.gc || globalThis.__gc;
228
- if (gc) return gc;
229
- console.warn("gc() not available, run node/bun with --expose-gc");
230
- return () => {};
231
- }
232
- /** @return function to get V8 optimization status (requires --allow-natives-syntax) */
233
- function createOptStatusGetter() {
234
- try {
235
- const getter = new Function("f", "return %GetOptimizationStatus(f)");
236
- getter(() => {});
237
- return getter;
238
- } catch {
239
- return;
240
- }
241
- }
242
- /** Estimate sample count for pre-allocation */
243
- function estimateSampleCount(maxTime, maxIterations) {
244
- return maxIterations || Math.ceil(maxTime / .1);
245
- }
246
- /** Pre-allocate arrays to reduce GC pressure during measurement */
247
- function createSampleArrays(n, trackHeap, trackOpt) {
248
- const arr = (track) => track ? new Array(n) : [];
249
- return {
250
- samples: new Array(n),
251
- timestamps: new Array(n),
252
- heapSamples: arr(trackHeap),
253
- optStatuses: arr(trackOpt),
254
- pausePoints: []
255
- };
256
- }
257
- /** Check if we should pause at this iteration for V8 optimization */
258
- function shouldPause(iter, first, interval) {
259
- if (first !== void 0 && iter === first) return true;
260
- if (interval <= 0) return false;
261
- if (first === void 0) return iter % interval === 0;
262
- return (iter - first) % interval === 0;
263
- }
264
- /** Trim arrays to actual sample count */
265
- function trimArrays(a, count, trackHeap, trackOpt) {
266
- a.samples.length = a.timestamps.length = count;
267
- if (trackHeap) a.heapSamples.length = count;
268
- if (trackOpt) a.optStatuses.length = count;
269
- }
270
-
271
- //#endregion
272
- //#region src/StatisticalUtils.ts
273
- const confidence = .95;
274
- const bootstrapSamples = 1e4;
275
- /** @return relative standard deviation (coefficient of variation) */
276
- function coefficientOfVariation(samples) {
277
- const mean = average(samples);
278
- if (mean === 0) return 0;
279
- return standardDeviation(samples) / mean;
280
- }
281
- /** @return median absolute deviation for robust variability measure */
282
- function medianAbsoluteDeviation(samples) {
283
- const median = percentile(samples, .5);
284
- return percentile(samples.map((x) => Math.abs(x - median)), .5);
285
- }
286
- /** @return mean of values */
287
- function average(values) {
288
- return values.reduce((a, b) => a + b, 0) / values.length;
289
- }
290
- /** @return standard deviation with Bessel's correction */
291
- function standardDeviation(samples) {
292
- if (samples.length <= 1) return 0;
293
- const mean = average(samples);
294
- const variance = samples.reduce((sum, x) => sum + (x - mean) ** 2, 0) / (samples.length - 1);
295
- return Math.sqrt(variance);
296
- }
297
- /** @return value at percentile p (0-1) */
298
- function percentile(values, p) {
299
- const sorted = [...values].sort((a, b) => a - b);
300
- const index = Math.ceil(sorted.length * p) - 1;
301
- return sorted[Math.max(0, index)];
302
- }
303
- /** @return bootstrap resample with replacement */
304
- function createResample(samples) {
305
- const n = samples.length;
306
- const rand = () => samples[Math.floor(Math.random() * n)];
307
- return Array.from({ length: n }, rand);
308
- }
309
- /** @return bootstrap CI for percentage difference between baseline and current medians */
310
- function bootstrapDifferenceCI(baseline, current, options = {}) {
311
- const { resamples = bootstrapSamples, confidence: conf = confidence } = options;
312
- const baselineMedian = percentile(baseline, .5);
313
- const observedPercent = (percentile(current, .5) - baselineMedian) / baselineMedian * 100;
314
- const diffs = [];
315
- for (let i = 0; i < resamples; i++) {
316
- const resB = createResample(baseline);
317
- const resC = createResample(current);
318
- const medB = percentile(resB, .5);
319
- const medC = percentile(resC, .5);
320
- diffs.push((medC - medB) / medB * 100);
321
- }
322
- const ci = computeInterval(diffs, conf);
323
- const excludesZero = ci[0] > 0 || ci[1] < 0;
324
- let direction = "uncertain";
325
- if (excludesZero) direction = observedPercent < 0 ? "faster" : "slower";
326
- const histogram = binValues(diffs);
327
- return {
328
- percent: observedPercent,
329
- ci,
330
- direction,
331
- histogram
332
- };
333
- }
334
- /** @return confidence interval [lower, upper] */
335
- function computeInterval(medians, confidence) {
336
- const alpha = (1 - confidence) / 2;
337
- return [percentile(medians, alpha), percentile(medians, 1 - alpha)];
338
- }
339
- /** Bin values into histogram for compact visualization */
340
- function binValues(values, binCount = 30) {
341
- const sorted = [...values].sort((a, b) => a - b);
342
- const min = sorted[0];
343
- const max = sorted[sorted.length - 1];
344
- if (min === max) return [{
345
- x: min,
346
- count: values.length
347
- }];
348
- const step = (max - min) / binCount;
349
- const counts = new Array(binCount).fill(0);
350
- for (const v of values) {
351
- const bin = Math.min(Math.floor((v - min) / step), binCount - 1);
352
- counts[bin]++;
353
- }
354
- return counts.map((count, i) => ({
355
- x: min + (i + .5) * step,
356
- count
357
- }));
358
- }
359
-
360
- //#endregion
361
- //#region src/runners/RunnerUtils.ts
362
- const msToNs = 1e6;
363
-
364
- //#endregion
365
- //#region src/runners/AdaptiveWrapper.ts
366
- const minTime = 1e3;
367
- const maxTime = 1e4;
368
- const targetConfidence = 95;
369
- const fallbackThreshold = 80;
370
- const windowSize = 50;
371
- const stability = .05;
372
- const initialBatch = 100;
373
- const continueBatch = 100;
374
- const continueIterations = 10;
375
- /** @return adaptive sampling runner wrapper */
376
- function createAdaptiveWrapper(baseRunner, options) {
377
- return { async runBench(benchmark, runnerOptions, params) {
378
- return runAdaptiveBench(baseRunner, benchmark, runnerOptions, options, params);
379
- } };
380
- }
381
- /** @return convergence based on window stability */
382
- function checkConvergence(samples) {
383
- const windowSize = getWindowSize(samples);
384
- const minSamples = windowSize * 2;
385
- if (samples.length < minSamples) return buildProgressResult(samples.length, minSamples);
386
- return buildConvergence(getStability(samples, windowSize));
387
- }
388
- /** @return results using adaptive sampling strategy */
389
- async function runAdaptiveBench(baseRunner, benchmark, runnerOptions, options, params) {
390
- const { minTime: min = options.minTime ?? minTime, maxTime: max = options.maxTime ?? maxTime, targetConfidence: target = options.convergence ?? targetConfidence } = runnerOptions;
391
- const allSamples = [];
392
- const warmup = await collectInitial(baseRunner, benchmark, runnerOptions, params, allSamples);
393
- const startTime = performance.now();
394
- await collectAdaptive(baseRunner, benchmark, runnerOptions, params, allSamples, {
395
- minTime: min,
396
- maxTime: max,
397
- targetConfidence: target,
398
- startTime
399
- });
400
- return buildResults(allSamples, startTime, checkConvergence(allSamples.map((s) => s * msToNs)), benchmark.name, warmup);
401
- }
402
- /** @return window size scaled to execution time */
403
- function getWindowSize(samples) {
404
- if (samples.length < 20) return windowSize;
405
- const recentMedian = percentile(samples.slice(-20).map((s) => s / msToNs), .5);
406
- if (recentMedian < .01) return 200;
407
- if (recentMedian < .1) return 100;
408
- if (recentMedian < 1) return 50;
409
- if (recentMedian < 10) return 30;
410
- return 20;
411
- }
412
- /** @return progress when samples insufficient */
413
- function buildProgressResult(currentSamples, minSamples) {
414
- return {
415
- converged: false,
416
- confidence: currentSamples / minSamples * 100,
417
- reason: `Collecting samples: ${currentSamples}/${minSamples}`
418
- };
419
- }
420
- /** @return stability metrics between windows */
421
- function getStability(samples, windowSize) {
422
- const recent = samples.slice(-windowSize);
423
- const previous = samples.slice(-windowSize * 2, -windowSize);
424
- const recentMs = recent.map((s) => s / msToNs);
425
- const previousMs = previous.map((s) => s / msToNs);
426
- const medianRecent = percentile(recentMs, .5);
427
- const medianPrevious = percentile(previousMs, .5);
428
- const medianDrift = Math.abs(medianRecent - medianPrevious) / medianPrevious;
429
- const impactRecent = getOutlierImpact(recentMs);
430
- const impactPrevious = getOutlierImpact(previousMs);
431
- const impactDrift = Math.abs(impactRecent.ratio - impactPrevious.ratio);
432
- return {
433
- medianDrift,
434
- impactDrift,
435
- medianStable: medianDrift < stability,
436
- impactStable: impactDrift < stability
437
- };
438
- }
439
- /** @return convergence from stability metrics */
440
- function buildConvergence(metrics) {
441
- const { medianDrift, impactDrift, medianStable, impactStable } = metrics;
442
- if (medianStable && impactStable) return {
443
- converged: true,
444
- confidence: 100,
445
- reason: "Stable performance pattern"
446
- };
447
- const confidence = Math.min(100, (1 - medianDrift / stability) * 50 + (1 - impactDrift / stability) * 50);
448
- const reason = medianDrift > impactDrift ? `Median drifting: ${(medianDrift * 100).toFixed(1)}%` : `Outlier impact changing: ${(impactDrift * 100).toFixed(1)}%`;
449
- return {
450
- converged: false,
451
- confidence: Math.max(0, confidence),
452
- reason
453
- };
454
- }
455
- /** @return warmupSamples from initial batch */
456
- async function collectInitial(baseRunner, benchmark, runnerOptions, params, allSamples) {
457
- const opts = {
458
- ...runnerOptions,
459
- maxTime: initialBatch,
460
- maxIterations: void 0
461
- };
462
- const results = await baseRunner.runBench(benchmark, opts, params);
463
- appendSamples(results[0], allSamples);
464
- return results[0].warmupSamples;
465
- }
466
- /** @return samples until convergence or timeout */
467
- async function collectAdaptive(baseRunner, benchmark, runnerOptions, params, allSamples, limits) {
468
- const { minTime, maxTime, targetConfidence, startTime } = limits;
469
- let lastLog = 0;
470
- while (performance.now() - startTime < maxTime) {
471
- const convergence = checkConvergence(allSamples.map((s) => s * msToNs));
472
- const elapsed = performance.now() - startTime;
473
- if (elapsed - lastLog > 1e3) {
474
- const elapsedSec = (elapsed / 1e3).toFixed(1);
475
- const conf = convergence.confidence.toFixed(0);
476
- process.stderr.write(`\r◊ ${benchmark.name}: ${conf}% confident (${elapsedSec}s) `);
477
- lastLog = elapsed;
478
- }
479
- if (shouldStop(convergence, targetConfidence, elapsed, minTime)) break;
480
- const opts = {
481
- ...runnerOptions,
482
- maxTime: continueBatch,
483
- maxIterations: continueIterations,
484
- skipWarmup: true
485
- };
486
- appendSamples((await baseRunner.runBench(benchmark, opts, params))[0], allSamples);
487
- }
488
- process.stderr.write("\r" + " ".repeat(60) + "\r");
489
- }
490
- /** @return measured results with convergence metrics */
491
- function buildResults(samplesMs, startTime, convergence, name, warmupSamples) {
492
- const totalTime = (performance.now() - startTime) / 1e3;
493
- return [{
494
- name,
495
- samples: samplesMs,
496
- warmupSamples,
497
- time: computeTimeStats(samplesMs.map((s) => s * msToNs)),
498
- totalTime,
499
- convergence
500
- }];
501
- }
502
- /** @return outlier impact as proportion of total time */
503
- function getOutlierImpact(samples) {
504
- if (samples.length === 0) return {
505
- ratio: 0,
506
- count: 0
507
- };
508
- const median = percentile(samples, .5);
509
- const threshold = median + 1.5 * (percentile(samples, .75) - median);
510
- let excessTime = 0;
511
- let count = 0;
512
- for (const sample of samples) if (sample > threshold) {
513
- excessTime += sample - median;
514
- count++;
515
- }
516
- const totalTime = samples.reduce((a, b) => a + b, 0);
517
- return {
518
- ratio: totalTime > 0 ? excessTime / totalTime : 0,
519
- count
520
- };
521
- }
522
- /** Append samples one-by-one to avoid stack overflow from spread on large arrays */
523
- function appendSamples(result, samples) {
524
- if (!result.samples?.length) return;
525
- for (const sample of result.samples) samples.push(sample);
526
- }
527
- /** @return true if convergence reached or timeout */
528
- function shouldStop(convergence, targetConfidence, elapsedTime, minTime) {
529
- if (convergence.converged && convergence.confidence >= targetConfidence) return true;
530
- const threshold = Math.max(targetConfidence, fallbackThreshold);
531
- return elapsedTime >= minTime && convergence.confidence >= threshold;
532
- }
533
- /** @return time percentiles and statistics in ms */
534
- function computeTimeStats(samplesNs) {
535
- const samplesMs = samplesNs.map((s) => s / msToNs);
536
- const { min, max, sum } = getMinMaxSum(samplesNs);
537
- const percentiles = getPercentiles(samplesNs);
538
- const robust = getRobustMetrics(samplesMs);
539
- return {
540
- min: min / msToNs,
541
- max: max / msToNs,
542
- avg: sum / samplesNs.length / msToNs,
543
- ...percentiles,
544
- ...robust
545
- };
546
- }
547
- /** @return min, max, sum of samples */
548
- function getMinMaxSum(samples) {
549
- return {
550
- min: samples.reduce((a, b) => Math.min(a, b), Number.POSITIVE_INFINITY),
551
- max: samples.reduce((a, b) => Math.max(a, b), Number.NEGATIVE_INFINITY),
552
- sum: samples.reduce((a, b) => a + b, 0)
553
- };
554
- }
555
- /** @return percentiles in ms */
556
- function getPercentiles(samples) {
557
- return {
558
- p25: percentile(samples, .25) / msToNs,
559
- p50: percentile(samples, .5) / msToNs,
560
- p75: percentile(samples, .75) / msToNs,
561
- p95: percentile(samples, .95) / msToNs,
562
- p99: percentile(samples, .99) / msToNs,
563
- p999: percentile(samples, .999) / msToNs
564
- };
565
- }
566
- /** @return robust variability metrics */
567
- function getRobustMetrics(samplesMs) {
568
- const impact = getOutlierImpact(samplesMs);
569
- return {
570
- cv: coefficientOfVariation(samplesMs),
571
- mad: medianAbsoluteDeviation(samplesMs),
572
- outlierRate: impact.ratio
573
- };
574
- }
575
-
576
- //#endregion
577
- //#region src/runners/CreateRunner.ts
578
- /** @return benchmark runner */
579
- async function createRunner(_runnerName) {
580
- return new BasicRunner();
581
- }
582
-
583
- //#endregion
584
- //#region src/runners/TimingUtils.ts
585
- const debugWorkerTiming = false;
586
- /** Get current time or 0 if debugging disabled */
587
- function getPerfNow() {
588
- return 0;
589
- }
590
- /** Calculate elapsed milliseconds between marks */
591
- function getElapsed(startMark, endMark) {
592
- return 0;
593
- }
594
-
595
- //#endregion
596
- export { createAdaptiveWrapper as a, BasicRunner as c, variantModuleUrl as d, createRunner as i, computeStats as l, getElapsed as n, average as o, getPerfNow as r, bootstrapDifferenceCI as s, debugWorkerTiming as t, discoverVariants as u };
597
- //# sourceMappingURL=TimingUtils-DwOwkc8G.mjs.map