benchforge 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +432 -0
  2. package/bin/benchforge +3 -0
  3. package/dist/bin/benchforge.mjs +9 -0
  4. package/dist/bin/benchforge.mjs.map +1 -0
  5. package/dist/browser/index.js +914 -0
  6. package/dist/index.mjs +3 -0
  7. package/dist/src-CGuaC3Wo.mjs +3676 -0
  8. package/dist/src-CGuaC3Wo.mjs.map +1 -0
  9. package/package.json +49 -0
  10. package/src/BenchMatrix.ts +380 -0
  11. package/src/Benchmark.ts +33 -0
  12. package/src/BenchmarkReport.ts +156 -0
  13. package/src/GitUtils.ts +79 -0
  14. package/src/HtmlDataPrep.ts +148 -0
  15. package/src/MeasuredResults.ts +127 -0
  16. package/src/NodeGC.ts +48 -0
  17. package/src/PermutationTest.ts +115 -0
  18. package/src/StandardSections.ts +268 -0
  19. package/src/StatisticalUtils.ts +176 -0
  20. package/src/TypeUtil.ts +8 -0
  21. package/src/bin/benchforge.ts +4 -0
  22. package/src/browser/BrowserGcStats.ts +44 -0
  23. package/src/browser/BrowserHeapSampler.ts +248 -0
  24. package/src/cli/CliArgs.ts +64 -0
  25. package/src/cli/FilterBenchmarks.ts +68 -0
  26. package/src/cli/RunBenchCLI.ts +856 -0
  27. package/src/export/JsonExport.ts +103 -0
  28. package/src/export/JsonFormat.ts +91 -0
  29. package/src/export/PerfettoExport.ts +203 -0
  30. package/src/heap-sample/HeapSampleReport.ts +196 -0
  31. package/src/heap-sample/HeapSampler.ts +78 -0
  32. package/src/html/HtmlReport.ts +131 -0
  33. package/src/html/HtmlTemplate.ts +284 -0
  34. package/src/html/Types.ts +88 -0
  35. package/src/html/browser/CIPlot.ts +287 -0
  36. package/src/html/browser/HistogramKde.ts +118 -0
  37. package/src/html/browser/LegendUtils.ts +163 -0
  38. package/src/html/browser/RenderPlots.ts +263 -0
  39. package/src/html/browser/SampleTimeSeries.ts +389 -0
  40. package/src/html/browser/Types.ts +96 -0
  41. package/src/html/browser/index.ts +1 -0
  42. package/src/html/index.ts +17 -0
  43. package/src/index.ts +92 -0
  44. package/src/matrix/CaseLoader.ts +36 -0
  45. package/src/matrix/MatrixFilter.ts +103 -0
  46. package/src/matrix/MatrixReport.ts +290 -0
  47. package/src/matrix/VariantLoader.ts +46 -0
  48. package/src/runners/AdaptiveWrapper.ts +391 -0
  49. package/src/runners/BasicRunner.ts +368 -0
  50. package/src/runners/BenchRunner.ts +60 -0
  51. package/src/runners/CreateRunner.ts +11 -0
  52. package/src/runners/GcStats.ts +107 -0
  53. package/src/runners/RunnerOrchestrator.ts +374 -0
  54. package/src/runners/RunnerUtils.ts +2 -0
  55. package/src/runners/TimingUtils.ts +13 -0
  56. package/src/runners/WorkerScript.ts +256 -0
  57. package/src/table-util/ConvergenceFormatters.ts +19 -0
  58. package/src/table-util/Formatters.ts +152 -0
  59. package/src/table-util/README.md +70 -0
  60. package/src/table-util/TableReport.ts +293 -0
  61. package/src/table-util/test/TableReport.test.ts +105 -0
  62. package/src/table-util/test/TableValueExtractor.test.ts +41 -0
  63. package/src/table-util/test/TableValueExtractor.ts +100 -0
  64. package/src/test/AdaptiveRunner.test.ts +185 -0
  65. package/src/test/AdaptiveStatistics.integration.ts +119 -0
  66. package/src/test/BenchmarkReport.test.ts +82 -0
  67. package/src/test/BrowserBench.e2e.test.ts +44 -0
  68. package/src/test/BrowserBench.test.ts +79 -0
  69. package/src/test/GcStats.test.ts +94 -0
  70. package/src/test/PermutationTest.test.ts +121 -0
  71. package/src/test/RunBenchCLI.test.ts +166 -0
  72. package/src/test/RunnerOrchestrator.test.ts +102 -0
  73. package/src/test/StatisticalUtils.test.ts +112 -0
  74. package/src/test/TestUtils.ts +93 -0
  75. package/src/test/fixtures/test-bench-script.ts +30 -0
  76. package/src/tests/AdaptiveConvergence.test.ts +177 -0
  77. package/src/tests/AdaptiveSampling.test.ts +240 -0
  78. package/src/tests/BenchMatrix.test.ts +366 -0
  79. package/src/tests/MatrixFilter.test.ts +117 -0
  80. package/src/tests/MatrixReport.test.ts +139 -0
  81. package/src/tests/RealDataValidation.test.ts +177 -0
  82. package/src/tests/fixtures/baseline/impl.ts +4 -0
  83. package/src/tests/fixtures/bevy30-samples.ts +158 -0
  84. package/src/tests/fixtures/cases/asyncCases.ts +7 -0
  85. package/src/tests/fixtures/cases/cases.ts +8 -0
  86. package/src/tests/fixtures/cases/variants/product.ts +2 -0
  87. package/src/tests/fixtures/cases/variants/sum.ts +2 -0
  88. package/src/tests/fixtures/discover/fast.ts +1 -0
  89. package/src/tests/fixtures/discover/slow.ts +4 -0
  90. package/src/tests/fixtures/invalid/bad.ts +1 -0
  91. package/src/tests/fixtures/loader/fast.ts +1 -0
  92. package/src/tests/fixtures/loader/slow.ts +4 -0
  93. package/src/tests/fixtures/loader/stateful.ts +2 -0
  94. package/src/tests/fixtures/stateful/stateful.ts +2 -0
  95. package/src/tests/fixtures/variants/extra.ts +1 -0
  96. package/src/tests/fixtures/variants/impl.ts +1 -0
  97. package/src/tests/fixtures/worker/fast.ts +1 -0
  98. package/src/tests/fixtures/worker/slow.ts +4 -0
@@ -0,0 +1,100 @@
1
+ /** Extract numeric values from benchmark tables with Unicode borders. */
2
+
3
+ /** Extract a numeric value from a table by row name and column header. */
4
+ export function extractValue(
5
+ table: string,
6
+ row: string,
7
+ column: string,
8
+ group?: string,
9
+ ): number | undefined {
10
+ const lines = trimmedBody(table);
11
+ const dataRow = lines.find(l => l.includes(row));
12
+ if (!dataRow) return undefined;
13
+
14
+ const colIndex = group
15
+ ? getGroupColumnIndex(lines, group, column)
16
+ : getColumnIndex(
17
+ lines.find(line => line.includes(column)),
18
+ column,
19
+ );
20
+
21
+ if (colIndex === undefined) return undefined;
22
+ return parseCell(dataRow, colIndex);
23
+ }
24
+
25
+ /** @return the table lines w/borders and blank rows removed */
26
+ function trimmedBody(table: string): string[] {
27
+ return table
28
+ .split("\n")
29
+ .filter(line => line.includes("║"))
30
+ .map(line => line.replaceAll("║", "").trim())
31
+ .filter(line => !line.match(/^[\s│]+$/));
32
+ }
33
+
34
+ /** Get the column index for a specific group and column combination. */
35
+ function getGroupColumnIndex(
36
+ lines: string[],
37
+ group: string,
38
+ column: string,
39
+ ): number | undefined {
40
+ // assume the first line with the group or column name is the header line
41
+ const groupLine = lines.find(line => line.includes(group));
42
+ const columnLine = lines.find(line => line.includes(column));
43
+ if (!columnLine || !groupLine) return undefined;
44
+
45
+ const groupHeaders = splitColumnGroups(groupLine);
46
+ const groupedColumns = splitColumnGroups(columnLine);
47
+
48
+ const groupIndex = groupHeaders.findIndex(col => col.includes(group));
49
+ if (groupIndex === -1) return undefined;
50
+
51
+ const columnsBefore = countColumnsBeforeGroup(groupedColumns, groupIndex);
52
+ const columnHeaders = splitColumns(columnLine);
53
+ return columnHeaders.findIndex(
54
+ (c, i) => c.includes(column) && i >= columnsBefore,
55
+ );
56
+ }
57
+
58
+ /** Count total columns in groups before the target group index. */
59
+ function countColumnsBeforeGroup(
60
+ headers: string[],
61
+ groupIndex: number,
62
+ ): number {
63
+ const perGroup = headers.map(col => splitColumns(col).length);
64
+ return perGroup.slice(0, groupIndex).reduce((sum, n) => sum + n, 0);
65
+ }
66
+
67
+ /** Find a column's position index in a header line. */
68
+ function getColumnIndex(
69
+ header: string | undefined,
70
+ column: string,
71
+ ): number | undefined {
72
+ if (!header) return undefined;
73
+
74
+ const columns = splitColumns(header);
75
+ const index = columns.findIndex(col => col.includes(column));
76
+ return index !== -1 ? index : undefined;
77
+ }
78
+
79
+ /** Extract and parse the numeric value from a cell at the given column index. */
80
+ function parseCell(row: string, index: number): number | undefined {
81
+ const text = splitColumns(row)[index];
82
+ if (!text) return undefined;
83
+ const match = text.match(/[\d,]+\.?\d*/);
84
+ if (!match) return undefined;
85
+ const value = Number.parseFloat(match[0].replace(/,/g, ""));
86
+ return Number.isNaN(value) ? undefined : value;
87
+ }
88
+
89
+ /** split column groups along '│' borders */
90
+ function splitColumnGroups(line: string): string[] {
91
+ return line.split("│").map(col => col.trim());
92
+ }
93
+
94
+ /** Split on 2+ whitespace or '│' borders, so single-space titles like "L1 miss" survive. */
95
+ function splitColumns(line: string): string[] {
96
+ return line
97
+ .split(/(?:[\s│]{2,}|│)/)
98
+ .map(col => col.trim())
99
+ .filter(Boolean);
100
+ }
@@ -0,0 +1,185 @@
1
+ import { expect, test } from "vitest";
2
+ import type { BenchmarkSpec } from "../Benchmark.ts";
3
+ import {
4
+ checkConvergence,
5
+ createAdaptiveWrapper,
6
+ } from "../runners/AdaptiveWrapper.ts";
7
+ import { BasicRunner } from "../runners/BasicRunner.ts";
8
+
9
+ test(
10
+ "adaptive runner collects samples for minimum time",
11
+ { timeout: 10000 },
12
+ async () => {
13
+ const runner = new BasicRunner();
14
+ const adaptive = createAdaptiveWrapper(runner, {
15
+ minTime: 100,
16
+ maxTime: 300,
17
+ });
18
+
19
+ const benchmark: BenchmarkSpec = {
20
+ name: "test-min-time",
21
+ fn: () => {
22
+ let sum = 0;
23
+ for (let i = 0; i < 1000; i++) sum += i;
24
+ return sum;
25
+ },
26
+ };
27
+
28
+ const start = performance.now();
29
+ const results = await adaptive.runBench(benchmark, { minTime: 100 });
30
+ const elapsed = performance.now() - start;
31
+
32
+ expect(results).toHaveLength(1);
33
+ expect(results[0].samples.length).toBeGreaterThan(0);
34
+ expect(elapsed).toBeGreaterThanOrEqual(100);
35
+ },
36
+ );
37
+
38
+ test("adaptive runner respects max time limit", async () => {
39
+ const runner = new BasicRunner();
40
+ const adaptive = createAdaptiveWrapper(runner, {
41
+ minTime: 100,
42
+ maxTime: 2000,
43
+ });
44
+
45
+ const benchmark: BenchmarkSpec = {
46
+ name: "test-max-time",
47
+ fn: () => {
48
+ let sum = 0;
49
+ for (let i = 0; i < 10000; i++) sum += Math.sqrt(i);
50
+ return sum;
51
+ },
52
+ };
53
+
54
+ const results = await adaptive.runBench(benchmark, {
55
+ minTime: 250,
56
+ maxTime: 500,
57
+ });
58
+
59
+ expect(results).toHaveLength(1);
60
+ expect(results[0].totalTime).toBeGreaterThanOrEqual(0);
61
+ // 1s warmup overhead + 500ms maxTime + some tolerance
62
+ expect(results[0].totalTime).toBeLessThanOrEqual(2.0);
63
+ });
64
+
65
+ test("adaptive runner merges results correctly", async () => {
66
+ const runner = new BasicRunner();
67
+ const adaptive = createAdaptiveWrapper(runner, {
68
+ minTime: 100,
69
+ maxTime: 200,
70
+ });
71
+
72
+ const benchmark: BenchmarkSpec = {
73
+ name: "test-merge",
74
+ fn: () => {
75
+ let sum = 0;
76
+ for (let i = 0; i < 100; i++) sum += i;
77
+ return sum;
78
+ },
79
+ };
80
+
81
+ const results = await adaptive.runBench(benchmark, { minTime: 50 });
82
+
83
+ expect(results).toHaveLength(1);
84
+ const result = results[0];
85
+
86
+ expect(result.samples.length).toBeGreaterThan(0);
87
+ expect(result.time.min).toBeLessThanOrEqual(result.time.max);
88
+ expect(result.time.avg).toBeGreaterThan(0);
89
+ expect(result.time.p50).toBeDefined();
90
+ expect(result.time.p99).toBeDefined();
91
+
92
+ if (result.time.p25 !== undefined && result.time.p75 !== undefined) {
93
+ expect(result.time.p25).toBeLessThanOrEqual(result.time.p50);
94
+ expect(result.time.p50).toBeLessThanOrEqual(result.time.p75);
95
+ }
96
+ if (result.time.p99 !== undefined && result.time.p999 !== undefined) {
97
+ expect(result.time.p99).toBeLessThanOrEqual(result.time.p999);
98
+ }
99
+
100
+ expect(result.totalTime).toBeDefined();
101
+ expect(result.totalTime).toBeGreaterThan(0);
102
+ }, 10000);
103
+
104
+ test("convergence detection with stable benchmark", async () => {
105
+ const runner = new BasicRunner();
106
+ const adaptive = createAdaptiveWrapper(runner, {
107
+ minTime: 100,
108
+ maxTime: 2000,
109
+ targetConfidence: 95,
110
+ });
111
+
112
+ const benchmark: BenchmarkSpec = {
113
+ name: "stable-convergence-test",
114
+ fn: () => {
115
+ let sum = 0;
116
+ for (let i = 0; i < 100; i++) sum += i;
117
+ return sum;
118
+ },
119
+ };
120
+
121
+ const results = await adaptive.runBench(benchmark, { minTime: 50 });
122
+
123
+ expect(results).toHaveLength(1);
124
+ const result = results[0];
125
+
126
+ expect(result.convergence).toBeDefined();
127
+ expect(result.convergence?.confidence).toBeGreaterThanOrEqual(0);
128
+ expect(result.convergence?.confidence).toBeLessThanOrEqual(100);
129
+ expect(result.convergence?.reason).toBeDefined();
130
+ });
131
+
132
+ test("convergence detection with variable benchmark", async () => {
133
+ const runner = new BasicRunner();
134
+ const adaptive = createAdaptiveWrapper(runner, {
135
+ minTime: 100,
136
+ maxTime: 1000,
137
+ });
138
+
139
+ let callCount = 0;
140
+ const benchmark: BenchmarkSpec = {
141
+ name: "variable-convergence-test",
142
+ fn: () => {
143
+ // Variable operation - alternates between fast and slow
144
+ callCount++;
145
+ const iterations = callCount % 10 === 0 ? 1000 : 100;
146
+ let sum = 0;
147
+ for (let i = 0; i < iterations; i++) {
148
+ sum += i;
149
+ }
150
+ return sum;
151
+ },
152
+ };
153
+
154
+ const results = await adaptive.runBench(benchmark, { minTime: 100 });
155
+
156
+ expect(results).toHaveLength(1);
157
+ const result = results[0];
158
+
159
+ expect(result.convergence).toBeDefined();
160
+ // Pattern may be detected as stable if predictable
161
+ expect(result.convergence?.confidence).toBeGreaterThanOrEqual(0);
162
+ expect(result.convergence?.confidence).toBeLessThanOrEqual(100);
163
+ });
164
+
165
+ test("checkConvergence function basics", () => {
166
+ // Not enough samples
167
+ const fewSamples = [1e6, 1.1e6, 1e6];
168
+ const fewResult = checkConvergence(fewSamples);
169
+ expect(fewResult.converged).toBe(false);
170
+ expect(fewResult.reason).toContain("Collecting samples");
171
+
172
+ // Many stable samples
173
+ const stableSamples = Array(100).fill(1e6);
174
+ const stableResult = checkConvergence(stableSamples);
175
+ expect(stableResult.confidence).toBeGreaterThan(50);
176
+
177
+ // Variable samples - alternating pattern may be detected as stable
178
+ const variableSamples = Array.from({ length: 100 }, (_, i) =>
179
+ i % 2 === 0 ? 1e6 : 2e6,
180
+ );
181
+ const variableResult = checkConvergence(variableSamples);
182
+ // Just verify confidence is calculated, alternating may be seen as stable
183
+ expect(variableResult.confidence).toBeGreaterThanOrEqual(0);
184
+ expect(variableResult.confidence).toBeLessThanOrEqual(100);
185
+ });
@@ -0,0 +1,119 @@
1
+ import { expect, test } from "vitest";
2
+ import type { BenchSuite } from "../Benchmark.ts";
3
+ import type { BenchmarkReport } from "../BenchmarkReport.ts";
4
+ import { parseBenchArgs, runBenchmarks } from "../cli/RunBenchCLI.ts";
5
+
6
+ const statisticalSuite: BenchSuite = {
7
+ name: "Statistical Test Suite",
8
+ groups: [
9
+ {
10
+ name: "Test Group",
11
+ benchmarks: [
12
+ {
13
+ name: "stable-benchmark",
14
+ fn: () => {
15
+ let sum = 0;
16
+ for (let i = 0; i < 100; i++) sum += i;
17
+ return sum;
18
+ },
19
+ },
20
+ {
21
+ name: "variable-benchmark",
22
+ fn: () => {
23
+ const iterations = Math.random() > 0.9 ? 1000 : 100;
24
+ let sum = 0;
25
+ for (let i = 0; i < iterations; i++) sum += i;
26
+ return sum;
27
+ },
28
+ },
29
+ ],
30
+ },
31
+ ],
32
+ };
33
+
34
+ const gcSuite: BenchSuite = {
35
+ name: "GC Test Suite",
36
+ groups: [
37
+ {
38
+ name: "Memory Group",
39
+ benchmarks: [
40
+ {
41
+ name: "gc-heavy",
42
+ fn: () => {
43
+ const arrays = [];
44
+ for (let i = 0; i < 50; i++) {
45
+ arrays.push(Array.from({ length: 1000 }, () => Math.random()));
46
+ }
47
+ return arrays.length;
48
+ },
49
+ },
50
+ ],
51
+ },
52
+ ],
53
+ };
54
+
55
+ function parseAdaptiveArgs() {
56
+ return parseBenchArgs((yargs: any) =>
57
+ yargs
58
+ .option("adaptive", { type: "boolean", default: true })
59
+ .option("time", { type: "number", default: 0.1 })
60
+ .option("min-time", { type: "number", default: 0.1 }),
61
+ );
62
+ }
63
+
64
+ function verifyStatisticalMetrics(report: BenchmarkReport): void {
65
+ const { time, convergence } = report.measuredResults;
66
+ expect(time).toBeDefined();
67
+ expect(time?.p25).toBeDefined();
68
+ expect(time?.p50).toBeDefined();
69
+ expect(time?.p75).toBeDefined();
70
+ expect(time?.p95).toBeDefined();
71
+ expect(time?.cv).toBeGreaterThanOrEqual(0);
72
+ expect(time?.mad).toBeGreaterThanOrEqual(0);
73
+ expect(time?.outlierRate).toBeGreaterThanOrEqual(0);
74
+ expect(time?.outlierRate).toBeLessThanOrEqual(1);
75
+ expect(convergence?.confidence).toBeGreaterThanOrEqual(0);
76
+ expect(convergence?.confidence).toBeLessThanOrEqual(100);
77
+ expect(convergence?.reason).toBeDefined();
78
+ }
79
+
80
+ function verifyPercentileOrdering(report: BenchmarkReport): void {
81
+ const t = report.measuredResults.time;
82
+ if (t?.p25 && t?.p50 && t?.p75 && t?.p95) {
83
+ expect(t.p25).toBeLessThanOrEqual(t.p50);
84
+ expect(t.p50).toBeLessThanOrEqual(t.p75);
85
+ expect(t.p75).toBeLessThanOrEqual(t.p95);
86
+ }
87
+ }
88
+
89
+ test("adaptive mode reports statistical metrics correctly", async () => {
90
+ const results = await runBenchmarks(statisticalSuite, parseAdaptiveArgs());
91
+ expect(results).toHaveLength(1);
92
+ expect(results[0].reports).toHaveLength(2);
93
+
94
+ for (const report of results[0].reports) {
95
+ verifyStatisticalMetrics(report);
96
+ verifyPercentileOrdering(report);
97
+ }
98
+
99
+ const reports = results[0].reports;
100
+ const stableCV = reports.find(r => r.name === "stable-benchmark")
101
+ ?.measuredResults.time?.cv;
102
+ const variableCV = reports.find(r => r.name === "variable-benchmark")
103
+ ?.measuredResults.time?.cv;
104
+ if (stableCV && variableCV) {
105
+ expect(variableCV).toBeGreaterThanOrEqual(stableCV);
106
+ }
107
+ }, 20000);
108
+
109
+ test("adaptive mode handles GC-heavy workload", async () => {
110
+ const results = await runBenchmarks(gcSuite, parseAdaptiveArgs());
111
+ expect(results).toHaveLength(1);
112
+ expect(results[0].reports).toHaveLength(1);
113
+
114
+ const gcResult = results[0].reports[0].measuredResults;
115
+ expect(gcResult.convergence).toBeDefined();
116
+ expect(gcResult.time?.outlierRate).toBeDefined();
117
+ expect(gcResult.time?.cv).toBeDefined();
118
+ expect(gcResult.time?.mad).toBeGreaterThanOrEqual(0);
119
+ }, 20000);
@@ -0,0 +1,82 @@
1
+ import { expect, test } from "vitest";
2
+ import {
3
+ type BenchmarkReport,
4
+ reportResults,
5
+ valuesForReports,
6
+ } from "../BenchmarkReport.ts";
7
+ import {
8
+ adaptiveSection,
9
+ gcSection,
10
+ timeSection,
11
+ } from "../StandardSections.ts";
12
+ import { createBenchmarkReport, createMeasuredResults } from "./TestUtils.ts";
13
+
14
+ test("combines time and gc sections into report", () => {
15
+ const sections = [timeSection, gcSection] as const;
16
+ const report = createBenchmarkReport("test", [100, 150]);
17
+ const rows = valuesForReports([report], sections);
18
+
19
+ expect(rows[0].name).toBe("test");
20
+ expect(rows[0].mean).toBeCloseTo(report.measuredResults.time.avg, 1);
21
+ expect(rows[0].p50).toBeCloseTo(report.measuredResults.time.p50, 1);
22
+ expect(rows[0].p99).toBeCloseTo(report.measuredResults.time.p99, 1);
23
+ expect(rows[0].gc).toBeDefined();
24
+ });
25
+
26
+ test("generates diff columns for baseline comparison", () => {
27
+ const faster = createMeasuredResults([250, 300]);
28
+ const slower = createMeasuredResults([0, 50]);
29
+
30
+ const scale = (results: typeof faster, factor: number) => ({
31
+ ...results,
32
+ time: {
33
+ ...results.time,
34
+ avg: results.time.avg * factor,
35
+ p50: results.time.p50 * factor,
36
+ p99: results.time.p99 * factor,
37
+ },
38
+ });
39
+
40
+ const group1Reports: BenchmarkReport[] = [
41
+ { name: "version1", measuredResults: scale(faster, 0.8) },
42
+ { name: "version2", measuredResults: scale(slower, 1.2) },
43
+ ];
44
+
45
+ const baseline = createBenchmarkReport("baseVersion", [200, 250]);
46
+ const group2Reports = [
47
+ createBenchmarkReport("test3", [300, 350]),
48
+ createBenchmarkReport("test4", [350, 400]),
49
+ ];
50
+
51
+ const groups = [
52
+ { name: "group1", reports: group1Reports, baseline },
53
+ { name: "group2", reports: group2Reports },
54
+ ];
55
+
56
+ const table = reportResults(groups, [timeSection]);
57
+ const names = ["version1", "version2", "baseVersion", "test3", "test4"];
58
+ for (const name of names) expect(table).toContain(name);
59
+ expect(table).toContain("Δ%");
60
+ });
61
+
62
+ test("formats adaptive convergence statistics", () => {
63
+ const reports: BenchmarkReport[] = [
64
+ createBenchmarkReport("test-adaptive", [400, 500], {
65
+ convergence: { converged: true, confidence: 95, reason: "stable" },
66
+ }),
67
+ createBenchmarkReport("test-low-confidence", [0, 30], {
68
+ convergence: { converged: false, confidence: 65, reason: "unstable" },
69
+ }),
70
+ ];
71
+
72
+ const rows = valuesForReports(reports, [adaptiveSection]);
73
+ expect(rows[0].convergence).toBe(95);
74
+ expect(rows[1].convergence).toBe(65);
75
+
76
+ const table = reportResults(
77
+ [{ name: "adaptive", reports }],
78
+ [adaptiveSection],
79
+ );
80
+ expect(table).toContain("95%");
81
+ expect(table).toMatch(/65%/);
82
+ });
@@ -0,0 +1,44 @@
1
+ import path from "node:path";
2
+ import { expect, test } from "vitest";
3
+ import { profileBrowser } from "../browser/BrowserHeapSampler.ts";
4
+
5
+ const examplesDir = path.resolve(import.meta.dirname!, "../../examples");
6
+
7
+ test("bench function mode (window.__bench)", { timeout: 30000 }, async () => {
8
+ const url = `file://${examplesDir}/browser-bench/index.html`;
9
+ const result = await profileBrowser({ url, maxTime: 500, gcStats: true });
10
+
11
+ expect(result.samples).toBeDefined();
12
+ expect(result.samples!.length).toBeGreaterThan(5);
13
+ expect(result.wallTimeMs).toBeGreaterThan(0);
14
+ expect(result.gcStats).toBeDefined();
15
+ expect(result.gcStats!.scavenges).toBeGreaterThanOrEqual(0);
16
+ for (const s of result.samples!) {
17
+ expect(s).toBeGreaterThan(0);
18
+ expect(s).toBeLessThan(1000);
19
+ }
20
+ });
21
+
22
+ test("lap mode with N laps", { timeout: 30000 }, async () => {
23
+ const url = `file://${examplesDir}/browser-lap/index.html`;
24
+ const result = await profileBrowser({ url, gcStats: true });
25
+
26
+ expect(result.samples).toBeDefined();
27
+ expect(result.samples!).toHaveLength(100);
28
+ expect(result.wallTimeMs).toBeGreaterThan(0);
29
+ expect(result.gcStats).toBeDefined();
30
+ for (const s of result.samples!) {
31
+ expect(s).toBeGreaterThanOrEqual(0);
32
+ }
33
+ });
34
+
35
+ test("lap mode 0 laps with heap profiling", { timeout: 30000 }, async () => {
36
+ const url = `file://${examplesDir}/browser-heap/index.html`;
37
+ const result = await profileBrowser({ url, heapSample: true });
38
+
39
+ expect(result.samples).toBeDefined();
40
+ expect(result.samples!).toHaveLength(0);
41
+ expect(result.wallTimeMs).toBeGreaterThan(0);
42
+ expect(result.heapProfile).toBeDefined();
43
+ expect(result.heapProfile!.head).toBeDefined();
44
+ });
@@ -0,0 +1,79 @@
1
+ import { expect, test } from "vitest";
2
+ import {
3
+ browserGcStats,
4
+ parseGcTraceEvents,
5
+ type TraceEvent,
6
+ } from "../browser/BrowserGcStats.ts";
7
+
8
+ test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
9
+ const events: TraceEvent[] = [
10
+ {
11
+ cat: "v8.gc",
12
+ name: "MinorGC",
13
+ ph: "X",
14
+ dur: 500,
15
+ args: { usedHeapSizeBefore: 10000, usedHeapSizeAfter: 8000 },
16
+ },
17
+ {
18
+ cat: "v8.gc",
19
+ name: "MajorGC",
20
+ ph: "X",
21
+ dur: 12000,
22
+ args: { usedHeapSizeBefore: 50000, usedHeapSizeAfter: 30000 },
23
+ },
24
+ ];
25
+ const parsed = parseGcTraceEvents(events);
26
+ expect(parsed).toHaveLength(2);
27
+ expect(parsed[0]).toEqual({
28
+ type: "scavenge",
29
+ pauseMs: 0.5,
30
+ collected: 2000,
31
+ });
32
+ expect(parsed[1]).toEqual({
33
+ type: "mark-compact",
34
+ pauseMs: 12,
35
+ collected: 20000,
36
+ });
37
+ });
38
+
39
+ test("parseGcTraceEvents ignores non-complete and non-GC events", () => {
40
+ const events: TraceEvent[] = [
41
+ { cat: "v8.gc", name: "MinorGC", ph: "B", dur: 500 }, // not complete
42
+ { cat: "v8", name: "V8.Execute", ph: "X", dur: 100 }, // not GC
43
+ { cat: "v8.gc", name: "MinorGC", ph: "X" }, // valid, missing dur/args
44
+ ];
45
+ const parsed = parseGcTraceEvents(events);
46
+ expect(parsed).toHaveLength(1);
47
+ expect(parsed[0]).toEqual({ type: "scavenge", pauseMs: 0, collected: 0 });
48
+ });
49
+
50
+ test("browserGcStats aggregates trace events into GcStats", () => {
51
+ const events: TraceEvent[] = [
52
+ {
53
+ cat: "v8.gc",
54
+ name: "MinorGC",
55
+ ph: "X",
56
+ dur: 300,
57
+ args: { usedHeapSizeBefore: 5000, usedHeapSizeAfter: 3000 },
58
+ },
59
+ {
60
+ cat: "v8.gc",
61
+ name: "MinorGC",
62
+ ph: "X",
63
+ dur: 200,
64
+ args: { usedHeapSizeBefore: 6000, usedHeapSizeAfter: 4000 },
65
+ },
66
+ {
67
+ cat: "v8.gc",
68
+ name: "MajorGC",
69
+ ph: "X",
70
+ dur: 8000,
71
+ args: { usedHeapSizeBefore: 40000, usedHeapSizeAfter: 20000 },
72
+ },
73
+ ];
74
+ const stats = browserGcStats(events);
75
+ expect(stats.scavenges).toBe(2);
76
+ expect(stats.markCompacts).toBe(1);
77
+ expect(stats.totalCollected).toBe(24000);
78
+ expect(stats.gcPauseTime).toBeCloseTo(8.5, 2);
79
+ });
@@ -0,0 +1,94 @@
1
+ import { expect, test } from "vitest";
2
+ import { aggregateGcStats, parseGcLine } from "../runners/GcStats.ts";
3
+
4
+ test("parseGcLine parses scavenge event from real V8 output", () => {
5
+ // Real V8 --trace-gc-nvp format
6
+ const line =
7
+ "[71753:0x83280c000:0] 9 ms: pause=0.5 mutator=0.1 gc=s allocated=293224 promoted=653480 new_space_survived=290176 start_object_size=4392688 end_object_size=4287840";
8
+ const event = parseGcLine(line);
9
+ expect(event).toMatchObject({
10
+ type: "scavenge",
11
+ pauseMs: 0.5,
12
+ allocated: 293224,
13
+ promoted: 653480,
14
+ survived: 290176,
15
+ collected: 4392688 - 4287840,
16
+ });
17
+ });
18
+
19
+ test("parseGcLine parses mark-sweep event", () => {
20
+ const line =
21
+ "[1234:0x12345:0] 100 ms: pause=12.3 gc=ms allocated=2097152 promoted=0 new_space_survived=0 start_object_size=5000000 end_object_size=3000000";
22
+ const event = parseGcLine(line);
23
+ expect(event).toMatchObject({
24
+ type: "mark-compact",
25
+ pauseMs: 12.3,
26
+ allocated: 2097152,
27
+ collected: 2000000,
28
+ });
29
+ });
30
+
31
+ test("parseGcLine returns undefined for non-GC lines", () => {
32
+ expect(parseGcLine("some random stderr output")).toBeUndefined();
33
+ expect(parseGcLine("")).toBeUndefined();
34
+ expect(parseGcLine(" ")).toBeUndefined();
35
+ });
36
+
37
+ test("parseGcLine handles missing fields", () => {
38
+ const line = "gc=s pause=1.0";
39
+ const event = parseGcLine(line);
40
+ expect(event).toMatchObject({
41
+ type: "scavenge",
42
+ pauseMs: 1.0,
43
+ allocated: 0,
44
+ collected: 0,
45
+ promoted: 0,
46
+ survived: 0,
47
+ });
48
+ });
49
+
50
+ test("aggregateGcStats aggregates multiple events", () => {
51
+ const scav = "scavenge" as const;
52
+ const mc = "mark-compact" as const;
53
+ const events = [
54
+ {
55
+ type: scav,
56
+ pauseMs: 0.5,
57
+ allocated: 1000,
58
+ collected: 500,
59
+ promoted: 100,
60
+ survived: 400,
61
+ },
62
+ {
63
+ type: scav,
64
+ pauseMs: 0.3,
65
+ allocated: 2000,
66
+ collected: 800,
67
+ promoted: 200,
68
+ survived: 1000,
69
+ },
70
+ {
71
+ type: mc,
72
+ pauseMs: 10.0,
73
+ allocated: 0,
74
+ collected: 5000,
75
+ promoted: 0,
76
+ survived: 0,
77
+ },
78
+ ];
79
+ const stats = aggregateGcStats(events);
80
+ expect(stats.scavenges).toBe(2);
81
+ expect(stats.markCompacts).toBe(1);
82
+ expect(stats.totalAllocated).toBe(3000);
83
+ expect(stats.totalCollected).toBe(6300);
84
+ expect(stats.totalPromoted).toBe(300);
85
+ expect(stats.totalSurvived).toBe(1400);
86
+ expect(stats.gcPauseTime).toBeCloseTo(10.8, 2);
87
+ });
88
+
89
+ test("aggregateGcStats handles empty events", () => {
90
+ const stats = aggregateGcStats([]);
91
+ expect(stats.scavenges).toBe(0);
92
+ expect(stats.markCompacts).toBe(0);
93
+ expect(stats.gcPauseTime).toBe(0);
94
+ });