benchforge 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +432 -0
- package/bin/benchforge +3 -0
- package/dist/bin/benchforge.mjs +9 -0
- package/dist/bin/benchforge.mjs.map +1 -0
- package/dist/browser/index.js +914 -0
- package/dist/index.mjs +3 -0
- package/dist/src-CGuaC3Wo.mjs +3676 -0
- package/dist/src-CGuaC3Wo.mjs.map +1 -0
- package/package.json +49 -0
- package/src/BenchMatrix.ts +380 -0
- package/src/Benchmark.ts +33 -0
- package/src/BenchmarkReport.ts +156 -0
- package/src/GitUtils.ts +79 -0
- package/src/HtmlDataPrep.ts +148 -0
- package/src/MeasuredResults.ts +127 -0
- package/src/NodeGC.ts +48 -0
- package/src/PermutationTest.ts +115 -0
- package/src/StandardSections.ts +268 -0
- package/src/StatisticalUtils.ts +176 -0
- package/src/TypeUtil.ts +8 -0
- package/src/bin/benchforge.ts +4 -0
- package/src/browser/BrowserGcStats.ts +44 -0
- package/src/browser/BrowserHeapSampler.ts +248 -0
- package/src/cli/CliArgs.ts +64 -0
- package/src/cli/FilterBenchmarks.ts +68 -0
- package/src/cli/RunBenchCLI.ts +856 -0
- package/src/export/JsonExport.ts +103 -0
- package/src/export/JsonFormat.ts +91 -0
- package/src/export/PerfettoExport.ts +203 -0
- package/src/heap-sample/HeapSampleReport.ts +196 -0
- package/src/heap-sample/HeapSampler.ts +78 -0
- package/src/html/HtmlReport.ts +131 -0
- package/src/html/HtmlTemplate.ts +284 -0
- package/src/html/Types.ts +88 -0
- package/src/html/browser/CIPlot.ts +287 -0
- package/src/html/browser/HistogramKde.ts +118 -0
- package/src/html/browser/LegendUtils.ts +163 -0
- package/src/html/browser/RenderPlots.ts +263 -0
- package/src/html/browser/SampleTimeSeries.ts +389 -0
- package/src/html/browser/Types.ts +96 -0
- package/src/html/browser/index.ts +1 -0
- package/src/html/index.ts +17 -0
- package/src/index.ts +92 -0
- package/src/matrix/CaseLoader.ts +36 -0
- package/src/matrix/MatrixFilter.ts +103 -0
- package/src/matrix/MatrixReport.ts +290 -0
- package/src/matrix/VariantLoader.ts +46 -0
- package/src/runners/AdaptiveWrapper.ts +391 -0
- package/src/runners/BasicRunner.ts +368 -0
- package/src/runners/BenchRunner.ts +60 -0
- package/src/runners/CreateRunner.ts +11 -0
- package/src/runners/GcStats.ts +107 -0
- package/src/runners/RunnerOrchestrator.ts +374 -0
- package/src/runners/RunnerUtils.ts +2 -0
- package/src/runners/TimingUtils.ts +13 -0
- package/src/runners/WorkerScript.ts +256 -0
- package/src/table-util/ConvergenceFormatters.ts +19 -0
- package/src/table-util/Formatters.ts +152 -0
- package/src/table-util/README.md +70 -0
- package/src/table-util/TableReport.ts +293 -0
- package/src/table-util/test/TableReport.test.ts +105 -0
- package/src/table-util/test/TableValueExtractor.test.ts +41 -0
- package/src/table-util/test/TableValueExtractor.ts +100 -0
- package/src/test/AdaptiveRunner.test.ts +185 -0
- package/src/test/AdaptiveStatistics.integration.ts +119 -0
- package/src/test/BenchmarkReport.test.ts +82 -0
- package/src/test/BrowserBench.e2e.test.ts +44 -0
- package/src/test/BrowserBench.test.ts +79 -0
- package/src/test/GcStats.test.ts +94 -0
- package/src/test/PermutationTest.test.ts +121 -0
- package/src/test/RunBenchCLI.test.ts +166 -0
- package/src/test/RunnerOrchestrator.test.ts +102 -0
- package/src/test/StatisticalUtils.test.ts +112 -0
- package/src/test/TestUtils.ts +93 -0
- package/src/test/fixtures/test-bench-script.ts +30 -0
- package/src/tests/AdaptiveConvergence.test.ts +177 -0
- package/src/tests/AdaptiveSampling.test.ts +240 -0
- package/src/tests/BenchMatrix.test.ts +366 -0
- package/src/tests/MatrixFilter.test.ts +117 -0
- package/src/tests/MatrixReport.test.ts +139 -0
- package/src/tests/RealDataValidation.test.ts +177 -0
- package/src/tests/fixtures/baseline/impl.ts +4 -0
- package/src/tests/fixtures/bevy30-samples.ts +158 -0
- package/src/tests/fixtures/cases/asyncCases.ts +7 -0
- package/src/tests/fixtures/cases/cases.ts +8 -0
- package/src/tests/fixtures/cases/variants/product.ts +2 -0
- package/src/tests/fixtures/cases/variants/sum.ts +2 -0
- package/src/tests/fixtures/discover/fast.ts +1 -0
- package/src/tests/fixtures/discover/slow.ts +4 -0
- package/src/tests/fixtures/invalid/bad.ts +1 -0
- package/src/tests/fixtures/loader/fast.ts +1 -0
- package/src/tests/fixtures/loader/slow.ts +4 -0
- package/src/tests/fixtures/loader/stateful.ts +2 -0
- package/src/tests/fixtures/stateful/stateful.ts +2 -0
- package/src/tests/fixtures/variants/extra.ts +1 -0
- package/src/tests/fixtures/variants/impl.ts +1 -0
- package/src/tests/fixtures/worker/fast.ts +1 -0
- package/src/tests/fixtures/worker/slow.ts +4 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/** Extract numeric values from benchmark tables with Unicode borders. */
|
|
2
|
+
|
|
3
|
+
/** Extract a numeric value from a table by row name and column header. */
|
|
4
|
+
export function extractValue(
|
|
5
|
+
table: string,
|
|
6
|
+
row: string,
|
|
7
|
+
column: string,
|
|
8
|
+
group?: string,
|
|
9
|
+
): number | undefined {
|
|
10
|
+
const lines = trimmedBody(table);
|
|
11
|
+
const dataRow = lines.find(l => l.includes(row));
|
|
12
|
+
if (!dataRow) return undefined;
|
|
13
|
+
|
|
14
|
+
const colIndex = group
|
|
15
|
+
? getGroupColumnIndex(lines, group, column)
|
|
16
|
+
: getColumnIndex(
|
|
17
|
+
lines.find(line => line.includes(column)),
|
|
18
|
+
column,
|
|
19
|
+
);
|
|
20
|
+
|
|
21
|
+
if (colIndex === undefined) return undefined;
|
|
22
|
+
return parseCell(dataRow, colIndex);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** @return the table lines w/borders and blank rows removed */
|
|
26
|
+
function trimmedBody(table: string): string[] {
|
|
27
|
+
return table
|
|
28
|
+
.split("\n")
|
|
29
|
+
.filter(line => line.includes("║"))
|
|
30
|
+
.map(line => line.replaceAll("║", "").trim())
|
|
31
|
+
.filter(line => !line.match(/^[\s│]+$/));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Get the column index for a specific group and column combination. */
|
|
35
|
+
function getGroupColumnIndex(
|
|
36
|
+
lines: string[],
|
|
37
|
+
group: string,
|
|
38
|
+
column: string,
|
|
39
|
+
): number | undefined {
|
|
40
|
+
// assume the first line with the group or column name is the header line
|
|
41
|
+
const groupLine = lines.find(line => line.includes(group));
|
|
42
|
+
const columnLine = lines.find(line => line.includes(column));
|
|
43
|
+
if (!columnLine || !groupLine) return undefined;
|
|
44
|
+
|
|
45
|
+
const groupHeaders = splitColumnGroups(groupLine);
|
|
46
|
+
const groupedColumns = splitColumnGroups(columnLine);
|
|
47
|
+
|
|
48
|
+
const groupIndex = groupHeaders.findIndex(col => col.includes(group));
|
|
49
|
+
if (groupIndex === -1) return undefined;
|
|
50
|
+
|
|
51
|
+
const columnsBefore = countColumnsBeforeGroup(groupedColumns, groupIndex);
|
|
52
|
+
const columnHeaders = splitColumns(columnLine);
|
|
53
|
+
return columnHeaders.findIndex(
|
|
54
|
+
(c, i) => c.includes(column) && i >= columnsBefore,
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Count total columns in groups before the target group index. */
|
|
59
|
+
function countColumnsBeforeGroup(
|
|
60
|
+
headers: string[],
|
|
61
|
+
groupIndex: number,
|
|
62
|
+
): number {
|
|
63
|
+
const perGroup = headers.map(col => splitColumns(col).length);
|
|
64
|
+
return perGroup.slice(0, groupIndex).reduce((sum, n) => sum + n, 0);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Find a column's position index in a header line. */
|
|
68
|
+
function getColumnIndex(
|
|
69
|
+
header: string | undefined,
|
|
70
|
+
column: string,
|
|
71
|
+
): number | undefined {
|
|
72
|
+
if (!header) return undefined;
|
|
73
|
+
|
|
74
|
+
const columns = splitColumns(header);
|
|
75
|
+
const index = columns.findIndex(col => col.includes(column));
|
|
76
|
+
return index !== -1 ? index : undefined;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Extract and parse the numeric value from a cell at the given column index. */
|
|
80
|
+
function parseCell(row: string, index: number): number | undefined {
|
|
81
|
+
const text = splitColumns(row)[index];
|
|
82
|
+
if (!text) return undefined;
|
|
83
|
+
const match = text.match(/[\d,]+\.?\d*/);
|
|
84
|
+
if (!match) return undefined;
|
|
85
|
+
const value = Number.parseFloat(match[0].replace(/,/g, ""));
|
|
86
|
+
return Number.isNaN(value) ? undefined : value;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** split column groups along '│' borders */
|
|
90
|
+
function splitColumnGroups(line: string): string[] {
|
|
91
|
+
return line.split("│").map(col => col.trim());
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Split on 2+ whitespace or '│' borders, so single-space titles like "L1 miss" survive. */
|
|
95
|
+
function splitColumns(line: string): string[] {
|
|
96
|
+
return line
|
|
97
|
+
.split(/(?:[\s│]{2,}|│)/)
|
|
98
|
+
.map(col => col.trim())
|
|
99
|
+
.filter(Boolean);
|
|
100
|
+
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { expect, test } from "vitest";
|
|
2
|
+
import type { BenchmarkSpec } from "../Benchmark.ts";
|
|
3
|
+
import {
|
|
4
|
+
checkConvergence,
|
|
5
|
+
createAdaptiveWrapper,
|
|
6
|
+
} from "../runners/AdaptiveWrapper.ts";
|
|
7
|
+
import { BasicRunner } from "../runners/BasicRunner.ts";
|
|
8
|
+
|
|
9
|
+
test(
|
|
10
|
+
"adaptive runner collects samples for minimum time",
|
|
11
|
+
{ timeout: 10000 },
|
|
12
|
+
async () => {
|
|
13
|
+
const runner = new BasicRunner();
|
|
14
|
+
const adaptive = createAdaptiveWrapper(runner, {
|
|
15
|
+
minTime: 100,
|
|
16
|
+
maxTime: 300,
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
const benchmark: BenchmarkSpec = {
|
|
20
|
+
name: "test-min-time",
|
|
21
|
+
fn: () => {
|
|
22
|
+
let sum = 0;
|
|
23
|
+
for (let i = 0; i < 1000; i++) sum += i;
|
|
24
|
+
return sum;
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const start = performance.now();
|
|
29
|
+
const results = await adaptive.runBench(benchmark, { minTime: 100 });
|
|
30
|
+
const elapsed = performance.now() - start;
|
|
31
|
+
|
|
32
|
+
expect(results).toHaveLength(1);
|
|
33
|
+
expect(results[0].samples.length).toBeGreaterThan(0);
|
|
34
|
+
expect(elapsed).toBeGreaterThanOrEqual(100);
|
|
35
|
+
},
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
test("adaptive runner respects max time limit", async () => {
|
|
39
|
+
const runner = new BasicRunner();
|
|
40
|
+
const adaptive = createAdaptiveWrapper(runner, {
|
|
41
|
+
minTime: 100,
|
|
42
|
+
maxTime: 2000,
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
const benchmark: BenchmarkSpec = {
|
|
46
|
+
name: "test-max-time",
|
|
47
|
+
fn: () => {
|
|
48
|
+
let sum = 0;
|
|
49
|
+
for (let i = 0; i < 10000; i++) sum += Math.sqrt(i);
|
|
50
|
+
return sum;
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
const results = await adaptive.runBench(benchmark, {
|
|
55
|
+
minTime: 250,
|
|
56
|
+
maxTime: 500,
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
expect(results).toHaveLength(1);
|
|
60
|
+
expect(results[0].totalTime).toBeGreaterThanOrEqual(0);
|
|
61
|
+
// 1s warmup overhead + 500ms maxTime + some tolerance
|
|
62
|
+
expect(results[0].totalTime).toBeLessThanOrEqual(2.0);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("adaptive runner merges results correctly", async () => {
|
|
66
|
+
const runner = new BasicRunner();
|
|
67
|
+
const adaptive = createAdaptiveWrapper(runner, {
|
|
68
|
+
minTime: 100,
|
|
69
|
+
maxTime: 200,
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
const benchmark: BenchmarkSpec = {
|
|
73
|
+
name: "test-merge",
|
|
74
|
+
fn: () => {
|
|
75
|
+
let sum = 0;
|
|
76
|
+
for (let i = 0; i < 100; i++) sum += i;
|
|
77
|
+
return sum;
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
const results = await adaptive.runBench(benchmark, { minTime: 50 });
|
|
82
|
+
|
|
83
|
+
expect(results).toHaveLength(1);
|
|
84
|
+
const result = results[0];
|
|
85
|
+
|
|
86
|
+
expect(result.samples.length).toBeGreaterThan(0);
|
|
87
|
+
expect(result.time.min).toBeLessThanOrEqual(result.time.max);
|
|
88
|
+
expect(result.time.avg).toBeGreaterThan(0);
|
|
89
|
+
expect(result.time.p50).toBeDefined();
|
|
90
|
+
expect(result.time.p99).toBeDefined();
|
|
91
|
+
|
|
92
|
+
if (result.time.p25 !== undefined && result.time.p75 !== undefined) {
|
|
93
|
+
expect(result.time.p25).toBeLessThanOrEqual(result.time.p50);
|
|
94
|
+
expect(result.time.p50).toBeLessThanOrEqual(result.time.p75);
|
|
95
|
+
}
|
|
96
|
+
if (result.time.p99 !== undefined && result.time.p999 !== undefined) {
|
|
97
|
+
expect(result.time.p99).toBeLessThanOrEqual(result.time.p999);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
expect(result.totalTime).toBeDefined();
|
|
101
|
+
expect(result.totalTime).toBeGreaterThan(0);
|
|
102
|
+
}, 10000);
|
|
103
|
+
|
|
104
|
+
test("convergence detection with stable benchmark", async () => {
|
|
105
|
+
const runner = new BasicRunner();
|
|
106
|
+
const adaptive = createAdaptiveWrapper(runner, {
|
|
107
|
+
minTime: 100,
|
|
108
|
+
maxTime: 2000,
|
|
109
|
+
targetConfidence: 95,
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
const benchmark: BenchmarkSpec = {
|
|
113
|
+
name: "stable-convergence-test",
|
|
114
|
+
fn: () => {
|
|
115
|
+
let sum = 0;
|
|
116
|
+
for (let i = 0; i < 100; i++) sum += i;
|
|
117
|
+
return sum;
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const results = await adaptive.runBench(benchmark, { minTime: 50 });
|
|
122
|
+
|
|
123
|
+
expect(results).toHaveLength(1);
|
|
124
|
+
const result = results[0];
|
|
125
|
+
|
|
126
|
+
expect(result.convergence).toBeDefined();
|
|
127
|
+
expect(result.convergence?.confidence).toBeGreaterThanOrEqual(0);
|
|
128
|
+
expect(result.convergence?.confidence).toBeLessThanOrEqual(100);
|
|
129
|
+
expect(result.convergence?.reason).toBeDefined();
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("convergence detection with variable benchmark", async () => {
|
|
133
|
+
const runner = new BasicRunner();
|
|
134
|
+
const adaptive = createAdaptiveWrapper(runner, {
|
|
135
|
+
minTime: 100,
|
|
136
|
+
maxTime: 1000,
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
let callCount = 0;
|
|
140
|
+
const benchmark: BenchmarkSpec = {
|
|
141
|
+
name: "variable-convergence-test",
|
|
142
|
+
fn: () => {
|
|
143
|
+
// Variable operation - alternates between fast and slow
|
|
144
|
+
callCount++;
|
|
145
|
+
const iterations = callCount % 10 === 0 ? 1000 : 100;
|
|
146
|
+
let sum = 0;
|
|
147
|
+
for (let i = 0; i < iterations; i++) {
|
|
148
|
+
sum += i;
|
|
149
|
+
}
|
|
150
|
+
return sum;
|
|
151
|
+
},
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
const results = await adaptive.runBench(benchmark, { minTime: 100 });
|
|
155
|
+
|
|
156
|
+
expect(results).toHaveLength(1);
|
|
157
|
+
const result = results[0];
|
|
158
|
+
|
|
159
|
+
expect(result.convergence).toBeDefined();
|
|
160
|
+
// Pattern may be detected as stable if predictable
|
|
161
|
+
expect(result.convergence?.confidence).toBeGreaterThanOrEqual(0);
|
|
162
|
+
expect(result.convergence?.confidence).toBeLessThanOrEqual(100);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
test("checkConvergence function basics", () => {
|
|
166
|
+
// Not enough samples
|
|
167
|
+
const fewSamples = [1e6, 1.1e6, 1e6];
|
|
168
|
+
const fewResult = checkConvergence(fewSamples);
|
|
169
|
+
expect(fewResult.converged).toBe(false);
|
|
170
|
+
expect(fewResult.reason).toContain("Collecting samples");
|
|
171
|
+
|
|
172
|
+
// Many stable samples
|
|
173
|
+
const stableSamples = Array(100).fill(1e6);
|
|
174
|
+
const stableResult = checkConvergence(stableSamples);
|
|
175
|
+
expect(stableResult.confidence).toBeGreaterThan(50);
|
|
176
|
+
|
|
177
|
+
// Variable samples - alternating pattern may be detected as stable
|
|
178
|
+
const variableSamples = Array.from({ length: 100 }, (_, i) =>
|
|
179
|
+
i % 2 === 0 ? 1e6 : 2e6,
|
|
180
|
+
);
|
|
181
|
+
const variableResult = checkConvergence(variableSamples);
|
|
182
|
+
// Just verify confidence is calculated, alternating may be seen as stable
|
|
183
|
+
expect(variableResult.confidence).toBeGreaterThanOrEqual(0);
|
|
184
|
+
expect(variableResult.confidence).toBeLessThanOrEqual(100);
|
|
185
|
+
});
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { expect, test } from "vitest";
|
|
2
|
+
import type { BenchSuite } from "../Benchmark.ts";
|
|
3
|
+
import type { BenchmarkReport } from "../BenchmarkReport.ts";
|
|
4
|
+
import { parseBenchArgs, runBenchmarks } from "../cli/RunBenchCLI.ts";
|
|
5
|
+
|
|
6
|
+
const statisticalSuite: BenchSuite = {
|
|
7
|
+
name: "Statistical Test Suite",
|
|
8
|
+
groups: [
|
|
9
|
+
{
|
|
10
|
+
name: "Test Group",
|
|
11
|
+
benchmarks: [
|
|
12
|
+
{
|
|
13
|
+
name: "stable-benchmark",
|
|
14
|
+
fn: () => {
|
|
15
|
+
let sum = 0;
|
|
16
|
+
for (let i = 0; i < 100; i++) sum += i;
|
|
17
|
+
return sum;
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
name: "variable-benchmark",
|
|
22
|
+
fn: () => {
|
|
23
|
+
const iterations = Math.random() > 0.9 ? 1000 : 100;
|
|
24
|
+
let sum = 0;
|
|
25
|
+
for (let i = 0; i < iterations; i++) sum += i;
|
|
26
|
+
return sum;
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
],
|
|
30
|
+
},
|
|
31
|
+
],
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const gcSuite: BenchSuite = {
|
|
35
|
+
name: "GC Test Suite",
|
|
36
|
+
groups: [
|
|
37
|
+
{
|
|
38
|
+
name: "Memory Group",
|
|
39
|
+
benchmarks: [
|
|
40
|
+
{
|
|
41
|
+
name: "gc-heavy",
|
|
42
|
+
fn: () => {
|
|
43
|
+
const arrays = [];
|
|
44
|
+
for (let i = 0; i < 50; i++) {
|
|
45
|
+
arrays.push(Array.from({ length: 1000 }, () => Math.random()));
|
|
46
|
+
}
|
|
47
|
+
return arrays.length;
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
],
|
|
51
|
+
},
|
|
52
|
+
],
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
function parseAdaptiveArgs() {
|
|
56
|
+
return parseBenchArgs((yargs: any) =>
|
|
57
|
+
yargs
|
|
58
|
+
.option("adaptive", { type: "boolean", default: true })
|
|
59
|
+
.option("time", { type: "number", default: 0.1 })
|
|
60
|
+
.option("min-time", { type: "number", default: 0.1 }),
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function verifyStatisticalMetrics(report: BenchmarkReport): void {
|
|
65
|
+
const { time, convergence } = report.measuredResults;
|
|
66
|
+
expect(time).toBeDefined();
|
|
67
|
+
expect(time?.p25).toBeDefined();
|
|
68
|
+
expect(time?.p50).toBeDefined();
|
|
69
|
+
expect(time?.p75).toBeDefined();
|
|
70
|
+
expect(time?.p95).toBeDefined();
|
|
71
|
+
expect(time?.cv).toBeGreaterThanOrEqual(0);
|
|
72
|
+
expect(time?.mad).toBeGreaterThanOrEqual(0);
|
|
73
|
+
expect(time?.outlierRate).toBeGreaterThanOrEqual(0);
|
|
74
|
+
expect(time?.outlierRate).toBeLessThanOrEqual(1);
|
|
75
|
+
expect(convergence?.confidence).toBeGreaterThanOrEqual(0);
|
|
76
|
+
expect(convergence?.confidence).toBeLessThanOrEqual(100);
|
|
77
|
+
expect(convergence?.reason).toBeDefined();
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function verifyPercentileOrdering(report: BenchmarkReport): void {
|
|
81
|
+
const t = report.measuredResults.time;
|
|
82
|
+
if (t?.p25 && t?.p50 && t?.p75 && t?.p95) {
|
|
83
|
+
expect(t.p25).toBeLessThanOrEqual(t.p50);
|
|
84
|
+
expect(t.p50).toBeLessThanOrEqual(t.p75);
|
|
85
|
+
expect(t.p75).toBeLessThanOrEqual(t.p95);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
test("adaptive mode reports statistical metrics correctly", async () => {
|
|
90
|
+
const results = await runBenchmarks(statisticalSuite, parseAdaptiveArgs());
|
|
91
|
+
expect(results).toHaveLength(1);
|
|
92
|
+
expect(results[0].reports).toHaveLength(2);
|
|
93
|
+
|
|
94
|
+
for (const report of results[0].reports) {
|
|
95
|
+
verifyStatisticalMetrics(report);
|
|
96
|
+
verifyPercentileOrdering(report);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const reports = results[0].reports;
|
|
100
|
+
const stableCV = reports.find(r => r.name === "stable-benchmark")
|
|
101
|
+
?.measuredResults.time?.cv;
|
|
102
|
+
const variableCV = reports.find(r => r.name === "variable-benchmark")
|
|
103
|
+
?.measuredResults.time?.cv;
|
|
104
|
+
if (stableCV && variableCV) {
|
|
105
|
+
expect(variableCV).toBeGreaterThanOrEqual(stableCV);
|
|
106
|
+
}
|
|
107
|
+
}, 20000);
|
|
108
|
+
|
|
109
|
+
test("adaptive mode handles GC-heavy workload", async () => {
|
|
110
|
+
const results = await runBenchmarks(gcSuite, parseAdaptiveArgs());
|
|
111
|
+
expect(results).toHaveLength(1);
|
|
112
|
+
expect(results[0].reports).toHaveLength(1);
|
|
113
|
+
|
|
114
|
+
const gcResult = results[0].reports[0].measuredResults;
|
|
115
|
+
expect(gcResult.convergence).toBeDefined();
|
|
116
|
+
expect(gcResult.time?.outlierRate).toBeDefined();
|
|
117
|
+
expect(gcResult.time?.cv).toBeDefined();
|
|
118
|
+
expect(gcResult.time?.mad).toBeGreaterThanOrEqual(0);
|
|
119
|
+
}, 20000);
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { expect, test } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
type BenchmarkReport,
|
|
4
|
+
reportResults,
|
|
5
|
+
valuesForReports,
|
|
6
|
+
} from "../BenchmarkReport.ts";
|
|
7
|
+
import {
|
|
8
|
+
adaptiveSection,
|
|
9
|
+
gcSection,
|
|
10
|
+
timeSection,
|
|
11
|
+
} from "../StandardSections.ts";
|
|
12
|
+
import { createBenchmarkReport, createMeasuredResults } from "./TestUtils.ts";
|
|
13
|
+
|
|
14
|
+
test("combines time and gc sections into report", () => {
|
|
15
|
+
const sections = [timeSection, gcSection] as const;
|
|
16
|
+
const report = createBenchmarkReport("test", [100, 150]);
|
|
17
|
+
const rows = valuesForReports([report], sections);
|
|
18
|
+
|
|
19
|
+
expect(rows[0].name).toBe("test");
|
|
20
|
+
expect(rows[0].mean).toBeCloseTo(report.measuredResults.time.avg, 1);
|
|
21
|
+
expect(rows[0].p50).toBeCloseTo(report.measuredResults.time.p50, 1);
|
|
22
|
+
expect(rows[0].p99).toBeCloseTo(report.measuredResults.time.p99, 1);
|
|
23
|
+
expect(rows[0].gc).toBeDefined();
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
test("generates diff columns for baseline comparison", () => {
|
|
27
|
+
const faster = createMeasuredResults([250, 300]);
|
|
28
|
+
const slower = createMeasuredResults([0, 50]);
|
|
29
|
+
|
|
30
|
+
const scale = (results: typeof faster, factor: number) => ({
|
|
31
|
+
...results,
|
|
32
|
+
time: {
|
|
33
|
+
...results.time,
|
|
34
|
+
avg: results.time.avg * factor,
|
|
35
|
+
p50: results.time.p50 * factor,
|
|
36
|
+
p99: results.time.p99 * factor,
|
|
37
|
+
},
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
const group1Reports: BenchmarkReport[] = [
|
|
41
|
+
{ name: "version1", measuredResults: scale(faster, 0.8) },
|
|
42
|
+
{ name: "version2", measuredResults: scale(slower, 1.2) },
|
|
43
|
+
];
|
|
44
|
+
|
|
45
|
+
const baseline = createBenchmarkReport("baseVersion", [200, 250]);
|
|
46
|
+
const group2Reports = [
|
|
47
|
+
createBenchmarkReport("test3", [300, 350]),
|
|
48
|
+
createBenchmarkReport("test4", [350, 400]),
|
|
49
|
+
];
|
|
50
|
+
|
|
51
|
+
const groups = [
|
|
52
|
+
{ name: "group1", reports: group1Reports, baseline },
|
|
53
|
+
{ name: "group2", reports: group2Reports },
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
const table = reportResults(groups, [timeSection]);
|
|
57
|
+
const names = ["version1", "version2", "baseVersion", "test3", "test4"];
|
|
58
|
+
for (const name of names) expect(table).toContain(name);
|
|
59
|
+
expect(table).toContain("Δ%");
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("formats adaptive convergence statistics", () => {
|
|
63
|
+
const reports: BenchmarkReport[] = [
|
|
64
|
+
createBenchmarkReport("test-adaptive", [400, 500], {
|
|
65
|
+
convergence: { converged: true, confidence: 95, reason: "stable" },
|
|
66
|
+
}),
|
|
67
|
+
createBenchmarkReport("test-low-confidence", [0, 30], {
|
|
68
|
+
convergence: { converged: false, confidence: 65, reason: "unstable" },
|
|
69
|
+
}),
|
|
70
|
+
];
|
|
71
|
+
|
|
72
|
+
const rows = valuesForReports(reports, [adaptiveSection]);
|
|
73
|
+
expect(rows[0].convergence).toBe(95);
|
|
74
|
+
expect(rows[1].convergence).toBe(65);
|
|
75
|
+
|
|
76
|
+
const table = reportResults(
|
|
77
|
+
[{ name: "adaptive", reports }],
|
|
78
|
+
[adaptiveSection],
|
|
79
|
+
);
|
|
80
|
+
expect(table).toContain("95%");
|
|
81
|
+
expect(table).toMatch(/65%/);
|
|
82
|
+
});
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { expect, test } from "vitest";
|
|
3
|
+
import { profileBrowser } from "../browser/BrowserHeapSampler.ts";
|
|
4
|
+
|
|
5
|
+
const examplesDir = path.resolve(import.meta.dirname!, "../../examples");
|
|
6
|
+
|
|
7
|
+
test("bench function mode (window.__bench)", { timeout: 30000 }, async () => {
|
|
8
|
+
const url = `file://${examplesDir}/browser-bench/index.html`;
|
|
9
|
+
const result = await profileBrowser({ url, maxTime: 500, gcStats: true });
|
|
10
|
+
|
|
11
|
+
expect(result.samples).toBeDefined();
|
|
12
|
+
expect(result.samples!.length).toBeGreaterThan(5);
|
|
13
|
+
expect(result.wallTimeMs).toBeGreaterThan(0);
|
|
14
|
+
expect(result.gcStats).toBeDefined();
|
|
15
|
+
expect(result.gcStats!.scavenges).toBeGreaterThanOrEqual(0);
|
|
16
|
+
for (const s of result.samples!) {
|
|
17
|
+
expect(s).toBeGreaterThan(0);
|
|
18
|
+
expect(s).toBeLessThan(1000);
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test("lap mode with N laps", { timeout: 30000 }, async () => {
|
|
23
|
+
const url = `file://${examplesDir}/browser-lap/index.html`;
|
|
24
|
+
const result = await profileBrowser({ url, gcStats: true });
|
|
25
|
+
|
|
26
|
+
expect(result.samples).toBeDefined();
|
|
27
|
+
expect(result.samples!).toHaveLength(100);
|
|
28
|
+
expect(result.wallTimeMs).toBeGreaterThan(0);
|
|
29
|
+
expect(result.gcStats).toBeDefined();
|
|
30
|
+
for (const s of result.samples!) {
|
|
31
|
+
expect(s).toBeGreaterThanOrEqual(0);
|
|
32
|
+
}
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("lap mode 0 laps with heap profiling", { timeout: 30000 }, async () => {
|
|
36
|
+
const url = `file://${examplesDir}/browser-heap/index.html`;
|
|
37
|
+
const result = await profileBrowser({ url, heapSample: true });
|
|
38
|
+
|
|
39
|
+
expect(result.samples).toBeDefined();
|
|
40
|
+
expect(result.samples!).toHaveLength(0);
|
|
41
|
+
expect(result.wallTimeMs).toBeGreaterThan(0);
|
|
42
|
+
expect(result.heapProfile).toBeDefined();
|
|
43
|
+
expect(result.heapProfile!.head).toBeDefined();
|
|
44
|
+
});
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { expect, test } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
browserGcStats,
|
|
4
|
+
parseGcTraceEvents,
|
|
5
|
+
type TraceEvent,
|
|
6
|
+
} from "../browser/BrowserGcStats.ts";
|
|
7
|
+
|
|
8
|
+
test("parseGcTraceEvents parses MinorGC and MajorGC events", () => {
|
|
9
|
+
const events: TraceEvent[] = [
|
|
10
|
+
{
|
|
11
|
+
cat: "v8.gc",
|
|
12
|
+
name: "MinorGC",
|
|
13
|
+
ph: "X",
|
|
14
|
+
dur: 500,
|
|
15
|
+
args: { usedHeapSizeBefore: 10000, usedHeapSizeAfter: 8000 },
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
cat: "v8.gc",
|
|
19
|
+
name: "MajorGC",
|
|
20
|
+
ph: "X",
|
|
21
|
+
dur: 12000,
|
|
22
|
+
args: { usedHeapSizeBefore: 50000, usedHeapSizeAfter: 30000 },
|
|
23
|
+
},
|
|
24
|
+
];
|
|
25
|
+
const parsed = parseGcTraceEvents(events);
|
|
26
|
+
expect(parsed).toHaveLength(2);
|
|
27
|
+
expect(parsed[0]).toEqual({
|
|
28
|
+
type: "scavenge",
|
|
29
|
+
pauseMs: 0.5,
|
|
30
|
+
collected: 2000,
|
|
31
|
+
});
|
|
32
|
+
expect(parsed[1]).toEqual({
|
|
33
|
+
type: "mark-compact",
|
|
34
|
+
pauseMs: 12,
|
|
35
|
+
collected: 20000,
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("parseGcTraceEvents ignores non-complete and non-GC events", () => {
|
|
40
|
+
const events: TraceEvent[] = [
|
|
41
|
+
{ cat: "v8.gc", name: "MinorGC", ph: "B", dur: 500 }, // not complete
|
|
42
|
+
{ cat: "v8", name: "V8.Execute", ph: "X", dur: 100 }, // not GC
|
|
43
|
+
{ cat: "v8.gc", name: "MinorGC", ph: "X" }, // valid, missing dur/args
|
|
44
|
+
];
|
|
45
|
+
const parsed = parseGcTraceEvents(events);
|
|
46
|
+
expect(parsed).toHaveLength(1);
|
|
47
|
+
expect(parsed[0]).toEqual({ type: "scavenge", pauseMs: 0, collected: 0 });
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("browserGcStats aggregates trace events into GcStats", () => {
|
|
51
|
+
const events: TraceEvent[] = [
|
|
52
|
+
{
|
|
53
|
+
cat: "v8.gc",
|
|
54
|
+
name: "MinorGC",
|
|
55
|
+
ph: "X",
|
|
56
|
+
dur: 300,
|
|
57
|
+
args: { usedHeapSizeBefore: 5000, usedHeapSizeAfter: 3000 },
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
cat: "v8.gc",
|
|
61
|
+
name: "MinorGC",
|
|
62
|
+
ph: "X",
|
|
63
|
+
dur: 200,
|
|
64
|
+
args: { usedHeapSizeBefore: 6000, usedHeapSizeAfter: 4000 },
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
cat: "v8.gc",
|
|
68
|
+
name: "MajorGC",
|
|
69
|
+
ph: "X",
|
|
70
|
+
dur: 8000,
|
|
71
|
+
args: { usedHeapSizeBefore: 40000, usedHeapSizeAfter: 20000 },
|
|
72
|
+
},
|
|
73
|
+
];
|
|
74
|
+
const stats = browserGcStats(events);
|
|
75
|
+
expect(stats.scavenges).toBe(2);
|
|
76
|
+
expect(stats.markCompacts).toBe(1);
|
|
77
|
+
expect(stats.totalCollected).toBe(24000);
|
|
78
|
+
expect(stats.gcPauseTime).toBeCloseTo(8.5, 2);
|
|
79
|
+
});
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { expect, test } from "vitest";
|
|
2
|
+
import { aggregateGcStats, parseGcLine } from "../runners/GcStats.ts";
|
|
3
|
+
|
|
4
|
+
test("parseGcLine parses scavenge event from real V8 output", () => {
|
|
5
|
+
// Real V8 --trace-gc-nvp format
|
|
6
|
+
const line =
|
|
7
|
+
"[71753:0x83280c000:0] 9 ms: pause=0.5 mutator=0.1 gc=s allocated=293224 promoted=653480 new_space_survived=290176 start_object_size=4392688 end_object_size=4287840";
|
|
8
|
+
const event = parseGcLine(line);
|
|
9
|
+
expect(event).toMatchObject({
|
|
10
|
+
type: "scavenge",
|
|
11
|
+
pauseMs: 0.5,
|
|
12
|
+
allocated: 293224,
|
|
13
|
+
promoted: 653480,
|
|
14
|
+
survived: 290176,
|
|
15
|
+
collected: 4392688 - 4287840,
|
|
16
|
+
});
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test("parseGcLine parses mark-sweep event", () => {
|
|
20
|
+
const line =
|
|
21
|
+
"[1234:0x12345:0] 100 ms: pause=12.3 gc=ms allocated=2097152 promoted=0 new_space_survived=0 start_object_size=5000000 end_object_size=3000000";
|
|
22
|
+
const event = parseGcLine(line);
|
|
23
|
+
expect(event).toMatchObject({
|
|
24
|
+
type: "mark-compact",
|
|
25
|
+
pauseMs: 12.3,
|
|
26
|
+
allocated: 2097152,
|
|
27
|
+
collected: 2000000,
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("parseGcLine returns undefined for non-GC lines", () => {
|
|
32
|
+
expect(parseGcLine("some random stderr output")).toBeUndefined();
|
|
33
|
+
expect(parseGcLine("")).toBeUndefined();
|
|
34
|
+
expect(parseGcLine(" ")).toBeUndefined();
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test("parseGcLine handles missing fields", () => {
|
|
38
|
+
const line = "gc=s pause=1.0";
|
|
39
|
+
const event = parseGcLine(line);
|
|
40
|
+
expect(event).toMatchObject({
|
|
41
|
+
type: "scavenge",
|
|
42
|
+
pauseMs: 1.0,
|
|
43
|
+
allocated: 0,
|
|
44
|
+
collected: 0,
|
|
45
|
+
promoted: 0,
|
|
46
|
+
survived: 0,
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("aggregateGcStats aggregates multiple events", () => {
|
|
51
|
+
const scav = "scavenge" as const;
|
|
52
|
+
const mc = "mark-compact" as const;
|
|
53
|
+
const events = [
|
|
54
|
+
{
|
|
55
|
+
type: scav,
|
|
56
|
+
pauseMs: 0.5,
|
|
57
|
+
allocated: 1000,
|
|
58
|
+
collected: 500,
|
|
59
|
+
promoted: 100,
|
|
60
|
+
survived: 400,
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
type: scav,
|
|
64
|
+
pauseMs: 0.3,
|
|
65
|
+
allocated: 2000,
|
|
66
|
+
collected: 800,
|
|
67
|
+
promoted: 200,
|
|
68
|
+
survived: 1000,
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
type: mc,
|
|
72
|
+
pauseMs: 10.0,
|
|
73
|
+
allocated: 0,
|
|
74
|
+
collected: 5000,
|
|
75
|
+
promoted: 0,
|
|
76
|
+
survived: 0,
|
|
77
|
+
},
|
|
78
|
+
];
|
|
79
|
+
const stats = aggregateGcStats(events);
|
|
80
|
+
expect(stats.scavenges).toBe(2);
|
|
81
|
+
expect(stats.markCompacts).toBe(1);
|
|
82
|
+
expect(stats.totalAllocated).toBe(3000);
|
|
83
|
+
expect(stats.totalCollected).toBe(6300);
|
|
84
|
+
expect(stats.totalPromoted).toBe(300);
|
|
85
|
+
expect(stats.totalSurvived).toBe(1400);
|
|
86
|
+
expect(stats.gcPauseTime).toBeCloseTo(10.8, 2);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("aggregateGcStats handles empty events", () => {
|
|
90
|
+
const stats = aggregateGcStats([]);
|
|
91
|
+
expect(stats.scavenges).toBe(0);
|
|
92
|
+
expect(stats.markCompacts).toBe(0);
|
|
93
|
+
expect(stats.gcPauseTime).toBe(0);
|
|
94
|
+
});
|