@vercel/devlow-bench 0.3.4 → 16.4.0-canary.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -33
- package/dist/browser.d.ts +1 -1
- package/dist/browser.js +52 -44
- package/dist/cli.js +114 -69
- package/dist/compare.d.ts +14 -0
- package/dist/compare.js +209 -0
- package/dist/describe.d.ts +1 -1
- package/dist/describe.js +18 -18
- package/dist/file.js +5 -5
- package/dist/index.d.ts +16 -3
- package/dist/index.js +3 -2
- package/dist/interfaces/compare.d.ts +4 -0
- package/dist/interfaces/compare.js +27 -0
- package/dist/interfaces/compose.d.ts +1 -1
- package/dist/interfaces/compose.js +1 -1
- package/dist/interfaces/console.d.ts +4 -2
- package/dist/interfaces/console.js +30 -8
- package/dist/interfaces/constants.js +7 -7
- package/dist/interfaces/datadog.d.ts +1 -1
- package/dist/interfaces/datadog.js +13 -13
- package/dist/interfaces/interactive.d.ts +1 -1
- package/dist/interfaces/interactive.js +8 -8
- package/dist/interfaces/json.d.ts +4 -2
- package/dist/interfaces/json.js +36 -17
- package/dist/interfaces/shell-test.d.ts +1 -0
- package/dist/interfaces/shell-test.js +56 -0
- package/dist/interfaces/snapshot.d.ts +6 -0
- package/dist/interfaces/snapshot.js +60 -0
- package/dist/interfaces/snowflake-test.js +3 -3
- package/dist/interfaces/snowflake.d.ts +1 -1
- package/dist/interfaces/snowflake.js +15 -13
- package/dist/runner.d.ts +5 -2
- package/dist/runner.js +132 -24
- package/dist/shell.d.ts +5 -3
- package/dist/shell.js +56 -28
- package/dist/snapshot.d.ts +18 -0
- package/dist/snapshot.js +162 -0
- package/dist/statistics-test.d.ts +1 -0
- package/dist/statistics-test.js +102 -0
- package/dist/statistics.d.ts +19 -0
- package/dist/statistics.js +123 -0
- package/dist/table.js +51 -51
- package/dist/units.js +6 -6
- package/dist/utils.d.ts +1 -0
- package/dist/utils.js +8 -6
- package/package.json +6 -5
package/README.md
CHANGED
|
@@ -10,19 +10,36 @@ npm install devlow-bench
|
|
|
10
10
|
|
|
11
11
|
## Usage
|
|
12
12
|
|
|
13
|
-
```
|
|
14
|
-
Usage: devlow-bench [options]
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
13
|
+
```text
|
|
14
|
+
Usage: devlow-bench [options] [command]
|
|
15
|
+
|
|
16
|
+
Run developer-workflow benchmarks.
|
|
17
|
+
|
|
18
|
+
Options:
|
|
19
|
+
-h, --help display help for command
|
|
20
|
+
|
|
21
|
+
Commands:
|
|
22
|
+
run [options] [scenarios...] Run scenario files and report measurements.
|
|
23
|
+
compare <baseline> <current> Compare two snapshot CSVs side-by-side.
|
|
24
|
+
help [command] display help for command
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
`run` is the default command, so scenario paths can still be passed without
|
|
28
|
+
writing `run` explicitly. Its options include:
|
|
29
|
+
|
|
30
|
+
```text
|
|
31
|
+
-s, --scenario <filter> Only run scenarios whose name matches the filter (repeatable).
|
|
32
|
+
-F, --filter <pair> Filter variants by property: key=value (repeatable).
|
|
33
|
+
-i, --interactive Select scenarios and variants interactively.
|
|
34
|
+
--n <number> Run each variant N times.
|
|
35
|
+
--warmup <number> Discard the first N runs before sampling.
|
|
36
|
+
--snapshot <path> Override the snapshot CSV path.
|
|
37
|
+
--compare Print a comparison table after the run.
|
|
38
|
+
--baseline <path> Select a comparison baseline; implies --compare.
|
|
39
|
+
-j, --json <path> Write results as JSON.
|
|
40
|
+
--no-console Suppress console output.
|
|
41
|
+
--datadog [host] Upload results to Datadog.
|
|
42
|
+
--snowflake [batchUri] Upload results to Snowflake.
|
|
26
43
|
```
|
|
27
44
|
|
|
28
45
|
## Scenarios
|
|
@@ -30,10 +47,10 @@ Usage: devlow-bench [options] <scenario files>
|
|
|
30
47
|
A scenario file is similar to a test case file. It can contain one or multiple scenarios by using the `describe()` method to define them.
|
|
31
48
|
|
|
32
49
|
```js
|
|
33
|
-
import { describe } from
|
|
50
|
+
import { describe } from 'devlow-bench'
|
|
34
51
|
|
|
35
52
|
describe(
|
|
36
|
-
|
|
53
|
+
'my scenario',
|
|
37
54
|
{
|
|
38
55
|
/* property options */
|
|
39
56
|
},
|
|
@@ -44,7 +61,7 @@ describe(
|
|
|
44
61
|
) => {
|
|
45
62
|
// run the scenario
|
|
46
63
|
}
|
|
47
|
-
)
|
|
64
|
+
)
|
|
48
65
|
```
|
|
49
66
|
|
|
50
67
|
The `describe()` method takes three arguments:
|
|
@@ -58,18 +75,18 @@ The `props` object can contain any number of properties. The key is the name of
|
|
|
58
75
|
### Example
|
|
59
76
|
|
|
60
77
|
```js
|
|
61
|
-
import { describe } from
|
|
78
|
+
import { describe } from 'devlow-bench'
|
|
62
79
|
|
|
63
80
|
describe(
|
|
64
|
-
|
|
81
|
+
'my scenario',
|
|
65
82
|
{
|
|
66
83
|
myProperty: [1, 2, 3],
|
|
67
84
|
myOtherProperty: true,
|
|
68
85
|
},
|
|
69
86
|
async ({ myProperty, myOtherProperty }) => {
|
|
70
|
-
console.log(myProperty, myOtherProperty)
|
|
87
|
+
console.log(myProperty, myOtherProperty)
|
|
71
88
|
}
|
|
72
|
-
)
|
|
89
|
+
)
|
|
73
90
|
|
|
74
91
|
// will print:
|
|
75
92
|
// 1 true
|
|
@@ -83,17 +100,17 @@ describe(
|
|
|
83
100
|
## Reporting measurements
|
|
84
101
|
|
|
85
102
|
```js
|
|
86
|
-
import { measureTime, reportMeasurement } from
|
|
103
|
+
import { measureTime, reportMeasurement } from 'devlow-bench'
|
|
87
104
|
|
|
88
105
|
// Measure a time
|
|
89
|
-
await measureTime(
|
|
106
|
+
await measureTime('name of the timing', {
|
|
90
107
|
/* optional options */
|
|
91
|
-
})
|
|
108
|
+
})
|
|
92
109
|
|
|
93
110
|
// Report some other measurement
|
|
94
|
-
await reportMeasurement(
|
|
111
|
+
await reportMeasurement('name of the measurement', value, unit, {
|
|
95
112
|
/* optional options */
|
|
96
|
-
})
|
|
113
|
+
})
|
|
97
114
|
```
|
|
98
115
|
|
|
99
116
|
Options:
|
|
@@ -107,15 +124,15 @@ Options:
|
|
|
107
124
|
The `devlow-bench` package provides a few helper functions to run operations in the browser.
|
|
108
125
|
|
|
109
126
|
```js
|
|
110
|
-
import { newBrowserSession } from
|
|
127
|
+
import { newBrowserSession } from 'devlow-bench/browser'
|
|
111
128
|
|
|
112
129
|
const session = await newBrowserSession({
|
|
113
130
|
// options
|
|
114
|
-
})
|
|
115
|
-
await session.hardNavigation(
|
|
116
|
-
await session.reload(
|
|
117
|
-
await session.softNavigationByClick(
|
|
118
|
-
await session.close()
|
|
131
|
+
})
|
|
132
|
+
await session.hardNavigation('metric name', 'https://example.com')
|
|
133
|
+
await session.reload('metric name')
|
|
134
|
+
await session.softNavigationByClick('metric name', '.selector-to-click')
|
|
135
|
+
await session.close()
|
|
119
136
|
```
|
|
120
137
|
|
|
121
138
|
Run with `BROWSER_OUTPUT=1` to show the output of the browser.
|
|
@@ -162,8 +179,8 @@ Run with `SHELL_OUTPUT=1` to show the output of the shell commands.
|
|
|
162
179
|
The `devlow-bench` package provides a few helper functions to run operations on the file system.
|
|
163
180
|
|
|
164
181
|
```js
|
|
165
|
-
import { waitForFile } from
|
|
182
|
+
import { waitForFile } from 'devlow-bench/file'
|
|
166
183
|
|
|
167
184
|
// wait for file to exist
|
|
168
|
-
await waitForFile(
|
|
185
|
+
await waitForFile('/path/to/file', /* timeout = */ 30000)
|
|
169
186
|
```
|
package/dist/browser.d.ts
CHANGED
package/dist/browser.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { chromium } from
|
|
2
|
-
import { measureTime, reportMeasurement } from
|
|
1
|
+
import { chromium } from 'playwright-chromium';
|
|
2
|
+
import { measureTime, reportMeasurement } from './index.js';
|
|
3
3
|
const browserOutput = Boolean(process.env.BROWSER_OUTPUT);
|
|
4
4
|
async function withRequestMetrics(metricName, page, fn) {
|
|
5
5
|
const activePromises = [];
|
|
@@ -9,9 +9,7 @@ async function withRequestMetrics(metricName, page, fn) {
|
|
|
9
9
|
activePromises.push((async () => {
|
|
10
10
|
const url = response.request().url();
|
|
11
11
|
const status = response.status();
|
|
12
|
-
const extension =
|
|
13
|
-
// eslint-disable-next-line prefer-named-capture-group -- TODO: address lint
|
|
14
|
-
/^[^?#]+\.([a-z0-9]+)(?:[?#]|$)/i.exec(url)?.[1] ?? "none";
|
|
12
|
+
const extension = /^[^?#]+\.([a-z0-9]+)(?:[?#]|$)/i.exec(url)?.[1] ?? 'none';
|
|
15
13
|
const currentRequests = requestsByExtension.get(extension) ?? 0;
|
|
16
14
|
requestsByExtension.set(extension, currentRequests + 1);
|
|
17
15
|
if (status >= 200 && status < 300) {
|
|
@@ -35,10 +33,10 @@ async function withRequestMetrics(metricName, page, fn) {
|
|
|
35
33
|
let logCount = 0;
|
|
36
34
|
const consoleHandler = (message) => {
|
|
37
35
|
const type = message.type();
|
|
38
|
-
if (type ===
|
|
36
|
+
if (type === 'error') {
|
|
39
37
|
errorCount++;
|
|
40
38
|
}
|
|
41
|
-
else if (type ===
|
|
39
|
+
else if (type === 'warning') {
|
|
42
40
|
warningCount++;
|
|
43
41
|
}
|
|
44
42
|
else {
|
|
@@ -68,31 +66,31 @@ async function withRequestMetrics(metricName, page, fn) {
|
|
|
68
66
|
}
|
|
69
67
|
};
|
|
70
68
|
try {
|
|
71
|
-
page.on(
|
|
72
|
-
page.on(
|
|
73
|
-
page.on(
|
|
69
|
+
page.on('response', responseHandler);
|
|
70
|
+
page.on('console', consoleHandler);
|
|
71
|
+
page.on('pageerror', exceptionHandler);
|
|
74
72
|
await fn();
|
|
75
73
|
await Promise.all(activePromises);
|
|
76
74
|
let totalDownload = 0;
|
|
77
75
|
for (const [extension, size] of sizeByExtension.entries()) {
|
|
78
|
-
await reportMeasurement(`${metricName}/responseSizes/${extension}`, size,
|
|
76
|
+
await reportMeasurement(`${metricName}/responseSizes/${extension}`, size, 'bytes');
|
|
79
77
|
totalDownload += size;
|
|
80
78
|
}
|
|
81
|
-
await reportMeasurement(`${metricName}/responseSizes`, totalDownload,
|
|
79
|
+
await reportMeasurement(`${metricName}/responseSizes`, totalDownload, 'bytes');
|
|
82
80
|
let totalRequests = 0;
|
|
83
81
|
for (const [extension, count] of requestsByExtension.entries()) {
|
|
84
|
-
await reportMeasurement(`${metricName}/requests/${extension}`, count,
|
|
82
|
+
await reportMeasurement(`${metricName}/requests/${extension}`, count, 'requests');
|
|
85
83
|
totalRequests += count;
|
|
86
84
|
}
|
|
87
|
-
await reportMeasurement(`${metricName}/requests`, totalRequests,
|
|
88
|
-
await reportMeasurement(`${metricName}/console/logs`, logCount,
|
|
89
|
-
await reportMeasurement(`${metricName}/console/warnings`, warningCount,
|
|
90
|
-
await reportMeasurement(`${metricName}/console/errors`, errorCount,
|
|
91
|
-
await reportMeasurement(`${metricName}/console/uncaught`, uncaughtCount,
|
|
92
|
-
await reportMeasurement(`${metricName}/console`, logCount + warningCount + errorCount + uncaughtCount,
|
|
85
|
+
await reportMeasurement(`${metricName}/requests`, totalRequests, 'requests');
|
|
86
|
+
await reportMeasurement(`${metricName}/console/logs`, logCount, 'messages');
|
|
87
|
+
await reportMeasurement(`${metricName}/console/warnings`, warningCount, 'messages');
|
|
88
|
+
await reportMeasurement(`${metricName}/console/errors`, errorCount, 'messages');
|
|
89
|
+
await reportMeasurement(`${metricName}/console/uncaught`, uncaughtCount, 'messages');
|
|
90
|
+
await reportMeasurement(`${metricName}/console`, logCount + warningCount + errorCount + uncaughtCount, 'messages');
|
|
93
91
|
}
|
|
94
92
|
finally {
|
|
95
|
-
page.off(
|
|
93
|
+
page.off('response', responseHandler);
|
|
96
94
|
}
|
|
97
95
|
}
|
|
98
96
|
/**
|
|
@@ -105,9 +103,9 @@ async function withRequestMetrics(metricName, page, fn) {
|
|
|
105
103
|
function networkIdle(page, delayMs = 300, timeoutMs = 180000) {
|
|
106
104
|
return new Promise((resolve) => {
|
|
107
105
|
const cleanup = () => {
|
|
108
|
-
page.off(
|
|
109
|
-
page.off(
|
|
110
|
-
page.off(
|
|
106
|
+
page.off('request', requestHandler);
|
|
107
|
+
page.off('requestfailed', requestFinishedHandler);
|
|
108
|
+
page.off('requestfinished', requestFinishedHandler);
|
|
111
109
|
clearTimeout(fullTimeout);
|
|
112
110
|
if (timeout) {
|
|
113
111
|
clearTimeout(timeout);
|
|
@@ -119,12 +117,11 @@ function networkIdle(page, delayMs = 300, timeoutMs = 180000) {
|
|
|
119
117
|
let timeout = null;
|
|
120
118
|
const fullTimeout = setTimeout(() => {
|
|
121
119
|
cleanup();
|
|
122
|
-
|
|
123
|
-
console.error(`Timeout while waiting for network idle. These requests are still pending: ${Array.from(requests).join(", ")}} time is ${lastRequest - start}`);
|
|
120
|
+
console.error(`Timeout while waiting for network idle. These requests are still pending: ${Array.from(requests).join(', ')}} time is ${lastRequest - start}`);
|
|
124
121
|
resolve(Date.now() - lastRequest);
|
|
125
122
|
}, timeoutMs);
|
|
126
123
|
const requestFilter = (request) => {
|
|
127
|
-
return request.headers().accept !==
|
|
124
|
+
return request.headers().accept !== 'text/event-stream';
|
|
128
125
|
};
|
|
129
126
|
const requestHandler = (request) => {
|
|
130
127
|
requests.set(request.url(), (requests.get(request.url()) ?? 0) + 1);
|
|
@@ -147,7 +144,6 @@ function networkIdle(page, delayMs = 300, timeoutMs = 180000) {
|
|
|
147
144
|
lastRequest = Date.now();
|
|
148
145
|
const currentCount = requests.get(request.url());
|
|
149
146
|
if (currentCount === undefined) {
|
|
150
|
-
// eslint-disable-next-line no-console -- basic logging
|
|
151
147
|
console.error(`Unexpected untracked but completed request ${request.url()}`);
|
|
152
148
|
return;
|
|
153
149
|
}
|
|
@@ -164,9 +160,9 @@ function networkIdle(page, delayMs = 300, timeoutMs = 180000) {
|
|
|
164
160
|
}, delayMs);
|
|
165
161
|
}
|
|
166
162
|
};
|
|
167
|
-
page.on(
|
|
168
|
-
page.on(
|
|
169
|
-
page.on(
|
|
163
|
+
page.on('request', requestHandler);
|
|
164
|
+
page.on('requestfailed', requestFinishedHandler);
|
|
165
|
+
page.on('requestfinished', requestFinishedHandler);
|
|
170
166
|
});
|
|
171
167
|
}
|
|
172
168
|
class BrowserSessionImpl {
|
|
@@ -191,17 +187,23 @@ class BrowserSessionImpl {
|
|
|
191
187
|
await withRequestMetrics(metricName, page, async () => {
|
|
192
188
|
await measureTime(`${metricName}/start`);
|
|
193
189
|
const idle = networkIdle(page, 3000);
|
|
194
|
-
await page.goto(url, {
|
|
195
|
-
waitUntil:
|
|
190
|
+
const response = await page.goto(url, {
|
|
191
|
+
waitUntil: 'commit',
|
|
196
192
|
});
|
|
193
|
+
if (!response) {
|
|
194
|
+
throw new Error(`Navigation to ${url} produced no response`);
|
|
195
|
+
}
|
|
196
|
+
if (!response.ok()) {
|
|
197
|
+
throw new Error(`Navigation to ${url} returned HTTP ${response.status()}`);
|
|
198
|
+
}
|
|
197
199
|
await measureTime(`${metricName}/html`, {
|
|
198
200
|
relativeTo: `${metricName}/start`,
|
|
199
201
|
});
|
|
200
|
-
await page.waitForLoadState(
|
|
202
|
+
await page.waitForLoadState('domcontentloaded');
|
|
201
203
|
await measureTime(`${metricName}/dom`, {
|
|
202
204
|
relativeTo: `${metricName}/start`,
|
|
203
205
|
});
|
|
204
|
-
await page.waitForLoadState(
|
|
206
|
+
await page.waitForLoadState('load');
|
|
205
207
|
await measureTime(`${metricName}/load`, {
|
|
206
208
|
relativeTo: `${metricName}/start`,
|
|
207
209
|
});
|
|
@@ -216,12 +218,12 @@ class BrowserSessionImpl {
|
|
|
216
218
|
async softNavigationByClick(metricName, selector) {
|
|
217
219
|
const page = this.page;
|
|
218
220
|
if (!page) {
|
|
219
|
-
throw new Error(
|
|
221
|
+
throw new Error('softNavigationByClick() must be called after hardNavigation()');
|
|
220
222
|
}
|
|
221
223
|
await withRequestMetrics(metricName, page, async () => {
|
|
222
224
|
await measureTime(`${metricName}/start`);
|
|
223
225
|
const firstResponse = new Promise((resolve) => {
|
|
224
|
-
page.once(
|
|
226
|
+
page.once('response', () => {
|
|
225
227
|
resolve();
|
|
226
228
|
});
|
|
227
229
|
});
|
|
@@ -241,22 +243,28 @@ class BrowserSessionImpl {
|
|
|
241
243
|
async reload(metricName) {
|
|
242
244
|
const page = this.page;
|
|
243
245
|
if (!page) {
|
|
244
|
-
throw new Error(
|
|
246
|
+
throw new Error('reload() must be called after hardNavigation()');
|
|
245
247
|
}
|
|
246
248
|
await withRequestMetrics(metricName, page, async () => {
|
|
247
249
|
await measureTime(`${metricName}/start`);
|
|
248
250
|
const idle = networkIdle(page, 3000);
|
|
249
|
-
await page.reload({
|
|
250
|
-
waitUntil:
|
|
251
|
+
const response = await page.reload({
|
|
252
|
+
waitUntil: 'commit',
|
|
251
253
|
});
|
|
254
|
+
if (!response) {
|
|
255
|
+
throw new Error('Reload produced no response');
|
|
256
|
+
}
|
|
257
|
+
if (!response.ok()) {
|
|
258
|
+
throw new Error(`Reload returned HTTP ${response.status()}`);
|
|
259
|
+
}
|
|
252
260
|
await measureTime(`${metricName}/html`, {
|
|
253
261
|
relativeTo: `${metricName}/start`,
|
|
254
262
|
});
|
|
255
|
-
await page.waitForLoadState(
|
|
263
|
+
await page.waitForLoadState('domcontentloaded');
|
|
256
264
|
await measureTime(`${metricName}/dom`, {
|
|
257
265
|
relativeTo: `${metricName}/start`,
|
|
258
266
|
});
|
|
259
|
-
await page.waitForLoadState(
|
|
267
|
+
await page.waitForLoadState('load');
|
|
260
268
|
await measureTime(`${metricName}/load`, {
|
|
261
269
|
relativeTo: `${metricName}/start`,
|
|
262
270
|
});
|
|
@@ -270,12 +278,12 @@ class BrowserSessionImpl {
|
|
|
270
278
|
}
|
|
271
279
|
export async function newBrowserSession(options) {
|
|
272
280
|
const browser = await chromium.launch({
|
|
273
|
-
headless: options.headless ?? process.env.HEADLESS !==
|
|
274
|
-
|
|
281
|
+
headless: options.headless ?? process.env.HEADLESS !== 'false',
|
|
282
|
+
args: options.headless ? undefined : ['--auto-open-devtools-for-tabs'],
|
|
275
283
|
timeout: 60000,
|
|
276
284
|
});
|
|
277
285
|
const context = await browser.newContext({
|
|
278
|
-
baseURL: options.baseURL ??
|
|
286
|
+
baseURL: options.baseURL ?? 'http://localhost:3000',
|
|
279
287
|
viewport: { width: 1280, height: 720 },
|
|
280
288
|
});
|
|
281
289
|
context.setDefaultTimeout(120000);
|
package/dist/cli.js
CHANGED
|
@@ -1,73 +1,94 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import
|
|
6
|
-
import {
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { pathToFileURL } from 'url';
|
|
4
|
+
import { groupRows, printComparison } from './compare.js';
|
|
5
|
+
import { setCurrentScenarios } from './describe.js';
|
|
6
|
+
import { runScenarios } from './index.js';
|
|
7
|
+
import compose from './interfaces/compose.js';
|
|
8
|
+
import { readSnapshot, resolveCompareTarget } from './snapshot.js';
|
|
9
|
+
;
|
|
7
10
|
(async () => {
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
11
|
+
const program = new Command()
|
|
12
|
+
.name('devlow-bench')
|
|
13
|
+
.description('Run developer-workflow benchmarks.')
|
|
14
|
+
.showHelpAfterError();
|
|
15
|
+
program
|
|
16
|
+
.command('run', { isDefault: true })
|
|
17
|
+
.description('Run scenario files and report measurements.')
|
|
18
|
+
.argument('[scenarios...]', 'Scenario module paths to load.')
|
|
19
|
+
.option('-s, --scenario <filter>', 'Only run scenarios whose name matches the filter (repeatable).', (v, prev) => (prev ?? []).concat(v), [])
|
|
20
|
+
.option('-F, --filter <pair>', 'Filter variants by property: -F key=value (repeatable; same key repeated = OR).', (v, prev) => (prev ?? []).concat(v), [])
|
|
21
|
+
.option('-i, --interactive', 'Select scenarios and variants interactively.')
|
|
22
|
+
.option('--n <number>', 'Run each variant N times; reports mean/p50/p90 per metric. Default: 1.', (v) => Number(v))
|
|
23
|
+
.option('--warmup <number>', 'Discard the first N runs of each variant before sampling. Default: 0. Do NOT enable when measuring cold-start metrics.', (v) => Number(v))
|
|
24
|
+
.option('--snapshot <path>', 'Override the snapshot CSV path. Default: ./.devlow-bench/snapshots/<ts>.csv. Snapshots are always written.')
|
|
25
|
+
.option('--compare', 'Print a comparison table at end of run. Baseline = newest snapshot, unless --baseline overrides.')
|
|
26
|
+
.option('--baseline <path>', 'Explicit baseline (file or directory). Implies --compare.')
|
|
27
|
+
.option('-j, --json <path>', 'Write the results to the given path as JSON.')
|
|
28
|
+
.option('--no-console', 'Suppress console output.')
|
|
29
|
+
.option('--datadog [host]', 'Upload the results to Datadog (requires DATADOG_API_KEY).')
|
|
30
|
+
.option('--snowflake [batchUri]', 'Upload the results to Snowflake (requires SNOWFLAKE_TOPIC_NAME and SNOWFLAKE_SCHEMA_ID).')
|
|
31
|
+
.action(runRun);
|
|
32
|
+
program
|
|
33
|
+
.command('compare')
|
|
34
|
+
.description('Compare two snapshot CSVs side-by-side, with p50/p90/p99 plus a Mann–Whitney U p-value per metric.')
|
|
35
|
+
.argument('<baseline>', 'Baseline snapshot CSV path.')
|
|
36
|
+
.argument('<current>', 'Current snapshot CSV path.')
|
|
37
|
+
.action(runCompare);
|
|
38
|
+
await program.parseAsync();
|
|
39
|
+
})().catch((e) => {
|
|
40
|
+
console.error(e.stack);
|
|
41
|
+
process.exit(1);
|
|
42
|
+
});
|
|
43
|
+
async function runCompare(baselinePath, currentPath) {
|
|
44
|
+
const [baseRows, curRows] = await Promise.all([
|
|
45
|
+
readSnapshot(baselinePath),
|
|
46
|
+
readSnapshot(currentPath),
|
|
22
47
|
]);
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
j: "json",
|
|
27
|
-
i: "interactive",
|
|
28
|
-
"?": "help",
|
|
29
|
-
h: "help",
|
|
30
|
-
},
|
|
48
|
+
printComparison(groupRows(baseRows), groupRows(curRows), {
|
|
49
|
+
baselineLabel: baselinePath,
|
|
50
|
+
currentLabel: currentPath,
|
|
31
51
|
});
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
console.log(" (requires DATADOG_API_KEY environment variables)");
|
|
43
|
-
console.log(" --snowflake[=<batch-uri>] Upload the results to Snowflake");
|
|
44
|
-
console.log(" (requires SNOWFLAKE_TOPIC_NAME and SNOWFLAKE_SCHEMA_ID and environment variables)");
|
|
45
|
-
console.log("## Help");
|
|
46
|
-
console.log(" --help, -h, -? Show this help");
|
|
52
|
+
}
|
|
53
|
+
async function runRun(scenarioPaths, opts) {
|
|
54
|
+
const propFilters = [];
|
|
55
|
+
for (const pair of opts.filter ?? []) {
|
|
56
|
+
const eq = pair.indexOf('=');
|
|
57
|
+
if (eq === -1) {
|
|
58
|
+
console.error(`devlow-bench: invalid -F ${pair} (expected key=value).`);
|
|
59
|
+
process.exit(1);
|
|
60
|
+
}
|
|
61
|
+
propFilters.push([pair.slice(0, eq), pair.slice(eq + 1)]);
|
|
47
62
|
}
|
|
48
63
|
const scenarios = [];
|
|
49
64
|
setCurrentScenarios(scenarios);
|
|
50
|
-
for (const path of
|
|
65
|
+
for (const path of scenarioPaths) {
|
|
51
66
|
await import(pathToFileURL(join(process.cwd(), path)).toString());
|
|
52
67
|
}
|
|
53
68
|
setCurrentScenarios(null);
|
|
54
69
|
const cliIface = {
|
|
55
|
-
filterScenarios: async (
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
return
|
|
59
|
-
|
|
60
|
-
return scenarios;
|
|
70
|
+
filterScenarios: async (allScenarios) => {
|
|
71
|
+
const filters = opts.scenario;
|
|
72
|
+
if (!filters || filters.length === 0)
|
|
73
|
+
return allScenarios;
|
|
74
|
+
return allScenarios.filter((s) => filters.some((f) => s.name.includes(f)));
|
|
61
75
|
},
|
|
62
76
|
filterScenarioVariants: async (variants) => {
|
|
63
|
-
|
|
64
|
-
if (propEntries.length === 0)
|
|
77
|
+
if (propFilters.length === 0)
|
|
65
78
|
return variants;
|
|
66
|
-
|
|
67
|
-
|
|
79
|
+
// Group multiple -F key=value with the same key into an OR set.
|
|
80
|
+
const byKey = new Map();
|
|
81
|
+
for (const [k, v] of propFilters) {
|
|
82
|
+
const existing = byKey.get(k);
|
|
83
|
+
if (existing)
|
|
84
|
+
existing.push(v);
|
|
85
|
+
else
|
|
86
|
+
byKey.set(k, [v]);
|
|
87
|
+
}
|
|
88
|
+
for (const [key, values] of byKey) {
|
|
68
89
|
variants = variants.filter((variant) => {
|
|
69
90
|
const prop = variant.props[key];
|
|
70
|
-
if (typeof prop ===
|
|
91
|
+
if (typeof prop === 'undefined')
|
|
71
92
|
return false;
|
|
72
93
|
const str = prop.toString();
|
|
73
94
|
return values.some((v) => str.includes(v));
|
|
@@ -76,19 +97,43 @@ import { pathToFileURL } from "url";
|
|
|
76
97
|
return variants;
|
|
77
98
|
},
|
|
78
99
|
};
|
|
79
|
-
|
|
100
|
+
// Validation (clamp to non-negative integers, default n=1/warmup=0) is
|
|
101
|
+
// delegated to runScenarios — see runner.ts.
|
|
102
|
+
const n = typeof opts.n === 'number' && Number.isFinite(opts.n) ? opts.n : 1;
|
|
103
|
+
const warmup = typeof opts.warmup === 'number' && Number.isFinite(opts.warmup)
|
|
104
|
+
? opts.warmup
|
|
105
|
+
: 0;
|
|
106
|
+
// Snapshot is always on. --snapshot=<path> overrides the default path.
|
|
107
|
+
const snapshotIface = (await import('./interfaces/snapshot.js')).default({
|
|
108
|
+
path: opts.snapshot,
|
|
109
|
+
});
|
|
110
|
+
// Comparison: enabled by --compare or by giving an explicit --baseline.
|
|
111
|
+
const compareEnabled = opts.compare === true || typeof opts.baseline === 'string';
|
|
112
|
+
let compareIface = null;
|
|
113
|
+
if (compareEnabled) {
|
|
114
|
+
const baselineArg = typeof opts.baseline === 'string' ? opts.baseline : true;
|
|
115
|
+
const baselinePath = await resolveCompareTarget(baselineArg, snapshotIface.resolvedPath);
|
|
116
|
+
if (baselinePath == null) {
|
|
117
|
+
console.error('No baseline snapshot found. Run devlow-bench at least once, or pass --baseline=<path>.');
|
|
118
|
+
process.exit(1);
|
|
119
|
+
}
|
|
120
|
+
compareIface = await (await import('./interfaces/compare.js')).default({ baselinePath });
|
|
121
|
+
}
|
|
122
|
+
const ifaces = [
|
|
80
123
|
cliIface,
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
124
|
+
opts.interactive && (await import('./interfaces/interactive.js')).default(),
|
|
125
|
+
opts.json &&
|
|
126
|
+
(await import('./interfaces/json.js')).default(opts.json, { n }),
|
|
127
|
+
opts.datadog &&
|
|
128
|
+
(await import('./interfaces/datadog.js')).default(typeof opts.datadog === 'string' ? { host: opts.datadog } : undefined),
|
|
129
|
+
opts.snowflake &&
|
|
130
|
+
(await import('./interfaces/snowflake.js')).default(typeof opts.snowflake === 'string'
|
|
131
|
+
? { gatewayUri: opts.snowflake }
|
|
132
|
+
: undefined),
|
|
133
|
+
opts.console !== false &&
|
|
134
|
+
(await import('./interfaces/console.js')).default({ n }),
|
|
135
|
+
compareIface,
|
|
136
|
+
snapshotIface,
|
|
89
137
|
].filter((x) => x);
|
|
90
|
-
await runScenarios(scenarios, compose(...ifaces));
|
|
91
|
-
}
|
|
92
|
-
console.error(e.stack);
|
|
93
|
-
process.exit(1);
|
|
94
|
-
});
|
|
138
|
+
await runScenarios(scenarios, compose(...ifaces), { n, warmup });
|
|
139
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { SnapshotRow } from './snapshot.js';
|
|
2
|
+
export interface SampleGroup {
|
|
3
|
+
scenario: string;
|
|
4
|
+
variant: string;
|
|
5
|
+
metric: string;
|
|
6
|
+
unit: string;
|
|
7
|
+
samples: number[];
|
|
8
|
+
}
|
|
9
|
+
export declare function groupRows(rows: SnapshotRow[]): Map<string, SampleGroup>;
|
|
10
|
+
export declare function makeKey(scenario: string, variant: string, metric: string): string;
|
|
11
|
+
export declare function printComparison(baseline: Map<string, SampleGroup>, current: Map<string, SampleGroup>, options?: {
|
|
12
|
+
baselineLabel: string;
|
|
13
|
+
currentLabel?: string;
|
|
14
|
+
}): void;
|