@vercel/devlow-bench 0.3.5 → 16.4.0-canary.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +50 -33
  2. package/dist/browser.d.ts +1 -1
  3. package/dist/browser.js +52 -44
  4. package/dist/cli.js +114 -69
  5. package/dist/compare.d.ts +14 -0
  6. package/dist/compare.js +209 -0
  7. package/dist/describe.d.ts +1 -1
  8. package/dist/describe.js +18 -18
  9. package/dist/file.js +5 -5
  10. package/dist/index.d.ts +16 -3
  11. package/dist/index.js +3 -2
  12. package/dist/interfaces/compare.d.ts +4 -0
  13. package/dist/interfaces/compare.js +27 -0
  14. package/dist/interfaces/compose.d.ts +1 -1
  15. package/dist/interfaces/compose.js +1 -1
  16. package/dist/interfaces/console.d.ts +4 -2
  17. package/dist/interfaces/console.js +30 -8
  18. package/dist/interfaces/constants.js +7 -7
  19. package/dist/interfaces/datadog.d.ts +1 -1
  20. package/dist/interfaces/datadog.js +13 -13
  21. package/dist/interfaces/interactive.d.ts +1 -1
  22. package/dist/interfaces/interactive.js +8 -8
  23. package/dist/interfaces/json.d.ts +4 -2
  24. package/dist/interfaces/json.js +36 -17
  25. package/dist/interfaces/shell-test.d.ts +1 -0
  26. package/dist/interfaces/shell-test.js +56 -0
  27. package/dist/interfaces/snapshot.d.ts +6 -0
  28. package/dist/interfaces/snapshot.js +60 -0
  29. package/dist/interfaces/snowflake-test.js +3 -3
  30. package/dist/interfaces/snowflake.d.ts +1 -1
  31. package/dist/interfaces/snowflake.js +15 -13
  32. package/dist/runner.d.ts +5 -2
  33. package/dist/runner.js +131 -24
  34. package/dist/shell.d.ts +5 -3
  35. package/dist/shell.js +56 -28
  36. package/dist/snapshot.d.ts +18 -0
  37. package/dist/snapshot.js +162 -0
  38. package/dist/statistics-test.d.ts +1 -0
  39. package/dist/statistics-test.js +102 -0
  40. package/dist/statistics.d.ts +19 -0
  41. package/dist/statistics.js +123 -0
  42. package/dist/table.js +51 -51
  43. package/dist/units.js +6 -6
  44. package/dist/utils.d.ts +1 -0
  45. package/dist/utils.js +8 -6
  46. package/package.json +6 -5
package/README.md CHANGED
@@ -10,19 +10,36 @@ npm install devlow-bench
10
10
 
11
11
  ## Usage
12
12
 
13
- ```bash
14
- Usage: devlow-bench [options] <scenario files>
15
- ## Selecting scenarios
16
- --scenario=<filter>, -s=<filter> Only run the scenario with the given name
17
- --interactive, -i Select scenarios and variants interactively
18
- --<prop>=<value> Filter by any variant property defined in scenarios
19
- ## Output
20
- --json=<path>, -j=<path> Write the results to the given path as JSON
21
- --console Print the results to the console
22
- --datadog[=<hostname>] Upload the results to Datadog
23
- (requires DATADOG_API_KEY environment variables)
24
- ## Help
25
- --help, -h, -? Show this help
13
+ ```text
14
+ Usage: devlow-bench [options] [command]
15
+
16
+ Run developer-workflow benchmarks.
17
+
18
+ Options:
19
+ -h, --help display help for command
20
+
21
+ Commands:
22
+ run [options] [scenarios...] Run scenario files and report measurements.
23
+ compare <baseline> <current> Compare two snapshot CSVs side-by-side.
24
+ help [command] display help for command
25
+ ```
26
+
27
+ `run` is the default command, so scenario paths can still be passed without
28
+ writing `run` explicitly. Its options include:
29
+
30
+ ```text
31
+ -s, --scenario <filter> Only run scenarios whose name matches the filter (repeatable).
32
+ -F, --filter <pair> Filter variants by property: key=value (repeatable).
33
+ -i, --interactive Select scenarios and variants interactively.
34
+ --n <number> Run each variant N times.
35
+ --warmup <number> Discard the first N runs before sampling.
36
+ --snapshot <path> Override the snapshot CSV path.
37
+ --compare Print a comparison table after the run.
38
+ --baseline <path> Select a comparison baseline; implies --compare.
39
+ -j, --json <path> Write results as JSON.
40
+ --no-console Suppress console output.
41
+ --datadog [host] Upload results to Datadog.
42
+ --snowflake [batchUri] Upload results to Snowflake.
26
43
  ```
27
44
 
28
45
  ## Scenarios
@@ -30,10 +47,10 @@ Usage: devlow-bench [options] <scenario files>
30
47
  A scenario file is similar to a test case file. It can contain one or multiple scenarios by using the `describe()` method to define them.
31
48
 
32
49
  ```js
33
- import { describe } from "devlow-bench";
50
+ import { describe } from 'devlow-bench'
34
51
 
35
52
  describe(
36
- "my scenario",
53
+ 'my scenario',
37
54
  {
38
55
  /* property options */
39
56
  },
@@ -44,7 +61,7 @@ describe(
44
61
  ) => {
45
62
  // run the scenario
46
63
  }
47
- );
64
+ )
48
65
  ```
49
66
 
50
67
  The `describe()` method takes three arguments:
@@ -58,18 +75,18 @@ The `props` object can contain any number of properties. The key is the name of
58
75
  ### Example
59
76
 
60
77
  ```js
61
- import { describe } from "devlow-bench";
78
+ import { describe } from 'devlow-bench'
62
79
 
63
80
  describe(
64
- "my scenario",
81
+ 'my scenario',
65
82
  {
66
83
  myProperty: [1, 2, 3],
67
84
  myOtherProperty: true,
68
85
  },
69
86
  async ({ myProperty, myOtherProperty }) => {
70
- console.log(myProperty, myOtherProperty);
87
+ console.log(myProperty, myOtherProperty)
71
88
  }
72
- );
89
+ )
73
90
 
74
91
  // will print:
75
92
  // 1 true
@@ -83,17 +100,17 @@ describe(
83
100
  ## Reporting measurements
84
101
 
85
102
  ```js
86
- import { measureTime, reportMeasurement } from "devlow-bench";
103
+ import { measureTime, reportMeasurement } from 'devlow-bench'
87
104
 
88
105
  // Measure a time
89
- await measureTime("name of the timing", {
106
+ await measureTime('name of the timing', {
90
107
  /* optional options */
91
- });
108
+ })
92
109
 
93
110
  // Report some other measurement
94
- await reportMeasurement("name of the measurement", value, unit, {
111
+ await reportMeasurement('name of the measurement', value, unit, {
95
112
  /* optional options */
96
- });
113
+ })
97
114
  ```
98
115
 
99
116
  Options:
@@ -107,15 +124,15 @@ Options:
107
124
  The `devlow-bench` package provides a few helper functions to run operations in the browser.
108
125
 
109
126
  ```js
110
- import { newBrowserSession } from "devlow-bench/browser";
127
+ import { newBrowserSession } from 'devlow-bench/browser'
111
128
 
112
129
  const session = await newBrowserSession({
113
130
  // options
114
- });
115
- await session.hardNavigation("metric name", "https://example.com");
116
- await session.reload("metric name");
117
- await session.softNavigationByClick("metric name", ".selector-to-click");
118
- await session.close();
131
+ })
132
+ await session.hardNavigation('metric name', 'https://example.com')
133
+ await session.reload('metric name')
134
+ await session.softNavigationByClick('metric name', '.selector-to-click')
135
+ await session.close()
119
136
  ```
120
137
 
121
138
  Run with `BROWSER_OUTPUT=1` to show the output of the browser.
@@ -162,8 +179,8 @@ Run with `SHELL_OUTPUT=1` to show the output of the shell commands.
162
179
  The `devlow-bench` package provides a few helper functions to run operations on the file system.
163
180
 
164
181
  ```js
165
- import { waitForFile } from "devlow-bench/file";
182
+ import { waitForFile } from 'devlow-bench/file'
166
183
 
167
184
  // wait for file to exist
168
- await waitForFile("/path/to/file", /* timeout = */ 30000);
185
+ await waitForFile('/path/to/file', /* timeout = */ 30000)
169
186
  ```
package/dist/browser.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { Page } from "playwright-chromium";
1
+ import type { Page } from 'playwright-chromium';
2
2
  interface BrowserSession {
3
3
  close(): Promise<void>;
4
4
  hardNavigation(metricName: string, url: string): Promise<Page>;
package/dist/browser.js CHANGED
@@ -1,5 +1,5 @@
1
- import { chromium } from "playwright-chromium";
2
- import { measureTime, reportMeasurement } from "./index.js";
1
+ import { chromium } from 'playwright-chromium';
2
+ import { measureTime, reportMeasurement } from './index.js';
3
3
  const browserOutput = Boolean(process.env.BROWSER_OUTPUT);
4
4
  async function withRequestMetrics(metricName, page, fn) {
5
5
  const activePromises = [];
@@ -9,9 +9,7 @@ async function withRequestMetrics(metricName, page, fn) {
9
9
  activePromises.push((async () => {
10
10
  const url = response.request().url();
11
11
  const status = response.status();
12
- const extension =
13
- // eslint-disable-next-line prefer-named-capture-group -- TODO: address lint
14
- /^[^?#]+\.([a-z0-9]+)(?:[?#]|$)/i.exec(url)?.[1] ?? "none";
12
+ const extension = /^[^?#]+\.([a-z0-9]+)(?:[?#]|$)/i.exec(url)?.[1] ?? 'none';
15
13
  const currentRequests = requestsByExtension.get(extension) ?? 0;
16
14
  requestsByExtension.set(extension, currentRequests + 1);
17
15
  if (status >= 200 && status < 300) {
@@ -35,10 +33,10 @@ async function withRequestMetrics(metricName, page, fn) {
35
33
  let logCount = 0;
36
34
  const consoleHandler = (message) => {
37
35
  const type = message.type();
38
- if (type === "error") {
36
+ if (type === 'error') {
39
37
  errorCount++;
40
38
  }
41
- else if (type === "warning") {
39
+ else if (type === 'warning') {
42
40
  warningCount++;
43
41
  }
44
42
  else {
@@ -68,31 +66,31 @@ async function withRequestMetrics(metricName, page, fn) {
68
66
  }
69
67
  };
70
68
  try {
71
- page.on("response", responseHandler);
72
- page.on("console", consoleHandler);
73
- page.on("pageerror", exceptionHandler);
69
+ page.on('response', responseHandler);
70
+ page.on('console', consoleHandler);
71
+ page.on('pageerror', exceptionHandler);
74
72
  await fn();
75
73
  await Promise.all(activePromises);
76
74
  let totalDownload = 0;
77
75
  for (const [extension, size] of sizeByExtension.entries()) {
78
- await reportMeasurement(`${metricName}/responseSizes/${extension}`, size, "bytes");
76
+ await reportMeasurement(`${metricName}/responseSizes/${extension}`, size, 'bytes');
79
77
  totalDownload += size;
80
78
  }
81
- await reportMeasurement(`${metricName}/responseSizes`, totalDownload, "bytes");
79
+ await reportMeasurement(`${metricName}/responseSizes`, totalDownload, 'bytes');
82
80
  let totalRequests = 0;
83
81
  for (const [extension, count] of requestsByExtension.entries()) {
84
- await reportMeasurement(`${metricName}/requests/${extension}`, count, "requests");
82
+ await reportMeasurement(`${metricName}/requests/${extension}`, count, 'requests');
85
83
  totalRequests += count;
86
84
  }
87
- await reportMeasurement(`${metricName}/requests`, totalRequests, "requests");
88
- await reportMeasurement(`${metricName}/console/logs`, logCount, "messages");
89
- await reportMeasurement(`${metricName}/console/warnings`, warningCount, "messages");
90
- await reportMeasurement(`${metricName}/console/errors`, errorCount, "messages");
91
- await reportMeasurement(`${metricName}/console/uncaught`, uncaughtCount, "messages");
92
- await reportMeasurement(`${metricName}/console`, logCount + warningCount + errorCount + uncaughtCount, "messages");
85
+ await reportMeasurement(`${metricName}/requests`, totalRequests, 'requests');
86
+ await reportMeasurement(`${metricName}/console/logs`, logCount, 'messages');
87
+ await reportMeasurement(`${metricName}/console/warnings`, warningCount, 'messages');
88
+ await reportMeasurement(`${metricName}/console/errors`, errorCount, 'messages');
89
+ await reportMeasurement(`${metricName}/console/uncaught`, uncaughtCount, 'messages');
90
+ await reportMeasurement(`${metricName}/console`, logCount + warningCount + errorCount + uncaughtCount, 'messages');
93
91
  }
94
92
  finally {
95
- page.off("response", responseHandler);
93
+ page.off('response', responseHandler);
96
94
  }
97
95
  }
98
96
  /**
@@ -105,9 +103,9 @@ async function withRequestMetrics(metricName, page, fn) {
105
103
  function networkIdle(page, delayMs = 300, timeoutMs = 180000) {
106
104
  return new Promise((resolve) => {
107
105
  const cleanup = () => {
108
- page.off("request", requestHandler);
109
- page.off("requestfailed", requestFinishedHandler);
110
- page.off("requestfinished", requestFinishedHandler);
106
+ page.off('request', requestHandler);
107
+ page.off('requestfailed', requestFinishedHandler);
108
+ page.off('requestfinished', requestFinishedHandler);
111
109
  clearTimeout(fullTimeout);
112
110
  if (timeout) {
113
111
  clearTimeout(timeout);
@@ -119,12 +117,11 @@ function networkIdle(page, delayMs = 300, timeoutMs = 180000) {
119
117
  let timeout = null;
120
118
  const fullTimeout = setTimeout(() => {
121
119
  cleanup();
122
- // eslint-disable-next-line no-console -- logging
123
- console.error(`Timeout while waiting for network idle. These requests are still pending: ${Array.from(requests).join(", ")}} time is ${lastRequest - start}`);
120
+ console.error(`Timeout while waiting for network idle. These requests are still pending: ${Array.from(requests).join(', ')}} time is ${lastRequest - start}`);
124
121
  resolve(Date.now() - lastRequest);
125
122
  }, timeoutMs);
126
123
  const requestFilter = (request) => {
127
- return request.headers().accept !== "text/event-stream";
124
+ return request.headers().accept !== 'text/event-stream';
128
125
  };
129
126
  const requestHandler = (request) => {
130
127
  requests.set(request.url(), (requests.get(request.url()) ?? 0) + 1);
@@ -147,7 +144,6 @@ function networkIdle(page, delayMs = 300, timeoutMs = 180000) {
147
144
  lastRequest = Date.now();
148
145
  const currentCount = requests.get(request.url());
149
146
  if (currentCount === undefined) {
150
- // eslint-disable-next-line no-console -- basic logging
151
147
  console.error(`Unexpected untracked but completed request ${request.url()}`);
152
148
  return;
153
149
  }
@@ -164,9 +160,9 @@ function networkIdle(page, delayMs = 300, timeoutMs = 180000) {
164
160
  }, delayMs);
165
161
  }
166
162
  };
167
- page.on("request", requestHandler);
168
- page.on("requestfailed", requestFinishedHandler);
169
- page.on("requestfinished", requestFinishedHandler);
163
+ page.on('request', requestHandler);
164
+ page.on('requestfailed', requestFinishedHandler);
165
+ page.on('requestfinished', requestFinishedHandler);
170
166
  });
171
167
  }
172
168
  class BrowserSessionImpl {
@@ -191,17 +187,23 @@ class BrowserSessionImpl {
191
187
  await withRequestMetrics(metricName, page, async () => {
192
188
  await measureTime(`${metricName}/start`);
193
189
  const idle = networkIdle(page, 3000);
194
- await page.goto(url, {
195
- waitUntil: "commit",
190
+ const response = await page.goto(url, {
191
+ waitUntil: 'commit',
196
192
  });
193
+ if (!response) {
194
+ throw new Error(`Navigation to ${url} produced no response`);
195
+ }
196
+ if (!response.ok()) {
197
+ throw new Error(`Navigation to ${url} returned HTTP ${response.status()}`);
198
+ }
197
199
  await measureTime(`${metricName}/html`, {
198
200
  relativeTo: `${metricName}/start`,
199
201
  });
200
- await page.waitForLoadState("domcontentloaded");
202
+ await page.waitForLoadState('domcontentloaded');
201
203
  await measureTime(`${metricName}/dom`, {
202
204
  relativeTo: `${metricName}/start`,
203
205
  });
204
- await page.waitForLoadState("load");
206
+ await page.waitForLoadState('load');
205
207
  await measureTime(`${metricName}/load`, {
206
208
  relativeTo: `${metricName}/start`,
207
209
  });
@@ -216,12 +218,12 @@ class BrowserSessionImpl {
216
218
  async softNavigationByClick(metricName, selector) {
217
219
  const page = this.page;
218
220
  if (!page) {
219
- throw new Error("softNavigationByClick() must be called after hardNavigation()");
221
+ throw new Error('softNavigationByClick() must be called after hardNavigation()');
220
222
  }
221
223
  await withRequestMetrics(metricName, page, async () => {
222
224
  await measureTime(`${metricName}/start`);
223
225
  const firstResponse = new Promise((resolve) => {
224
- page.once("response", () => {
226
+ page.once('response', () => {
225
227
  resolve();
226
228
  });
227
229
  });
@@ -241,22 +243,28 @@ class BrowserSessionImpl {
241
243
  async reload(metricName) {
242
244
  const page = this.page;
243
245
  if (!page) {
244
- throw new Error("reload() must be called after hardNavigation()");
246
+ throw new Error('reload() must be called after hardNavigation()');
245
247
  }
246
248
  await withRequestMetrics(metricName, page, async () => {
247
249
  await measureTime(`${metricName}/start`);
248
250
  const idle = networkIdle(page, 3000);
249
- await page.reload({
250
- waitUntil: "commit",
251
+ const response = await page.reload({
252
+ waitUntil: 'commit',
251
253
  });
254
+ if (!response) {
255
+ throw new Error('Reload produced no response');
256
+ }
257
+ if (!response.ok()) {
258
+ throw new Error(`Reload returned HTTP ${response.status()}`);
259
+ }
252
260
  await measureTime(`${metricName}/html`, {
253
261
  relativeTo: `${metricName}/start`,
254
262
  });
255
- await page.waitForLoadState("domcontentloaded");
263
+ await page.waitForLoadState('domcontentloaded');
256
264
  await measureTime(`${metricName}/dom`, {
257
265
  relativeTo: `${metricName}/start`,
258
266
  });
259
- await page.waitForLoadState("load");
267
+ await page.waitForLoadState('load');
260
268
  await measureTime(`${metricName}/load`, {
261
269
  relativeTo: `${metricName}/start`,
262
270
  });
@@ -270,12 +278,12 @@ class BrowserSessionImpl {
270
278
  }
271
279
  export async function newBrowserSession(options) {
272
280
  const browser = await chromium.launch({
273
- headless: options.headless ?? process.env.HEADLESS !== "false",
274
- devtools: true,
281
+ headless: options.headless ?? process.env.HEADLESS !== 'false',
282
+ args: options.headless ? undefined : ['--auto-open-devtools-for-tabs'],
275
283
  timeout: 60000,
276
284
  });
277
285
  const context = await browser.newContext({
278
- baseURL: options.baseURL ?? "http://localhost:3000",
286
+ baseURL: options.baseURL ?? 'http://localhost:3000',
279
287
  viewport: { width: 1280, height: 720 },
280
288
  });
281
289
  context.setDefaultTimeout(120000);
package/dist/cli.js CHANGED
@@ -1,73 +1,94 @@
1
- import minimist from "minimist";
2
- import { setCurrentScenarios } from "./describe.js";
3
- import { join } from "path";
4
- import { runScenarios } from "./index.js";
5
- import compose from "./interfaces/compose.js";
6
- import { pathToFileURL } from "url";
1
+ import { Command } from 'commander';
2
+ import { join } from 'path';
3
+ import { pathToFileURL } from 'url';
4
+ import { groupRows, printComparison } from './compare.js';
5
+ import { setCurrentScenarios } from './describe.js';
6
+ import { runScenarios } from './index.js';
7
+ import compose from './interfaces/compose.js';
8
+ import { readSnapshot, resolveCompareTarget } from './snapshot.js';
9
+ ;
7
10
  (async () => {
8
- const knownArgs = new Set([
9
- "scenario",
10
- "s",
11
- "json",
12
- "j",
13
- "console",
14
- "datadog",
15
- "snowflake",
16
- "interactive",
17
- "i",
18
- "help",
19
- "h",
20
- "?",
21
- "_",
11
+ const program = new Command()
12
+ .name('devlow-bench')
13
+ .description('Run developer-workflow benchmarks.')
14
+ .showHelpAfterError();
15
+ program
16
+ .command('run', { isDefault: true })
17
+ .description('Run scenario files and report measurements.')
18
+ .argument('[scenarios...]', 'Scenario module paths to load.')
19
+ .option('-s, --scenario <filter>', 'Only run scenarios whose name matches the filter (repeatable).', (v, prev) => (prev ?? []).concat(v), [])
20
+ .option('-F, --filter <pair>', 'Filter variants by property: -F key=value (repeatable; same key repeated = OR).', (v, prev) => (prev ?? []).concat(v), [])
21
+ .option('-i, --interactive', 'Select scenarios and variants interactively.')
22
+ .option('--n <number>', 'Run each variant N times; reports mean/p50/p90 per metric. Default: 1.', (v) => Number(v))
23
+ .option('--warmup <number>', 'Discard the first N runs of each variant before sampling. Default: 0. Do NOT enable when measuring cold-start metrics.', (v) => Number(v))
24
+ .option('--snapshot <path>', 'Override the snapshot CSV path. Default: ./.devlow-bench/snapshots/<ts>.csv. Snapshots are always written.')
25
+ .option('--compare', 'Print a comparison table at end of run. Baseline = newest snapshot, unless --baseline overrides.')
26
+ .option('--baseline <path>', 'Explicit baseline (file or directory). Implies --compare.')
27
+ .option('-j, --json <path>', 'Write the results to the given path as JSON.')
28
+ .option('--no-console', 'Suppress console output.')
29
+ .option('--datadog [host]', 'Upload the results to Datadog (requires DATADOG_API_KEY).')
30
+ .option('--snowflake [batchUri]', 'Upload the results to Snowflake (requires SNOWFLAKE_TOPIC_NAME and SNOWFLAKE_SCHEMA_ID).')
31
+ .action(runRun);
32
+ program
33
+ .command('compare')
34
+ .description('Compare two snapshot CSVs side-by-side, with p50/p90/p99 plus a Mann–Whitney U p-value per metric.')
35
+ .argument('<baseline>', 'Baseline snapshot CSV path.')
36
+ .argument('<current>', 'Current snapshot CSV path.')
37
+ .action(runCompare);
38
+ await program.parseAsync();
39
+ })().catch((e) => {
40
+ console.error(e.stack);
41
+ process.exit(1);
42
+ });
43
+ async function runCompare(baselinePath, currentPath) {
44
+ const [baseRows, curRows] = await Promise.all([
45
+ readSnapshot(baselinePath),
46
+ readSnapshot(currentPath),
22
47
  ]);
23
- const args = minimist(process.argv.slice(2), {
24
- alias: {
25
- s: "scenario",
26
- j: "json",
27
- i: "interactive",
28
- "?": "help",
29
- h: "help",
30
- },
48
+ printComparison(groupRows(baseRows), groupRows(curRows), {
49
+ baselineLabel: baselinePath,
50
+ currentLabel: currentPath,
31
51
  });
32
- if (args.help || (Object.keys(args).length === 1 && args._.length === 0)) {
33
- console.log("Usage: devlow-bench [options] <scenario files>");
34
- console.log("## Selecting scenarios");
35
- console.log(" --scenario=<filter>, -s=<filter> Only run the scenario with the given name");
36
- console.log(" --interactive, -i Select scenarios and variants interactively");
37
- console.log(" --<prop>=<value> Filter by any variant property defined in scenarios");
38
- console.log("## Output");
39
- console.log(" --json=<path>, -j=<path> Write the results to the given path as JSON");
40
- console.log(" --console Print the results to the console");
41
- console.log(" --datadog[=<hostname>] Upload the results to Datadog");
42
- console.log(" (requires DATADOG_API_KEY environment variables)");
43
- console.log(" --snowflake[=<batch-uri>] Upload the results to Snowflake");
44
- console.log(" (requires SNOWFLAKE_TOPIC_NAME and SNOWFLAKE_SCHEMA_ID and environment variables)");
45
- console.log("## Help");
46
- console.log(" --help, -h, -? Show this help");
52
+ }
53
+ async function runRun(scenarioPaths, opts) {
54
+ const propFilters = [];
55
+ for (const pair of opts.filter ?? []) {
56
+ const eq = pair.indexOf('=');
57
+ if (eq === -1) {
58
+ console.error(`devlow-bench: invalid -F ${pair} (expected key=value).`);
59
+ process.exit(1);
60
+ }
61
+ propFilters.push([pair.slice(0, eq), pair.slice(eq + 1)]);
47
62
  }
48
63
  const scenarios = [];
49
64
  setCurrentScenarios(scenarios);
50
- for (const path of args._) {
65
+ for (const path of scenarioPaths) {
51
66
  await import(pathToFileURL(join(process.cwd(), path)).toString());
52
67
  }
53
68
  setCurrentScenarios(null);
54
69
  const cliIface = {
55
- filterScenarios: async (scenarios) => {
56
- if (args.scenario) {
57
- const filter = [].concat(args.scenario);
58
- return scenarios.filter((s) => filter.some((filter) => s.name.includes(filter)));
59
- }
60
- return scenarios;
70
+ filterScenarios: async (allScenarios) => {
71
+ const filters = opts.scenario;
72
+ if (!filters || filters.length === 0)
73
+ return allScenarios;
74
+ return allScenarios.filter((s) => filters.some((f) => s.name.includes(f)));
61
75
  },
62
76
  filterScenarioVariants: async (variants) => {
63
- const propEntries = Object.entries(args).filter(([key]) => !knownArgs.has(key));
64
- if (propEntries.length === 0)
77
+ if (propFilters.length === 0)
65
78
  return variants;
66
- for (const [key, value] of propEntries) {
67
- const values = (Array.isArray(value) ? value : [value]).map((v) => v.toString());
79
+ // Group multiple -F key=value with the same key into an OR set.
80
+ const byKey = new Map();
81
+ for (const [k, v] of propFilters) {
82
+ const existing = byKey.get(k);
83
+ if (existing)
84
+ existing.push(v);
85
+ else
86
+ byKey.set(k, [v]);
87
+ }
88
+ for (const [key, values] of byKey) {
68
89
  variants = variants.filter((variant) => {
69
90
  const prop = variant.props[key];
70
- if (typeof prop === "undefined")
91
+ if (typeof prop === 'undefined')
71
92
  return false;
72
93
  const str = prop.toString();
73
94
  return values.some((v) => str.includes(v));
@@ -76,19 +97,43 @@ import { pathToFileURL } from "url";
76
97
  return variants;
77
98
  },
78
99
  };
79
- let ifaces = [
100
+ // Validation (clamp to non-negative integers, default n=1/warmup=0) is
101
+ // delegated to runScenarios — see runner.ts.
102
+ const n = typeof opts.n === 'number' && Number.isFinite(opts.n) ? opts.n : 1;
103
+ const warmup = typeof opts.warmup === 'number' && Number.isFinite(opts.warmup)
104
+ ? opts.warmup
105
+ : 0;
106
+ // Snapshot is always on. --snapshot=<path> overrides the default path.
107
+ const snapshotIface = (await import('./interfaces/snapshot.js')).default({
108
+ path: opts.snapshot,
109
+ });
110
+ // Comparison: enabled by --compare or by giving an explicit --baseline.
111
+ const compareEnabled = opts.compare === true || typeof opts.baseline === 'string';
112
+ let compareIface = null;
113
+ if (compareEnabled) {
114
+ const baselineArg = typeof opts.baseline === 'string' ? opts.baseline : true;
115
+ const baselinePath = await resolveCompareTarget(baselineArg, snapshotIface.resolvedPath);
116
+ if (baselinePath == null) {
117
+ console.error('No baseline snapshot found. Run devlow-bench at least once, or pass --baseline=<path>.');
118
+ process.exit(1);
119
+ }
120
+ compareIface = await (await import('./interfaces/compare.js')).default({ baselinePath });
121
+ }
122
+ const ifaces = [
80
123
  cliIface,
81
- args.interactive && (await import("./interfaces/interactive.js")).default(),
82
- args.json && (await import("./interfaces/json.js")).default(args.json),
83
- args.datadog &&
84
- (await import("./interfaces/datadog.js")).default(typeof args.datadog === "string" ? { host: args.datadog } : undefined),
85
- args.snowflake &&
86
- (await import("./interfaces/snowflake.js")).default(typeof args.snowflake === "string" ? { gatewayUri: args.snowflake } : undefined),
87
- args.console !== false &&
88
- (await import("./interfaces/console.js")).default(),
124
+ opts.interactive && (await import('./interfaces/interactive.js')).default(),
125
+ opts.json &&
126
+ (await import('./interfaces/json.js')).default(opts.json, { n }),
127
+ opts.datadog &&
128
+ (await import('./interfaces/datadog.js')).default(typeof opts.datadog === 'string' ? { host: opts.datadog } : undefined),
129
+ opts.snowflake &&
130
+ (await import('./interfaces/snowflake.js')).default(typeof opts.snowflake === 'string'
131
+ ? { gatewayUri: opts.snowflake }
132
+ : undefined),
133
+ opts.console !== false &&
134
+ (await import('./interfaces/console.js')).default({ n }),
135
+ compareIface,
136
+ snapshotIface,
89
137
  ].filter((x) => x);
90
- await runScenarios(scenarios, compose(...ifaces));
91
- })().catch((e) => {
92
- console.error(e.stack);
93
- process.exit(1);
94
- });
138
+ await runScenarios(scenarios, compose(...ifaces), { n, warmup });
139
+ }
@@ -0,0 +1,14 @@
1
+ import type { SnapshotRow } from './snapshot.js';
2
+ export interface SampleGroup {
3
+ scenario: string;
4
+ variant: string;
5
+ metric: string;
6
+ unit: string;
7
+ samples: number[];
8
+ }
9
+ export declare function groupRows(rows: SnapshotRow[]): Map<string, SampleGroup>;
10
+ export declare function makeKey(scenario: string, variant: string, metric: string): string;
11
+ export declare function printComparison(baseline: Map<string, SampleGroup>, current: Map<string, SampleGroup>, options?: {
12
+ baselineLabel: string;
13
+ currentLabel?: string;
14
+ }): void;