@ac-bench/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +50 -0
- package/bin/measure.js +15 -0
- package/dist/config.d.ts +80 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +5 -0
- package/dist/config.js.map +1 -0
- package/dist/main.d.ts +2 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +71 -0
- package/dist/main.js.map +1 -0
- package/dist/mode.d.ts +27 -0
- package/dist/mode.d.ts.map +1 -0
- package/dist/mode.js +65 -0
- package/dist/mode.js.map +1 -0
- package/dist/resolve-reporter-plugins.d.ts +13 -0
- package/dist/resolve-reporter-plugins.d.ts.map +1 -0
- package/dist/resolve-reporter-plugins.js +48 -0
- package/dist/resolve-reporter-plugins.js.map +1 -0
- package/dist/runner.d.ts +45 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +70 -0
- package/dist/runner.js.map +1 -0
- package/package.json +83 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024-2026 Anthony Champagne and contributors
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
package/README.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# @ac-bench/cli
|
|
2
|
+
|
|
3
|
+
The benchmark CLI: argument parsing, config-file loading, and the resolution of
|
|
4
|
+
everything a run needs — which reporters to use, what `--mode` stands for, where
|
|
5
|
+
output files go.
|
|
6
|
+
|
|
7
|
+
The engine underneath is `@ac-bench/lib`, which takes those already resolved.
|
|
8
|
+
Use it directly to run benchmarks from your own code instead of a config file.
|
|
9
|
+
|
|
10
|
+
## Configuration
|
|
11
|
+
|
|
12
|
+
```ts
|
|
13
|
+
// bench.config.ts
|
|
14
|
+
import { defineConfig } from "@ac-bench/cli";
|
|
15
|
+
import durationPlugin from "@ac-bench/measure-duration/plugin";
|
|
16
|
+
import jsonReporter from "@ac-bench/reporter-json";
|
|
17
|
+
|
|
18
|
+
export default defineConfig({
|
|
19
|
+
plugins: [durationPlugin, jsonReporter({ output: "artifacts/bench.json" })],
|
|
20
|
+
reporters: ["table", "json"],
|
|
21
|
+
});
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
`plugins` holds measure and reporter plugins together — import each package and
|
|
25
|
+
list what it exports. A package that takes options default-exports a factory
|
|
26
|
+
(`jsonReporter({ … })`); one that does not default-exports the plugin
|
|
27
|
+
(`durationPlugin`).
|
|
28
|
+
|
|
29
|
+
`reporters` selects which reporters run, by id. `"table"`, `"json"`,
|
|
30
|
+
`"markdown"` and `"csv"` resolve to the reporters this CLI ships, with their
|
|
31
|
+
default options; any other id must name a reporter listed in `plugins`.
|
|
32
|
+
|
|
33
|
+
Listing a reporter in `plugins` **configures** it, it does not select it. That
|
|
34
|
+
separation is what lets `--reporter csv` turn on a reporter the config file has
|
|
35
|
+
already configured, without naming it twice.
|
|
36
|
+
|
|
37
|
+
Every option resolves in the same order: **command-line flag, then config file,
|
|
38
|
+
then default**.
|
|
39
|
+
|
|
40
|
+
## Configuring a built-in reporter
|
|
41
|
+
|
|
42
|
+
Construct it yourself in `plugins` and keep selecting it by name — the
|
|
43
|
+
configured instance wins over the built-in of the same id:
|
|
44
|
+
|
|
45
|
+
```ts
|
|
46
|
+
export default defineConfig({
|
|
47
|
+
plugins: [jsonReporter({ output: "artifacts/bench.json" })],
|
|
48
|
+
reporters: ["json"],
|
|
49
|
+
});
|
|
50
|
+
```
|
package/bin/measure.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
|
|
5
|
+
import packageJson from "../package.json" with { type: "json" };
|
|
6
|
+
|
|
7
|
+
const entryPath = packageJson.exports["./cli"];
|
|
8
|
+
|
|
9
|
+
if (path.extname(entryPath) === ".ts") {
|
|
10
|
+
await import("tsx/esm");
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const entryUrl = new URL(entryPath, new URL("../", import.meta.url));
|
|
14
|
+
|
|
15
|
+
await import(entryUrl.href);
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { BenchPlugin } from "@ac-bench/core/plugin";
|
|
2
|
+
import { BenchConditionOptions } from "@ac-bench/core/runner";
|
|
3
|
+
import { MeasurementPlan } from "@ac-bench/lib";
|
|
4
|
+
import { BenchMode } from "./mode.js";
|
|
5
|
+
export interface BenchConfig {
|
|
6
|
+
/** Glob patterns for bench files. Default: `["**\/*.bench.ts"]` */
|
|
7
|
+
include?: string[];
|
|
8
|
+
/** Glob patterns to exclude. Default: `["node_modules/**"]` */
|
|
9
|
+
exclude?: string[];
|
|
10
|
+
/**
|
|
11
|
+
* Ids of the reporters to run, in output order. Default: `["table"]`. Also
|
|
12
|
+
* settable with `--reporter`.
|
|
13
|
+
*
|
|
14
|
+
* `"table"`, `"json"`, `"markdown"` and `"csv"` resolve to the reporters this
|
|
15
|
+
* CLI ships, with their default options; any other id must name a reporter
|
|
16
|
+
* plugin listed in `plugins`. Listing a reporter in `plugins` configures it,
|
|
17
|
+
* it does not select it — that way `--reporter csv` can turn on a reporter
|
|
18
|
+
* this file has already configured.
|
|
19
|
+
*/
|
|
20
|
+
reporters?: string[];
|
|
21
|
+
/** Default output base name (without extension) for file reporters. */
|
|
22
|
+
output?: string;
|
|
23
|
+
/**
|
|
24
|
+
* Default sampling options applied to all conditions (overridden per
|
|
25
|
+
* condition).
|
|
26
|
+
*/
|
|
27
|
+
sampling?: Omit<BenchConditionOptions, "spawnBaselines">;
|
|
28
|
+
/**
|
|
29
|
+
* Measure and reporter plugins available to this run.
|
|
30
|
+
*
|
|
31
|
+
* Import each package directly and list what it exports: a package that takes
|
|
32
|
+
* options default-exports a factory (`jsonReporter({ output })`), one that
|
|
33
|
+
* does not default-exports the plugin (`durationPlugin`). The CLI never
|
|
34
|
+
* imports a measure package itself.
|
|
35
|
+
*/
|
|
36
|
+
plugins?: readonly BenchPlugin[];
|
|
37
|
+
/**
|
|
38
|
+
* Modules imported inside every child before the bench file, in order.
|
|
39
|
+
*
|
|
40
|
+
* `beforeAll`/`afterAll` are condition-scoped and nothing wider exists, so
|
|
41
|
+
* this is the only per-child setup mechanism. Its cost is paid once per fork
|
|
42
|
+
* unit rather than once per run — which is what asking for it means when no
|
|
43
|
+
* state crosses a process boundary.
|
|
44
|
+
*/
|
|
45
|
+
setupFiles?: string[];
|
|
46
|
+
/**
|
|
47
|
+
* Time without any message from a child before it is considered stuck.
|
|
48
|
+
* Default 30_000. Reset by every message it sends.
|
|
49
|
+
*/
|
|
50
|
+
heartbeatTimeoutMs?: number;
|
|
51
|
+
/** Wall-clock cap on one fork unit. Default `0`, meaning no cap. */
|
|
52
|
+
conditionTimeoutMs?: number;
|
|
53
|
+
/**
|
|
54
|
+
* Wall-clock cap on the whole run, retries and cooldown included. Default
|
|
55
|
+
* `0`, meaning no cap. Also settable with `--max-total-time`.
|
|
56
|
+
*/
|
|
57
|
+
maxTotalTimeMs?: number;
|
|
58
|
+
/**
|
|
59
|
+
* How much rigour to apply. Default `"standard"`. Also settable with
|
|
60
|
+
* `--mode`, which is the intended way in.
|
|
61
|
+
*/
|
|
62
|
+
mode?: BenchMode;
|
|
63
|
+
/**
|
|
64
|
+
* Pins individual measurement-plan parameters, overriding what `mode`
|
|
65
|
+
* derives.
|
|
66
|
+
*
|
|
67
|
+
* For a run that must reproduce an exact plan rather than express an intent —
|
|
68
|
+
* a published comparison, or a CI job that has to stay comparable with last
|
|
69
|
+
* month's. Everyone else should use `mode`.
|
|
70
|
+
*/
|
|
71
|
+
measurementPlan?: Partial<MeasurementPlan>;
|
|
72
|
+
/**
|
|
73
|
+
* Seeds replicate and case ordering. Default: derived from the clock, so each
|
|
74
|
+
* run randomises differently. Pin it to reproduce a run exactly.
|
|
75
|
+
*/
|
|
76
|
+
orderSeed?: number;
|
|
77
|
+
}
|
|
78
|
+
/** Returns a typed config object. Use in `bench.config.ts`. */
|
|
79
|
+
export declare function defineConfig(config: BenchConfig): BenchConfig;
|
|
80
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,MAAM,WAAW,WAAW;IAC3B,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;;;;OASG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,uEAAuE;IACvE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,QAAQ,CAAC,EAAE,IAAI,CAAC,qBAAqB,EAAE,gBAAgB,CAAC,CAAC;IACzD;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IACjC;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,oEAAoE;IACpE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IAC3C;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,+DAA+D;AAC/D,wBAAgB,YAAY,CAAC,MAAM,EAAE,WAAW,GAAG,WAAW,CAE7D"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAgFA,+DAA+D;AAC/D,MAAM,UAAU,YAAY,CAAC,MAAmB;IAC/C,OAAO,MAAM,CAAC;AACf,CAAC"}
|
package/dist/main.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":""}
|
package/dist/main.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Measure CLI entry point.
|
|
3
|
+
*
|
|
4
|
+
* Invoked via `bin/measure.js`, which registers tsx before importing this
|
|
5
|
+
* module.
|
|
6
|
+
*/
|
|
7
|
+
import { Command, InvalidArgumentError } from "commander";
|
|
8
|
+
import packageJson from "../package.json" with { type: "json" };
|
|
9
|
+
import { benchModeSchema } from "./mode.js";
|
|
10
|
+
import { loadConfig, runAll } from "./runner.js";
|
|
11
|
+
const program = new Command("measure")
|
|
12
|
+
.description("Discover and run TypeScript benchmark, latency, and timing files")
|
|
13
|
+
.version(packageJson.version)
|
|
14
|
+
.option("-c, --config <path>", "Path to bench config file")
|
|
15
|
+
.option("--include <pattern>", "Glob pattern for bench files (repeatable)", collect, [])
|
|
16
|
+
.option("--exclude <pattern>", "Glob pattern to exclude (repeatable)", collect, [])
|
|
17
|
+
.option("--filter <pattern>", "Only run conditions/cases whose title/name contains this substring (repeatable)", collect, [])
|
|
18
|
+
.option("--list", "List discovered conditions and cases without running them")
|
|
19
|
+
.option("--dry-run", "Alias for --list")
|
|
20
|
+
.option("--reporter <type>", "Reporter to use: table, json, markdown, csv (repeatable)", collect, [])
|
|
21
|
+
.option("--output <path>", "Output base name for file reporters")
|
|
22
|
+
.option("--mode <mode>", "How much rigour to apply: quick, standard or rigorous", parseMode)
|
|
23
|
+
.option("--order-seed <n>", "Reproduce a run's replicate and case ordering exactly", parseIntArg)
|
|
24
|
+
.option("--max-total-time <ms>", "Stop the run after this many milliseconds, retries and cooldown included", parseIntArg)
|
|
25
|
+
.option("--emit-samples", "Attach every case's raw samples; they are copied across the process boundary")
|
|
26
|
+
.option("--verbose", "Show every benchmark child's output as it happens, and full stack traces");
|
|
27
|
+
program.action(async () => {
|
|
28
|
+
const opts = program.opts();
|
|
29
|
+
const config = opts.config ? await loadConfig(opts.config) : {};
|
|
30
|
+
const cli = {
|
|
31
|
+
...(opts.include.length > 0 ? { include: opts.include } : {}),
|
|
32
|
+
...(opts.exclude.length > 0 ? { exclude: opts.exclude } : {}),
|
|
33
|
+
...(opts.filter.length > 0 ? { filter: opts.filter } : {}),
|
|
34
|
+
...(opts.reporter.length > 0 ? { reporters: opts.reporter } : {}),
|
|
35
|
+
...(opts.output !== undefined ? { output: opts.output } : {}),
|
|
36
|
+
...(opts.list || opts.dryRun ? { list: true } : {}),
|
|
37
|
+
...(opts.mode !== undefined ? { mode: opts.mode } : {}),
|
|
38
|
+
...(opts.orderSeed !== undefined ? { orderSeed: opts.orderSeed } : {}),
|
|
39
|
+
...(opts.maxTotalTime !== undefined
|
|
40
|
+
? { maxTotalTime: opts.maxTotalTime }
|
|
41
|
+
: {}),
|
|
42
|
+
...(opts.emitSamples ? { emitSamples: true } : {}),
|
|
43
|
+
...(opts.verbose ? { verbose: true } : {}),
|
|
44
|
+
};
|
|
45
|
+
const controller = new AbortController();
|
|
46
|
+
process.once("SIGINT", () => {
|
|
47
|
+
process.stderr.write("\nmeasure: interrupted, finishing in-flight condition…\n");
|
|
48
|
+
controller.abort();
|
|
49
|
+
});
|
|
50
|
+
await runAll(config, cli, controller.signal);
|
|
51
|
+
if (controller.signal.aborted) {
|
|
52
|
+
process.exitCode = 130;
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
await program.parseAsync(process.argv);
|
|
56
|
+
function collect(value, previous) {
|
|
57
|
+
return [...previous, value];
|
|
58
|
+
}
|
|
59
|
+
function parseIntArg(value) {
|
|
60
|
+
const parsed = Number.parseInt(value, 10);
|
|
61
|
+
if (Number.isNaN(parsed)) {
|
|
62
|
+
// Commander turns this one into a usage message; a plain Error would reach
|
|
63
|
+
// the user as a stack trace over a typo.
|
|
64
|
+
throw new InvalidArgumentError(`Expected an integer, got: ${value}`);
|
|
65
|
+
}
|
|
66
|
+
return parsed;
|
|
67
|
+
}
|
|
68
|
+
function parseMode(value) {
|
|
69
|
+
return benchModeSchema.parse(value);
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=main.js.map
|
package/dist/main.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AAE1D,OAAO,WAAW,MAAM,iBAAiB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AAChE,OAAO,EAAa,eAAe,EAAE,MAAM,WAAW,CAAC;AAEvD,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAEjD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,SAAS,CAAC;KACpC,WAAW,CACX,kEAAkE,CAClE;KACA,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;KAC5B,MAAM,CAAC,qBAAqB,EAAE,2BAA2B,CAAC;KAC1D,MAAM,CACN,qBAAqB,EACrB,2CAA2C,EAC3C,OAAO,EACP,EAAc,CACd;KACA,MAAM,CACN,qBAAqB,EACrB,sCAAsC,EACtC,OAAO,EACP,EAAc,CACd;KACA,MAAM,CACN,oBAAoB,EACpB,iFAAiF,EACjF,OAAO,EACP,EAAc,CACd;KACA,MAAM,CAAC,QAAQ,EAAE,2DAA2D,CAAC;KAC7E,MAAM,CAAC,WAAW,EAAE,kBAAkB,CAAC;KACvC,MAAM,CACN,mBAAmB,EACnB,0DAA0D,EAC1D,OAAO,EACP,EAAc,CACd;KACA,MAAM,CAAC,iBAAiB,EAAE,qCAAqC,CAAC;KAChE,MAAM,CACN,eAAe,EACf,uDAAuD,EACvD,SAAS,CACT;KACA,MAAM,CACN,kBAAkB,EAClB,uDAAuD,EACvD,WAAW,CACX;KACA,MAAM,CACN,uBAAuB,EACvB,0EAA0E,EAC1E,WAAW,CACX;KACA,MAAM,CACN,gBAAgB,EAChB,8EAA8E,CAC9E;KACA,MAAM,CACN,WAAW,EACX,0EAA0E,CAC1E,CAAC;AAEH,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE;IACzB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAcrB,CAAC;IAEL,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEhE,MAAM,GAAG,GAAiB;QACzB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,GAAG,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;YAClC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE;YACrC,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1C,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE;QAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,0DAA0D,CAC1D,CAAC;QACF,UAAU,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAE7C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC;IACxB,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAEvC,SAAS,OAAO,CAAC,KAAa,EAAE,QAAkB;IACjD,OAAO,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAE1C,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,2EAA2E;QAC3E,yCAAyC;QACzC,MAAM,IAAI,oBAAoB,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC/B,OAAO,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACrC,CAAC"}
|
package/dist/mode.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { MeasurementPlan } from "@ac-bench/lib";
|
|
2
|
+
import { z } from "zod/mini";
|
|
3
|
+
/**
|
|
4
|
+
* What the user wants to know, not how to find it out.
|
|
5
|
+
*
|
|
6
|
+
* None of the measurement plan's parameters is a flag. A user asking for a
|
|
7
|
+
* trustworthy comparison should not also have to know how many processes that
|
|
8
|
+
* takes on their machine — the same move the sampler already makes for sample
|
|
9
|
+
* counts with `relativeError: "auto"`.
|
|
10
|
+
*/
|
|
11
|
+
export declare const benchModeSchema: z.ZodMiniEnum<{
|
|
12
|
+
quick: "quick";
|
|
13
|
+
rigorous: "rigorous";
|
|
14
|
+
standard: "standard";
|
|
15
|
+
}>;
|
|
16
|
+
export type BenchMode = z.infer<typeof benchModeSchema>;
|
|
17
|
+
export declare const DEFAULT_BENCH_MODE: BenchMode;
|
|
18
|
+
/**
|
|
19
|
+
* Derives the measurement plan a mode stands for.
|
|
20
|
+
*
|
|
21
|
+
* @param mode Defaults to {@link DEFAULT_BENCH_MODE}.
|
|
22
|
+
* @param overrides Mechanism the config file may pin, for a run that needs an
|
|
23
|
+
* exact plan rather than an intent.
|
|
24
|
+
* @returns The plan, frozen so a later stage cannot quietly retune it.
|
|
25
|
+
*/
|
|
26
|
+
export declare function resolveMeasurementPlan(mode?: BenchMode, overrides?: Partial<MeasurementPlan>): MeasurementPlan;
|
|
27
|
+
//# sourceMappingURL=mode.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mode.d.ts","sourceRoot":"","sources":["../src/mode.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,CAAC,EAAE,MAAM,UAAU,CAAC;AAE7B;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe;;;;EAA4C,CAAC;AACzE,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAqCxD,eAAO,MAAM,kBAAkB,EAAE,SAAsB,CAAC;AAExD;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CACrC,IAAI,GAAE,SAA8B,EACpC,SAAS,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAClC,eAAe,CAejB"}
|
package/dist/mode.js
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { z } from "zod/mini";
|
|
2
|
+
/**
|
|
3
|
+
* What the user wants to know, not how to find it out.
|
|
4
|
+
*
|
|
5
|
+
* None of the measurement plan's parameters is a flag. A user asking for a
|
|
6
|
+
* trustworthy comparison should not also have to know how many processes that
|
|
7
|
+
* takes on their machine — the same move the sampler already makes for sample
|
|
8
|
+
* counts with `relativeError: "auto"`.
|
|
9
|
+
*/
|
|
10
|
+
export const benchModeSchema = z.enum(["quick", "standard", "rigorous"]);
|
|
11
|
+
const PROFILES_ = {
|
|
12
|
+
// One process, no replication: the process-level offset is unmeasured and
|
|
13
|
+
// the answer is a rough one. Says so by being called `quick`.
|
|
14
|
+
quick: {
|
|
15
|
+
replicates: 1,
|
|
16
|
+
maxReplicates: 1,
|
|
17
|
+
isolatedRuns: 0,
|
|
18
|
+
rounds: 1,
|
|
19
|
+
cooldown: false,
|
|
20
|
+
retryOnInstability: 0,
|
|
21
|
+
},
|
|
22
|
+
// Two processes is the cheapest plan that can see the process lottery at
|
|
23
|
+
// all: with one, the only interval available is the within-process one,
|
|
24
|
+
// which is exactly the number that says nothing about it. `standard` starts
|
|
25
|
+
// there and buys more only for a comparison those intervals leave open.
|
|
26
|
+
standard: {
|
|
27
|
+
replicates: 2,
|
|
28
|
+
maxReplicates: 5,
|
|
29
|
+
isolatedRuns: 0,
|
|
30
|
+
rounds: 16,
|
|
31
|
+
cooldown: false,
|
|
32
|
+
retryOnInstability: 0,
|
|
33
|
+
},
|
|
34
|
+
// Everything on: replicates for the process lottery, isolated runs for
|
|
35
|
+
// order sensitivity, and a machine that has been made to sit still first.
|
|
36
|
+
rigorous: {
|
|
37
|
+
replicates: 5,
|
|
38
|
+
maxReplicates: 9,
|
|
39
|
+
isolatedRuns: 3,
|
|
40
|
+
rounds: 16,
|
|
41
|
+
cooldown: true,
|
|
42
|
+
retryOnInstability: 1,
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
export const DEFAULT_BENCH_MODE = "standard";
|
|
46
|
+
/**
|
|
47
|
+
* Derives the measurement plan a mode stands for.
|
|
48
|
+
*
|
|
49
|
+
* @param mode Defaults to {@link DEFAULT_BENCH_MODE}.
|
|
50
|
+
* @param overrides Mechanism the config file may pin, for a run that needs an
|
|
51
|
+
* exact plan rather than an intent.
|
|
52
|
+
* @returns The plan, frozen so a later stage cannot quietly retune it.
|
|
53
|
+
*/
|
|
54
|
+
export function resolveMeasurementPlan(mode = DEFAULT_BENCH_MODE, overrides) {
|
|
55
|
+
const profile = PROFILES_[mode];
|
|
56
|
+
return {
|
|
57
|
+
replicates: overrides?.replicates ?? profile.replicates,
|
|
58
|
+
maxReplicates: Math.max(overrides?.maxReplicates ?? profile.maxReplicates, overrides?.replicates ?? profile.replicates),
|
|
59
|
+
isolatedRuns: overrides?.isolatedRuns ?? profile.isolatedRuns,
|
|
60
|
+
rounds: overrides?.rounds ?? profile.rounds,
|
|
61
|
+
cooldown: overrides?.cooldown ?? profile.cooldown,
|
|
62
|
+
retryOnInstability: overrides?.retryOnInstability ?? profile.retryOnInstability,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=mode.js.map
|
package/dist/mode.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mode.js","sourceRoot":"","sources":["../src/mode.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,UAAU,CAAC;AAE7B;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;AAGzE,MAAM,SAAS,GAAiD;IAC/D,0EAA0E;IAC1E,8DAA8D;IAC9D,KAAK,EAAE;QACN,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,CAAC;QAChB,YAAY,EAAE,CAAC;QACf,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,KAAK;QACf,kBAAkB,EAAE,CAAC;KACrB;IACD,yEAAyE;IACzE,wEAAwE;IACxE,4EAA4E;IAC5E,wEAAwE;IACxE,QAAQ,EAAE;QACT,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,CAAC;QAChB,YAAY,EAAE,CAAC;QACf,MAAM,EAAE,EAAE;QACV,QAAQ,EAAE,KAAK;QACf,kBAAkB,EAAE,CAAC;KACrB;IACD,uEAAuE;IACvE,0EAA0E;IAC1E,QAAQ,EAAE;QACT,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,CAAC;QAChB,YAAY,EAAE,CAAC;QACf,MAAM,EAAE,EAAE;QACV,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,CAAC;KACrB;CACD,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAc,UAAU,CAAC;AAExD;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CACrC,IAAI,GAAc,kBAAkB,EACpC,SAAoC;IAEpC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEhC,OAAO;QACN,UAAU,EAAE,SAAS,EAAE,UAAU,IAAI,OAAO,CAAC,UAAU;QACvD,aAAa,EAAE,IAAI,CAAC,GAAG,CACtB,SAAS,EAAE,aAAa,IAAI,OAAO,CAAC,aAAa,EACjD,SAAS,EAAE,UAAU,IAAI,OAAO,CAAC,UAAU,CAC3C;QACD,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,OAAO,CAAC,YAAY;QAC7D,MAAM,EAAE,SAAS,EAAE,MAAM,IAAI,OAAO,CAAC,MAAM;QAC3C,QAAQ,EAAE,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,QAAQ;QACjD,kBAAkB,EACjB,SAAS,EAAE,kBAAkB,IAAI,OAAO,CAAC,kBAAkB;KAC5D,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { ReporterPlugin } from "@ac-bench/core/plugin";
|
|
2
|
+
/**
|
|
3
|
+
* Resolves reporter ids into the plugins that will run.
|
|
4
|
+
*
|
|
5
|
+
* A configured instance wins over the built-in of the same id, so a config file
|
|
6
|
+
* that lists `jsonReporter({ output: … })` in `plugins` gets its options
|
|
7
|
+
* honoured while still selecting the reporter by name.
|
|
8
|
+
*
|
|
9
|
+
* @throws {Error} An id names neither a configured plugin nor a built-in, or
|
|
10
|
+
* two configured plugins claim the same id.
|
|
11
|
+
*/
|
|
12
|
+
export declare function resolveReporterPlugins(ids: readonly string[], configured: readonly ReporterPlugin[]): ReporterPlugin[];
|
|
13
|
+
//# sourceMappingURL=resolve-reporter-plugins.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resolve-reporter-plugins.d.ts","sourceRoot":"","sources":["../src/resolve-reporter-plugins.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAmB5D;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACrC,GAAG,EAAE,SAAS,MAAM,EAAE,EACtB,UAAU,EAAE,SAAS,cAAc,EAAE,GACnC,cAAc,EAAE,CA0BlB"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import csvReporter from "@ac-bench/reporter-csv";
|
|
2
|
+
import jsonReporter from "@ac-bench/reporter-json";
|
|
3
|
+
import markdownReporter from "@ac-bench/reporter-markdown";
|
|
4
|
+
import tableReporter from "@ac-bench/reporter-table";
|
|
5
|
+
/**
|
|
6
|
+
* The reporters this CLI ships, each with its own default options.
|
|
7
|
+
*
|
|
8
|
+
* Uniformly a thunk because a reporter package default-exports a factory only
|
|
9
|
+
* when it has options: `table` has none and exports the plugin itself.
|
|
10
|
+
*/
|
|
11
|
+
const BUILT_INS_ = {
|
|
12
|
+
table: () => tableReporter,
|
|
13
|
+
json: () => jsonReporter(),
|
|
14
|
+
markdown: () => markdownReporter(),
|
|
15
|
+
csv: () => csvReporter(),
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Resolves reporter ids into the plugins that will run.
|
|
19
|
+
*
|
|
20
|
+
* A configured instance wins over the built-in of the same id, so a config file
|
|
21
|
+
* that lists `jsonReporter({ output: … })` in `plugins` gets its options
|
|
22
|
+
* honoured while still selecting the reporter by name.
|
|
23
|
+
*
|
|
24
|
+
* @throws {Error} An id names neither a configured plugin nor a built-in, or
|
|
25
|
+
* two configured plugins claim the same id.
|
|
26
|
+
*/
|
|
27
|
+
export function resolveReporterPlugins(ids, configured) {
|
|
28
|
+
const byId = new Map();
|
|
29
|
+
for (const plugin of configured) {
|
|
30
|
+
if (byId.has(plugin.id)) {
|
|
31
|
+
throw new Error(`Duplicate reporter plugin id: ${plugin.id}`);
|
|
32
|
+
}
|
|
33
|
+
byId.set(plugin.id, plugin);
|
|
34
|
+
}
|
|
35
|
+
return ids.map((id) => {
|
|
36
|
+
const configuredPlugin = byId.get(id);
|
|
37
|
+
if (configuredPlugin) {
|
|
38
|
+
return configuredPlugin;
|
|
39
|
+
}
|
|
40
|
+
const builtIn = BUILT_INS_[id];
|
|
41
|
+
if (builtIn) {
|
|
42
|
+
return builtIn();
|
|
43
|
+
}
|
|
44
|
+
const known = [...new Set([...Object.keys(BUILT_INS_), ...byId.keys()])];
|
|
45
|
+
throw new Error(`Unknown reporter type: ${id}. Known reporters: ${known.join(", ")}`);
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=resolve-reporter-plugins.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resolve-reporter-plugins.js","sourceRoot":"","sources":["../src/resolve-reporter-plugins.ts"],"names":[],"mappings":"AACA,OAAO,WAAW,MAAM,wBAAwB,CAAC;AACjD,OAAO,YAAY,MAAM,yBAAyB,CAAC;AACnD,OAAO,gBAAgB,MAAM,6BAA6B,CAAC;AAC3D,OAAO,aAAa,MAAM,0BAA0B,CAAC;AAErD;;;;;GAKG;AACH,MAAM,UAAU,GAAmD;IAClE,KAAK,EAAE,GAAG,EAAE,CAAC,aAAa;IAC1B,IAAI,EAAE,GAAG,EAAE,CAAC,YAAY,EAAE;IAC1B,QAAQ,EAAE,GAAG,EAAE,CAAC,gBAAgB,EAAE;IAClC,GAAG,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE;CACxB,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CACrC,GAAsB,EACtB,UAAqC;IAErC,MAAM,IAAI,GAAG,IAAI,GAAG,EAA0B,CAAC;IAE/C,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACrB,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,IAAI,gBAAgB,EAAE,CAAC;YACtB,OAAO,gBAAgB,CAAC;QACzB,CAAC;QAED,MAAM,OAAO,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,OAAO,EAAE,CAAC;YACb,OAAO,OAAO,EAAE,CAAC;QAClB,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACzE,MAAM,IAAI,KAAK,CACd,0BAA0B,EAAE,sBAAsB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpE,CAAC;IACH,CAAC,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/runner.d.ts
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { BenchConfig } from "./config.js";
|
|
2
|
+
import { BenchMode } from "./mode.js";
|
|
3
|
+
/** Load a config file by path (must default-export a BenchConfig). */
|
|
4
|
+
export declare function loadConfig(configPath: string): Promise<BenchConfig>;
|
|
5
|
+
/** CLI overrides from Commander flags. */
|
|
6
|
+
export interface CliOverrides {
|
|
7
|
+
include?: string[];
|
|
8
|
+
exclude?: string[];
|
|
9
|
+
/**
|
|
10
|
+
* Case-insensitive substring patterns matched against a condition's title or
|
|
11
|
+
* a case's name.
|
|
12
|
+
*/
|
|
13
|
+
filter?: string[];
|
|
14
|
+
/** Ids of the reporters to run, overriding the config file's list. */
|
|
15
|
+
reporters?: string[];
|
|
16
|
+
/** Output base name (without extension) for file reporters. */
|
|
17
|
+
output?: string;
|
|
18
|
+
/** Print discovered conditions/cases without running them. */
|
|
19
|
+
list?: boolean;
|
|
20
|
+
/**
|
|
21
|
+
* How much rigour the run should apply.
|
|
22
|
+
*
|
|
23
|
+
* The only rigour control on the CLI: replicate counts, isolated runs,
|
|
24
|
+
* cooldown gating and retries are all mechanism, and live in the config file
|
|
25
|
+
* for the rare run that needs to pin them exactly.
|
|
26
|
+
*/
|
|
27
|
+
mode?: BenchMode;
|
|
28
|
+
/** Reproduces one run's replicate and case orderings exactly. */
|
|
29
|
+
orderSeed?: number;
|
|
30
|
+
/** Wall-clock cap on the whole run, in milliseconds. */
|
|
31
|
+
maxTotalTime?: number;
|
|
32
|
+
/** Attach each case's raw samples to its scope. */
|
|
33
|
+
emitSamples?: boolean;
|
|
34
|
+
/** Show every child's output as it happens, and full stack traces. */
|
|
35
|
+
verbose?: boolean;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Resolves flags and config into run parameters, then hands them to the
|
|
39
|
+
* programmatic API.
|
|
40
|
+
*
|
|
41
|
+
* Exits with status 1 when the include patterns match nothing: a run that
|
|
42
|
+
* measured nothing is a mistake to report, not an empty result to hand back.
|
|
43
|
+
*/
|
|
44
|
+
export declare function runAll(config: BenchConfig, cli: CliOverrides, signal: AbortSignal): Promise<void>;
|
|
45
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,SAAS,EAA0B,MAAM,WAAW,CAAC;AAU9D,sEAAsE;AACtE,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAIzE;AAED,0CAA0C;AAC1C,MAAM,WAAW,YAAY;IAC5B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,sEAAsE;IACtE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,sEAAsE;IACtE,OAAO,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,MAAM,CAC3B,MAAM,EAAE,WAAW,EACnB,GAAG,EAAE,YAAY,EACjB,MAAM,EAAE,WAAW,GACjB,OAAO,CAAC,IAAI,CAAC,CA4Df"}
|
package/dist/runner.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { pathToFileURL } from "node:url";
|
|
2
|
+
import { partitionPlugins } from "@ac-bench/core/plugin";
|
|
3
|
+
import { runBench } from "@ac-bench/lib";
|
|
4
|
+
import { formatError } from "@ac-kit/core";
|
|
5
|
+
import { walkPaths } from "@ac-kit/node";
|
|
6
|
+
import { resolveMeasurementPlan } from "./mode.js";
|
|
7
|
+
import { resolveReporterPlugins } from "./resolve-reporter-plugins.js";
|
|
8
|
+
/** Applied when neither the command line nor the config file names any. */
|
|
9
|
+
const DEFAULT_INCLUDE = ["**/*.bench.ts"];
|
|
10
|
+
// Matches the directory itself, at any depth, so the walk never enters it.
|
|
11
|
+
const DEFAULT_EXCLUDE = ["**/node_modules"];
|
|
12
|
+
const DEFAULT_REPORTERS = ["table"];
|
|
13
|
+
const DEFAULT_OUTPUT = "bench-results";
|
|
14
|
+
/** Load a config file by path (must default-export a BenchConfig). */
|
|
15
|
+
export async function loadConfig(configPath) {
|
|
16
|
+
const url = pathToFileURL(configPath).href;
|
|
17
|
+
const mod = (await import(url));
|
|
18
|
+
return mod.default ?? {};
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Resolves flags and config into run parameters, then hands them to the
|
|
22
|
+
* programmatic API.
|
|
23
|
+
*
|
|
24
|
+
* Exits with status 1 when the include patterns match nothing: a run that
|
|
25
|
+
* measured nothing is a mistake to report, not an empty result to hand back.
|
|
26
|
+
*/
|
|
27
|
+
export async function runAll(config, cli, signal) {
|
|
28
|
+
const include = cli.include ?? config.include ?? DEFAULT_INCLUDE;
|
|
29
|
+
const exclude = cli.exclude ?? config.exclude ?? DEFAULT_EXCLUDE;
|
|
30
|
+
if (cli.verbose === true) {
|
|
31
|
+
// Set once rather than threaded through every `formatError` call site: the
|
|
32
|
+
// switch is run-wide, and nothing in a bench run wants the short form while
|
|
33
|
+
// the user is asking to see everything.
|
|
34
|
+
formatError.defaultOptions.stackTrace = true;
|
|
35
|
+
}
|
|
36
|
+
const files = await Array.fromAsync(walkPaths({
|
|
37
|
+
include: include.length > 0 ? include : DEFAULT_INCLUDE,
|
|
38
|
+
exclude: exclude.length > 0 ? exclude : DEFAULT_EXCLUDE,
|
|
39
|
+
root: process.cwd(),
|
|
40
|
+
descend: true,
|
|
41
|
+
signal,
|
|
42
|
+
}), (walked) => walked.path);
|
|
43
|
+
if (files.length === 0) {
|
|
44
|
+
process.stderr.write("measure: no files found. Use --include to specify patterns.\n");
|
|
45
|
+
process.exit(1);
|
|
46
|
+
}
|
|
47
|
+
const { measures, reporters: configuredReporters } = partitionPlugins(config.plugins ?? []);
|
|
48
|
+
await runBench({
|
|
49
|
+
files,
|
|
50
|
+
measures,
|
|
51
|
+
reporters: resolveReporterPlugins(cli.reporters ?? config.reporters ?? DEFAULT_REPORTERS, configuredReporters),
|
|
52
|
+
output: cli.output ?? config.output ?? DEFAULT_OUTPUT,
|
|
53
|
+
plan: resolveMeasurementPlan(cli.mode ?? config.mode, config.measurementPlan),
|
|
54
|
+
orderSeed: cli.orderSeed ?? config.orderSeed ?? Date.now() | 0,
|
|
55
|
+
signal,
|
|
56
|
+
maxTotalTimeMs: cli.maxTotalTime ?? config.maxTotalTimeMs ?? 0,
|
|
57
|
+
...(cli.filter ? { filter: cli.filter } : {}),
|
|
58
|
+
...(cli.list === true ? { list: true } : {}),
|
|
59
|
+
...(cli.emitSamples ? { emitSamples: true } : {}),
|
|
60
|
+
...(cli.verbose ? { verbose: true } : {}),
|
|
61
|
+
...(config.setupFiles ? { setupFiles: config.setupFiles } : {}),
|
|
62
|
+
...(config.heartbeatTimeoutMs !== undefined
|
|
63
|
+
? { heartbeatTimeoutMs: config.heartbeatTimeoutMs }
|
|
64
|
+
: {}),
|
|
65
|
+
...(config.conditionTimeoutMs !== undefined
|
|
66
|
+
? { conditionTimeoutMs: config.conditionTimeoutMs }
|
|
67
|
+
: {}),
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAa,sBAAsB,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAEvE,2EAA2E;AAC3E,MAAM,eAAe,GAAG,CAAC,eAAe,CAAC,CAAC;AAC1C,2EAA2E;AAC3E,MAAM,eAAe,GAAG,CAAC,iBAAiB,CAAC,CAAC;AAC5C,MAAM,iBAAiB,GAAG,CAAC,OAAO,CAAC,CAAC;AACpC,MAAM,cAAc,GAAG,eAAe,CAAC;AAEvC,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,UAAkB;IAClD,MAAM,GAAG,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC;IAC3C,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAA8B,CAAC;IAC7D,OAAO,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;AAC1B,CAAC;AAmCD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC3B,MAAmB,EACnB,GAAiB,EACjB,MAAmB;IAEnB,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC;IACjE,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC;IAEjE,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC1B,2EAA2E;QAC3E,4EAA4E;QAC5E,wCAAwC;QACxC,WAAW,CAAC,cAAc,CAAC,UAAU,GAAG,IAAI,CAAC;IAC9C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,CAClC,SAAS,CAAC;QACT,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;QACvD,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;QACvD,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;QACnB,OAAO,EAAE,IAAI;QACb,MAAM;KACN,CAAC,EACF,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CACvB,CAAC;IAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,+DAA+D,CAC/D,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CACpE,MAAM,CAAC,OAAO,IAAI,EAAE,CACpB,CAAC;IAEF,MAAM,QAAQ,CAAC;QACd,KAAK;QACL,QAAQ;QACR,SAAS,EAAE,sBAAsB,CAChC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,iBAAiB,EACtD,mBAAmB,CACnB;QACD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,cAAc;QACrD,IAAI,EAAE,sBAAsB,CAC3B,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EACvB,MAAM,CAAC,eAAe,CACtB;QACD,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC;QAC9D,MAAM;QACN,cAAc,EAAE,GAAG,CAAC,YAAY,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC;QAC9D,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,MAAM,CAAC,kBAAkB,KAAK,SAAS;YAC1C,CAAC,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,EAAE;YACnD,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,MAAM,CAAC,kBAAkB,KAAK,SAAS;YAC1C,CAAC,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,EAAE;YACnD,CAAC,CAAC,EAAE,CAAC;KACN,CAAC,CAAC;AACJ,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@ac-bench/cli",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Command-line runner for @ac-bench benchmark suites: argument parsing, config-file loading and run resolution.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"benchmark",
|
|
7
|
+
"cli",
|
|
8
|
+
"measurement",
|
|
9
|
+
"performance"
|
|
10
|
+
],
|
|
11
|
+
"homepage": "https://github.com/anthochamp/node-essentials/tree/main/bench/cli#readme",
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/anthochamp/node-essentials/issues"
|
|
14
|
+
},
|
|
15
|
+
"license": "BSD-3-Clause",
|
|
16
|
+
"author": "Anthony Champagne <dev@anthonychampagne.fr>",
|
|
17
|
+
"repository": {
|
|
18
|
+
"type": "git",
|
|
19
|
+
"url": "git+https://github.com/anthochamp/node-essentials.git",
|
|
20
|
+
"directory": "bench/cli"
|
|
21
|
+
},
|
|
22
|
+
"bin": {
|
|
23
|
+
"measure": "./bin/measure.js"
|
|
24
|
+
},
|
|
25
|
+
"files": [
|
|
26
|
+
"dist",
|
|
27
|
+
"bin"
|
|
28
|
+
],
|
|
29
|
+
"type": "module",
|
|
30
|
+
"sideEffects": false,
|
|
31
|
+
"main": "./dist/config.js",
|
|
32
|
+
"exports": {
|
|
33
|
+
".": "./dist/config.js",
|
|
34
|
+
"./cli": "./dist/main.js"
|
|
35
|
+
},
|
|
36
|
+
"publishConfig": {
|
|
37
|
+
"main": "./dist/config.js",
|
|
38
|
+
"exports": {
|
|
39
|
+
".": "./dist/config.js",
|
|
40
|
+
"./cli": "./dist/main.js"
|
|
41
|
+
},
|
|
42
|
+
"access": "public"
|
|
43
|
+
},
|
|
44
|
+
"scripts": {
|
|
45
|
+
"compile": "tsc -p ./tsconfig.build.json --outDir dist",
|
|
46
|
+
"compile:check": "tsc -noEmit -p ./tsconfig.json",
|
|
47
|
+
"clean": "rimraf dist",
|
|
48
|
+
"build": "yarn clean && yarn compile",
|
|
49
|
+
"test": "yarn run -T vitest run",
|
|
50
|
+
"prepack": "yarn build"
|
|
51
|
+
},
|
|
52
|
+
"dependencies": {
|
|
53
|
+
"@ac-bench/core": "^0.1.0",
|
|
54
|
+
"@ac-bench/lib": "^0.1.0",
|
|
55
|
+
"@ac-bench/reporter-csv": "^0.1.0",
|
|
56
|
+
"@ac-bench/reporter-json": "^0.1.0",
|
|
57
|
+
"@ac-bench/reporter-markdown": "^0.1.0",
|
|
58
|
+
"@ac-bench/reporter-table": "^0.1.0",
|
|
59
|
+
"@ac-kit/algo": "^0.1.0",
|
|
60
|
+
"@ac-kit/app-report": "^0.1.0",
|
|
61
|
+
"@ac-kit/app-system": "^0.1.0",
|
|
62
|
+
"@ac-kit/core": "^0.1.0",
|
|
63
|
+
"@ac-kit/node": "^0.1.0",
|
|
64
|
+
"commander": "^15.0.0",
|
|
65
|
+
"zod": "^4.5.4"
|
|
66
|
+
},
|
|
67
|
+
"devDependencies": {
|
|
68
|
+
"@ac-kit/tsconfig": "0.1.0",
|
|
69
|
+
"@ac-kit/typedoc-config": "0.1.0",
|
|
70
|
+
"@ac-kit/vitest-config": "0.1.0",
|
|
71
|
+
"@types/node": "26.5.1",
|
|
72
|
+
"@typescript/native": "typescript@7.0.2",
|
|
73
|
+
"rimraf": "6.1.3",
|
|
74
|
+
"tsx": "4.23.12",
|
|
75
|
+
"typedoc": "0.28.20",
|
|
76
|
+
"typescript": "@typescript/typescript6@6.0.2",
|
|
77
|
+
"vite": "8.3.0",
|
|
78
|
+
"vitest": "5.0.0"
|
|
79
|
+
},
|
|
80
|
+
"engines": {
|
|
81
|
+
"node": ">=24"
|
|
82
|
+
}
|
|
83
|
+
}
|