@autor3search/javascript 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +500 -0
- package/bin/autor3search-javascript.js +4 -0
- package/package.json +50 -0
- package/src/adapters/bench/index.js +31 -0
- package/src/adapters/bench/vitest.js +128 -0
- package/src/adapters/driver-child.js +72 -0
- package/src/adapters/driver-hooks.js +18 -0
- package/src/adapters/driver.js +192 -0
- package/src/adapters/gates/index.js +64 -0
- package/src/adapters/gates/lint.js +46 -0
- package/src/adapters/gates/test.js +27 -0
- package/src/adapters/gates/typecheck.js +98 -0
- package/src/adapters/gates/util.js +45 -0
- package/src/adapters/vitest-shim.js +51 -0
- package/src/bench/parse.js +120 -0
- package/src/bench/set.js +101 -0
- package/src/bench/stats.js +443 -0
- package/src/cli/cmd-baseline.js +136 -0
- package/src/cli/cmd-doctor.js +38 -0
- package/src/cli/cmd-eval.js +231 -0
- package/src/cli/cmd-init.js +136 -0
- package/src/cli/cmd-profile.js +38 -0
- package/src/cli/cmd-report.js +96 -0
- package/src/cli/cmd-status.js +68 -0
- package/src/cli/cmd-stop.js +98 -0
- package/src/cli/cmd-version.js +31 -0
- package/src/cli/context.js +98 -0
- package/src/cli/main.js +62 -0
- package/src/config.js +209 -0
- package/src/discover.js +239 -0
- package/src/doctor.js +282 -0
- package/src/duration.js +75 -0
- package/src/freeze.js +234 -0
- package/src/gitx.js +115 -0
- package/src/measure.js +127 -0
- package/src/pipeline.js +391 -0
- package/src/profile.js +100 -0
- package/src/results.js +124 -0
- package/src/runner.js +198 -0
- package/src/scope.js +92 -0
- package/src/state/index.js +312 -0
- package/src/state/lock.js +189 -0
- package/src/state/stop.js +56 -0
- package/src/verdict.js +214 -0
- package/templates/program.md +264 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Prints which build of the harness is running.
|
|
3
|
+
*
|
|
4
|
+
* A results.tsv row is only as reproducible as the harness that produced it,
|
|
5
|
+
* so this reports the package version and, when running from a checkout, the
|
|
6
|
+
* commit with a `dirty` marker.
|
|
7
|
+
*/
|
|
8
|
+
import { readFile } from 'node:fs/promises'
|
|
9
|
+
import { dirname, join } from 'node:path'
|
|
10
|
+
import { fileURLToPath } from 'node:url'
|
|
11
|
+
import { parseArgs } from 'node:util'
|
|
12
|
+
import * as gitx from '../gitx.js'
|
|
13
|
+
|
|
14
|
+
const HERE = dirname(fileURLToPath(import.meta.url))
|
|
15
|
+
const PACKAGE_ROOT = join(HERE, '..', '..')
|
|
16
|
+
|
|
17
|
+
export async function runVersion(args, io) {
|
|
18
|
+
parseArgs({ args, options: { C: { type: 'string', default: '.' } }, allowPositionals: false })
|
|
19
|
+
|
|
20
|
+
const pkg = JSON.parse(await readFile(join(PACKAGE_ROOT, 'package.json'), 'utf8'))
|
|
21
|
+
let provenance = 'not a git checkout'
|
|
22
|
+
try {
|
|
23
|
+
const commit = await gitx.headCommit(PACKAGE_ROOT)
|
|
24
|
+
const clean = await gitx.isClean(PACKAGE_ROOT)
|
|
25
|
+
provenance = `commit ${commit}${clean ? '' : ' (dirty)'}`
|
|
26
|
+
} catch {
|
|
27
|
+
// Installed from a registry rather than run from a checkout.
|
|
28
|
+
}
|
|
29
|
+
io.out.write(`autor3search-javascript ${pkg.version} (${provenance})\n`)
|
|
30
|
+
return 0
|
|
31
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared resolution for every command: which repository, which run.
|
|
3
|
+
*
|
|
4
|
+
* Nothing here calls process.chdir. `-C` is threaded through as a value
|
|
5
|
+
* instead, which is safer under concurrent invocations and lets the tests
|
|
6
|
+
* exercise commands without mutating global state.
|
|
7
|
+
*/
|
|
8
|
+
import { CONFIG_PATH, loadConfig } from '../config.js'
|
|
9
|
+
import * as gitx from '../gitx.js'
|
|
10
|
+
import { BRANCH_PREFIX, stateDir, validTag } from '../state/index.js'
|
|
11
|
+
import { join } from 'node:path'
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Rewrites single-dash multi-character flags into the long form parseArgs
|
|
15
|
+
* accepts.
|
|
16
|
+
*
|
|
17
|
+
* Node's parseArgs recognises a single dash ONLY for a single-character option
|
|
18
|
+
* name, so `-C` works but `-tag` fails with "Unknown option '-t'" and `-desc`
|
|
19
|
+
* with "Unknown option '-d'". This harness documents the single-dash spelling
|
|
20
|
+
* throughout — program.md tells the agent to run `eval -desc "..."` and
|
|
21
|
+
* `baseline -tag sep7` — so the tokens are normalised here rather than
|
|
22
|
+
* changing a contract an agent already follows.
|
|
23
|
+
*
|
|
24
|
+
* Only an exact whole-token match is rewritten, so a VALUE that happens to
|
|
25
|
+
* look like a flag (`-desc "-tag is confusing"`) is left alone.
|
|
26
|
+
*
|
|
27
|
+
* @param {string[]} args
|
|
28
|
+
* @param {string[]} names long option names to accept in single-dash form
|
|
29
|
+
* @returns {string[]}
|
|
30
|
+
*/
|
|
31
|
+
export function expandSingleDashFlags(args, names) {
|
|
32
|
+
const single = new Set(names.map((name) => `-${name}`))
|
|
33
|
+
let expectingValue = false
|
|
34
|
+
return args.map((token) => {
|
|
35
|
+
if (expectingValue) {
|
|
36
|
+
expectingValue = false
|
|
37
|
+
return token
|
|
38
|
+
}
|
|
39
|
+
if (single.has(token)) {
|
|
40
|
+
expectingValue = true
|
|
41
|
+
return `--${token.slice(1)}`
|
|
42
|
+
}
|
|
43
|
+
return token
|
|
44
|
+
})
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The repository root containing dir. */
|
|
48
|
+
export async function resolveRepo(dir) {
|
|
49
|
+
try {
|
|
50
|
+
return await gitx.root(dir)
|
|
51
|
+
} catch (err) {
|
|
52
|
+
throw new Error(`${dir} is not inside a git repository: ${err.message}`, { cause: err })
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Resolves the run a command should act on.
|
|
58
|
+
*
|
|
59
|
+
* The tag comes from `-tag` when given, otherwise from the checked-out branch
|
|
60
|
+
* — which is why `status` and `stop` accept `-tag`: they are meant to work
|
|
61
|
+
* from any branch, including one the agent is not on.
|
|
62
|
+
*
|
|
63
|
+
* @param {string} dir
|
|
64
|
+
* @param {string} [tag]
|
|
65
|
+
*/
|
|
66
|
+
export async function resolveRun(dir, tag) {
|
|
67
|
+
const root = await resolveRepo(dir)
|
|
68
|
+
let resolved = tag
|
|
69
|
+
if (!resolved) {
|
|
70
|
+
const branch = await gitx.currentBranch(root)
|
|
71
|
+
if (!branch.startsWith(BRANCH_PREFIX)) {
|
|
72
|
+
throw new Error(
|
|
73
|
+
`the checked-out branch ${JSON.stringify(branch)} is not a run branch, so there is no tag to ` +
|
|
74
|
+
`infer — pass -tag <tag>`,
|
|
75
|
+
)
|
|
76
|
+
}
|
|
77
|
+
resolved = branch.slice(BRANCH_PREFIX.length)
|
|
78
|
+
}
|
|
79
|
+
validTag(resolved)
|
|
80
|
+
return {
|
|
81
|
+
root,
|
|
82
|
+
tag: resolved,
|
|
83
|
+
branch: `${BRANCH_PREFIX}${resolved}`,
|
|
84
|
+
stateDir: await stateDir(root, resolved),
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Loads the in-repo config, with a message naming what to run when absent. */
|
|
89
|
+
export async function loadRepoConfig(root) {
|
|
90
|
+
try {
|
|
91
|
+
return await loadConfig(join(root, CONFIG_PATH))
|
|
92
|
+
} catch (err) {
|
|
93
|
+
if (err.cause?.code === 'ENOENT') {
|
|
94
|
+
throw new Error(`no ${CONFIG_PATH} in ${root}: run 'autor3search-javascript init' first`, { cause: err })
|
|
95
|
+
}
|
|
96
|
+
throw err
|
|
97
|
+
}
|
|
98
|
+
}
|
package/src/cli/main.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Subcommand registry and dispatch.
|
|
3
|
+
*
|
|
4
|
+
* The binary never edits source code. It gates correctness, measures
|
|
5
|
+
* candidate against baseline, and returns a verdict.
|
|
6
|
+
*/
|
|
7
|
+
import { runVersion } from './cmd-version.js'
|
|
8
|
+
|
|
9
|
+
/** Exit codes. program.md branches on 0-3; 64 is the conventional usage code. */
|
|
10
|
+
export const EXIT_USAGE = 64
|
|
11
|
+
|
|
12
|
+
const COMMANDS = [
|
|
13
|
+
['init', 'scan the repo, discover benchmarks, write config and program.md', () => import('./cmd-init.js').then((m) => m.runInit)],
|
|
14
|
+
['doctor', 'check whether this machine can measure reliably', () => import('./cmd-doctor.js').then((m) => m.runDoctor)],
|
|
15
|
+
['baseline', 'create the run branch, freeze tests and benchmarks, record the baseline', () => import('./cmd-baseline.js').then((m) => m.runBaseline)],
|
|
16
|
+
['profile', 'profile the declared benchmarks and report hot spots', () => import('./cmd-profile.js').then((m) => m.runProfile)],
|
|
17
|
+
['eval', 'run one experiment step and return a verdict', () => import('./cmd-eval.js').then((m) => m.runEval)],
|
|
18
|
+
['status', 'show where the run is: branch, worktree, experiments, stop state', () => import('./cmd-status.js').then((m) => m.runStatus)],
|
|
19
|
+
['stop', 'ask the agent to end the run after the current experiment', () => import('./cmd-stop.js').then((m) => m.runStop)],
|
|
20
|
+
['report', 'summarize results.tsv', () => import('./cmd-report.js').then((m) => m.runReport)],
|
|
21
|
+
['version', 'print which build of the harness this is', async () => runVersion],
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
function usage(io) {
|
|
25
|
+
io.err.write('autor3search-javascript — autonomous JavaScript performance optimization harness\n')
|
|
26
|
+
io.err.write('\nusage: autor3search-javascript <command> [flags]\n\ncommands:\n')
|
|
27
|
+
for (const [name, summary] of COMMANDS) io.err.write(` ${name.padEnd(9)} ${summary}\n`)
|
|
28
|
+
io.err.write('\nevery command accepts -C <dir> to run against another repository\n')
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {string[]} argv arguments after the program name
|
|
33
|
+
* @param {{out: {write(s: string): void}, err: {write(s: string): void}}} io
|
|
34
|
+
* @returns {Promise<number>} the process exit code
|
|
35
|
+
*/
|
|
36
|
+
export async function dispatch(argv, io) {
|
|
37
|
+
if (argv.length === 0) {
|
|
38
|
+
usage(io)
|
|
39
|
+
return EXIT_USAGE
|
|
40
|
+
}
|
|
41
|
+
const entry = COMMANDS.find(([name]) => name === argv[0])
|
|
42
|
+
if (!entry) {
|
|
43
|
+
io.err.write(`unknown command "${argv[0]}"\n\n`)
|
|
44
|
+
usage(io)
|
|
45
|
+
return EXIT_USAGE
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
try {
|
|
49
|
+
const run = await entry[2]()
|
|
50
|
+
return await run(argv.slice(1), io)
|
|
51
|
+
} catch (err) {
|
|
52
|
+
// A bad flag is a usage error, not a crash: report it in one line rather
|
|
53
|
+
// than as a stack trace an unattended agent would have to parse.
|
|
54
|
+
if (err.code?.startsWith('ERR_PARSE_ARGS')) {
|
|
55
|
+
io.err.write(`${err.message}\n\n`)
|
|
56
|
+
usage(io)
|
|
57
|
+
return EXIT_USAGE
|
|
58
|
+
}
|
|
59
|
+
io.err.write(`${err.message}\n`)
|
|
60
|
+
return 2
|
|
61
|
+
}
|
|
62
|
+
}
|
package/src/config.js
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Loads and validates the autor3search-javascript run configuration.
|
|
3
|
+
*
|
|
4
|
+
* The file lives inside the repository because humans own it and want it in
|
|
5
|
+
* version control. That means the agent can reach it, so it is protected by
|
|
6
|
+
* integrity checking rather than relocation: `baseline` records its hash and
|
|
7
|
+
* `eval` fails the run if it has changed. See src/pipeline.js.
|
|
8
|
+
*/
|
|
9
|
+
import { readFile } from 'node:fs/promises'
|
|
10
|
+
import { parse as parseYaml } from 'yaml'
|
|
11
|
+
import { parseDuration } from './duration.js'
|
|
12
|
+
|
|
13
|
+
/** Config location, relative to the repository root. */
|
|
14
|
+
export const CONFIG_PATH = '.autor3search/config.yaml'
|
|
15
|
+
|
|
16
|
+
/** The only bench runner adapter registered in this version. */
|
|
17
|
+
export const RUNNERS = ['vitest']
|
|
18
|
+
|
|
19
|
+
/** Permitted values for each entry under `gates:`. */
|
|
20
|
+
export const GATE_MODES = ['auto', 'on', 'off']
|
|
21
|
+
|
|
22
|
+
/** The gates a config may name. Anything else under `gates:` is a typo. */
|
|
23
|
+
export const GATE_NAMES = ['typecheck', 'lint', 'test']
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The smallest `count` at which the exact Mann-Whitney test used by
|
|
27
|
+
* src/bench/stats.js can ever report p < 0.05, however large or clean the
|
|
28
|
+
* improvement is. At 2 and 3 rounds per side the best achievable two-sided
|
|
29
|
+
* p-value is 0.3333 and 0.1 — both above the default alpha — so every single
|
|
30
|
+
* experiment would be discarded on a technicality rather than on its merits,
|
|
31
|
+
* with nothing in the output explaining why.
|
|
32
|
+
*/
|
|
33
|
+
export const MIN_COUNT = 4
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The largest `count` a config may ask for.
|
|
37
|
+
*
|
|
38
|
+
* Not a statistical limit — past a few dozen rounds per side the estimate has
|
|
39
|
+
* long stopped moving. It is a typo guard. `timeout` bounds each child process,
|
|
40
|
+
* not the run, so an extra zero does not fail: it quietly turns an overnight
|
|
41
|
+
* run into one that never finishes, and unattended is exactly how this tool is
|
|
42
|
+
* meant to be used. 1000 rounds per side is already 2000+ measurement
|
|
43
|
+
* processes and hours of wall clock, so a legitimate run stays well under it
|
|
44
|
+
* while `10000` or `1000000` stops at the config instead of at dawn.
|
|
45
|
+
*/
|
|
46
|
+
export const MAX_COUNT = 1000
|
|
47
|
+
|
|
48
|
+
/** Returns the configuration used when a field is omitted. */
|
|
49
|
+
export function defaultConfig() {
|
|
50
|
+
return {
|
|
51
|
+
benchmarks: [],
|
|
52
|
+
scope: ['**'],
|
|
53
|
+
count: 10,
|
|
54
|
+
maxRegressPct: 5,
|
|
55
|
+
minEffectPct: 1,
|
|
56
|
+
timeout: '15m',
|
|
57
|
+
unfreeze: [],
|
|
58
|
+
runner: 'vitest',
|
|
59
|
+
heapHint: true,
|
|
60
|
+
gates: { typecheck: 'auto', lint: 'auto', test: 'auto' },
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** YAML key -> in-memory field. Keys absent here keep their own name. */
|
|
65
|
+
const KEY_MAP = {
|
|
66
|
+
max_regress_pct: 'maxRegressPct',
|
|
67
|
+
min_effect_pct: 'minEffectPct',
|
|
68
|
+
heap_hint: 'heapHint',
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Every in-memory field name, derived from the defaults so the two cannot drift. */
|
|
72
|
+
const KNOWN_FIELDS = new Set(Object.keys(defaultConfig()))
|
|
73
|
+
|
|
74
|
+
/** Every key a user may legally write in the YAML, for the error message. */
|
|
75
|
+
const KNOWN_YAML_KEYS = new Set([
|
|
76
|
+
...Object.keys(KEY_MAP),
|
|
77
|
+
...[...KNOWN_FIELDS].filter((f) => !Object.values(KEY_MAP).includes(f)),
|
|
78
|
+
])
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Reads a config file, applying defaults for omitted fields and validating
|
|
82
|
+
* the result.
|
|
83
|
+
*
|
|
84
|
+
* @param {string} path
|
|
85
|
+
* @returns {Promise<object>}
|
|
86
|
+
*/
|
|
87
|
+
export async function loadConfig(path) {
|
|
88
|
+
let text
|
|
89
|
+
try {
|
|
90
|
+
text = await readFile(path, 'utf8')
|
|
91
|
+
} catch (err) {
|
|
92
|
+
throw new Error(`read config ${path}: ${err.message}`, { cause: err })
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
let raw
|
|
96
|
+
try {
|
|
97
|
+
raw = parseYaml(text) ?? {}
|
|
98
|
+
} catch (err) {
|
|
99
|
+
throw new Error(`parse ${path}: ${err.message}`, { cause: err })
|
|
100
|
+
}
|
|
101
|
+
if (typeof raw !== 'object' || Array.isArray(raw)) {
|
|
102
|
+
throw new Error(`parse ${path}: expected a mapping of settings at the top level`)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const cfg = defaultConfig()
|
|
106
|
+
for (const [key, value] of Object.entries(raw)) {
|
|
107
|
+
if (value === undefined || value === null) continue
|
|
108
|
+
const field = KEY_MAP[key] ?? key
|
|
109
|
+
// An unrecognised key is an ERROR, never a stray property. Without this, a
|
|
110
|
+
// typo like `max_regres_pct` is silently accepted while the real field
|
|
111
|
+
// keeps its default — so the run gates on a threshold the user never set
|
|
112
|
+
// and nothing says so. config.yaml is hashed at baseline, which means that
|
|
113
|
+
// typo is then locked in for the entire run.
|
|
114
|
+
if (!KNOWN_FIELDS.has(field)) {
|
|
115
|
+
throw new Error(
|
|
116
|
+
`${path}: unknown setting ${JSON.stringify(key)} — known settings are: ${[...KNOWN_YAML_KEYS].sort().join(', ')}`,
|
|
117
|
+
)
|
|
118
|
+
}
|
|
119
|
+
// `gates` merges rather than replaces, so a config setting one gate does
|
|
120
|
+
// not silently drop the defaults for the other two.
|
|
121
|
+
if (field === 'gates' && typeof value === 'object' && !Array.isArray(value)) {
|
|
122
|
+
// The same rule as above, one level down. `gates: {tests: "on"}` would
|
|
123
|
+
// otherwise merge a key nothing reads while `test` kept its default, so
|
|
124
|
+
// a user who asked for the test gate ON gets "auto" — which may skip the
|
|
125
|
+
// gate entirely — and the baseline hash locks that in for the whole run.
|
|
126
|
+
// Own keys only: a `__proto__` entry must be rejected, not walked into.
|
|
127
|
+
for (const name of Object.keys(value)) {
|
|
128
|
+
if (!GATE_NAMES.includes(name)) {
|
|
129
|
+
throw new Error(
|
|
130
|
+
`${path}: unknown gate ${JSON.stringify(name)} under 'gates' — known gates are: ${GATE_NAMES.join(', ')}`,
|
|
131
|
+
)
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
for (const name of GATE_NAMES) {
|
|
135
|
+
if (Object.hasOwn(value, name)) cfg.gates[name] = value[name]
|
|
136
|
+
}
|
|
137
|
+
continue
|
|
138
|
+
}
|
|
139
|
+
cfg[field] = value
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
try {
|
|
143
|
+
validate(cfg)
|
|
144
|
+
} catch (err) {
|
|
145
|
+
throw new Error(`invalid ${path}: ${err.message}`, { cause: err })
|
|
146
|
+
}
|
|
147
|
+
return cfg
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Reports whether the configuration is usable, throwing an Error whose
|
|
152
|
+
* message explains the problem in terms of what it would do to a run.
|
|
153
|
+
*
|
|
154
|
+
* @param {object} c
|
|
155
|
+
* @throws {Error}
|
|
156
|
+
*/
|
|
157
|
+
export function validate(c) {
|
|
158
|
+
if (!Number.isInteger(c.count) || c.count < MIN_COUNT) {
|
|
159
|
+
throw new Error(
|
|
160
|
+
`count must be at least ${MIN_COUNT}: the significance test cannot report p < 0.05 with fewer ` +
|
|
161
|
+
`than ${MIN_COUNT} measured rounds per side no matter how large the improvement is, so every ` +
|
|
162
|
+
`experiment would be discarded regardless of what changed (the default is 10)`,
|
|
163
|
+
)
|
|
164
|
+
}
|
|
165
|
+
if (c.count > MAX_COUNT) {
|
|
166
|
+
throw new Error(
|
|
167
|
+
`count ${c.count} is above the ${MAX_COUNT} round limit: timeout bounds each measurement process, ` +
|
|
168
|
+
`not the run, so a count this high does not fail — it runs for hours or days without finishing. ` +
|
|
169
|
+
`If you really want more than ${MAX_COUNT} rounds per side, raise MAX_COUNT deliberately ` +
|
|
170
|
+
`(the default is 10, and the estimate stops moving well before 100)`,
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
if (typeof c.maxRegressPct !== 'number' || c.maxRegressPct < 0) {
|
|
174
|
+
throw new Error('max_regress_pct must not be negative')
|
|
175
|
+
}
|
|
176
|
+
if (typeof c.minEffectPct !== 'number' || c.minEffectPct < 0 || c.minEffectPct >= 100) {
|
|
177
|
+
throw new Error('min_effect_pct must be at least 0 and less than 100')
|
|
178
|
+
}
|
|
179
|
+
if (!Array.isArray(c.scope) || c.scope.length === 0) {
|
|
180
|
+
throw new Error('scope must list at least one path pattern')
|
|
181
|
+
}
|
|
182
|
+
for (const s of c.scope) {
|
|
183
|
+
if (typeof s !== 'string' || s.trim() === '') {
|
|
184
|
+
throw new Error('scope must not contain an empty or whitespace-only entry')
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
if (!Array.isArray(c.unfreeze) || c.unfreeze.some((u) => typeof u !== 'string')) {
|
|
188
|
+
throw new Error('unfreeze must be a list of file paths')
|
|
189
|
+
}
|
|
190
|
+
if (!Array.isArray(c.benchmarks) || c.benchmarks.some((b) => typeof b !== 'string')) {
|
|
191
|
+
throw new Error('benchmarks must be a list of benchmark names')
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
parseDuration(c.timeout)
|
|
195
|
+
|
|
196
|
+
if (!RUNNERS.includes(c.runner)) {
|
|
197
|
+
throw new Error(
|
|
198
|
+
`runner ${JSON.stringify(c.runner)} is not a registered bench adapter (have: ${RUNNERS.join(', ')})`,
|
|
199
|
+
)
|
|
200
|
+
}
|
|
201
|
+
if (typeof c.heapHint !== 'boolean') {
|
|
202
|
+
throw new Error('heap_hint must be true or false')
|
|
203
|
+
}
|
|
204
|
+
for (const name of GATE_NAMES) {
|
|
205
|
+
if (!GATE_MODES.includes(c.gates?.[name])) {
|
|
206
|
+
throw new Error(`gates.${name} must be one of ${GATE_MODES.join(', ')}`)
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
package/src/discover.js
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finds benchmarks and test files by PARSING them, never by running Vitest.
|
|
3
|
+
*
|
|
4
|
+
* Parsing means discovery works on a tree that does not build — which is
|
|
5
|
+
* exactly when `init` is run, and exactly when a broken candidate needs to be
|
|
6
|
+
* reported as a typecheck failure rather than as "no benchmarks found".
|
|
7
|
+
*/
|
|
8
|
+
import { readdir, readFile } from 'node:fs/promises'
|
|
9
|
+
import { join, posix, relative, sep } from 'node:path'
|
|
10
|
+
import { parse } from '@babel/parser'
|
|
11
|
+
|
|
12
|
+
/** Extensions a bench or test file may carry. */
|
|
13
|
+
export const SOURCE_EXTS = ['.js', '.mjs', '.cjs', '.jsx', '.ts', '.mts', '.cts', '.tsx']
|
|
14
|
+
|
|
15
|
+
/** Directories never descended, matching what a JS toolchain itself ignores. */
|
|
16
|
+
const SKIP_DIRS = new Set(['node_modules', 'dist', 'build', 'coverage', 'out', '.git'])
|
|
17
|
+
|
|
18
|
+
const EXT_GROUP = SOURCE_EXTS.map((e) => e.slice(1)).join('|')
|
|
19
|
+
const BENCH_RE = new RegExp(`\\.bench\\.(${EXT_GROUP})$`)
|
|
20
|
+
const TEST_RE = new RegExp(`(\\.(test|spec)\\.(${EXT_GROUP})$)|(^|/)__tests__/`)
|
|
21
|
+
|
|
22
|
+
/** Identifiers that register a benchmark. */
|
|
23
|
+
const BENCH_CALLEES = new Set(['bench'])
|
|
24
|
+
/** Identifiers that open a naming scope around benchmarks. */
|
|
25
|
+
const SUITE_CALLEES = new Set(['describe', 'suite'])
|
|
26
|
+
|
|
27
|
+
/** @param {string} rel repo-relative, slash-separated */
|
|
28
|
+
export const isBenchFile = (rel) => BENCH_RE.test(rel)
|
|
29
|
+
|
|
30
|
+
/** @param {string} rel repo-relative, slash-separated */
|
|
31
|
+
export const isTestFile = (rel) => !BENCH_RE.test(rel) && TEST_RE.test(rel)
|
|
32
|
+
|
|
33
|
+
/** Every `*.bench.*` path in the repository, sorted. */
|
|
34
|
+
export async function benchFiles(root) {
|
|
35
|
+
return (await walk(root)).filter(isBenchFile)
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Every test path in the repository, sorted. */
|
|
39
|
+
export async function testFiles(root) {
|
|
40
|
+
return (await walk(root)).filter(isTestFile)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Every file that must be frozen at baseline: tests AND benchmarks.
|
|
45
|
+
*
|
|
46
|
+
* Freezing the benchmarks is not optional. In Vitest a benchmark lives in its
|
|
47
|
+
* own `*.bench.*` file, separate from the tests — so freezing only the tests
|
|
48
|
+
* would leave the metric itself editable, and an agent could rewrite the
|
|
49
|
+
* benchmark to measure something easier with every other gate still passing.
|
|
50
|
+
* Tests protect correctness; freezing the bench files protects the number.
|
|
51
|
+
*
|
|
52
|
+
* @param {string} root
|
|
53
|
+
* @param {string[]} [exclude] repo-relative paths deliberately left unfrozen
|
|
54
|
+
*/
|
|
55
|
+
export async function frozenFiles(root, exclude = []) {
|
|
56
|
+
const skip = new Set(exclude.map(toSlash))
|
|
57
|
+
return (await walk(root)).filter((rel) => (isBenchFile(rel) || isTestFile(rel)) && !skip.has(rel))
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Every benchmark declared in the repository, sorted by full path.
|
|
62
|
+
*
|
|
63
|
+
* @returns {Promise<{name: string, path: string, file: string, dir: string}[]>}
|
|
64
|
+
*/
|
|
65
|
+
export async function benchmarks(root) {
|
|
66
|
+
const out = []
|
|
67
|
+
for (const rel of await benchFiles(root)) {
|
|
68
|
+
let ast
|
|
69
|
+
try {
|
|
70
|
+
ast = parse(await readFile(join(root, rel), 'utf8'), {
|
|
71
|
+
sourceType: 'unambiguous',
|
|
72
|
+
errorRecovery: false,
|
|
73
|
+
plugins: pluginsFor(rel),
|
|
74
|
+
})
|
|
75
|
+
} catch {
|
|
76
|
+
// An unparseable bench file is not fatal to discovery. `init` reporting
|
|
77
|
+
// "no benchmarks" for a whole repository because one file has a syntax
|
|
78
|
+
// error would be a worse failure than silently skipping it.
|
|
79
|
+
continue
|
|
80
|
+
}
|
|
81
|
+
const dir = posix.dirname(rel)
|
|
82
|
+
collect(ast.program.body, [], (name, path) => {
|
|
83
|
+
out.push({ name, path, file: rel, dir })
|
|
84
|
+
})
|
|
85
|
+
}
|
|
86
|
+
out.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))
|
|
87
|
+
return out
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The bench FILES that contain any of the declared benchmarks.
|
|
92
|
+
*
|
|
93
|
+
* Vitest has no working name filter for benchmarks — `--testNamePattern` does
|
|
94
|
+
* not apply to them — so the declared set is selected after parsing, by
|
|
95
|
+
* BenchSet.selectByBase. Without this, every round runs every benchmark in the
|
|
96
|
+
* repository and throws almost all of it away: on a repository with 286 bench
|
|
97
|
+
* files and 1131 `bench()` calls, one round costs ~9 minutes instead of ~8
|
|
98
|
+
* seconds, and an eval needs fourteen of them.
|
|
99
|
+
*
|
|
100
|
+
* Filtering by FILE is safe where filtering by name is not, because it cannot
|
|
101
|
+
* change which benchmarks are scored — selectByBase still decides that. It
|
|
102
|
+
* only stops Vitest measuring benchmarks nothing will look at. The same list
|
|
103
|
+
* is used for both sides of the comparison, so the two sides stay identical.
|
|
104
|
+
*
|
|
105
|
+
* An empty `declared` means "every benchmark" and returns every bench file.
|
|
106
|
+
* A declared name matching nothing also returns [], which runs everything and
|
|
107
|
+
* lets selectByBase fail loudly naming what it could not match — the same
|
|
108
|
+
* behaviour as before this filter existed.
|
|
109
|
+
*
|
|
110
|
+
* @param {string} root
|
|
111
|
+
* @param {string[]} declared leaf benchmark names, as recorded at baseline
|
|
112
|
+
* @returns {Promise<string[]>} repo-relative paths, sorted
|
|
113
|
+
*/
|
|
114
|
+
export async function benchFilesFor(root, declared = []) {
|
|
115
|
+
if (declared.length === 0) return benchFiles(root)
|
|
116
|
+
const want = new Set(declared)
|
|
117
|
+
const files = new Set()
|
|
118
|
+
for (const b of await benchmarks(root)) {
|
|
119
|
+
if (want.has(b.name)) files.add(b.file)
|
|
120
|
+
}
|
|
121
|
+
return [...files].sort()
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** The leaf benchmark names, sorted and deduplicated. */
|
|
125
|
+
export function baseNames(list) {
|
|
126
|
+
return [...new Set(list.map((b) => b.name))].sort()
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Walks the statements of a scope, recording `bench(...)` calls and
|
|
131
|
+
* descending into `describe(...)` bodies so a nested benchmark's reported
|
|
132
|
+
* path matches how Vitest names the task.
|
|
133
|
+
*/
|
|
134
|
+
function collect(nodes, prefix, emit) {
|
|
135
|
+
for (const node of nodes ?? []) {
|
|
136
|
+
const call = asCall(node)
|
|
137
|
+
if (!call) continue
|
|
138
|
+
const callee = rootCalleeName(call.callee)
|
|
139
|
+
const name = literalName(call.arguments?.[0])
|
|
140
|
+
if (name === null) continue
|
|
141
|
+
|
|
142
|
+
if (BENCH_CALLEES.has(callee)) {
|
|
143
|
+
emit(name, [...prefix, name].join(' > '))
|
|
144
|
+
continue
|
|
145
|
+
}
|
|
146
|
+
if (SUITE_CALLEES.has(callee)) {
|
|
147
|
+
const body = call.arguments[1]
|
|
148
|
+
if (body?.body?.type === 'BlockStatement') {
|
|
149
|
+
collect(body.body.body, [...prefix, name], emit)
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* The Babel plugins to parse one file with, keyed by extension.
|
|
157
|
+
*
|
|
158
|
+
* `jsx` must NOT be enabled for a plain `.ts` file. There, `<number>value` is
|
|
159
|
+
* a legacy type assertion, but with `jsx` on, Babel reads it as an unclosed
|
|
160
|
+
* JSX element and the whole file fails to parse — so its benchmarks silently
|
|
161
|
+
* vanish from discovery. TypeScript itself forbids JSX syntax in `.ts` for
|
|
162
|
+
* exactly this ambiguity, which is why `.tsx` exists, so keying on the
|
|
163
|
+
* extension costs nothing and cannot regress a legitimate construct.
|
|
164
|
+
*/
|
|
165
|
+
export function pluginsFor(rel) {
|
|
166
|
+
const common = ['decorators-legacy', 'importAttributes']
|
|
167
|
+
if (/\.(mts|cts|ts)$/.test(rel)) return ['typescript', ...common]
|
|
168
|
+
if (rel.endsWith('.tsx')) return ['typescript', 'jsx', ...common]
|
|
169
|
+
return ['jsx', ...common]
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Unwraps an expression statement into its call expression, if it is one. */
|
|
173
|
+
function asCall(node) {
|
|
174
|
+
const expr = node?.type === 'ExpressionStatement' ? node.expression : null
|
|
175
|
+
return expr?.type === 'CallExpression' ? expr : null
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* The base identifier of a callee, so `bench.skip` and `describe.each(...)`
|
|
180
|
+
* are recognised as `bench` and `describe`.
|
|
181
|
+
*/
|
|
182
|
+
function rootCalleeName(callee) {
|
|
183
|
+
let node = callee
|
|
184
|
+
while (node) {
|
|
185
|
+
if (node.type === 'Identifier') return node.name
|
|
186
|
+
if (node.type === 'MemberExpression') node = node.object
|
|
187
|
+
else if (node.type === 'CallExpression') node = node.callee
|
|
188
|
+
else if (node.type === 'TaggedTemplateExpression') node = node.tag
|
|
189
|
+
else return null
|
|
190
|
+
}
|
|
191
|
+
return null
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* The static name of a task, from a string or a template literal with no
|
|
196
|
+
* substitutions. A computed name is returned as null and the call is skipped:
|
|
197
|
+
* the harness must never guess at a name it will later have to match against
|
|
198
|
+
* Vitest's own output.
|
|
199
|
+
*/
|
|
200
|
+
function literalName(node) {
|
|
201
|
+
if (node?.type === 'StringLiteral') return node.value
|
|
202
|
+
if (node?.type === 'TemplateLiteral' && node.expressions.length === 0) {
|
|
203
|
+
return node.quasis.map((q) => q.value.cooked).join('')
|
|
204
|
+
}
|
|
205
|
+
return null
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Every file under root, repo-relative and slash-separated, sorted. */
|
|
209
|
+
async function walk(root) {
|
|
210
|
+
const out = []
|
|
211
|
+
async function visit(absolute) {
|
|
212
|
+
let entries
|
|
213
|
+
try {
|
|
214
|
+
entries = await readdir(absolute, { withFileTypes: true })
|
|
215
|
+
} catch {
|
|
216
|
+
return
|
|
217
|
+
}
|
|
218
|
+
for (const entry of entries) {
|
|
219
|
+
const child = join(absolute, entry.name)
|
|
220
|
+
if (entry.isDirectory()) {
|
|
221
|
+
if (SKIP_DIRS.has(entry.name) || entry.name.startsWith('.') || entry.name.startsWith('_')) {
|
|
222
|
+
// __tests__ is the one underscore-prefixed directory that matters.
|
|
223
|
+
if (entry.name !== '__tests__') continue
|
|
224
|
+
}
|
|
225
|
+
await visit(child)
|
|
226
|
+
continue
|
|
227
|
+
}
|
|
228
|
+
// Deliberately does NOT follow symlinked entries: a symlink is handled
|
|
229
|
+
// by src/freeze.js, which refuses them loudly rather than walking
|
|
230
|
+
// through one into a tree outside the repository.
|
|
231
|
+
if (!entry.isFile()) continue
|
|
232
|
+
out.push(toSlash(relative(root, child)))
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
await visit(root)
|
|
236
|
+
return out.sort()
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const toSlash = (p) => p.split(sep).join('/')
|