@autor3search/javascript 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +500 -0
- package/bin/autor3search-javascript.js +4 -0
- package/package.json +50 -0
- package/src/adapters/bench/index.js +31 -0
- package/src/adapters/bench/vitest.js +128 -0
- package/src/adapters/driver-child.js +72 -0
- package/src/adapters/driver-hooks.js +18 -0
- package/src/adapters/driver.js +192 -0
- package/src/adapters/gates/index.js +64 -0
- package/src/adapters/gates/lint.js +46 -0
- package/src/adapters/gates/test.js +27 -0
- package/src/adapters/gates/typecheck.js +98 -0
- package/src/adapters/gates/util.js +45 -0
- package/src/adapters/vitest-shim.js +51 -0
- package/src/bench/parse.js +120 -0
- package/src/bench/set.js +101 -0
- package/src/bench/stats.js +443 -0
- package/src/cli/cmd-baseline.js +136 -0
- package/src/cli/cmd-doctor.js +38 -0
- package/src/cli/cmd-eval.js +231 -0
- package/src/cli/cmd-init.js +136 -0
- package/src/cli/cmd-profile.js +38 -0
- package/src/cli/cmd-report.js +96 -0
- package/src/cli/cmd-status.js +68 -0
- package/src/cli/cmd-stop.js +98 -0
- package/src/cli/cmd-version.js +31 -0
- package/src/cli/context.js +98 -0
- package/src/cli/main.js +62 -0
- package/src/config.js +209 -0
- package/src/discover.js +239 -0
- package/src/doctor.js +282 -0
- package/src/duration.js +75 -0
- package/src/freeze.js +234 -0
- package/src/gitx.js +115 -0
- package/src/measure.js +127 -0
- package/src/pipeline.js +391 -0
- package/src/profile.js +100 -0
- package/src/results.js +124 -0
- package/src/runner.js +198 -0
- package/src/scope.js +92 -0
- package/src/state/index.js +312 -0
- package/src/state/lock.js +189 -0
- package/src/state/stop.js +56 -0
- package/src/verdict.js +214 -0
- package/templates/program.md +264 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Vitest implementation of the BenchRunner interface.
|
|
3
|
+
*
|
|
4
|
+
* One call to run() is ONE MEASURED ROUND: a single `vitest bench` process
|
|
5
|
+
* invocation, contributing one observation per benchmark. Rounds are what the
|
|
6
|
+
* significance test counts, so this must never be asked to loop internally.
|
|
7
|
+
*/
|
|
8
|
+
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
|
9
|
+
import { tmpdir } from 'node:os'
|
|
10
|
+
import { join } from 'node:path'
|
|
11
|
+
import { parseVitestBench } from '../../bench/parse.js'
|
|
12
|
+
import { Runner } from '../../runner.js'
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Runs one measured round.
|
|
16
|
+
*
|
|
17
|
+
* The report is written to a temp file via --outputJson rather than parsed
|
|
18
|
+
* from stdout: Vitest interleaves progress output with the report, and a
|
|
19
|
+
* stray console.log from the measured code would corrupt a stdout parse.
|
|
20
|
+
*
|
|
21
|
+
* @param {string} dir the worktree to measure
|
|
22
|
+
* @param {{benchmarks?: string[], timeoutMs: number, env?: object, log?: {write(s: string): void}, signal?: AbortSignal}} opts
|
|
23
|
+
* `benchmarks` selects the declared set AFTER parsing. Vitest has no working
|
|
24
|
+
* benchmark name filter — `--testNamePattern` is accepted but does not filter
|
|
25
|
+
* benches (confirmed against Vitest 2.1.9: `^(alpha)$` still returns `beta`
|
|
26
|
+
* too) — so selection happens here rather than on the command line.
|
|
27
|
+
* @returns {Promise<import('../../bench/set.js').BenchSet>}
|
|
28
|
+
*/
|
|
29
|
+
async function run(dir, opts) {
|
|
30
|
+
const scratch = await mkdtemp(join(tmpdir(), 'a3s-bench-'))
|
|
31
|
+
const reportPath = join(scratch, 'bench.json')
|
|
32
|
+
try {
|
|
33
|
+
const runner = new Runner(dir, opts.timeoutMs, opts.log ?? null)
|
|
34
|
+
const result = await runner.run(
|
|
35
|
+
process.execPath,
|
|
36
|
+
[
|
|
37
|
+
vitestBin(dir),
|
|
38
|
+
'bench',
|
|
39
|
+
'--run',
|
|
40
|
+
// `--outputJson` is the ONLY working way to get a machine-readable
|
|
41
|
+
// benchmark report. `--reporter=json --outputFile=…` fails outright
|
|
42
|
+
// ("Failed to load custom Reporter from json") and writes nothing.
|
|
43
|
+
// There is no `--benchmark.time=…` flag either; `BenchmarkUserOptions`
|
|
44
|
+
// exposes only include/exclude/includeSource/reporters/outputFile/
|
|
45
|
+
// compare/outputJson/includeSamples.
|
|
46
|
+
`--outputJson=${reportPath}`,
|
|
47
|
+
`--root=${dir}`,
|
|
48
|
+
// Positional args are Vitest's FILENAME filter, which does work (the
|
|
49
|
+
// name filter does not). Narrowing to the files that hold the declared
|
|
50
|
+
// benchmarks is what keeps a round proportional to what was declared
|
|
51
|
+
// rather than to the size of the repository. Empty means no filter, so
|
|
52
|
+
// an undeclared run still measures everything.
|
|
53
|
+
...(opts.benchFiles ?? []),
|
|
54
|
+
],
|
|
55
|
+
{ env: opts.env ?? process.env, signal: opts.signal },
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
if (result.timedOut) {
|
|
59
|
+
throw new Error(`benchmark round timed out in ${dir}`)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// A non-zero exit means some task (an import, a bench body) blew up. The
|
|
63
|
+
// JSON report may still exist in this case — Vitest writes it with an
|
|
64
|
+
// empty `groups` array for the failed file — but it cannot be trusted as
|
|
65
|
+
// a complete measurement, and the excerpt naming the real failure lives
|
|
66
|
+
// in the process output, not in the report. So this is checked BEFORE
|
|
67
|
+
// reading the report, not after, and carries the tail with it.
|
|
68
|
+
if (!result.ok()) {
|
|
69
|
+
throw new Error(`bench run failed in ${dir} (exit ${result.exitCode}):\n${result.tail(30)}`)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const report = await readFile(reportPath, 'utf8').catch(() => null)
|
|
73
|
+
if (report === null) {
|
|
74
|
+
throw new Error(
|
|
75
|
+
`bench run exited cleanly in ${dir} but wrote no report at ${reportPath}:\n${result.tail(30)}`,
|
|
76
|
+
)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// A bench body that throws at CALL time is NOT caught above: Vitest exits
|
|
80
|
+
// 0 for it and writes a report with zeros and no `median`. It is caught
|
|
81
|
+
// here instead, because parseVitestBench accepts only tinybench's median
|
|
82
|
+
// and refuses anything else. That coupling is deliberate and load-bearing
|
|
83
|
+
// — if a timing fallback were ever reintroduced in the parser, a
|
|
84
|
+
// half-failed benchmark would start being scored as a real measurement.
|
|
85
|
+
// test/adapters/vitest.test.js pins this.
|
|
86
|
+
let parsed
|
|
87
|
+
try {
|
|
88
|
+
parsed = parseVitestBench(report)
|
|
89
|
+
} catch (err) {
|
|
90
|
+
if (/contained no benchmarks/.test(err.message)) {
|
|
91
|
+
throw new Error(`Vitest measured no benchmarks at all in ${dir}: ${err.message}`, { cause: err })
|
|
92
|
+
}
|
|
93
|
+
throw err
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Selection happens HERE, not on the command line: Vitest accepts
|
|
97
|
+
// --testNamePattern for benches but does not act on it, so a run measures
|
|
98
|
+
// everything and the declared set is chosen afterwards. Costs wall-clock
|
|
99
|
+
// on undeclared benchmarks; correctness first.
|
|
100
|
+
const declared = opts.benchmarks ?? []
|
|
101
|
+
const set = parsed.selectByBase(declared)
|
|
102
|
+
if (set.names().length === 0) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
`no benchmarks matched ${JSON.stringify(declared)} in ${dir} (measured: ${JSON.stringify(parsed.bases())})`,
|
|
105
|
+
)
|
|
106
|
+
}
|
|
107
|
+
return set
|
|
108
|
+
} finally {
|
|
109
|
+
await rm(scratch, { recursive: true, force: true })
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The Vitest CLI entry point inside the measured repository.
|
|
115
|
+
*
|
|
116
|
+
* Exported so `doctor` can check for the SAME file this actually spawns.
|
|
117
|
+
* A module resolution (`require.resolve('vitest/vitest.mjs')`) is not
|
|
118
|
+
* equivalent: it honours the package's `exports` map, and Vitest 2 has a
|
|
119
|
+
* `./*` wildcard there while Vitest 3 and 4 do not — so the resolution
|
|
120
|
+
* reports "not installed" on a repository where this path exists and
|
|
121
|
+
* measurement works.
|
|
122
|
+
*/
|
|
123
|
+
export function vitestBin(dir) {
|
|
124
|
+
return join(dir, 'node_modules', 'vitest', 'vitest.mjs')
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** @type {{name: string, run: typeof run}} */
|
|
128
|
+
export const vitestRunner = { name: 'vitest', run }
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The child process the driver spawns.
|
|
3
|
+
*
|
|
4
|
+
* Invoked as:
|
|
5
|
+
* node [--expose-gc] driver-child.js <mode> <jsonOptionsPath>
|
|
6
|
+
* where mode is "list", "heap" or "profile". It prints ONE JSON object to
|
|
7
|
+
* stdout and nothing else, so the parent never has to parse around noise.
|
|
8
|
+
*/
|
|
9
|
+
import { readFile } from 'node:fs/promises'
|
|
10
|
+
import { register } from 'node:module'
|
|
11
|
+
import { pathToFileURL } from 'node:url'
|
|
12
|
+
|
|
13
|
+
const [, , mode, optionsPath] = process.argv
|
|
14
|
+
|
|
15
|
+
async function main() {
|
|
16
|
+
register('./driver-hooks.js', import.meta.url)
|
|
17
|
+
const options = JSON.parse(await readFile(optionsPath, 'utf8'))
|
|
18
|
+
|
|
19
|
+
const tasks = []
|
|
20
|
+
for (const file of options.benchFiles) {
|
|
21
|
+
globalThis.__a3sTasks = []
|
|
22
|
+
globalThis.__a3sSuite = []
|
|
23
|
+
await import(pathToFileURL(file).href)
|
|
24
|
+
for (const task of globalThis.__a3sTasks) tasks.push({ ...task, file })
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Selection is by NAME, matching how the rest of the harness selects (see
|
|
28
|
+
// BenchSet.selectByBase). An empty list selects everything. Deliberately not
|
|
29
|
+
// a regexp: a hand-edited config name containing a metacharacter must never
|
|
30
|
+
// silently widen what gets measured.
|
|
31
|
+
const wanted = new Set(options.benchmarks ?? [])
|
|
32
|
+
const selected = wanted.size === 0 ? tasks : tasks.filter((t) => wanted.has(t.name))
|
|
33
|
+
|
|
34
|
+
if (mode === 'list') {
|
|
35
|
+
return { tasks: selected.map(({ name, path, file }) => ({ name, path, file })) }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const results = []
|
|
39
|
+
for (const task of selected) {
|
|
40
|
+
// A short warmup lets V8 tier the function up, so the measured window
|
|
41
|
+
// reflects steady-state allocation rather than first-call overhead.
|
|
42
|
+
for (let i = 0; i < Math.min(50, options.iterations); i++) await task.fn()
|
|
43
|
+
|
|
44
|
+
globalThis.gc?.()
|
|
45
|
+
const before = process.memoryUsage().heapUsed
|
|
46
|
+
for (let i = 0; i < options.iterations; i++) await task.fn()
|
|
47
|
+
// Deliberately NOT collecting here: a collection would discard exactly the
|
|
48
|
+
// allocations being counted. The cost is that a collection running INSIDE
|
|
49
|
+
// the loop shows up as a negative delta, which is reported as no
|
|
50
|
+
// observation rather than as a negative allocation figure.
|
|
51
|
+
const after = process.memoryUsage().heapUsed
|
|
52
|
+
|
|
53
|
+
const delta = after - before
|
|
54
|
+
results.push({
|
|
55
|
+
name: task.name,
|
|
56
|
+
path: task.path,
|
|
57
|
+
file: task.file,
|
|
58
|
+
bytesPerOp: delta > 0 ? delta / options.iterations : null,
|
|
59
|
+
})
|
|
60
|
+
}
|
|
61
|
+
return { results, gcAvailable: typeof globalThis.gc === 'function' }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
main().then(
|
|
65
|
+
(payload) => {
|
|
66
|
+
process.stdout.write(JSON.stringify(payload))
|
|
67
|
+
},
|
|
68
|
+
(err) => {
|
|
69
|
+
process.stdout.write(JSON.stringify({ error: err?.stack ?? String(err) }))
|
|
70
|
+
process.exitCode = 1
|
|
71
|
+
},
|
|
72
|
+
)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Module-resolution hooks redirecting the `vitest` specifier to our shim.
|
|
3
|
+
*
|
|
4
|
+
* Registered with module.register() (Node >= 20.6) from driver-child.js, so
|
|
5
|
+
* a bench file's `import { bench } from 'vitest'` resolves to
|
|
6
|
+
* vitest-shim.js instead of the real package.
|
|
7
|
+
*/
|
|
8
|
+
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
9
|
+
import { dirname, join } from 'node:path'
|
|
10
|
+
|
|
11
|
+
const SHIM = pathToFileURL(join(dirname(fileURLToPath(import.meta.url)), 'vitest-shim.js')).href
|
|
12
|
+
|
|
13
|
+
export async function resolve(specifier, context, nextResolve) {
|
|
14
|
+
if (specifier === 'vitest' || specifier.startsWith('vitest/')) {
|
|
15
|
+
return { url: SHIM, shortCircuit: true }
|
|
16
|
+
}
|
|
17
|
+
return nextResolve(specifier, context)
|
|
18
|
+
}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drives bench files outside Vitest, for the two things Vitest's benchmark
|
|
3
|
+
* reporter cannot give us: an allocation hint, and a CPU/heap profile.
|
|
4
|
+
*
|
|
5
|
+
* Everything here is BEST-EFFORT. V8 has no allocation counter, the sampling
|
|
6
|
+
* window is at the mercy of GC scheduling, and a bench file may do something
|
|
7
|
+
* the shim cannot drive. Any failure yields an empty set and a note in the
|
|
8
|
+
* log — never an exception that would sink an otherwise-valid experiment.
|
|
9
|
+
* src/pipeline.js relies on that: a missing hint must not cost a KEEP.
|
|
10
|
+
*/
|
|
11
|
+
import { mkdir, mkdtemp, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
|
12
|
+
import { tmpdir } from 'node:os'
|
|
13
|
+
import { dirname, join, relative, resolve as resolvePath } from 'node:path'
|
|
14
|
+
import { fileURLToPath } from 'node:url'
|
|
15
|
+
import { BenchSet, UNIT_BYTES } from '../bench/set.js'
|
|
16
|
+
import { Runner } from '../runner.js'
|
|
17
|
+
|
|
18
|
+
const CHILD = join(dirname(fileURLToPath(import.meta.url)), 'driver-child.js')
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Lists the benchmarks a bench file registers, without measuring anything.
|
|
22
|
+
*
|
|
23
|
+
* @param {string} benchFile absolute path
|
|
24
|
+
* @returns {Promise<{name: string, path: string}[]>}
|
|
25
|
+
*/
|
|
26
|
+
export async function collectTasks(benchFile) {
|
|
27
|
+
const payload = await invoke(dirname(benchFile), 'list', {
|
|
28
|
+
benchFiles: [benchFile],
|
|
29
|
+
benchmarks: [],
|
|
30
|
+
iterations: 0,
|
|
31
|
+
})
|
|
32
|
+
return (payload?.tasks ?? []).map(({ name, path }) => ({ name, path }))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Measures the approximate bytes allocated per operation, one observation per
|
|
37
|
+
* benchmark. Returns an EMPTY set when the measurement is unavailable.
|
|
38
|
+
*
|
|
39
|
+
* @param {string} dir the worktree to measure
|
|
40
|
+
* @param {{benchFiles: string[], benchmarks?: string[], iterations: number, timeoutMs: number, env?: object, log?: {write(s: string): void}, signal?: AbortSignal}} opts
|
|
41
|
+
* @returns {Promise<BenchSet>}
|
|
42
|
+
*/
|
|
43
|
+
export async function measureHeap(dir, opts) {
|
|
44
|
+
const set = new BenchSet()
|
|
45
|
+
let payload
|
|
46
|
+
try {
|
|
47
|
+
payload = await invoke(dir, 'heap', {
|
|
48
|
+
benchFiles: opts.benchFiles.map((f) => resolvePath(dir, f)),
|
|
49
|
+
benchmarks: opts.benchmarks ?? [],
|
|
50
|
+
iterations: opts.iterations ?? 1000,
|
|
51
|
+
timeoutMs: opts.timeoutMs,
|
|
52
|
+
env: opts.env,
|
|
53
|
+
execArgv: ['--expose-gc'],
|
|
54
|
+
signal: opts.signal,
|
|
55
|
+
})
|
|
56
|
+
} catch (err) {
|
|
57
|
+
opts.log?.write(`heap hint unavailable, continuing without it: ${err.message}\n`)
|
|
58
|
+
return set
|
|
59
|
+
}
|
|
60
|
+
if (payload?.error) {
|
|
61
|
+
opts.log?.write(`heap hint unavailable, continuing without it: ${payload.error}\n`)
|
|
62
|
+
return set
|
|
63
|
+
}
|
|
64
|
+
if (payload?.gcAvailable === false) {
|
|
65
|
+
opts.log?.write('heap hint unavailable: node was not started with --expose-gc\n')
|
|
66
|
+
return set
|
|
67
|
+
}
|
|
68
|
+
for (const r of payload?.results ?? []) {
|
|
69
|
+
if (r.bytesPerOp === null) continue
|
|
70
|
+
// Key on a path RELATIVE to `dir`, never the absolute one. `dir` is the
|
|
71
|
+
// pinned baseline worktree for one side and the repository root for the
|
|
72
|
+
// other, so absolute keys can never match across the two and every
|
|
73
|
+
// cross-directory comparison silently produced no hint at all. The time
|
|
74
|
+
// path avoids this only because Vitest's own report yields root-relative
|
|
75
|
+
// names — see taskPath in src/bench/parse.js — so this mirrors it.
|
|
76
|
+
set.record(`${relative(dir, r.file)} > ${r.path}`, r.name, UNIT_BYTES, r.bytesPerOp)
|
|
77
|
+
}
|
|
78
|
+
return set
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Runs the benchmarks under V8's CPU and heap profilers, writing the raw
|
|
83
|
+
* profiles into outDir. Both files are openable in Chrome DevTools or
|
|
84
|
+
* speedscope; src/profile.js summarises the CPU one.
|
|
85
|
+
*
|
|
86
|
+
* @returns {Promise<{cpuProfile: string, heapProfile: string}>}
|
|
87
|
+
*/
|
|
88
|
+
export async function runProfiled(dir, opts) {
|
|
89
|
+
await mkdir(opts.outDir, { recursive: true })
|
|
90
|
+
const scratch = await mkdtemp(join(tmpdir(), 'a3s-prof-'))
|
|
91
|
+
try {
|
|
92
|
+
await invoke(dir, 'heap', {
|
|
93
|
+
benchFiles: opts.benchFiles.map((f) => resolvePath(dir, f)),
|
|
94
|
+
benchmarks: opts.benchmarks ?? [],
|
|
95
|
+
iterations: opts.iterations ?? 1000,
|
|
96
|
+
timeoutMs: opts.timeoutMs,
|
|
97
|
+
env: opts.env,
|
|
98
|
+
execArgv: [
|
|
99
|
+
'--expose-gc',
|
|
100
|
+
'--cpu-prof',
|
|
101
|
+
`--cpu-prof-dir=${scratch}`,
|
|
102
|
+
'--heap-prof',
|
|
103
|
+
`--heap-prof-dir=${scratch}`,
|
|
104
|
+
],
|
|
105
|
+
})
|
|
106
|
+
// V8 names the profiles after a timestamp and pid — verified on Node
|
|
107
|
+
// 22.23.1 as `CPU.<timestamp>.<pid>.0.001.cpuprofile` and
|
|
108
|
+
// `Heap.<timestamp>.<pid>.0.002.heapprofile`. There is NO fixed filename to
|
|
109
|
+
// predict, so they are matched by extension.
|
|
110
|
+
//
|
|
111
|
+
// CRUCIALLY, there is more than one of each. `module.register` runs the
|
|
112
|
+
// resolve hooks on their own thread, and --cpu-prof/--heap-prof profile
|
|
113
|
+
// that thread too — so a run yields both the MAIN thread's profile (the
|
|
114
|
+
// benchmark frames we want) and the LOADER thread's (module-resolution
|
|
115
|
+
// internals, no user code at all). Taking whichever readdir happened to
|
|
116
|
+
// return last picked the loader's profile in 3 of 3 trials, handing back a
|
|
117
|
+
// perfectly valid, non-empty profile containing ZERO frames of the
|
|
118
|
+
// benchmarked code.
|
|
119
|
+
//
|
|
120
|
+
// Content-based heuristics were tried and rejected: CPU sample count
|
|
121
|
+
// discriminated correctly in every trial run during verification, but the
|
|
122
|
+
// equivalent heap "most tree nodes" heuristic FLIPPED between trials (the
|
|
123
|
+
// main thread's heap-prof tree was sometimes smaller than the loader
|
|
124
|
+
// thread's, since heap-prof sampling is allocation-triggered and its yield
|
|
125
|
+
// is luck-of-the-draw, not proportional to "is this the thread that ran
|
|
126
|
+
// the benchmark"). So instead this parses the filename's thread-id field:
|
|
127
|
+
// Node names each profile `<Kind>.<date>.<time>.<pid>.<threadId>.<seq>.<ext>`,
|
|
128
|
+
// and the main thread's threadId is always 0 — a structural guarantee
|
|
129
|
+
// (`node:worker_threads` documents `threadId === 0` off the main thread),
|
|
130
|
+
// not an empirical proxy. Verified across 5 trials: the threadId-0 file is
|
|
131
|
+
// the one containing `driver-child.js`/benchmark frames in every case.
|
|
132
|
+
const written = await readdir(scratch)
|
|
133
|
+
const out = { cpuProfile: join(opts.outDir, 'cpu.cpuprofile'), heapProfile: join(opts.outDir, 'heap.heapprofile') }
|
|
134
|
+
const cpu = mainThreadFile(written, '.cpuprofile')
|
|
135
|
+
const heap = mainThreadFile(written, '.heapprofile')
|
|
136
|
+
if (cpu) await rename(join(scratch, cpu), out.cpuProfile)
|
|
137
|
+
if (heap) await rename(join(scratch, heap), out.heapProfile)
|
|
138
|
+
return out
|
|
139
|
+
} finally {
|
|
140
|
+
await rm(scratch, { recursive: true, force: true })
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Of several profile files V8 wrote for one run, picks the one written by the
|
|
146
|
+
* MAIN thread — see the long comment in runProfiled for why this can't be a
|
|
147
|
+
* content heuristic. Falls back to the alphabetically-first file (still
|
|
148
|
+
* deterministic) on the Node versions this was verified against, every
|
|
149
|
+
* profile filename carries the thread-id field, so the fallback is not
|
|
150
|
+
* expected to trigger in practice.
|
|
151
|
+
*/
|
|
152
|
+
function mainThreadFile(names, extension) {
|
|
153
|
+
const candidates = names
|
|
154
|
+
.filter((n) => n.endsWith(extension))
|
|
155
|
+
.map((n) => ({ name: n, threadId: n.split('.').at(-3) }))
|
|
156
|
+
.sort((a, b) => a.name.localeCompare(b.name))
|
|
157
|
+
return (candidates.find((c) => c.threadId === '0') ?? candidates[0])?.name ?? null
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Spawns the driver child and parses its single JSON object.
|
|
162
|
+
*
|
|
163
|
+
* `options.signal`, when present, is threaded into the Runner that spawns
|
|
164
|
+
* the child — NOT into the JSON file the child itself reads. An AbortSignal
|
|
165
|
+
* is not data for driver-child.js (JSON.stringify would silently drop it
|
|
166
|
+
* anyway, since it carries no own enumerable properties), and without this
|
|
167
|
+
* split an abort during this call — Ctrl+C or `stop --force` during the
|
|
168
|
+
* heap-hint phase, most concretely — would never reach runner.run, leaving
|
|
169
|
+
* the `node --expose-gc` child running until its own timeout (15m by
|
|
170
|
+
* default) instead of being killed promptly, same as `Runner.onAbort`
|
|
171
|
+
* already does for every other subprocess this project spawns.
|
|
172
|
+
*/
|
|
173
|
+
async function invoke(dir, mode, options) {
|
|
174
|
+
const { signal, ...forChild } = options
|
|
175
|
+
const scratch = await mkdtemp(join(tmpdir(), 'a3s-driver-'))
|
|
176
|
+
const optionsPath = join(scratch, 'options.json')
|
|
177
|
+
try {
|
|
178
|
+
await writeFile(optionsPath, JSON.stringify(forChild))
|
|
179
|
+
const runner = new Runner(dir, options.timeoutMs ?? 120_000, null)
|
|
180
|
+
const result = await runner.run(
|
|
181
|
+
process.execPath,
|
|
182
|
+
[...(options.execArgv ?? []), CHILD, mode, optionsPath],
|
|
183
|
+
{ env: options.env ?? process.env, signal },
|
|
184
|
+
)
|
|
185
|
+
if (result.stdout.trim() === '') {
|
|
186
|
+
throw new Error(`driver produced no output (exit ${result.exitCode}):\n${result.tail(20)}`)
|
|
187
|
+
}
|
|
188
|
+
return JSON.parse(result.stdout)
|
|
189
|
+
} finally {
|
|
190
|
+
await rm(scratch, { recursive: true, force: true })
|
|
191
|
+
}
|
|
192
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs the correctness gates in order, stopping at the first failure.
|
|
3
|
+
*
|
|
4
|
+
* Stopping early is deliberate: once a gate has rejected the candidate the
|
|
5
|
+
* experiment is over, and running the rest would spend minutes producing
|
|
6
|
+
* output nobody will read.
|
|
7
|
+
*/
|
|
8
|
+
import * as typecheck from './typecheck.js'
|
|
9
|
+
import * as lint from './lint.js'
|
|
10
|
+
import * as test from './test.js'
|
|
11
|
+
|
|
12
|
+
/** Order matters: cheapest and most fundamental first. */
|
|
13
|
+
const GATES = [typecheck, lint, test]
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {string} dir
|
|
17
|
+
* @param {{modes: Record<string,string>, scope: string[], timeoutMs: number, log?: object, signal?: AbortSignal}} opts
|
|
18
|
+
* @returns {Promise<{name: string, ran: boolean, ok: boolean, timedOut: boolean, skipped: string|null, detail: string}[]>}
|
|
19
|
+
*/
|
|
20
|
+
export async function runGates(dir, opts) {
|
|
21
|
+
const outcomes = []
|
|
22
|
+
for (const gate of GATES) {
|
|
23
|
+
const mode = opts.modes?.[gate.name] ?? 'auto'
|
|
24
|
+
if (mode === 'off') {
|
|
25
|
+
outcomes.push(skip(gate.name, 'gates.' + gate.name + ' is "off"'))
|
|
26
|
+
continue
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const why = await gate.unavailable(dir)
|
|
30
|
+
if (why) {
|
|
31
|
+
if (mode === 'on') {
|
|
32
|
+
// A gate the user explicitly asked for must not silently vanish.
|
|
33
|
+
outcomes.push({
|
|
34
|
+
name: gate.name,
|
|
35
|
+
ran: false,
|
|
36
|
+
ok: false,
|
|
37
|
+
timedOut: false,
|
|
38
|
+
skipped: null,
|
|
39
|
+
detail: `gates.${gate.name} is "on" but the gate cannot run: ${why}`,
|
|
40
|
+
})
|
|
41
|
+
return outcomes
|
|
42
|
+
}
|
|
43
|
+
outcomes.push(skip(gate.name, why))
|
|
44
|
+
continue
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const outcome = { name: gate.name, ...(await gate.run(dir, opts)) }
|
|
48
|
+
// Stop early if the caller aborted while this gate ran; the remaining
|
|
49
|
+
// gates would only spend minutes producing output nobody will read.
|
|
50
|
+
if (opts.signal?.aborted) return outcomes.concat(outcome)
|
|
51
|
+
// A gate that ran but only in a WEAKER form than the repository calls for
|
|
52
|
+
// fails when the user required it. Silently giving someone a lesser check
|
|
53
|
+
// than they asked for is the same defect as skipping it, but harder to notice.
|
|
54
|
+
if (mode === 'on' && outcome.degraded && outcome.ok) {
|
|
55
|
+
outcome.ok = false
|
|
56
|
+
outcome.detail = `gates.${gate.name} is "on" but only a degraded check was possible: ${outcome.detail}`
|
|
57
|
+
}
|
|
58
|
+
outcomes.push(outcome)
|
|
59
|
+
if (!outcome.ok) return outcomes
|
|
60
|
+
}
|
|
61
|
+
return outcomes
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const skip = (name, why) => ({ name, ran: false, ok: true, timedOut: false, skipped: why, detail: '' })
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/** The lint gate: ESLint, when the repository is configured for it. */
|
|
2
|
+
import { stat } from 'node:fs/promises'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { Runner } from '../../runner.js'
|
|
5
|
+
import { resolveFrom } from './util.js'
|
|
6
|
+
|
|
7
|
+
export const name = 'lint'
|
|
8
|
+
|
|
9
|
+
const CONFIGS = [
|
|
10
|
+
'eslint.config.js',
|
|
11
|
+
'eslint.config.mjs',
|
|
12
|
+
'eslint.config.cjs',
|
|
13
|
+
'.eslintrc',
|
|
14
|
+
'.eslintrc.js',
|
|
15
|
+
'.eslintrc.cjs',
|
|
16
|
+
'.eslintrc.json',
|
|
17
|
+
'.eslintrc.yml',
|
|
18
|
+
'.eslintrc.yaml',
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
export async function unavailable(dir) {
|
|
22
|
+
for (const candidate of CONFIGS) {
|
|
23
|
+
const found = await stat(join(dir, candidate)).then(
|
|
24
|
+
() => true,
|
|
25
|
+
() => false,
|
|
26
|
+
)
|
|
27
|
+
if (found) {
|
|
28
|
+
return resolveFrom(dir, 'eslint/bin/eslint.js') ? null : 'ESLint is configured but not installed'
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return 'no ESLint config found'
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export async function run(dir, opts) {
|
|
35
|
+
const bin = resolveFrom(dir, 'eslint/bin/eslint.js')
|
|
36
|
+
const result = await new Runner(dir, opts.timeoutMs, opts.log).run(process.execPath, [bin, '.'], {
|
|
37
|
+
signal: opts.signal,
|
|
38
|
+
})
|
|
39
|
+
return {
|
|
40
|
+
ran: true,
|
|
41
|
+
ok: result.ok(),
|
|
42
|
+
timedOut: result.timedOut,
|
|
43
|
+
skipped: null,
|
|
44
|
+
detail: result.ok() ? 'eslint .' : result.tail(30),
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The test gate. Correctness is never traded for speed: this runs the frozen
|
|
3
|
+
* tests, which were restored moments earlier, so a candidate that broke
|
|
4
|
+
* behaviour fails here before anything is measured.
|
|
5
|
+
*/
|
|
6
|
+
import { Runner } from '../../runner.js'
|
|
7
|
+
import { resolveFrom } from './util.js'
|
|
8
|
+
|
|
9
|
+
export const name = 'test'
|
|
10
|
+
|
|
11
|
+
export async function unavailable(dir) {
|
|
12
|
+
return resolveFrom(dir, 'vitest/vitest.mjs') ? null : 'vitest is not installed in this repository'
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export async function run(dir, opts) {
|
|
16
|
+
const bin = resolveFrom(dir, 'vitest/vitest.mjs')
|
|
17
|
+
const result = await new Runner(dir, opts.timeoutMs, opts.log).run(process.execPath, [bin, 'run'], {
|
|
18
|
+
signal: opts.signal,
|
|
19
|
+
})
|
|
20
|
+
return {
|
|
21
|
+
ran: true,
|
|
22
|
+
ok: result.ok(),
|
|
23
|
+
timedOut: result.timedOut,
|
|
24
|
+
skipped: null,
|
|
25
|
+
detail: result.ok() ? 'vitest run' : result.tail(40),
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The build gate.
|
|
3
|
+
*
|
|
4
|
+
* A TypeScript repository gets `tsc --noEmit`. A plain-JavaScript repository
|
|
5
|
+
* has no build step, but it must still get a real "is this even valid code"
|
|
6
|
+
* gate — otherwise a candidate with a syntax error would reach the test gate
|
|
7
|
+
* and be reported as a FAILING TEST rather than as a CRASH, losing exactly
|
|
8
|
+
* the distinction the exit codes exist to draw. So it gets a parse of every
|
|
9
|
+
* in-scope source file instead.
|
|
10
|
+
*
|
|
11
|
+
* The parse fallback reuses discover.js's per-extension plugin choice
|
|
12
|
+
* (`pluginsFor`): `jsx` must not be enabled for a plain `.ts` file, where
|
|
13
|
+
* `<number>value` is a legacy type assertion rather than an unclosed JSX
|
|
14
|
+
* element. Hardcoding a single plugin list here would silently reject valid
|
|
15
|
+
* TypeScript that discover.js itself parses correctly.
|
|
16
|
+
*/
|
|
17
|
+
import { readFile, stat } from 'node:fs/promises'
|
|
18
|
+
import { join } from 'node:path'
|
|
19
|
+
import { parse } from '@babel/parser'
|
|
20
|
+
import { createMatcher } from '../../scope.js'
|
|
21
|
+
import { SOURCE_EXTS, pluginsFor } from '../../discover.js'
|
|
22
|
+
import { Runner } from '../../runner.js'
|
|
23
|
+
import { resolveFrom, walkSources } from './util.js'
|
|
24
|
+
|
|
25
|
+
export const name = 'typecheck'
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Reports why this gate cannot run here, or null when it can.
|
|
29
|
+
*
|
|
30
|
+
* Always null: the parse fallback works everywhere, so there is no repository
|
|
31
|
+
* where this gate cannot run AT ALL. Degradation is reported separately — see
|
|
32
|
+
* the `degraded` flag in `run` — because a repo with a tsconfig but no
|
|
33
|
+
* installed typescript can still be parse-checked, just not type-checked.
|
|
34
|
+
*/
|
|
35
|
+
export async function unavailable() {
|
|
36
|
+
return null // the parse fallback always works
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function run(dir, opts) {
|
|
40
|
+
const hasTsconfig = await stat(join(dir, 'tsconfig.json')).then(
|
|
41
|
+
() => true,
|
|
42
|
+
() => false,
|
|
43
|
+
)
|
|
44
|
+
const tsc = hasTsconfig ? resolveFrom(dir, 'typescript/bin/tsc') : null
|
|
45
|
+
|
|
46
|
+
if (tsc) {
|
|
47
|
+
const result = await new Runner(dir, opts.timeoutMs, opts.log).run(process.execPath, [tsc, '--noEmit'], {
|
|
48
|
+
signal: opts.signal,
|
|
49
|
+
})
|
|
50
|
+
return {
|
|
51
|
+
ran: true,
|
|
52
|
+
ok: result.ok(),
|
|
53
|
+
timedOut: result.timedOut,
|
|
54
|
+
skipped: null,
|
|
55
|
+
degraded: false,
|
|
56
|
+
detail: result.ok() ? 'tsc --noEmit' : result.tail(30),
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const matcher = createMatcher(opts.scope)
|
|
61
|
+
const failures = []
|
|
62
|
+
for (const rel of await walkSources(dir, SOURCE_EXTS)) {
|
|
63
|
+
// Cheap: a boolean read, not a listener. The parse fallback has no
|
|
64
|
+
// subprocess for the Runner to kill, so without this an abort mid-walk
|
|
65
|
+
// would wait out the rest of a large repository's file list.
|
|
66
|
+
if (opts.signal?.aborted) break
|
|
67
|
+
if (!matcher.match(rel)) continue
|
|
68
|
+
try {
|
|
69
|
+
parse(await readFile(join(dir, rel), 'utf8'), {
|
|
70
|
+
sourceType: 'unambiguous',
|
|
71
|
+
plugins: pluginsFor(rel),
|
|
72
|
+
})
|
|
73
|
+
} catch (err) {
|
|
74
|
+
failures.push(`${rel}: ${err.message}`)
|
|
75
|
+
if (failures.length >= 20) break
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
// `degraded` marks a check WEAKER than the repository asked for: a
|
|
79
|
+
// tsconfig.json is present, so this repo wants real type checking, but
|
|
80
|
+
// typescript is not installed and only a syntax parse was possible. Under
|
|
81
|
+
// gates.typecheck: "on" that is a failure — a user who required type
|
|
82
|
+
// checking must not silently receive syntax checking instead. Under "auto"
|
|
83
|
+
// it runs and says so.
|
|
84
|
+
const degraded = hasTsconfig
|
|
85
|
+
return {
|
|
86
|
+
ran: true,
|
|
87
|
+
ok: failures.length === 0,
|
|
88
|
+
timedOut: false,
|
|
89
|
+
skipped: null,
|
|
90
|
+
degraded,
|
|
91
|
+
detail:
|
|
92
|
+
failures.length > 0
|
|
93
|
+
? failures.join('\n')
|
|
94
|
+
: degraded
|
|
95
|
+
? 'parse check ONLY — tsconfig.json is present but typescript is not installed, so types were not checked'
|
|
96
|
+
: 'parse check (no tsconfig.json)',
|
|
97
|
+
}
|
|
98
|
+
}
|