bmad-method-test-architecture-enterprise 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/cli/automate.js +7 -0
- package/cli/examples/gitlab-ci.yaml +1 -1
- package/cli/examples/pr-test-review.yml +3 -3
- package/cli/lib/automate-inventory.js +258 -25
- package/cli/lib/automate-prompt.js +34 -2
- package/cli/lib/automate-result.js +74 -2
- package/cli/lib/unit-runners.js +219 -0
- package/cli/lib/vitest-attempts-reporter.mjs +30 -0
- package/cli/unit-check.js +595 -0
- package/package.json +12 -3
- package/skills/bmad-tea-automate/red/checklist.md +1 -1
- package/skills/bmad-tea-automate/resources/run-and-heal.md +3 -1
- package/skills/bmad-tea-evaluate/references/evaluator.md +2 -2
- package/skills/bmad-tea-evaluate/references/gaps.md +1 -1
- package/skills/bmad-tea-test-review/steps-c/step-03f-aggregate-scores.md +3 -3
- package/skills/bmod-tea/bmod.toml +1 -1
- package/skills/bmod-tea/knowledge/auth-session.md +25 -20
- package/skills/bmod-tea/knowledge/test-healing-patterns.md +2 -0
|
@@ -0,0 +1,595 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* tea-unit-check: runs generated Vitest or Jest tests under the project's own
|
|
4
|
+
* runner and writes the execution evidence tea-automate verifies.
|
|
5
|
+
*
|
|
6
|
+
* Expand mode runs each generated file in the project. Red mode proves a red
|
|
7
|
+
* scaffold fails for its intended reason while the permanent scaffold stays
|
|
8
|
+
* skipped:
|
|
9
|
+
*
|
|
10
|
+
* the consuming project with generated `test.skip()` / `it.skip()` leaves
|
|
11
|
+
* -> a disposable copy of the project, or of the whole workspace for a
|
|
12
|
+
* workspace package (node_modules linked, never copied)
|
|
13
|
+
* -> in the copy only, each generated leaf's `.skip` is blanked to spaces at
|
|
14
|
+
* the location the source inventory derived, so no unrelated skip moves
|
|
15
|
+
* and no other registration changes column
|
|
16
|
+
* -> one JSON report, bound to the permanent scaffold bytes by SHA-256
|
|
17
|
+
*
|
|
18
|
+
* Each file runs alone, so one file's load failure never masks another file's
|
|
19
|
+
* result. The runner writes its native JSON report; this command adds what that
|
|
20
|
+
* report cannot carry: Vitest's JSON reporter omits unhandled errors, so the
|
|
21
|
+
* process exit code and Vitest's unhandled-error banner are recorded beside it.
|
|
22
|
+
*
|
|
23
|
+
* The runner is detected from the project's package.json and config files.
|
|
24
|
+
* `--runner` only chooses between runners the project already declares.
|
|
25
|
+
*
|
|
26
|
+
* Usage:
|
|
27
|
+
* tea-unit-check --mode expand|red --project-root <dir> --report <path> --file <path> [--file <path>]
|
|
28
|
+
* [--runner vitest|jest] [--per-file-timeout-ms 120000] [--work-dir <dir>]
|
|
29
|
+
*
|
|
30
|
+
* Exit codes:
|
|
31
|
+
* 0 every file ran to some conclusion and the report was written
|
|
32
|
+
* 2 a usage error, or the run could not be set up
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
'use strict';
|
|
36
|
+
|
|
37
|
+
const fs = require('node:fs');
|
|
38
|
+
const os = require('node:os');
|
|
39
|
+
const path = require('node:path');
|
|
40
|
+
const { createHash } = require('node:crypto');
|
|
41
|
+
const { spawn, spawnSync } = require('node:child_process');
|
|
42
|
+
const { Command } = require('commander');
|
|
43
|
+
const { generatedInventory } = require('./lib/automate-inventory');
|
|
44
|
+
const {
|
|
45
|
+
UNIT_CHECK_KIND,
|
|
46
|
+
UNIT_RUNNERS,
|
|
47
|
+
detectRunners,
|
|
48
|
+
isWorkspaceRoot,
|
|
49
|
+
normalizeUnitReport,
|
|
50
|
+
workspaceAncestors,
|
|
51
|
+
} = require('./lib/unit-runners');
|
|
52
|
+
|
|
53
|
+
const NAME = 'tea-unit-check';
|
|
54
|
+
// Directories the copy never carries: dependency trees are linked in place and
|
|
55
|
+
// TeA's own output holds evidence, never product or test source.
|
|
56
|
+
const COPY_EXCLUDED = new Set(['node_modules', '.git', '_bmad-output']);
|
|
57
|
+
// Directories the integrity digest skips: dependencies, VCS state, evidence and
|
|
58
|
+
// runner output that a test run writes by design.
|
|
59
|
+
const DIGEST_EXCLUDED = new Set(['node_modules', '.git', '_bmad-output', 'test-results', '.cache', 'coverage', '__snapshots__']);
|
|
60
|
+
// Records each Vitest test's actual retry and repeat counts, which its JSON omits.
|
|
61
|
+
const ATTEMPTS_REPORTER = path.join(__dirname, 'lib', 'vitest-attempts-reporter.mjs');
|
|
62
|
+
// Vitest's default reporter prints this banner when it caught unhandled errors; its JSON report omits them.
|
|
63
|
+
const VITEST_UNHANDLED = /Vitest caught \d+ unhandled errors? during the test run/;
|
|
64
|
+
|
|
65
|
+
class SetupError extends Error {}
|
|
66
|
+
|
|
67
|
+
// The running runner and the disposable copy, so an interrupted check removes both.
|
|
68
|
+
const active = { child: null, scratch: null };
|
|
69
|
+
|
|
70
|
+
// The whole process group, even after its leader exited: a descendant that
|
|
71
|
+
// still holds the output pipes would otherwise keep the run open.
|
|
72
|
+
function killGroup(child) {
|
|
73
|
+
if (!child?.pid) return;
|
|
74
|
+
if (process.platform === 'win32') {
|
|
75
|
+
if (child.exitCode === null && child.signalCode === null)
|
|
76
|
+
spawnSync('taskkill', ['/pid', String(child.pid), '/T', '/F'], { windowsHide: true });
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
try {
|
|
80
|
+
process.kill(-child.pid, 'SIGKILL');
|
|
81
|
+
} catch {
|
|
82
|
+
// the group is already gone
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function cleanUpOnSignals() {
|
|
87
|
+
for (const [signal, number] of Object.entries({ SIGINT: 2, SIGTERM: 15, SIGHUP: 1 }))
|
|
88
|
+
process.on(signal, () => {
|
|
89
|
+
killGroup(active.child);
|
|
90
|
+
if (active.scratch) fs.rmSync(active.scratch, { recursive: true, force: true });
|
|
91
|
+
process.exit(128 + number);
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function fail(message) {
|
|
96
|
+
process.stderr.write(`${NAME}: ${message}\n`);
|
|
97
|
+
process.exit(2);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
|
|
101
|
+
const slash = (value) => value.split(path.sep).join('/');
|
|
102
|
+
const real = (value) => fs.realpathSync.native(value);
|
|
103
|
+
const inside = (base, absolute) => {
|
|
104
|
+
const relative = path.relative(base, absolute);
|
|
105
|
+
return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
function projectFile(root, value) {
|
|
109
|
+
const absolute = path.resolve(root, value);
|
|
110
|
+
if (absolute === root || !inside(root, absolute)) throw new SetupError(`--file must name a file inside the project: ${value}`);
|
|
111
|
+
if (!fs.existsSync(absolute) || !fs.statSync(absolute).isFile()) throw new SetupError(`--file does not exist: ${value}`);
|
|
112
|
+
const resolved = real(absolute);
|
|
113
|
+
if (!inside(root, resolved)) throw new SetupError(`--file resolves outside the project: ${value}`);
|
|
114
|
+
return slash(path.relative(root, resolved));
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** The project's own installed runner, found where Node resolves it from the project or its workspace. */
|
|
118
|
+
function runnerEntry(root, runner) {
|
|
119
|
+
for (const directory of [root, ...workspaceAncestors(root)]) {
|
|
120
|
+
const manifest = path.join(directory, 'node_modules', runner, 'package.json');
|
|
121
|
+
if (!fs.existsSync(manifest)) continue;
|
|
122
|
+
const pkg = JSON.parse(fs.readFileSync(manifest, 'utf8'));
|
|
123
|
+
const bin = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin?.[runner];
|
|
124
|
+
if (!bin) throw new SetupError(`${runner} at ${manifest} declares no ${runner} executable`);
|
|
125
|
+
const entry = path.join(path.dirname(manifest), bin);
|
|
126
|
+
if (!fs.existsSync(entry)) throw new SetupError(`${runner} declares its executable at ${entry}, which does not exist`);
|
|
127
|
+
return { entry, version: pkg.version };
|
|
128
|
+
}
|
|
129
|
+
throw new SetupError(`the project's ${runner} is not installed; run its package install first`);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Content, permission and link-target digest of every file under `root`, so any write, chmod or relink shows. */
|
|
133
|
+
function digestTree(root, skip = new Set()) {
|
|
134
|
+
const digest = new Map();
|
|
135
|
+
const walk = (directory) => {
|
|
136
|
+
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
137
|
+
const absolute = path.join(directory, entry.name);
|
|
138
|
+
const relative = slash(path.relative(root, absolute));
|
|
139
|
+
if (skip.has(absolute)) continue;
|
|
140
|
+
if (entry.isDirectory()) {
|
|
141
|
+
if (!DIGEST_EXCLUDED.has(entry.name)) walk(absolute);
|
|
142
|
+
} else if (entry.isSymbolicLink()) digest.set(relative, `link:${fs.readlinkSync(absolute)}`);
|
|
143
|
+
else if (entry.isFile())
|
|
144
|
+
digest.set(relative, `${(fs.statSync(absolute).mode & 0o777).toString(8)}:${sha256(fs.readFileSync(absolute))}`);
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
walk(root);
|
|
148
|
+
return digest;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const changed = (before, after) => [...new Set([...before.keys(), ...after.keys()])].filter((key) => before.get(key) !== after.get(key));
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* What the red copy carries. `base` is the outermost ancestor kept for relative
|
|
155
|
+
* paths: the workspace or repository root. `top` is the tree copied whole: the
|
|
156
|
+
* workspace for a workspace package, otherwise the project alone, so a project
|
|
157
|
+
* in a repository subfolder never copies or digests the rest of the repository.
|
|
158
|
+
* `linked` lists what the project's own symlinks reach elsewhere under `base`:
|
|
159
|
+
* those targets are copied and digested too, so no write through a link can
|
|
160
|
+
* reach the permanent repository unseen.
|
|
161
|
+
*/
|
|
162
|
+
function copyScope(root) {
|
|
163
|
+
const ancestors = workspaceAncestors(root);
|
|
164
|
+
const base = ancestors.at(-1) ?? root;
|
|
165
|
+
const top = base !== root && isWorkspaceRoot(base) ? base : root;
|
|
166
|
+
const linked = new Set();
|
|
167
|
+
const scan = (directory) => {
|
|
168
|
+
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
169
|
+
const absolute = path.join(directory, entry.name);
|
|
170
|
+
if (COPY_EXCLUDED.has(entry.name)) continue;
|
|
171
|
+
if (entry.isSymbolicLink()) {
|
|
172
|
+
let pointed;
|
|
173
|
+
try {
|
|
174
|
+
pointed = real(absolute);
|
|
175
|
+
} catch {
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (inside(base, pointed) && !inside(top, pointed)) {
|
|
179
|
+
// A link to an ancestor of the copied tree would pull the whole repository into it.
|
|
180
|
+
if (inside(pointed, top))
|
|
181
|
+
throw new SetupError(
|
|
182
|
+
`${slash(path.relative(root, absolute))} links to an ancestor of the project; the red copy cannot isolate it`,
|
|
183
|
+
);
|
|
184
|
+
if (![...linked].some((item) => inside(item, pointed))) {
|
|
185
|
+
linked.add(pointed);
|
|
186
|
+
if (fs.statSync(pointed).isDirectory()) scan(pointed);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
} else if (entry.isDirectory()) scan(absolute);
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
scan(top);
|
|
193
|
+
// Keep only the outermost targets, so no tree is copied twice whatever order the links were found in.
|
|
194
|
+
const outermost = [...linked].filter((item) => ![...linked].some((other) => other !== item && inside(other, item)));
|
|
195
|
+
return { ancestors, base, top, linked: outermost };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Digest every root of the scope, keyed by path relative to the project root. */
|
|
199
|
+
function digestScope(root, roots, locate = (item) => item, skip = new Set()) {
|
|
200
|
+
const digest = new Map();
|
|
201
|
+
for (const item of roots) {
|
|
202
|
+
const located = locate(item);
|
|
203
|
+
if (!fs.existsSync(located)) continue;
|
|
204
|
+
if (fs.statSync(located).isDirectory()) {
|
|
205
|
+
for (const [relative, value] of digestTree(located, skip)) digest.set(slash(path.relative(root, path.join(item, relative))), value);
|
|
206
|
+
} else
|
|
207
|
+
digest.set(
|
|
208
|
+
slash(path.relative(root, item)),
|
|
209
|
+
`${(fs.statSync(located).mode & 0o777).toString(8)}:${sha256(fs.readFileSync(located))}`,
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
return digest;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Copy the project without dependency trees and link each one so the copy
|
|
217
|
+
* resolves the project's own packages. A workspace package is copied with its
|
|
218
|
+
* whole workspace, so hoisted dependencies, sibling packages and relative
|
|
219
|
+
* `extends` resolve as they do in place, and a test writing into a sibling
|
|
220
|
+
* package writes into the copy. A project inside a plain repository keeps its
|
|
221
|
+
* ancestors' own files and dependency links above it, and the targets of its
|
|
222
|
+
* symlinks into the rest of the repository.
|
|
223
|
+
*/
|
|
224
|
+
function disposableCopy(root, scratch, { ancestors, base, top, linked }) {
|
|
225
|
+
const mirror = path.join(scratch, 'workspace');
|
|
226
|
+
const mirrorOf = (absolute) => path.join(mirror, path.relative(base, absolute));
|
|
227
|
+
for (const ancestor of ancestors.filter((directory) => !inside(top, directory))) {
|
|
228
|
+
fs.mkdirSync(mirrorOf(ancestor), { recursive: true });
|
|
229
|
+
for (const entry of fs.readdirSync(ancestor, { withFileTypes: true })) {
|
|
230
|
+
const source = path.join(ancestor, entry.name);
|
|
231
|
+
if (entry.name === 'node_modules') fs.symlinkSync(real(source), mirrorOf(source), 'junction');
|
|
232
|
+
else if (entry.isFile()) fs.copyFileSync(source, mirrorOf(source));
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
const links = [];
|
|
236
|
+
const copyFile = (from, to) => {
|
|
237
|
+
fs.mkdirSync(path.dirname(to), { recursive: true });
|
|
238
|
+
fs.copyFileSync(from, to);
|
|
239
|
+
fs.chmodSync(to, fs.statSync(from).mode & 0o7777);
|
|
240
|
+
};
|
|
241
|
+
// A hand-rolled walk: fs.cpSync refuses any copy whose destination lies inside
|
|
242
|
+
// its source, which a work directory inside the project makes true.
|
|
243
|
+
const copy = (source, target) => {
|
|
244
|
+
fs.mkdirSync(target, { recursive: true });
|
|
245
|
+
for (const entry of fs.readdirSync(source, { withFileTypes: true })) {
|
|
246
|
+
const from = path.join(source, entry.name);
|
|
247
|
+
const to = path.join(target, entry.name);
|
|
248
|
+
if (from === scratch || inside(scratch, from)) continue;
|
|
249
|
+
if (COPY_EXCLUDED.has(entry.name)) {
|
|
250
|
+
if (entry.name === 'node_modules') links.push(from);
|
|
251
|
+
} else if (entry.isSymbolicLink()) fs.symlinkSync(fs.readlinkSync(from), to);
|
|
252
|
+
else if (entry.isDirectory()) copy(from, to);
|
|
253
|
+
else if (entry.isFile()) copyFile(from, to);
|
|
254
|
+
}
|
|
255
|
+
};
|
|
256
|
+
const roots = [top, ...linked];
|
|
257
|
+
for (const item of roots) {
|
|
258
|
+
if (fs.statSync(item).isDirectory()) copy(item, mirrorOf(item));
|
|
259
|
+
else copyFile(item, mirrorOf(item));
|
|
260
|
+
}
|
|
261
|
+
// A link that reaches back into the copied scope must reach the copy instead,
|
|
262
|
+
// or a test writing through it would change the permanent project.
|
|
263
|
+
const rebase = (directory) => {
|
|
264
|
+
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
265
|
+
const absolute = path.join(directory, entry.name);
|
|
266
|
+
if (entry.isDirectory()) rebase(absolute);
|
|
267
|
+
else if (entry.isSymbolicLink()) {
|
|
268
|
+
const original = path.join(base, path.relative(mirror, absolute));
|
|
269
|
+
const text = fs.readlinkSync(absolute);
|
|
270
|
+
let pointed;
|
|
271
|
+
try {
|
|
272
|
+
pointed = real(original);
|
|
273
|
+
} catch {
|
|
274
|
+
pointed = path.resolve(path.dirname(original), text);
|
|
275
|
+
}
|
|
276
|
+
if (roots.some((item) => item === pointed || inside(item, pointed))) {
|
|
277
|
+
fs.unlinkSync(absolute);
|
|
278
|
+
fs.symlinkSync(mirrorOf(pointed), absolute);
|
|
279
|
+
} else if (!path.isAbsolute(text)) {
|
|
280
|
+
// The copy sits elsewhere, so a relative link out of the scope must name its target absolutely.
|
|
281
|
+
fs.unlinkSync(absolute);
|
|
282
|
+
fs.symlinkSync(pointed, absolute);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
};
|
|
287
|
+
for (const item of roots) if (fs.statSync(item).isDirectory()) rebase(mirrorOf(item));
|
|
288
|
+
for (const from of links) {
|
|
289
|
+
const target = mirrorOf(from);
|
|
290
|
+
if (fs.existsSync(path.dirname(target))) fs.symlinkSync(real(from), target, 'junction');
|
|
291
|
+
}
|
|
292
|
+
return { copyRoot: mirrorOf(root), mirrorOf, roots };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function runnerArguments(runner, entry, file, output) {
|
|
296
|
+
const attempts = `--reporter=${ATTEMPTS_REPORTER}`;
|
|
297
|
+
return runner === 'vitest'
|
|
298
|
+
? // The default reporter prints the unhandled-error banner the JSON reporter omits.
|
|
299
|
+
[
|
|
300
|
+
entry,
|
|
301
|
+
'run',
|
|
302
|
+
file,
|
|
303
|
+
'--reporter=default',
|
|
304
|
+
'--reporter=json',
|
|
305
|
+
`--outputFile.json=${output}`,
|
|
306
|
+
attempts,
|
|
307
|
+
'--includeTaskLocation',
|
|
308
|
+
'--coverage.enabled=false',
|
|
309
|
+
// One attempt per test, so every failure message belongs to that attempt.
|
|
310
|
+
'--retry=0',
|
|
311
|
+
]
|
|
312
|
+
: [
|
|
313
|
+
entry,
|
|
314
|
+
'--json',
|
|
315
|
+
'--testLocationInResults',
|
|
316
|
+
`--outputFile=${output}`,
|
|
317
|
+
'--ci',
|
|
318
|
+
// The exit code then means what the evidence check expects, whatever the project configures.
|
|
319
|
+
'--testFailureExitCode=1',
|
|
320
|
+
'--coverage=false',
|
|
321
|
+
'--no-watchman',
|
|
322
|
+
'--runTestsByPath',
|
|
323
|
+
file,
|
|
324
|
+
];
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Run the runner as the leader of its own process group, so a timeout kills
|
|
329
|
+
* its worker processes too: a hung test file must not leave a busy worker behind.
|
|
330
|
+
*/
|
|
331
|
+
function execute(args, { cwd, timeoutMs, env }) {
|
|
332
|
+
return new Promise((resolve) => {
|
|
333
|
+
const child = spawn(process.execPath, args, {
|
|
334
|
+
cwd,
|
|
335
|
+
env: { ...process.env, FORCE_COLOR: '0', NO_COLOR: '1', ...env },
|
|
336
|
+
detached: process.platform !== 'win32',
|
|
337
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
338
|
+
windowsHide: true,
|
|
339
|
+
});
|
|
340
|
+
const streams = { stdout: '', stderr: '' };
|
|
341
|
+
// Retained output is capped, so the unhandled-error banner is matched as it
|
|
342
|
+
// streams, across chunk boundaries, and can never fall past the cap.
|
|
343
|
+
let unhandled = false;
|
|
344
|
+
const carry = { stdout: '', stderr: '' };
|
|
345
|
+
for (const name of ['stdout', 'stderr']) {
|
|
346
|
+
child[name].setEncoding('utf8');
|
|
347
|
+
child[name].on('data', (chunk) => {
|
|
348
|
+
const window = carry[name] + chunk;
|
|
349
|
+
if (VITEST_UNHANDLED.test(window)) unhandled = true;
|
|
350
|
+
carry[name] = window.slice(-256);
|
|
351
|
+
if (streams[name].length < 4 * 1024 * 1024) streams[name] += chunk;
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
let timedOut = false;
|
|
355
|
+
active.child = child;
|
|
356
|
+
const timer = setTimeout(() => {
|
|
357
|
+
timedOut = true;
|
|
358
|
+
killGroup(child);
|
|
359
|
+
}, timeoutMs);
|
|
360
|
+
child.once('error', (error) => {
|
|
361
|
+
clearTimeout(timer);
|
|
362
|
+
resolve({ status: null, signal: null, error, timedOut, unhandled, ...streams });
|
|
363
|
+
});
|
|
364
|
+
// Leftover group members after the runner exits are this run's own processes.
|
|
365
|
+
child.once('exit', () => killGroup(child));
|
|
366
|
+
child.once('close', (status, signal) => {
|
|
367
|
+
clearTimeout(timer);
|
|
368
|
+
active.child = null;
|
|
369
|
+
resolve({ status, signal, error: null, timedOut, unhandled, ...streams });
|
|
370
|
+
});
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
async function runFile({ runner, entry, cwd, file, output, timeoutMs }) {
|
|
375
|
+
const absolute = path.join(cwd, file);
|
|
376
|
+
const attemptsFile = `${output}.attempts.json`;
|
|
377
|
+
const result = await execute(runnerArguments(runner, entry, absolute, output), {
|
|
378
|
+
cwd,
|
|
379
|
+
timeoutMs,
|
|
380
|
+
env: { TEA_UNIT_CHECK_ATTEMPTS: attemptsFile },
|
|
381
|
+
});
|
|
382
|
+
const exitCode = result.status;
|
|
383
|
+
const outcome = (loadError, runError = null, tests = []) => ({ exitCode, loadError, runError, tests });
|
|
384
|
+
const tail = String(result.stderr || result.stdout || '')
|
|
385
|
+
.trim()
|
|
386
|
+
.split('\n')
|
|
387
|
+
.filter(Boolean)
|
|
388
|
+
.slice(-6)
|
|
389
|
+
.join(' | ');
|
|
390
|
+
if (result.timedOut) return outcome(`timed out after ${timeoutMs}ms; the runner and its workers were killed`);
|
|
391
|
+
if (result.signal) return outcome(`the runner was killed by ${result.signal}`);
|
|
392
|
+
if (result.error) return outcome(`spawn failed: ${result.error.message}`);
|
|
393
|
+
if (!fs.existsSync(output)) return outcome(`no JSON report was written (exit ${exitCode}); ${tail || 'nothing on stderr'}`);
|
|
394
|
+
let normalized;
|
|
395
|
+
try {
|
|
396
|
+
normalized = normalizeUnitReport(JSON.parse(fs.readFileSync(output, 'utf8')));
|
|
397
|
+
} catch (error) {
|
|
398
|
+
return outcome(`the runner report could not be read: ${error.message}`);
|
|
399
|
+
}
|
|
400
|
+
const target = real(absolute);
|
|
401
|
+
const own = normalized.files.filter((item) => fs.existsSync(item.name) && real(item.name) === target);
|
|
402
|
+
if (own.length === 0) return outcome(`the runner report has no result for ${file} (exit ${exitCode}); ${tail || 'nothing on stderr'}`);
|
|
403
|
+
const fileErrors = own.filter((item) => item.error);
|
|
404
|
+
if (fileErrors.length > 0)
|
|
405
|
+
return outcome(
|
|
406
|
+
fileErrors
|
|
407
|
+
.map((item) => item.error)
|
|
408
|
+
.join('\n')
|
|
409
|
+
.split('\n')
|
|
410
|
+
.slice(0, 6)
|
|
411
|
+
.join(' | '),
|
|
412
|
+
);
|
|
413
|
+
// Stack frames name the disposable copy; the message itself is the evidence.
|
|
414
|
+
const tests = own.flatMap((item) =>
|
|
415
|
+
item.tests.map((test) => {
|
|
416
|
+
const last = test.attempts.at(-1);
|
|
417
|
+
const message = test.message?.replaceAll(/^\s+at .*$\n?/gm, '').trim() || null;
|
|
418
|
+
return {
|
|
419
|
+
titlePath: test.titlePath,
|
|
420
|
+
line: test.line,
|
|
421
|
+
column: test.column,
|
|
422
|
+
status: last.status,
|
|
423
|
+
failure: last.failure,
|
|
424
|
+
attempts: test.attempts,
|
|
425
|
+
message,
|
|
426
|
+
};
|
|
427
|
+
}),
|
|
428
|
+
);
|
|
429
|
+
const failed = tests.some((test) => test.status === 'failed');
|
|
430
|
+
const unhandled = runner === 'vitest' && result.unhandled;
|
|
431
|
+
// Vitest's JSON keeps one result per test; a retry or repeat, from any config, must not hide behind it.
|
|
432
|
+
let repeated = null;
|
|
433
|
+
if (runner === 'vitest') {
|
|
434
|
+
let diagnostics = null;
|
|
435
|
+
try {
|
|
436
|
+
diagnostics = JSON.parse(fs.readFileSync(attemptsFile, 'utf8'));
|
|
437
|
+
} catch {
|
|
438
|
+
repeated = 'Vitest wrote no attempt diagnostics, so its retries and repeats cannot be ruled out';
|
|
439
|
+
}
|
|
440
|
+
const extra = (diagnostics ?? []).filter(
|
|
441
|
+
(item) =>
|
|
442
|
+
item.moduleId &&
|
|
443
|
+
fs.existsSync(item.moduleId) &&
|
|
444
|
+
real(item.moduleId) === target &&
|
|
445
|
+
(item.retryCount !== 0 || item.repeatCount !== 0),
|
|
446
|
+
);
|
|
447
|
+
if (extra.length > 0)
|
|
448
|
+
repeated = `Vitest retried or repeated ${extra.map((item) => item.fullName).join(', ')}; one attempt per generated test is required`;
|
|
449
|
+
}
|
|
450
|
+
const runError =
|
|
451
|
+
repeated ??
|
|
452
|
+
(own.length > 1
|
|
453
|
+
? `${file} ran in ${own.length} runner projects; one result per generated test is required`
|
|
454
|
+
: unhandled
|
|
455
|
+
? 'Vitest caught unhandled errors during the run; its JSON report omits them'
|
|
456
|
+
: normalized.errors > 0
|
|
457
|
+
? `the runner reported ${normalized.errors} run-level error(s)`
|
|
458
|
+
: exitCode !== 0 && !failed
|
|
459
|
+
? `the runner exited ${exitCode} with no failed test`
|
|
460
|
+
: null);
|
|
461
|
+
return outcome(null, runError, tests);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
async function main(argv) {
|
|
465
|
+
const program = new Command();
|
|
466
|
+
program
|
|
467
|
+
.name(NAME)
|
|
468
|
+
.description('Run generated Vitest or Jest tests, red scaffolds in a disposable activated copy, and report what happened.')
|
|
469
|
+
.requiredOption('--mode <mode>', 'expand runs the files in place; red activates skipped scaffolds in a disposable copy')
|
|
470
|
+
.requiredOption('--project-root <dir>', 'the consuming project')
|
|
471
|
+
.requiredOption('--report <path>', 'where to write the JSON report')
|
|
472
|
+
.option('--file <path>', 'generated test file inside the project (repeatable)', (value, previous) => [...previous, value], [])
|
|
473
|
+
.option('--runner <name>', 'vitest or jest; only needed when the project declares both')
|
|
474
|
+
.option('--per-file-timeout-ms <n>', 'wall-clock bound per file', String(120_000))
|
|
475
|
+
.option('--work-dir <dir>', 'parent directory for the disposable red copy', os.tmpdir());
|
|
476
|
+
program.exitOverride();
|
|
477
|
+
try {
|
|
478
|
+
program.parse(argv);
|
|
479
|
+
} catch (error) {
|
|
480
|
+
if (error.exitCode === 0) return;
|
|
481
|
+
fail(error.message);
|
|
482
|
+
}
|
|
483
|
+
const options = program.opts();
|
|
484
|
+
let scratch;
|
|
485
|
+
try {
|
|
486
|
+
if (!['expand', 'red'].includes(options.mode)) throw new SetupError('--mode must be expand or red');
|
|
487
|
+
const red = options.mode === 'red';
|
|
488
|
+
if (!fs.existsSync(options.projectRoot) || !fs.statSync(options.projectRoot).isDirectory())
|
|
489
|
+
throw new SetupError(`--project-root ${options.projectRoot} is not a directory`);
|
|
490
|
+
const root = real(options.projectRoot);
|
|
491
|
+
const timeoutMs = Number.parseInt(options.perFileTimeoutMs, 10);
|
|
492
|
+
if (!Number.isInteger(timeoutMs) || timeoutMs <= 0 || String(timeoutMs) !== options.perFileTimeoutMs)
|
|
493
|
+
throw new SetupError('--per-file-timeout-ms must be a positive integer');
|
|
494
|
+
if (options.file.length === 0) throw new SetupError('name at least one generated test file with --file');
|
|
495
|
+
const files = options.file.map((file) => projectFile(root, file));
|
|
496
|
+
if (new Set(files).size !== files.length) throw new SetupError('--file names the same file twice');
|
|
497
|
+
|
|
498
|
+
const detected = detectRunners(root).filter(({ runner }) => UNIT_RUNNERS.includes(runner));
|
|
499
|
+
if (options.runner !== undefined && !UNIT_RUNNERS.includes(options.runner)) throw new SetupError('--runner must be vitest or jest');
|
|
500
|
+
const candidates = options.runner ? detected.filter(({ runner }) => runner === options.runner) : detected;
|
|
501
|
+
if (candidates.length === 0)
|
|
502
|
+
throw new SetupError(
|
|
503
|
+
options.runner ? `the project does not declare ${options.runner}` : 'the project declares neither Vitest nor Jest',
|
|
504
|
+
);
|
|
505
|
+
if (candidates.length > 1) throw new SetupError('the project declares both Vitest and Jest; pass --runner to choose one');
|
|
506
|
+
const [{ runner, configFile }] = candidates;
|
|
507
|
+
const { entry, version } = runnerEntry(root, runner);
|
|
508
|
+
// The attempt diagnostics need the reporter API Vitest 3 introduced.
|
|
509
|
+
if (runner === 'vitest' && Number.parseInt(version, 10) < 3)
|
|
510
|
+
throw new SetupError(`tea-unit-check needs Vitest 3 or newer; the project has ${version}`);
|
|
511
|
+
|
|
512
|
+
const activations = [];
|
|
513
|
+
const inventory = generatedInventory(root, files, { runner, activations });
|
|
514
|
+
if (red) {
|
|
515
|
+
const active = inventory.tests.filter((leaf) => leaf.activationLine === undefined);
|
|
516
|
+
if (active.length > 0)
|
|
517
|
+
throw new SetupError(
|
|
518
|
+
`red scaffolds must register every generated leaf with .skip: ${active.map((leaf) => `${leaf.file} ${leaf.titlePath.join(' > ')}`).join('; ')}`,
|
|
519
|
+
);
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
const reportPath = path.resolve(options.report);
|
|
523
|
+
if (!fs.existsSync(options.workDir) || !fs.statSync(options.workDir).isDirectory())
|
|
524
|
+
throw new SetupError(`--work-dir ${options.workDir} is not a directory`);
|
|
525
|
+
scratch = real(fs.mkdtempSync(path.join(real(options.workDir), `${NAME}-`)));
|
|
526
|
+
active.scratch = scratch;
|
|
527
|
+
const outputs = path.join(scratch, 'reports');
|
|
528
|
+
fs.mkdirSync(outputs);
|
|
529
|
+
const before = new Map(files.map((file) => [file, sha256(fs.readFileSync(path.join(root, file)))]));
|
|
530
|
+
// Red integrity covers the whole tree the copy carries: the project, or its workspace.
|
|
531
|
+
// Expand runs in place, so only red needs the copy scope and its isolation checks.
|
|
532
|
+
const scope = red ? copyScope(root) : null;
|
|
533
|
+
const roots = red ? [scope.top, ...scope.linked] : [];
|
|
534
|
+
// The report and the work directory change during the run by design.
|
|
535
|
+
const skip = new Set([reportPath, scratch]);
|
|
536
|
+
const projectBefore = red ? digestScope(root, roots, undefined, skip) : null;
|
|
537
|
+
let cwd = root;
|
|
538
|
+
let copy;
|
|
539
|
+
let copyBefore;
|
|
540
|
+
if (red) {
|
|
541
|
+
copy = disposableCopy(root, scratch, scope);
|
|
542
|
+
cwd = copy.copyRoot;
|
|
543
|
+
for (const file of files) {
|
|
544
|
+
const copied = path.join(cwd, file);
|
|
545
|
+
let text = fs.readFileSync(copied, 'utf8');
|
|
546
|
+
// Blank `.skip` to spaces (newlines kept), so every later registration keeps its line and column.
|
|
547
|
+
for (const { start, end } of activations.filter((edit) => edit.file === file))
|
|
548
|
+
text = `${text.slice(0, start)}${text.slice(start, end).replaceAll(/[^\n\r]/g, ' ')}${text.slice(end)}`;
|
|
549
|
+
fs.writeFileSync(copied, text);
|
|
550
|
+
}
|
|
551
|
+
copyBefore = digestScope(root, roots, copy.mirrorOf);
|
|
552
|
+
}
|
|
553
|
+
const results = [];
|
|
554
|
+
for (const [index, file] of files.entries())
|
|
555
|
+
results.push({
|
|
556
|
+
file,
|
|
557
|
+
sha256: before.get(file),
|
|
558
|
+
...(red ? { activated: activations.filter((edit) => edit.file === file).length } : {}),
|
|
559
|
+
...(await runFile({ runner, entry, cwd, file, output: path.join(outputs, `${index}.json`), timeoutMs })),
|
|
560
|
+
});
|
|
561
|
+
const report = {
|
|
562
|
+
schemaVersion: 1,
|
|
563
|
+
kind: UNIT_CHECK_KIND,
|
|
564
|
+
mode: options.mode,
|
|
565
|
+
runner,
|
|
566
|
+
runnerVersion: version,
|
|
567
|
+
configFile,
|
|
568
|
+
projectRoot: root,
|
|
569
|
+
files: results,
|
|
570
|
+
};
|
|
571
|
+
if (red) {
|
|
572
|
+
// Writes in the copy and writes that escaped into the permanent project both count.
|
|
573
|
+
const touched = new Set(changed(copyBefore, digestScope(root, roots, copy.mirrorOf)));
|
|
574
|
+
for (const file of changed(projectBefore, digestScope(root, roots, undefined, skip))) touched.add(file);
|
|
575
|
+
for (const file of files) if (sha256(fs.readFileSync(path.join(root, file))) !== before.get(file)) touched.add(file);
|
|
576
|
+
report.productionFilesTouched = [...touched].sort();
|
|
577
|
+
}
|
|
578
|
+
fs.mkdirSync(path.dirname(reportPath), { recursive: true });
|
|
579
|
+
fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
|
580
|
+
process.stdout.write(`Wrote ${reportPath}.\n`);
|
|
581
|
+
} catch (error) {
|
|
582
|
+
// Exit through exitCode so the finally block still removes the disposable copy.
|
|
583
|
+
process.stderr.write(`${NAME}: ${error instanceof SetupError ? error.message : `the run could not be set up: ${error.message}`}\n`);
|
|
584
|
+
process.exitCode = 2;
|
|
585
|
+
} finally {
|
|
586
|
+
if (scratch) fs.rmSync(scratch, { recursive: true, force: true });
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
if (require.main === module) {
|
|
591
|
+
cleanUpOnSignals();
|
|
592
|
+
main(process.argv).catch((error) => fail(error?.message ?? String(error)));
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
module.exports = { main, runnerArguments };
|