@descryy/runtime-test-runner 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/collector-version.d.ts +10 -0
- package/dist/collector-version.d.ts.map +1 -0
- package/dist/collector-version.js +12 -0
- package/dist/collector-version.js.map +1 -0
- package/dist/go-test.d.ts +81 -0
- package/dist/go-test.d.ts.map +1 -0
- package/dist/go-test.js +224 -0
- package/dist/go-test.js.map +1 -0
- package/dist/graph-mapping.d.ts +44 -0
- package/dist/graph-mapping.d.ts.map +1 -0
- package/dist/graph-mapping.js +72 -0
- package/dist/graph-mapping.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -0
- package/dist/jest-reporter.d.ts +72 -0
- package/dist/jest-reporter.d.ts.map +1 -0
- package/dist/jest-reporter.js +110 -0
- package/dist/jest-reporter.js.map +1 -0
- package/dist/playwright-reporter.d.ts +67 -0
- package/dist/playwright-reporter.d.ts.map +1 -0
- package/dist/playwright-reporter.js +93 -0
- package/dist/playwright-reporter.js.map +1 -0
- package/dist/reporter.d.ts +35 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporter.js +74 -0
- package/dist/reporter.js.map +1 -0
- package/dist/swift-xctest.d.ts +138 -0
- package/dist/swift-xctest.d.ts.map +1 -0
- package/dist/swift-xctest.js +384 -0
- package/dist/swift-xctest.js.map +1 -0
- package/dist/test-runner-collector.d.ts +370 -0
- package/dist/test-runner-collector.d.ts.map +1 -0
- package/dist/test-runner-collector.js +1014 -0
- package/dist/test-runner-collector.js.map +1 -0
- package/dist/vitest-reporter.d.ts +76 -0
- package/dist/vitest-reporter.d.ts.map +1 -0
- package/dist/vitest-reporter.js +131 -0
- package/dist/vitest-reporter.js.map +1 -0
- package/package.json +33 -0
|
@@ -0,0 +1,1014 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* §17's test-runner observation: **`node:test` (RT-073), Jest and Vitest
|
|
3
|
+
* (RT-123)**, each observed through that runner's own structured reporter
|
|
4
|
+
* API rather than TAP, a JSON summary file, or any other printed format.
|
|
5
|
+
*
|
|
6
|
+
* ## Three reporters, one envelope, no per-runner branch below this line
|
|
7
|
+
*
|
|
8
|
+
* The per-runner knowledge is *what a runner's own API calls things*, and
|
|
9
|
+
* that only exists inside the runner's process. So each runner gets a
|
|
10
|
+
* reporter that speaks its API (`reporter.ts`, `jest-reporter.ts`,
|
|
11
|
+
* `vitest-reporter.ts`) and writes one shared NDJSON envelope — node:test's
|
|
12
|
+
* own event and field names, kept because they were first. Everything from
|
|
13
|
+
* `createEventTranslator` down is runner-agnostic, the same boundary rule
|
|
14
|
+
* the IR has. The only per-runner logic is `PROFILES` below: how a command
|
|
15
|
+
* is recognised, what it is refused for, and which flags are injected.
|
|
16
|
+
*
|
|
17
|
+
* ## The defect a second runner exposed in code that had already shipped
|
|
18
|
+
*
|
|
19
|
+
* A `test.skip` emits **`test:pass` carrying `skip: true`** on node:test,
|
|
20
|
+
* and the translator mapped every `test:pass` to TEST_PASSED — so a
|
|
21
|
+
* skipped test was evidence of a passing one, the shape this checklist
|
|
22
|
+
* exists to ban, invisible because every fixture was fully-running. They
|
|
23
|
+
* are counted now, not emitted; see `TraceCoverage.notRun`.
|
|
24
|
+
*
|
|
25
|
+
* The node:test notes below still describe what this file's node path
|
|
26
|
+
* does, unchanged.
|
|
27
|
+
*
|
|
28
|
+
* `test:start` / `test:pass` / `test:fail` translate directly into
|
|
29
|
+
* `TEST_STARTED` / `TEST_PASSED` / `TEST_FAILED` Evidence, with the real
|
|
30
|
+
* `file`/`line`/`column` Node's own reporter stream already carries —
|
|
31
|
+
* `reliability: "self-contained"`, the same class a Python traceback or
|
|
32
|
+
* Ruby backtrace gets, since this came from the runtime's own captured
|
|
33
|
+
* state, not derivation.
|
|
34
|
+
*
|
|
35
|
+
* **Discovery is declared, never inferred** (RT-032's own precedent):
|
|
36
|
+
* `discoverRunner` refuses a command that names none of the three runners
|
|
37
|
+
* it observes, rather than sniffing `package.json`/lockfiles to guess which
|
|
38
|
+
* runner a project uses. Also refuses a command that already carries its
|
|
39
|
+
* own `--test-reporter` — safely co-existing with an unknown pre-existing
|
|
40
|
+
* reporter configuration (matching destinations positionally, possibly
|
|
41
|
+
* conflicting output) is real complexity this collector does not attempt
|
|
42
|
+
* to get right blindly; a caller who wants both gets an honest refusal
|
|
43
|
+
* naming why, not a best-effort guess.
|
|
44
|
+
*
|
|
45
|
+
* **Trace capture (RT-074): declared, via `TestContext.diagnostic()`, not
|
|
46
|
+
* inferred from anything.** A test that makes a backend call and wants its
|
|
47
|
+
* TEST_PASSED/TEST_FAILED evidence to correlate with that call's backend
|
|
48
|
+
* evidence calls `t.diagnostic("requestId: <the id it used>")` (and/or
|
|
49
|
+
* `traceId: ...`) — a real node:test API, surfaced as its own
|
|
50
|
+
* `test:diagnostic` reporter event, distinguishable from suite-level
|
|
51
|
+
* summary diagnostics by carrying the same `file`/`line` as the test
|
|
52
|
+
* itself. Nothing here guesses a request id from timing or proximity; a
|
|
53
|
+
* test that says nothing gets `requestId: null`, same as before.
|
|
54
|
+
*
|
|
55
|
+
* **TEST_CASE graph mapping (RT-076): `payload.qualifiedName`, not
|
|
56
|
+
* `payload.name`, is what a real graph resolves against.** node:test's own
|
|
57
|
+
* events carry only a test's innermost title; `adapter-typescript`'s
|
|
58
|
+
* TEST_CASE node name is the full `describe()` chain, space-joined ahead
|
|
59
|
+
* of the title. `SuiteStack` (below) reconstructs that chain purely from
|
|
60
|
+
* the reporter stream this collector already reads. `graphNodeId` on
|
|
61
|
+
* emitted evidence is still always null here, same as every other
|
|
62
|
+
* collector — `graph-mapping.ts`'s `resolveObservedTestCase` is what a
|
|
63
|
+
* caller holding both the store and the graph driver uses to resolve and
|
|
64
|
+
* attribute it, after `write()`, the same way `vertical-slice` already
|
|
65
|
+
* does for every other evidence type.
|
|
66
|
+
*
|
|
67
|
+
* **A note for whoever next mutation-tests this file (RT-080).** This
|
|
68
|
+
* package's own tests import `@descryy/runtime-test-runner` by package
|
|
69
|
+
* name, resolving through `exports` to `dist/` — not `../src/*.ts`
|
|
70
|
+
* directly. Mutating this file and re-running with a bare `node --test
|
|
71
|
+
* <file>` proves nothing: the compiled `dist/` is untouched and the
|
|
72
|
+
* mutation is never in the code path actually executed, an inert check
|
|
73
|
+
* that reads as a passing safety net. Always rebuild between mutate and
|
|
74
|
+
* run — `npm test` at the repo root does this automatically (`npm run
|
|
75
|
+
* build && node --test ...`); a scoped `npx tsc -b packages/test-runner`
|
|
76
|
+
* before a scoped `node --test packages/test-runner/test/*.test.ts` also
|
|
77
|
+
* works and is faster while iterating on one file.
|
|
78
|
+
*/
|
|
79
|
+
import { fileURLToPath } from "node:url";
|
|
80
|
+
import { spawnProcess, tokenizeCommand } from "@descryy/runtime-controller";
|
|
81
|
+
import { LineSplitter } from "@descryy/runtime-backend-observation";
|
|
82
|
+
import { COLLECTOR_VERSION } from "./collector-version.js";
|
|
83
|
+
/**
|
|
84
|
+
* The version of the NDJSON envelope every reporter speaks (RT-130).
|
|
85
|
+
*
|
|
86
|
+
* ## Why a version, and why now rather than when it first hurts
|
|
87
|
+
*
|
|
88
|
+
* The three reporters that exist today are TypeScript in this package,
|
|
89
|
+
* compiled by the same `tsc -b` as the translator — so they cannot drift
|
|
90
|
+
* from it without the build saying so. **The JUnit listener will not be**:
|
|
91
|
+
* it is a Java source built by a separate step, and a compiled artifact
|
|
92
|
+
* that still runs while the envelope moves underneath it produces *wrong
|
|
93
|
+
* events*, not a build error.
|
|
94
|
+
*
|
|
95
|
+
* That is the failure the placement argument was really about. Wiring a
|
|
96
|
+
* build step can only tell you a file did not compile; it cannot tell you
|
|
97
|
+
* the thing that *did* compile is no longer telling the truth. A version
|
|
98
|
+
* in the stream can, at the moment it would otherwise be believed.
|
|
99
|
+
*
|
|
100
|
+
* **Landed before the first out-of-band reporter exists, deliberately.** A
|
|
101
|
+
* version added afterwards has nothing to check on the transition it was
|
|
102
|
+
* added for: the first listener would speak an unversioned envelope, and
|
|
103
|
+
* the absence-is-an-error rule below could not be turned on without
|
|
104
|
+
* breaking it.
|
|
105
|
+
*
|
|
106
|
+
* Bump this only when the envelope's *meaning* changes for an existing
|
|
107
|
+
* field. Adding an optional field a reporter may omit — `skipReason`,
|
|
108
|
+
* `isSuite`, `qualifiedName` — is not a bump: an older reporter that never
|
|
109
|
+
* sets it is still telling the truth, and the translator already has a
|
|
110
|
+
* documented answer for its absence.
|
|
111
|
+
*/
|
|
112
|
+
export const ENVELOPE_VERSION = 1;
|
|
113
|
+
/** The handshake line every reporter emits before its first event. */
|
|
114
|
+
export const ENVELOPE_HANDSHAKE_TYPE = "descry:envelope";
|
|
115
|
+
/**
|
|
116
|
+
* The envelope gate, a function of its own because the case it exists for
|
|
117
|
+
* **cannot be reached end to end** (RT-130).
|
|
118
|
+
*
|
|
119
|
+
* Measured: node:test redirects a test's stdout — the patched
|
|
120
|
+
* `process.stdout.write` and the raw fd alike — and re-emits its output as
|
|
121
|
+
* a `test:stdout` event. So a fixture cannot put a stale handshake on the
|
|
122
|
+
* wire, and there is no way to drive the mismatch path through a real
|
|
123
|
+
* `node --test` run. That is a genuine safety property of the design (a
|
|
124
|
+
* test's own `console.log` can never be mistaken for a reporter line) and
|
|
125
|
+
* simultaneously the reason this logic is a pure function with its own
|
|
126
|
+
* tests rather than four lines inside `start()`.
|
|
127
|
+
*
|
|
128
|
+
* Stated rather than left as a coverage gap: the wiring is asserted end to
|
|
129
|
+
* end — a real run declares version 1, produces no `COLLECTOR_ERROR`, and
|
|
130
|
+
* the handshake never becomes an event — and the refusal is asserted here.
|
|
131
|
+
*/
|
|
132
|
+
export function createEnvelopeGate() {
|
|
133
|
+
let seen = false;
|
|
134
|
+
let refused = false;
|
|
135
|
+
return {
|
|
136
|
+
inspect(parsed, isEvent) {
|
|
137
|
+
const declared = envelopeVersionOf(parsed);
|
|
138
|
+
if (declared !== null) {
|
|
139
|
+
if (declared !== ENVELOPE_VERSION && !refused) {
|
|
140
|
+
refused = true;
|
|
141
|
+
return {
|
|
142
|
+
kind: "refuse",
|
|
143
|
+
reason: `reporter declared envelope version ${declared}; this collector speaks ${ENVELOPE_VERSION}. ` +
|
|
144
|
+
"A reporter built out of band — a compiled listener rather than this package's TypeScript — can run " +
|
|
145
|
+
"against a moved envelope with no build failing, which is the case this check exists for.",
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
seen = true;
|
|
149
|
+
return { kind: "handshake" };
|
|
150
|
+
}
|
|
151
|
+
if (!isEvent || refused)
|
|
152
|
+
return { kind: "ignore" };
|
|
153
|
+
if (!seen) {
|
|
154
|
+
refused = true;
|
|
155
|
+
return {
|
|
156
|
+
kind: "refuse",
|
|
157
|
+
reason: "a reporter event arrived before any envelope declaration. A reporter that never declares one is older " +
|
|
158
|
+
`than envelope ${ENVELOPE_VERSION}, and its events cannot be read as this version's.`,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
return { kind: "event" };
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
/** The declared envelope version of a handshake line, or null for any other line. */
|
|
166
|
+
function envelopeVersionOf(parsed) {
|
|
167
|
+
if (typeof parsed !== "object" || parsed === null)
|
|
168
|
+
return null;
|
|
169
|
+
const line = parsed;
|
|
170
|
+
if (line.type !== ENVELOPE_HANDSHAKE_TYPE)
|
|
171
|
+
return null;
|
|
172
|
+
// A handshake whose version is not a number is a broken reporter, not a
|
|
173
|
+
// missing one -- reported as version -1 so it fails the equality check
|
|
174
|
+
// and produces the mismatch error rather than being ignored as noise.
|
|
175
|
+
return typeof line.data?.version === "number" ? line.data.version : -1;
|
|
176
|
+
}
|
|
177
|
+
const REPORTER_MODULE_PATH = fileURLToPath(new URL("./reporter.js", import.meta.url));
|
|
178
|
+
const JEST_REPORTER_MODULE_PATH = fileURLToPath(new URL("./jest-reporter.js", import.meta.url));
|
|
179
|
+
const VITEST_REPORTER_MODULE_PATH = fileURLToPath(new URL("./vitest-reporter.js", import.meta.url));
|
|
180
|
+
const PLAYWRIGHT_REPORTER_MODULE_PATH = fileURLToPath(new URL("./playwright-reporter.js", import.meta.url));
|
|
181
|
+
/**
|
|
182
|
+
* **JUnit has no reporter flag.** `DescryJUnitListener` is discovered by
|
|
183
|
+
* `ServiceLoader` from a `META-INF/services` entry, so this profile's job is
|
|
184
|
+
* to put the listener's classes on the launcher's `--class-path`, not to
|
|
185
|
+
* inject an argv flag the way the other three do (RT-137).
|
|
186
|
+
*
|
|
187
|
+
* Computed from `import.meta.url` like the other reporter paths rather than
|
|
188
|
+
* imported from `reporters/junit/build.mjs`: that file is outside this
|
|
189
|
+
* package's `rootDir`, so a static import would not compile. The duplication
|
|
190
|
+
* is guarded by a test asserting this equals the build's own
|
|
191
|
+
* `LISTENER_CLASSPATH` — a red test rather than a silent drift if either
|
|
192
|
+
* side moves.
|
|
193
|
+
*/
|
|
194
|
+
const JUNIT_LISTENER_CLASSPATH = fileURLToPath(new URL("../reporters/junit/classes", import.meta.url));
|
|
195
|
+
/** `:` everywhere but Windows, where the JVM splits a classpath on `;`. */
|
|
196
|
+
const JUNIT_CLASSPATH_SEPARATOR = process.platform === "win32" ? ";" : ":";
|
|
197
|
+
/**
|
|
198
|
+
* **The .NET logger is loaded by friendly name from an adapter path**, so
|
|
199
|
+
* this profile injects two flags and one RunSettings entry rather than a
|
|
200
|
+
* reporter module (RT-139). Recomputed from `import.meta.url` for the same
|
|
201
|
+
* `rootDir` reason as JUnit's, and guarded by the same equality test.
|
|
202
|
+
*/
|
|
203
|
+
const XUNIT_ADAPTER_PATH = fileURLToPath(new URL("../reporters/xunit/bin/Release/netstandard2.0", import.meta.url));
|
|
204
|
+
const XUNIT_LOGGER_NAME = "descry";
|
|
205
|
+
/**
|
|
206
|
+
* **pytest loads a plugin by importable name, not by path** (RT-107), so
|
|
207
|
+
* this is a `PYTHONPATH` entry, not a reporter module the way node:test's,
|
|
208
|
+
* Jest's and Vitest's are. Computed from `import.meta.url` for the same
|
|
209
|
+
* `rootDir` reason JUnit's and xUnit's paths are.
|
|
210
|
+
*/
|
|
211
|
+
const PYTEST_PLUGIN_DIR = fileURLToPath(new URL("../reporters/pytest", import.meta.url));
|
|
212
|
+
const PYTEST_PLUGIN_MODULE = "descry_pytest_plugin";
|
|
213
|
+
const PYTHONPATH_SEPARATOR = process.platform === "win32" ? ";" : ":";
|
|
214
|
+
/**
|
|
215
|
+
* RSpec's formatter is loaded the same way Jest's/Vitest's reporter modules
|
|
216
|
+
* are — a path handed straight to the runner's own loading flag — so unlike
|
|
217
|
+
* pytest this needs no `reporterEnv`.
|
|
218
|
+
*/
|
|
219
|
+
const RSPEC_FORMATTER_PATH = fileURLToPath(new URL("../reporters/rspec/descry_rspec_formatter.rb", import.meta.url));
|
|
220
|
+
const RSPEC_FORMATTER_CLASS = "DescryRSpecFormatter";
|
|
221
|
+
/**
|
|
222
|
+
* **PHPUnit's extension is registered by class name, and the class must
|
|
223
|
+
* already be declared when `--extension` is processed** (RT-109's own
|
|
224
|
+
* fixture proves this: its `bootstrap.php` requires the file before
|
|
225
|
+
* PHPUnit's `<extensions>` block runs). `--bootstrap <file>` runs after
|
|
226
|
+
* PHPUnit's own autoloading is live — unlike `-d auto_prepend_file`, which
|
|
227
|
+
* runs before it and was measured to fatal on `implements Extension` not
|
|
228
|
+
* yet being defined. Verified directly: `php <phar> --no-configuration
|
|
229
|
+
* --bootstrap <this file> --extension DescryPhpUnitExtension <test file>`
|
|
230
|
+
* emits a full, real envelope with no project config file at all.
|
|
231
|
+
*/
|
|
232
|
+
const PHPUNIT_EXTENSION_PATH = fileURLToPath(new URL("../reporters/phpunit/DescryPhpUnitExtension.php", import.meta.url));
|
|
233
|
+
const PHPUNIT_EXTENSION_CLASS = "DescryPhpUnitExtension";
|
|
234
|
+
/**
|
|
235
|
+
* Minitest's reporter is loaded the same way RSpec's formatter is — a path
|
|
236
|
+
* handed straight to `-r` — so like RSpec, and unlike pytest, this needs no
|
|
237
|
+
* `reporterEnv`.
|
|
238
|
+
*/
|
|
239
|
+
const MINITEST_REPORTER_PATH = fileURLToPath(new URL("../reporters/minitest/descry_minitest_reporter.rb", import.meta.url));
|
|
240
|
+
/**
|
|
241
|
+
* **The only real, caller-declared signal Minitest has** (RT-minitest-
|
|
242
|
+
* discovery-is-not-argv-disambiguable.md). Minitest has no dedicated binary
|
|
243
|
+
* and no distinguishing flag — `ruby test/foo_test.rb` is indistinguishable
|
|
244
|
+
* by argv alone from any other Ruby script; the file's own `require
|
|
245
|
+
* "minitest/autorun"` line is invisible to a collector that only reads the
|
|
246
|
+
* command, and reading the file to check would be exactly the sniffing
|
|
247
|
+
* RT-032 forbids. An explicit `-r minitest/autorun` on the command line is
|
|
248
|
+
* different in kind: it is the caller declaring, in argv, the same fact the
|
|
249
|
+
* bare-file form only states inside the file. Ruby accepts a `-r` library
|
|
250
|
+
* name either as its own token or concatenated (`-rminitest/autorun`); both
|
|
251
|
+
* are checked, since a caller who wrote either spelling declared the same
|
|
252
|
+
* thing.
|
|
253
|
+
*/
|
|
254
|
+
function declaresRequire(args, library) {
|
|
255
|
+
return args.some((arg, index) => (arg === "-r" && args[index + 1] === library) || arg === `-r${library}`);
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* `dotnet test` ends argv parsing at the first bare `--`: everything after
|
|
259
|
+
* it is RunSettings, and a second `--` is an argument rather than a second
|
|
260
|
+
* section.
|
|
261
|
+
*
|
|
262
|
+
* **So flags and settings cannot both be appended.** The first version of
|
|
263
|
+
* this profile appended `--logger descry` to the end of a command that
|
|
264
|
+
* already had a `--`, which handed the logger name to VSTest as a *setting*
|
|
265
|
+
* — a run that would have started, found no logger, and produced no events
|
|
266
|
+
* while looking entirely healthy. Caught by the test written for the
|
|
267
|
+
* setting, not by review.
|
|
268
|
+
*/
|
|
269
|
+
function withDotnetArgs(args, flags, settings) {
|
|
270
|
+
const separator = args.indexOf("--");
|
|
271
|
+
const argv = separator === -1 ? args : args.slice(0, separator);
|
|
272
|
+
const existing = separator === -1 ? [] : args.slice(separator + 1);
|
|
273
|
+
const allSettings = [...existing, ...settings];
|
|
274
|
+
return allSettings.length === 0 ? [...argv, ...flags] : [...argv, ...flags, "--", ...allSettings];
|
|
275
|
+
}
|
|
276
|
+
/** Where the classpath VALUE sits in argv, for `--class-path V`, `--class-path=V`, `--classpath` and `-cp`. */
|
|
277
|
+
function classPathArg(args) {
|
|
278
|
+
const NAMES = ["--class-path", "--classpath", "-cp"];
|
|
279
|
+
for (const [index, arg] of args.entries()) {
|
|
280
|
+
if (NAMES.includes(arg))
|
|
281
|
+
return index + 1 < args.length ? { index: index + 1, inline: false } : null;
|
|
282
|
+
if (NAMES.some((n) => arg.startsWith(`${n}=`)))
|
|
283
|
+
return { index, inline: true };
|
|
284
|
+
}
|
|
285
|
+
return null;
|
|
286
|
+
}
|
|
287
|
+
const hasFlag = (args, flag) => args.some((arg) => arg === flag || arg.startsWith(`${flag}=`));
|
|
288
|
+
/**
|
|
289
|
+
* **Where the reporter flags land in argv is load-bearing for node, not
|
|
290
|
+
* cosmetic.** Measured while building RT-073: `node --test sample.test.mjs
|
|
291
|
+
* --test-reporter=... --test-reporter-destination=...` silently falls back
|
|
292
|
+
* to the default TAP reporter -- once node's `--test` argument parser sees
|
|
293
|
+
* a positional file argument, anything after it stops being read as a
|
|
294
|
+
* flag. `node --test --test-reporter=... --test-reporter-destination=...
|
|
295
|
+
* sample.test.mjs` works. `discoverRunner` only guarantees `--test` is
|
|
296
|
+
* present *somewhere* in `args`, not that it is the last flag before any
|
|
297
|
+
* file arguments (a caller may pass `node --test --concurrency=4
|
|
298
|
+
* some.test.mjs`, and the reporter flags need to land after `--test` but
|
|
299
|
+
* still before `some.test.mjs`) -- so this inserts them immediately after
|
|
300
|
+
* the `--test` token specifically, not at the end of the array.
|
|
301
|
+
*
|
|
302
|
+
* Jest and Vitest both use full CLI parsers (yargs / cac) that read flags
|
|
303
|
+
* anywhere, so theirs are appended. That is a real difference between the
|
|
304
|
+
* three and is why this is per-runner rather than one shared rule.
|
|
305
|
+
*/
|
|
306
|
+
export function withReporterFlags(args) {
|
|
307
|
+
const testIndex = args.indexOf("--test");
|
|
308
|
+
const before = args.slice(0, testIndex + 1);
|
|
309
|
+
const after = args.slice(testIndex + 1);
|
|
310
|
+
return [...before, `--test-reporter=${REPORTER_MODULE_PATH}`, "--test-reporter-destination=stdout", ...after];
|
|
311
|
+
}
|
|
312
|
+
const PROFILES = [
|
|
313
|
+
{
|
|
314
|
+
kind: "node:test",
|
|
315
|
+
// Matched on the executable alone, with `--test` checked in `refuse`
|
|
316
|
+
// rather than here. Matching on both would make `node server.mjs` fall
|
|
317
|
+
// through to the generic "names no runner" message, which is true and
|
|
318
|
+
// useless -- the specific "you gave me node without --test" is the one
|
|
319
|
+
// a caller can act on.
|
|
320
|
+
matches: (baseName) => baseName === "node" || baseName === "node.exe",
|
|
321
|
+
refuse: (args) => {
|
|
322
|
+
if (!args.includes("--test")) {
|
|
323
|
+
return "command does not include --test -- this collector only observes node's own built-in test runner (node --test ...), never a different one inferred from the command shape.";
|
|
324
|
+
}
|
|
325
|
+
return hasFlag(args, "--test-reporter")
|
|
326
|
+
? "command already configures --test-reporter -- this collector needs to own the reporter to observe structured events, and safely co-existing with an unknown existing reporter configuration is not attempted. Remove the existing --test-reporter/--test-reporter-destination flags."
|
|
327
|
+
: null;
|
|
328
|
+
},
|
|
329
|
+
withReporterFlags,
|
|
330
|
+
},
|
|
331
|
+
{
|
|
332
|
+
kind: "jest",
|
|
333
|
+
matches: (baseName) => baseName === "jest" || baseName === "jest.js",
|
|
334
|
+
refuse: (args) => hasFlag(args, "--reporters")
|
|
335
|
+
? "command already configures --reporters -- this collector needs to own the reporter to observe structured events, and Jest's --reporters REPLACES the default set rather than adding to it, so co-existing safely with an unknown one is not attempted. Remove the existing --reporters flag."
|
|
336
|
+
: null,
|
|
337
|
+
// `--testLocationInResults` is not optional decoration: measured,
|
|
338
|
+
// without it every `testCaseResult.location` is `undefined` and every
|
|
339
|
+
// emitted event silently carries no line or column.
|
|
340
|
+
withReporterFlags: (args) => [...args, `--reporters=${JEST_REPORTER_MODULE_PATH}`, "--testLocationInResults"],
|
|
341
|
+
},
|
|
342
|
+
{
|
|
343
|
+
kind: "vitest",
|
|
344
|
+
matches: (baseName) => baseName === "vitest" || baseName === "vitest.js",
|
|
345
|
+
refuse: (args) => {
|
|
346
|
+
// **Vitest defaults to WATCH mode**, and a watching runner never
|
|
347
|
+
// exits -- `pumpLines` would poll a live buffer until the observation
|
|
348
|
+
// window closed and the process was killed, reporting whatever
|
|
349
|
+
// partial run it had seen. Refused by name rather than "fixed" by
|
|
350
|
+
// injecting `run`, because silently changing what a declared command
|
|
351
|
+
// does is the guessing this seam exists to avoid.
|
|
352
|
+
if (!args.includes("run") && !hasFlag(args, "--run")) {
|
|
353
|
+
return "command does not include `run` -- `vitest` alone starts WATCH mode, which never exits, so this collector would observe a partial run and report it as a complete one. Declare `vitest run ...` explicitly rather than have this collector rewrite the command.";
|
|
354
|
+
}
|
|
355
|
+
return hasFlag(args, "--reporter")
|
|
356
|
+
? "command already configures --reporter -- this collector needs to own the reporter to observe structured events, and co-existing safely with an unknown one is not attempted. Remove the existing --reporter flag."
|
|
357
|
+
: null;
|
|
358
|
+
},
|
|
359
|
+
// `--includeTaskLocation` is the counterpart of Jest's
|
|
360
|
+
// `--testLocationInResults`, and silent in exactly the same way:
|
|
361
|
+
// measured, without it every `testCase.location` is `undefined`.
|
|
362
|
+
withReporterFlags: (args) => [...args, `--reporter=${VITEST_REPORTER_MODULE_PATH}`, "--includeTaskLocation"],
|
|
363
|
+
},
|
|
364
|
+
{
|
|
365
|
+
kind: "junit",
|
|
366
|
+
// `java` alone, with the specifics in `refuse` — the node:test
|
|
367
|
+
// precedent. `java -jar app.jar` is an ordinary command and the useful
|
|
368
|
+
// answer for it is "that is not the console launcher", not the generic
|
|
369
|
+
// "names no runner this collector observes".
|
|
370
|
+
matches: (baseName) => baseName === "java" || baseName === "java.exe",
|
|
371
|
+
refuse: (args) => {
|
|
372
|
+
const isConsoleLauncher = args.some((a) => /junit-platform-console-standalone.*\.jar$/.test(a)) ||
|
|
373
|
+
args.includes("org.junit.platform.console.ConsoleLauncher");
|
|
374
|
+
if (!isConsoleLauncher) {
|
|
375
|
+
return "command runs `java` but does not name the JUnit Platform Console Launcher — this collector observes JUnit through that launcher (java -jar junit-platform-console-standalone-<version>.jar execute ...), never a JVM process inferred from the command shape.";
|
|
376
|
+
}
|
|
377
|
+
if (classPathArg(args) === null) {
|
|
378
|
+
return ("command has no --class-path to extend, and this collector will not add one. The listener is ServiceLoader-discovered from the classpath, so it must join an existing one; " +
|
|
379
|
+
"introducing the first --class-path would change how the launcher resolves the tests themselves rather than adding to it. Declare --class-path explicitly.");
|
|
380
|
+
}
|
|
381
|
+
return null;
|
|
382
|
+
},
|
|
383
|
+
// **Appended to the existing value, never passed as a second flag.** A
|
|
384
|
+
// second --class-path does not merge, it replaces — which would take
|
|
385
|
+
// the caller's own test classes off the path and report zero tests
|
|
386
|
+
// found: a green run that observed nothing.
|
|
387
|
+
withReporterFlags: (args) => {
|
|
388
|
+
const found = classPathArg(args);
|
|
389
|
+
if (found === null)
|
|
390
|
+
return args;
|
|
391
|
+
const next = [...args];
|
|
392
|
+
next[found.index] = `${next[found.index]}${JUNIT_CLASSPATH_SEPARATOR}${JUNIT_LISTENER_CLASSPATH}`;
|
|
393
|
+
return next;
|
|
394
|
+
},
|
|
395
|
+
},
|
|
396
|
+
{
|
|
397
|
+
kind: "playwright",
|
|
398
|
+
matches: (baseName) => baseName === "playwright" || baseName === "playwright.js",
|
|
399
|
+
refuse: (args) => {
|
|
400
|
+
// `playwright` alone is the CLI (codegen, install, show-report); only
|
|
401
|
+
// `test` runs tests. Named rather than left to the generic message,
|
|
402
|
+
// the node:test precedent.
|
|
403
|
+
if (!args.includes("test")) {
|
|
404
|
+
return "command runs `playwright` but not `playwright test` — this collector observes the Playwright *test runner*, not the CLI's other subcommands (codegen, install, show-report).";
|
|
405
|
+
}
|
|
406
|
+
if (hasFlag(args, "--reporter")) {
|
|
407
|
+
return "command already configures --reporter — this collector needs to own the reporter to observe structured events, and co-existing safely with an unknown one is not attempted. Remove the existing --reporter flag.";
|
|
408
|
+
}
|
|
409
|
+
return null;
|
|
410
|
+
},
|
|
411
|
+
// **No location flag, and that is the finding.** Playwright is the only
|
|
412
|
+
// runner on this seam that populates `TestCase.location` unasked
|
|
413
|
+
// (RT-144); every other one either needs a flag or cannot do it at all.
|
|
414
|
+
withReporterFlags: (args) => [...args, `--reporter=${PLAYWRIGHT_REPORTER_MODULE_PATH}`],
|
|
415
|
+
},
|
|
416
|
+
{
|
|
417
|
+
kind: "xunit",
|
|
418
|
+
matches: (baseName) => baseName === "dotnet" || baseName === "dotnet.exe",
|
|
419
|
+
refuse: (args) => {
|
|
420
|
+
if (args[0] !== "test") {
|
|
421
|
+
return "command runs `dotnet` but not `dotnet test` — this collector observes .NET tests through the VSTest logger interface, never a dotnet subcommand inferred from the command shape.";
|
|
422
|
+
}
|
|
423
|
+
if (hasFlag(args, "--logger")) {
|
|
424
|
+
return "command already configures --logger — this collector needs to own the logger to observe structured events, and co-existing safely with an unknown one is not attempted. Remove the existing --logger flag.";
|
|
425
|
+
}
|
|
426
|
+
if (hasFlag(args, "--test-adapter-path")) {
|
|
427
|
+
return "command already configures --test-adapter-path — the logger is loaded from that path, so a second one would decide which logger is found. Remove the existing --test-adapter-path flag.";
|
|
428
|
+
}
|
|
429
|
+
// Not silently left alone: if the caller has already declared this
|
|
430
|
+
// setting we cannot know whether they set it false, and a false value
|
|
431
|
+
// empties `TestCase.CodeFilePath` — every event silently carrying no
|
|
432
|
+
// location, which is Jest's `--testLocationInResults` failure exactly
|
|
433
|
+
// (RT-123) and invisible in a green run.
|
|
434
|
+
if (args.some((a) => a.includes("RunConfiguration.CollectSourceInformation"))) {
|
|
435
|
+
return "command already declares RunConfiguration.CollectSourceInformation — this collector must own it, because a false value empties TestCase.CodeFilePath and every emitted event then silently carries no source location. Remove it.";
|
|
436
|
+
}
|
|
437
|
+
return null;
|
|
438
|
+
},
|
|
439
|
+
withReporterFlags: (args) => withDotnetArgs(args, ["--logger", XUNIT_LOGGER_NAME, "--test-adapter-path", XUNIT_ADAPTER_PATH], ["RunConfiguration.CollectSourceInformation=true"]),
|
|
440
|
+
},
|
|
441
|
+
{
|
|
442
|
+
kind: "pytest",
|
|
443
|
+
// `pytest ...` directly, or `python -m pytest ...` (and the `python3`/
|
|
444
|
+
// `.exe` spellings) — the node:test precedent of matching the
|
|
445
|
+
// interpreter and checking the invocation shape in `refuse`, since
|
|
446
|
+
// `python -m pytest` and `python other_script.py` share an executable.
|
|
447
|
+
matches: (baseName, args) => baseName === "pytest" ||
|
|
448
|
+
baseName === "pytest.exe" ||
|
|
449
|
+
((baseName === "python" || baseName === "python3" || baseName === "python.exe") &&
|
|
450
|
+
args[0] === "-m" &&
|
|
451
|
+
args[1] === "pytest"),
|
|
452
|
+
refuse: (args) => hasFlag(args, "-p")
|
|
453
|
+
? "command already configures -p (a plugin) -- this collector needs to own the reporter plugin to observe structured events, and co-existing safely with an unknown existing one is not attempted. Remove the existing -p flag."
|
|
454
|
+
: null,
|
|
455
|
+
withReporterFlags: (args) => [...args, "-p", PYTEST_PLUGIN_MODULE],
|
|
456
|
+
// Prepended so the caller's own PYTHONPATH (if any) still resolves —
|
|
457
|
+
// this only needs to ADD the plugin's directory, never replace what a
|
|
458
|
+
// caller already declared.
|
|
459
|
+
reporterEnv: () => ({
|
|
460
|
+
PYTHONPATH: [PYTEST_PLUGIN_DIR, process.env["PYTHONPATH"]].filter((v) => Boolean(v)).join(PYTHONPATH_SEPARATOR),
|
|
461
|
+
}),
|
|
462
|
+
},
|
|
463
|
+
{
|
|
464
|
+
kind: "rspec",
|
|
465
|
+
// `rspec ...` directly, or `ruby <path/to/rspec> ...` — RSpec's own
|
|
466
|
+
// installed binary is itself a Ruby script, and some environments (this
|
|
467
|
+
// one included — see rspec-reporter.test.ts) invoke it through an
|
|
468
|
+
// explicit `ruby` interpreter rather than relying on its shebang. Told
|
|
469
|
+
// apart from an arbitrary `ruby script.rb` the same way pytest's `python
|
|
470
|
+
// -m pytest` is told apart from `python other_script.py`: by the
|
|
471
|
+
// declared script name, never by sniffing its contents.
|
|
472
|
+
matches: (baseName, args) => baseName === "rspec" || ((baseName === "ruby" || baseName === "ruby.exe") && /(^|[\\/])rspec$/.test(args[0] ?? "")),
|
|
473
|
+
refuse: (args) => hasFlag(args, "-f") || hasFlag(args, "--format")
|
|
474
|
+
? "command already configures -f/--format -- this collector needs to own the formatter to observe structured events, and co-existing safely with an unknown existing one is not attempted. Remove the existing -f/--format flag."
|
|
475
|
+
: null,
|
|
476
|
+
// `-r` is additive in RSpec (each use adds a require, never replaces),
|
|
477
|
+
// so unlike `-f` it is always safe to add ours alongside whatever the
|
|
478
|
+
// caller already declared.
|
|
479
|
+
withReporterFlags: (args) => [...args, "-r", RSPEC_FORMATTER_PATH, "-f", RSPEC_FORMATTER_CLASS],
|
|
480
|
+
},
|
|
481
|
+
{
|
|
482
|
+
kind: "phpunit",
|
|
483
|
+
// `phpunit ...` directly (the composer-installed binary, itself a PHP
|
|
484
|
+
// script with a shebang), or `php <path/to/phpunit-or-.phar> ...` — the
|
|
485
|
+
// same dual shape as rspec's, for the same reason: some environments
|
|
486
|
+
// invoke it through an explicit `php` interpreter (this repo's own
|
|
487
|
+
// phpunit-reporter.test.ts does, against the pinned phar) rather than
|
|
488
|
+
// relying on the shebang.
|
|
489
|
+
// The pinned phar this repo's own reporter test runs is
|
|
490
|
+
// `phpunit-11.5.56.phar`, not a bare `phpunit.phar` — Composer's
|
|
491
|
+
// installed binary and the project's own downloaded phars are both
|
|
492
|
+
// routinely version-suffixed, so the match allows an optional
|
|
493
|
+
// `-<version>` between the name and the extension.
|
|
494
|
+
matches: (baseName, args) => baseName === "phpunit" ||
|
|
495
|
+
((baseName === "php" || baseName === "php.exe") && /(^|[\\/])phpunit(-[^\\/]*)?(\.phar)?$/.test(args[0] ?? "")),
|
|
496
|
+
refuse: (args) => {
|
|
497
|
+
if (hasFlag(args, "--bootstrap")) {
|
|
498
|
+
return "command already configures --bootstrap -- this collector needs to own the bootstrap to register its extension, and PHPUnit accepts only one. Remove the existing --bootstrap flag.";
|
|
499
|
+
}
|
|
500
|
+
if (hasFlag(args, "--extension") || args.includes("--no-extensions")) {
|
|
501
|
+
return "command already configures --extension/--no-extensions -- this collector needs to register its own extension to observe structured events, and co-existing safely with an unknown existing configuration is not attempted. Remove the existing flag.";
|
|
502
|
+
}
|
|
503
|
+
return null;
|
|
504
|
+
},
|
|
505
|
+
// **`--bootstrap` before `--extension`, not the other way round** — the
|
|
506
|
+
// extension class must already be `require`-d (which `--bootstrap`
|
|
507
|
+
// does) before PHPUnit resolves the class name `--extension` names.
|
|
508
|
+
// Verified directly against the real phar; see the constant's own
|
|
509
|
+
// comment for the command.
|
|
510
|
+
withReporterFlags: (args) => [...args, "--bootstrap", PHPUNIT_EXTENSION_PATH, "--extension", PHPUNIT_EXTENSION_CLASS],
|
|
511
|
+
},
|
|
512
|
+
{
|
|
513
|
+
kind: "minitest",
|
|
514
|
+
// The interpreter alone is not enough — every `ruby <file>` command
|
|
515
|
+
// shares it, tests and non-tests alike. `declaresRequire` is what
|
|
516
|
+
// actually distinguishes a Minitest invocation, so it is checked here in
|
|
517
|
+
// `matches` rather than deferred to `refuse`, unlike every other profile
|
|
518
|
+
// on this seam where the executable name alone is sufficient signal.
|
|
519
|
+
matches: (baseName, args) => (baseName === "ruby" || baseName === "ruby.exe") && declaresRequire(args, "minitest/autorun"),
|
|
520
|
+
// `-r` is additive in Ruby (each use adds a require, never replaces),
|
|
521
|
+
// the same fact RSpec's own profile relies on, so there is nothing for
|
|
522
|
+
// a second `-r` to conflict with.
|
|
523
|
+
refuse: () => null,
|
|
524
|
+
// **Inserted right after the caller's own declared `-r`, never
|
|
525
|
+
// appended.** Unlike `rspec`'s own binary, which parses its flags with
|
|
526
|
+
// a full option parser regardless of position, `ruby` the interpreter
|
|
527
|
+
// stops reading flags at the first positional argument — the test
|
|
528
|
+
// file. Appending at the end (measured: `ruby -r minitest/autorun
|
|
529
|
+
// <file> -r <reporter>`) hands `-r <reporter>` to the *script* as
|
|
530
|
+
// `ARGV`, which Minitest's own option parser then rejects outright
|
|
531
|
+
// (`invalid option: -r`) rather than loading it — the same insertion-
|
|
532
|
+
// point lesson `node --test`'s own profile already learned, for the
|
|
533
|
+
// same reason: a positional argument ends flag parsing.
|
|
534
|
+
withReporterFlags: (args) => {
|
|
535
|
+
const index = args.findIndex((arg, i) => (arg === "-r" && args[i + 1] === "minitest/autorun") || arg === "-rminitest/autorun");
|
|
536
|
+
const insertAt = args[index] === "-r" ? index + 2 : index + 1;
|
|
537
|
+
return [...args.slice(0, insertAt), "-r", MINITEST_REPORTER_PATH, ...args.slice(insertAt)];
|
|
538
|
+
},
|
|
539
|
+
},
|
|
540
|
+
];
|
|
541
|
+
/**
|
|
542
|
+
* Exported on its own so a caller can validate a configured command before
|
|
543
|
+
* ever attempting to spawn it — the same "refuse atomically before anything
|
|
544
|
+
* is spawned" shape `validateServicesConfiguration` already established.
|
|
545
|
+
*
|
|
546
|
+
* **Declared, never inferred** (RT-032's precedent): this reads the command
|
|
547
|
+
* the caller wrote and nothing else. It does not open `package.json`, a
|
|
548
|
+
* lockfile, or a config file to guess which runner a project uses, and a
|
|
549
|
+
* command it does not recognise gets a named refusal rather than a
|
|
550
|
+
* best-effort guess.
|
|
551
|
+
*/
|
|
552
|
+
export function discoverRunner(rawCommand) {
|
|
553
|
+
const tokenized = tokenizeCommand(rawCommand);
|
|
554
|
+
if (!tokenized.ok) {
|
|
555
|
+
return { ok: false, reason: tokenized.reason };
|
|
556
|
+
}
|
|
557
|
+
const { command, args } = tokenized;
|
|
558
|
+
const baseName = command.split(/[\\/]/).pop() ?? command;
|
|
559
|
+
// **`npx jest` is refused by name, not supported and not lumped in with
|
|
560
|
+
// an unrecognised command.** Unwrapping it means deciding which tokens
|
|
561
|
+
// belong to npx and which to the runner, and `npx -p vitest@4 vitest run`
|
|
562
|
+
// makes that genuinely ambiguous -- a wrong split silently drops a flag
|
|
563
|
+
// the caller wrote. The runner's own binary is one path away
|
|
564
|
+
// (`node_modules/.bin/jest`), so the refusal names the fix rather than
|
|
565
|
+
// guessing at one.
|
|
566
|
+
if (baseName === "npx" || baseName === "npx.cmd") {
|
|
567
|
+
return {
|
|
568
|
+
ok: false,
|
|
569
|
+
reason: `"${rawCommand}" runs the runner through npx, whose own arguments cannot be told apart from the runner's ` +
|
|
570
|
+
"without guessing (`npx -p vitest@4 vitest run`). Declare the runner binary directly -- e.g. " +
|
|
571
|
+
"`node_modules/.bin/jest ...` or `node_modules/.bin/vitest run ...`.",
|
|
572
|
+
};
|
|
573
|
+
}
|
|
574
|
+
const profile = PROFILES.find((candidate) => candidate.matches(baseName, args));
|
|
575
|
+
if (profile === undefined) {
|
|
576
|
+
return {
|
|
577
|
+
ok: false,
|
|
578
|
+
reason: `"${rawCommand}" names no runner this collector observes. Supported, and each declared explicitly: ` +
|
|
579
|
+
`\`node ... --test ...\`, \`jest ...\`, \`vitest run ...\`. ` +
|
|
580
|
+
"A command is never matched by sniffing package.json or a lockfile.",
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
const refusal = profile.refuse(args);
|
|
584
|
+
if (refusal !== null) {
|
|
585
|
+
return { ok: false, reason: refusal };
|
|
586
|
+
}
|
|
587
|
+
return { ok: true, kind: profile.kind, command, args };
|
|
588
|
+
}
|
|
589
|
+
export function reporterFlagsFor(kind, args) {
|
|
590
|
+
return PROFILES.find((profile) => profile.kind === kind).withReporterFlags(args);
|
|
591
|
+
}
|
|
592
|
+
/** `{}` for every profile that has no `reporterEnv` — `start()`'s merge then adds nothing. */
|
|
593
|
+
export function reporterEnvFor(kind, args) {
|
|
594
|
+
const profile = PROFILES.find((candidate) => candidate.kind === kind);
|
|
595
|
+
return profile.reporterEnv?.(args) ?? {};
|
|
596
|
+
}
|
|
597
|
+
export function isNodeTestEvent(value) {
|
|
598
|
+
return typeof value === "object" && value !== null && "type" in value && "data" in value;
|
|
599
|
+
}
|
|
600
|
+
function sourceLocationOf(data) {
|
|
601
|
+
return {
|
|
602
|
+
file: data.file ?? null,
|
|
603
|
+
line: data.line ?? null,
|
|
604
|
+
column: data.column ?? null,
|
|
605
|
+
functionName: null,
|
|
606
|
+
reliability: "self-contained",
|
|
607
|
+
resolvedVia: null,
|
|
608
|
+
};
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* §17's "TEST_CASE graph mapping" (RT-076) found a real mismatch before it
|
|
612
|
+
* ever reached graph-correlator: `adapter-typescript`'s TEST_CASE node
|
|
613
|
+
* `name` is `[...suite, title].join(" ")` -- the full `describe()` chain,
|
|
614
|
+
* space-joined, ahead of the test's own title. node:test's own reporter
|
|
615
|
+
* events do NOT do this: a nested test's `test:pass`/`test:fail` carries
|
|
616
|
+
* only its own innermost title (`"deletes an invoice"`), never
|
|
617
|
+
* `"invoice deletion deletes an invoice"` -- confirmed directly against a
|
|
618
|
+
* real `describe()`/`it()` file before assuming otherwise. A naive
|
|
619
|
+
* `resolveTestCaseNode(driver, { testName: event.data.name, ... }, repo)`
|
|
620
|
+
* would silently `not-found` every nested test while looking correct for
|
|
621
|
+
* every *top-level* one (this package's own earlier fixtures happened to
|
|
622
|
+
* have none nested, which is exactly how this would have shipped unnoticed).
|
|
623
|
+
*
|
|
624
|
+
* `describe`/`suite` blocks fire their own `test:start`/`test:pass` events
|
|
625
|
+
* too, at one nesting level shallower than what they contain (also
|
|
626
|
+
* confirmed directly, two levels deep) -- so the ancestor chain is
|
|
627
|
+
* reconstructable purely from the stream this collector already reads,
|
|
628
|
+
* with no new node:test API. `SuiteStack` tracks it: on every `test:start`,
|
|
629
|
+
* truncate to the reported `nesting` (a same-or-shallower start means any
|
|
630
|
+
* deeper siblings from a previous branch are done) then record the name at
|
|
631
|
+
* that depth. `qualify(nesting, name)` joins everything shallower with the
|
|
632
|
+
* given name -- matching the adapter's own convention exactly, so a
|
|
633
|
+
* caller doing graph resolution uses `payload.qualifiedName`, never
|
|
634
|
+
* `payload.name` (which stays the plain, human-readable title).
|
|
635
|
+
*/
|
|
636
|
+
class SuiteStack {
|
|
637
|
+
#names = [];
|
|
638
|
+
observeStart(nesting, name) {
|
|
639
|
+
this.#names.length = nesting;
|
|
640
|
+
this.#names[nesting] = name;
|
|
641
|
+
}
|
|
642
|
+
qualify(nesting, name) {
|
|
643
|
+
return [...this.#names.slice(0, nesting), name].join(" ");
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
/**
|
|
647
|
+
* The one convention `t.diagnostic()` messages are read for: a test author
|
|
648
|
+
* writing exactly `requestId: <value>` or `traceId: <value>` (case-
|
|
649
|
+
* sensitive on the key, matching the Evidence field names exactly, so a
|
|
650
|
+
* typo is a miss rather than a silent near-match). Anything else --
|
|
651
|
+
* `t.diagnostic()` is a general-purpose node:test API, not exclusively
|
|
652
|
+
* this collector's -- is ignored, not treated as malformed input.
|
|
653
|
+
*/
|
|
654
|
+
function parseCorrelationDiagnostic(message) {
|
|
655
|
+
const match = /^(requestId|traceId):\s*(.+)$/.exec(message.trim());
|
|
656
|
+
if (match === null)
|
|
657
|
+
return null;
|
|
658
|
+
const [, kind, value] = match;
|
|
659
|
+
if (kind === undefined || value === undefined || value.trim() === "")
|
|
660
|
+
return null;
|
|
661
|
+
return { kind: kind, value: value.trim() };
|
|
662
|
+
}
|
|
663
|
+
export function createEventTranslator(emit, service, processId) {
|
|
664
|
+
let pending = null;
|
|
665
|
+
let testsObserved = 0;
|
|
666
|
+
let idsPresent = 0;
|
|
667
|
+
let notRun = 0;
|
|
668
|
+
const suiteStack = new SuiteStack();
|
|
669
|
+
function flush() {
|
|
670
|
+
if (pending === null)
|
|
671
|
+
return;
|
|
672
|
+
const { eventType, name, qualifiedName, durationMs, error, location, requestId, traceId, skipReason } = pending;
|
|
673
|
+
pending = null;
|
|
674
|
+
// `testsObserved` counts results that represent a test that RAN.
|
|
675
|
+
// Folding skips into it would make "3 of 40 correlated" a ratio over a
|
|
676
|
+
// denominator that includes tests which could not have correlated
|
|
677
|
+
// with anything — the exact overclaim RT-074 added the denominator to
|
|
678
|
+
// prevent, one level along.
|
|
679
|
+
if (eventType === "TEST_SKIPPED")
|
|
680
|
+
notRun++;
|
|
681
|
+
else
|
|
682
|
+
testsObserved++;
|
|
683
|
+
if (requestId !== null || traceId !== null)
|
|
684
|
+
idsPresent++;
|
|
685
|
+
emit({
|
|
686
|
+
timestamp: new Date().toISOString(),
|
|
687
|
+
source: "test-runner",
|
|
688
|
+
service: service ?? null,
|
|
689
|
+
process: processId,
|
|
690
|
+
traceId,
|
|
691
|
+
requestId,
|
|
692
|
+
correlationId: null,
|
|
693
|
+
graphNodeId: null,
|
|
694
|
+
stackTrace: null,
|
|
695
|
+
confidence: 1,
|
|
696
|
+
redactionStatus: "pending-redaction",
|
|
697
|
+
collectorVersion: COLLECTOR_VERSION,
|
|
698
|
+
eventType,
|
|
699
|
+
payload: eventType === "TEST_FAILED"
|
|
700
|
+
? { name, qualifiedName, durationMs, error }
|
|
701
|
+
: eventType === "TEST_SKIPPED"
|
|
702
|
+
? // `durationMs` is null for a skip by construction and carrying it
|
|
703
|
+
// would invite a consumer to average it in. `skipReason` replaces
|
|
704
|
+
// it: the one thing there is to say about a test that did not run.
|
|
705
|
+
{ name, qualifiedName, skipReason: skipReason ?? "unspecified" }
|
|
706
|
+
: { name, qualifiedName, durationMs },
|
|
707
|
+
sourceLocation: location,
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
function handle(event) {
|
|
711
|
+
const { data } = event;
|
|
712
|
+
const name = data.name ?? "(unnamed test)";
|
|
713
|
+
const nesting = data.nesting ?? 0;
|
|
714
|
+
switch (event.type) {
|
|
715
|
+
case "test:start": {
|
|
716
|
+
flush(); // whatever was pending belongs to the PREVIOUS test -- nothing more can merge into it now
|
|
717
|
+
// Recorded BEFORE emitting: a describe/suite's own test:start is
|
|
718
|
+
// itself an ancestor for whatever test:start events nest under it
|
|
719
|
+
// next, and the stack has to reflect that by the time they arrive.
|
|
720
|
+
suiteStack.observeStart(nesting, name);
|
|
721
|
+
emit({
|
|
722
|
+
timestamp: new Date().toISOString(),
|
|
723
|
+
source: "test-runner",
|
|
724
|
+
service: service ?? null,
|
|
725
|
+
process: processId,
|
|
726
|
+
traceId: null,
|
|
727
|
+
requestId: null,
|
|
728
|
+
correlationId: null,
|
|
729
|
+
graphNodeId: null,
|
|
730
|
+
stackTrace: null,
|
|
731
|
+
confidence: 1,
|
|
732
|
+
redactionStatus: "pending-redaction",
|
|
733
|
+
collectorVersion: COLLECTOR_VERSION,
|
|
734
|
+
eventType: "TEST_STARTED",
|
|
735
|
+
payload: { name, qualifiedName: data.qualifiedName ?? suiteStack.qualify(nesting, name) },
|
|
736
|
+
sourceLocation: sourceLocationOf(data),
|
|
737
|
+
});
|
|
738
|
+
return;
|
|
739
|
+
}
|
|
740
|
+
case "test:pass":
|
|
741
|
+
case "test:fail":
|
|
742
|
+
flush(); // safety: should already be null here, but never silently drop a prior pending result
|
|
743
|
+
// **A `describe()` block that passed is not a test that passed.**
|
|
744
|
+
// node:test emits `test:pass` for a container as well as for each
|
|
745
|
+
// test inside it (measured), so a three-test file with one
|
|
746
|
+
// `describe()` reported four passes — inflating the pass count and
|
|
747
|
+
// inflating RT-074's own denominator, the number added precisely so
|
|
748
|
+
// "3 of 40 correlated" could not overclaim. It would also resolve
|
|
749
|
+
// to a `TEST_CASE` node that no adapter mints, since adapters mint
|
|
750
|
+
// them for tests and not for containers.
|
|
751
|
+
//
|
|
752
|
+
// A suite's **failure** is kept: it means a `before`/`after` hook
|
|
753
|
+
// threw, which is a real failure and the only signal that a whole
|
|
754
|
+
// block never ran. Suppressing that to make the rule symmetrical
|
|
755
|
+
// would trade an overcount for a silence.
|
|
756
|
+
if (data.isSuite === true && event.type === "test:pass")
|
|
757
|
+
return;
|
|
758
|
+
// **A test that did not run is not a pass.** Emitted as its own
|
|
759
|
+
// `TEST_SKIPPED` (RT-124) rather than counted-and-discarded, which
|
|
760
|
+
// is what RT-123 shipped: a bare counter carries no
|
|
761
|
+
// `sourceLocation`, no `qualifiedName` and no graph node, so "the
|
|
762
|
+
// function you changed has a test and it is disabled" could not be
|
|
763
|
+
// expressed at all. `notRun` is kept as the denominator.
|
|
764
|
+
//
|
|
765
|
+
// Held rather than emitted immediately, exactly like a pass or a
|
|
766
|
+
// failure, so a `t.diagnostic()` on a skipped test still merges —
|
|
767
|
+
// node:test emits diagnostics after the result for skipped tests
|
|
768
|
+
// too, and a separate emit path here would be a second shape that
|
|
769
|
+
// could drift from the one above.
|
|
770
|
+
if (data.skip === true || data.todo === true) {
|
|
771
|
+
pending = {
|
|
772
|
+
eventType: "TEST_SKIPPED",
|
|
773
|
+
name,
|
|
774
|
+
qualifiedName: data.qualifiedName ?? suiteStack.qualify(nesting, name),
|
|
775
|
+
durationMs: null,
|
|
776
|
+
error: null,
|
|
777
|
+
location: sourceLocationOf(data),
|
|
778
|
+
requestId: null,
|
|
779
|
+
traceId: null,
|
|
780
|
+
// Passed through, never derived. A translator that inferred
|
|
781
|
+
// `disabled` from `skip === true` would state it for Vitest,
|
|
782
|
+
// which sets that flag for `.todo` as well and cannot tell them
|
|
783
|
+
// apart — a false fact manufactured by the layer whose job is
|
|
784
|
+
// to be runner-agnostic.
|
|
785
|
+
skipReason: data.skipReason ?? "unspecified",
|
|
786
|
+
};
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
pending = {
|
|
790
|
+
eventType: event.type === "test:pass" ? "TEST_PASSED" : "TEST_FAILED",
|
|
791
|
+
name,
|
|
792
|
+
// A reporter that can name the full chain itself is believed over
|
|
793
|
+
// this collector's reconstruction of it. `SuiteStack` exists
|
|
794
|
+
// because node:test's events carry only the innermost title;
|
|
795
|
+
// Jest and Vitest carry the chain, and re-deriving it from
|
|
796
|
+
// `nesting` when the runner already said so would be a second
|
|
797
|
+
// source of truth that can disagree.
|
|
798
|
+
qualifiedName: data.qualifiedName ?? suiteStack.qualify(nesting, name),
|
|
799
|
+
durationMs: data.details?.duration_ms ?? null,
|
|
800
|
+
error: event.type === "test:fail" ? (data.details?.error ?? null) : null,
|
|
801
|
+
location: sourceLocationOf(data),
|
|
802
|
+
requestId: null,
|
|
803
|
+
traceId: null,
|
|
804
|
+
};
|
|
805
|
+
return;
|
|
806
|
+
case "test:diagnostic": {
|
|
807
|
+
if (data.file === undefined) {
|
|
808
|
+
// A suite-level summary line ("tests 3", "pass 1", ...) -- these
|
|
809
|
+
// never carry a file, and their arrival means every per-test
|
|
810
|
+
// diagnostic for this file has already been seen.
|
|
811
|
+
flush();
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
814
|
+
if (pending === null || data.message === undefined)
|
|
815
|
+
return;
|
|
816
|
+
const parsed = parseCorrelationDiagnostic(data.message);
|
|
817
|
+
if (parsed === null)
|
|
818
|
+
return;
|
|
819
|
+
pending[parsed.kind] = parsed.value;
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
default:
|
|
823
|
+
// test:enqueue/dequeue/complete/plan -- not yet translated.
|
|
824
|
+
// `test:complete` in particular duplicates pass/fail and also
|
|
825
|
+
// wraps the whole file as its own pseudo-test; deliberately not
|
|
826
|
+
// read, so the file-level wrapper never becomes a fourth,
|
|
827
|
+
// unasked-for TEST_* event.
|
|
828
|
+
return;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
return { handle, flush, coverage: () => ({ testsObserved, idsPresent, notRun }) };
|
|
832
|
+
}
|
|
833
|
+
/**
|
|
834
|
+
* Feeds complete NDJSON lines from `managed`'s growing output buffer to
|
|
835
|
+
* `onLine`, exactly once each, as they arrive -- streaming-first, not
|
|
836
|
+
* batched until the process exits. A narrower, single-purpose relative of
|
|
837
|
+
* `orchestrator`'s `createPollingProcessOutputSource` (same buffer-growth
|
|
838
|
+
* assumption, same "ends only once the process is finished AND the buffer
|
|
839
|
+
* stopped growing" ordering) rather than a shared dependency on it: this
|
|
840
|
+
* collector only ever needs complete lines fed to a callback, not the
|
|
841
|
+
* general `ProcessOutputSource` shape, and taking a dependency on
|
|
842
|
+
* `@descryy/runtime-orchestrator` -- the assembly layer -- from a collector
|
|
843
|
+
* would invert the direction every other package in this repo depends in.
|
|
844
|
+
*/
|
|
845
|
+
export async function pumpLines(managed, onLine, pollIntervalMs = 20) {
|
|
846
|
+
const splitter = new LineSplitter();
|
|
847
|
+
let consumed = 0;
|
|
848
|
+
let finished = false;
|
|
849
|
+
void managed.waitForExit().then(() => {
|
|
850
|
+
finished = true;
|
|
851
|
+
});
|
|
852
|
+
for (;;) {
|
|
853
|
+
const whole = managed.readOutput();
|
|
854
|
+
if (whole.length > consumed) {
|
|
855
|
+
const chunk = whole.slice(consumed);
|
|
856
|
+
consumed = whole.length;
|
|
857
|
+
for (const line of splitter.push(chunk)) {
|
|
858
|
+
onLine(line);
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
if (finished && whole.length === consumed) {
|
|
862
|
+
for (const line of splitter.flush()) {
|
|
863
|
+
onLine(line);
|
|
864
|
+
}
|
|
865
|
+
return;
|
|
866
|
+
}
|
|
867
|
+
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
function computeCapabilities() {
|
|
871
|
+
const reason = "TestRunnerCollector observes test-runner events (started/passed/failed) only -- the capability model has no field for that yet.";
|
|
872
|
+
const status = { availability: "unavailable", reason };
|
|
873
|
+
return {
|
|
874
|
+
domObservation: status,
|
|
875
|
+
consoleObservation: status,
|
|
876
|
+
networkObservation: status,
|
|
877
|
+
backendLogAccess: status,
|
|
878
|
+
distributedTrace: status,
|
|
879
|
+
sourceMapping: { availability: "available", reason: null },
|
|
880
|
+
stackCapture: status,
|
|
881
|
+
processLifecycle: status,
|
|
882
|
+
databaseObservation: status,
|
|
883
|
+
externalServiceObservation: status,
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
export function createTestRunnerCollector(options) {
|
|
887
|
+
const capabilities = computeCapabilities();
|
|
888
|
+
let context = null;
|
|
889
|
+
let managed = null;
|
|
890
|
+
let completion = Promise.resolve();
|
|
891
|
+
let translator = null;
|
|
892
|
+
return {
|
|
893
|
+
collectorId: "test-runner-collector",
|
|
894
|
+
async start(ctx) {
|
|
895
|
+
context = ctx;
|
|
896
|
+
const discovery = discoverRunner(options.configuration.command);
|
|
897
|
+
if (!discovery.ok) {
|
|
898
|
+
return { available: false, reason: discovery.reason };
|
|
899
|
+
}
|
|
900
|
+
const processId = `test-runner-${Date.now()}`;
|
|
901
|
+
managed = spawnProcess({
|
|
902
|
+
processId,
|
|
903
|
+
command: discovery.command,
|
|
904
|
+
args: reporterFlagsFor(discovery.kind, discovery.args),
|
|
905
|
+
cwd: options.configuration.cwd,
|
|
906
|
+
// `NODE_TEST_CONTEXT` (set by node's own `--test` on itself, inherited
|
|
907
|
+
// by every child process unless cleared) is load-bearing here, and it
|
|
908
|
+
// took a real, silent, empty-output failure to find: when this
|
|
909
|
+
// collector's own test suite -- itself running under `node --test`
|
|
910
|
+
// -- spawns a NESTED `node --test` child, the child sees this env var
|
|
911
|
+
// inherited from the parent and behaves as though it is a
|
|
912
|
+
// sub-process of an existing test run, producing NO stdout at all
|
|
913
|
+
// regardless of --test-reporter. `undefined` here removes the
|
|
914
|
+
// inherited key (spawnProcess's env merge, like child_process.spawn
|
|
915
|
+
// itself, drops undefined-valued keys rather than stringifying
|
|
916
|
+
// them) so a spawned runner always starts as its own independent,
|
|
917
|
+
// top-level `node --test` invocation -- which is what it actually
|
|
918
|
+
// is, whether or not this collector's own process happens to be
|
|
919
|
+
// running under a test runner too.
|
|
920
|
+
//
|
|
921
|
+
// `reporterEnvFor` adds nothing for every runner but pytest, whose
|
|
922
|
+
// plugin is loaded by importable name and needs `PYTHONPATH` to
|
|
923
|
+
// resolve it (see `RunnerProfile.reporterEnv`'s own doc comment).
|
|
924
|
+
env: { NODE_TEST_CONTEXT: undefined, ...reporterEnvFor(discovery.kind, discovery.args) },
|
|
925
|
+
});
|
|
926
|
+
translator = createEventTranslator(context.emit, options.service, processId);
|
|
927
|
+
const activeTranslator = translator;
|
|
928
|
+
const gate = createEnvelopeGate();
|
|
929
|
+
const reportEnvelopeError = (reason) => {
|
|
930
|
+
ctx.emit({
|
|
931
|
+
timestamp: new Date().toISOString(),
|
|
932
|
+
source: "test-runner",
|
|
933
|
+
service: options.service ?? null,
|
|
934
|
+
process: processId,
|
|
935
|
+
traceId: null,
|
|
936
|
+
requestId: null,
|
|
937
|
+
correlationId: null,
|
|
938
|
+
graphNodeId: null,
|
|
939
|
+
stackTrace: null,
|
|
940
|
+
confidence: 1,
|
|
941
|
+
redactionStatus: "pending-redaction",
|
|
942
|
+
collectorVersion: COLLECTOR_VERSION,
|
|
943
|
+
eventType: "COLLECTOR_ERROR",
|
|
944
|
+
payload: { collector: "test-runner", reason },
|
|
945
|
+
sourceLocation: null,
|
|
946
|
+
});
|
|
947
|
+
};
|
|
948
|
+
completion = pumpLines(managed, (line) => {
|
|
949
|
+
if (line.trim() === "")
|
|
950
|
+
return;
|
|
951
|
+
let parsed;
|
|
952
|
+
try {
|
|
953
|
+
parsed = JSON.parse(line);
|
|
954
|
+
}
|
|
955
|
+
catch {
|
|
956
|
+
// A non-JSON line on stdout means something other than this
|
|
957
|
+
// collector's own reporter wrote it (e.g. the process under
|
|
958
|
+
// test's own console.log) -- ignored rather than treated as a
|
|
959
|
+
// parse failure of the reporter stream itself, which is a real,
|
|
960
|
+
// expected, ordinary case for any real test suite.
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
// **The handshake, checked before anything is believed.** A
|
|
964
|
+
// reporter declares which envelope it speaks; a mismatch, or an
|
|
965
|
+
// event arriving with no declaration at all, means what follows
|
|
966
|
+
// cannot be trusted to mean what this translator will read it as.
|
|
967
|
+
// COLLECTOR_ERROR and then silence, never confident wrong evidence.
|
|
968
|
+
// The gate sees every line, not only translatable ones: a
|
|
969
|
+
// handshake is not a `NodeTestEvent` and would otherwise be
|
|
970
|
+
// dropped as noise before it could be read.
|
|
971
|
+
if (!isNodeTestEvent(parsed)) {
|
|
972
|
+
const noise = gate.inspect(parsed, false);
|
|
973
|
+
if (noise.kind === "refuse")
|
|
974
|
+
reportEnvelopeError(noise.reason);
|
|
975
|
+
return;
|
|
976
|
+
}
|
|
977
|
+
const verdict = gate.inspect(parsed, true);
|
|
978
|
+
if (verdict.kind === "refuse") {
|
|
979
|
+
reportEnvelopeError(verdict.reason);
|
|
980
|
+
return;
|
|
981
|
+
}
|
|
982
|
+
if (verdict.kind !== "event")
|
|
983
|
+
return;
|
|
984
|
+
activeTranslator.handle(parsed);
|
|
985
|
+
}).then(() => {
|
|
986
|
+
// The very last test's result has no "next test:start" or
|
|
987
|
+
// no-`file` diagnostic to flush it -- node's own suite-level
|
|
988
|
+
// summary diagnostics almost always follow, but relying on that
|
|
989
|
+
// rather than flushing explicitly here would mean a truncated or
|
|
990
|
+
// unusual stream (the process killed mid-run, `stop()` called
|
|
991
|
+
// early) leaves the last real result buffered and never emitted.
|
|
992
|
+
activeTranslator.flush();
|
|
993
|
+
});
|
|
994
|
+
return { available: true };
|
|
995
|
+
},
|
|
996
|
+
async stop() {
|
|
997
|
+
if (managed !== null) {
|
|
998
|
+
await managed.kill();
|
|
999
|
+
}
|
|
1000
|
+
await completion;
|
|
1001
|
+
context = null;
|
|
1002
|
+
},
|
|
1003
|
+
capabilities() {
|
|
1004
|
+
return capabilities;
|
|
1005
|
+
},
|
|
1006
|
+
async waitForCompletion() {
|
|
1007
|
+
await completion;
|
|
1008
|
+
},
|
|
1009
|
+
traceCoverage() {
|
|
1010
|
+
return translator?.coverage() ?? { testsObserved: 0, idsPresent: 0, notRun: 0 };
|
|
1011
|
+
},
|
|
1012
|
+
};
|
|
1013
|
+
}
|
|
1014
|
+
//# sourceMappingURL=test-runner-collector.js.map
|