@bluewombat/slots 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -63
- package/agents/agent-plugins.mjs +154 -6
- package/agents/claude.mjs +2 -1
- package/agents/cursor.mjs +2 -0
- package/builders/{e2e.mjs → feature-test.mjs} +27 -17
- package/gates/base-diff.mjs +276 -0
- package/gates/ci-green.mjs +83 -3
- package/gates/feature-tests.mjs +280 -0
- package/gates/parent-clean.mjs +46 -8
- package/gates/tests-only.mjs +65 -0
- package/package.json +2 -2
- package/publishers/git.mjs +117 -16
- package/builders/tests.mjs +0 -139
- package/gates/e2e.mjs +0 -538
- package/gates/tests-frozen.mjs +0 -96
- package/gates/tests-red.mjs +0 -227
package/builders/tests.mjs
DELETED
|
@@ -1,139 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
BUILDER_FLAGS,
|
|
4
|
-
emitFailure,
|
|
5
|
-
fillPrompt,
|
|
6
|
-
finishRun,
|
|
7
|
-
loadPrompt,
|
|
8
|
-
loadRules,
|
|
9
|
-
PROMPT_RULES,
|
|
10
|
-
parseRole,
|
|
11
|
-
spawnFilled,
|
|
12
|
-
take,
|
|
13
|
-
textOption,
|
|
14
|
-
transcriptArgs,
|
|
15
|
-
} from "@bluewombat/slot-kit";
|
|
16
|
-
import { attemptOf } from "../lib/fill.mjs";
|
|
17
|
-
|
|
18
|
-
const NAME = "builder tests";
|
|
19
|
-
|
|
20
|
-
const DEFAULT_TEMPLATE = `You are a coding agent working unattended in a checkout of the project.
|
|
21
|
-
|
|
22
|
-
Task:
|
|
23
|
-
{{task}}
|
|
24
|
-
|
|
25
|
-
YOUR WORK NOW IS TO WRITE THE TESTS OF THIS TASK, AND ONLY THE TESTS. Another agent
|
|
26
|
-
writes the code afterwards and must make your tests pass without editing them. A test it
|
|
27
|
-
cannot pass is not sent back to you: it stops the task and waits for a person. So a test
|
|
28
|
-
you deliver must be one that a correct implementation of the task passes, and you prove
|
|
29
|
-
it before you hand it over (see below).
|
|
30
|
-
|
|
31
|
-
## The cases
|
|
32
|
-
|
|
33
|
-
- The task has a "Tests" section: write one test per case of it, in the file and the
|
|
34
|
-
framework it names (or the ones this repository already uses), each named after the
|
|
35
|
-
case it covers. Test the behaviour at the seam the section names, not how it is built.
|
|
36
|
-
- The task has no "Tests" section: do not look for one elsewhere. Derive the cases from
|
|
37
|
-
its acceptance criteria ("Done when" or the like), one case per criterion, and name each
|
|
38
|
-
test after the criterion and the identifiers it carries. A criterion that describes two
|
|
39
|
-
sets of data (sixty rows for a page, more than a cap for the cap) is two cases, never one
|
|
40
|
-
test with two batches of data.
|
|
41
|
-
- Do not write the behaviour. If the interface under test does not exist yet, create the
|
|
42
|
-
minimal stub — the name and signature the task gives, a body that does nothing useful —
|
|
43
|
-
so that the tests compile and fail on what they check, not on a missing import.
|
|
44
|
-
- Add or update no other test, and no production code beyond the stubs.
|
|
45
|
-
|
|
46
|
-
## How a test stays passable
|
|
47
|
-
|
|
48
|
-
- One data set per test, seeded at the top. Every expectation derives from that set: the
|
|
49
|
-
rows a filter keeps, the order they take, the page a limit and an offset cut out of it.
|
|
50
|
-
Write the expected count from the seeded data, not as a figure recounted by hand.
|
|
51
|
-
- To observe a bound, seed past it: a cap of 200 is only visible with more than 200 rows
|
|
52
|
-
that the filters keep. "At most N" is asserted with \`<= N\`, or by seeding more than N;
|
|
53
|
-
never as exactly N when fewer exist.
|
|
54
|
-
|
|
55
|
-
## Prove them: red, green, red
|
|
56
|
-
|
|
57
|
-
A test you have only seen fail proves nothing about itself: a test no implementation
|
|
58
|
-
could pass fails exactly the same way. The tests must FAIL now, because the behaviour is
|
|
59
|
-
missing, and you prove every one of them passable before you finish:
|
|
60
|
-
|
|
61
|
-
1. **Red.** Run the tests. Each fails on an assertion (not on an import, a type, a syntax
|
|
62
|
-
error).
|
|
63
|
-
2. **Green.** In the stub bodies — the same files, no new file — write the shortest
|
|
64
|
-
implementation of the task that makes every test pass: naive, in memory, unoptimised;
|
|
65
|
-
it is thrown away. Run the tests. A test still red here is wrong, not the task: fix its
|
|
66
|
-
data or its expectation (never drop a case, never replace a value the task gives with a
|
|
67
|
-
looser one), and run again until all are green.
|
|
68
|
-
3. **Red again.** Put every stub body back to no behaviour. Run the tests: each fails
|
|
69
|
-
again on an assertion. Leave the tree in this state.
|
|
70
|
-
|
|
71
|
-
Leave no trace of the green step: no helper, no import, no comment. The only non-test
|
|
72
|
-
lines you deliver are the stubs.
|
|
73
|
-
|
|
74
|
-
{{history}}
|
|
75
|
-
|
|
76
|
-
## When you cannot act
|
|
77
|
-
|
|
78
|
-
If the cases are missing, or contradict the task or the code, do not invent them. Change
|
|
79
|
-
nothing and end your final message with exactly one line:
|
|
80
|
-
|
|
81
|
-
MASON_BLOCKED: <what is missing, in one line>
|
|
82
|
-
|
|
83
|
-
A person reads that line and decides. It is a correct answer, not a failure.
|
|
84
|
-
|
|
85
|
-
Rules of this run:
|
|
86
|
-
{{rules}}
|
|
87
|
-
`;
|
|
88
|
-
|
|
89
|
-
// A retry carries the refusal in --history; the flags themselves are consumed so the
|
|
90
|
-
// role options see only their own.
|
|
91
|
-
take(process.argv, "--report");
|
|
92
|
-
take(process.argv, "--report-from");
|
|
93
|
-
|
|
94
|
-
const intention = (take(process.argv, "--intention") ?? "").trim();
|
|
95
|
-
if (intention.length === 0) {
|
|
96
|
-
emitFailure("fail-blocking", "The Builder received no --intention.");
|
|
97
|
-
process.exit(1);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
const parsed = parseRole(process.argv, BUILDER_FLAGS, NAME);
|
|
101
|
-
if (!parsed.ok) {
|
|
102
|
-
emitFailure("fail-blocking", parsed.reason);
|
|
103
|
-
process.exit(1);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
const template = loadPrompt(parsed.options, DEFAULT_TEMPLATE);
|
|
107
|
-
if (!template.ok) {
|
|
108
|
-
emitFailure("fail-blocking", template.reason);
|
|
109
|
-
process.exit(1);
|
|
110
|
-
}
|
|
111
|
-
const rules = loadRules(parsed.options, PROMPT_RULES);
|
|
112
|
-
if (!rules.ok) {
|
|
113
|
-
emitFailure("fail-blocking", rules.reason);
|
|
114
|
-
process.exit(1);
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
const rendered = fillPrompt(
|
|
118
|
-
template.value,
|
|
119
|
-
{
|
|
120
|
-
task: intention,
|
|
121
|
-
history: (take(process.argv, "--history") ?? "").trim(),
|
|
122
|
-
rules: rules.value.trim(),
|
|
123
|
-
id: take(process.argv, "--id") ?? "",
|
|
124
|
-
attempt: String(attemptOf(process.argv)),
|
|
125
|
-
},
|
|
126
|
-
["task", "rules"],
|
|
127
|
-
textOption(parsed.options, "promptFile") ?? "built-in template",
|
|
128
|
-
);
|
|
129
|
-
if (!rendered.ok) {
|
|
130
|
-
emitFailure("fail-blocking", rendered.reason);
|
|
131
|
-
process.exit(1);
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
const spawned = await spawnFilled(parsed.runner, rendered.prompt, transcriptArgs(process.argv));
|
|
135
|
-
if (!spawned.ok) {
|
|
136
|
-
emitFailure("fail-blocking", spawned.reason);
|
|
137
|
-
process.exit(1);
|
|
138
|
-
}
|
|
139
|
-
finishRun({ run: spawned.run, bin: spawned.about.bin, name: spawned.about.name });
|
package/gates/e2e.mjs
DELETED
|
@@ -1,538 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { spawn, spawnSync } from "node:child_process";
|
|
3
|
-
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
4
|
-
import { tmpdir } from "node:os";
|
|
5
|
-
import { isAbsolute, join, resolve } from "node:path";
|
|
6
|
-
import { setTimeout as sleep } from "node:timers/promises";
|
|
7
|
-
import {
|
|
8
|
-
emitVerdict,
|
|
9
|
-
GATE_FLAGS,
|
|
10
|
-
ownArgv,
|
|
11
|
-
parseOptions,
|
|
12
|
-
take,
|
|
13
|
-
textOption,
|
|
14
|
-
} from "@bluewombat/slot-kit";
|
|
15
|
-
import { parseJunit, scenarioOf } from "../lib/junit.mjs";
|
|
16
|
-
import { createOutputTail } from "../lib/output-tail.mjs";
|
|
17
|
-
|
|
18
|
-
// The acceptance suite of a Feature, run on the assembled Feature line: does the product do
|
|
19
|
-
// what the operator agreed it should, seen from the outside?
|
|
20
|
-
//
|
|
21
|
-
// Nothing here knows a stack. The Project supplies four commands, run in the workspace through
|
|
22
|
-
// the shell, and the Gate owns the rest: starting, waiting, running, comparing, always stopping.
|
|
23
|
-
//
|
|
24
|
-
// --up COMMAND (required): starts the application. It may return once started (a daemon, a
|
|
25
|
-
// compose stack) or keep running (a dev server): either way the Gate does not wait for it, it
|
|
26
|
-
// waits for --ready. An `up` that exits non-zero before the app is ready is a start failure.
|
|
27
|
-
// --ready COMMAND (required): exit 0 when the application answers. Polled until --start-timeout-ms.
|
|
28
|
-
// --run COMMAND (required): runs the suite and writes a JUnit XML report to --junit.
|
|
29
|
-
// --rerun COMMAND: runs the failed scenarios again, `{ids}` standing for their ids joined with
|
|
30
|
-
// "|" (a regular expression: `pnpm e2e --grep "{ids}"`). Absent: --run again, whole.
|
|
31
|
-
// --down COMMAND (required): stops the application. Always run, whatever happened, and the
|
|
32
|
-
// process group of --up is killed after it.
|
|
33
|
-
// --manifest PATH: the scenarios the suite must hold. Default e2e/scenarios.json: a JSON array of
|
|
34
|
-
// ids, or { "scenarios": [ "E1" | { "id": "E1", "title": "…" } ] }.
|
|
35
|
-
// --junit PATH: where --run writes its report. Default e2e/results/junit.xml. Removed before the run.
|
|
36
|
-
// --server-log PATH: the application's log, whose tail goes in a red report. Default: what --up
|
|
37
|
-
// printed.
|
|
38
|
-
// --max-scenarios N: the most scenarios a suite may hold. Default 5.
|
|
39
|
-
// --max-run-ms N: the most the whole Gate may take. Default 600000 (10 minutes), and never more
|
|
40
|
-
// than the Gate's own ceiling.
|
|
41
|
-
// --start-timeout-ms N: the most --up and --ready may take together. Default 120000.
|
|
42
|
-
// --port N: handed to every command as E2E_PORT, for an application that takes its port from the
|
|
43
|
-
// environment. Absent: the application's own choice.
|
|
44
|
-
// --host HOST: handed to every command as E2E_HOST. Default 127.0.0.1; anything but a loopback
|
|
45
|
-
// address is refused.
|
|
46
|
-
//
|
|
47
|
-
// Every command also gets E2E_DATA_DIR (an empty directory made for this run and removed after:
|
|
48
|
-
// the data is disposable) and E2E_RESULTS_DIR (where --junit lives). Secrets are not an option
|
|
49
|
-
// of the Gate: the commands read them from the environment the operator started mason in.
|
|
50
|
-
//
|
|
51
|
-
// A scenario is a manifest id; each test carries its id in its name. The comparison is exact: an
|
|
52
|
-
// id with no test, a test with no id, or a test the runner skipped is a failure. A test that
|
|
53
|
-
// fails is run once more: passing then is flagged unstable in the report, failing twice is red.
|
|
54
|
-
//
|
|
55
|
-
// Verdicts: pass; fail-retryable for a red suite (the application is the producer's to repair,
|
|
56
|
-
// the suite is frozen); fail-blocking for what no change to the application fixes: a missing
|
|
57
|
-
// or oversized manifest, an application that does not start or a run that outlives its ceiling.
|
|
58
|
-
|
|
59
|
-
const cwd = process.cwd();
|
|
60
|
-
const KEEP_BYTES = 12_000;
|
|
61
|
-
const DOWN_MS = 20_000;
|
|
62
|
-
// The Gate answers before its own ceiling, and leaves --down the time to stop the application.
|
|
63
|
-
const MARGIN_MS = 5_000 + DOWN_MS;
|
|
64
|
-
const MIN_PHASE_MS = 1_000;
|
|
65
|
-
const READY_CALL_MS = 15_000;
|
|
66
|
-
const READY_EVERY_MS = 1_000;
|
|
67
|
-
const LOG_LINES = 40;
|
|
68
|
-
const REPORT_CHARS = 8_000;
|
|
69
|
-
const LOOPBACK = new Set(["127.0.0.1", "localhost", "::1", "[::1]"]);
|
|
70
|
-
|
|
71
|
-
const parsed = parseOptions(
|
|
72
|
-
ownArgv(process.argv, GATE_FLAGS),
|
|
73
|
-
{
|
|
74
|
-
"--up": {},
|
|
75
|
-
"--ready": {},
|
|
76
|
-
"--run": {},
|
|
77
|
-
"--rerun": {},
|
|
78
|
-
"--down": {},
|
|
79
|
-
"--manifest": { fallback: "e2e/scenarios.json" },
|
|
80
|
-
"--junit": { fallback: "e2e/results/junit.xml" },
|
|
81
|
-
"--server-log": {},
|
|
82
|
-
"--max-scenarios": { fallback: "5" },
|
|
83
|
-
"--max-run-ms": { fallback: "600000" },
|
|
84
|
-
"--start-timeout-ms": { fallback: "120000" },
|
|
85
|
-
"--host": { fallback: "127.0.0.1" },
|
|
86
|
-
"--port": {},
|
|
87
|
-
},
|
|
88
|
-
"e2e",
|
|
89
|
-
);
|
|
90
|
-
if (!parsed.ok) {
|
|
91
|
-
emitVerdict("fail-blocking", parsed.reason);
|
|
92
|
-
process.exit(0);
|
|
93
|
-
}
|
|
94
|
-
const up = textOption(parsed.values, "up");
|
|
95
|
-
const ready = textOption(parsed.values, "ready");
|
|
96
|
-
const runCommand = textOption(parsed.values, "run");
|
|
97
|
-
const rerunCommand = textOption(parsed.values, "rerun");
|
|
98
|
-
const down = textOption(parsed.values, "down");
|
|
99
|
-
for (const [flag, value] of [
|
|
100
|
-
["--up", up],
|
|
101
|
-
["--ready", ready],
|
|
102
|
-
["--run", runCommand],
|
|
103
|
-
["--down", down],
|
|
104
|
-
]) {
|
|
105
|
-
if (value === undefined || value.trim().length === 0) {
|
|
106
|
-
emitVerdict(
|
|
107
|
-
"fail-blocking",
|
|
108
|
-
`e2e needs ${flag} "<command>". The Project supplies --up, --ready, --run and --down.`,
|
|
109
|
-
);
|
|
110
|
-
process.exit(0);
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
const host = textOption(parsed.values, "host") ?? "127.0.0.1";
|
|
114
|
-
if (!LOOPBACK.has(host)) {
|
|
115
|
-
emitVerdict(
|
|
116
|
-
"fail-blocking",
|
|
117
|
-
`e2e serves the application on a loopback address only: "${host}" is not one.`,
|
|
118
|
-
);
|
|
119
|
-
process.exit(0);
|
|
120
|
-
}
|
|
121
|
-
const maxScenarios = positive(textOption(parsed.values, "maxScenarios"));
|
|
122
|
-
const maxRunMs = positive(textOption(parsed.values, "maxRunMs"));
|
|
123
|
-
const startTimeoutMs = positive(textOption(parsed.values, "startTimeoutMs"));
|
|
124
|
-
if (maxScenarios === undefined || maxRunMs === undefined || startTimeoutMs === undefined) {
|
|
125
|
-
emitVerdict(
|
|
126
|
-
"fail-blocking",
|
|
127
|
-
"e2e: --max-scenarios, --max-run-ms and --start-timeout-ms take a positive integer.",
|
|
128
|
-
);
|
|
129
|
-
process.exit(0);
|
|
130
|
-
}
|
|
131
|
-
const portText = textOption(parsed.values, "port");
|
|
132
|
-
if (portText !== undefined && positive(portText) === undefined) {
|
|
133
|
-
emitVerdict("fail-blocking", `e2e: --port takes a port number, not "${portText}".`);
|
|
134
|
-
process.exit(0);
|
|
135
|
-
}
|
|
136
|
-
const manifestPath = inWorkspace(textOption(parsed.values, "manifest") ?? "");
|
|
137
|
-
const junitPath = inWorkspace(textOption(parsed.values, "junit") ?? "");
|
|
138
|
-
const serverLogPath = textOption(parsed.values, "serverLog");
|
|
139
|
-
|
|
140
|
-
const manifest = readManifest();
|
|
141
|
-
if (typeof manifest === "string") {
|
|
142
|
-
emitVerdict("fail-blocking", manifest);
|
|
143
|
-
process.exit(0);
|
|
144
|
-
}
|
|
145
|
-
// Typed apart from the early exit above: the functions below are hoisted, and a narrowing does not reach them.
|
|
146
|
-
/** @type {string[]} */
|
|
147
|
-
const ids = manifest;
|
|
148
|
-
|
|
149
|
-
const ceilingMs = Number(take(process.argv, "--timeout-ms"));
|
|
150
|
-
const budgetMs =
|
|
151
|
-
Number.isFinite(ceilingMs) && ceilingMs > 0
|
|
152
|
-
? Math.min(maxRunMs, Math.max(MIN_PHASE_MS, ceilingMs - MARGIN_MS))
|
|
153
|
-
: maxRunMs;
|
|
154
|
-
const startedAt = Date.now();
|
|
155
|
-
const remainingMs = () => budgetMs - (Date.now() - startedAt);
|
|
156
|
-
|
|
157
|
-
const dataDir = mkdtempSync(join(tmpdir(), "e2e-data-"));
|
|
158
|
-
const env = {
|
|
159
|
-
...process.env,
|
|
160
|
-
CI: "true",
|
|
161
|
-
NO_COLOR: "1",
|
|
162
|
-
FORCE_COLOR: "0",
|
|
163
|
-
E2E_HOST: host,
|
|
164
|
-
...(portText === undefined ? {} : { E2E_PORT: portText }),
|
|
165
|
-
E2E_DATA_DIR: dataDir,
|
|
166
|
-
E2E_RESULTS_DIR: dirOf(junitPath),
|
|
167
|
-
};
|
|
168
|
-
|
|
169
|
-
/** @type {ReturnType<typeof startBackground> | undefined} */
|
|
170
|
-
let server;
|
|
171
|
-
// Host stopping the run must not leave an application behind.
|
|
172
|
-
for (const signal of ["SIGTERM", "SIGINT"]) {
|
|
173
|
-
process.on(signal, () => {
|
|
174
|
-
server?.kill();
|
|
175
|
-
spawnSync("sh", ["-c", /** @type {string} */ (down)], { cwd, env, timeout: DOWN_MS });
|
|
176
|
-
rmSync(dataDir, { recursive: true, force: true });
|
|
177
|
-
process.exit(1);
|
|
178
|
-
});
|
|
179
|
-
}
|
|
180
|
-
let verdict = /** @type {[string, string]} */ (["fail-blocking", "e2e ended without a verdict."]);
|
|
181
|
-
try {
|
|
182
|
-
verdict = await judge();
|
|
183
|
-
} catch (error) {
|
|
184
|
-
verdict = [
|
|
185
|
-
"fail-blocking",
|
|
186
|
-
`e2e could not run: ${error instanceof Error ? error.message : error}`,
|
|
187
|
-
];
|
|
188
|
-
} finally {
|
|
189
|
-
await stop();
|
|
190
|
-
}
|
|
191
|
-
emitVerdict(/** @type {"pass" | "fail-retryable" | "fail-blocking"} */ (verdict[0]), verdict[1]);
|
|
192
|
-
process.exit(0);
|
|
193
|
-
|
|
194
|
-
/** @returns {Promise<[string, string]>} */
|
|
195
|
-
async function judge() {
|
|
196
|
-
server = startBackground(/** @type {string} */ (up));
|
|
197
|
-
const started = await waitReady(/** @type {string} */ (ready));
|
|
198
|
-
if (started !== undefined) {
|
|
199
|
-
return ["fail-blocking", started];
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
rmSync(junitPath, { force: true });
|
|
203
|
-
const first = await runShell(/** @type {string} */ (runCommand), remainingMs());
|
|
204
|
-
if (first.timedOut) {
|
|
205
|
-
return ["fail-blocking", timeoutReport("The suite", first.output)];
|
|
206
|
-
}
|
|
207
|
-
const firstResults = readResults();
|
|
208
|
-
if (typeof firstResults === "string") {
|
|
209
|
-
return first.code === 0
|
|
210
|
-
? ["fail-blocking", firstResults]
|
|
211
|
-
: [
|
|
212
|
-
"fail-retryable",
|
|
213
|
-
`${firstResults}\n${exitLine(first)} End of its output:\n${first.output}`,
|
|
214
|
-
];
|
|
215
|
-
}
|
|
216
|
-
let results = firstResults;
|
|
217
|
-
let failing = failingOf(results);
|
|
218
|
-
/** @type {string[]} */
|
|
219
|
-
let unstable = [];
|
|
220
|
-
if (failing.length > 0) {
|
|
221
|
-
const again =
|
|
222
|
-
rerunCommand === undefined ? runCommand : rerunCommand.replaceAll("{ids}", failing.join("|"));
|
|
223
|
-
rmSync(junitPath, { force: true });
|
|
224
|
-
const second = await runShell(/** @type {string} */ (again), remainingMs());
|
|
225
|
-
if (second.timedOut) {
|
|
226
|
-
return ["fail-blocking", timeoutReport("The retry of the failed scenarios", second.output)];
|
|
227
|
-
}
|
|
228
|
-
const secondResults = readResults();
|
|
229
|
-
const retried = typeof secondResults === "string" ? undefined : secondResults;
|
|
230
|
-
const stillFailing = retried === undefined ? failing : failingOf(retried, failing);
|
|
231
|
-
unstable = failing.filter((id) => !stillFailing.includes(id));
|
|
232
|
-
// What the retry did not run keeps its first verdict; what it ran takes the second.
|
|
233
|
-
results = retried === undefined ? results : mergeRetry(results, retried, failing);
|
|
234
|
-
failing = stillFailing;
|
|
235
|
-
}
|
|
236
|
-
if (failing.length > 0) {
|
|
237
|
-
return ["fail-retryable", redReport(results, failing)];
|
|
238
|
-
}
|
|
239
|
-
const problems = comparison(results);
|
|
240
|
-
if (problems.length > 0) {
|
|
241
|
-
return ["fail-retryable", problems.join("\n")];
|
|
242
|
-
}
|
|
243
|
-
return ["pass", passReport(results, unstable)];
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
/**
|
|
247
|
-
* Scenario ids failing, for the ids in `among` only when it is given. The comparison with the
|
|
248
|
-
* manifest counts: an id that did not run, a test without an id, a skipped test.
|
|
249
|
-
*
|
|
250
|
-
* @param {Result} results
|
|
251
|
-
* @param {string[]} [among]
|
|
252
|
-
*/
|
|
253
|
-
function failingOf(results, among) {
|
|
254
|
-
const bad = new Set(comparisonFailures(results));
|
|
255
|
-
const failing = ids.filter((id) => bad.has(id));
|
|
256
|
-
return among === undefined ? failing : failing.filter((id) => among.includes(id));
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
/**
|
|
260
|
-
* @typedef {{ cases: import("../lib/junit.mjs").TestCase[] }} Result
|
|
261
|
-
*/
|
|
262
|
-
|
|
263
|
-
/** @returns {Result | string} */
|
|
264
|
-
function readResults() {
|
|
265
|
-
if (!existsSync(junitPath)) {
|
|
266
|
-
return `The suite wrote no JUnit report at ${junitPath}. --run must write one there.`;
|
|
267
|
-
}
|
|
268
|
-
const cases = parseJunit(readFileSync(junitPath, "utf8"));
|
|
269
|
-
if (cases.length === 0) {
|
|
270
|
-
return `The JUnit report at ${junitPath} holds no test case.`;
|
|
271
|
-
}
|
|
272
|
-
return { cases };
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
/** The scenario ids that are not a pass: failed, skipped, or never run. */
|
|
276
|
-
function comparisonFailures(/** @type {Result} */ results) {
|
|
277
|
-
const out = new Set();
|
|
278
|
-
for (const id of ids) {
|
|
279
|
-
const mine = results.cases.filter((c) => scenarioOf(c.name, ids) === id);
|
|
280
|
-
if (mine.length === 0 || mine.some((c) => c.status !== "passed")) {
|
|
281
|
-
out.add(id);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
return [...out];
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
/** What the manifest and the report disagree on, beyond a red test: tests no scenario owns. */
|
|
288
|
-
function comparison(/** @type {Result} */ results) {
|
|
289
|
-
const foreign = results.cases.filter((c) => scenarioOf(c.name, ids) === undefined);
|
|
290
|
-
return foreign.length === 0
|
|
291
|
-
? []
|
|
292
|
-
: [
|
|
293
|
-
`e2e: ${foreign.length} test(s) carry no scenario id of the manifest (${ids.join(", ")}): the suite may hold nothing else.`,
|
|
294
|
-
...foreign.map((c) => ` ${c.name}`),
|
|
295
|
-
"Every test names its scenario id, and every scenario is in the manifest.",
|
|
296
|
-
];
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
/**
|
|
300
|
-
* @param {Result} first
|
|
301
|
-
* @param {Result} second
|
|
302
|
-
* @param {string[]} retried
|
|
303
|
-
* @returns {Result}
|
|
304
|
-
*/
|
|
305
|
-
function mergeRetry(first, second, retried) {
|
|
306
|
-
const kept = first.cases.filter((c) => !retried.includes(scenarioOf(c.name, ids) ?? ""));
|
|
307
|
-
const reran = second.cases.filter((c) => retried.includes(scenarioOf(c.name, ids) ?? ""));
|
|
308
|
-
return { cases: [...kept, ...reran] };
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
function passReport(/** @type {Result} */ results, /** @type {string[]} */ unstable) {
|
|
312
|
-
const lines = [
|
|
313
|
-
`e2e: ${ids.length} scenario(s) ran on the assembled feature and passed: ${ids.join(", ")}.`,
|
|
314
|
-
];
|
|
315
|
-
if (unstable.length > 0) {
|
|
316
|
-
lines.push(`UNSTABLE: ${unstable.join(", ")} passed only on the retry.`);
|
|
317
|
-
}
|
|
318
|
-
const extra = results.cases.length - ids.length;
|
|
319
|
-
if (extra > 0) {
|
|
320
|
-
lines.push(`(${results.cases.length} test cases carry those ids.)`);
|
|
321
|
-
}
|
|
322
|
-
return lines.join("\n");
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
function redReport(/** @type {Result} */ results, /** @type {string[]} */ failing) {
|
|
326
|
-
const lines = [
|
|
327
|
-
`e2e: ${failing.length} of ${ids.length} scenario(s) failed on the assembled feature, twice: ${failing.join(", ")}.`,
|
|
328
|
-
"The suite is frozen: repair the application, never the tests.",
|
|
329
|
-
];
|
|
330
|
-
for (const id of failing) {
|
|
331
|
-
const mine = results.cases.filter((c) => scenarioOf(c.name, ids) === id);
|
|
332
|
-
if (mine.length === 0) {
|
|
333
|
-
lines.push(`${id} — no test ran for it.`);
|
|
334
|
-
continue;
|
|
335
|
-
}
|
|
336
|
-
for (const c of mine.filter((m) => m.status !== "passed")) {
|
|
337
|
-
lines.push(`${id} — ${c.name}`);
|
|
338
|
-
if (c.status === "skipped") {
|
|
339
|
-
lines.push(" skipped: a skipped scenario is not a passing one.");
|
|
340
|
-
continue;
|
|
341
|
-
}
|
|
342
|
-
lines.push(` at: ${firstFrame(c.detail) ?? "(no step given)"}`);
|
|
343
|
-
lines.push(` message: ${c.message.slice(0, 600)}`);
|
|
344
|
-
const trace = c.attachments.find((a) => /trace/i.test(a)) ?? c.attachments[0];
|
|
345
|
-
if (trace !== undefined) {
|
|
346
|
-
lines.push(` trace: ${trace}`);
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
const log = logTail();
|
|
351
|
-
if (log.length > 0) {
|
|
352
|
-
lines.push(`Server log, last ${LOG_LINES} lines:`, log);
|
|
353
|
-
}
|
|
354
|
-
lines.push(`Report: ${junitPath}. Screenshots and traces stay on disk beside it.`);
|
|
355
|
-
const text = lines.join("\n");
|
|
356
|
-
return text.length <= REPORT_CHARS ? text : `${text.slice(0, REPORT_CHARS)}\n…`;
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
/** The first stack line that is the suite's own, where a step failed. */
|
|
360
|
-
function firstFrame(/** @type {string} */ detail) {
|
|
361
|
-
return detail
|
|
362
|
-
.split("\n")
|
|
363
|
-
.map((line) => line.trim())
|
|
364
|
-
.find((line) => /^at\s|\.(spec|test)\.[cm]?[jt]sx?:\d+/.test(line));
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
function logTail() {
|
|
368
|
-
const fromFile =
|
|
369
|
-
serverLogPath !== undefined && existsSync(inWorkspace(serverLogPath))
|
|
370
|
-
? readFileSync(inWorkspace(serverLogPath), "utf8")
|
|
371
|
-
: undefined;
|
|
372
|
-
const text = fromFile ?? server?.output() ?? "";
|
|
373
|
-
return text.split("\n").slice(-LOG_LINES).join("\n").trim();
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
/** @returns {Promise<string | undefined>} Why it did not start, or nothing. */
|
|
377
|
-
async function waitReady(/** @type {string} */ command) {
|
|
378
|
-
const limit = Math.min(startTimeoutMs ?? 0, remainingMs());
|
|
379
|
-
const until = Date.now() + limit;
|
|
380
|
-
while (Date.now() < until) {
|
|
381
|
-
if (server?.failed() !== undefined) {
|
|
382
|
-
return `The application did not start: "${up}" ${server.failed()}. Its output:\n${server.output()}`;
|
|
383
|
-
}
|
|
384
|
-
const probe = await runShell(command, Math.min(READY_CALL_MS, until - Date.now()));
|
|
385
|
-
if (!probe.timedOut && probe.code === 0) {
|
|
386
|
-
return undefined;
|
|
387
|
-
}
|
|
388
|
-
await sleep(READY_EVERY_MS);
|
|
389
|
-
}
|
|
390
|
-
return `The application was not ready after ${Math.round(limit / 1000)}s ("${command}" never exited 0). Output of "${up}":\n${server?.output() ?? ""}`;
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
/** @param {string} command */
|
|
394
|
-
function startBackground(command) {
|
|
395
|
-
const child = spawn("sh", ["-c", command], {
|
|
396
|
-
cwd,
|
|
397
|
-
env,
|
|
398
|
-
detached: true,
|
|
399
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
400
|
-
});
|
|
401
|
-
const tail = createOutputTail(KEEP_BYTES);
|
|
402
|
-
child.stdout.on("data", (chunk) => tail.push("stdout", chunk));
|
|
403
|
-
child.stderr.on("data", (chunk) => tail.push("stderr", chunk));
|
|
404
|
-
/** @type {string | undefined} */
|
|
405
|
-
let failure;
|
|
406
|
-
child.on("error", (error) => {
|
|
407
|
-
failure = `cannot start: ${error.message}`;
|
|
408
|
-
});
|
|
409
|
-
child.on("close", (code, signal) => {
|
|
410
|
-
if (code !== 0 && code !== null) {
|
|
411
|
-
failure = `exited with ${code}`;
|
|
412
|
-
} else if (signal !== null && !stopping) {
|
|
413
|
-
failure = `was killed by ${signal}`;
|
|
414
|
-
}
|
|
415
|
-
});
|
|
416
|
-
let stopping = false;
|
|
417
|
-
return {
|
|
418
|
-
output: () => tail.text(),
|
|
419
|
-
failed: () => failure,
|
|
420
|
-
kill() {
|
|
421
|
-
stopping = true;
|
|
422
|
-
killGroup(child);
|
|
423
|
-
},
|
|
424
|
-
};
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
/**
|
|
428
|
-
* @param {string} command
|
|
429
|
-
* @param {number} limitMs
|
|
430
|
-
* @returns {Promise<{ code: number | null, signal: string | null, timedOut: boolean, output: string }>}
|
|
431
|
-
*/
|
|
432
|
-
function runShell(command, limitMs) {
|
|
433
|
-
return new Promise((done) => {
|
|
434
|
-
const child = spawn("sh", ["-c", command], {
|
|
435
|
-
cwd,
|
|
436
|
-
env,
|
|
437
|
-
detached: true,
|
|
438
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
439
|
-
});
|
|
440
|
-
const tail = createOutputTail(KEEP_BYTES);
|
|
441
|
-
child.stdout.on("data", (chunk) => tail.push("stdout", chunk));
|
|
442
|
-
child.stderr.on("data", (chunk) => tail.push("stderr", chunk));
|
|
443
|
-
let timedOut = false;
|
|
444
|
-
const timer = globalThis.setTimeout(
|
|
445
|
-
() => {
|
|
446
|
-
timedOut = true;
|
|
447
|
-
killGroup(child);
|
|
448
|
-
},
|
|
449
|
-
Math.max(MIN_PHASE_MS, limitMs),
|
|
450
|
-
);
|
|
451
|
-
child.on("error", (error) => {
|
|
452
|
-
clearTimeout(timer);
|
|
453
|
-
done({ code: 127, signal: null, timedOut: false, output: `cannot start: ${error.message}` });
|
|
454
|
-
});
|
|
455
|
-
child.on("close", (code, signal) => {
|
|
456
|
-
clearTimeout(timer);
|
|
457
|
-
done({ code, signal, timedOut, output: tail.text() });
|
|
458
|
-
});
|
|
459
|
-
});
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
async function stop() {
|
|
463
|
-
// The application is stopped the way the Project says, then whatever `up` left is killed.
|
|
464
|
-
try {
|
|
465
|
-
await runShell(/** @type {string} */ (down), DOWN_MS);
|
|
466
|
-
} finally {
|
|
467
|
-
server?.kill();
|
|
468
|
-
rmSync(dataDir, { recursive: true, force: true });
|
|
469
|
-
}
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
/** @param {import("node:child_process").ChildProcess} child */
|
|
473
|
-
function killGroup(child) {
|
|
474
|
-
try {
|
|
475
|
-
if (child.pid !== undefined) {
|
|
476
|
-
process.kill(-child.pid, "SIGKILL");
|
|
477
|
-
}
|
|
478
|
-
} catch {
|
|
479
|
-
child.kill("SIGKILL");
|
|
480
|
-
}
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
function timeoutReport(/** @type {string} */ what, /** @type {string} */ output) {
|
|
484
|
-
return [
|
|
485
|
-
`${what} did not finish within the ${Math.round(budgetMs / 1000)}s this Gate may take, and was stopped.`,
|
|
486
|
-
"A person decides: the suite hangs, or the application does.",
|
|
487
|
-
output.length > 0 ? `Last output:\n${output}` : "It printed nothing.",
|
|
488
|
-
].join("\n");
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
function exitLine(/** @type {{ code: number | null, signal: string | null }} */ run) {
|
|
492
|
-
return run.signal === null ? `The suite exited ${run.code}.` : `The suite got ${run.signal}.`;
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
/** @returns {string[] | string} The ids, or why the manifest is not usable. */
|
|
496
|
-
function readManifest() {
|
|
497
|
-
if (!existsSync(manifestPath)) {
|
|
498
|
-
return `The scenarios manifest ${manifestPath} does not exist. The e2e Ticket writes it: one id per scenario.`;
|
|
499
|
-
}
|
|
500
|
-
let raw;
|
|
501
|
-
try {
|
|
502
|
-
raw = JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
503
|
-
} catch (error) {
|
|
504
|
-
return `The scenarios manifest ${manifestPath} is not JSON: ${error instanceof Error ? error.message : error}`;
|
|
505
|
-
}
|
|
506
|
-
const list = Array.isArray(raw) ? raw : raw?.scenarios;
|
|
507
|
-
if (!Array.isArray(list)) {
|
|
508
|
-
return `The scenarios manifest ${manifestPath} must be an array of ids, or { "scenarios": [...] }.`;
|
|
509
|
-
}
|
|
510
|
-
const found = list.map((entry) => (typeof entry === "string" ? entry : entry?.id));
|
|
511
|
-
if (found.some((id) => typeof id !== "string" || id.trim().length === 0)) {
|
|
512
|
-
return `Every scenario of ${manifestPath} needs a non-empty id.`;
|
|
513
|
-
}
|
|
514
|
-
const unique = [...new Set(found.map((id) => id.trim()))];
|
|
515
|
-
if (unique.length !== found.length) {
|
|
516
|
-
return `The scenarios manifest ${manifestPath} lists an id twice.`;
|
|
517
|
-
}
|
|
518
|
-
if (unique.length === 0) {
|
|
519
|
-
return `The scenarios manifest ${manifestPath} lists no scenario: there is nothing to prove.`;
|
|
520
|
-
}
|
|
521
|
-
if (unique.length > (maxScenarios ?? 0)) {
|
|
522
|
-
return `The scenarios manifest lists ${unique.length} scenarios; at most ${maxScenarios} are allowed (--max-scenarios). A person decides which to keep.`;
|
|
523
|
-
}
|
|
524
|
-
return unique;
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
function positive(/** @type {string | undefined} */ text) {
|
|
528
|
-
const value = Number(text);
|
|
529
|
-
return Number.isInteger(value) && value > 0 ? value : undefined;
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
function inWorkspace(/** @type {string} */ path) {
|
|
533
|
-
return isAbsolute(path) ? path : resolve(cwd, path);
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
function dirOf(/** @type {string} */ path) {
|
|
537
|
-
return resolve(path, "..");
|
|
538
|
-
}
|