@bluewombat/slots 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -50
- package/builders/{e2e.mjs → feature-test.mjs} +27 -17
- package/gates/base-diff.mjs +276 -0
- package/gates/ci-green.mjs +83 -3
- package/gates/feature-tests.mjs +280 -0
- package/gates/tests-only.mjs +65 -0
- package/package.json +2 -2
- package/builders/tests.mjs +0 -139
- package/gates/e2e.mjs +0 -538
- package/gates/tests-frozen.mjs +0 -96
- package/gates/tests-red.mjs +0 -227
package/gates/ci-green.mjs
CHANGED
|
@@ -20,6 +20,14 @@ import { emitVerdict, GATE_FLAGS, ownArgv, take } from "@bluewombat/slot-kit";
|
|
|
20
20
|
// to it failed only at the first assembly of a real run
|
|
21
21
|
// --require-checks wait for a check to appear; without it, a commit with no
|
|
22
22
|
// check at all passes
|
|
23
|
+
// --require-check NAME repeatable. The check run (or status context) called
|
|
24
|
+
// exactly NAME must exist and have succeeded. A `skipped` or
|
|
25
|
+
// `neutral` one is not a pass, which is what the general
|
|
26
|
+
// rule lets through (a job skipped for want of its secrets
|
|
27
|
+
// would let the line go green): it is refused. One not
|
|
28
|
+
// there yet is waited for like --require-checks, and at the
|
|
29
|
+
// ceiling the CI did not answer. A matrix is one check per
|
|
30
|
+
// cell, named as GitHub names it ("e2e (chrome)"): list each one.
|
|
23
31
|
//
|
|
24
32
|
// It also receives `--timeout-ms` from Implementer: the ceiling it will be
|
|
25
33
|
// killed at. A Gate that waits on a CI it does not run must not wait to be cut
|
|
@@ -38,6 +46,7 @@ if (!options.ok) {
|
|
|
38
46
|
process.exit(0);
|
|
39
47
|
}
|
|
40
48
|
const apiBase = options.apiBase;
|
|
49
|
+
const requireCheck = options.requireCheck;
|
|
41
50
|
|
|
42
51
|
if (git(["rev-parse", "--is-inside-work-tree"]).stdout.trim() !== "true") {
|
|
43
52
|
emitVerdict("fail-blocking", "ci-green needs a git workspace.");
|
|
@@ -93,6 +102,25 @@ for (;;) {
|
|
|
93
102
|
await sleep(Math.min(options.pollMs, Math.max(0, stopAt - Date.now())));
|
|
94
103
|
continue;
|
|
95
104
|
}
|
|
105
|
+
const required = judgeRequired(checks.named);
|
|
106
|
+
if (required.refused.length > 0) {
|
|
107
|
+
// A check that settled without running says nothing of the work, and nothing
|
|
108
|
+
// the work could change (a job skipped for want of its secrets): a person's,
|
|
109
|
+
// not an agent's to repair.
|
|
110
|
+
emitVerdict(
|
|
111
|
+
"fail-blocking",
|
|
112
|
+
`${required.refused.join("\n")}\nThis is not a refusal of the work: the CI did not run a check the Project requires, and no change to the work makes it run.`,
|
|
113
|
+
);
|
|
114
|
+
process.exit(0);
|
|
115
|
+
}
|
|
116
|
+
if (required.missing.length > 0) {
|
|
117
|
+
// Same window as above, for a check the Project named.
|
|
118
|
+
if (Date.now() >= stopAt) {
|
|
119
|
+
stopWaiting(pull, `required check ${quoted(required.missing)} has not appeared`, true);
|
|
120
|
+
}
|
|
121
|
+
await sleep(Math.min(options.pollMs, Math.max(0, stopAt - Date.now())));
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
96
124
|
if (checks.running === 0) {
|
|
97
125
|
emitVerdict("pass");
|
|
98
126
|
process.exit(0);
|
|
@@ -106,6 +134,40 @@ for (;;) {
|
|
|
106
134
|
await sleep(Math.min(options.pollMs, Math.max(0, stopAt - Date.now())));
|
|
107
135
|
}
|
|
108
136
|
|
|
137
|
+
/**
|
|
138
|
+
* What `--require-check` asks of the checks read: each name must have a
|
|
139
|
+
* success. A name not there yet is `missing` (the CI may still create it); one
|
|
140
|
+
* still running is waited for with the rest; one that settled without success
|
|
141
|
+
* (`skipped`, `neutral`) is `refused`: nothing will turn it green, and no change
|
|
142
|
+
* to the work can, so it goes to a person.
|
|
143
|
+
*
|
|
144
|
+
* @param {Map<string, string[]>} named every check by name, with its outcome:
|
|
145
|
+
* `success`, `pending`, or the conclusion it settled on
|
|
146
|
+
* @returns {{ missing: string[]; refused: string[] }}
|
|
147
|
+
*/
|
|
148
|
+
function judgeRequired(named) {
|
|
149
|
+
/** @type {string[]} */
|
|
150
|
+
const missing = [];
|
|
151
|
+
/** @type {string[]} */
|
|
152
|
+
const refused = [];
|
|
153
|
+
for (const name of requireCheck) {
|
|
154
|
+
const outcomes = named.get(name);
|
|
155
|
+
if (outcomes === undefined) {
|
|
156
|
+
missing.push(name);
|
|
157
|
+
} else if (!outcomes.includes("success") && !outcomes.includes("pending")) {
|
|
158
|
+
refused.push(
|
|
159
|
+
`${name}: ${outcomes[0]}. It is required to succeed, and a check that did not run is not a pass.`,
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return { missing, refused };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** @param {readonly string[]} names */
|
|
167
|
+
function quoted(names) {
|
|
168
|
+
return names.map((name) => `"${name}"`).join(", ");
|
|
169
|
+
}
|
|
170
|
+
|
|
109
171
|
/**
|
|
110
172
|
* The moment to stop waiting: a margin before Implementer's ceiling (5 %, between
|
|
111
173
|
* 2 and 30 seconds, and at most half of it), so the verdict is out before the
|
|
@@ -153,7 +215,7 @@ function stopWaiting(pr, what, never = false) {
|
|
|
153
215
|
|
|
154
216
|
/**
|
|
155
217
|
* @param {readonly string[]} tokens
|
|
156
|
-
* @returns {{ ok: true; remote: string; apiBase: string; pollMs: number; tokenEnv: string; requireChecks: boolean } | { ok: false; reason: string }}
|
|
218
|
+
* @returns {{ ok: true; remote: string; apiBase: string; pollMs: number; tokenEnv: string; requireChecks: boolean; requireCheck: string[] } | { ok: false; reason: string }}
|
|
157
219
|
*/
|
|
158
220
|
function parseOwnArgv(tokens) {
|
|
159
221
|
let remote = "origin";
|
|
@@ -161,6 +223,8 @@ function parseOwnArgv(tokens) {
|
|
|
161
223
|
let pollMs = 15_000;
|
|
162
224
|
let tokenEnv;
|
|
163
225
|
let requireChecks = false;
|
|
226
|
+
/** @type {string[]} */
|
|
227
|
+
const requireCheck = [];
|
|
164
228
|
for (let i = 0; i < tokens.length; i++) {
|
|
165
229
|
const token = tokens[i];
|
|
166
230
|
if (token === "--require-checks") {
|
|
@@ -176,6 +240,8 @@ function parseOwnArgv(tokens) {
|
|
|
176
240
|
remote = value;
|
|
177
241
|
} else if (token === "--api-base") {
|
|
178
242
|
apiBase = value.replace(/\/+$/, "");
|
|
243
|
+
} else if (token === "--require-check") {
|
|
244
|
+
requireCheck.push(value);
|
|
179
245
|
} else if (token === "--token-env") {
|
|
180
246
|
tokenEnv = value;
|
|
181
247
|
} else if (token === "--poll-ms") {
|
|
@@ -195,7 +261,7 @@ function parseOwnArgv(tokens) {
|
|
|
195
261
|
"ci-green needs --token-env NAME: the environment variable holding a token that can read the checks (the same one @bluewombat/manager-github's tokenEnv names, never GITHUB_TOKEN).",
|
|
196
262
|
};
|
|
197
263
|
}
|
|
198
|
-
return { ok: true, remote, apiBase, pollMs, tokenEnv, requireChecks };
|
|
264
|
+
return { ok: true, remote, apiBase, pollMs, tokenEnv, requireChecks, requireCheck };
|
|
199
265
|
}
|
|
200
266
|
|
|
201
267
|
/**
|
|
@@ -269,12 +335,25 @@ async function readChecks(sha) {
|
|
|
269
335
|
// on the same commit: same name, same log. Said once.
|
|
270
336
|
const seen = new Set();
|
|
271
337
|
let running = 0;
|
|
338
|
+
/** @type {Map<string, string[]>} */
|
|
339
|
+
const named = new Map();
|
|
340
|
+
/**
|
|
341
|
+
* @param {unknown} name
|
|
342
|
+
* @param {string} outcome
|
|
343
|
+
*/
|
|
344
|
+
const note = (name, outcome) => {
|
|
345
|
+
if (typeof name === "string") {
|
|
346
|
+
named.set(name, [...(named.get(name) ?? []), outcome]);
|
|
347
|
+
}
|
|
348
|
+
};
|
|
272
349
|
for (const entry of checkRuns) {
|
|
273
350
|
const run = record(entry) ?? {};
|
|
274
351
|
if (run.status !== "completed") {
|
|
275
352
|
running += 1;
|
|
353
|
+
note(run.name, "pending");
|
|
276
354
|
continue;
|
|
277
355
|
}
|
|
356
|
+
note(run.name, String(run.conclusion));
|
|
278
357
|
if (
|
|
279
358
|
run.conclusion === "success" ||
|
|
280
359
|
run.conclusion === "neutral" ||
|
|
@@ -293,6 +372,7 @@ async function readChecks(sha) {
|
|
|
293
372
|
// and where it said more is all there is.
|
|
294
373
|
for (const entry of statuses) {
|
|
295
374
|
const status = record(entry) ?? {};
|
|
375
|
+
note(status.context, status.state === "pending" ? "pending" : String(status.state));
|
|
296
376
|
if (status.state === "pending") {
|
|
297
377
|
running += 1;
|
|
298
378
|
continue;
|
|
@@ -310,7 +390,7 @@ async function readChecks(sha) {
|
|
|
310
390
|
);
|
|
311
391
|
failed.push([heading, ...more].join("\n"));
|
|
312
392
|
}
|
|
313
|
-
return { total: checkRuns.length + statuses.length, running, failed };
|
|
393
|
+
return { total: checkRuns.length + statuses.length, running, failed, named };
|
|
314
394
|
}
|
|
315
395
|
|
|
316
396
|
/**
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { spawn } from "node:child_process";
|
|
3
|
+
import { existsSync, readFileSync, rmSync } from "node:fs";
|
|
4
|
+
import { isAbsolute, resolve } from "node:path";
|
|
5
|
+
import {
|
|
6
|
+
emitVerdict,
|
|
7
|
+
GATE_FLAGS,
|
|
8
|
+
ownArgv,
|
|
9
|
+
parseOptions,
|
|
10
|
+
take,
|
|
11
|
+
textOption,
|
|
12
|
+
} from "@bluewombat/slot-kit";
|
|
13
|
+
import { parseJunit, scenarioOf } from "../lib/junit.mjs";
|
|
14
|
+
import { createOutputTail } from "../lib/output-tail.mjs";
|
|
15
|
+
|
|
16
|
+
// The tests of a Feature, run on the assembled Feature line: does the Feature do what the
|
|
17
|
+
// operator agreed it should? Only the scenarios of this Feature run, not the Project's whole suite.
|
|
18
|
+
//
|
|
19
|
+
// Nothing here knows a stack. The Project supplies one command, run in the workspace through the
|
|
20
|
+
// shell, and the Gate owns the rest: running, comparing.
|
|
21
|
+
//
|
|
22
|
+
// --run COMMAND (required): runs the tests of the Feature and writes a JUnit XML report. `{ids}`
|
|
23
|
+
// stands for the scenario ids of the manifest joined with "|" (a regular expression:
|
|
24
|
+
// `pnpm test --grep "{ids}"`), `{junit}` for the path the report must be written to.
|
|
25
|
+
// --manifest PATH: the scenarios of the Feature. Default feature-tests/scenarios.json: a JSON
|
|
26
|
+
// array of ids, or { "scenarios": [ "S1" | { "id": "S1", "title": "…" } ] }.
|
|
27
|
+
// --junit PATH: where --run writes its report. Default feature-tests/results/junit.xml. Removed
|
|
28
|
+
// before the run.
|
|
29
|
+
// --max-run-ms N: the most the whole Gate may take. Default 600000 (10 minutes), and never more
|
|
30
|
+
// than the Gate's own ceiling.
|
|
31
|
+
//
|
|
32
|
+
// A scenario is a manifest id; each test carries its id in its name. The comparison is exact: an
|
|
33
|
+
// id with no test, a test with no id, or a test the runner skipped is a failure.
|
|
34
|
+
//
|
|
35
|
+
// Verdicts: pass; fail-retryable for a red run (a wrong test is its author's to fix, a wrong
|
|
36
|
+
// application the producer's: what the caller does with the report is its own rule); fail-blocking for what no change to the application fixes: a missing manifest,
|
|
37
|
+
// a run that writes no report or outlives its ceiling.
|
|
38
|
+
|
|
39
|
+
const cwd = process.cwd();
|
|
40
|
+
const KEEP_BYTES = 12_000;
|
|
41
|
+
const MARGIN_MS = 5_000;
|
|
42
|
+
const MIN_PHASE_MS = 1_000;
|
|
43
|
+
const REPORT_CHARS = 8_000;
|
|
44
|
+
|
|
45
|
+
const parsed = parseOptions(
|
|
46
|
+
ownArgv(process.argv, GATE_FLAGS),
|
|
47
|
+
{
|
|
48
|
+
"--run": {},
|
|
49
|
+
"--manifest": { fallback: "feature-tests/scenarios.json" },
|
|
50
|
+
"--junit": { fallback: "feature-tests/results/junit.xml" },
|
|
51
|
+
"--max-run-ms": { fallback: "600000" },
|
|
52
|
+
},
|
|
53
|
+
"feature-tests",
|
|
54
|
+
);
|
|
55
|
+
if (!parsed.ok) {
|
|
56
|
+
emitVerdict("fail-blocking", parsed.reason);
|
|
57
|
+
process.exit(0);
|
|
58
|
+
}
|
|
59
|
+
const runTemplate = textOption(parsed.values, "run");
|
|
60
|
+
if (runTemplate === undefined || runTemplate.trim().length === 0) {
|
|
61
|
+
emitVerdict(
|
|
62
|
+
"fail-blocking",
|
|
63
|
+
'feature-tests needs --run "<command>". The Project supplies the command that runs the tests.',
|
|
64
|
+
);
|
|
65
|
+
process.exit(0);
|
|
66
|
+
}
|
|
67
|
+
const maxRunMs = positive(textOption(parsed.values, "maxRunMs"));
|
|
68
|
+
if (maxRunMs === undefined) {
|
|
69
|
+
emitVerdict("fail-blocking", "feature-tests: --max-run-ms takes a positive integer.");
|
|
70
|
+
process.exit(0);
|
|
71
|
+
}
|
|
72
|
+
const manifestPath = inWorkspace(textOption(parsed.values, "manifest") ?? "");
|
|
73
|
+
const junitPath = inWorkspace(textOption(parsed.values, "junit") ?? "");
|
|
74
|
+
|
|
75
|
+
const manifest = readManifest();
|
|
76
|
+
if (typeof manifest === "string") {
|
|
77
|
+
emitVerdict("fail-blocking", manifest);
|
|
78
|
+
process.exit(0);
|
|
79
|
+
}
|
|
80
|
+
// Typed apart from the early exit above: the functions below are hoisted, and a narrowing does not reach them.
|
|
81
|
+
/** @type {string[]} */
|
|
82
|
+
const ids = manifest;
|
|
83
|
+
|
|
84
|
+
const ceilingMs = Number(take(process.argv, "--timeout-ms"));
|
|
85
|
+
const budgetMs =
|
|
86
|
+
Number.isFinite(ceilingMs) && ceilingMs > 0
|
|
87
|
+
? Math.min(maxRunMs, Math.max(MIN_PHASE_MS, ceilingMs - MARGIN_MS))
|
|
88
|
+
: maxRunMs;
|
|
89
|
+
|
|
90
|
+
const [verdict, report] = await judge();
|
|
91
|
+
emitVerdict(/** @type {"pass" | "fail-retryable" | "fail-blocking"} */ (verdict), report);
|
|
92
|
+
process.exit(0);
|
|
93
|
+
|
|
94
|
+
/** @returns {Promise<[string, string]>} */
|
|
95
|
+
async function judge() {
|
|
96
|
+
rmSync(junitPath, { force: true });
|
|
97
|
+
const command = runTemplate?.replaceAll("{ids}", ids.join("|")).replaceAll("{junit}", junitPath);
|
|
98
|
+
const run = await runShell(/** @type {string} */ (command), budgetMs);
|
|
99
|
+
if (run.timedOut) {
|
|
100
|
+
return ["fail-blocking", timeoutReport(run.output)];
|
|
101
|
+
}
|
|
102
|
+
const results = readResults();
|
|
103
|
+
if (typeof results === "string") {
|
|
104
|
+
return run.code === 0
|
|
105
|
+
? ["fail-blocking", results]
|
|
106
|
+
: ["fail-retryable", `${results}\n${exitLine(run)} End of its output:\n${run.output}`];
|
|
107
|
+
}
|
|
108
|
+
const failing = failingOf(results);
|
|
109
|
+
if (failing.length > 0) {
|
|
110
|
+
return ["fail-retryable", redReport(results, failing)];
|
|
111
|
+
}
|
|
112
|
+
const foreign = results.cases.filter((c) => scenarioOf(c.name, ids) === undefined);
|
|
113
|
+
if (foreign.length > 0) {
|
|
114
|
+
return [
|
|
115
|
+
"fail-retryable",
|
|
116
|
+
[
|
|
117
|
+
`feature-tests: ${foreign.length} test(s) carry no scenario id of the manifest (${ids.join(", ")}): only the scenarios of the Feature run.`,
|
|
118
|
+
...foreign.map((c) => ` ${c.name}`),
|
|
119
|
+
"Every test names its scenario id, and --run must run the ids of the manifest only ({ids}).",
|
|
120
|
+
].join("\n"),
|
|
121
|
+
];
|
|
122
|
+
}
|
|
123
|
+
return [
|
|
124
|
+
"pass",
|
|
125
|
+
`feature-tests: ${ids.length} scenario(s) of the feature ran and passed: ${ids.join(", ")}.`,
|
|
126
|
+
];
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* @typedef {{ cases: import("../lib/junit.mjs").TestCase[] }} Result
|
|
131
|
+
*/
|
|
132
|
+
|
|
133
|
+
/** @returns {Result | string} */
|
|
134
|
+
function readResults() {
|
|
135
|
+
if (!existsSync(junitPath)) {
|
|
136
|
+
return `The run wrote no JUnit report at ${junitPath}. --run must write one there ({junit}).`;
|
|
137
|
+
}
|
|
138
|
+
const cases = parseJunit(readFileSync(junitPath, "utf8"));
|
|
139
|
+
if (cases.length === 0) {
|
|
140
|
+
return `The JUnit report at ${junitPath} holds no test case.`;
|
|
141
|
+
}
|
|
142
|
+
return { cases };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** The scenario ids that are not a pass: failed, skipped, or never run. */
|
|
146
|
+
function failingOf(/** @type {Result} */ results) {
|
|
147
|
+
return ids.filter((id) => {
|
|
148
|
+
const mine = results.cases.filter((c) => scenarioOf(c.name, ids) === id);
|
|
149
|
+
return mine.length === 0 || mine.some((c) => c.status !== "passed");
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function redReport(/** @type {Result} */ results, /** @type {string[]} */ failing) {
|
|
154
|
+
const lines = [
|
|
155
|
+
`feature-tests: ${failing.length} of ${ids.length} scenario(s) failed on the assembled feature: ${failing.join(", ")}.`,
|
|
156
|
+
];
|
|
157
|
+
for (const id of failing) {
|
|
158
|
+
const mine = results.cases.filter((c) => scenarioOf(c.name, ids) === id);
|
|
159
|
+
if (mine.length === 0) {
|
|
160
|
+
lines.push(`${id} — no test ran for it.`);
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
for (const c of mine.filter((m) => m.status !== "passed")) {
|
|
164
|
+
lines.push(`${id} — ${c.name}`);
|
|
165
|
+
if (c.status === "skipped") {
|
|
166
|
+
lines.push(" skipped: a skipped scenario is not a passing one.");
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
lines.push(` at: ${firstFrame(c.detail) ?? "(no step given)"}`);
|
|
170
|
+
lines.push(` message: ${c.message.slice(0, 600)}`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
lines.push(`Report: ${junitPath}.`);
|
|
174
|
+
const text = lines.join("\n");
|
|
175
|
+
return text.length <= REPORT_CHARS ? text : `${text.slice(0, REPORT_CHARS)}\n…`;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** The first stack line that is the test's own, where a step failed. */
|
|
179
|
+
function firstFrame(/** @type {string} */ detail) {
|
|
180
|
+
return detail
|
|
181
|
+
.split("\n")
|
|
182
|
+
.map((line) => line.trim())
|
|
183
|
+
.find((line) => /^at\s|\.(spec|test)\.[cm]?[jt]sx?:\d+/.test(line));
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* @param {string} command
|
|
188
|
+
* @param {number} limitMs
|
|
189
|
+
* @returns {Promise<{ code: number | null, signal: string | null, timedOut: boolean, output: string }>}
|
|
190
|
+
*/
|
|
191
|
+
function runShell(command, limitMs) {
|
|
192
|
+
return new Promise((done) => {
|
|
193
|
+
const child = spawn("sh", ["-c", command], {
|
|
194
|
+
cwd,
|
|
195
|
+
env: { ...process.env, CI: "true", NO_COLOR: "1", FORCE_COLOR: "0" },
|
|
196
|
+
detached: true,
|
|
197
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
198
|
+
});
|
|
199
|
+
const tail = createOutputTail(KEEP_BYTES);
|
|
200
|
+
child.stdout.on("data", (chunk) => tail.push("stdout", chunk));
|
|
201
|
+
child.stderr.on("data", (chunk) => tail.push("stderr", chunk));
|
|
202
|
+
let timedOut = false;
|
|
203
|
+
const timer = globalThis.setTimeout(
|
|
204
|
+
() => {
|
|
205
|
+
timedOut = true;
|
|
206
|
+
killGroup(child);
|
|
207
|
+
},
|
|
208
|
+
Math.max(MIN_PHASE_MS, limitMs),
|
|
209
|
+
);
|
|
210
|
+
child.on("error", (error) => {
|
|
211
|
+
clearTimeout(timer);
|
|
212
|
+
done({ code: 127, signal: null, timedOut: false, output: `cannot start: ${error.message}` });
|
|
213
|
+
});
|
|
214
|
+
child.on("close", (code, signal) => {
|
|
215
|
+
clearTimeout(timer);
|
|
216
|
+
done({ code, signal, timedOut, output: tail.text() });
|
|
217
|
+
});
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** @param {import("node:child_process").ChildProcess} child */
|
|
222
|
+
function killGroup(child) {
|
|
223
|
+
try {
|
|
224
|
+
if (child.pid !== undefined) {
|
|
225
|
+
process.kill(-child.pid, "SIGKILL");
|
|
226
|
+
}
|
|
227
|
+
} catch {
|
|
228
|
+
child.kill("SIGKILL");
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function timeoutReport(/** @type {string} */ output) {
|
|
233
|
+
return [
|
|
234
|
+
`The run did not finish within the ${Math.round(budgetMs / 1000)}s this Gate may take, and was stopped.`,
|
|
235
|
+
"A person decides: the tests hang.",
|
|
236
|
+
output.length > 0 ? `Last output:\n${output}` : "It printed nothing.",
|
|
237
|
+
].join("\n");
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function exitLine(/** @type {{ code: number | null, signal: string | null }} */ run) {
|
|
241
|
+
return run.signal === null ? `The run exited ${run.code}.` : `The run got ${run.signal}.`;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** @returns {string[] | string} The ids, or why the manifest is not usable. */
|
|
245
|
+
function readManifest() {
|
|
246
|
+
if (!existsSync(manifestPath)) {
|
|
247
|
+
return `The scenarios manifest ${manifestPath} does not exist. The feature-test Ticket writes it: one id per scenario.`;
|
|
248
|
+
}
|
|
249
|
+
let raw;
|
|
250
|
+
try {
|
|
251
|
+
raw = JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
252
|
+
} catch (error) {
|
|
253
|
+
return `The scenarios manifest ${manifestPath} is not JSON: ${error instanceof Error ? error.message : error}`;
|
|
254
|
+
}
|
|
255
|
+
const list = Array.isArray(raw) ? raw : raw?.scenarios;
|
|
256
|
+
if (!Array.isArray(list)) {
|
|
257
|
+
return `The scenarios manifest ${manifestPath} must be an array of ids, or { "scenarios": [...] }.`;
|
|
258
|
+
}
|
|
259
|
+
const found = list.map((entry) => (typeof entry === "string" ? entry : entry?.id));
|
|
260
|
+
if (found.some((id) => typeof id !== "string" || id.trim().length === 0)) {
|
|
261
|
+
return `Every scenario of ${manifestPath} needs a non-empty id.`;
|
|
262
|
+
}
|
|
263
|
+
const unique = [...new Set(found.map((id) => id.trim()))];
|
|
264
|
+
if (unique.length !== found.length) {
|
|
265
|
+
return `The scenarios manifest ${manifestPath} lists an id twice.`;
|
|
266
|
+
}
|
|
267
|
+
if (unique.length === 0) {
|
|
268
|
+
return `The scenarios manifest ${manifestPath} lists no scenario: there is nothing to prove.`;
|
|
269
|
+
}
|
|
270
|
+
return unique;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function positive(/** @type {string | undefined} */ text) {
|
|
274
|
+
const value = Number(text);
|
|
275
|
+
return Number.isInteger(value) && value > 0 ? value : undefined;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function inWorkspace(/** @type {string} */ path) {
|
|
279
|
+
return isAbsolute(path) ? path : resolve(cwd, path);
|
|
280
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { spawnSync } from "node:child_process";
|
|
3
|
+
import { emitVerdict, GATE_FLAGS, ownArgv, pathMatchesGlob } from "@bluewombat/slot-kit";
|
|
4
|
+
|
|
5
|
+
// Fail-retryable when the workspace changed a path that matches none of the globs: the one place
|
|
6
|
+
// the workspace may be written is what the globs name (the tests and their manifest). The other
|
|
7
|
+
// side of `sensitive-path`, which names what may not be touched; a Project whose tests sit among its
|
|
8
|
+
// code (`**/*.test.ts` in every package) cannot say "everything but" with globs that only deny.
|
|
9
|
+
// Remaining argv after Implementer's flags are the globs.
|
|
10
|
+
//
|
|
11
|
+
// What counts is what a fold can carry: tracked paths that differ from HEAD (added, modified,
|
|
12
|
+
// deleted, renamed) and new paths .gitignore does not exclude.
|
|
13
|
+
|
|
14
|
+
const cwd = process.cwd();
|
|
15
|
+
const globs = ownArgv(process.argv, GATE_FLAGS).filter((token) => token.length > 0);
|
|
16
|
+
|
|
17
|
+
if (globs.length === 0) {
|
|
18
|
+
emitVerdict("fail-blocking", "tests-only needs at least one glob.");
|
|
19
|
+
process.exit(0);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const status = spawnSync(
|
|
23
|
+
"git",
|
|
24
|
+
["-C", cwd, "status", "--porcelain=v1", "-z", "--untracked-files=all"],
|
|
25
|
+
{
|
|
26
|
+
encoding: "utf8",
|
|
27
|
+
},
|
|
28
|
+
);
|
|
29
|
+
if (status.status !== 0) {
|
|
30
|
+
emitVerdict("fail-blocking", `tests-only needs a git workspace: ${status.stderr.trim()}`);
|
|
31
|
+
process.exit(0);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** @type {Set<string>} */
|
|
35
|
+
const changed = new Set();
|
|
36
|
+
const fields = status.stdout.split("\0");
|
|
37
|
+
for (let i = 0; i < fields.length; i++) {
|
|
38
|
+
const entry = fields[i] ?? "";
|
|
39
|
+
if (entry.length < 4) {
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
changed.add(entry.slice(3));
|
|
43
|
+
// A rename or a copy carries the old path in the next field.
|
|
44
|
+
if (entry[0] === "R" || entry[0] === "C") {
|
|
45
|
+
const old = fields[++i];
|
|
46
|
+
if (old !== undefined && old.length > 0) {
|
|
47
|
+
changed.add(old);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const outside = [...changed]
|
|
53
|
+
.map((path) => path.replaceAll("\\", "/"))
|
|
54
|
+
.filter((path) => !globs.some((glob) => pathMatchesGlob(path, glob)))
|
|
55
|
+
.sort();
|
|
56
|
+
|
|
57
|
+
if (outside.length > 0) {
|
|
58
|
+
emitVerdict(
|
|
59
|
+
"fail-retryable",
|
|
60
|
+
`Paths changed that are not tests or their manifest (${globs.join(", ")}):\n${outside.join("\n")}`,
|
|
61
|
+
);
|
|
62
|
+
process.exit(0);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
emitVerdict("pass");
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bluewombat/slots",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Shipped mason slots: Builders, Gates, publishers, and refreshers a Project names in its config.",
|
|
6
6
|
"keywords": [
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"test": "tsx --test $(find test -name '*.test.ts')"
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
|
-
"@bluewombat/slot-kit": "^0.
|
|
55
|
+
"@bluewombat/slot-kit": "^0.11.0",
|
|
56
56
|
"@pinkpuma/deck": "1.1.0"
|
|
57
57
|
}
|
|
58
58
|
}
|
package/builders/tests.mjs
DELETED
|
@@ -1,139 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
BUILDER_FLAGS,
|
|
4
|
-
emitFailure,
|
|
5
|
-
fillPrompt,
|
|
6
|
-
finishRun,
|
|
7
|
-
loadPrompt,
|
|
8
|
-
loadRules,
|
|
9
|
-
PROMPT_RULES,
|
|
10
|
-
parseRole,
|
|
11
|
-
spawnFilled,
|
|
12
|
-
take,
|
|
13
|
-
textOption,
|
|
14
|
-
transcriptArgs,
|
|
15
|
-
} from "@bluewombat/slot-kit";
|
|
16
|
-
import { attemptOf } from "../lib/fill.mjs";
|
|
17
|
-
|
|
18
|
-
const NAME = "builder tests";
|
|
19
|
-
|
|
20
|
-
const DEFAULT_TEMPLATE = `You are a coding agent working unattended in a checkout of the project.
|
|
21
|
-
|
|
22
|
-
Task:
|
|
23
|
-
{{task}}
|
|
24
|
-
|
|
25
|
-
YOUR WORK NOW IS TO WRITE THE TESTS OF THIS TASK, AND ONLY THE TESTS. Another agent
|
|
26
|
-
writes the code afterwards and must make your tests pass without editing them. A test it
|
|
27
|
-
cannot pass is not sent back to you: it stops the task and waits for a person. So a test
|
|
28
|
-
you deliver must be one that a correct implementation of the task passes, and you prove
|
|
29
|
-
it before you hand it over (see below).
|
|
30
|
-
|
|
31
|
-
## The cases
|
|
32
|
-
|
|
33
|
-
- The task has a "Tests" section: write one test per case of it, in the file and the
|
|
34
|
-
framework it names (or the ones this repository already uses), each named after the
|
|
35
|
-
case it covers. Test the behaviour at the seam the section names, not how it is built.
|
|
36
|
-
- The task has no "Tests" section: do not look for one elsewhere. Derive the cases from
|
|
37
|
-
its acceptance criteria ("Done when" or the like), one case per criterion, and name each
|
|
38
|
-
test after the criterion and the identifiers it carries. A criterion that describes two
|
|
39
|
-
sets of data (sixty rows for a page, more than a cap for the cap) is two cases, never one
|
|
40
|
-
test with two batches of data.
|
|
41
|
-
- Do not write the behaviour. If the interface under test does not exist yet, create the
|
|
42
|
-
minimal stub — the name and signature the task gives, a body that does nothing useful —
|
|
43
|
-
so that the tests compile and fail on what they check, not on a missing import.
|
|
44
|
-
- Add or update no other test, and no production code beyond the stubs.
|
|
45
|
-
|
|
46
|
-
## How a test stays passable
|
|
47
|
-
|
|
48
|
-
- One data set per test, seeded at the top. Every expectation derives from that set: the
|
|
49
|
-
rows a filter keeps, the order they take, the page a limit and an offset cut out of it.
|
|
50
|
-
Write the expected count from the seeded data, not as a figure recounted by hand.
|
|
51
|
-
- To observe a bound, seed past it: a cap of 200 is only visible with more than 200 rows
|
|
52
|
-
that the filters keep. "At most N" is asserted with \`<= N\`, or by seeding more than N;
|
|
53
|
-
never as exactly N when fewer exist.
|
|
54
|
-
|
|
55
|
-
## Prove them: red, green, red
|
|
56
|
-
|
|
57
|
-
A test you have only seen fail proves nothing about itself: a test no implementation
|
|
58
|
-
could pass fails exactly the same way. The tests must FAIL now, because the behaviour is
|
|
59
|
-
missing, and you prove every one of them passable before you finish:
|
|
60
|
-
|
|
61
|
-
1. **Red.** Run the tests. Each fails on an assertion (not on an import, a type, a syntax
|
|
62
|
-
error).
|
|
63
|
-
2. **Green.** In the stub bodies — the same files, no new file — write the shortest
|
|
64
|
-
implementation of the task that makes every test pass: naive, in memory, unoptimised;
|
|
65
|
-
it is thrown away. Run the tests. A test still red here is wrong, not the task: fix its
|
|
66
|
-
data or its expectation (never drop a case, never replace a value the task gives with a
|
|
67
|
-
looser one), and run again until all are green.
|
|
68
|
-
3. **Red again.** Put every stub body back to no behaviour. Run the tests: each fails
|
|
69
|
-
again on an assertion. Leave the tree in this state.
|
|
70
|
-
|
|
71
|
-
Leave no trace of the green step: no helper, no import, no comment. The only non-test
|
|
72
|
-
lines you deliver are the stubs.
|
|
73
|
-
|
|
74
|
-
{{history}}
|
|
75
|
-
|
|
76
|
-
## When you cannot act
|
|
77
|
-
|
|
78
|
-
If the cases are missing, or contradict the task or the code, do not invent them. Change
|
|
79
|
-
nothing and end your final message with exactly one line:
|
|
80
|
-
|
|
81
|
-
MASON_BLOCKED: <what is missing, in one line>
|
|
82
|
-
|
|
83
|
-
A person reads that line and decides. It is a correct answer, not a failure.
|
|
84
|
-
|
|
85
|
-
Rules of this run:
|
|
86
|
-
{{rules}}
|
|
87
|
-
`;
|
|
88
|
-
|
|
89
|
-
// A retry carries the refusal in --history; the flags themselves are consumed so the
|
|
90
|
-
// role options see only their own.
|
|
91
|
-
take(process.argv, "--report");
|
|
92
|
-
take(process.argv, "--report-from");
|
|
93
|
-
|
|
94
|
-
const intention = (take(process.argv, "--intention") ?? "").trim();
|
|
95
|
-
if (intention.length === 0) {
|
|
96
|
-
emitFailure("fail-blocking", "The Builder received no --intention.");
|
|
97
|
-
process.exit(1);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
const parsed = parseRole(process.argv, BUILDER_FLAGS, NAME);
|
|
101
|
-
if (!parsed.ok) {
|
|
102
|
-
emitFailure("fail-blocking", parsed.reason);
|
|
103
|
-
process.exit(1);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
const template = loadPrompt(parsed.options, DEFAULT_TEMPLATE);
|
|
107
|
-
if (!template.ok) {
|
|
108
|
-
emitFailure("fail-blocking", template.reason);
|
|
109
|
-
process.exit(1);
|
|
110
|
-
}
|
|
111
|
-
const rules = loadRules(parsed.options, PROMPT_RULES);
|
|
112
|
-
if (!rules.ok) {
|
|
113
|
-
emitFailure("fail-blocking", rules.reason);
|
|
114
|
-
process.exit(1);
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
const rendered = fillPrompt(
|
|
118
|
-
template.value,
|
|
119
|
-
{
|
|
120
|
-
task: intention,
|
|
121
|
-
history: (take(process.argv, "--history") ?? "").trim(),
|
|
122
|
-
rules: rules.value.trim(),
|
|
123
|
-
id: take(process.argv, "--id") ?? "",
|
|
124
|
-
attempt: String(attemptOf(process.argv)),
|
|
125
|
-
},
|
|
126
|
-
["task", "rules"],
|
|
127
|
-
textOption(parsed.options, "promptFile") ?? "built-in template",
|
|
128
|
-
);
|
|
129
|
-
if (!rendered.ok) {
|
|
130
|
-
emitFailure("fail-blocking", rendered.reason);
|
|
131
|
-
process.exit(1);
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
const spawned = await spawnFilled(parsed.runner, rendered.prompt, transcriptArgs(process.argv));
|
|
135
|
-
if (!spawned.ok) {
|
|
136
|
-
emitFailure("fail-blocking", spawned.reason);
|
|
137
|
-
process.exit(1);
|
|
138
|
-
}
|
|
139
|
-
finishRun({ run: spawned.run, bin: spawned.about.bin, name: spawned.about.name });
|