@sleepy_panda_srl/portulan 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/cli/README.md +69 -2
- package/cli/compile.mjs +555 -37
- package/cli/doctor.mjs +5 -1
- package/cli/drills.mjs +1541 -0
- package/cli/fuzz-shell.mjs +880 -0
- package/cli/gate.mjs +3 -2
- package/cli/goldens.mjs +469 -0
- package/cli/init.mjs +4 -1
- package/cli/mutants.mjs +925 -0
- package/cli/pack-identity.mjs +9 -2
- package/cli/pack-version.mjs +7 -0
- package/cli/recipe-set.mjs +6 -0
- package/cli/release-eval.mjs +1068 -0
- package/cli/review-meter.mjs +741 -0
- package/cli/skill-goldens.mjs +590 -0
- package/cli/stop-gate.mjs +107 -4
- package/cli/telemetry.mjs +1047 -0
- package/cli/version-carriers.mjs +13 -0
- package/core/templates/gate-policy.md +27 -8
- package/evals/README.md +910 -0
- package/evals/releases/0.1.3.json +134 -0
- package/evals/releases/0.1.3.md +64 -0
- package/evals/releases/README.md +32 -0
- package/package.json +7 -2
- package/packs/rituals/checkpoints/README.md +8 -1
- package/packs/rituals/checkpoints/pack.json +1 -1
- package/packs/rituals/checkpoints/skills/pre-commit/SKILL.md +32 -2
- package/spec/slots.md +26 -2
package/cli/mutants.mjs
ADDED
|
@@ -0,0 +1,925 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Mutation-test this repository's two gate matchers against the corpus that claims to cover them.
|
|
3
|
+
//
|
|
4
|
+
// Milestone 8, clause (b), first half: *mutation testing over both matchers.* The second half —
|
|
5
|
+
// grammar-aware fuzzing over the shell segmenter — is ./fuzz-shell.mjs, and the two are separate
|
|
6
|
+
// because a surviving mutant and a grammar bypass are different verdicts with different repairs.
|
|
7
|
+
//
|
|
8
|
+
// ## The question this answers, and it is the one `./goldens.mjs` says out loud that it cannot
|
|
9
|
+
//
|
|
10
|
+
// That runner prints on every green: *"this is a PRESENCE floor — whether a corpus is a real attack
|
|
11
|
+
// is a reviewer's judgement, not this rail's."* One trivial fixture per rule satisfies it while
|
|
12
|
+
// proving nothing adversarial. This runner stands in exactly that stated gap. It does not ask whether
|
|
13
|
+
// a fixture exists; it breaks a matcher on purpose and asks whether the corpus NOTICES.
|
|
14
|
+
//
|
|
15
|
+
// A corpus that cannot tell a working matcher from a broken one is a corpus whose green means
|
|
16
|
+
// nothing, and until this file existed no check in this repository could tell those two apart.
|
|
17
|
+
//
|
|
18
|
+
// ## What a mutant is here — anchored, not generated
|
|
19
|
+
//
|
|
20
|
+
// An operator is a **declared textual substitution** into `./compile.mjs`, anchored to a named
|
|
21
|
+
// construct, required to match **exactly once**. Not a random AST perturbation: a random mutator over
|
|
22
|
+
// a 2681-line module spends most of its budget outside the matchers, and this clause is about the
|
|
23
|
+
// matchers. Every operator names the region member it attacks, and the census prints coverage per
|
|
24
|
+
// member INCLUDING THE ZEROES — otherwise "mutation testing over both matchers" is satisfiable by
|
|
25
|
+
// three operators inside `matchesRule` and no reader would see the narrowing.
|
|
26
|
+
//
|
|
27
|
+
// ## The record, and why it reads in both directions
|
|
28
|
+
//
|
|
29
|
+
// Each operator declares the outcome expected of it:
|
|
30
|
+
//
|
|
31
|
+
// - `killed` — the corpus must catch this. If it stops catching it, the kill-set has weakened.
|
|
32
|
+
// - `survives` — the corpus cannot catch it AND THAT IS PROVEN, not observed. Two proofs are
|
|
33
|
+
// admissible and each entry states which: the mutant is semantically **equivalent**
|
|
34
|
+
// to the original, or it is equivalent **under the yielded policy** — no rule of the
|
|
35
|
+
// shape that would distinguish it is declared, so no fixture could exist to kill it.
|
|
36
|
+
//
|
|
37
|
+
// **A survivor that is neither is not a resting state.** `matchesRule` is a pure function of
|
|
38
|
+
// `(rule, tool, input)` and a fixture is exactly that triple, so any non-equivalent mutant is killable
|
|
39
|
+
// by adding one case. A standing ledger of named-but-unfilled gaps would rebuild the prose hole list
|
|
40
|
+
// that clause (a) exists to have replaced, one altitude up. The repair for such a survivor is a new
|
|
41
|
+
// fixture in the corpus, in the corpus's own shape — never a new `survives` entry.
|
|
42
|
+
//
|
|
43
|
+
// Both directions are red, which is `documented-hole`'s discipline in `./goldens.mjs` applied to a
|
|
44
|
+
// second subject: an operator recorded `killed` that now survives, and an operator recorded `survives`
|
|
45
|
+
// that is now killed. The second is good news and the message says so — but a record that still
|
|
46
|
+
// describes a gap somebody closed is as wrong as one that hides a gap somebody opened.
|
|
47
|
+
//
|
|
48
|
+
// ## Three things it deliberately does not do
|
|
49
|
+
//
|
|
50
|
+
// **It does not run the `node --test` suite as a kill-set.** The corpus is the kill-set this
|
|
51
|
+
// repository designated — `../evals/README.md` says clause (b) *"needs this corpus as its kill-set
|
|
52
|
+
// and this fixture format as its output shape"* — running the suite per mutant would need the
|
|
53
|
+
// subprocesses this design refuses, and a `survives` record citing the suite as the killer would put
|
|
54
|
+
// the kill-claim in a second carrier.
|
|
55
|
+
//
|
|
56
|
+
// **It executes nothing.** Fixture command strings are data here exactly as they are in
|
|
57
|
+
// `./goldens.mjs`; the corpus holds `git push --force` and constitution-write spellings by design.
|
|
58
|
+
// The only thing this module runs is `import()` on a JavaScript file it wrote itself, and
|
|
59
|
+
// `./mutants.test.mjs` asserts it imports no process-spawning API.
|
|
60
|
+
//
|
|
61
|
+
// **It writes nothing inside the tree.** Mutants live under `os.tmpdir()`, one fresh directory each,
|
|
62
|
+
// removed in a `finally`.
|
|
63
|
+
//
|
|
64
|
+
// Exit 0 green · 1 red · 2 could not run.
|
|
65
|
+
|
|
66
|
+
import fs from "node:fs";
|
|
67
|
+
import os from "node:os";
|
|
68
|
+
import path from "node:path";
|
|
69
|
+
import { pathToFileURL } from "node:url";
|
|
70
|
+
|
|
71
|
+
import { CompileError } from "./compile.mjs";
|
|
72
|
+
import { CouldNotRun, partition, readCorpus, yieldedRules } from "./goldens.mjs";
|
|
73
|
+
|
|
74
|
+
/** The module under mutation, relative to the repository root. */
|
|
75
|
+
export const SUBJECT = "cli/compile.mjs";
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The matcher region — every function and table `matchesRule` can reach, named.
|
|
79
|
+
*
|
|
80
|
+
* This list is the **coverage floor**, and it is written down rather than derived because deriving it
|
|
81
|
+
* would mean a call-graph walker, and a walker that is wrong is wrong silently. Each member must carry
|
|
82
|
+
* at least one operator or the table below must say why not; `census` prints the per-member count
|
|
83
|
+
* including zeroes, the way `./goldens.mjs` prints its per-matcher-path census including zeroes.
|
|
84
|
+
*
|
|
85
|
+
* The three entry branches of `matchesRule` are the two matchers the clause names plus `read:`, which
|
|
86
|
+
* shares `matchesPath` with the write branch and so cannot be mutated separately from it.
|
|
87
|
+
*/
|
|
88
|
+
export const REGION = [
|
|
89
|
+
"matchesRule",
|
|
90
|
+
"matchesPath",
|
|
91
|
+
"shellWrites",
|
|
92
|
+
"shellSegments",
|
|
93
|
+
"commandSegments",
|
|
94
|
+
"shellWords",
|
|
95
|
+
"spellings",
|
|
96
|
+
"stripHeredocs",
|
|
97
|
+
"stripLeadingRedirections",
|
|
98
|
+
"REDIRECTION_TARGET",
|
|
99
|
+
"LEADING_REDIRECTION",
|
|
100
|
+
"namesTarget",
|
|
101
|
+
"normalisePath",
|
|
102
|
+
"ancestors",
|
|
103
|
+
"writesWhatItNames",
|
|
104
|
+
"FILE_WRITERS",
|
|
105
|
+
"IN_PLACE_EDITORS",
|
|
106
|
+
"COMMAND_PREFIXES",
|
|
107
|
+
"SEGMENT_LEADERS",
|
|
108
|
+
"OPERATOR",
|
|
109
|
+
"ASSIGNMENT",
|
|
110
|
+
// Added at the pre-commit checkpoint, which derived the reachable set rather than reading this
|
|
111
|
+
// list and found both missing while the sentence above said "every function and table
|
|
112
|
+
// `matchesRule` can reach". They are the tool tables the write and read branches dispatch on —
|
|
113
|
+
// `compile.mjs:92` and `:93`, read at `:1003` and `:1033` — and a list that omits a dispatch
|
|
114
|
+
// table is exactly the narrowing this census exists to make visible. Adding them found a real
|
|
115
|
+
// gap: the corpus had no `NotebookEdit` case at all.
|
|
116
|
+
"WRITE_TOOLS",
|
|
117
|
+
"READ_TOOLS",
|
|
118
|
+
];
|
|
119
|
+
|
|
120
|
+
/** The two outcomes an operator may record. */
|
|
121
|
+
export const OUTCOMES = ["killed", "survives"];
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The operators, and the outcome recorded for each.
|
|
125
|
+
*
|
|
126
|
+
* **One table, in this module, deliberately.** An operator is an anchor into `./compile.mjs`'s source
|
|
127
|
+
* text plus the outcome expected of it; splitting those across code and a JSON file would make two
|
|
128
|
+
* carriers of one operator, which is the defect this repository names more often than any other. The
|
|
129
|
+
* corpus is data because it is per-rule and large; an operator is neither.
|
|
130
|
+
*
|
|
131
|
+
* `find` must occur **exactly once** in the subject. A missing or ambiguous anchor is could-not-run,
|
|
132
|
+
* never a skip — see `mutate`.
|
|
133
|
+
*/
|
|
134
|
+
export const OPERATORS = [
|
|
135
|
+
// ---------------------------------------------------------------------------------- matchesRule
|
|
136
|
+
{
|
|
137
|
+
id: "matchesRule-shell-drops-exact-match",
|
|
138
|
+
member: "matchesRule",
|
|
139
|
+
find: "action.shell.endsWith(\"/\") ? s.startsWith(action.shell) : s === action.shell || s.startsWith(`${action.shell} `)",
|
|
140
|
+
replace: "action.shell.endsWith(\"/\") ? s.startsWith(action.shell) : s.startsWith(`${action.shell} `)",
|
|
141
|
+
outcome: "killed",
|
|
142
|
+
why: "A command spelled EXACTLY as its target — `git commit` with no arguments — stops matching.",
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
id: "matchesRule-shell-drops-word-boundary",
|
|
146
|
+
member: "matchesRule",
|
|
147
|
+
find: "s === action.shell || s.startsWith(`${action.shell} `)",
|
|
148
|
+
replace: "s === action.shell || s.startsWith(action.shell)",
|
|
149
|
+
outcome: "killed",
|
|
150
|
+
why: "The whitespace boundary is what stops `git push` covering `git pushall`. Removing it is a false red on an unrelated command.",
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
id: "matchesRule-shell-drops-segment-arm",
|
|
154
|
+
member: "matchesRule",
|
|
155
|
+
find: "return spellings(input.command).some(reach) || commandSegments(input.command).some((seg) => spellings(seg).some(reach));",
|
|
156
|
+
replace: "return spellings(input.command).some(reach);",
|
|
157
|
+
outcome: "killed",
|
|
158
|
+
why: "Reopens hole 2 wholesale: a gated command behind any separator reaches nothing.",
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
id: "matchesRule-inserts-a-no-op",
|
|
162
|
+
member: "matchesRule",
|
|
163
|
+
find: "const action = rule?.action ?? {};",
|
|
164
|
+
replace: "const action = rule?.action ?? {};\n void action;",
|
|
165
|
+
outcome: "survives",
|
|
166
|
+
why: "EQUIVALENT: a no-op statement. Present as the table's own control — an operator that changes the source and not the behaviour must survive, or the harness is reporting kills it did not make.",
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
id: "matchesRule-shell-stops-segmenting-a-spelling",
|
|
170
|
+
member: "matchesRule",
|
|
171
|
+
find: "const reach = (s) => hit(s) || commandSegments(s).some(hit);",
|
|
172
|
+
replace: "const reach = (s) => hit(s);",
|
|
173
|
+
outcome: "killed",
|
|
174
|
+
why: "Reverts the 2026-08-25 repair and reopens the live bypass it closed: `bash -c \"ls; git push --force origin main\"` — one wrapper, one separator, every Gated shell action.",
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
id: "matchesRule-write-drops-bash-branch",
|
|
178
|
+
member: "matchesRule",
|
|
179
|
+
find: " if (tool === \"Bash\") {\n // Whole line and per segment, exactly as the `shell` branch above",
|
|
180
|
+
replace: " if (false && tool === \"Bash\") {\n // Whole line and per segment, exactly as the `shell` branch above",
|
|
181
|
+
outcome: "killed",
|
|
182
|
+
why: "Reopens the constitution's shell half entirely — `echo x >> docs/vision.md` gated by neither layer.",
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
id: "matchesRule-write-drops-segment-arm",
|
|
186
|
+
member: "matchesRule",
|
|
187
|
+
find: " spellings(input.command).some(writes) ||\n commandSegments(input.command).some((seg) => spellings(seg).some(writes))",
|
|
188
|
+
replace: " spellings(input.command).some(writes)",
|
|
189
|
+
outcome: "killed",
|
|
190
|
+
why: "The four write-gate holes #71 closed reopen; each is reachable only through this second arm.",
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
id: "matchesRule-read-accepts-any-tool",
|
|
194
|
+
member: "matchesRule",
|
|
195
|
+
find: "if (typeof action.read === \"string\" && READ_TOOLS.includes(tool)) {",
|
|
196
|
+
replace: "if (typeof action.read === \"string\") {",
|
|
197
|
+
outcome: "survives",
|
|
198
|
+
why: "EQUIVALENT UNDER THE YIELDED POLICY: the one `read:` rule declared is `read-anything-in-the-repository`, whose target is `./` — hole 8, `matchesPath` answers false for every input whatever the tool. No fixture can distinguish this, and none should be invented to; the mutant becomes killable the day a `read:` rule with a real target is declared.",
|
|
199
|
+
},
|
|
200
|
+
// ---------------------------------------------------------------------------------- matchesPath
|
|
201
|
+
{
|
|
202
|
+
id: "matchesPath-subtree-becomes-tail",
|
|
203
|
+
member: "matchesPath",
|
|
204
|
+
find: "return clean.endsWith(\"/\") ? normalised.includes(`/${clean}`) : normalised.endsWith(`/${clean}`);",
|
|
205
|
+
replace: "return normalised.endsWith(`/${clean}`);",
|
|
206
|
+
outcome: "killed",
|
|
207
|
+
why: "A subtree target stops covering its subtree: `core/` no longer matches `core/engine.md`.",
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
id: "matchesPath-tail-becomes-substring",
|
|
211
|
+
member: "matchesPath",
|
|
212
|
+
find: "clean.endsWith(\"/\") ? normalised.includes(`/${clean}`) : normalised.endsWith(`/${clean}`)",
|
|
213
|
+
replace: "normalised.includes(`/${clean}`)",
|
|
214
|
+
outcome: "killed",
|
|
215
|
+
why: "An exact target starts matching any path CONTAINING it — `docs/vision.md.bak` becomes the constitution.",
|
|
216
|
+
},
|
|
217
|
+
{
|
|
218
|
+
id: "matchesPath-admits-the-empty-target",
|
|
219
|
+
member: "matchesPath",
|
|
220
|
+
find: "if (clean === \"\" || clean === \"/\") return false;",
|
|
221
|
+
replace: "if (clean === \"/\") return false;",
|
|
222
|
+
outcome: "killed",
|
|
223
|
+
why: "CLOSES hole 8 — a `./` target would start matching. The documented-hole cases must go red, which is the good-news direction the corpus exists to catch.",
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
id: "matchesPath-drops-the-empty-candidate-guard",
|
|
227
|
+
member: "matchesPath",
|
|
228
|
+
find: "if (typeof candidate !== \"string\" || candidate === \"\") return false;",
|
|
229
|
+
replace: "if (typeof candidate !== \"string\") return false;",
|
|
230
|
+
outcome: "survives",
|
|
231
|
+
why: "EQUIVALENT: with `candidate` empty, `\"\".endsWith(`/${clean}`)` is false for every non-empty `clean`, and `clean` is guarded non-empty two lines down. The guard is a readability rail, not a behavioural one.",
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
id: "matchesPath-stops-stripping-a-leading-dot-slash",
|
|
235
|
+
member: "matchesPath",
|
|
236
|
+
// **The anchor carries the line BELOW it, and that is not decoration.** `neverMatches` in the
|
|
237
|
+
// same file re-spells this strip — deliberately, so that predicate reads without `matchesPath`
|
|
238
|
+
// in front of you — and the moment it did, this operator placed TWICE and the whole census
|
|
239
|
+
// exited 2: could-not-run wearing a pass's clothes, over a cause no reader would guess from the
|
|
240
|
+
// exit code. The recipe's own advice is to lengthen the anchor until it is unique, and the
|
|
241
|
+
// `if (clean === ...)` guard is the line only `matchesPath` has. Measured 2026-09-03.
|
|
242
|
+
find: "const clean = String(target ?? \"\").replace(/^\\.\\//, \"\").replace(/^\\/+/, \"\");\n if (clean === \"\" || clean === \"/\") return false;",
|
|
243
|
+
replace: "const clean = String(target ?? \"\").replace(/^\\/+/, \"\");\n if (clean === \"\" || clean === \"/\") return false;",
|
|
244
|
+
outcome: "killed",
|
|
245
|
+
why: "A target written `./docs/vision.md` stops matching; and `./` stops reducing to empty, which moves hole 8.",
|
|
246
|
+
},
|
|
247
|
+
// ---------------------------------------------------------------------------------- shellWrites
|
|
248
|
+
{
|
|
249
|
+
id: "shellWrites-drops-redirection-recognition",
|
|
250
|
+
member: "shellWrites",
|
|
251
|
+
find: "if (segment.redirects.some((word) => namesTarget(word, target))) return true;",
|
|
252
|
+
replace: "",
|
|
253
|
+
outcome: "killed",
|
|
254
|
+
why: "`echo x > docs/vision.md` stops being a write. The plainest spelling the shell half exists for.",
|
|
255
|
+
},
|
|
256
|
+
{
|
|
257
|
+
id: "shellWrites-drops-the-writer-table",
|
|
258
|
+
member: "shellWrites",
|
|
259
|
+
find: "if (writesWhatItNames(segment) && segment.args.some((word) => namesTarget(word, target, true))) return true;",
|
|
260
|
+
replace: "",
|
|
261
|
+
outcome: "killed",
|
|
262
|
+
why: "`cp /tmp/x docs/vision.md` stops being a write.",
|
|
263
|
+
},
|
|
264
|
+
{
|
|
265
|
+
id: "shellWrites-drops-ancestors",
|
|
266
|
+
member: "shellWrites",
|
|
267
|
+
find: "segment.args.some((word) => namesTarget(word, target, true))",
|
|
268
|
+
replace: "segment.args.some((word) => namesTarget(word, target, false))",
|
|
269
|
+
outcome: "killed",
|
|
270
|
+
why: "`rm -rf docs` stops reaching a rule protecting `docs/vision.md` — destroying the container stops counting.",
|
|
271
|
+
},
|
|
272
|
+
// -------------------------------------------------------------------------------- shellSegments
|
|
273
|
+
{
|
|
274
|
+
id: "shellSegments-forgets-write-redirections",
|
|
275
|
+
member: "shellSegments",
|
|
276
|
+
find: " if (word.text.includes(\">\")) pending = \"written\";",
|
|
277
|
+
replace: " if (false) pending = \"written\";",
|
|
278
|
+
outcome: "killed",
|
|
279
|
+
why: "Redirection targets stop being collected, so the first arm of `shellWrites` sees nothing.",
|
|
280
|
+
},
|
|
281
|
+
{
|
|
282
|
+
id: "shellSegments-treats-read-redirection-as-a-separator",
|
|
283
|
+
member: "shellSegments",
|
|
284
|
+
find: " else if (word.text.includes(\"<\")) pending = \"read\";",
|
|
285
|
+
replace: " else if (false) pending = \"read\";",
|
|
286
|
+
outcome: "killed",
|
|
287
|
+
why: "`tee < /tmp/in docs/vision.md` loses its head and its argument — the case that docblock says goes red if the branch is removed.",
|
|
288
|
+
},
|
|
289
|
+
{
|
|
290
|
+
id: "shellSegments-stops-skipping-segment-leaders",
|
|
291
|
+
member: "shellSegments",
|
|
292
|
+
find: "if (ASSIGNMENT.test(word.text) || COMMAND_PREFIXES.has(bare) || SEGMENT_LEADERS.has(bare)) continue;",
|
|
293
|
+
replace: "if (ASSIGNMENT.test(word.text) || COMMAND_PREFIXES.has(bare)) continue;",
|
|
294
|
+
outcome: "killed",
|
|
295
|
+
why: "`{ cp … ; }` and `if …; then cp …` hide the writer behind the leader — hole 2's write half reopens.",
|
|
296
|
+
},
|
|
297
|
+
{
|
|
298
|
+
id: "shellSegments-stops-skipping-command-prefixes",
|
|
299
|
+
member: "shellSegments",
|
|
300
|
+
find: "if (ASSIGNMENT.test(word.text) || COMMAND_PREFIXES.has(bare) || SEGMENT_LEADERS.has(bare)) continue;\n current.head = bare;",
|
|
301
|
+
replace: "if (ASSIGNMENT.test(word.text) || SEGMENT_LEADERS.has(bare)) continue;\n current.head = bare;",
|
|
302
|
+
outcome: "killed",
|
|
303
|
+
why: "`sudo cp /tmp/x docs/vision.md` takes `sudo` as its head, which is not a writer.",
|
|
304
|
+
},
|
|
305
|
+
{
|
|
306
|
+
id: "shellSegments-stops-skipping-assignments",
|
|
307
|
+
member: "shellSegments",
|
|
308
|
+
find: "if (ASSIGNMENT.test(word.text) || COMMAND_PREFIXES.has(bare) || SEGMENT_LEADERS.has(bare)) continue;\n current.head = bare;\n continue;",
|
|
309
|
+
replace: "if (COMMAND_PREFIXES.has(bare) || SEGMENT_LEADERS.has(bare)) continue;\n current.head = bare;\n continue;",
|
|
310
|
+
outcome: "killed",
|
|
311
|
+
why: "`FOO=bar cp /tmp/x docs/vision.md` takes `FOO=bar` as its head.",
|
|
312
|
+
},
|
|
313
|
+
// ------------------------------------------------------------------------------- commandSegments
|
|
314
|
+
{
|
|
315
|
+
id: "commandSegments-forgets-quoting",
|
|
316
|
+
member: "commandSegments",
|
|
317
|
+
find: " if (quote) {",
|
|
318
|
+
replace: " if (false && quote) {",
|
|
319
|
+
outcome: "killed",
|
|
320
|
+
why: "A separator inside a quoted string starts splitting, so `echo \"git push --force\"` becomes a gated command — a false red on data.",
|
|
321
|
+
},
|
|
322
|
+
{
|
|
323
|
+
id: "commandSegments-forgets-escapes-inside-quotes",
|
|
324
|
+
member: "commandSegments",
|
|
325
|
+
find: " escaped = i;\n continue;",
|
|
326
|
+
replace: " escaped = -1;\n continue;",
|
|
327
|
+
outcome: "killed",
|
|
328
|
+
why: "The escape-aware split the pre-commit checkpoint added on #336 reverts: `echo \\>| git push --force origin main` stops splitting at a real pipe.",
|
|
329
|
+
},
|
|
330
|
+
{
|
|
331
|
+
id: "commandSegments-splits-redirection-operators",
|
|
332
|
+
member: "commandSegments",
|
|
333
|
+
find: "if (c === \"&\" && (opBefore || command[i + 1] === \">\")) continue;",
|
|
334
|
+
replace: "if (false) continue;",
|
|
335
|
+
outcome: "killed",
|
|
336
|
+
why: "`2>&1 git push --force …` breaks into pieces at the `&` before the redirection can be stripped — the #71 regression.",
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
id: "commandSegments-drops-heredoc-stripping",
|
|
340
|
+
member: "commandSegments",
|
|
341
|
+
find: "const command = stripHeredocs(String(raw ?? \"\"));",
|
|
342
|
+
replace: "const command = String(raw ?? \"\");",
|
|
343
|
+
outcome: "killed",
|
|
344
|
+
why: "A heredoc BODY is read as command text again, so its lines become segments — the false red that made this repository's own commit message ungatable.",
|
|
345
|
+
},
|
|
346
|
+
{
|
|
347
|
+
id: "commandSegments-declares-a-dead-local",
|
|
348
|
+
member: "commandSegments",
|
|
349
|
+
find: "let quote = null;",
|
|
350
|
+
replace: "let quote = null;\n let unusedByDesign = 0;\n void unusedByDesign;",
|
|
351
|
+
outcome: "survives",
|
|
352
|
+
why: "EQUIVALENT: an unread local, declared and immediately voided. A second control alongside `matchesRule-inserts-a-no-op`, at a different member, because a harness that only ever reports kills is indistinguishable from one that reports kills wrongly. _(This `why` described a `String(x).toString()` substitution until the pre-commit checkpoint read it — the text of an EARLIER draft of this operator, left behind when the anchor moved. A `survives` record whose proof describes a different mutant is a wrong record even where the verdict is right, and it is this session's own signature defect in its own record layer.)_",
|
|
353
|
+
},
|
|
354
|
+
// ----------------------------------------------------------------------------------- shellWords
|
|
355
|
+
{
|
|
356
|
+
id: "shellWords-forgets-ansi-c-quoting",
|
|
357
|
+
member: "shellWords",
|
|
358
|
+
find: "if (c === \"$\" && (command[i + 1] === \"'\" || command[i + 1] === '\"')) continue;",
|
|
359
|
+
replace: "if (false && (command[i + 1] === \"'\" || command[i + 1] === '\"')) continue;",
|
|
360
|
+
outcome: "killed",
|
|
361
|
+
why: "`cp /tmp/x $'docs/vision.md'` glues the `$` onto the word and the path stops being recognised — the Copilot finding from #60.",
|
|
362
|
+
},
|
|
363
|
+
{
|
|
364
|
+
id: "shellWords-stops-emitting-operator-words",
|
|
365
|
+
member: "shellWords",
|
|
366
|
+
find: "words.push({ text: run, op: true });",
|
|
367
|
+
replace: "words.push({ text: run, op: false });",
|
|
368
|
+
outcome: "killed",
|
|
369
|
+
why: "Operators become ordinary words, so redirection targets are never collected and segments never close.",
|
|
370
|
+
},
|
|
371
|
+
// ------------------------------------------------------------------------------------ spellings
|
|
372
|
+
{
|
|
373
|
+
id: "spellings-stops-unwrapping",
|
|
374
|
+
member: "spellings",
|
|
375
|
+
find: " if (wrapper) {",
|
|
376
|
+
replace: " if (false && wrapper) {",
|
|
377
|
+
outcome: "killed",
|
|
378
|
+
why: "`sh -c \"git push --force origin main\"` reaches nothing — the one level of unwrapping that is the hook's whole reach past a literal prefix.",
|
|
379
|
+
},
|
|
380
|
+
{
|
|
381
|
+
id: "spellings-stops-stripping-the-wrapper-quotes",
|
|
382
|
+
member: "spellings",
|
|
383
|
+
find: " if ((quote === '\"' || quote === \"'\") && inner.endsWith(quote) && inner.length > 1) {",
|
|
384
|
+
replace: " if (false && inner.endsWith(quote) && inner.length > 1) {",
|
|
385
|
+
outcome: "killed",
|
|
386
|
+
why: "The unwrapped command keeps its quotes, so the prefix compare fails on the leading quote character.",
|
|
387
|
+
},
|
|
388
|
+
// --------------------------------------------------------------------------------- stripHeredocs
|
|
389
|
+
{
|
|
390
|
+
id: "stripHeredocs-swallows-an-unterminated-opener",
|
|
391
|
+
member: "stripHeredocs",
|
|
392
|
+
find: " if (delimiter !== null) out.push(...held);",
|
|
393
|
+
replace: "",
|
|
394
|
+
outcome: "killed",
|
|
395
|
+
why: "The fail-open Copilot found on #60 returns: `echo \"not a heredoc <<EOF\"` swallows every later line, hiding a gated command on any of them.",
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
id: "stripHeredocs-forgets-the-quoted-delimiter-form",
|
|
399
|
+
member: "stripHeredocs",
|
|
400
|
+
find: "const opened = /<<-?\\s*(['\"]?)([A-Za-z_][A-Za-z0-9_]*)\\1/.exec(line);",
|
|
401
|
+
replace: "const opened = /<<-?\\s*([A-Za-z_][A-Za-z0-9_]*)/.exec(line);",
|
|
402
|
+
outcome: "killed",
|
|
403
|
+
why: "`<<'EOF'` stops being recognised as an opener — and with capture group 2 now undefined the delimiter never matches, so the body is read as commands.",
|
|
404
|
+
},
|
|
405
|
+
// --------------------------------------------------------------- stripLeadingRedirections + regexes
|
|
406
|
+
{
|
|
407
|
+
id: "stripLeadingRedirections-strips-only-once",
|
|
408
|
+
member: "stripLeadingRedirections",
|
|
409
|
+
find: " for (;;) {\n const shorter = text.replace(LEADING_REDIRECTION, \"\");",
|
|
410
|
+
replace: " for (let once = 0; once < 1; once += 1) {\n const shorter = text.replace(LEADING_REDIRECTION, \"\");",
|
|
411
|
+
outcome: "killed",
|
|
412
|
+
why: "Two stacked redirections — `> a 2> b git push --force …` — leave the second in front of the command.",
|
|
413
|
+
},
|
|
414
|
+
{
|
|
415
|
+
id: "REDIRECTION_TARGET-narrows-to-non-whitespace",
|
|
416
|
+
member: "REDIRECTION_TARGET",
|
|
417
|
+
find: "const REDIRECTION_TARGET = String.raw`(?:\"(?:\\\\[\\s\\S]|[^\"\\\\])*\"|'[^']*'|\\\\[\\s\\S]|[^\\s])+`;",
|
|
418
|
+
replace: "const REDIRECTION_TARGET = String.raw`[^\\s]+`;",
|
|
419
|
+
outcome: "killed",
|
|
420
|
+
why: "The exact round-1 defect on #336: `> \"foo bar\" git push --force …` strips `> \"foo` and leaves the command behind it ungated.",
|
|
421
|
+
},
|
|
422
|
+
{
|
|
423
|
+
id: "LEADING_REDIRECTION-forgets-the-clobber-form",
|
|
424
|
+
member: "LEADING_REDIRECTION",
|
|
425
|
+
find: "^\\d*(?:&>>|&>|>>|>&|>\\||<&|<>|<|>)\\s*",
|
|
426
|
+
replace: "^\\d*(?:&>>|&>|>>|>&|<&|<>|<|>)\\s*",
|
|
427
|
+
outcome: "killed",
|
|
428
|
+
why: "`>| f git push --force …` stops stripping — one of the two spellings #71's own table never named.",
|
|
429
|
+
},
|
|
430
|
+
// ---------------------------------------------------------------------------------- namesTarget
|
|
431
|
+
{
|
|
432
|
+
id: "namesTarget-forgets-the-equals-form",
|
|
433
|
+
member: "namesTarget",
|
|
434
|
+
find: "const candidates = eq > 0 ? [word, word.slice(eq + 1)] : [word];",
|
|
435
|
+
replace: "const candidates = [word];",
|
|
436
|
+
outcome: "killed",
|
|
437
|
+
why: "`dd of=docs/vision.md` stops naming the target.",
|
|
438
|
+
},
|
|
439
|
+
{
|
|
440
|
+
id: "namesTarget-stops-rooting-a-relative-word",
|
|
441
|
+
member: "namesTarget",
|
|
442
|
+
find: "const rooted = clean.startsWith(\"/\") ? clean : `/${clean}`;",
|
|
443
|
+
replace: "const rooted = clean;",
|
|
444
|
+
outcome: "killed",
|
|
445
|
+
why: "A relative word loses the leading separator the tail comparison needs, so `cp /tmp/x docs/vision.md` stops matching.",
|
|
446
|
+
},
|
|
447
|
+
// --------------------------------------------------------------------------------- normalisePath
|
|
448
|
+
{
|
|
449
|
+
id: "normalisePath-keeps-dot-segments",
|
|
450
|
+
member: "normalisePath",
|
|
451
|
+
find: " if (part === \"\" || part === \".\") continue;",
|
|
452
|
+
replace: " if (part === \"\") continue;",
|
|
453
|
+
outcome: "killed",
|
|
454
|
+
why: "`docs/./vision.md` stops being the constitution — a live escape for one review round.",
|
|
455
|
+
},
|
|
456
|
+
{
|
|
457
|
+
id: "normalisePath-keeps-empty-segments",
|
|
458
|
+
member: "normalisePath",
|
|
459
|
+
find: "if (part === \"\" || part === \".\") continue;\n if (part === \"..\"",
|
|
460
|
+
replace: "if (part === \".\") continue;\n if (part === \"..\"",
|
|
461
|
+
outcome: "killed",
|
|
462
|
+
why: "`docs//vision.md` stops being the constitution — the other half of the same escape.",
|
|
463
|
+
},
|
|
464
|
+
{
|
|
465
|
+
id: "normalisePath-stops-resolving-dotdot",
|
|
466
|
+
member: "normalisePath",
|
|
467
|
+
find: " if (part === \"..\" && out.length && out[out.length - 1] !== \"..\") out.pop();\n else out.push(part);",
|
|
468
|
+
replace: " out.push(part);",
|
|
469
|
+
outcome: "killed",
|
|
470
|
+
why: "`foo/../docs/vision.md` stops being the constitution.",
|
|
471
|
+
},
|
|
472
|
+
// ------------------------------------------------------------------------------------ ancestors
|
|
473
|
+
{
|
|
474
|
+
id: "ancestors-includes-the-file-itself",
|
|
475
|
+
member: "ancestors",
|
|
476
|
+
find: "return parts.slice(0, -1).map((_, i) => parts.slice(0, i + 1).join(\"/\"));",
|
|
477
|
+
replace: "return parts.map((_, i) => parts.slice(0, i + 1).join(\"/\"));",
|
|
478
|
+
outcome: "survives",
|
|
479
|
+
why: "EQUIVALENT AT THE CALL SITE: the extra entry is the target's own full path, and `namesTarget` already tests that path in the branch above the ancestor one — so the added candidate can only ever repeat a `true` the first branch has already returned. Recorded rather than deleted because the equivalence is a property of the CALLER, not of `ancestors`, and a reader of this function alone would expect a kill.",
|
|
480
|
+
},
|
|
481
|
+
// ----------------------------------------------------------------------------- writesWhatItNames
|
|
482
|
+
{
|
|
483
|
+
id: "writesWhatItNames-drops-the-writer-table",
|
|
484
|
+
member: "writesWhatItNames",
|
|
485
|
+
find: " if (FILE_WRITERS.has(head)) return true;",
|
|
486
|
+
replace: "",
|
|
487
|
+
outcome: "killed",
|
|
488
|
+
why: "Nothing in the writer table writes any more.",
|
|
489
|
+
},
|
|
490
|
+
{
|
|
491
|
+
id: "writesWhatItNames-drops-the-in-place-flag-check",
|
|
492
|
+
member: "writesWhatItNames",
|
|
493
|
+
find: "return IN_PLACE_EDITORS.has(head) && args.some((a) => /^--in-place(=|$)/.test(a) || /^-[a-zA-Z]*i/.test(a));",
|
|
494
|
+
replace: "return IN_PLACE_EDITORS.has(head);",
|
|
495
|
+
outcome: "killed",
|
|
496
|
+
why: "`sed -n '1,5p' docs/vision.md` — a READ this policy declares Auto — starts being gated as a write.",
|
|
497
|
+
},
|
|
498
|
+
// --------------------------------------------------------------------------------- the tables
|
|
499
|
+
{
|
|
500
|
+
id: "FILE_WRITERS-loses-cp",
|
|
501
|
+
member: "FILE_WRITERS",
|
|
502
|
+
find: "new Set([\"cp\", \"mv\", \"ln\", \"rm\", \"tee\", \"dd\", \"install\", \"truncate\", \"shred\", \"patch\"])",
|
|
503
|
+
replace: "new Set([\"mv\", \"ln\", \"rm\", \"tee\", \"dd\", \"install\", \"truncate\", \"shred\", \"patch\"])",
|
|
504
|
+
outcome: "killed",
|
|
505
|
+
why: "One entry removed from a recognition table. If the corpus does not notice, the table is prose.",
|
|
506
|
+
},
|
|
507
|
+
{
|
|
508
|
+
id: "IN_PLACE_EDITORS-loses-sed",
|
|
509
|
+
member: "IN_PLACE_EDITORS",
|
|
510
|
+
find: "export const IN_PLACE_EDITORS = new Set([\"sed\", \"gsed\", \"perl\", \"ruby\"]);",
|
|
511
|
+
replace: "export const IN_PLACE_EDITORS = new Set([\"gsed\", \"perl\", \"ruby\"]);",
|
|
512
|
+
outcome: "killed",
|
|
513
|
+
why: "`sed -i … docs/vision.md` stops being a write.",
|
|
514
|
+
},
|
|
515
|
+
{
|
|
516
|
+
id: "COMMAND_PREFIXES-loses-sudo",
|
|
517
|
+
member: "COMMAND_PREFIXES",
|
|
518
|
+
find: "new Set([\"sudo\", \"env\", \"command\", \"builtin\", \"exec\", \"nohup\", \"nice\", \"time\"])",
|
|
519
|
+
replace: "new Set([\"env\", \"command\", \"builtin\", \"exec\", \"nohup\", \"nice\", \"time\"])",
|
|
520
|
+
outcome: "killed",
|
|
521
|
+
why: "`sudo cp /tmp/x docs/vision.md` takes `sudo` as its head.",
|
|
522
|
+
},
|
|
523
|
+
{
|
|
524
|
+
id: "SEGMENT_LEADERS-loses-then",
|
|
525
|
+
member: "SEGMENT_LEADERS",
|
|
526
|
+
find: "new Set([\"{\", \"}\", \"!\", \"then\", \"else\", \"elif\", \"do\", \"done\", \"fi\", \"esac\"])",
|
|
527
|
+
replace: "new Set([\"{\", \"}\", \"!\", \"else\", \"elif\", \"do\", \"done\", \"fi\", \"esac\"])",
|
|
528
|
+
outcome: "killed",
|
|
529
|
+
why: "`if true; then cp /tmp/x docs/vision.md; fi` takes `then` as its head — and this is the asymmetry the corpus records between the two segmenters, so it must be the WRITE path that notices.",
|
|
530
|
+
},
|
|
531
|
+
{
|
|
532
|
+
id: "OPERATOR-loses-the-newline",
|
|
533
|
+
member: "OPERATOR",
|
|
534
|
+
find: "const OPERATOR = /[|&;<>()\\n\\r]/;",
|
|
535
|
+
replace: "const OPERATOR = /[|&;<>()\\r]/;",
|
|
536
|
+
outcome: "killed",
|
|
537
|
+
why: "The plainest spelling there is, and the one the first hole list missed: a two-line script folds into one segment whose head is the first command.",
|
|
538
|
+
},
|
|
539
|
+
{
|
|
540
|
+
id: "WRITE_TOOLS-loses-Write",
|
|
541
|
+
member: "WRITE_TOOLS",
|
|
542
|
+
find: 'export const WRITE_TOOLS = ["Edit", "Write", "NotebookEdit"];',
|
|
543
|
+
replace: 'export const WRITE_TOOLS = ["Edit", "NotebookEdit"];',
|
|
544
|
+
outcome: "killed",
|
|
545
|
+
why: "The `Write` tool stops reaching any `write:` rule, so the constitution is editable through the plainest tool there is.",
|
|
546
|
+
},
|
|
547
|
+
{
|
|
548
|
+
id: "WRITE_TOOLS-loses-NotebookEdit",
|
|
549
|
+
member: "WRITE_TOOLS",
|
|
550
|
+
find: 'export const WRITE_TOOLS = ["Edit", "Write", "NotebookEdit"];\nexport const READ_TOOLS = ["Read"];',
|
|
551
|
+
replace: 'export const WRITE_TOOLS = ["Edit", "Write"];\nexport const READ_TOOLS = ["Read"];',
|
|
552
|
+
outcome: "killed",
|
|
553
|
+
why: "The third write tool, which the corpus did not exercise AT ALL until this operator was added — twenty-one write-rule fixtures across five files and not one `NotebookEdit` among them. Found by widening REGION at the pre-commit checkpoint, which is the coverage floor earning its place.",
|
|
554
|
+
},
|
|
555
|
+
{
|
|
556
|
+
id: "matchesRule-write-drops-the-notebook-path-fallback",
|
|
557
|
+
member: "matchesRule",
|
|
558
|
+
find: "return matchesPath(input.file_path ?? input.notebook_path, action.write);",
|
|
559
|
+
replace: "return matchesPath(input.file_path, action.write);",
|
|
560
|
+
outcome: "killed",
|
|
561
|
+
why: "`NotebookEdit` carries `notebook_path` rather than `file_path`, so without the fallback the tool is in the table and its payload reaches nothing — a gate that looks whole from the outside. Untested until the same widening.",
|
|
562
|
+
},
|
|
563
|
+
{
|
|
564
|
+
id: "READ_TOOLS-loses-Read",
|
|
565
|
+
member: "READ_TOOLS",
|
|
566
|
+
find: 'export const READ_TOOLS = ["Read"];',
|
|
567
|
+
replace: "export const READ_TOOLS = [];",
|
|
568
|
+
outcome: "survives",
|
|
569
|
+
why: "EQUIVALENT UNDER THE YIELDED POLICY, by the same argument as `matchesRule-read-accepts-any-tool`: the one `read:` rule declared is `read-anything-in-the-repository`, whose `./` target reduces to the empty string and is refused — hole 8 — so the branch answers false whether it is reached or not. No fixture can distinguish this and none should be invented to; it becomes killable the day a `read:` rule with a real target is declared.",
|
|
570
|
+
},
|
|
571
|
+
{
|
|
572
|
+
id: "ASSIGNMENT-requires-two-characters",
|
|
573
|
+
member: "ASSIGNMENT",
|
|
574
|
+
find: "const ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=/;",
|
|
575
|
+
replace: "const ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]+=/;",
|
|
576
|
+
outcome: "killed",
|
|
577
|
+
why: "A single-letter assignment — `A=1 cp /tmp/x docs/vision.md` — stops being recognised as one.",
|
|
578
|
+
},
|
|
579
|
+
];
|
|
580
|
+
|
|
581
|
+
/**
|
|
582
|
+
* Apply one operator to the subject's source.
|
|
583
|
+
*
|
|
584
|
+
* **A missing or ambiguous anchor is could-not-run, never a skip.** If `compile.mjs` moves and an
|
|
585
|
+
* anchor no longer places, a harness that skipped it would report a clean sweep of the operators it
|
|
586
|
+
* happened to apply — `../.portulan/memory/a-checker-must-refuse-what-it-cannot-check.md`, in the one
|
|
587
|
+
* tool whose entire subject is checks that must not be trusted on their own word.
|
|
588
|
+
*/
|
|
589
|
+
export function mutate(source, op) {
|
|
590
|
+
let count = 0;
|
|
591
|
+
let at = 0;
|
|
592
|
+
for (;;) {
|
|
593
|
+
const i = source.indexOf(op.find, at);
|
|
594
|
+
if (i === -1) break;
|
|
595
|
+
count += 1;
|
|
596
|
+
at = i + 1;
|
|
597
|
+
}
|
|
598
|
+
if (count === 0) {
|
|
599
|
+
throw new CouldNotRun(
|
|
600
|
+
`operator \`${op.id}\` does not place: its anchor is absent from ${SUBJECT}. The subject moved and this ` +
|
|
601
|
+
`operator did not — re-anchor it or retire it, but do not let it be skipped`,
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
if (count > 1) {
|
|
605
|
+
throw new CouldNotRun(
|
|
606
|
+
`operator \`${op.id}\` places ${count} times in ${SUBJECT} and an operator must be exactly one edit. ` +
|
|
607
|
+
`Lengthen its anchor until it is unique`,
|
|
608
|
+
);
|
|
609
|
+
}
|
|
610
|
+
return source.replace(op.find, () => op.replace);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* Rewrite the subject's relative imports to absolute `file://` URLs.
|
|
615
|
+
*
|
|
616
|
+
* A mutant lives in a temp directory, so `./discover.mjs` and `./inside.mjs` would not resolve from
|
|
617
|
+
* there. Both import node builtins only, so pointing at the real modules costs nothing and copies
|
|
618
|
+
* nothing. Measured before this was written rather than assumed: `${SUBJECT}` has exactly two relative
|
|
619
|
+
* imports, no dynamic `import(`, and its only top-level `import.meta.url` use is the entry guard —
|
|
620
|
+
* which compares against `process.argv[1]` and therefore cannot fire for a module imported from a
|
|
621
|
+
* temp directory while argv[1] is this runner.
|
|
622
|
+
*/
|
|
623
|
+
export function absolutiseImports(source, cliDir) {
|
|
624
|
+
return source.replace(/from "\.\/([A-Za-z0-9._-]+\.mjs)"/g, (_, name) => `from "${pathToFileURL(path.join(cliDir, name)).href}"`);
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/** Run the whole corpus through one `matchesRule`, and report the first disagreement. */
|
|
628
|
+
export function runCorpus(matchesRule, byId, corpus) {
|
|
629
|
+
for (const { where, doc } of corpus) {
|
|
630
|
+
const rule = byId.get(doc.rule);
|
|
631
|
+
// **Never a skip.** This read `if (!rule) continue;`, which let a fixture file whose rule the
|
|
632
|
+
// policy does not declare — renamed, misfiled, outlived its gate — sit ungraded while the
|
|
633
|
+
// census reported green. A coverage tool that silently thins its own kill-set and then reports
|
|
634
|
+
// on it is the loudest false green available here, and it is the failure
|
|
635
|
+
// `../.portulan/memory/a-checker-must-refuse-what-it-cannot-check.md` names, in the module
|
|
636
|
+
// whose whole subject is that class. `run` refuses the whole census before the baseline for
|
|
637
|
+
// exactly this, so this arm is unreachable from there and is kept because a direct caller of
|
|
638
|
+
// an exported function must not get the silent behaviour back. Reported as a suppressed note
|
|
639
|
+
// by Copilot, round 5 on #338.
|
|
640
|
+
if (!rule) {
|
|
641
|
+
throw new CouldNotRun(
|
|
642
|
+
`${where} attacks \`${doc.rule}\`, which the yielded policy does not declare — its cases cannot be ` +
|
|
643
|
+
`graded, and a census that skipped them would report on a kill-set smaller than the one it names`,
|
|
644
|
+
);
|
|
645
|
+
}
|
|
646
|
+
for (const c of doc.cases) {
|
|
647
|
+
let actual;
|
|
648
|
+
try {
|
|
649
|
+
actual = matchesRule(rule, c.tool, c.input);
|
|
650
|
+
} catch (error) {
|
|
651
|
+
// A mutant that THROWS is caught, and loudly: `matchesRule` promises never to throw,
|
|
652
|
+
// so a throw is a disagreement with the contract as much as a wrong boolean is.
|
|
653
|
+
return { agreed: false, at: `${where} → ${c.id}`, how: `threw ${error?.name ?? "an error"}` };
|
|
654
|
+
}
|
|
655
|
+
if (actual !== c.expect) {
|
|
656
|
+
return { agreed: false, at: `${where} → ${c.id}`, how: `answered ${actual}, corpus records ${c.expect}` };
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
return { agreed: true };
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
/**
|
|
664
|
+
* Import a mutated subject from a fresh temp directory.
|
|
665
|
+
*
|
|
666
|
+
* **A fresh directory per mutant, and that is load-bearing rather than tidy.** ESM caches by resolved
|
|
667
|
+
* URL and offers no invalidation, so writing every mutant to one reused path would import mutant 1
|
|
668
|
+
* and then grade it forty more times — silently, and *including while the record is first being
|
|
669
|
+
* written*, where the two-directional rail cannot see it because the record would be built from the
|
|
670
|
+
* same wrong readings. `./mutants.test.mjs` pins it with two mutants that must answer differently in
|
|
671
|
+
* one process.
|
|
672
|
+
*/
|
|
673
|
+
async function importMutant(source, cliDir, op) {
|
|
674
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "portulan-mutant-"));
|
|
675
|
+
const file = path.join(dir, "compile.mjs");
|
|
676
|
+
try {
|
|
677
|
+
fs.writeFileSync(file, absolutiseImports(source, cliDir), "utf8");
|
|
678
|
+
try {
|
|
679
|
+
return { module: await import(pathToFileURL(file).href), dir };
|
|
680
|
+
} catch (cause) {
|
|
681
|
+
// **An unimportable mutant is could-not-run, never a kill.** A substitution that produces a
|
|
682
|
+
// syntax error checks nothing about the corpus, and counting it as killed would let a
|
|
683
|
+
// broken operator masquerade as a well-covered one — the loudest possible false green in a
|
|
684
|
+
// tool whose whole output is a coverage claim.
|
|
685
|
+
throw new CouldNotRun(
|
|
686
|
+
`operator \`${op.id}\` produced a module that will not import — ${cause?.message ?? cause}. ` +
|
|
687
|
+
`A mutant that cannot load tests nothing; fix the substitution`,
|
|
688
|
+
);
|
|
689
|
+
}
|
|
690
|
+
} catch (error) {
|
|
691
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
692
|
+
throw error;
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/** Per-region-member operator counts, zeroes included. */
|
|
697
|
+
export function census(operators = OPERATORS) {
|
|
698
|
+
const counts = new Map(REGION.map((m) => [m, 0]));
|
|
699
|
+
const stray = [];
|
|
700
|
+
for (const op of operators) {
|
|
701
|
+
if (!counts.has(op.member)) stray.push(op);
|
|
702
|
+
else counts.set(op.member, counts.get(op.member) + 1);
|
|
703
|
+
}
|
|
704
|
+
return { counts, stray, uncovered: REGION.filter((m) => counts.get(m) === 0) };
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
function usage() {
|
|
708
|
+
return [
|
|
709
|
+
"usage: node cli/mutants.mjs [--check] [--workspace <dir>] [--pack-root <dir>] [--only <operator-id>]",
|
|
710
|
+
"",
|
|
711
|
+
" Mutates cli/compile.mjs's matcher region, one declared operator at a time, and grades each",
|
|
712
|
+
" mutant against evals/goldens/gates/ — the corpus that claims to cover those matchers.",
|
|
713
|
+
"",
|
|
714
|
+
" --check is the default and the only mode; the flag is accepted so the recipe reads",
|
|
715
|
+
" like its siblings. --only narrows to one operator, for a session hunting a single survivor.",
|
|
716
|
+
"",
|
|
717
|
+
" Exit 0 green · 1 red · 2 could not run.",
|
|
718
|
+
].join("\n");
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
export async function run(argv = [], { stdout = process.stdout, stderr = process.stderr, cwd = process.cwd() } = {}) {
|
|
722
|
+
const say = (line = "") => stdout.write(`${line}\n`);
|
|
723
|
+
if (argv.includes("--help") || argv.includes("-h")) {
|
|
724
|
+
say(usage());
|
|
725
|
+
return 0;
|
|
726
|
+
}
|
|
727
|
+
let named = cwd;
|
|
728
|
+
let packRoots = null;
|
|
729
|
+
let only = null;
|
|
730
|
+
const dirs = [];
|
|
731
|
+
try {
|
|
732
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
733
|
+
if (argv[i] === "--check") continue;
|
|
734
|
+
if (argv[i] === "--workspace") {
|
|
735
|
+
named = argv[i + 1];
|
|
736
|
+
i += 1;
|
|
737
|
+
if (named === undefined) throw new CouldNotRun("--workspace needs a directory");
|
|
738
|
+
} else if (argv[i] === "--only") {
|
|
739
|
+
only = argv[i + 1];
|
|
740
|
+
i += 1;
|
|
741
|
+
if (only === undefined || only.startsWith("-")) throw new CouldNotRun("--only needs an operator id");
|
|
742
|
+
} else if (argv[i] === "--pack-root") {
|
|
743
|
+
const root = argv[i + 1];
|
|
744
|
+
i += 1;
|
|
745
|
+
if (root === undefined || root.startsWith("-")) throw new CouldNotRun("--pack-root needs a directory");
|
|
746
|
+
let stat = null;
|
|
747
|
+
try {
|
|
748
|
+
stat = fs.statSync(root);
|
|
749
|
+
} catch (cause) {
|
|
750
|
+
throw new CouldNotRun(`--pack-root ${root} cannot be read — ${cause.code ?? cause.message}`);
|
|
751
|
+
}
|
|
752
|
+
if (!stat.isDirectory()) throw new CouldNotRun(`--pack-root ${root} is not a directory`);
|
|
753
|
+
(packRoots ??= []).push(path.resolve(root));
|
|
754
|
+
} else throw new CouldNotRun(`unknown argument ${JSON.stringify(argv[i])}`);
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
// The coverage floor is checked before a single mutant is written, because it is a property of
|
|
758
|
+
// the TABLE and a run that spends thirty seconds before reporting a structural gap is a run
|
|
759
|
+
// nobody waits for.
|
|
760
|
+
const { counts, stray, uncovered } = census();
|
|
761
|
+
if (stray.length) {
|
|
762
|
+
throw new CouldNotRun(
|
|
763
|
+
`operator(s) ${stray.map((o) => `\`${o.id}\``).join(", ")} name a region member that REGION does not list. ` +
|
|
764
|
+
`Add the member to REGION or correct the operator — an operator outside the census is coverage nobody counts`,
|
|
765
|
+
);
|
|
766
|
+
}
|
|
767
|
+
// **The WHOLE table, in both modes.** This read `only === null ? OPERATORS : []`, so `--only`
|
|
768
|
+
// — the mode a person reaches for precisely when something is wrong — validated nothing, and a
|
|
769
|
+
// malformed operator slipped through exactly where somebody was looking closely. `--only`
|
|
770
|
+
// narrows what RUNS; it does not narrow what must be well formed, because the table is the
|
|
771
|
+
// artifact under review and a reviewer reading it in `--only` mode is reading the same table.
|
|
772
|
+
// Reported by Copilot, round 1 on #338.
|
|
773
|
+
for (const op of OPERATORS) {
|
|
774
|
+
if (!OUTCOMES.includes(op.outcome)) {
|
|
775
|
+
throw new CouldNotRun(`operator \`${op.id}\` records outcome ${JSON.stringify(op.outcome)} — one of ${OUTCOMES.join(" / ")}`);
|
|
776
|
+
}
|
|
777
|
+
if (typeof op.why !== "string" || op.why.trim() === "") {
|
|
778
|
+
throw new CouldNotRun(`operator \`${op.id}\` carries no \`why\` — a mutation nobody can read is not reviewable`);
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
const { workspaceRoot, rules, unresolved } = yieldedRules(named, { packRoots });
|
|
783
|
+
const corpus = readCorpus(workspaceRoot);
|
|
784
|
+
for (const u of unresolved) say(`pack ${u.name} UNRESOLVED — ${u.why}; its gate fragments are not in this census`);
|
|
785
|
+
|
|
786
|
+
const cliDir = path.join(workspaceRoot, "cli");
|
|
787
|
+
const subject = path.join(workspaceRoot, SUBJECT);
|
|
788
|
+
let source;
|
|
789
|
+
try {
|
|
790
|
+
source = fs.readFileSync(subject, "utf8");
|
|
791
|
+
} catch (cause) {
|
|
792
|
+
throw new CouldNotRun(`${SUBJECT} cannot be read at ${subject} — ${cause.code ?? cause.message}`);
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
const { matchable, exempt } = partition(rules);
|
|
796
|
+
const byId = new Map(matchable.map((r) => [r.id, r]));
|
|
797
|
+
|
|
798
|
+
// **The corpus and the policy must correspond before a single mutant is written.** A fixture
|
|
799
|
+
// naming a rule this census cannot grade is not a case that happens to pass — it is a piece of
|
|
800
|
+
// the kill-set that never runs, and the census's whole output is a claim about how much the
|
|
801
|
+
// kill-set catches. Refused here rather than at the first mutant so the message names the
|
|
802
|
+
// repair once instead of forty-nine times, and refused at all because the alternative measured
|
|
803
|
+
// green. `goldens` reds on the same condition; this does not lean on that recipe having run,
|
|
804
|
+
// since a rail that depends on a sibling rail is a rail with a precondition nobody states.
|
|
805
|
+
const ungradable = corpus
|
|
806
|
+
.filter(({ doc }) => !byId.has(doc.rule))
|
|
807
|
+
.map(({ where, doc }) => {
|
|
808
|
+
const why = exempt.some((e) => e.id === doc.rule)
|
|
809
|
+
? "declares no matchable action, so there is nothing for a mutant to change"
|
|
810
|
+
: "is not declared by the yielded policy at all";
|
|
811
|
+
return `${where} attacks \`${doc.rule}\`, which ${why}`;
|
|
812
|
+
});
|
|
813
|
+
if (ungradable.length) {
|
|
814
|
+
throw new CouldNotRun(
|
|
815
|
+
`${ungradable.length} fixture file(s) cannot be graded, so the kill-set is smaller than it looks:\n ` +
|
|
816
|
+
`${ungradable.join("\n ")}\n ` +
|
|
817
|
+
`Fix the corpus (the \`goldens\` recipe grades that) before reading a census over it`,
|
|
818
|
+
);
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
// **The baseline, before any mutant.** A mutation census over a corpus that is already red
|
|
822
|
+
// measures nothing — every mutant would "survive" against a kill-set that cannot even agree
|
|
823
|
+
// with the unmutated matcher. `goldens` is the recipe that grades this; here it is a
|
|
824
|
+
// precondition, and a failed precondition is could-not-run rather than a verdict.
|
|
825
|
+
const { matchesRule } = await import(pathToFileURL(subject).href);
|
|
826
|
+
const baseline = runCorpus(matchesRule, byId, corpus);
|
|
827
|
+
if (!baseline.agreed) {
|
|
828
|
+
throw new CouldNotRun(
|
|
829
|
+
`the corpus does not agree with the UNMUTATED matcher at ${baseline.at} (${baseline.how}). ` +
|
|
830
|
+
`A mutation census over a red corpus measures nothing — run the \`goldens\` recipe and fix that first`,
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
const selected = only === null ? OPERATORS : OPERATORS.filter((op) => op.id === only);
|
|
835
|
+
if (only !== null && selected.length === 0) {
|
|
836
|
+
throw new CouldNotRun(`no operator with id \`${only}\` — see the OPERATORS table in cli/mutants.mjs`);
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
const findings = [];
|
|
840
|
+
let killed = 0;
|
|
841
|
+
let survived = 0;
|
|
842
|
+
for (const op of selected) {
|
|
843
|
+
const mutated = mutate(source, op);
|
|
844
|
+
if (mutated === source) {
|
|
845
|
+
throw new CouldNotRun(
|
|
846
|
+
`operator \`${op.id}\` placed but changed nothing — its \`find\` and \`replace\` are the same text. ` +
|
|
847
|
+
`A no-op operator is an operator that reports coverage it does not provide`,
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
const { module, dir } = await importMutant(mutated, cliDir, op);
|
|
851
|
+
dirs.push(dir);
|
|
852
|
+
const result = runCorpus(module.matchesRule, byId, corpus);
|
|
853
|
+
const actual = result.agreed ? "survives" : "killed";
|
|
854
|
+
if (actual === "killed") killed += 1;
|
|
855
|
+
else survived += 1;
|
|
856
|
+
if (actual === op.outcome) continue;
|
|
857
|
+
findings.push({
|
|
858
|
+
where: op.id,
|
|
859
|
+
what:
|
|
860
|
+
op.outcome === "killed"
|
|
861
|
+
? `is recorded KILLED and SURVIVED. The corpus no longer notices this mutation, so its coverage of ` +
|
|
862
|
+
`\`${op.member}\` has weakened. ${op.why} Add the case that kills it to evals/goldens/gates/ — ` +
|
|
863
|
+
`\`matchesRule\` is a pure function of (rule, tool, input) and a fixture is exactly that triple, ` +
|
|
864
|
+
`so a non-equivalent mutant is always killable by one`
|
|
865
|
+
: `is recorded SURVIVES and was KILLED at ${result.at} (${result.how}). That is good news and the ` +
|
|
866
|
+
`record must absorb it: the equivalence claimed for this operator no longer holds — either the ` +
|
|
867
|
+
`matcher changed or the claim was wrong. Re-read the \`why\` and change the outcome to \`killed\`. ` +
|
|
868
|
+
`A record that still describes a gap somebody closed is as wrong as one that hides a gap somebody opened`,
|
|
869
|
+
});
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
say(`mutants: ${selected.length} operator(s) over ${corpus.length} fixture file(s) — ${killed} killed, ${survived} survived`);
|
|
873
|
+
// Per region member, zeroes included — see REGION. A total says nothing about where the
|
|
874
|
+
// operators landed, and "mutation testing over both matchers" is satisfiable by three
|
|
875
|
+
// operators in one function if nobody prints the distribution.
|
|
876
|
+
say(`mutants: by matcher-region member —`);
|
|
877
|
+
for (const member of REGION) say(` ${String(counts.get(member)).padStart(2)} ${member}`);
|
|
878
|
+
if (uncovered.length) {
|
|
879
|
+
// A gap in the floor is a RED rather than a could-not-run: the table is well-formed, it
|
|
880
|
+
// simply does not reach part of what the clause names.
|
|
881
|
+
findings.push({
|
|
882
|
+
where: "the operator table",
|
|
883
|
+
what:
|
|
884
|
+
`leaves ${uncovered.length} matcher-region member(s) with no operator at all — ` +
|
|
885
|
+
`${uncovered.map((m) => `\`${m}\``).join(", ")}. Every member either carries an operator or REGION ` +
|
|
886
|
+
`should not list it; an unmutated member is a part of the matcher this census makes no claim about`,
|
|
887
|
+
});
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
if (findings.length) {
|
|
891
|
+
for (const f of findings) stderr.write(`mutants: ${f.where}\n ${f.what}\n`);
|
|
892
|
+
stderr.write(`RED — ${findings.length} finding(s) in the mutation census\n`);
|
|
893
|
+
return 1;
|
|
894
|
+
}
|
|
895
|
+
say("GREEN — every mutant met its recorded outcome, and every matcher-region member carries one");
|
|
896
|
+
say("mutants: this measures whether the corpus NOTICES a broken matcher — not whether its attacks are realistic");
|
|
897
|
+
return 0;
|
|
898
|
+
} catch (error) {
|
|
899
|
+
if (error instanceof CouldNotRun || error instanceof CompileError) {
|
|
900
|
+
stderr.write(`mutants: ${error.message}\n`);
|
|
901
|
+
return 2;
|
|
902
|
+
}
|
|
903
|
+
stderr.write(`mutants: could not finish the census — ${error?.stack ?? error}\n`);
|
|
904
|
+
return 2;
|
|
905
|
+
} finally {
|
|
906
|
+
for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true });
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
// The entry guard, in the ONE form `./rule-carriers.mjs` designates. `file://${argv[1]}` is NOT that
|
|
911
|
+
// form: `import.meta.url` percent-encodes, this working copy lives under a path with spaces, and the
|
|
912
|
+
// comparison fails — so the tool exits 0 having run nothing. Copied rather than re-derived, for the
|
|
913
|
+
// reason `./goldens.mjs` states after meeting the false green a third time.
|
|
914
|
+
function isMain() {
|
|
915
|
+
const invoked = process.argv[1];
|
|
916
|
+
if (!invoked) return false;
|
|
917
|
+
if (import.meta.url === pathToFileURL(invoked).href) return true;
|
|
918
|
+
try {
|
|
919
|
+
return import.meta.url === pathToFileURL(fs.realpathSync(invoked)).href;
|
|
920
|
+
} catch {
|
|
921
|
+
return false;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
if (isMain()) process.exitCode = await run(process.argv.slice(2));
|