@sleepy_panda_srl/portulan 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/cli/README.md +69 -2
- package/cli/compile.mjs +555 -37
- package/cli/doctor.mjs +5 -1
- package/cli/drills.mjs +1541 -0
- package/cli/fuzz-shell.mjs +880 -0
- package/cli/gate.mjs +3 -2
- package/cli/goldens.mjs +469 -0
- package/cli/init.mjs +4 -1
- package/cli/mutants.mjs +925 -0
- package/cli/pack-identity.mjs +9 -2
- package/cli/pack-version.mjs +7 -0
- package/cli/recipe-set.mjs +6 -0
- package/cli/release-eval.mjs +1068 -0
- package/cli/review-meter.mjs +741 -0
- package/cli/skill-goldens.mjs +590 -0
- package/cli/stop-gate.mjs +107 -4
- package/cli/telemetry.mjs +1047 -0
- package/cli/version-carriers.mjs +13 -0
- package/core/templates/gate-policy.md +27 -8
- package/evals/README.md +910 -0
- package/evals/releases/0.1.3.json +134 -0
- package/evals/releases/0.1.3.md +64 -0
- package/evals/releases/README.md +32 -0
- package/package.json +7 -2
- package/packs/rituals/checkpoints/README.md +8 -1
- package/packs/rituals/checkpoints/pack.json +1 -1
- package/packs/rituals/checkpoints/skills/pre-commit/SKILL.md +32 -2
- package/spec/slots.md +26 -2
package/cli/drills.mjs
ADDED
|
@@ -0,0 +1,1541 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The forced-red drill harness — every rail broken on purpose, and required to fire.
|
|
3
|
+
//
|
|
4
|
+
// Milestone 8, clause (d): *scheduled forced-red drills — every rail forced red on a calendar and
|
|
5
|
+
// required to fire.* The argument for the clause is in `../docs/milestones/m08.md`; the calendar is
|
|
6
|
+
// `../.github/workflows/drills.yml`; what belongs here is what a drill is and why the shape is this
|
|
7
|
+
// one.
|
|
8
|
+
//
|
|
9
|
+
// ## What it answers, and what its two siblings answer
|
|
10
|
+
//
|
|
11
|
+
// `./goldens.mjs` asks whether every compiled gate has adversarial fixtures — **presence**.
|
|
12
|
+
// `./mutants.mjs` asks whether those fixtures can tell a working matcher from a broken one —
|
|
13
|
+
// **discrimination**. Neither asks the question one level out: **does the rail still fire at all?**
|
|
14
|
+
// A recipe whose precondition quietly started exiting 0 over an empty file list, a hook that fails
|
|
15
|
+
// open on a crash, a check whose enumeration went empty — each reports green and each has stopped
|
|
16
|
+
// being a rail. Nothing in this repository could see that, and the way it was found until now was an
|
|
17
|
+
// incident.
|
|
18
|
+
//
|
|
19
|
+
// ## The clause's provenance is two sessions doing this by hand
|
|
20
|
+
//
|
|
21
|
+
// `../.portulan/proposals/0007-every-watcher-ships-with-its-observation-procedure.md` — *a watcher
|
|
22
|
+
// earns its place by being watched* — asks for a procedure *"run once and its result recorded"*. The
|
|
23
|
+
// 2026-07-28 amendment generalises it along both axes: from one demonstration at adoption **to a
|
|
24
|
+
// calendar**, and from **watchers to every rail**. Milestone 8 sessions 0 and 1 ran eleven forced-red
|
|
25
|
+
// drills by hand and recorded them in their handoffs, which is precisely the state this file exists to
|
|
26
|
+
// replace: two sessions running them by hand is evidence for the clause, not a substitute for it.
|
|
27
|
+
//
|
|
28
|
+
// **And both sessions had a drill that did not fire.** One anchored substitution missed by four spaces
|
|
29
|
+
// of indentation; one patch script's quoting broke. Both times the recipe ran green against an
|
|
30
|
+
// unmodified file and the drill reported on nothing — the same false green the whole milestone is
|
|
31
|
+
// about, inside the instrument built to detect it. Every guard below traces to one of those two.
|
|
32
|
+
//
|
|
33
|
+
// ## A drill is a PAIR, and the pair is the oracle
|
|
34
|
+
//
|
|
35
|
+
// Per rail: a **control** run on a pristine tree, then a **fire** run after the perturbation. Both are
|
|
36
|
+
// required, because drill 1 of 2026-07-30 recorded the reason — *"a rail that only ever reds proves
|
|
37
|
+
// nothing about its green: a recipe hard-wired to fail would have produced the identical red
|
|
38
|
+
// transcript."*
|
|
39
|
+
//
|
|
40
|
+
// The control does a second job the drills of the two hand sessions could not do: it catches an
|
|
41
|
+
// **environment** difference. A rail that is red for a reason having nothing to do with the
|
|
42
|
+
// perturbation — a path-sensitive check, a clean checkout that is not a working copy — makes its drill
|
|
43
|
+
// exit 2 rather than report a fire it did not cause.
|
|
44
|
+
//
|
|
45
|
+
// ## Three guards, each from a defect this repository has already shipped
|
|
46
|
+
//
|
|
47
|
+
// 1. **A `tell` is required, never optional.** A perturbation can red a rail for the wrong reason — a
|
|
48
|
+
// syntax error instead of the finding. So every drill declares a substring the rail's own output
|
|
49
|
+
// must carry when it fires, and the harness requires it **absent in the control and present in the
|
|
50
|
+
// fire**. `../.portulan/verify/README.md` already records one instance caught *"only because the
|
|
51
|
+
// drill asserts the message"*.
|
|
52
|
+
// 2. **A perturbation may not no-op.** An anchored substitution must place **exactly once** —
|
|
53
|
+
// `./mutants.mjs`'s discipline, adopted rather than re-derived — *and* the bytes on disk are
|
|
54
|
+
// hashed before and after and must differ. A missing or ambiguous anchor is could-not-run, never
|
|
55
|
+
// a skip.
|
|
56
|
+
// 3. **`--only` narrows what RUNS, not what must be well formed.** The whole table is validated in
|
|
57
|
+
// every mode. Session 1's round 1 found the opposite in `./mutants.mjs`: `--only` skipped
|
|
58
|
+
// validation in exactly the mode a person reaches for when something is already wrong.
|
|
59
|
+
//
|
|
60
|
+
// ## Isolation: one throwaway `git worktree` per drill
|
|
61
|
+
//
|
|
62
|
+
// `git worktree add --detach <tmp> <tree>` shares the object store, so there is no clone and no
|
|
63
|
+
// network, and history is complete — which `pack-version` and `pack-identity` both need. The
|
|
64
|
+
// developer's working tree is never perturbed. Both hand sessions perturbed in place and restored by
|
|
65
|
+
// hand, which is one crash away from leaving a repository broken.
|
|
66
|
+
//
|
|
67
|
+
// ## Which tree is drilled, said out loud on every run
|
|
68
|
+
//
|
|
69
|
+
// A worktree is a **commit**, so this tool reports on a commit and prints which one. By default it
|
|
70
|
+
// refuses a dirty tree rather than quietly drilling `HEAD` while the reader believes it drilled what
|
|
71
|
+
// they are looking at — the shape of *the gate allows in silence when it reads the wrong tree*.
|
|
72
|
+
// `--working-copy` synthesizes a commit from the working copy with `git stash create` and prints
|
|
73
|
+
// **that** sha; because `stash create` does not carry untracked-and-unstaged files, that mode refuses
|
|
74
|
+
// while any exist, naming them, rather than drilling a tree missing the very file under review.
|
|
75
|
+
//
|
|
76
|
+
// ## Why the sweep is NOT a verify recipe, and what is
|
|
77
|
+
//
|
|
78
|
+
// `../.portulan/dod.md` condition 1 asks that every recipe ran green *in this working copy*. A recipe
|
|
79
|
+
// that reported on `HEAD` would be a green about a different tree. So the **sweep** belongs to the
|
|
80
|
+
// calendar and to `--working-copy` by hand, and what is declared as the `drills` recipe is `--check`:
|
|
81
|
+
// the correspondence pass over the working tree — every yielded rail has a drill, every drill names a
|
|
82
|
+
// declared rail, every anchor still places exactly once. That is the half that can drift on any commit,
|
|
83
|
+
// it runs no rail, and it costs milliseconds.
|
|
84
|
+
//
|
|
85
|
+
// ## This module SPAWNS, and its siblings assert they do not
|
|
86
|
+
//
|
|
87
|
+
// `./goldens.mjs`, `./mutants.mjs` and `./fuzz-shell.mjs` are each held by their suites to importing no
|
|
88
|
+
// process-spawning API, because a corpus of `git push --force` spellings must never reach a shell.
|
|
89
|
+
// This module is the opposite by nature: running a rail *is* spawning it. What its suite pins instead
|
|
90
|
+
// is that nothing it spawns is composed from a payload — every command comes from the yielded recipe
|
|
91
|
+
// set or from this file's own declarations, and no drill's `perturb` value is ever passed to a shell.
|
|
92
|
+
|
|
93
|
+
import fs from "node:fs";
|
|
94
|
+
import os from "node:os";
|
|
95
|
+
import path from "node:path";
|
|
96
|
+
import crypto from "node:crypto";
|
|
97
|
+
import { spawnSync } from "node:child_process";
|
|
98
|
+
import { pathToFileURL } from "node:url";
|
|
99
|
+
|
|
100
|
+
import { CouldNotRun } from "./goldens.mjs";
|
|
101
|
+
import { isInside } from "./inside.mjs";
|
|
102
|
+
import { recipeSet, resolverFor } from "./recipe-set.mjs";
|
|
103
|
+
|
|
104
|
+
/** How long any one rail may take before the drill calls it a could-not-run rather than a verdict. */
|
|
105
|
+
const RAIL_TIMEOUT_MS = 10 * 60 * 1000;
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The prefix every session id this harness invents carries.
|
|
109
|
+
*
|
|
110
|
+
* It is load-bearing twice. A hook keyed by session id must not share state between runs, and — because
|
|
111
|
+
* `./stop-gate.mjs` writes its refusal counter into the OS temp directory under
|
|
112
|
+
* `portulan-stopgate-<readable-session-id>-…` — a prefix this harness owns is what lets it retire its
|
|
113
|
+
* own counter files without touching a real session's. Measured before it was relied on: each sweep was
|
|
114
|
+
* leaving one 45-byte file behind, found at the pre-commit checkpoint.
|
|
115
|
+
*/
|
|
116
|
+
export const DRILL_SESSION_PREFIX = "portulan-drill";
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The rails that are NOT recipes, declared as an allow-list.
|
|
120
|
+
*
|
|
121
|
+
* **The allow-list is the point rather than the convenience.** Without it a drill whose `rail` is a
|
|
122
|
+
* misspelled recipe id — `doc` for `docs` — would be silently reclassified as a rail of its own, and
|
|
123
|
+
* the coverage check would pass while the real recipe went undrilled. A rail is a yielded recipe id or
|
|
124
|
+
* a member of this list, and nothing else.
|
|
125
|
+
*
|
|
126
|
+
* Both members are hooks: they exit 0 whether they refuse or allow, because a hook communicates through
|
|
127
|
+
* its stdout JSON and not through its status. So their drills declare `exit: 0` and lean entirely on the
|
|
128
|
+
* tell — which is why the tell is mandatory rather than a nicety.
|
|
129
|
+
*
|
|
130
|
+
* `tellStream: "stdout"` for both, measured rather than assumed: `stop-gate` writes git's own
|
|
131
|
+
* diagnostics to stderr when it probes an upstream in a detached worktree (`fatal: HEAD does not point
|
|
132
|
+
* to a branch`), so a tell read from the combined streams would be read out of noise the rail did not
|
|
133
|
+
* choose to say.
|
|
134
|
+
*/
|
|
135
|
+
export const NON_RECIPE_RAILS = [
|
|
136
|
+
{
|
|
137
|
+
id: "stop-gate",
|
|
138
|
+
what: "the Stop-gate — `./stop-gate.mjs`, wired by the compiler as a `Stop` hook. It runs the workspace's default recipe and blocks the turn ending when it is not green.",
|
|
139
|
+
argv: ["cli/stop-gate.mjs"],
|
|
140
|
+
tellStream: "stdout",
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
id: "gate",
|
|
144
|
+
what: "the PreToolUse gate runner — `./gate.mjs`, which finds the rule of the yielded policy an attempted action matches and returns that rule's own sentence.",
|
|
145
|
+
argv: ["cli/gate.mjs"],
|
|
146
|
+
tellStream: "stdout",
|
|
147
|
+
},
|
|
148
|
+
];
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* The rails this harness does NOT drill, each with the reason.
|
|
152
|
+
*
|
|
153
|
+
* Printed on every run, green included. The clause's whole subject is the honesty of the word *every*,
|
|
154
|
+
* and a scope claim with no carrier is the defect `../.portulan/verify/README.md`'s register was
|
|
155
|
+
* already carrying by hand. These are rails; they are simply not rails a local process can force.
|
|
156
|
+
*/
|
|
157
|
+
export const NOT_DRILLED = [
|
|
158
|
+
{
|
|
159
|
+
rail: "the platform floor — branch protection, the required checks, `enforce_admins`",
|
|
160
|
+
why: "forcing it red means a direct push to `main` or a merge of a red pull request: outward, Gated, and the maintainer's. `../.portulan/verify/README.md` records that `enforce_admins` is what stands behind the last inch and that no pull request can demonstrate it about itself.",
|
|
161
|
+
},
|
|
162
|
+
{
|
|
163
|
+
rail: "the `permissions` layer the compiler emits",
|
|
164
|
+
why: "that gate is the host's, not this repository's. `compile` byte-checks what is emitted and `goldens`/`mutants` grade the matcher behind it; forcing the host to refuse is a measurement of Claude Code, taken by hand and recorded in `../.portulan/gate-map.md`.",
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
rail: "the CI seam — a non-zero recipe becoming a failed check becoming `BLOCKED`",
|
|
168
|
+
why: "drill 1 of 2026-07-30 established it in both directions on #118, and re-drilling it needs a pull request opened for the purpose. Covered once, deliberately not recurring: the seam is shared by every recipe in one job, so it does not need covering once per rail.",
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
rail: "the pre-commit seam scan (`../.portulan/dod.md` condition 5)",
|
|
172
|
+
why: "its term list lives outside this repository by design, so no scheduled job here can run it. It is drilled once per session instead, by the planted-term control every session's attestation records.",
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
rail: "`claude plugin validate --strict`",
|
|
176
|
+
why: "deliberately not a verify recipe — `../.portulan/identity.md` argues why: CI installs nothing, so declaring it would make a recipe that exits 2 on every run. It is checked at the supervised checkpoints and before a release, which is a rail on no calendar, and that cost is already stated in `../.portulan/verify/README.md`.",
|
|
177
|
+
},
|
|
178
|
+
{
|
|
179
|
+
rail: "Dependabot, the `copilot auto-review` ruleset, and `pr-labels`",
|
|
180
|
+
why: "platform watchers whose triggering condition is an event this repository cannot manufacture on a schedule. `0007`'s own honest limit applies — not every watcher can be forced red safely — and each carries its own recorded observation instead.",
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
rail: "the librarian's scheduled pass",
|
|
184
|
+
why: "not a rail in this sense: it files work for a human rather than refusing an action, so there is no red to force. Its own failure is a failed workflow run, and `0007` covers it as a watcher.",
|
|
185
|
+
},
|
|
186
|
+
];
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* The drills. One per rail, and the harness refuses a yielded rail that has none.
|
|
190
|
+
*
|
|
191
|
+
* **One table, in this module** — `./mutants.mjs`'s argument, unchanged: a drill is an anchor into a
|
|
192
|
+
* file plus the outcome expected of it, and splitting those across code and JSON would make two
|
|
193
|
+
* carriers of one drill.
|
|
194
|
+
*
|
|
195
|
+
* Fields:
|
|
196
|
+
* `rail` a yielded recipe id, or a `NON_RECIPE_RAILS` member's id
|
|
197
|
+
* `perturb` `{file, find, replace}` — anchored, must place exactly once; or `{create, content}`;
|
|
198
|
+
* or `null` for a rail whose control and fire differ by INPUT rather than by tree
|
|
199
|
+
* `stage` `git add -A` in the drill worktree before the fire, for a rail that reads the index
|
|
200
|
+
* `exit` the status the fire must return
|
|
201
|
+
* `tell` a substring the rail's own output must carry when it fires, and must not carry before
|
|
202
|
+
* `why` what this drill proves about this rail — not what the perturbation does
|
|
203
|
+
*
|
|
204
|
+
* **Prefer an in-place edit of a tracked file to a creation.** Measured: `docs.sh` and
|
|
205
|
+
* `control-chars.sh` both walk `--cached --others --exclude-standard` and see a new file, while a pass
|
|
206
|
+
* reading `--cached` alone does not — so a creation-shaped perturbation is invisible to some rails and
|
|
207
|
+
* would read as a rail that did not fire. Where a creation is the only honest shape, `stage` makes it
|
|
208
|
+
* visible and says so.
|
|
209
|
+
*/
|
|
210
|
+
export const DRILLS = [
|
|
211
|
+
// ------------------------------------------------------------------------- the yielded recipes
|
|
212
|
+
// (Including the composed one at the end of this section. This read *the eighteen yielded recipes*,
|
|
213
|
+
// which was both a count in prose and wrong — the section holds the workspace's own plus the pack's.
|
|
214
|
+
// The number has one carrier and it is the runner. Copilot, round 2's notes.)
|
|
215
|
+
{
|
|
216
|
+
rail: "docs",
|
|
217
|
+
perturb: {
|
|
218
|
+
file: "CONTRIBUTING.md",
|
|
219
|
+
find: "- [`docs/plan.md`](docs/plan.md) — the milestones, and the Session log every session appends to.",
|
|
220
|
+
replace: "- [`docs/plan.md`](docs/plan-the-drill-broke-this.md) — the milestones, and the Session log every session appends to.",
|
|
221
|
+
},
|
|
222
|
+
exit: 1,
|
|
223
|
+
tell: "link(s) that do not resolve in the repository",
|
|
224
|
+
why: "The `links` check is the one that has fired for real here, twice on 2026-07-30, over a path that existed in a working copy and not in a clean checkout. This keeps it firing.",
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
rail: "json",
|
|
228
|
+
perturb: {
|
|
229
|
+
file: ".portulan/labels.json",
|
|
230
|
+
find: '"policy": "at-least-one",',
|
|
231
|
+
replace: '"policy": "at-least-one",,',
|
|
232
|
+
},
|
|
233
|
+
exit: 1,
|
|
234
|
+
tell: "Expected double-quoted property name",
|
|
235
|
+
why: "A manifest that does not parse gates nothing. This rail is the floor every JSON-reading rail above it stands on, so its silence would be the widest.",
|
|
236
|
+
},
|
|
237
|
+
{
|
|
238
|
+
rail: "doctor",
|
|
239
|
+
perturb: {
|
|
240
|
+
file: ".portulan/workspace.json",
|
|
241
|
+
find: '"identity": "identity.md",',
|
|
242
|
+
replace: '"identity": "identity-the-drill-removed.md",',
|
|
243
|
+
},
|
|
244
|
+
exit: 1,
|
|
245
|
+
tell: "slots.identity points at",
|
|
246
|
+
why: "`doctor` resolving a slot path is the check the Workspace Definition rests on: a manifest naming a file that is not there must never validate.",
|
|
247
|
+
},
|
|
248
|
+
{
|
|
249
|
+
// ---- the SECOND drill on the `doctor` rail, and the reason there are two
|
|
250
|
+
//
|
|
251
|
+
// `check()` counts a rail as drilled once any drill names it, so a second entry adds coverage
|
|
252
|
+
// rather than a duplicate — the DRILLS docblock's "one per rail" is a floor, not a cap. This
|
|
253
|
+
// arm needs its own because the one above perturbs a slot path and this one perturbs the gate
|
|
254
|
+
// POLICY: different file, different check inside `doctor`, and nothing about the first would
|
|
255
|
+
// have noticed if this one stopped firing.
|
|
256
|
+
//
|
|
257
|
+
// **Why it is a `doctor` drill and not a `compile` one.** The refusal it forces is a
|
|
258
|
+
// `CompileError`, so `compile` exits **2** — and `check()` refuses any drill declaring
|
|
259
|
+
// `exit: 2`, correctly, because it "would read a refusal as a verdict". `doctor` runs the
|
|
260
|
+
// backends inside its own try and turns the same refusal into a `fail`, which is exit 1 and is
|
|
261
|
+
// a verdict. Measured on this tree, 2026-09-03: with `edit-on-a-working-branch` flipped to
|
|
262
|
+
// `gated`, `compile` exits 2, `goldens` stays 0, and `doctor` exits 1 printing the sentence
|
|
263
|
+
// below.
|
|
264
|
+
rail: "doctor",
|
|
265
|
+
perturb: {
|
|
266
|
+
file: ".portulan/gates.json",
|
|
267
|
+
// The flip is the whole hazard in one word: this rule's target is `./`, which matches no
|
|
268
|
+
// path a host submits, and at `auto` that is harmless — the tier is refused before any
|
|
269
|
+
// target is read. At `gated` it becomes a hollow gate, reported compiled and enforcing
|
|
270
|
+
// nothing. So the perturbation manufactures exactly the state hole 8 describes, from the
|
|
271
|
+
// policy this repository actually ships.
|
|
272
|
+
find: '"id": "edit-on-a-working-branch",\n "tier": "auto",',
|
|
273
|
+
replace: '"id": "edit-on-a-working-branch",\n "tier": "gated",',
|
|
274
|
+
},
|
|
275
|
+
exit: 1,
|
|
276
|
+
// Scoped to the sentence only this refusal prints. `doctor` has other enforcement failures and
|
|
277
|
+
// a tell like "enforcement" would be satisfied by any of them.
|
|
278
|
+
tell: "matches no path a host can submit",
|
|
279
|
+
why: "A gate whose target can never match is the one defect this repository cannot see by reading a policy: the compiler reports it COMPILED and `doctor` counts it covered, so the gate map, the artifact and the report all agree about a gate that enforces nothing. Hole 8 of the gate map, closed by #337's option 3 — and the only rail that can watch the closure is the one whose exit is a verdict rather than a refusal.",
|
|
280
|
+
},
|
|
281
|
+
{
|
|
282
|
+
rail: "tests",
|
|
283
|
+
perturb: {
|
|
284
|
+
create: "cli/drill.test.mjs",
|
|
285
|
+
content: [
|
|
286
|
+
"// A forced-red drill fixture, written by `cli/drills.mjs` into a throwaway worktree and never",
|
|
287
|
+
"// committed. If you are reading this file inside the repository, a drill did not clean up after",
|
|
288
|
+
"// itself and that is the bug.",
|
|
289
|
+
'import { test } from "node:test";',
|
|
290
|
+
'import assert from "node:assert/strict";',
|
|
291
|
+
"",
|
|
292
|
+
'test("forced-red drill: the tests recipe reports a failing assertion", () => {',
|
|
293
|
+
" assert.equal(1, 2);",
|
|
294
|
+
"});",
|
|
295
|
+
"",
|
|
296
|
+
].join("\n"),
|
|
297
|
+
},
|
|
298
|
+
exit: 1,
|
|
299
|
+
tell: "forced-red drill: the tests recipe reports a failing assertion",
|
|
300
|
+
why: "The shape drill 1 used on #118 on 2026-07-30, kept as the recurring form of the only rail this repository has ever observed firing on purpose in CI.",
|
|
301
|
+
},
|
|
302
|
+
{
|
|
303
|
+
rail: "plugin",
|
|
304
|
+
perturb: {
|
|
305
|
+
file: ".claude-plugin/plugin.json",
|
|
306
|
+
find: '"./plugin/skills/",',
|
|
307
|
+
replace: '"./plugin/skills-the-drill-moved/",',
|
|
308
|
+
},
|
|
309
|
+
exit: 1,
|
|
310
|
+
tell: "plugin/skills-the-drill-moved",
|
|
311
|
+
why: "A declared component path that resolves nowhere is how a plugin ships registering nothing — the defect `skills-set` exists beside this check to make impossible.",
|
|
312
|
+
},
|
|
313
|
+
{
|
|
314
|
+
rail: "compile",
|
|
315
|
+
perturb: {
|
|
316
|
+
file: ".claude/settings.json",
|
|
317
|
+
find: '"Bash(gh pr merge:*)",',
|
|
318
|
+
replace: '"Bash(gh pr merge:*)",\n "Bash(the-drill-added-this:*)",',
|
|
319
|
+
},
|
|
320
|
+
exit: 1,
|
|
321
|
+
tell: "has drifted from",
|
|
322
|
+
why: "A committed generated file invites the hand-fix that works until the next compile silently reverts it. This is the drift rail for the enforcement itself, so its firing is the difference between a compiled gate and a decorative one.",
|
|
323
|
+
},
|
|
324
|
+
{
|
|
325
|
+
rail: "workflow-filters",
|
|
326
|
+
perturb: {
|
|
327
|
+
file: ".github/workflows/pr-labels.yml",
|
|
328
|
+
find: "if ! declared=$(jq -er '.labels[].name' \"$POLICY\"); then",
|
|
329
|
+
// The program's OUTPUT is changed rather than its shape: dropping a character made the
|
|
330
|
+
// fixture's anchor match no program at all, and the rail then exited 2 — a legitimate
|
|
331
|
+
// refusal, but the arm this drill is not for. Appending a suffix keeps the anchor findable
|
|
332
|
+
// and makes the bytes the surrounding shell branches on differ, which is the exit-1 arm.
|
|
333
|
+
// Measured both ways before this line was written.
|
|
334
|
+
replace: "if ! declared=$(jq -er '.labels[].name + \"-drilled\"' \"$POLICY\"); then",
|
|
335
|
+
},
|
|
336
|
+
exit: 1,
|
|
337
|
+
tell: "fixture(s) failed",
|
|
338
|
+
why: "The jq programs the merge gates branch on are executed rather than described. A filter that silently changed shape would take the label gate down with it.",
|
|
339
|
+
},
|
|
340
|
+
{
|
|
341
|
+
rail: "index",
|
|
342
|
+
perturb: {
|
|
343
|
+
file: ".portulan/memory-index.md",
|
|
344
|
+
find: "- [A branch syncs with main before it merges](memory/a-branch-syncs-with-main-before-it-merges.md) — rule",
|
|
345
|
+
replace: "- [A branch syncs with main before it merges](memory/a-branch-syncs-with-main-before-it-merges.md) — rule (edited by hand)",
|
|
346
|
+
},
|
|
347
|
+
exit: 1,
|
|
348
|
+
tell: "is out of date against the store",
|
|
349
|
+
why: "The index is generated and byte-compared, which is the only thing that makes `core/operating/memory.md`'s *generated, never hand-maintained* a fact rather than a preference.",
|
|
350
|
+
},
|
|
351
|
+
{
|
|
352
|
+
rail: "control-chars",
|
|
353
|
+
perturb: {
|
|
354
|
+
file: "CONTRIBUTING.md",
|
|
355
|
+
find: "# Contributing",
|
|
356
|
+
// A real NUL, written as an escape so THIS file carries no control byte of its own. Session 0
|
|
357
|
+
// shipped a literal NUL as a literal character while writing prose about storing bytes escaped.
|
|
358
|
+
replace: "# Contributing\u0000",
|
|
359
|
+
},
|
|
360
|
+
exit: 1,
|
|
361
|
+
tell: "control character(s) — first at line",
|
|
362
|
+
why: "A raw NUL shipped here once inside a template literal, and the tool most likely to have shown it — `grep` — is the tool the byte silences. Only a byte reader can fire on this, so only a byte reader's silence would hide it.",
|
|
363
|
+
},
|
|
364
|
+
{
|
|
365
|
+
rail: "rule-carriers",
|
|
366
|
+
perturb: {
|
|
367
|
+
file: ".portulan/README.md",
|
|
368
|
+
find: "# `.portulan/` — Portulan's own workspace",
|
|
369
|
+
replace: "# `.portulan/` — Portulan's own workspace\n\nCI runs every recipe the manifest declares.",
|
|
370
|
+
},
|
|
371
|
+
exit: 1,
|
|
372
|
+
tell: "recipe the manifest declares",
|
|
373
|
+
why: "A rule an incident reduced to one carrier must stay reduced. The registered spelling reappearing uncited is exactly the drift proposal 0027 was written for.",
|
|
374
|
+
},
|
|
375
|
+
{
|
|
376
|
+
rail: "pack-version",
|
|
377
|
+
perturb: {
|
|
378
|
+
file: "packs/tools/github/pack.json",
|
|
379
|
+
find: ' "tr"\n ],',
|
|
380
|
+
replace: ' "tr",\n "awk"\n ],',
|
|
381
|
+
},
|
|
382
|
+
exit: 1,
|
|
383
|
+
tell: "without moving `portulan.version`",
|
|
384
|
+
why: "A pack whose contributions moved without its version moving is a pin that no longer names what it pins — the whole basis on which an adopter runs third-party code in their CI.",
|
|
385
|
+
},
|
|
386
|
+
{
|
|
387
|
+
rail: "pack-identity",
|
|
388
|
+
perturb: {
|
|
389
|
+
file: "README.md",
|
|
390
|
+
find: "**Current release: `0.1.3`**",
|
|
391
|
+
replace: "**Current release: `0.1.2`** <!-- the drill edited this and did not stage it -->",
|
|
392
|
+
},
|
|
393
|
+
stage: false,
|
|
394
|
+
exit: 1,
|
|
395
|
+
tell: "the package would not install the tree's bytes",
|
|
396
|
+
why: "`.portulan/identity.md` claims the `npx` path installs the same bytes as the tree. This rail is that claim's only continuous check; an unstaged edit is precisely the drift it owns.",
|
|
397
|
+
},
|
|
398
|
+
{
|
|
399
|
+
rail: "payload",
|
|
400
|
+
perturb: {
|
|
401
|
+
create: "cli/a-module-the-drill-left-unclassified.mjs",
|
|
402
|
+
content: [
|
|
403
|
+
"// A forced-red drill fixture, written by `cli/drills.mjs` into a throwaway worktree and never",
|
|
404
|
+
"// committed. If you are reading this file inside the repository, a drill did not clean up after",
|
|
405
|
+
"// itself and that is the bug.",
|
|
406
|
+
"//",
|
|
407
|
+
"// It is here because `package.json`'s `files` sweeps `cli/`, so ANY module landing here joins the",
|
|
408
|
+
"// npm payload. `cli/pack-identity.mjs` would report green over it — it checks the bytes of what is",
|
|
409
|
+
"// packed and never the roster — which is how `ab.mjs`, `ab-run.mjs` and `ab-grade.mjs` shipped for",
|
|
410
|
+
"// three sessions before #382 removed them.",
|
|
411
|
+
"",
|
|
412
|
+
"export const theDrillPlantedThis = true;",
|
|
413
|
+
"",
|
|
414
|
+
].join("\n"),
|
|
415
|
+
},
|
|
416
|
+
// **NOT staged, and that is the drill's second subject.** `npm pack` reads the WORKING TREE, so an
|
|
417
|
+
// untracked module ships — measured before this drill was written. A drill that staged first would
|
|
418
|
+
// pass while leaving the likelier arrival untested: a module written, packed and published before
|
|
419
|
+
// anyone committed it. `pack-identity` above is the mirror case and stages for the opposite reason.
|
|
420
|
+
stage: false,
|
|
421
|
+
exit: 1,
|
|
422
|
+
tell: "SHIPS and is classified by nothing",
|
|
423
|
+
why:
|
|
424
|
+
"`package.json`'s `files` array governs the payload and nothing derived its membership until #383 — " +
|
|
425
|
+
"`pack-identity` holds the packed BYTES and is silent on which files are packed. The drill proves the " +
|
|
426
|
+
"rail sees a real arrival in the tree rather than only a synthetic report handed to `findings()`, " +
|
|
427
|
+
"which is the half a unit test cannot reach.",
|
|
428
|
+
},
|
|
429
|
+
{
|
|
430
|
+
rail: "eval-bundle",
|
|
431
|
+
perturb: {
|
|
432
|
+
create: "drill-top-level-path/README.md",
|
|
433
|
+
content: "A top-level path the eval-bundle roster has never seen.\n",
|
|
434
|
+
},
|
|
435
|
+
stage: true,
|
|
436
|
+
exit: 1,
|
|
437
|
+
tell: "drill-top-level-path",
|
|
438
|
+
why: "The bundle's pinned top-level roster is what stops the next licensed cut silently thinning or mislicensing. It reads the index, so this drill stages — a creation it cannot see is a rail that reads as not firing.",
|
|
439
|
+
},
|
|
440
|
+
{
|
|
441
|
+
rail: "goldens",
|
|
442
|
+
perturb: {
|
|
443
|
+
file: "evals/goldens/gates/force-push-without-a-lease.json",
|
|
444
|
+
find: ' "command": "git push --force origin main"\n },\n "expect": true,',
|
|
445
|
+
replace: ' "command": "git push --force origin main"\n },\n "expect": false,',
|
|
446
|
+
},
|
|
447
|
+
exit: 1,
|
|
448
|
+
tell: "the-bare-spelling",
|
|
449
|
+
why: "The corpus is only worth its green if a case that stops answering as recorded reds it. This flips the control every other case in that file is measured against.",
|
|
450
|
+
},
|
|
451
|
+
{
|
|
452
|
+
rail: "mutants",
|
|
453
|
+
perturb: {
|
|
454
|
+
file: "cli/mutants.mjs",
|
|
455
|
+
find: ' replace: "s === action.shell || s.startsWith(action.shell)",\n outcome: "killed",',
|
|
456
|
+
replace: ' replace: "s === action.shell || s.startsWith(action.shell)",\n outcome: "survives",',
|
|
457
|
+
},
|
|
458
|
+
exit: 1,
|
|
459
|
+
tell: "is recorded SURVIVES and was KILLED",
|
|
460
|
+
why: "A recorded outcome that stops being true must red in BOTH directions — a `killed` operator that survives means the kill-set weakened, and a `survives` one that is killed is good news the record has to absorb. This drills the second direction, which is the one a reader is least likely to expect a rail to hold. _(The first draft renamed an operator id instead, on the assumption that the census pins its own roster. Measured: it does not — the census stayed GREEN. The drill was rewritten around what the rail actually does rather than around what its author assumed.)_",
|
|
461
|
+
},
|
|
462
|
+
{
|
|
463
|
+
rail: "fuzz-shell",
|
|
464
|
+
perturb: {
|
|
465
|
+
file: "cli/compile.mjs",
|
|
466
|
+
find: 'const REDIRECTION_TARGET = String.raw`(?:"(?:\\\\[\\s\\S]|[^"\\\\])*"|\'[^\']*\'|\\\\[\\s\\S]|[^\\s])+`;',
|
|
467
|
+
replace: "const REDIRECTION_TARGET = String.raw`[^\\s]+`;",
|
|
468
|
+
},
|
|
469
|
+
exit: 1,
|
|
470
|
+
tell: "disagree with the recorded grammar",
|
|
471
|
+
why: "Session 1's own drill, kept: this is the exact #336 defect — a redirection target reader narrower than a shell word — and the fuzzer is what turns that class from three review rounds into a generated red.",
|
|
472
|
+
},
|
|
473
|
+
{
|
|
474
|
+
rail: "version-carriers",
|
|
475
|
+
perturb: {
|
|
476
|
+
file: "README.md",
|
|
477
|
+
find: "**Current release: `0.1.3`**",
|
|
478
|
+
replace: "**Current release: `0.2.0`**",
|
|
479
|
+
},
|
|
480
|
+
// **Staged, because this rail reads the INDEX** — `git show :<path>`, the same way
|
|
481
|
+
// `pack-identity` does and for the reason that module states: enumerating from the index and
|
|
482
|
+
// then reading the worktree lets a staged drift with a reverted worktree copy report green.
|
|
483
|
+
// Measured: unstaged, the perturbation is invisible and the rail reports its ordinary green,
|
|
484
|
+
// which would have read as a rail that stopped firing.
|
|
485
|
+
stage: true,
|
|
486
|
+
exit: 1,
|
|
487
|
+
tell: "but package.json declares",
|
|
488
|
+
why: "This repository shipped that exact defect twice, and `README.md` is in `package.json`'s `files` — npm freezes a README per version, so a wrong sentence that reaches a publish needs another release to correct.",
|
|
489
|
+
},
|
|
490
|
+
{
|
|
491
|
+
rail: "skill-goldens",
|
|
492
|
+
perturb: {
|
|
493
|
+
file: "core/skills/clarify/SKILL.md",
|
|
494
|
+
// **The perturbation ADDS A MANDATE**, which puts this rail's central claim under test: the
|
|
495
|
+
// denominator is derived from the skill's own pass, so a step nobody wrote a case for is a
|
|
496
|
+
// red rather than a default. A corpus whose author picks which mandates to answer for
|
|
497
|
+
// reports "5 of 5" of a set it drew, and that is the version this drill exists to refuse.
|
|
498
|
+
//
|
|
499
|
+
// **Rewording a mandate was the first spelling and it is the wrong arm.** An anchor that
|
|
500
|
+
// stops placing is could-not-run, and this harness refuses `exit: 2` as a fire outright —
|
|
501
|
+
// *"counting it as a fire would read a refusal as a verdict"*. Measured rather than
|
|
502
|
+
// reasoned about: `--check` rejected the first version of this entry for exactly that.
|
|
503
|
+
find: "## Why it earns its tokens",
|
|
504
|
+
replace: "5. **A step the drill added** — bound to nothing, answered by no case.\n\n## Why it earns its tokens",
|
|
505
|
+
},
|
|
506
|
+
exit: 1,
|
|
507
|
+
tell: "is in the skill's pass and in no case",
|
|
508
|
+
why: "A core skill states its mandates in prose, and prose is what this repository ships. Before this rail a step could be added, reworded or deleted with every check still green. The derived denominator is what makes the corpus a rail rather than a self-graded exam, so it is the half worth drilling.",
|
|
509
|
+
},
|
|
510
|
+
{
|
|
511
|
+
rail: "review-loop",
|
|
512
|
+
perturb: {
|
|
513
|
+
file: "evals/review-loop/register.md",
|
|
514
|
+
find: "- **Repository:** `sleepy-panda-srl/portulan`",
|
|
515
|
+
replace: "- **Repository:** `sleepy-panda-srl/portulan` (edited by hand)",
|
|
516
|
+
},
|
|
517
|
+
exit: 1,
|
|
518
|
+
tell: "is out of date against the snapshot",
|
|
519
|
+
// The perturbation edits the REGISTER rather than the snapshot, and that is the direction the
|
|
520
|
+
// rail exists for. Editing the snapshot would make both sides move together in a rerun and
|
|
521
|
+
// prove nothing; editing the published figures is the failure the recipe is there to catch,
|
|
522
|
+
// since a committed Markdown table is the one artifact here a person can quietly correct.
|
|
523
|
+
why: "The register is the review loop's published figures, and this clause exists because the figures it replaces were hand-maintained. A register that could drift from its own snapshot would be that hand-maintained tally wearing a generated file's clothes.",
|
|
524
|
+
},
|
|
525
|
+
{
|
|
526
|
+
rail: "telemetry",
|
|
527
|
+
perturb: {
|
|
528
|
+
file: ".portulan/verify/review-loop.sh",
|
|
529
|
+
find: " --snapshot evals/review-loop/snapshot.json \\",
|
|
530
|
+
replace: " --fetch \\\n --snapshot evals/review-loop/snapshot.json \\",
|
|
531
|
+
},
|
|
532
|
+
exit: 1,
|
|
533
|
+
tell: "invokes cli/review-meter.mjs --fetch",
|
|
534
|
+
// **The perturbation is a network mode reaching a recipe, which is the rail's actual subject —
|
|
535
|
+
// not a drifted payload, which would have been the cheaper drill.** Byte-drift is already the
|
|
536
|
+
// shape `review-loop` and `index` drill; what nothing else here watches is the offline
|
|
537
|
+
// property, so that is what this forces.
|
|
538
|
+
//
|
|
539
|
+
// And it forces it through the mode this session did NOT introduce. `--fetch` predates the
|
|
540
|
+
// telemetry clause by a session, so a drill that fired on `--export` would prove only that the
|
|
541
|
+
// checker recognises its author's own module — while the whole argument for railing a class
|
|
542
|
+
// rather than a file is that the OLDER network mode was the unrailed one. A drill on the new
|
|
543
|
+
// mode would have passed while leaving exactly the gap the class exists to close.
|
|
544
|
+
why: "A verify recipe that can reach the network goes red about the world rather than about the tree, and this repository prohibits it in three carriers while nothing checked it. The drill proves the checker sees a real invocation — spread across continued lines, the spelling a recipe actually uses — rather than only the literal string a test would hand it.",
|
|
545
|
+
},
|
|
546
|
+
{
|
|
547
|
+
rail: "ab",
|
|
548
|
+
perturb: {
|
|
549
|
+
create: ".portulan/a-file-the-drill-left-unclassified.md",
|
|
550
|
+
content: [
|
|
551
|
+
"A forced-red drill fixture, written by `cli/drills.mjs` into a throwaway worktree and never",
|
|
552
|
+
"committed. If you are reading this file inside the repository, a drill did not clean up after",
|
|
553
|
+
"itself and that is the bug.",
|
|
554
|
+
"",
|
|
555
|
+
"It is here because `cli/ab.mjs`'s disposition table has to be TOTAL over `.portulan/`: anything",
|
|
556
|
+
"it does not classify is carried into the A/B treatment arm by `cli/vendor.mjs`, which walks the",
|
|
557
|
+
"whole workspace directory.",
|
|
558
|
+
"",
|
|
559
|
+
].join("\n"),
|
|
560
|
+
},
|
|
561
|
+
exit: 1,
|
|
562
|
+
// **The perturbation ADDS a file rather than editing the table, and that is the rail's actual
|
|
563
|
+
// subject.** Deleting a disposition would fire the *stale declaration* arm — a legitimate
|
|
564
|
+
// refusal, and exit 2, which is the arm this drill is not for. What the rail exists to catch is
|
|
565
|
+
// the opposite direction: a file arriving in `.portulan/` that nobody classified, which is
|
|
566
|
+
// precisely how customer zero's `memory-index.md`, `handoffs-index.md`, `rule-carriers.json`
|
|
567
|
+
// and `labels.json` reached an arm built to `evals/ab/arm.md`'s six rows with `doctor` GREEN
|
|
568
|
+
// over all of it. The drill reproduces the arrival, not the forgetting.
|
|
569
|
+
tell: "classified by no disposition",
|
|
570
|
+
why:
|
|
571
|
+
"The A/B treatment arm is built by removing things from customer zero's workspace, and `cli/vendor.mjs` carries " +
|
|
572
|
+
"every ordinary file under it — so a specification that names what to remove leaks whatever it forgot, silently, " +
|
|
573
|
+
"into the arm an experiment is about to measure. That leak has already happened once and `doctor` was green on it. " +
|
|
574
|
+
"This is the rail that turns the next one into a red on the commit that adds the file.",
|
|
575
|
+
},
|
|
576
|
+
{
|
|
577
|
+
rail: "ab-grade",
|
|
578
|
+
perturb: {
|
|
579
|
+
file: "cli/ab-grade.mjs",
|
|
580
|
+
// **The perturbation makes a grader a CONSTANT, which is the rail's actual subject.** Drifting
|
|
581
|
+
// the register would have been the cheaper drill and would have proved only the byte compare —
|
|
582
|
+
// a shape `review-loop`, `index` and `ab` already force. What nothing else here watches is a
|
|
583
|
+
// grader that has stopped reading the arm, and that is the failure this instrument was built
|
|
584
|
+
// around: the prototype returned "30 records, each with its own provenance and retirement
|
|
585
|
+
// condition" against an arm that had done nothing at all.
|
|
586
|
+
//
|
|
587
|
+
// It is planted in `gradeCuratedLayer` deliberately — the prototype's own scenario family —
|
|
588
|
+
// and it is an early `return` rather than an edited branch, so it fires whatever the delta,
|
|
589
|
+
// which is what a constant is.
|
|
590
|
+
find: ' const scenario = "curated-layer";\n const anchor = anchored(root, scenario, nonce);',
|
|
591
|
+
replace:
|
|
592
|
+
' const scenario = "curated-layer";\n' +
|
|
593
|
+
' if (fs.existsSync(path.join(root, "AGENTS.md"))) return verdict(scenario, "proposals", "a drill-planted constant about the BASE", []);\n' +
|
|
594
|
+
" const anchor = anchored(root, scenario, nonce);",
|
|
595
|
+
},
|
|
596
|
+
exit: 1,
|
|
597
|
+
// The tell is level 2's, not level 1's. A constant about the base ALSO fails level 1 and
|
|
598
|
+
// inertness here, and naming one of those would let a later edit that fixed only the cheap half
|
|
599
|
+
// pass the drill — while the property this rail is for is that the inverted fixture cannot be
|
|
600
|
+
// inverted by a grader that ignores the delta.
|
|
601
|
+
tell: "did not invert the figures",
|
|
602
|
+
why:
|
|
603
|
+
"A grader that has stopped reading the arm produces a baseline about file copying, and it does so silently: the " +
|
|
604
|
+
"prototype this instrument was built around passed level-1 discrimination and was still a constant. Only inversion " +
|
|
605
|
+
"caught it. This drill plants that constant back and requires the rail to fire — on the level-2 tell specifically, " +
|
|
606
|
+
"because a fix that restored level 1 alone would leave the arm-blindness in place.",
|
|
607
|
+
},
|
|
608
|
+
{
|
|
609
|
+
rail: "release-eval",
|
|
610
|
+
perturb: {
|
|
611
|
+
file: "cli/release-eval.mjs",
|
|
612
|
+
// **The perturbation moves the CLAUSE BOUNDARY, which is the rail's actual subject.**
|
|
613
|
+
// Editing a committed record would have been the cheaper drill and would have proved only
|
|
614
|
+
// the byte compare — a shape `review-loop`, `index`, `ab` and `ab-run` already force. What
|
|
615
|
+
// nothing else here watches is the check this clause exists for: *a release the clause
|
|
616
|
+
// governs, carrying no eval result*. Every governed release this repository has cut carries its
|
|
617
|
+
// record, so that arm is unreachable from the committed tree — until the boundary moves. Dropping it
|
|
618
|
+
// to `0.1.2` makes an already-cut release governed, and the rail must then say the record
|
|
619
|
+
// is missing.
|
|
620
|
+
//
|
|
621
|
+
// It is the boundary rather than a record because a boundary that silently widened is the
|
|
622
|
+
// failure mode with teeth: a release would be graded that nobody captured for, or — the
|
|
623
|
+
// direction that matters — one that should be graded would fall outside and the rail would
|
|
624
|
+
// report green having looked at nothing.
|
|
625
|
+
find: 'export const FIRST_GOVERNED_VERSION = "0.1.3";',
|
|
626
|
+
replace: 'export const FIRST_GOVERNED_VERSION = "0.1.2";',
|
|
627
|
+
},
|
|
628
|
+
exit: 1,
|
|
629
|
+
// The tell is the missing-record arm specifically, not the boundary arithmetic. A fix that
|
|
630
|
+
// restored only the constant while leaving the presence check broken would pass a looser tell.
|
|
631
|
+
tell: "is a release from milestone 8 onward and there is no",
|
|
632
|
+
why:
|
|
633
|
+
"Milestone 8's ninth clause is that a release carries an eval result, and the check that carries it is *a governed " +
|
|
634
|
+
"release with no record reds*. Every governed release this repository has cut carries its record, so that arm cannot " +
|
|
635
|
+
"be exercised from the committed tree at all — the rail's green grades a non-empty set and finds every member " +
|
|
636
|
+
"complete, which is a measurement whose red arm nothing in the tree can reach. This drill moves the boundary so " +
|
|
637
|
+
"the arm becomes reachable and requires it to fire, so the " +
|
|
638
|
+
"clause's central check is watched working rather than assumed from a green over an empty set.",
|
|
639
|
+
},
|
|
640
|
+
{
|
|
641
|
+
rail: "ab-run",
|
|
642
|
+
perturb: {
|
|
643
|
+
file: "evals/ab/baseline.json",
|
|
644
|
+
// **The perturbation edits a PUBLISHED FIGURE, which is the rail's actual subject.** The
|
|
645
|
+
// baseline is the one record here that cannot be re-derived — its events do not repeat — so
|
|
646
|
+
// what holds it honest is that the register agrees with the capture beside it, and that the
|
|
647
|
+
// capture folds to the figures it publishes. Editing a cell is exactly the drift
|
|
648
|
+
// `review-loop.sh` names: *"a published figure that can drift from its own data is the
|
|
649
|
+
// hand-maintained tally in a new costume"*.
|
|
650
|
+
//
|
|
651
|
+
// A drifted REGISTER would have been the cheaper drill and would prove only the byte
|
|
652
|
+
// compare, which `review-loop`, `index` and `ab-grade` already force between them. This
|
|
653
|
+
// forces the arithmetic instead.
|
|
654
|
+
find: '"operatorEnv": "isolated"',
|
|
655
|
+
replace: '"operatorEnv": "inherit"',
|
|
656
|
+
},
|
|
657
|
+
exit: 1,
|
|
658
|
+
// The tell is the isolation clause, not the byte compare: a baseline recorded over an unisolated
|
|
659
|
+
// arm is the one thing `evals/ab/corpus.md` forbids outright, and a repair that fixed only the
|
|
660
|
+
// rendering would leave that hole open.
|
|
661
|
+
tell: "no baseline may be recorded under an unisolated arm",
|
|
662
|
+
why:
|
|
663
|
+
"A baseline is the only record in this repository derived from events rather than from the tree, so it cannot be " +
|
|
664
|
+
"re-derived and a reader has nothing but the capture beside it to trust. This rail holds the published document to " +
|
|
665
|
+
"that capture, and refuses a baseline claiming an arm that is not the ruled one — the departure `evals/ab/corpus.md` " +
|
|
666
|
+
"forbids in terms, and the one a later session is likeliest to reach for when a credential is inconvenient.",
|
|
667
|
+
},
|
|
668
|
+
{
|
|
669
|
+
rail: "tools/github:actions-pinned",
|
|
670
|
+
perturb: {
|
|
671
|
+
file: ".github/workflows/verify.yml",
|
|
672
|
+
find: " - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1",
|
|
673
|
+
replace: " - uses: actions/checkout@v7",
|
|
674
|
+
},
|
|
675
|
+
exit: 1,
|
|
676
|
+
tell: "is a tag or branch, not a commit",
|
|
677
|
+
why: "The composed pack's own rail, drilled from the adopting workspace exactly as a workspace-owned one is — which is the property milestone 7's composition amendment bought and nothing else here re-checks.",
|
|
678
|
+
},
|
|
679
|
+
|
|
680
|
+
// ------------------------------------------------------- the coverage rail, which is itself a rail
|
|
681
|
+
{
|
|
682
|
+
rail: "drills",
|
|
683
|
+
perturb: {
|
|
684
|
+
file: "cli/drills.mjs",
|
|
685
|
+
// **The perturbation is ADDITIVE and the anchor is CONCATENATED, and neither is a style
|
|
686
|
+
// choice.** A drill perturbing its own module meets two traps, and the first draft met
|
|
687
|
+
// both. Renaming an existing declaration destroys the very anchor some drill depends on —
|
|
688
|
+
// measured: the child `--check` then exited 2 saying the drill no longer places, instead
|
|
689
|
+
// of 1 reporting the coverage hole. And an anchor written as one literal would place
|
|
690
|
+
// TWICE, once at its target and once in this line, so `--check` would refuse the whole
|
|
691
|
+
// roster as ambiguous. Splitting the literal keeps this line from being what it searches
|
|
692
|
+
// for; inserting rather than editing leaves every other anchor standing.
|
|
693
|
+
find: "export const DRILL" + "S = [",
|
|
694
|
+
replace:
|
|
695
|
+
"export const DRILL" +
|
|
696
|
+
'S = [\n { rail: "a-rail-this-workspace-does-not-yield", perturb: null, stdinControl: {}, exit: 1, tell: "an inserted drill", why: "inserted by the drill for the `drills` rail" },',
|
|
697
|
+
},
|
|
698
|
+
exit: 1,
|
|
699
|
+
tell: "a-rail-this-workspace-does-not-yield",
|
|
700
|
+
why: "The allow-list is what stops a misspelled recipe id being silently reclassified as a rail of its own, leaving the real recipe undrilled while the coverage check reports green. A drill naming a rail nothing yields is the shortest path to that state.",
|
|
701
|
+
},
|
|
702
|
+
|
|
703
|
+
// ------------------------------------------------------------------------- the two non-recipe rails
|
|
704
|
+
{
|
|
705
|
+
rail: "stop-gate",
|
|
706
|
+
// The same perturbation the `docs` drill uses, because `docs` is this workspace's DEFAULT recipe
|
|
707
|
+
// and the default is what this hook runs. The tell is scoped to the recipe reason on purpose:
|
|
708
|
+
// this gate also blocks for a missing same-day handoff, and whether it does depends on the
|
|
709
|
+
// calendar date and on whether HEAD's patches are on a remote — measured at the session-open
|
|
710
|
+
// checkpoint, which saw the identical tree allow on one date and block on the next. A drill whose
|
|
711
|
+
// control demanded *no block at all* would therefore be green or red by the day of the week.
|
|
712
|
+
perturb: {
|
|
713
|
+
file: "CONTRIBUTING.md",
|
|
714
|
+
find: "- [`docs/plan.md`](docs/plan.md) — the milestones, and the Session log every session appends to.",
|
|
715
|
+
replace: "- [`docs/plan.md`](docs/plan-the-drill-broke-this.md) — the milestones, and the Session log every session appends to.",
|
|
716
|
+
},
|
|
717
|
+
stdin: {
|
|
718
|
+
hook_event_name: "Stop",
|
|
719
|
+
// **A session id unique per run**, completed by `runRail` from the worktree's own name. The
|
|
720
|
+
// constant this replaced was harmless and not obviously so, which is the reason it moved:
|
|
721
|
+
// `stop-gate`'s counter file is keyed by session id **and** by the tree, and the tree is a
|
|
722
|
+
// fresh `mkdtemp` every drill, so two runs could not share a counter either way. Measured
|
|
723
|
+
// at the pre-commit checkpoint. Relying on that is relying on another module's key, and the
|
|
724
|
+
// prefix below is also what lets this harness retire the counter file it leaves behind.
|
|
725
|
+
session_id: DRILL_SESSION_PREFIX,
|
|
726
|
+
},
|
|
727
|
+
exit: 0,
|
|
728
|
+
tell: "PORTULAN STOP-GATE (recipe",
|
|
729
|
+
why: "This gate is what makes a red unmissable rather than merely recorded. It fails open on a crash by design, so a crashed Stop-gate and a green one are indistinguishable from outside — and only a drill that requires the refusal can tell them apart.",
|
|
730
|
+
},
|
|
731
|
+
{
|
|
732
|
+
rail: "gate",
|
|
733
|
+
// No perturbation: this rail's control and fire differ by INPUT. The control is a benign command
|
|
734
|
+
// and the fire is a gated one.
|
|
735
|
+
//
|
|
736
|
+
// **The control is byte-identical to this runner having crashed**, and that is worth knowing
|
|
737
|
+
// rather than fixing: `./gate.mjs` fails open deliberately — any internal error exits 0 emitting
|
|
738
|
+
// nothing, so the permission layer governs unchanged. Empty stdout therefore means *allowed* or
|
|
739
|
+
// *crashed*, indistinguishably. What separates them is the FIRE: a crashing runner is silent on
|
|
740
|
+
// both inputs, so the pair is the oracle even though neither half is.
|
|
741
|
+
perturb: null,
|
|
742
|
+
stdinControl: { tool_name: "Bash", tool_input: { command: "git status --short" } },
|
|
743
|
+
stdin: { tool_name: "Bash", tool_input: { command: "git push --force origin main" } },
|
|
744
|
+
exit: 0,
|
|
745
|
+
tell: "PORTULAN GATE `force-push-without-a-lease`",
|
|
746
|
+
why: "The wrapper spellings and the shell writes are the ground the permission pattern cannot reach, so for those this hook is the only layer. A hook that had stopped emitting would leave that ground uncovered and say nothing.",
|
|
747
|
+
},
|
|
748
|
+
];
|
|
749
|
+
|
|
750
|
+
/** The digest used to prove a perturbation moved bytes. Not security — just enough to differ. */
|
|
751
|
+
const digest = (buffer) => crypto.createHash("sha256").update(buffer).digest("hex");
|
|
752
|
+
|
|
753
|
+
/**
|
|
754
|
+
* Apply a drill's perturbation inside `worktree`.
|
|
755
|
+
*
|
|
756
|
+
* Refuses — could-not-run, never a skip — when the anchor is absent, when it places more than once, or
|
|
757
|
+
* when the bytes on disk did not move. The third is not redundant with the first two: both hand sessions
|
|
758
|
+
* had a drill whose substitution placed and changed nothing that mattered, and `count === 1` alone would
|
|
759
|
+
* have passed a replacement identical to what it replaced.
|
|
760
|
+
*/
|
|
761
|
+
export function perturb(worktree, drill) {
|
|
762
|
+
if (!drill.perturb) return null;
|
|
763
|
+
|
|
764
|
+
/**
|
|
765
|
+
* Refuse a target that does not stay inside the worktree, **before anything is written**.
|
|
766
|
+
*
|
|
767
|
+
* The isolation guarantee is the whole reason a drill runs in a throwaway tree, and nothing checked
|
|
768
|
+
* it: a `..` in a declaration, or a symlinked parent in the checkout, would have let `writeFileSync`
|
|
769
|
+
* reach the caller's own repository — the tree this harness promises never to perturb. `isInside` is
|
|
770
|
+
* the one carrier of the predicate and `realpathSync` is what makes it answer about the tree rather
|
|
771
|
+
* than about two spellings of it; for a path that does not exist yet the nearest existing ancestor
|
|
772
|
+
* is resolved instead, since that is what a write would actually follow. Copilot round 2.
|
|
773
|
+
*/
|
|
774
|
+
const contained = (rel) => {
|
|
775
|
+
// The worktree side is resolved tolerantly too. `realpathSync` on a directory that is not there
|
|
776
|
+
// throws a bare ENOENT, and an unguarded throw here would arrive as *could not finish the sweep*
|
|
777
|
+
// with a stack trace instead of a sentence — the first version did exactly that, and its own test
|
|
778
|
+
// caught it.
|
|
779
|
+
let root;
|
|
780
|
+
try {
|
|
781
|
+
root = fs.realpathSync(worktree);
|
|
782
|
+
} catch (cause) {
|
|
783
|
+
throw new CouldNotRun(
|
|
784
|
+
`the drill worktree ${worktree} cannot be resolved — ${cause.code ?? cause.message}. ` +
|
|
785
|
+
"Nothing is perturbed against a tree that is not there",
|
|
786
|
+
);
|
|
787
|
+
}
|
|
788
|
+
const target = path.resolve(worktree, rel);
|
|
789
|
+
|
|
790
|
+
// **A symlink AT THE TARGET is refused outright, and this is the sibling the first containment
|
|
791
|
+
// fix left open.** `writeFileSync` follows a link, and the resolve loop below skips a *broken*
|
|
792
|
+
// leaf up to its parent — so a target that is itself a dangling symlink pointing outside passed
|
|
793
|
+
// both checks and the write landed outside the worktree. Measured at the final checkpoint, which
|
|
794
|
+
// produced a file called `outside/ghost.txt` holding `PWNED`.
|
|
795
|
+
//
|
|
796
|
+
// `lstat`, not `stat`, because the question is whether *this name* is a link rather than what it
|
|
797
|
+
// points at — and it catches a live link and a dangling one with the same test. A drill has no
|
|
798
|
+
// business perturbing through a link in either case: the tree it is given is a fresh `git
|
|
799
|
+
// worktree` of this repository, which commits none.
|
|
800
|
+
//
|
|
801
|
+
// The enumerated vectors — `..`, an absolute path, a symlinked *parent* — were all closed by the
|
|
802
|
+
// first fix, and this was not one of them. That is `0020` at its most literal: the repair held
|
|
803
|
+
// for the spellings its own docblock listed.
|
|
804
|
+
let leaf = null;
|
|
805
|
+
try {
|
|
806
|
+
leaf = fs.lstatSync(target);
|
|
807
|
+
} catch {
|
|
808
|
+
/* Not there at all, which is the ordinary case for a `create`. */
|
|
809
|
+
}
|
|
810
|
+
if (leaf?.isSymbolicLink()) {
|
|
811
|
+
throw new CouldNotRun(
|
|
812
|
+
`drill \`${drill.rail}\` names ${JSON.stringify(rel)}, which is a symlink. A write follows it, so it can ` +
|
|
813
|
+
"leave the drill worktree even when the link is dangling — refused whatever it points at",
|
|
814
|
+
);
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
let probe = target;
|
|
818
|
+
for (;;) {
|
|
819
|
+
try {
|
|
820
|
+
probe = fs.realpathSync(probe);
|
|
821
|
+
break;
|
|
822
|
+
} catch {
|
|
823
|
+
const parent = path.dirname(probe);
|
|
824
|
+
if (parent === probe) break;
|
|
825
|
+
probe = parent;
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
// The resolved ancestor must be inside, and so must the unresolved remainder — the first stops a
|
|
829
|
+
// symlink hop, the second stops a `..` that never touches an existing directory.
|
|
830
|
+
if (!isInside(root, probe) || !isInside(path.resolve(worktree), target)) {
|
|
831
|
+
throw new CouldNotRun(
|
|
832
|
+
`drill \`${drill.rail}\` names ${JSON.stringify(rel)}, which resolves outside the drill worktree. ` +
|
|
833
|
+
"A perturbation may only touch the throwaway tree — the isolation is the reason a drill is safe to run at all",
|
|
834
|
+
);
|
|
835
|
+
}
|
|
836
|
+
return target;
|
|
837
|
+
};
|
|
838
|
+
|
|
839
|
+
if (drill.perturb.create !== undefined) {
|
|
840
|
+
const target = contained(drill.perturb.create);
|
|
841
|
+
if (fs.existsSync(target)) {
|
|
842
|
+
throw new CouldNotRun(
|
|
843
|
+
`drill \`${drill.rail}\` would create ${drill.perturb.create}, which already exists in the tree — ` +
|
|
844
|
+
`a creation that overwrites is an edit wearing the wrong shape. Re-declare it as {file, find, replace} or pick another path`,
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
848
|
+
fs.writeFileSync(target, drill.perturb.content);
|
|
849
|
+
return { path: drill.perturb.create, before: null, after: digest(Buffer.from(drill.perturb.content)) };
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
const rel = drill.perturb.file;
|
|
853
|
+
const target = contained(rel);
|
|
854
|
+
let source;
|
|
855
|
+
try {
|
|
856
|
+
source = fs.readFileSync(target, "utf8");
|
|
857
|
+
} catch (cause) {
|
|
858
|
+
throw new CouldNotRun(
|
|
859
|
+
`drill \`${drill.rail}\` names ${rel}, which cannot be read in the drilled tree — ${cause.code ?? cause.message}. ` +
|
|
860
|
+
`The file moved and this drill did not: re-anchor it or retire it, but do not let it be skipped`,
|
|
861
|
+
);
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
let count = 0;
|
|
865
|
+
let at = 0;
|
|
866
|
+
for (;;) {
|
|
867
|
+
const i = source.indexOf(drill.perturb.find, at);
|
|
868
|
+
if (i === -1) break;
|
|
869
|
+
count += 1;
|
|
870
|
+
at = i + 1;
|
|
871
|
+
}
|
|
872
|
+
if (count === 0) {
|
|
873
|
+
throw new CouldNotRun(
|
|
874
|
+
`drill \`${drill.rail}\` does not place: its anchor is absent from ${rel}. The subject moved and this drill ` +
|
|
875
|
+
`did not — re-anchor it or retire it, but do not let it be skipped`,
|
|
876
|
+
);
|
|
877
|
+
}
|
|
878
|
+
if (count > 1) {
|
|
879
|
+
throw new CouldNotRun(
|
|
880
|
+
`drill \`${drill.rail}\` places ${count} times in ${rel} and a perturbation must be exactly one edit. ` +
|
|
881
|
+
`Lengthen its anchor until it is unique`,
|
|
882
|
+
);
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
const before = digest(Buffer.from(source, "utf8"));
|
|
886
|
+
const next = source.replace(drill.perturb.find, () => drill.perturb.replace);
|
|
887
|
+
fs.writeFileSync(target, next);
|
|
888
|
+
const after = digest(fs.readFileSync(target));
|
|
889
|
+
if (before === after) {
|
|
890
|
+
throw new CouldNotRun(
|
|
891
|
+
`drill \`${drill.rail}\` placed in ${rel} and the bytes did not move — the replacement is what it replaced. ` +
|
|
892
|
+
`A drill that does not fire reports on nothing`,
|
|
893
|
+
);
|
|
894
|
+
}
|
|
895
|
+
return { path: rel, before, after };
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* The yielded recipe set, at a PINNED root, with host discovery refused.
|
|
900
|
+
*
|
|
901
|
+
* The pin is the same argument `./goldens.sh` and `./compile.sh` state at length: a rail answers *does
|
|
902
|
+
* this tree hold its own claims*, so its answer may not move with what happens to be installed on the
|
|
903
|
+
* machine running it. `discovery: null` and `forced: false` are the second carrier of that, so a stray
|
|
904
|
+
* `--pack-root auto` on the command line cannot reach the host either.
|
|
905
|
+
*/
|
|
906
|
+
export function yieldedRecipes({ workspaceDir, repoRoot, packRoots }) {
|
|
907
|
+
let manifest;
|
|
908
|
+
try {
|
|
909
|
+
manifest = JSON.parse(fs.readFileSync(path.join(workspaceDir, "workspace.json"), "utf8"));
|
|
910
|
+
} catch (cause) {
|
|
911
|
+
throw new CouldNotRun(`the workspace manifest at ${workspaceDir} could not be read — ${cause.message}`);
|
|
912
|
+
}
|
|
913
|
+
let resolve;
|
|
914
|
+
try {
|
|
915
|
+
resolve = resolverFor({ workspaceDir, manifest, repoRoot, named: packRoots, discovery: null, forced: false });
|
|
916
|
+
} catch (cause) {
|
|
917
|
+
throw new CouldNotRun(cause.message);
|
|
918
|
+
}
|
|
919
|
+
const set = recipeSet(manifest, { resolve });
|
|
920
|
+
if (!set.ok) throw new CouldNotRun(set.reason);
|
|
921
|
+
return set.recipes;
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
/**
|
|
925
|
+
* Validate the whole table against a tree. Returns findings; throws `CouldNotRun` for a drill whose
|
|
926
|
+
* anchor no longer places, because that is `./mutants.mjs`'s ruling on the same condition: an anchor
|
|
927
|
+
* that has drifted must refuse rather than be skipped.
|
|
928
|
+
*
|
|
929
|
+
* Runs in **every** mode, `--only` included.
|
|
930
|
+
*/
|
|
931
|
+
export function check({ recipes, repoRoot, drills = DRILLS }) {
|
|
932
|
+
const findings = [];
|
|
933
|
+
const recipeIds = new Set(recipes.map((r) => r.id));
|
|
934
|
+
const nonRecipeIds = new Set(NON_RECIPE_RAILS.map((r) => r.id));
|
|
935
|
+
const drilled = new Set();
|
|
936
|
+
|
|
937
|
+
// **A yielded recipe sharing an id with a non-recipe rail is refused**, because the sweep's lookup
|
|
938
|
+
// map spreads the hooks after the recipes: a workspace recipe called `gate` would be silently
|
|
939
|
+
// shadowed by the hook, `check` would count that id as drilled, and the real recipe would never run
|
|
940
|
+
// while the sweep reported green. Nothing in `spec/workspace.schema.json` reserves these two slugs,
|
|
941
|
+
// so the refusal belongs here. Copilot round 1.
|
|
942
|
+
for (const rail of NON_RECIPE_RAILS) {
|
|
943
|
+
if (recipeIds.has(rail.id)) {
|
|
944
|
+
findings.push({
|
|
945
|
+
where: `rail \`${rail.id}\``,
|
|
946
|
+
what:
|
|
947
|
+
"is both a recipe this workspace yields and a declared non-recipe rail. One id cannot be two rails: " +
|
|
948
|
+
"the hook would shadow the recipe, and the recipe would go undrilled behind a green",
|
|
949
|
+
});
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
// `drills` is a seam for the suite and never a selection: the CLI never passes it, so the whole
|
|
954
|
+
// declared table is always what gets validated. It exists because a guard nothing can exercise
|
|
955
|
+
// positively is a guard nobody has seen work — this module's own subject, one altitude up.
|
|
956
|
+
for (const drill of drills) {
|
|
957
|
+
const where = `drill \`${drill.rail}\``;
|
|
958
|
+
if (!recipeIds.has(drill.rail) && !nonRecipeIds.has(drill.rail)) {
|
|
959
|
+
findings.push({
|
|
960
|
+
where,
|
|
961
|
+
what:
|
|
962
|
+
`names a rail that is neither a recipe the workspace yields nor a declared non-recipe rail. ` +
|
|
963
|
+
`A misspelled recipe id looks exactly like this, and it would leave the real recipe undrilled`,
|
|
964
|
+
});
|
|
965
|
+
continue;
|
|
966
|
+
}
|
|
967
|
+
drilled.add(drill.rail);
|
|
968
|
+
|
|
969
|
+
// **A hook drill's session id must be the prefix this harness owns.** The completion in
|
|
970
|
+
// `runRail` and the retirement in `drillOne` both key on it, so a drill inventing its own id
|
|
971
|
+
// would get neither: a counter shared between runs and a file nobody cleans. That is a rail
|
|
972
|
+
// with a precondition nobody states, refused here rather than left as a comment. Raised at the
|
|
973
|
+
// pre-commit checkpoint's second pass, as a trap the fold had left open.
|
|
974
|
+
if (nonRecipeIds.has(drill.rail) && drill.stdin?.session_id !== undefined && drill.stdin.session_id !== DRILL_SESSION_PREFIX) {
|
|
975
|
+
findings.push({
|
|
976
|
+
where,
|
|
977
|
+
what:
|
|
978
|
+
`declares its own \`session_id\` (${JSON.stringify(drill.stdin.session_id)}). A hook drill's id must be ` +
|
|
979
|
+
`\`${DRILL_SESSION_PREFIX}\`, which \`runRail\` completes per run and \`drillOne\` retires — an id outside it ` +
|
|
980
|
+
`is shared between runs and its counter file is never cleaned`,
|
|
981
|
+
});
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
if (typeof drill.tell !== "string" || drill.tell.length === 0) {
|
|
985
|
+
findings.push({ where, what: "declares no `tell`, so a red for the wrong reason would count as a fire" });
|
|
986
|
+
}
|
|
987
|
+
if (typeof drill.why !== "string" || drill.why.length === 0) {
|
|
988
|
+
findings.push({ where, what: "declares no `why`. A drill with no stated claim is a perturbation nobody can review" });
|
|
989
|
+
}
|
|
990
|
+
// **0 or 1, never 2.** Exit 2 is reserved throughout this repository for *could not run*, so a
|
|
991
|
+
// drill declaring it would count a rail that could not be judged as a rail that fired — the exact
|
|
992
|
+
// inversion `../.portulan/memory/verify-preconditions-fail-closed.md` exists to prevent, inside
|
|
993
|
+
// the harness built to detect it. A rail whose only non-green arm IS a refusal needs an argument
|
|
994
|
+
// and a different mechanism, not a declaration that quietly reads a refusal as a fire; the
|
|
995
|
+
// `workflow-filters` drill was drafted that way and was rewritten to force the exit-1 arm
|
|
996
|
+
// instead. Copilot round 1.
|
|
997
|
+
if (drill.exit !== 0 && drill.exit !== 1) {
|
|
998
|
+
findings.push({
|
|
999
|
+
where,
|
|
1000
|
+
what:
|
|
1001
|
+
`declares \`exit: ${JSON.stringify(drill.exit)}\`. A fire is 0 (a hook, which answers in its stdout) or ` +
|
|
1002
|
+
"1 (a red); 2 is could-not-run everywhere here, and counting it as a fire would read a refusal as a verdict",
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
// A drill whose control and fire are the same run proves nothing at all. One of the two must
|
|
1006
|
+
// differ: the tree, or the input.
|
|
1007
|
+
//
|
|
1008
|
+
// **The input half is only available to a non-recipe rail, and the test for it was three kinds of
|
|
1009
|
+
// weak.** `stdinControl !== undefined` did not ask whether it *differs*; `stdinControl: null`
|
|
1010
|
+
// falls back to `stdin` through `drillOne`'s `??`, so it is the same run wearing a different
|
|
1011
|
+
// declaration; and `runRail` passes stdin to `rail.argv` only — a **recipe** rail ignores it
|
|
1012
|
+
// entirely, so a recipe drill with no perturbation could satisfy this and never change a thing.
|
|
1013
|
+
// Raised as a suppressed note by Copilot, round 1.
|
|
1014
|
+
const inputCanDiffer =
|
|
1015
|
+
nonRecipeIds.has(drill.rail) &&
|
|
1016
|
+
drill.stdinControl !== undefined &&
|
|
1017
|
+
drill.stdinControl !== null &&
|
|
1018
|
+
JSON.stringify(drill.stdinControl) !== JSON.stringify(drill.stdin);
|
|
1019
|
+
if (!drill.perturb && !inputCanDiffer) {
|
|
1020
|
+
findings.push({
|
|
1021
|
+
where,
|
|
1022
|
+
what:
|
|
1023
|
+
nonRecipeIds.has(drill.rail)
|
|
1024
|
+
? "has no perturbation and no control input that differs from its fire input, so its control and its fire are the same run"
|
|
1025
|
+
: "has no perturbation, and a recipe rail is handed no stdin at all — so its control and its fire are the same run",
|
|
1026
|
+
});
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
// The anchor is checked against the WORKING TREE here, which is the whole value of `--check`
|
|
1030
|
+
// running on a pull request: the commit that moves an anchored line is the one that learns it.
|
|
1031
|
+
if (drill.perturb?.file !== undefined) {
|
|
1032
|
+
const target = path.join(repoRoot, drill.perturb.file);
|
|
1033
|
+
let source;
|
|
1034
|
+
try {
|
|
1035
|
+
source = fs.readFileSync(target, "utf8");
|
|
1036
|
+
} catch (cause) {
|
|
1037
|
+
throw new CouldNotRun(
|
|
1038
|
+
`${where} names ${drill.perturb.file}, which cannot be read — ${cause.code ?? cause.message}`,
|
|
1039
|
+
);
|
|
1040
|
+
}
|
|
1041
|
+
let count = 0;
|
|
1042
|
+
let at = 0;
|
|
1043
|
+
for (;;) {
|
|
1044
|
+
const i = source.indexOf(drill.perturb.find, at);
|
|
1045
|
+
if (i === -1) break;
|
|
1046
|
+
count += 1;
|
|
1047
|
+
at = i + 1;
|
|
1048
|
+
}
|
|
1049
|
+
if (count !== 1) {
|
|
1050
|
+
throw new CouldNotRun(
|
|
1051
|
+
`${where} places ${count} time(s) in ${drill.perturb.file} and a perturbation must be exactly one edit. ` +
|
|
1052
|
+
(count === 0
|
|
1053
|
+
? "The subject moved and this drill did not — re-anchor it or retire it, but do not let it be skipped"
|
|
1054
|
+
: "Lengthen its anchor until it is unique"),
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
if (drill.perturb.find === drill.perturb.replace) {
|
|
1058
|
+
findings.push({ where, what: "replaces its anchor with itself, so the perturbation cannot move a byte" });
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
if (drill.perturb?.create !== undefined && fs.existsSync(path.join(repoRoot, drill.perturb.create))) {
|
|
1062
|
+
findings.push({
|
|
1063
|
+
where,
|
|
1064
|
+
what: `would create ${drill.perturb.create}, which is already in the tree — a creation that overwrites is an edit wearing the wrong shape`,
|
|
1065
|
+
});
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
for (const recipe of recipes) {
|
|
1070
|
+
if (!drilled.has(recipe.id)) {
|
|
1071
|
+
findings.push({
|
|
1072
|
+
where: `rail \`${recipe.id}\``,
|
|
1073
|
+
what: "is yielded by this workspace and has no drill. Nothing here has ever watched it fire",
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
for (const rail of NON_RECIPE_RAILS) {
|
|
1078
|
+
if (!drilled.has(rail.id)) {
|
|
1079
|
+
findings.push({ where: `rail \`${rail.id}\``, what: "is a declared non-recipe rail and has no drill" });
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
return findings;
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
/** `git`, run for its stdout, with a could-not-run on failure rather than a throw nobody can read. */
|
|
1086
|
+
function git(args, { cwd }) {
|
|
1087
|
+
const result = spawnSync("git", args, { cwd, encoding: "utf8" });
|
|
1088
|
+
if (result.error) throw new CouldNotRun(`git ${args[0]} could not run — ${result.error.message}`);
|
|
1089
|
+
if (result.status !== 0) {
|
|
1090
|
+
throw new CouldNotRun(`git ${args.join(" ")} exited ${result.status} — ${(result.stderr || "").trim()}`);
|
|
1091
|
+
}
|
|
1092
|
+
return result.stdout;
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
/**
|
|
1096
|
+
* Which commit the sweep drills, and the refusals that keep the answer honest.
|
|
1097
|
+
*
|
|
1098
|
+
* A worktree is a commit, so this tool reports on a commit. Two modes:
|
|
1099
|
+
* default `HEAD`, and a dirty tree is refused rather than silently drilled around
|
|
1100
|
+
* --working-copy a commit synthesized from the working copy by `git stash create`
|
|
1101
|
+
*
|
|
1102
|
+
* `stash create` carries staged additions and unstaged edits to tracked files and does **not** carry
|
|
1103
|
+
* untracked-and-unstaged files — measured at the session-open checkpoint. So that mode refuses while any
|
|
1104
|
+
* exist, naming them: drilling a tree that is missing the file under review is the wrong-tree green this
|
|
1105
|
+
* whole design is arranged against.
|
|
1106
|
+
*/
|
|
1107
|
+
export function treeToDrill({ repoRoot, workingCopy }) {
|
|
1108
|
+
const head = git(["rev-parse", "HEAD"], { cwd: repoRoot }).trim();
|
|
1109
|
+
const dirty = git(["status", "--porcelain"], { cwd: repoRoot }).trim();
|
|
1110
|
+
|
|
1111
|
+
if (!workingCopy) {
|
|
1112
|
+
if (dirty) {
|
|
1113
|
+
throw new CouldNotRun(
|
|
1114
|
+
`the working tree is not clean, and the sweep drills a COMMIT. Reporting on ${head.slice(0, 7)} while you are ` +
|
|
1115
|
+
`looking at uncommitted work would be a green about a different tree. Pass --working-copy to drill the ` +
|
|
1116
|
+
`working copy through a synthesized commit, or commit first`,
|
|
1117
|
+
);
|
|
1118
|
+
}
|
|
1119
|
+
return { sha: head, kind: "HEAD" };
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
if (!dirty) return { sha: head, kind: "HEAD (the working tree is clean, so --working-copy has nothing to add)" };
|
|
1123
|
+
|
|
1124
|
+
const untracked = git(["ls-files", "--others", "--exclude-standard"], { cwd: repoRoot }).trim();
|
|
1125
|
+
if (untracked) {
|
|
1126
|
+
throw new CouldNotRun(
|
|
1127
|
+
"`git stash create` does not carry untracked files, so a synthesized commit would be missing these and the " +
|
|
1128
|
+
`drill would report on a tree that is not the one under review — stage them (\`git add\`) or remove them:\n` +
|
|
1129
|
+
untracked
|
|
1130
|
+
.split("\n")
|
|
1131
|
+
.map((f) => ` ${f}`)
|
|
1132
|
+
.join("\n"),
|
|
1133
|
+
);
|
|
1134
|
+
}
|
|
1135
|
+
const synthesized = git(["stash", "create"], { cwd: repoRoot }).trim();
|
|
1136
|
+
if (!synthesized) {
|
|
1137
|
+
throw new CouldNotRun(
|
|
1138
|
+
"`git stash create` produced no commit over a tree `git status` calls dirty — refusing to guess which tree to drill",
|
|
1139
|
+
);
|
|
1140
|
+
}
|
|
1141
|
+
return { sha: synthesized, kind: "a commit synthesized from the working copy" };
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
/** Run one rail in one tree. Returns `{status, stdout, stderr}`. */
|
|
1145
|
+
function runRail({ rail, worktree, stdin, workspaceRel }) {
|
|
1146
|
+
if (rail.argv) {
|
|
1147
|
+
// The id is completed here rather than in the declaration, so it is distinct per worktree and
|
|
1148
|
+
// still carries the prefix this harness cleans up under.
|
|
1149
|
+
const payload =
|
|
1150
|
+
stdin?.session_id === DRILL_SESSION_PREFIX
|
|
1151
|
+
? { ...stdin, session_id: `${DRILL_SESSION_PREFIX}-${path.basename(worktree)}` }
|
|
1152
|
+
: stdin;
|
|
1153
|
+
const result = spawnSync(process.execPath, rail.argv, {
|
|
1154
|
+
cwd: worktree,
|
|
1155
|
+
encoding: "utf8",
|
|
1156
|
+
// **The enforced fields come LAST**, and the order is a finding rather than a style: spread
|
|
1157
|
+
// first, a drill's own `cwd` silently overrode it, and `./stop-gate.mjs` resolves the session
|
|
1158
|
+
// tree from that field — so drill data could have pointed a control or a fire at another
|
|
1159
|
+
// repository entirely while the transcript said the worktree. The harness owns the execution
|
|
1160
|
+
// tree, never the declaration. Copilot round 1.
|
|
1161
|
+
input: `${JSON.stringify({ ...payload, cwd: worktree })}\n`,
|
|
1162
|
+
timeout: RAIL_TIMEOUT_MS,
|
|
1163
|
+
// The hooks are TOLD their project root rather than deriving it, so a drill that did not set
|
|
1164
|
+
// this would grade the repository this session is in and not the throwaway worktree.
|
|
1165
|
+
//
|
|
1166
|
+
// **And `PORTULAN_WORKSPACE` is set rather than inherited.** Both hooks read
|
|
1167
|
+
// `process.env.PORTULAN_WORKSPACE || ".portulan"`, so with `--workspace` naming anything else
|
|
1168
|
+
// — or with that variable merely present in the ambient environment — the sweep enumerated one
|
|
1169
|
+
// workspace's recipes while the hooks read another workspace's policy. A rail graded against a
|
|
1170
|
+
// policy the run did not choose is a verdict about the machine. `run` refuses a workspace that
|
|
1171
|
+
// is not inside the repository, so this relative spelling always resolves in the worktree.
|
|
1172
|
+
// Copilot round 1.
|
|
1173
|
+
env: { ...process.env, CLAUDE_PROJECT_DIR: worktree, PORTULAN_WORKSPACE: workspaceRel },
|
|
1174
|
+
});
|
|
1175
|
+
if (result.error) throw new CouldNotRun(`rail \`${rail.id}\` could not run — ${result.error.message}`);
|
|
1176
|
+
return { status: result.status, stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
|
|
1177
|
+
}
|
|
1178
|
+
// A recipe's `run` is a shell string in the manifest — `bash -c` is how CI runs it and how the
|
|
1179
|
+
// Stop-gate runs it, so it is how a drill must run it. Nothing from a drill's declarations reaches
|
|
1180
|
+
// this command line: the string comes from the yielded set and the perturbation is applied to a file.
|
|
1181
|
+
const result = spawnSync("bash", ["-c", rail.run], {
|
|
1182
|
+
cwd: worktree,
|
|
1183
|
+
encoding: "utf8",
|
|
1184
|
+
timeout: RAIL_TIMEOUT_MS,
|
|
1185
|
+
});
|
|
1186
|
+
if (result.error) throw new CouldNotRun(`rail \`${rail.id}\` could not run — ${result.error.message}`);
|
|
1187
|
+
return { status: result.status, stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
/** The text a tell is looked for in — see `NON_RECIPE_RAILS` for why one rail reads stdout alone. */
|
|
1191
|
+
const tellText = (rail, out) => (rail.tellStream === "stdout" ? out.stdout : `${out.stdout}${out.stderr}`);
|
|
1192
|
+
|
|
1193
|
+
/**
|
|
1194
|
+
* The part of a rail's output a reader needs, for a message that has to be legible in a CI log.
|
|
1195
|
+
*
|
|
1196
|
+
* **The finding-shaped lines first, and that is a defect this tool shipped once.** The first version
|
|
1197
|
+
* printed the last twelve lines, and this repository's recipes end with a long list of `ok` lines — so
|
|
1198
|
+
* the one `FAIL` that explained everything was scrolled off the top of the very diagnostic written to
|
|
1199
|
+
* explain it. A reader then has to re-run the rail by hand to find out what it said.
|
|
1200
|
+
*/
|
|
1201
|
+
const salient = (text, lines = 12) => {
|
|
1202
|
+
const all = text.split("\n").filter((l) => l.trim() !== "");
|
|
1203
|
+
const findings = all.filter((l) => /^\s*(FAIL|RED|✗|✖|not ok|UNPINNED)\b/.test(l) || /^RED —/.test(l));
|
|
1204
|
+
const chosen = findings.length ? [...findings.slice(0, lines), ...all.slice(-2)] : all.slice(-lines);
|
|
1205
|
+
return [...new Set(chosen)].map((l) => ` | ${l}`).join("\n");
|
|
1206
|
+
};
|
|
1207
|
+
|
|
1208
|
+
/**
|
|
1209
|
+
* Drill one rail: a pristine control, then the perturbation, then the fire.
|
|
1210
|
+
*
|
|
1211
|
+
* Returns a finding or `null`. Throws `CouldNotRun` when no verdict can be formed — a control that is
|
|
1212
|
+
* already red, a tell already present before the perturbation, an anchor that will not place.
|
|
1213
|
+
*/
|
|
1214
|
+
export function drillOne({ drill, rail, repoRoot, sha, say, workspaceRel = ".portulan" }) {
|
|
1215
|
+
const worktree = fs.mkdtempSync(path.join(os.tmpdir(), "portulan-drill-"));
|
|
1216
|
+
// `mkdtemp` creates the directory and `git worktree add` insists on creating it itself, so the
|
|
1217
|
+
// reservation is made and then handed back — which is still the right order: it is what guarantees
|
|
1218
|
+
// no two concurrent runs pick one path.
|
|
1219
|
+
fs.rmSync(worktree, { recursive: true, force: true });
|
|
1220
|
+
try {
|
|
1221
|
+
git(["worktree", "add", "--detach", worktree, sha], { cwd: repoRoot });
|
|
1222
|
+
|
|
1223
|
+
const control = runRail({ rail, worktree, stdin: drill.stdinControl ?? drill.stdin, workspaceRel });
|
|
1224
|
+
if (control.status !== 0) {
|
|
1225
|
+
throw new CouldNotRun(
|
|
1226
|
+
`rail \`${drill.rail}\` is not green on the drilled tree, so nothing it does next is attributable to the ` +
|
|
1227
|
+
`perturbation. Its control exited ${control.status}:\n${salient(`${control.stdout}${control.stderr}`)}`,
|
|
1228
|
+
);
|
|
1229
|
+
}
|
|
1230
|
+
if (tellText(rail, control).includes(drill.tell)) {
|
|
1231
|
+
throw new CouldNotRun(
|
|
1232
|
+
`rail \`${drill.rail}\` already says ${JSON.stringify(drill.tell)} before it is perturbed, so that tell ` +
|
|
1233
|
+
`cannot show the drill fired. Narrow it to something only the finding says`,
|
|
1234
|
+
);
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
const moved = perturb(worktree, drill);
|
|
1238
|
+
if (drill.stage) git(["add", "-A"], { cwd: worktree });
|
|
1239
|
+
|
|
1240
|
+
const fire = runRail({ rail, worktree, stdin: drill.stdin, workspaceRel });
|
|
1241
|
+
const text = tellText(rail, fire);
|
|
1242
|
+
// **A status that is not a verdict is could-not-run, never a rail that failed to fire.**
|
|
1243
|
+
// `spawnSync` reports `status: null` for a signal-killed child — a timeout, an OOM — and 2 is
|
|
1244
|
+
// could-not-run everywhere here. Both mean no verdict was formed, and reporting either as *this
|
|
1245
|
+
// rail did not fire* is a could-not-measure read as a measurement: session 1's round 3, in the
|
|
1246
|
+
// guard added to stop exactly that. Raised as a suppressed note by Copilot, round 1.
|
|
1247
|
+
if (fire.status === null) {
|
|
1248
|
+
throw new CouldNotRun(
|
|
1249
|
+
`rail \`${drill.rail}\` was killed by a signal rather than exiting, so no verdict was formed` +
|
|
1250
|
+
`${moved ? ` after ${moved.path} was perturbed` : ""}. Output:\n${salient(`${fire.stdout}${fire.stderr}`)}`,
|
|
1251
|
+
);
|
|
1252
|
+
}
|
|
1253
|
+
if (fire.status === 2 && drill.exit !== 2) {
|
|
1254
|
+
throw new CouldNotRun(
|
|
1255
|
+
`rail \`${drill.rail}\` exited 2 — could not run — so it formed no verdict about the perturbation` +
|
|
1256
|
+
`${moved ? ` in ${moved.path}` : ""}. Output:\n${salient(`${fire.stdout}${fire.stderr}`)}`,
|
|
1257
|
+
);
|
|
1258
|
+
}
|
|
1259
|
+
if (fire.status !== drill.exit) {
|
|
1260
|
+
return {
|
|
1261
|
+
where: `rail \`${drill.rail}\``,
|
|
1262
|
+
what:
|
|
1263
|
+
`did not fire as recorded: the drill expects exit ${drill.exit} and it exited ${fire.status}` +
|
|
1264
|
+
`${moved ? ` after ${moved.path} was perturbed` : ""}. Output:\n${salient(`${fire.stdout}${fire.stderr}`)}`,
|
|
1265
|
+
};
|
|
1266
|
+
}
|
|
1267
|
+
if (!text.includes(drill.tell)) {
|
|
1268
|
+
return {
|
|
1269
|
+
where: `rail \`${drill.rail}\``,
|
|
1270
|
+
what:
|
|
1271
|
+
`exited ${fire.status} as recorded and never said ${JSON.stringify(drill.tell)}, so it may have fired for ` +
|
|
1272
|
+
`another reason entirely. Output:\n${salient(text)}`,
|
|
1273
|
+
};
|
|
1274
|
+
}
|
|
1275
|
+
say(` fired ${drill.rail.padEnd(28)} exit ${fire.status} · said ${JSON.stringify(drill.tell)}`);
|
|
1276
|
+
return null;
|
|
1277
|
+
} finally {
|
|
1278
|
+
// **The counter files a hook rail leaves behind, retired.** `./stop-gate.mjs` writes one per
|
|
1279
|
+
// (session id, tree) into the OS temp directory, so a sweep that invented a session id and
|
|
1280
|
+
// walked away left one 45-byte file per run accumulating forever — the leak #340 names in a
|
|
1281
|
+
// sibling module, found here at the pre-commit checkpoint before it could become the same
|
|
1282
|
+
// issue. Scoped to this harness's own prefix, which is why the prefix is a constant: a wider
|
|
1283
|
+
// glob would retire a live session's counter and quietly disarm its cap.
|
|
1284
|
+
try {
|
|
1285
|
+
const dir = os.tmpdir();
|
|
1286
|
+
const mine = `portulan-stopgate-${DRILL_SESSION_PREFIX}-${path.basename(worktree)}`;
|
|
1287
|
+
for (const entry of fs.readdirSync(dir)) {
|
|
1288
|
+
if (entry.startsWith(mine)) fs.rmSync(path.join(dir, entry), { force: true });
|
|
1289
|
+
}
|
|
1290
|
+
} catch {
|
|
1291
|
+
/* A counter file this run cannot retire is litter in a temp directory, never a wrong verdict. */
|
|
1292
|
+
}
|
|
1293
|
+
// `--force` because the perturbation left the worktree dirty, which is the whole point of it.
|
|
1294
|
+
try {
|
|
1295
|
+
git(["worktree", "remove", "--force", worktree], { cwd: repoRoot });
|
|
1296
|
+
} catch {
|
|
1297
|
+
fs.rmSync(worktree, { recursive: true, force: true });
|
|
1298
|
+
try {
|
|
1299
|
+
git(["worktree", "prune"], { cwd: repoRoot });
|
|
1300
|
+
} catch {
|
|
1301
|
+
/* A worktree this run cannot retire is noise in `git worktree list`, never a wrong verdict. */
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
const USAGE = [
|
|
1308
|
+
"usage: node cli/drills.mjs [--check] [--only <rail>] [--working-copy] [--repo-root <dir>] [--workspace <dir>] [--pack-root <dir>]",
|
|
1309
|
+
"",
|
|
1310
|
+
" Forces every rail red and requires it to fire. Milestone 8, clause (d).",
|
|
1311
|
+
"",
|
|
1312
|
+
" --check the correspondence pass only: every yielded rail has a drill, every drill names a",
|
|
1313
|
+
" declared rail, every anchor still places exactly once. Runs no rail. This is what",
|
|
1314
|
+
" the `drills` verify recipe runs, because the sweep reports on a COMMIT and a recipe",
|
|
1315
|
+
" must answer about the working copy.",
|
|
1316
|
+
" --only <rail> drill one rail. Narrows what RUNS, never what must be well formed.",
|
|
1317
|
+
" --working-copy drill the working copy through a commit synthesized with `git stash create`,",
|
|
1318
|
+
" instead of refusing a dirty tree. The synthesized sha is printed.",
|
|
1319
|
+
"",
|
|
1320
|
+
" Exit 0 every drill fired · 1 a rail did not fire, or a rail has no drill · 2 could not run.",
|
|
1321
|
+
].join("\n");
|
|
1322
|
+
|
|
1323
|
+
export async function run(argv = [], { stdout = process.stdout, stderr = process.stderr, cwd = process.cwd() } = {}) {
|
|
1324
|
+
const say = (line) => stdout.write(`${line}\n`);
|
|
1325
|
+
try {
|
|
1326
|
+
let checkOnly = false;
|
|
1327
|
+
let workingCopy = false;
|
|
1328
|
+
let only = null;
|
|
1329
|
+
let repoRoot = cwd;
|
|
1330
|
+
let workspaceDir = null;
|
|
1331
|
+
const packRoots = [];
|
|
1332
|
+
|
|
1333
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
1334
|
+
const value = argv[i + 1];
|
|
1335
|
+
const needs = (flag) => {
|
|
1336
|
+
if (value === undefined || value.startsWith("-") || value.trim() === "") {
|
|
1337
|
+
throw new CouldNotRun(`${flag} needs a value`);
|
|
1338
|
+
}
|
|
1339
|
+
i += 1;
|
|
1340
|
+
return value;
|
|
1341
|
+
};
|
|
1342
|
+
if (argv[i] === "--check") checkOnly = true;
|
|
1343
|
+
else if (argv[i] === "--working-copy") workingCopy = true;
|
|
1344
|
+
else if (argv[i] === "--help" || argv[i] === "-h") {
|
|
1345
|
+
say(USAGE);
|
|
1346
|
+
return 0;
|
|
1347
|
+
} else if (argv[i] === "--only") only = needs("--only");
|
|
1348
|
+
else if (argv[i] === "--repo-root") repoRoot = path.resolve(needs("--repo-root"));
|
|
1349
|
+
else if (argv[i] === "--workspace") workspaceDir = path.resolve(needs("--workspace"));
|
|
1350
|
+
else if (argv[i] === "--pack-root") {
|
|
1351
|
+
const root = needs("--pack-root");
|
|
1352
|
+
let stat = null;
|
|
1353
|
+
try {
|
|
1354
|
+
stat = fs.statSync(root);
|
|
1355
|
+
} catch (cause) {
|
|
1356
|
+
throw new CouldNotRun(`--pack-root ${JSON.stringify(root)} cannot be read — ${cause.code ?? cause.message}`);
|
|
1357
|
+
}
|
|
1358
|
+
if (!stat.isDirectory()) throw new CouldNotRun(`--pack-root ${JSON.stringify(root)} is not a directory`);
|
|
1359
|
+
packRoots.push(path.resolve(root));
|
|
1360
|
+
} else throw new CouldNotRun(`unknown argument ${JSON.stringify(argv[i])}`);
|
|
1361
|
+
}
|
|
1362
|
+
workspaceDir ??= path.join(repoRoot, ".portulan");
|
|
1363
|
+
|
|
1364
|
+
const recipes = yieldedRecipes({ workspaceDir, repoRoot, packRoots });
|
|
1365
|
+
|
|
1366
|
+
// **The whole table, in every mode.** `--only` narrows what runs and not what must be well
|
|
1367
|
+
// formed — session 1's round 1 on `./mutants.mjs`, where `--only` skipped validation in exactly
|
|
1368
|
+
// the mode a person reaches for when something is already wrong.
|
|
1369
|
+
const findings = check({ recipes, repoRoot });
|
|
1370
|
+
|
|
1371
|
+
if (checkOnly) {
|
|
1372
|
+
say(
|
|
1373
|
+
`drills: ${DRILLS.length} drill(s) over ${recipes.length} yielded recipe(s) plus ` +
|
|
1374
|
+
`${NON_RECIPE_RAILS.length} declared non-recipe rail(s); anchors checked against the working tree`,
|
|
1375
|
+
);
|
|
1376
|
+
for (const excluded of NOT_DRILLED) say(` not drilled ${excluded.rail}`);
|
|
1377
|
+
if (findings.length) {
|
|
1378
|
+
for (const f of findings) stderr.write(`drills: ${f.where}\n ${f.what}\n`);
|
|
1379
|
+
stderr.write(`RED — ${findings.length} finding(s) in the drill roster\n`);
|
|
1380
|
+
return 1;
|
|
1381
|
+
}
|
|
1382
|
+
say("GREEN — every yielded rail has a drill and every drill's anchor places exactly once");
|
|
1383
|
+
say("drills: --check runs no rail. Whether each one still FIRES is the sweep's answer, on the calendar");
|
|
1384
|
+
return 0;
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
// **The argument is validated before the tree is chosen, and the order is a finding rather than
|
|
1388
|
+
// a preference.** With `treeToDrill` first, a typo'd `--only` on a dirty tree reported the dirty
|
|
1389
|
+
// tree — a refusal naming a cause it had not established, which sends the reader to look at
|
|
1390
|
+
// their working copy when what was wrong was what they typed. Caught by this module's own suite.
|
|
1391
|
+
const selected = only === null ? DRILLS : DRILLS.filter((d) => d.rail === only);
|
|
1392
|
+
if (only !== null && selected.length === 0) {
|
|
1393
|
+
throw new CouldNotRun(
|
|
1394
|
+
`--only ${JSON.stringify(only)} names no drill. The declared rails are: ` +
|
|
1395
|
+
`${DRILLS.map((d) => d.rail).join(", ")}`,
|
|
1396
|
+
);
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
// **The sweep runs every command from a throwaway worktree, so every path in one must resolve
|
|
1400
|
+
// there.** Two refusals, both measured rather than reasoned:
|
|
1401
|
+
//
|
|
1402
|
+
// * a **pack root outside the repository** makes `recipe-set` relativise `${PACK_ROOT}` against
|
|
1403
|
+
// the repo root, which produced `bash ../../../../../../../private/tmp/…/actions-pinned.sh`
|
|
1404
|
+
// — a path with `..` hops, executed from a different directory, landing somewhere nobody
|
|
1405
|
+
// chose. `--check` stays permissive because it runs no rail.
|
|
1406
|
+
// * a **workspace outside the repository** cannot be handed to the hooks, which read
|
|
1407
|
+
// `PORTULAN_WORKSPACE` as a path inside their project root.
|
|
1408
|
+
//
|
|
1409
|
+
// Both are could-not-run rather than a best effort: a rail run against files the sweep cannot
|
|
1410
|
+
// name is a verdict about neither tree. Copilot round 1.
|
|
1411
|
+
// **`isInside` is imported, not re-spelled.** The predicate this replaced was
|
|
1412
|
+
// `!path.relative(a, b).startsWith("..")` plus `rel !== ""` — the exact spelling
|
|
1413
|
+
// `./inside.mjs` exists to hold, and its docblock records that two copies of this rule drifted
|
|
1414
|
+
// into the identical defect before either shipped. Mine had both halves of it: a directory
|
|
1415
|
+
// legitimately named `..packs` reads as outside, and a root *equal* to the repository root was
|
|
1416
|
+
// rejected outright, so `--pack-root .` refused a tree that is trivially inside itself. A third
|
|
1417
|
+
// copy of a rule twice-corrected elsewhere is `0020` exactly. Copilot round 2.
|
|
1418
|
+
//
|
|
1419
|
+
// `fs.realpathSync` on both sides, for the reason `./compile.mjs` had to add it: a symlinked
|
|
1420
|
+
// checkout — `/var` against `/private/var` on macOS is the one this session already met — makes
|
|
1421
|
+
// two names for one directory compare as different trees.
|
|
1422
|
+
const real = (dir) => {
|
|
1423
|
+
try {
|
|
1424
|
+
return fs.realpathSync(dir);
|
|
1425
|
+
} catch {
|
|
1426
|
+
return path.resolve(dir);
|
|
1427
|
+
}
|
|
1428
|
+
};
|
|
1429
|
+
const repoReal = real(repoRoot);
|
|
1430
|
+
for (const root of packRoots) {
|
|
1431
|
+
if (!isInside(repoReal, real(root))) {
|
|
1432
|
+
throw new CouldNotRun(
|
|
1433
|
+
`--pack-root ${root} is outside ${repoRoot}, and the sweep runs each rail from a throwaway worktree. ` +
|
|
1434
|
+
"A composed recipe's `${PACK_ROOT}` is relativised against the repository, so from a worktree it " +
|
|
1435
|
+
"would point at neither copy. Pass a root inside the repository, or use --check, which runs no rail",
|
|
1436
|
+
);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
if (!isInside(repoReal, real(workspaceDir))) {
|
|
1440
|
+
throw new CouldNotRun(
|
|
1441
|
+
`--workspace ${workspaceDir} is outside ${repoRoot}, and the hook rails are handed their workspace as a ` +
|
|
1442
|
+
"path inside the tree being drilled. Pass a workspace inside the repository, or use --check",
|
|
1443
|
+
);
|
|
1444
|
+
}
|
|
1445
|
+
// **An empty relative path means *the repository root itself*, and the hooks read it as absent.**
|
|
1446
|
+
// `PORTULAN_WORKSPACE=""` falls through `process.env.PORTULAN_WORKSPACE || ".portulan"` in both
|
|
1447
|
+
// runners, so a workspace at the repo root would have silently drilled `.portulan` instead — a
|
|
1448
|
+
// rail graded against a policy the run did not choose, which is the defect the threading was
|
|
1449
|
+
// added to close, surviving in its own edge case. `.` is the spelling that says *here*. Copilot,
|
|
1450
|
+
// round 2's notes.
|
|
1451
|
+
const workspaceRel = path.relative(repoRoot, workspaceDir) || ".";
|
|
1452
|
+
|
|
1453
|
+
// **A malformed roster is reported here, not met as a throw inside the loop.** `check` had already
|
|
1454
|
+
// recorded a drill naming an undeclared rail as a finding, and the sweep then walked into the
|
|
1455
|
+
// guard below and turned that documented exit-1 roster failure into an exit-2 could-not-run,
|
|
1456
|
+
// abandoning the rest of the transcript. A roster failure is a finding in both modes. Raised as a
|
|
1457
|
+
// suppressed note by Copilot, round 1.
|
|
1458
|
+
if (findings.length) {
|
|
1459
|
+
for (const f of findings) stderr.write(`drills: ${f.where}\n ${f.what}\n`);
|
|
1460
|
+
stderr.write(`RED — ${findings.length} finding(s) in the drill roster; no rail was drilled\n`);
|
|
1461
|
+
return 1;
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
const tree = treeToDrill({ repoRoot, workingCopy });
|
|
1465
|
+
const byId = new Map([
|
|
1466
|
+
...recipes.map((r) => [r.id, { id: r.id, run: r.run }]),
|
|
1467
|
+
...NON_RECIPE_RAILS.map((r) => [r.id, r]),
|
|
1468
|
+
]);
|
|
1469
|
+
|
|
1470
|
+
say(`drills: forcing every rail red on ${tree.sha.slice(0, 7)} — ${tree.kind}`);
|
|
1471
|
+
say(`drills: ${selected.length} of ${DRILLS.length} drill(s), one throwaway git worktree each`);
|
|
1472
|
+
for (const excluded of NOT_DRILLED) say(` not drilled ${excluded.rail}`);
|
|
1473
|
+
|
|
1474
|
+
// **One rail that cannot be judged does not end the sweep**, and this is a defect the first run
|
|
1475
|
+
// of this tool shipped: a `CouldNotRun` thrown out of one drill aborted the loop and discarded
|
|
1476
|
+
// every finding already collected, so the run reported on one rail and said nothing about the
|
|
1477
|
+
// twenty behind it. That is the same argument `../.github/workflows/verify.yml` settles with
|
|
1478
|
+
// `set +e` — *a red recipe does not abort the loop* — and a sweep whose whole subject is rails
|
|
1479
|
+
// nobody has watched fire is the last place to stop looking at the first obstacle.
|
|
1480
|
+
//
|
|
1481
|
+
// The two outcomes stay distinguishable, per `../.portulan/memory/verify-preconditions-fail-closed.md`:
|
|
1482
|
+
// a **finding** is a rail that did not fire, and a **could-not-run** is a rail whose verdict
|
|
1483
|
+
// could not be formed. Both are printed; a single could-not-run makes the whole run exit 2,
|
|
1484
|
+
// because a set that was not fully judged has not been judged.
|
|
1485
|
+
const unjudged = [];
|
|
1486
|
+
for (const drill of selected) {
|
|
1487
|
+
const rail = byId.get(drill.rail);
|
|
1488
|
+
// `check` above has already refused a drill naming an undeclared rail, so this cannot be hit
|
|
1489
|
+
// through the CLI — it is the guard that keeps that true if the two ever drift.
|
|
1490
|
+
if (!rail) throw new CouldNotRun(`rail \`${drill.rail}\` is not in the yielded set nor declared`);
|
|
1491
|
+
try {
|
|
1492
|
+
const finding = drillOne({ drill, rail, repoRoot, sha: tree.sha, say, workspaceRel });
|
|
1493
|
+
if (finding) findings.push(finding);
|
|
1494
|
+
} catch (error) {
|
|
1495
|
+
if (!(error instanceof CouldNotRun)) throw error;
|
|
1496
|
+
unjudged.push({ where: `rail \`${drill.rail}\``, what: error.message });
|
|
1497
|
+
say(` UNJUDGED ${drill.rail}`);
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
for (const f of [...findings, ...unjudged]) stderr.write(`drills: ${f.where}\n ${f.what}\n`);
|
|
1502
|
+
if (unjudged.length) {
|
|
1503
|
+
stderr.write(
|
|
1504
|
+
`COULD NOT RUN — ${unjudged.length} rail(s) could not be judged` +
|
|
1505
|
+
`${findings.length ? ` and ${findings.length} did not fire` : ""}. A set that was not fully judged has not been judged\n`,
|
|
1506
|
+
);
|
|
1507
|
+
return 2;
|
|
1508
|
+
}
|
|
1509
|
+
if (findings.length) {
|
|
1510
|
+
stderr.write(`RED — ${findings.length} rail(s) did not fire as recorded\n`);
|
|
1511
|
+
return 1;
|
|
1512
|
+
}
|
|
1513
|
+
say(`GREEN — every drilled rail was forced red and fired, on ${tree.sha.slice(0, 7)}`);
|
|
1514
|
+
say("drills: a rail fires here on ONE known-bad input. That it fires is not that it catches everything");
|
|
1515
|
+
return 0;
|
|
1516
|
+
} catch (error) {
|
|
1517
|
+
if (error instanceof CouldNotRun) {
|
|
1518
|
+
stderr.write(`drills: ${error.message}\n`);
|
|
1519
|
+
return 2;
|
|
1520
|
+
}
|
|
1521
|
+
stderr.write(`drills: could not finish the sweep — ${error?.stack ?? error}\n`);
|
|
1522
|
+
return 2;
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
// The entry guard, in the ONE form `./rule-carriers.mjs` designates. `file://${argv[1]}` is NOT that
|
|
1527
|
+
// form: `import.meta.url` percent-encodes, this working copy lives under a path with spaces, and the
|
|
1528
|
+
// comparison fails — so the tool exits 0 having run nothing. Copied rather than re-derived, for the
|
|
1529
|
+
// reason `./goldens.mjs` states after meeting the false green a third time.
|
|
1530
|
+
function isMain() {
|
|
1531
|
+
const invoked = process.argv[1];
|
|
1532
|
+
if (!invoked) return false;
|
|
1533
|
+
if (import.meta.url === pathToFileURL(invoked).href) return true;
|
|
1534
|
+
try {
|
|
1535
|
+
return import.meta.url === pathToFileURL(fs.realpathSync(invoked)).href;
|
|
1536
|
+
} catch {
|
|
1537
|
+
return false;
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
if (isMain()) process.exitCode = await run(process.argv.slice(2));
|