@sleepy_panda_srl/portulan 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/cli/README.md +142 -291
- package/cli/advisory.mjs +354 -0
- package/cli/comments.mjs +932 -0
- package/cli/compile.mjs +1443 -1210
- package/cli/context.mjs +883 -0
- package/cli/control-chars.mjs +12 -301
- package/cli/discover.mjs +17 -574
- package/cli/doctor.mjs +225 -1071
- package/cli/drills.mjs +1107 -0
- package/cli/feedback.mjs +16 -237
- package/cli/finish.mjs +437 -0
- package/cli/form.mjs +644 -0
- package/cli/fuzz-shell.mjs +595 -0
- package/cli/gate.mjs +7 -198
- package/cli/goldens.mjs +311 -0
- package/cli/index.mjs +305 -655
- package/cli/init.mjs +244 -564
- package/cli/inside.mjs +3 -28
- package/cli/instructions.mjs +829 -0
- package/cli/ledger.mjs +778 -0
- package/cli/librarian.mjs +135 -599
- package/cli/manifest.mjs +2 -39
- package/cli/mutants.mjs +764 -0
- package/cli/new.mjs +12 -243
- package/cli/pack-identity.mjs +6 -68
- package/cli/pack-version.mjs +13 -248
- package/cli/plugin-lint.mjs +42 -658
- package/cli/portulan.mjs +5 -124
- package/cli/recipe-set.mjs +16 -240
- package/cli/release-eval.mjs +748 -0
- package/cli/review-meter.mjs +500 -0
- package/cli/rule-carriers.mjs +12 -175
- package/cli/sessions.mjs +82 -0
- package/cli/skill-goldens.mjs +424 -0
- package/cli/skills-set.mjs +12 -414
- package/cli/stop-gate.mjs +126 -454
- package/cli/symbols.mjs +1132 -0
- package/cli/telemetry.mjs +582 -0
- package/cli/upgrade.mjs +171 -314
- package/cli/vendor.mjs +146 -476
- package/cli/version-carriers.mjs +9 -74
- package/core/README.md +1 -1
- package/core/engine.md +3 -9
- package/core/operating/autonomy.md +1 -1
- package/core/operating/context.md +183 -0
- package/core/operating/evolution.md +3 -3
- package/core/operating/loop.md +13 -20
- package/core/operating/memory.md +12 -3
- package/core/operating/sessions.md +100 -0
- package/core/personas/librarian.md +2 -2
- package/core/skills/consolidate/SKILL.md +4 -2
- package/core/templates/gate-policy.md +27 -8
- package/core/templates/handoff.md +28 -15
- package/core/templates/memory-entry.md +1 -0
- package/evals/README.md +320 -0
- package/evals/releases/0.1.3.json +134 -0
- package/evals/releases/0.1.3.md +64 -0
- package/evals/releases/0.2.0.json +146 -0
- package/evals/releases/0.2.0.md +67 -0
- package/evals/releases/README.md +32 -0
- package/package.json +9 -2
- package/packs/rituals/checkpoints/README.md +9 -2
- package/packs/rituals/checkpoints/pack.json +1 -1
- package/packs/rituals/checkpoints/skills/pre-commit/SKILL.md +32 -2
- package/packs/tools/github/verify/actions-pinned.sh +10 -55
- package/spec/README.md +112 -6
- package/spec/migrations/0001-repository-declares-its-tree.mjs +1 -26
- package/spec/migrations/0002-bundle-fallback-path.mjs +13 -74
- package/spec/migrations/0003-handoff-index-not-kept.mjs +66 -0
- package/spec/migrations/0004-changelog-fragments.mjs +53 -0
- package/spec/migrations/0005-session-log-retired.mjs +69 -0
- package/spec/migrations/0006-boot-card.mjs +77 -0
- package/spec/migrations/0007-guidance-compiled.mjs +31 -0
- package/spec/migrations/0008-card-reading.mjs +55 -0
- package/spec/migrations/0009-instruction-sections-on-read.mjs +60 -0
- package/spec/migrations/0010-comment-rail.mjs +57 -0
- package/spec/migrations/README.md +45 -10
- package/spec/pack.schema.json +1 -1
- package/spec/slots.md +359 -23
- package/spec/workspace.schema.json +145 -4
package/cli/gate.mjs
CHANGED
|
@@ -1,184 +1,27 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// The PreToolUse gate runner — the *explanation* half of the enforcement compiler.
|
|
3
3
|
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// `.portulan/gates.json` plus the fragments its packs contribute — see "The policy this runner reads"
|
|
8
|
-
// below, which is the whole of what `#269` was.
|
|
9
|
-
//
|
|
10
|
-
// ## This file is not the gate. Read that twice.
|
|
11
|
-
//
|
|
12
|
-
// The gate is the `permissions` rule the same compiler emits. This runner exists because a bare
|
|
13
|
-
// "permission denied" teaches nobody anything, while `gates.json` already holds a sentence written
|
|
14
|
-
// for exactly this moment. The split is forced by a measurement, not by taste: on CLI 2.1.220 a
|
|
15
|
-
// PreToolUse hook that CRASHES **fails open** — the tool proceeds — on the identical wiring that
|
|
16
|
-
// blocks when the hook is healthy. So a hook can never be the load-bearing layer here, and this one
|
|
17
|
-
// is written to fail open *deliberately and visibly* rather than pretending otherwise:
|
|
18
|
-
//
|
|
19
|
-
// any internal error => exit 0, emit nothing => the permission rule governs, unchanged.
|
|
20
|
-
//
|
|
21
|
-
// Refusing loudly on error *is* available — a hook exiting 2 blocks — and it is still wrong here, for a
|
|
22
|
-
// reason worth stating precisely rather than with the hand-wave an earlier draft used. If this runner
|
|
23
|
-
// failed closed, a malformed `.portulan/gates.json` would deny EVERY matched tool call until someone fixed it:
|
|
24
|
-
// an undriveable session, whose only repair is inside the repository it can no longer edit. What is lost
|
|
25
|
-
// by stepping aside is bounded and known — the wrapper coverage and the sentences — because the
|
|
26
|
-
// permission layer never reads that file at runtime.
|
|
27
|
-
//
|
|
28
|
-
// ## Why the decision mirrors the permission rule
|
|
29
|
-
//
|
|
30
|
-
// `prohibited` => `deny`, `gated` => `ask`. The hook returns the SAME decision the permission rule
|
|
31
|
-
// carries. A hook returning `deny` for a Gated action would convert a per-action prompt into a hard
|
|
32
|
-
// block — that is the tier above it, and collapsing the two would make the constitution's
|
|
33
|
-
// protection indistinguishable from an ordinary push.
|
|
34
|
-
//
|
|
35
|
-
// ## What this layer is FOR, which is not what its author first assumed
|
|
36
|
-
//
|
|
37
|
-
// The first version of this runner existed to supply a better sentence than "permission denied".
|
|
38
|
-
// Measured, that was wrong: when a permission rule matches, **the host runs this hook and then
|
|
39
|
-
// discards its reason** — the permission layer's generic message is what the agent sees. Verified
|
|
40
|
-
// with a canary that recorded both the invocation and the command it saw. A hook emitted purely to
|
|
41
|
-
// improve a message it cannot reach would be an inert component that reads as an active one, which
|
|
42
|
-
// is the defect this repository already has a rule about.
|
|
43
|
-
//
|
|
44
|
-
// So the split earns its keep somewhere else: **this layer covers what the permission pattern
|
|
45
|
-
// cannot.** Two things, and in both the permission layer has nothing to say, so this decision AND
|
|
46
|
-
// its sentence are what the agent gets:
|
|
47
|
-
//
|
|
48
|
-
// 1. The WRAPPER spelling. `Bash(git push:*)` is a prefix match against the literal command, so
|
|
49
|
-
// `bash -c "git push …"` is invisible to it — measured, and the reason
|
|
50
|
-
// `../core/operating/autonomy.md` calls the platform floor the gate that holds when this
|
|
51
|
-
// layer fails. This runner unwraps one level of `sh -c` / `bash -c` / `zsh -c` before matching.
|
|
52
|
-
// 2. A SHELL WRITE to a path a `write:` rule protects. `Edit(./docs/vision.md)` denies three
|
|
53
|
-
// tools, and `echo x >> docs/vision.md` is a fourth way to the same bytes. `matchesRule` now
|
|
54
|
-
// answers for `Bash` on a write rule, by a table of redirections and file-writing commands
|
|
55
|
-
// that `./compile.mjs` states in full.
|
|
56
|
-
//
|
|
57
|
-
// Two layers, two jobs: the permission rule cannot fail open, and this one covers more ground.
|
|
58
|
-
// **The second case above is the uncomfortable one**, and it is named in `../gate-map.md`'s
|
|
59
|
-
// honest-holes list rather than left here: it is the only gate whose sole layer is this file, so
|
|
60
|
-
// every "fails open" sentence above is, for shell writes to the constitution, the whole story
|
|
61
|
-
// rather than a footnote about a lost message.
|
|
62
|
-
//
|
|
63
|
-
// **One level of unwrapping, and no more.** Deeper nesting, a heredoc, an interpolated variable, a
|
|
64
|
-
// command assembled at runtime — all still escape, and no amount of parsing here would close that.
|
|
65
|
-
// The same is true one layer over: a write spelled through a runtime (`python3 -c`) or a writer
|
|
66
|
-
// outside that table reaches the file. Anything that must not happen regardless of spelling belongs
|
|
67
|
-
// on the platform floor, not in a matcher.
|
|
4
|
+
// On Claude Code 2.1.220 a PreToolUse hook that crashes fails open, so the compiled permission rule is the gate.
|
|
5
|
+
// Any internal error exits 0 with no decision, so the permission rule governs unchanged.
|
|
6
|
+
// It covers what a permission pattern cannot: one level of `sh -c` wrapping, and a shell write to a path a `write:` rule protects.
|
|
68
7
|
|
|
69
8
|
import fs from "node:fs";
|
|
70
9
|
import path from "node:path";
|
|
71
10
|
import { fileURLToPath } from "node:url";
|
|
72
11
|
|
|
73
|
-
// The action vocabulary is defined ONCE, in the compiler, and imported here. Two implementations of
|
|
74
|
-
// one matcher is the drift this repository keeps finding — and a matcher that drifts does not look
|
|
75
|
-
// wrong, it looks like a gate that quietly stopped covering something.
|
|
76
|
-
//
|
|
77
|
-
// **The same argument reaches one level up, and for a milestone it was applied only to the matcher.**
|
|
78
|
-
// Importing `matchesRule` kept the two layers agreeing about what an action *is* while they disagreed
|
|
79
|
-
// about which rules *exist*: this file read the workspace's declared policy, and `./compile.mjs`
|
|
80
|
-
// enforced that policy plus its packs' contributions. `packContributions` and `composeFragments` are
|
|
81
|
-
// imported for the same reason as the matcher — a second composer would drift the same way — and
|
|
82
|
-
// `doctor` took this identical repair for this identical noun one session earlier.
|
|
83
12
|
import { matchesRule, policyPath, packContributions, composeFragments, parse } from "./compile.mjs";
|
|
84
13
|
|
|
85
|
-
//
|
|
86
|
-
//
|
|
87
|
-
// It used to be derived: this file lived at `.portulan/compile/gate.mjs`, so `HERE/../..` was the
|
|
88
|
-
// repository and `basename(HERE/..)` was the workspace directory. That worked for exactly one layout —
|
|
89
|
-
// the author's — and it is why the file could not ship. Milestone 7 moves both runners into `cli/` so an
|
|
90
|
-
// adopter actually receives them (`package.json`'s `files` never carried `.portulan/`, so every compiled
|
|
91
|
-
// policy named a file the adopter did not have, and a missing hook fails open). From `cli/`, and even
|
|
92
|
-
// more so from `node_modules/@sleepy_panda_srl/portulan/cli/`, this file has no idea where the
|
|
93
|
-
// adopter's workspace is — and inferring one would be #131's class, paths resolved against the author's
|
|
94
|
-
// layout, in the tool with the most to lose from it.
|
|
95
|
-
//
|
|
96
|
-
// `CLAUDE_PROJECT_DIR` is what the host sets and what the emitted hook already interpolates; `cwd` is the
|
|
97
|
-
// honest fallback, because a hook runs from the project. `||` rather than `??` on purpose: an env var set
|
|
98
|
-
// to the empty string must fall through to `cwd`, not resolve every path against `""`.
|
|
99
|
-
//
|
|
100
|
-
// **What happens when no workspace is found differs between the two runners, and the first draft of this
|
|
101
|
-
// paragraph claimed the stricter behaviour for both.** THIS file steps aside silently — `main()` catches
|
|
102
|
-
// and returns without a decision — which is its own defended design: a PreToolUse hook that cannot read
|
|
103
|
-
// the policy must not block every tool call, and the permission rules compiled beside it still hold, so
|
|
104
|
-
// the layer degrades rather than disappearing. `./stop-gate.mjs` does the opposite and blocks loudly,
|
|
105
|
-
// because a Stop-gate that cannot read the workspace has nothing beneath it. Stated per file rather than
|
|
106
|
-
// as one sentence covering both, since they genuinely differ and the difference is the interesting part.
|
|
14
|
+
// `||` so an empty CLAUDE_PROJECT_DIR falls back to cwd, the directory the host runs a hook from.
|
|
107
15
|
const PROJECT = process.env.CLAUDE_PROJECT_DIR || process.cwd();
|
|
108
16
|
const WORKSPACE_DIR = process.env.PORTULAN_WORKSPACE || ".portulan";
|
|
109
17
|
const POLICY = policyPath(PROJECT, WORKSPACE_DIR);
|
|
110
18
|
|
|
111
|
-
/** Exit without a decision. The permission rule still holds; only the sentence is lost. */
|
|
112
19
|
function stepAside() {
|
|
113
20
|
process.exit(0);
|
|
114
21
|
}
|
|
115
22
|
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
// **Declared plus composed, and until #269 it was declared alone.** A pack contributes gate fragments
|
|
119
|
-
// (`contributes.gates` in its `pack.json`); `./compile.mjs` composes them onto the workspace's own rules
|
|
120
|
-
// before it emits anything, so the permission rules sitting beside this hook are compiled from the
|
|
121
|
-
// composed set. This file read `.portulan/gates.json` and nothing else, so a rule a pack contributed was
|
|
122
|
-
// enforced by the layer that cannot say why and invisible to the layer whose whole job is the sentence —
|
|
123
|
-
// and for a `prohibited` fragment with a matcher, the two layers returned different decisions.
|
|
124
|
-
//
|
|
125
|
-
// **What this cost on THIS repository is nothing, and saying so is the point.** Its one gate-contributing
|
|
126
|
-
// pack (`rituals/checkpoints`) contributes two fragments and both carry `action: {none: …}`, which no
|
|
127
|
-
// matcher can reach — `compile --matrix` lists them among the gates no backend compiles. So the defect
|
|
128
|
-
// here was latent rather than live, and it is fixed on the shape rather than on an incident. The demonstration
|
|
129
|
-
// in `./gate.test.mjs` uses a fixture whose pack contributes a *matchable* fragment, because manufacturing
|
|
130
|
-
// a live case would have meant giving a `none` rule a matcher — reversing the ruling recorded in
|
|
131
|
-
// `../.portulan/proposals/0029-a-constraint-names-a-category-not-a-list.md` Q3, which removed one.
|
|
132
|
-
//
|
|
133
|
-
// **Three limits, named rather than left to be measured.** _(This said "Two" while enumerating three:
|
|
134
|
-
// the third arrived with the round-1 fix below and the count above it did not move. Exactly the
|
|
135
|
-
// undercount-by-enumeration `../.portulan/gate-map.md`'s own `action none` row was corrected for on
|
|
136
|
-
// 2026-08-14, repeated here within the week. Found by Copilot on #272, round 2.)_
|
|
137
|
-
//
|
|
138
|
-
// The three share one principle, and stating it is what keeps a later reader from deriving the wrong
|
|
139
|
-
// rule from the pair: **this runner degrades toward whatever intact policy remains — composed to
|
|
140
|
-
// declared, declared to nothing — and never toward a block.** It is never the load-bearing layer, so
|
|
141
|
-
// every arm below hands the decision back rather than taking it.
|
|
142
|
-
//
|
|
143
|
-
// 1. **No discovery.** `packContributions` is called with no `discovery` thunk, so `resolutionRoots`'
|
|
144
|
-
// unasked arm never consults the host's plugin cache and roots come from the workspace manifest's
|
|
145
|
-
// `tree` alone. Deliberate twice over: this runs on every tool call, and a gate whose answer moved
|
|
146
|
-
// with what is installed on the machine would be a gate nobody could review. A pack resolving ONLY
|
|
147
|
-
// from the host cache therefore contributes to what a bare `compile` yields and not to what this
|
|
148
|
-
// hook composes — the residual gap, stated in `../.portulan/gate-map.md` and owned by `#264`.
|
|
149
|
-
// 2. **A composition failure falls back to the DECLARED policy**, and does not step aside. Composition
|
|
150
|
-
// only ever adds a rule or raises a tier — `composeFragments` throws on a demotion rather than
|
|
151
|
-
// applying it — so the composed policy is never weaker than the declared one, and falling back can
|
|
152
|
-
// only lose coverage this file already had rather than drop below it. Stepping aside instead would
|
|
153
|
-
// let a malformed `pack.json` switch off gates the workspace declares in its own file, which hands a
|
|
154
|
-
// dependency a lever on the layer above it. `./compile.mjs` refuses to build in the same case, and
|
|
155
|
-
// the divergence is the intended one: it fails closed at build time against a file you can edit,
|
|
156
|
-
// while this runs on every tool call.
|
|
157
|
-
// 3. **The composed policy is VALIDATED, by the same `parse` the compiler validates with.** Composition
|
|
158
|
-
// checks a fragment's *tier* and, when tightening, that the action shape is unchanged; it does not
|
|
159
|
-
// check that an ADDED fragment is a well-formed rule. So a pack contributing
|
|
160
|
-
// `{id, tier: prohibited, action: {shell: curl}}` and no `reason` composed cleanly and this hook
|
|
161
|
-
// denied with the sentence `— undefined`, where `compile` refuses the same input at exit 2 with
|
|
162
|
-
// *"rule `no-reason` carries no reason — a gate with no sentence to show a human is not finished"*.
|
|
163
|
-
// Measured, on the runner. The two layers disagreeing about whether a policy is admissible is this
|
|
164
|
-
// file's own defect one step further in. Raised by Copilot on #272, round 1, in the suppressed
|
|
165
|
-
// channel. **The limit, since the fix is narrower than it reads:** this covers the COMPOSED arm. A
|
|
166
|
-
// rule missing its `reason` in the *declared* policy still denies with `— undefined`, byte-identically
|
|
167
|
-
// with every predecessor — because the fallback for an inadmissible declared policy is that same
|
|
168
|
-
// policy, and stepping aside there would drop a live prohibition over a missing string. `compile`
|
|
169
|
-
// refuses that shape at build time, which is where it belongs. Pinned in `./gate.test.mjs` so the
|
|
170
|
-
// arm cannot be quietly converted to a step-aside.
|
|
171
|
-
//
|
|
172
|
-
// **Validation only — the decision is still made on the composed policy, not on `parse`'s return.**
|
|
173
|
-
// Not because the parsed shape would not match: measured, `parse` returns rules carrying the SAME
|
|
174
|
-
// `action` object beside the added `kind`/`target`, and `matchesRule` answers correctly on them —
|
|
175
|
-
// a draft of this comment claimed otherwise and was wrong. The two real grounds: `parse` **refuses
|
|
176
|
-
// rather than normalises**, so once it passes, the composed and parsed forms are identical for
|
|
177
|
-
// matching and deciding on the parsed one buys nothing; and the fallback path has no parsed form to
|
|
178
|
-
// offer — an inadmissible policy produces none — so deciding on `parse`'s return would hand
|
|
179
|
-
// `decide()` two different shapes on its two paths, importing a representational seam rather than
|
|
180
|
-
// closing one. **If `parse` ever stops refusing and starts normalising, that change must sweep this
|
|
181
|
-
// hook too.** The check costs 0.003 ms against a ~30 ms invocation, 2000 iterations, so it is free.
|
|
23
|
+
// No `discovery` option: packs resolve from the manifest's `tree` alone, never from the host's plugin cache.
|
|
24
|
+
// Falls back to the declared policy when composing or `parse` fails: a broken pack cannot switch off the workspace's gates.
|
|
182
25
|
function yielded(declared) {
|
|
183
26
|
try {
|
|
184
27
|
const { contributions } = packContributions(PROJECT, WORKSPACE_DIR);
|
|
@@ -190,41 +33,7 @@ function yielded(declared) {
|
|
|
190
33
|
}
|
|
191
34
|
}
|
|
192
35
|
|
|
193
|
-
/**
|
|
194
|
-
* The rule this call answers to: the STRONGEST gate it falls under, not the first one listed.
|
|
195
|
-
*
|
|
196
|
-
* This returned the first match for one milestone, which is not the same rule whenever two of different
|
|
197
|
-
* tiers match one call: the weaker one wins if it is listed higher. The shape is ordinary rather than
|
|
198
|
-
* exotic — one dangerous spelling prohibited beneath a prefix the workspace already gates broadly,
|
|
199
|
-
* `git push` at `gated` and `git push --mirror` at `prohibited` — and it lands on `ask` for an action the
|
|
200
|
-
* policy prohibits.
|
|
201
|
-
*
|
|
202
|
-
* **Composition makes that systematic rather than causing it, and a draft of this paragraph had it the
|
|
203
|
-
* other way round** — *"indistinguishable from the strongest for as long as the policy came from one
|
|
204
|
-
* file"*. Measured on a workspace composing nothing, with those two rules in that order: the old scan
|
|
205
|
-
* answered `ask`, this one answers `deny`. What composition adds is that a contributed rule can never
|
|
206
|
-
* win the tie-break, because `composeFragments` APPENDS added rules after the workspace's own — so a
|
|
207
|
-
* pack-contributed `prohibited` rule is always the later one and always the one first-match discards.
|
|
208
|
-
* That is #269's case reached through the ordering rather than through the missing rules, and closing
|
|
209
|
-
* only the missing-rules half would have left it.
|
|
210
|
-
*
|
|
211
|
-
* Grounded in the policy's own semantics rather than in host precedence between two matching patterns,
|
|
212
|
-
* which this repository has never measured and which nothing here needs: `prohibited` means the action
|
|
213
|
-
* must not happen, so answering `ask` on a call the policy prohibits is wrong on the policy's terms
|
|
214
|
-
* whatever the host does with the compiled pair.
|
|
215
|
-
*
|
|
216
|
-
* **It only ever moves a decision UP, and that is the honest form of the guarantee.** `prohibited` is the
|
|
217
|
-
* top of the tier order and `gated` the only other tier considered, so "strongest, ties to the first" is
|
|
218
|
-
* exactly *the first `prohibited` match if there is one, otherwise the first `gated` match* — which is
|
|
219
|
-
* what the loop below is. Output is byte-identical wherever no two matching rules sit at different tiers,
|
|
220
|
-
* and `ask` becomes `deny` where they do. **Not** "byte-identical for any workspace that composes
|
|
221
|
-
* nothing" — that is the claim the counterexample above kills, and this repository is not evidence for it
|
|
222
|
-
* either: its own policy is unchanged only because `edit-the-constitution`, the one `prohibited` rule it
|
|
223
|
-
* declares, is listed FIRST, so first-match already returned it. Measured, on a command matching both it
|
|
224
|
-
* and a gated shell rule: `deny` from either runner. Move that rule down the file and the two disagree.
|
|
225
|
-
* The early return on `prohibited` also keeps the denied path's short-circuit: nothing later can outrank
|
|
226
|
-
* the top of the order.
|
|
227
|
-
*/
|
|
36
|
+
/** The strongest gate the call matches, not the first listed: a `prohibited` match outranks any `gated` one. */
|
|
228
37
|
function decide(payload, policy) {
|
|
229
38
|
const tool = payload.tool_name;
|
|
230
39
|
const input = payload.tool_input ?? {};
|
package/cli/goldens.mjs
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Grade this workspace's compiled gates against a corpus of adversarial fixtures.
|
|
3
|
+
//
|
|
4
|
+
// Exit 0 green · 1 red · 2 could not run.
|
|
5
|
+
|
|
6
|
+
import fs from "node:fs";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { pathToFileURL } from "node:url";
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
CompileError,
|
|
12
|
+
READ_TOOLS,
|
|
13
|
+
WRITE_TOOLS,
|
|
14
|
+
composeFragments,
|
|
15
|
+
matchesRule,
|
|
16
|
+
packContributions,
|
|
17
|
+
parse,
|
|
18
|
+
policyDeclaration,
|
|
19
|
+
resolveWorkspace,
|
|
20
|
+
unreadableManifest,
|
|
21
|
+
} from "./compile.mjs";
|
|
22
|
+
|
|
23
|
+
export const CORPUS_DIR = "evals/goldens/gates";
|
|
24
|
+
|
|
25
|
+
export const MATCHABLE = ["shell", "write", "read"];
|
|
26
|
+
|
|
27
|
+
/** `holds`: the matcher catches the case and must keep catching it; `documented-hole`: it does not, as the gate map records. */
|
|
28
|
+
export const CLASSES = ["holds", "documented-hole"];
|
|
29
|
+
|
|
30
|
+
/** The branch of `matchesRule` a case takes, since two branches segment a command differently. */
|
|
31
|
+
export const PATHS = ["matchesPath", "shell-write", "shell-prefix", "no-branch"];
|
|
32
|
+
|
|
33
|
+
/** `no-branch`: a kind and tool the matcher has no code for, such as `Bash` against a `read:` rule. */
|
|
34
|
+
export function matcherPath(kind, tool) {
|
|
35
|
+
if (kind === "shell") return tool === "Bash" ? "shell-prefix" : "no-branch";
|
|
36
|
+
if (kind === "write") {
|
|
37
|
+
if (WRITE_TOOLS.includes(tool)) return "matchesPath";
|
|
38
|
+
return tool === "Bash" ? "shell-write" : "no-branch";
|
|
39
|
+
}
|
|
40
|
+
if (kind === "read") return READ_TOOLS.includes(tool) ? "matchesPath" : "no-branch";
|
|
41
|
+
return "no-branch";
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export class CouldNotRun extends Error {}
|
|
45
|
+
|
|
46
|
+
function readJson(file, what) {
|
|
47
|
+
let text;
|
|
48
|
+
try {
|
|
49
|
+
text = fs.readFileSync(file, "utf8");
|
|
50
|
+
} catch (cause) {
|
|
51
|
+
throw new CouldNotRun(`${what} at ${file} cannot be read — ${cause.code ?? cause.message}`);
|
|
52
|
+
}
|
|
53
|
+
try {
|
|
54
|
+
return JSON.parse(text);
|
|
55
|
+
} catch (cause) {
|
|
56
|
+
throw new CouldNotRun(`${what} at ${file} is not valid JSON — ${cause.message}`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Composed before `parse`, as `compile` does, so a pack's fragment is graded like a hand-written rule. */
|
|
61
|
+
export function yieldedRules(named, { packRoots = null } = {}) {
|
|
62
|
+
const { workspaceRoot, workspaceDir } = resolveWorkspace(named);
|
|
63
|
+
const unreadable = unreadableManifest(workspaceRoot, workspaceDir);
|
|
64
|
+
if (unreadable !== null) {
|
|
65
|
+
throw new CouldNotRun(
|
|
66
|
+
`${unreadable.file} is not a manifest this tool can read: ${unreadable.why}. Read as one declaring nothing, ` +
|
|
67
|
+
`it would have fixtures graded against a \`gates.json\` found by convention, which it may not name. ` +
|
|
68
|
+
`There is nothing to grade fixtures against`,
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
const { file: policyFile, declared, reason } = policyDeclaration(workspaceRoot, workspaceDir);
|
|
72
|
+
if (reason === "refused" || (!declared && !fs.existsSync(policyFile))) {
|
|
73
|
+
const why =
|
|
74
|
+
reason === "no-manifest"
|
|
75
|
+
? `there is no ${workspaceDir}/workspace.json, so nothing here has been authored as a workspace yet`
|
|
76
|
+
: reason === "refused"
|
|
77
|
+
? `${workspaceDir}/workspace.json DOES name a gate policy, and it was refused — the manifest is not at fault, the path it names is`
|
|
78
|
+
: `${workspaceDir}/workspace.json declares no \`gates\` key, which is a legitimate shape and means the policy is expected at the default path`;
|
|
79
|
+
const head =
|
|
80
|
+
reason === "refused" && fs.existsSync(policyFile)
|
|
81
|
+
? `the \`gates.json\` at ${policyFile} is not the gate policy the manifest names`
|
|
82
|
+
: `no gate policy at ${policyFile}`;
|
|
83
|
+
throw new CouldNotRun(`${head}: ${why}. There is nothing to grade fixtures against`);
|
|
84
|
+
}
|
|
85
|
+
const policy = readJson(policyFile, "the gate policy");
|
|
86
|
+
// Never the host's plugin cache: a rail that consulted it would grade the machine, not the tree.
|
|
87
|
+
const { contributions, unresolved } = packContributions(workspaceRoot, workspaceDir, {
|
|
88
|
+
...(packRoots === null ? { named: [] } : { packRoots }),
|
|
89
|
+
discovery: null,
|
|
90
|
+
forced: false,
|
|
91
|
+
});
|
|
92
|
+
const composed = composeFragments(policy, contributions);
|
|
93
|
+
return { workspaceRoot, rules: parse(composed.policy).rules, unresolved };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function partition(rules) {
|
|
97
|
+
const matchable = [];
|
|
98
|
+
const exempt = [];
|
|
99
|
+
for (const rule of rules) {
|
|
100
|
+
const kind = MATCHABLE.find((k) => typeof rule.action?.[k] === "string");
|
|
101
|
+
if (kind) matchable.push({ ...rule, kind });
|
|
102
|
+
else exempt.push({ ...rule, why: Object.keys(rule.action ?? {}).join(", ") || "no action" });
|
|
103
|
+
}
|
|
104
|
+
return { matchable, exempt };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function readCorpus(repoRoot, dir = CORPUS_DIR) {
|
|
108
|
+
const base = path.join(repoRoot, dir);
|
|
109
|
+
let names;
|
|
110
|
+
try {
|
|
111
|
+
names = fs.readdirSync(base).filter((n) => n.endsWith(".json")).sort();
|
|
112
|
+
} catch (cause) {
|
|
113
|
+
throw new CouldNotRun(`the fixture corpus at ${base} cannot be read — ${cause.code ?? cause.message}`);
|
|
114
|
+
}
|
|
115
|
+
const files = [];
|
|
116
|
+
for (const name of names) {
|
|
117
|
+
const file = path.join(base, name);
|
|
118
|
+
const doc = readJson(file, "a fixture file");
|
|
119
|
+
const where = `${dir}/${name}`;
|
|
120
|
+
if (typeof doc.rule !== "string" || doc.rule === "") {
|
|
121
|
+
throw new CouldNotRun(`${where} names no \`rule\` — a fixture file attacks exactly one rule and must say which`);
|
|
122
|
+
}
|
|
123
|
+
if (name !== `${doc.rule}.json`) {
|
|
124
|
+
throw new CouldNotRun(
|
|
125
|
+
`${where} declares rule \`${doc.rule}\` — one fixture file per rule, named for it, so rename it to ` +
|
|
126
|
+
`${dir}/${doc.rule}.json or correct the \`rule\` field. A misfiled corpus grades correctly and ` +
|
|
127
|
+
`reviews badly, which is the worse of the two`,
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
if (!Array.isArray(doc.cases) || doc.cases.length === 0) {
|
|
131
|
+
throw new CouldNotRun(`${where} carries no \`cases\` — an empty fixture file is coverage that is not there`);
|
|
132
|
+
}
|
|
133
|
+
for (const [i, c] of doc.cases.entries()) {
|
|
134
|
+
const at = `${where} case ${i} (${c?.id ?? "unnamed"})`;
|
|
135
|
+
if (typeof c?.id !== "string" || c.id === "") throw new CouldNotRun(`${at} has no \`id\``);
|
|
136
|
+
if (!CLASSES.includes(c?.class)) {
|
|
137
|
+
throw new CouldNotRun(`${at} declares class ${JSON.stringify(c?.class)} — one of ${CLASSES.join(" / ")}`);
|
|
138
|
+
}
|
|
139
|
+
if (typeof c?.tool !== "string" || c.tool === "") throw new CouldNotRun(`${at} names no \`tool\``);
|
|
140
|
+
if (!PATHS.includes(c?.path)) {
|
|
141
|
+
throw new CouldNotRun(`${at} declares path ${JSON.stringify(c?.path)} — one of ${PATHS.join(" / ")}`);
|
|
142
|
+
}
|
|
143
|
+
if (typeof c?.expect !== "boolean") throw new CouldNotRun(`${at} declares no boolean \`expect\``);
|
|
144
|
+
if (typeof c?.why !== "string" || c.why.trim() === "") {
|
|
145
|
+
throw new CouldNotRun(`${at} carries no \`why\` — an attack case nobody can read is not reviewable`);
|
|
146
|
+
}
|
|
147
|
+
if (c.class === "documented-hole" && (typeof c.hole !== "string" || c.hole.trim() === "")) {
|
|
148
|
+
throw new CouldNotRun(`${at} is a documented-hole and names no \`hole\` — the record it keeps true is the point of the class`);
|
|
149
|
+
}
|
|
150
|
+
if (c.input === null || typeof c.input !== "object" || Array.isArray(c.input)) {
|
|
151
|
+
throw new CouldNotRun(`${at} declares no \`input\` object`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
files.push({ where, doc });
|
|
155
|
+
}
|
|
156
|
+
return files;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function grade(rules, corpus) {
|
|
160
|
+
const { matchable, exempt } = partition(rules);
|
|
161
|
+
const byId = new Map(matchable.map((r) => [r.id, r]));
|
|
162
|
+
const covered = new Set();
|
|
163
|
+
const findings = [];
|
|
164
|
+
const byPath = new Map();
|
|
165
|
+
let cases = 0;
|
|
166
|
+
|
|
167
|
+
for (const { where, doc } of corpus) {
|
|
168
|
+
const rule = byId.get(doc.rule);
|
|
169
|
+
if (!rule) {
|
|
170
|
+
const isExempt = exempt.some((e) => e.id === doc.rule);
|
|
171
|
+
findings.push({
|
|
172
|
+
where,
|
|
173
|
+
what: isExempt
|
|
174
|
+
? `attacks \`${doc.rule}\`, which declares no matchable action — it has no tool-level surface, so there is nothing to attack. Delete the file, or give the rule an action a matcher can read`
|
|
175
|
+
: `attacks \`${doc.rule}\`, which the yielded policy does not declare. Either the rule was renamed and this file was not, or the fixture outlived its gate`,
|
|
176
|
+
});
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
covered.add(rule.id);
|
|
180
|
+
for (const c of doc.cases) {
|
|
181
|
+
cases += 1;
|
|
182
|
+
const derived = matcherPath(rule.kind, c.tool);
|
|
183
|
+
byPath.set(derived, (byPath.get(derived) ?? 0) + 1);
|
|
184
|
+
if (c.path !== derived) {
|
|
185
|
+
findings.push({
|
|
186
|
+
where: `${where} → ${c.id}`,
|
|
187
|
+
what:
|
|
188
|
+
`declares path \`${c.path}\` and \`matchesRule\` takes \`${derived}\` for a ` +
|
|
189
|
+
`${rule.kind}: rule reached through ${c.tool}. The path is derived from the action kind and the ` +
|
|
190
|
+
`tool, so this is a mislabel rather than a disagreement — correct the case`,
|
|
191
|
+
});
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
const actual = matchesRule(rule, c.tool, c.input);
|
|
195
|
+
if (actual === c.expect) continue;
|
|
196
|
+
findings.push({
|
|
197
|
+
where: `${where} → ${c.id}`,
|
|
198
|
+
what:
|
|
199
|
+
c.class === "holds"
|
|
200
|
+
? `REGRESSION: \`${rule.id}\` used to answer ${c.expect} for this and now answers ${actual}. ${c.why}`
|
|
201
|
+
: `the documented hole \`${c.hole}\` has MOVED: this case expects ${c.expect} and the matcher answers ${actual}. ` +
|
|
202
|
+
`If the hole closed, that is good news and the record must say so — update ${c.hole}, then change this case to \`holds\`. ` +
|
|
203
|
+
`A hole list that still lists a closed hole is as wrong as one that hides an open one`,
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
for (const rule of matchable) {
|
|
209
|
+
if (covered.has(rule.id)) continue;
|
|
210
|
+
findings.push({
|
|
211
|
+
where: `${CORPUS_DIR}/`,
|
|
212
|
+
what:
|
|
213
|
+
`\`${rule.id}\` (${rule.tier}, a \`${rule.kind}:\` rule) compiles to a matcher and no fixture attacks it. ` +
|
|
214
|
+
`Coverage is measured, not named — add ${CORPUS_DIR}/${rule.id}.json with at least one case`,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
return { findings, cases, matchable, exempt, covered, byPath };
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function usage() {
|
|
222
|
+
return [
|
|
223
|
+
"usage: node cli/goldens.mjs [--check] [--workspace <dir>] [--pack-root <dir>]",
|
|
224
|
+
"",
|
|
225
|
+
" Grades evals/goldens/gates/ against the gate policy the workspace YIELDS.",
|
|
226
|
+
" --check is the default and the only mode; the flag is accepted so the recipe reads",
|
|
227
|
+
" like its siblings.",
|
|
228
|
+
"",
|
|
229
|
+
" Exit 0 green · 1 red · 2 could not run.",
|
|
230
|
+
].join("\n");
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
export function run(argv = [], { stdout = process.stdout, stderr = process.stderr, cwd = process.cwd() } = {}) {
|
|
234
|
+
const say = (line = "") => stdout.write(`${line}\n`);
|
|
235
|
+
// First, so `--help` is answered whatever else the command line holds.
|
|
236
|
+
if (argv.includes("--help") || argv.includes("-h")) {
|
|
237
|
+
say(usage());
|
|
238
|
+
return 0;
|
|
239
|
+
}
|
|
240
|
+
let named = cwd;
|
|
241
|
+
let packRoots = null;
|
|
242
|
+
try {
|
|
243
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
244
|
+
if (argv[i] === "--check") continue;
|
|
245
|
+
if (argv[i] === "--workspace") {
|
|
246
|
+
named = argv[i + 1];
|
|
247
|
+
i += 1;
|
|
248
|
+
if (named === undefined) throw new CouldNotRun("--workspace needs a directory");
|
|
249
|
+
} else if (argv[i] === "--pack-root") {
|
|
250
|
+
const root = argv[i + 1];
|
|
251
|
+
i += 1;
|
|
252
|
+
// A value starting with `-` is a flag, not a path; a directory so named is `./-name`.
|
|
253
|
+
if (root === undefined || root.startsWith("-")) throw new CouldNotRun("--pack-root needs a directory");
|
|
254
|
+
let stat = null;
|
|
255
|
+
try {
|
|
256
|
+
stat = fs.statSync(root);
|
|
257
|
+
} catch (cause) {
|
|
258
|
+
throw new CouldNotRun(`--pack-root ${root} cannot be read — ${cause.code ?? cause.message}`);
|
|
259
|
+
}
|
|
260
|
+
if (!stat.isDirectory()) throw new CouldNotRun(`--pack-root ${root} is not a directory`);
|
|
261
|
+
(packRoots ??= []).push(path.resolve(root));
|
|
262
|
+
} else throw new CouldNotRun(`unknown argument ${JSON.stringify(argv[i])}`);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const { workspaceRoot, rules, unresolved } = yieldedRules(named, { packRoots });
|
|
266
|
+
const corpus = readCorpus(workspaceRoot);
|
|
267
|
+
const { findings, cases, matchable, exempt, byPath } = grade(rules, corpus);
|
|
268
|
+
|
|
269
|
+
for (const u of unresolved) say(`pack ${u.name} UNRESOLVED — ${u.why}; its gate fragments are not in this census`);
|
|
270
|
+
|
|
271
|
+
say(`goldens: ${cases} case(s) over ${matchable.length} matchable rule(s) in ${corpus.length} fixture file(s)`);
|
|
272
|
+
say(`goldens: by matcher path — ${PATHS.map((p) => `${p} ${byPath.get(p) ?? 0}`).join(" · ")}`);
|
|
273
|
+
// Printed on every run, so the exemption cannot become a quiet way out of fixtures.
|
|
274
|
+
if (exempt.length) {
|
|
275
|
+
say(`goldens: ${exempt.length} rule(s) declare no matchable action and are exempt from fixtures:`);
|
|
276
|
+
for (const e of exempt) say(` ${e.id} (${e.tier}, ${e.why})`);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (findings.length) {
|
|
280
|
+
for (const f of findings) stderr.write(`goldens: ${f.where}\n ${f.what}\n`);
|
|
281
|
+
stderr.write(`RED — ${findings.length} finding(s) in the gate corpus\n`);
|
|
282
|
+
return 1;
|
|
283
|
+
}
|
|
284
|
+
say("GREEN — every matchable gate carries fixtures, and every case answers as recorded");
|
|
285
|
+
say("goldens: this is a PRESENCE floor — whether a corpus is a real attack is a reviewer's judgement, not this rail's");
|
|
286
|
+
return 0;
|
|
287
|
+
} catch (error) {
|
|
288
|
+
if (error instanceof CouldNotRun || error instanceof CompileError) {
|
|
289
|
+
stderr.write(`goldens: ${error.message}\n`);
|
|
290
|
+
return 2;
|
|
291
|
+
}
|
|
292
|
+
// An unexpected throw exits 2, never 1: a crash is not a verdict on the corpus.
|
|
293
|
+
stderr.write(`goldens: could not finish grading — ${error?.stack ?? error}\n`);
|
|
294
|
+
return 2;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// URLs on both sides, since `import.meta.url` percent-encodes; the realpath covers a symlinked `bin`.
|
|
299
|
+
function isMain() {
|
|
300
|
+
const invoked = process.argv[1];
|
|
301
|
+
if (!invoked) return false;
|
|
302
|
+
if (import.meta.url === pathToFileURL(invoked).href) return true;
|
|
303
|
+
try {
|
|
304
|
+
return import.meta.url === pathToFileURL(fs.realpathSync(invoked)).href;
|
|
305
|
+
} catch {
|
|
306
|
+
return false;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// `exitCode`, not `exit()`: exiting outright can cut off output a pipe has not drained.
|
|
311
|
+
if (isMain()) process.exitCode = run(process.argv.slice(2));
|