create-pmos 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/payload/KIT-VERSION +1 -1
- package/payload/scripts/pmos-log.sh +29 -2
- package/payload/scripts/workflow-state.js +40 -6
- package/payload/skills/evaluator.skill +45 -12
- package/payload/skills/pmos-router.skill +1 -1
- package/payload/skills-native/evaluator/SKILL.md +45 -12
- package/payload/skills-native/pmos-router/SKILL.md +1 -1
- package/workspace/lib/drive-guard.js +269 -13
- package/workspace/lib/engine/workflow-state.js +40 -6
- package/workspace/server.js +58 -8
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "create-pmos",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.29.0",
|
|
4
4
|
"description": "Scaffold the PMOS portable kit \u2014 run the PMOS product-management workflow (initiative \u2192 lane-fit spec \u2192 contract \u2192 build \u2192 three gates, with work-type lanes for features, research, maintenance, incidents, and content) entirely inside your own repo, with all state as git-versioned files. Includes `workflow-sweep`, which audits the state every board derives from and separates real defects from work you simply have not done. `create-pmos workspace` also installs + launches the local Terminal Workspace GUI. No database, no network, no telemetry.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"bin": {
|
package/payload/KIT-VERSION
CHANGED
|
@@ -45,10 +45,37 @@ case "$cmd" in
|
|
|
45
45
|
append runs.jsonl "{\"run_id\":$(js "$1"),\"initiative_id\":$(js "$2"),\"human_corrections\":$corr,\"date\":\"$today\"$model_field}"
|
|
46
46
|
;;
|
|
47
47
|
eval)
|
|
48
|
-
|
|
48
|
+
# SKIPPED is a verdict this writer must be able to record.
|
|
49
|
+
#
|
|
50
|
+
# The enum here was PASS|PASS-WITH-FINDINGS|FAIL, while AGENTS.md tells every adopter: "a gate
|
|
51
|
+
# that was skipped is recorded as skipped — never leave a missing eval to be read as a pass."
|
|
52
|
+
# An adopter following that instruction hit `fail` and had no way to comply except by hand-
|
|
53
|
+
# appending, which is the drift this writer exists to prevent. PMOS-self's own ledger carries 16
|
|
54
|
+
# SKIPPED rows; the kit could not have written one of them.
|
|
55
|
+
#
|
|
56
|
+
# A skip is NOT a soft pass: it takes no score (a skip has no score to give), it requires a
|
|
57
|
+
# stated reason — an unexplained skip is indistinguishable from a forgotten one — and it leaves
|
|
58
|
+
# the verification the run owes still owing.
|
|
59
|
+
if [ "${4:-}" = "SKIPPED" ] || [ "${3:-}" = "SKIPPED" ]; then
|
|
60
|
+
# tolerated both argument positions: `eval <run> <init> SKIPPED <reason>` is the natural form
|
|
61
|
+
# when there is no score to give, and refusing it on argument order would just push people
|
|
62
|
+
# back to hand-appending.
|
|
63
|
+
if [ "${3:-}" = "SKIPPED" ]; then skip_reason="${4:-}"; skip_rubric="${5:-}"; skip_model="${6:-}"
|
|
64
|
+
else skip_reason="${5:-}"; skip_rubric="${6:-}"; skip_model="${7:-}"; fi
|
|
65
|
+
nonempty run_id "$1"; nonempty initiative_id "$2"
|
|
66
|
+
[ -n "$skip_reason" ] || fail "SKIPPED needs a reason: eval <run_id> <initiative_id> SKIPPED <reason> [rubric_path] [evaluator_model]"
|
|
67
|
+
rubric_field=""
|
|
68
|
+
if [ -n "$skip_rubric" ]; then rubric_field=",\"rubric_path\":$(js "$skip_rubric")"; fi
|
|
69
|
+
emodel_field=""
|
|
70
|
+
if [ -n "$skip_model" ]; then emodel_field=",\"evaluator_model\":$(js "$skip_model")"; fi
|
|
71
|
+
append evals.jsonl "{\"run_id\":$(js "$1"),\"initiative_id\":$(js "$2"),\"score\":null,\"verdict\":\"SKIPPED\",\"skipped_reason\":$(js "$skip_reason")$rubric_field,\"date\":\"$today\"$emodel_field}"
|
|
72
|
+
exit 0
|
|
73
|
+
fi
|
|
74
|
+
[ $# -ge 5 ] || fail "usage: eval <run_id> <initiative_id> <score 0..1> <PASS|PASS-WITH-FINDINGS|FAIL> <rubric_path> [evaluator_model]
|
|
75
|
+
or: eval <run_id> <initiative_id> SKIPPED <reason> [rubric_path] [evaluator_model]"
|
|
49
76
|
nonempty run_id "$1"; nonempty initiative_id "$2"
|
|
50
77
|
printf '%s' "$3" | grep -qE '^(0?\.[0-9]+|0|1(\.0+)?)$' || fail "score must be 0..1 written as 0, 1, N.N or .N — got '$3'."
|
|
51
|
-
case "$4" in PASS|PASS-WITH-FINDINGS|FAIL) ;; *) fail "verdict must be PASS | PASS-WITH-FINDINGS | FAIL, got '$4'." ;; esac
|
|
78
|
+
case "$4" in PASS|PASS-WITH-FINDINGS|FAIL) ;; *) fail "verdict must be PASS | PASS-WITH-FINDINGS | FAIL | SKIPPED, got '$4'." ;; esac
|
|
52
79
|
score="$3"; case "$score" in .*) score="0$score" ;; esac # ".5" -> "0.5": bare .5 is not valid JSON
|
|
53
80
|
[ -f "$KIT/$5" ] || [ -f "$5" ] || fail "rubric_path '$5' not found (relative to the kit or the repo)."
|
|
54
81
|
# optional evaluator-model provenance (which model rendered the verdict). Calibration (κ) is a
|
|
@@ -859,14 +859,48 @@ function computeWorkflow(id, opts) {
|
|
|
859
859
|
// ── review ─ the Review Gate's verdict, where the lane has a rubric ──────────────────────────
|
|
860
860
|
const evals = readJsonl(path.join(L.runsDir, 'evals.jsonl')).filter((e) => e && e.initiative_id === id);
|
|
861
861
|
{
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
862
|
+
/* The rung reads the RUBRIC, not the lane.
|
|
863
|
+
*
|
|
864
|
+
* It keyed on `lane` alone, so a chore that deliberately carries a rubric could never surface
|
|
865
|
+
* its Review Gate — the ladder said "chore lane carries no rubric (D56)" while a SKIPPED verdict
|
|
866
|
+
* sat in evals.jsonl unread. That is how `guard-and-ledger-hardening`, a chore whose own record
|
|
867
|
+
* argues "blast radius overrides lane" and which carried a rubric on purpose, reached `accept`
|
|
868
|
+
* with the gate never run. The board could not show what the ledger already knew.
|
|
869
|
+
*
|
|
870
|
+
* A prototype is still never scored — D60 grills leg A as dialogue, and that is a property of
|
|
871
|
+
* the lane, not of whether someone attached a rubric. Every other lane defers to the artifact. */
|
|
872
|
+
const declaresRubric = !!String(fm.rubric_path || '').trim();
|
|
873
|
+
/* EVERY lane defers to the artifact — the first version said so in the comment and applied it
|
|
874
|
+
* only to `chore`, so a discovery or incident initiative with no rubric (legitimate under D56)
|
|
875
|
+
* showed a permanent `no Review-Gate verdict recorded`: a rung it could never satisfy. The lanes
|
|
876
|
+
* that carry no rubric by design are named once, here. */
|
|
877
|
+
const RUBRIC_OPTIONAL = ['chore', 'discovery', 'incident', 'content'];
|
|
878
|
+
if (effLane === 'prototype') {
|
|
879
|
+
steps.push(step('review', 'review', 'na', 'leg A is grilled as dialogue, never scored (D60)'));
|
|
880
|
+
} else if (RUBRIC_OPTIONAL.includes(effLane) && !declaresRubric && !evals.length) {
|
|
881
|
+
steps.push(step('review', 'review', 'na', `${effLane} lane carries no rubric (D56)`));
|
|
866
882
|
} else if (evals.length) {
|
|
867
883
|
const last = evals[evals.length - 1];
|
|
868
|
-
|
|
869
|
-
|
|
884
|
+
const verdict = String(last.verdict || '').toUpperCase();
|
|
885
|
+
const label = `${last.verdict}${last.score != null ? ` (${last.score})` : ''} — advisory until calibrated per evaluator model (D40)`;
|
|
886
|
+
/* A SKIPPED gate is NOT a satisfied gate, and a FAIL is emphatically not one.
|
|
887
|
+
*
|
|
888
|
+
* This rung rendered both as a green ✓, which is the watermelon this repo names as its own
|
|
889
|
+
* failure mode: the board said the Review Gate was done over a row that says judging never
|
|
890
|
+
* happened. Thirteen initiatives currently render `✓ review` over a SKIPPED verdict, and the
|
|
891
|
+
* fix that made a chore's gate visible ADDED one. The module's own rule, repeated in three
|
|
892
|
+
* files — "a skip is an honest RECORD that verification did not happen; it is not
|
|
893
|
+
* verification" — now applies to the surface that reports it. */
|
|
894
|
+
if (verdict === 'SKIPPED') {
|
|
895
|
+
steps.push(step('review', 'review', 'pending', label,
|
|
896
|
+
'the gate was skipped, so it is still owed — commission an independent evaluator and record a real verdict',
|
|
897
|
+
{ rule: 'R3' }));
|
|
898
|
+
} else if (verdict === 'FAIL') {
|
|
899
|
+
steps.push(step('review', 'review', 'blocked', label,
|
|
900
|
+
'the Review Gate FAILED — remediate the findings and record a new verdict', { rule: 'R3' }));
|
|
901
|
+
} else {
|
|
902
|
+
steps.push(step('review', 'review', 'done', label));
|
|
903
|
+
}
|
|
870
904
|
} else {
|
|
871
905
|
steps.push(step('review', 'review', 'pending', 'no Review-Gate verdict recorded',
|
|
872
906
|
'score the build against the rubric (build-skills/evaluator.skill), then append to evals.jsonl',
|
|
@@ -7,7 +7,7 @@ description: Adversarially evaluate an agent run's output against its rubric and
|
|
|
7
7
|
|
|
8
8
|
This skill is PMOS's **Review Gate** made a first-class, repeatable capability (P4). It formalizes the
|
|
9
9
|
adversarial independent evaluator run by hand through P1–P3: judge what an agent run produced against a
|
|
10
|
-
rubric, write a structured verdict
|
|
10
|
+
rubric, write a structured verdict through the validated writer (step 9), and surface it on the board. It runs in
|
|
11
11
|
the **interactive agent runtime** — any capable coding agent; heavy judgment belongs in the agent, not
|
|
12
12
|
an Edge Function ([D09](/DECISIONS.md)/[D34](/DECISIONS.md), amended 2026-07-23). PMOS-self runs it in
|
|
13
13
|
Claude Code, but the runtime is not Claude-specific.
|
|
@@ -32,7 +32,7 @@ Claude Code, but the runtime is not Claude-specific.
|
|
|
32
32
|
may under-reward an unfamiliar style. Either way: **default to skeptical**, and when a pass feels easy
|
|
33
33
|
or you are uncertain, **say so explicitly** and flag the run for the cross-family spot-check /
|
|
34
34
|
minority-veto jury ([calibration-method](/planning/evals/calibration-method.md)). Do not let stylistic
|
|
35
|
-
(un)familiarity read as quality. **Record your `evaluator_model` on the
|
|
35
|
+
(un)familiarity read as quality. **Record your `evaluator_model` on the verdict** — κ is
|
|
36
36
|
calibrated per evaluator model, and an unrecorded model silently pools calibration across model swaps.
|
|
37
37
|
- **Capability floor — this role assumes a frontier-tier reasoner.** The adversarial passes below
|
|
38
38
|
(default-to-refute, both-ways pairwise, semantic-pass-not-grep, injection resistance) degrade
|
|
@@ -57,8 +57,8 @@ Claude Code, but the runtime is not Claude-specific.
|
|
|
57
57
|
([injection-defenses](/planning/research/injection-defenses.md)).
|
|
58
58
|
- **Structural scope (binding).** You are **read-only**: no external-communication tool, no
|
|
59
59
|
repository writes, no artifact publishing; your **only write is the schema-constrained
|
|
60
|
-
|
|
61
|
-
that reads adversarial content must not be the component that can act. Any future widening of
|
|
60
|
+
verdict, written through the validated writer (step 9)**. This removes the exfiltration leg of the
|
|
61
|
+
lethal trifecta — the component that reads adversarial content must not be the component that can act. Any future widening of
|
|
62
62
|
this scope requires the trifecta audit in [AGENTS.md](/AGENTS.md) and a PM decision
|
|
63
63
|
([injection-defenses](/planning/research/injection-defenses.md)).
|
|
64
64
|
- **Advisory, not final.** Your verdict informs; it does not decide. The human PM **Acceptance Gate**
|
|
@@ -158,13 +158,45 @@ FAIL — never a clean PASS.) The split is **persisted**, not just narrated (ste
|
|
|
158
158
|
available, or state `pass-fraction: unknown (no backend)` in kit mode. A high pass-fraction is a
|
|
159
159
|
leniency-drift warning to the PM, not a compliment to the builders
|
|
160
160
|
([llm-judge-calibration](/planning/research/llm-judge-calibration.md)).
|
|
161
|
-
9. **Persist it
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
(
|
|
161
|
+
9. **Persist it through the validated writer — never by hand.** Which writer depends on where you
|
|
162
|
+
are running, and **both are validated; neither is a raw append**:
|
|
163
|
+
|
|
164
|
+
| you are in | the writer | why |
|
|
165
|
+
|---|---|---|
|
|
166
|
+
| **a vendored kit** (`pmos/` in a product repo) | `pmos/scripts/pmos-log.sh eval …` | ships with the kit; writes the kit's `state/evals.jsonl` |
|
|
167
|
+
| **PMOS-self** (file mode, [D64](/DECISIONS.md)) | `node scripts/record-eval.js --file verdict.json` | writes `planning/runs/evals.jsonl` |
|
|
168
|
+
| a hosted tenant | the `save_eval` MCP tool | **deprecated for PMOS-self**; does not exist in file mode |
|
|
169
|
+
|
|
170
|
+
```
|
|
171
|
+
# kit
|
|
172
|
+
pmos/scripts/pmos-log.sh eval <run_id> <initiative_id> <score> <PASS|PASS-WITH-FINDINGS|FAIL> <rubric_path> [evaluator_model]
|
|
173
|
+
pmos/scripts/pmos-log.sh eval <run_id> <initiative_id> SKIPPED "<reason>" [rubric_path] [evaluator_model]
|
|
174
|
+
|
|
175
|
+
# PMOS-self
|
|
176
|
+
node scripts/record-eval.js --file verdict.json # or pipe the JSON on stdin
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
Both **fail loud**: a rejected verdict exits non-zero with the specific reason and writes nothing,
|
|
180
|
+
so a malformed record can never light the gate green. `record-eval.js` additionally refuses any
|
|
181
|
+
sink but the eval ledger — resolved, not by name — which is what keeps a Review-Gate verdict out
|
|
182
|
+
of the PM's acceptance ledger.
|
|
183
|
+
|
|
184
|
+
If the writer your context calls for is not present, **say so in the verdict and stop** — do not
|
|
185
|
+
hand-append. A ledger written by hand is how a field ends up meaning three things.
|
|
186
|
+
|
|
187
|
+
The record carries `{ initiative_id, run_id, rubric_path, score, verdict, evaluator_model,
|
|
188
|
+
recorded_at, dimensions }`, where `dimensions` is `{ <name>: { score, weight, note, evidence,
|
|
189
|
+
axis } }` — `evidence` carries the quote/pointer from step 6 (or the string `unverified`), and
|
|
190
|
+
**`axis` is `"spec"` or `"standards"`** so the two-axis split is persisted, not just narrated. Put
|
|
191
|
+
the two axis pass/fail summaries in the top-level `note` and tag every dimension with its axis.
|
|
192
|
+
|
|
193
|
+
A `SKIPPED` verdict is **declared, not omitted** — it requires `skipped_reason` and carries
|
|
194
|
+
`score: null`. A skip is an honest record that judging did not happen; it does not close the
|
|
195
|
+
verification the run owes, and the queue checker (PMOS-self: `scripts/check-verify-queue.js`)
|
|
196
|
+
will keep reporting the debt.
|
|
197
|
+
|
|
198
|
+
This is what makes the gate self-hosting: the verdict lands in the eval ledger and surfaces on the
|
|
199
|
+
board.
|
|
168
200
|
10. **Hand to the PM.** The Acceptance Gate is theirs, and the PM records the ground-truth `pm_label` to
|
|
169
201
|
`acceptances` — paired against your verdict by `v_calibration` to compute the kappa that calibrates this
|
|
170
202
|
gate ([D40](/DECISIONS.md), [calibration-method](/planning/evals/calibration-method.md)). If the PM
|
|
@@ -177,5 +209,6 @@ A short report: **most important finding first**, then the **two axis verdicts (
|
|
|
177
209
|
one line; Standards: pass/fail — one line)** and the **overall VERDICT** (score vs threshold, under
|
|
178
210
|
the combine rule) with the pass-fraction disclosure, the calibration exemplars used (step 3), the
|
|
179
211
|
numbered defects, and the per-dimension scores **with their evidence quotes/pointers and `axis`
|
|
180
|
-
tag** — followed by the
|
|
212
|
+
tag** — followed by the verdict record, written through **the writer for your context (step 9)**:
|
|
213
|
+
`pmos/scripts/pmos-log.sh eval …` in a vendored kit, `node scripts/record-eval.js …` in PMOS-self. Be concrete; cite specifics; if it is genuinely solid, say
|
|
181
214
|
so — but only after a real refutation attempt.
|
|
@@ -19,7 +19,7 @@ lanes and gates that already exist; where it and an authority disagree, the auth
|
|
|
19
19
|
router is the bug (fix it via the weekly harness scan).
|
|
20
20
|
|
|
21
21
|
> **Format note:** three-level progressive disclosure ([SKILL-FORMAT](/skills/core/SKILL-FORMAT.md)).
|
|
22
|
-
> Sources of truth: the AGENTS.md lane list, `.github/scripts/initiative-gate.sh
|
|
22
|
+
> Sources of truth: the AGENTS.md lane list, the initiative gate (PMOS-self: `.github/scripts/initiative-gate.sh`; a vendored kit: `pmos/scripts/gate-initiative.sh`), D56/D58/D60.
|
|
23
23
|
|
|
24
24
|
## Step 0 — three checks before any route
|
|
25
25
|
|
|
@@ -8,7 +8,7 @@ license: MIT
|
|
|
8
8
|
|
|
9
9
|
This skill is PMOS's **Review Gate** made a first-class, repeatable capability (P4). It formalizes the
|
|
10
10
|
adversarial independent evaluator run by hand through P1–P3: judge what an agent run produced against a
|
|
11
|
-
rubric, write a structured verdict
|
|
11
|
+
rubric, write a structured verdict through the validated writer (step 9), and surface it on the board. It runs in
|
|
12
12
|
the **interactive agent runtime** — any capable coding agent; heavy judgment belongs in the agent, not
|
|
13
13
|
an Edge Function ([D09](/DECISIONS.md)/[D34](/DECISIONS.md), amended 2026-07-23). PMOS-self runs it in
|
|
14
14
|
Claude Code, but the runtime is not Claude-specific.
|
|
@@ -33,7 +33,7 @@ Claude Code, but the runtime is not Claude-specific.
|
|
|
33
33
|
may under-reward an unfamiliar style. Either way: **default to skeptical**, and when a pass feels easy
|
|
34
34
|
or you are uncertain, **say so explicitly** and flag the run for the cross-family spot-check /
|
|
35
35
|
minority-veto jury ([calibration-method](/planning/evals/calibration-method.md)). Do not let stylistic
|
|
36
|
-
(un)familiarity read as quality. **Record your `evaluator_model` on the
|
|
36
|
+
(un)familiarity read as quality. **Record your `evaluator_model` on the verdict** — κ is
|
|
37
37
|
calibrated per evaluator model, and an unrecorded model silently pools calibration across model swaps.
|
|
38
38
|
- **Capability floor — this role assumes a frontier-tier reasoner.** The adversarial passes below
|
|
39
39
|
(default-to-refute, both-ways pairwise, semantic-pass-not-grep, injection resistance) degrade
|
|
@@ -58,8 +58,8 @@ Claude Code, but the runtime is not Claude-specific.
|
|
|
58
58
|
([injection-defenses](/planning/research/injection-defenses.md)).
|
|
59
59
|
- **Structural scope (binding).** You are **read-only**: no external-communication tool, no
|
|
60
60
|
repository writes, no artifact publishing; your **only write is the schema-constrained
|
|
61
|
-
|
|
62
|
-
that reads adversarial content must not be the component that can act. Any future widening of
|
|
61
|
+
verdict, written through the validated writer (step 9)**. This removes the exfiltration leg of the
|
|
62
|
+
lethal trifecta — the component that reads adversarial content must not be the component that can act. Any future widening of
|
|
63
63
|
this scope requires the trifecta audit in [AGENTS.md](/AGENTS.md) and a PM decision
|
|
64
64
|
([injection-defenses](/planning/research/injection-defenses.md)).
|
|
65
65
|
- **Advisory, not final.** Your verdict informs; it does not decide. The human PM **Acceptance Gate**
|
|
@@ -159,13 +159,45 @@ FAIL — never a clean PASS.) The split is **persisted**, not just narrated (ste
|
|
|
159
159
|
available, or state `pass-fraction: unknown (no backend)` in kit mode. A high pass-fraction is a
|
|
160
160
|
leniency-drift warning to the PM, not a compliment to the builders
|
|
161
161
|
([llm-judge-calibration](/planning/research/llm-judge-calibration.md)).
|
|
162
|
-
9. **Persist it
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
(
|
|
162
|
+
9. **Persist it through the validated writer — never by hand.** Which writer depends on where you
|
|
163
|
+
are running, and **both are validated; neither is a raw append**:
|
|
164
|
+
|
|
165
|
+
| you are in | the writer | why |
|
|
166
|
+
|---|---|---|
|
|
167
|
+
| **a vendored kit** (`pmos/` in a product repo) | `pmos/scripts/pmos-log.sh eval …` | ships with the kit; writes the kit's `state/evals.jsonl` |
|
|
168
|
+
| **PMOS-self** (file mode, [D64](/DECISIONS.md)) | `node scripts/record-eval.js --file verdict.json` | writes `planning/runs/evals.jsonl` |
|
|
169
|
+
| a hosted tenant | the `save_eval` MCP tool | **deprecated for PMOS-self**; does not exist in file mode |
|
|
170
|
+
|
|
171
|
+
```
|
|
172
|
+
# kit
|
|
173
|
+
pmos/scripts/pmos-log.sh eval <run_id> <initiative_id> <score> <PASS|PASS-WITH-FINDINGS|FAIL> <rubric_path> [evaluator_model]
|
|
174
|
+
pmos/scripts/pmos-log.sh eval <run_id> <initiative_id> SKIPPED "<reason>" [rubric_path] [evaluator_model]
|
|
175
|
+
|
|
176
|
+
# PMOS-self
|
|
177
|
+
node scripts/record-eval.js --file verdict.json # or pipe the JSON on stdin
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Both **fail loud**: a rejected verdict exits non-zero with the specific reason and writes nothing,
|
|
181
|
+
so a malformed record can never light the gate green. `record-eval.js` additionally refuses any
|
|
182
|
+
sink but the eval ledger — resolved, not by name — which is what keeps a Review-Gate verdict out
|
|
183
|
+
of the PM's acceptance ledger.
|
|
184
|
+
|
|
185
|
+
If the writer your context calls for is not present, **say so in the verdict and stop** — do not
|
|
186
|
+
hand-append. A ledger written by hand is how a field ends up meaning three things.
|
|
187
|
+
|
|
188
|
+
The record carries `{ initiative_id, run_id, rubric_path, score, verdict, evaluator_model,
|
|
189
|
+
recorded_at, dimensions }`, where `dimensions` is `{ <name>: { score, weight, note, evidence,
|
|
190
|
+
axis } }` — `evidence` carries the quote/pointer from step 6 (or the string `unverified`), and
|
|
191
|
+
**`axis` is `"spec"` or `"standards"`** so the two-axis split is persisted, not just narrated. Put
|
|
192
|
+
the two axis pass/fail summaries in the top-level `note` and tag every dimension with its axis.
|
|
193
|
+
|
|
194
|
+
A `SKIPPED` verdict is **declared, not omitted** — it requires `skipped_reason` and carries
|
|
195
|
+
`score: null`. A skip is an honest record that judging did not happen; it does not close the
|
|
196
|
+
verification the run owes, and the queue checker (PMOS-self: `scripts/check-verify-queue.js`)
|
|
197
|
+
will keep reporting the debt.
|
|
198
|
+
|
|
199
|
+
This is what makes the gate self-hosting: the verdict lands in the eval ledger and surfaces on the
|
|
200
|
+
board.
|
|
169
201
|
10. **Hand to the PM.** The Acceptance Gate is theirs, and the PM records the ground-truth `pm_label` to
|
|
170
202
|
`acceptances` — paired against your verdict by `v_calibration` to compute the kappa that calibrates this
|
|
171
203
|
gate ([D40](/DECISIONS.md), [calibration-method](/planning/evals/calibration-method.md)). If the PM
|
|
@@ -178,5 +210,6 @@ A short report: **most important finding first**, then the **two axis verdicts (
|
|
|
178
210
|
one line; Standards: pass/fail — one line)** and the **overall VERDICT** (score vs threshold, under
|
|
179
211
|
the combine rule) with the pass-fraction disclosure, the calibration exemplars used (step 3), the
|
|
180
212
|
numbered defects, and the per-dimension scores **with their evidence quotes/pointers and `axis`
|
|
181
|
-
tag** — followed by the
|
|
213
|
+
tag** — followed by the verdict record, written through **the writer for your context (step 9)**:
|
|
214
|
+
`pmos/scripts/pmos-log.sh eval …` in a vendored kit, `node scripts/record-eval.js …` in PMOS-self. Be concrete; cite specifics; if it is genuinely solid, say
|
|
182
215
|
so — but only after a real refutation attempt.
|
|
@@ -21,7 +21,7 @@ lanes and gates that already exist; where it and an authority disagree, the auth
|
|
|
21
21
|
router is the bug (fix it via the weekly harness scan).
|
|
22
22
|
|
|
23
23
|
> **Format note:** three-level progressive disclosure ([SKILL-FORMAT](/skills/core/SKILL-FORMAT.md)).
|
|
24
|
-
> Sources of truth: the AGENTS.md lane list, `.github/scripts/initiative-gate.sh
|
|
24
|
+
> Sources of truth: the AGENTS.md lane list, the initiative gate (PMOS-self: `.github/scripts/initiative-gate.sh`; a vendored kit: `pmos/scripts/gate-initiative.sh`), D56/D58/D60.
|
|
25
25
|
|
|
26
26
|
## Step 0 — three checks before any route
|
|
27
27
|
|
|
@@ -102,16 +102,38 @@ const ALLOWED_DIR_PREFIXES = Object.freeze([
|
|
|
102
102
|
* its own run is the free zone working as designed, and a fabricated `PM_approved` INSIDE that write
|
|
103
103
|
* is caught by the content guard below, not by refusing the file.
|
|
104
104
|
*/
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
105
|
+
/**
|
|
106
|
+
* RESOLVE BEFORE ALLOWING — each ledger paired with the ONE directory it may live in.
|
|
107
|
+
*
|
|
108
|
+
* This was a bare basename list matched with `path.basename` in any directory, and an independent
|
|
109
|
+
* evaluator walked through it on 2026-08-04 with `/etc/agent-runs.jsonl`,
|
|
110
|
+
* `/home/user/.ssh/discoveries.md` and `workspace/lib/discoveries.md` — all permitted, because the
|
|
111
|
+
* name was the whole identity. Worse, the two halves of the allowlist disagreed: the PREFIX half
|
|
112
|
+
* correctly refused absolute paths while the basename half waved them through, so which rule
|
|
113
|
+
* happened to match decided whether `/etc/…` was inside the repo.
|
|
114
|
+
*
|
|
115
|
+
* A ledger is now its path, not its name.
|
|
116
|
+
*/
|
|
117
|
+
const ALLOWED_FILES = Object.freeze([
|
|
118
|
+
'planning/runs/agent-runs.jsonl',
|
|
119
|
+
'planning/runs/verify-queue.jsonl',
|
|
120
|
+
'planning/runs/work-items.jsonl',
|
|
109
121
|
'discoveries.md',
|
|
110
122
|
]);
|
|
111
123
|
|
|
124
|
+
/** Kept for the refusal message only — never for deciding permission. */
|
|
125
|
+
const ALLOWED_BASENAMES = Object.freeze(ALLOWED_FILES.map((f) => path.posix.basename(f)));
|
|
126
|
+
|
|
112
127
|
/** Named explicitly so the refusal says WHY, rather than "not on the allowlist". Each of these
|
|
113
128
|
* would otherwise be caught by the allowlist anyway — the value here is the message. */
|
|
114
129
|
const HUMAN_DECISION_SINKS = Object.freeze({
|
|
130
|
+
// The module argues that a DIRECTORY prefix "would admit the FOURTH ledger somebody adds next year
|
|
131
|
+
// without anyone deciding to — exactly the class-failure this rewrite exists to close" — and then
|
|
132
|
+
// allows `planning/evals/` as a prefix, which admits the PM's own override log and any future
|
|
133
|
+
// planning/evals/*.jsonl. The argument is right; it just was not applied to its own second
|
|
134
|
+
// directory. These two are the PM's calibration record, not the driver's scratch space.
|
|
135
|
+
'eval-calibration.md': 'the PM\'s log of every override of an evaluator verdict is the PM\'s (D40)',
|
|
136
|
+
'calibration-status.md': 'the declared calibration state is the PM\'s to assert, not the driver\'s',
|
|
115
137
|
'acceptances.jsonl': 'the Acceptance Gate is the PM\'s and non-delegable (D12)',
|
|
116
138
|
'evals.jsonl': 'a Review-Gate verdict on the driver\'s own work is self-grading — the driver may not record one (D40)',
|
|
117
139
|
'outcome_checks.jsonl': 'whether the bet paid off is the PM\'s judgement (D57)',
|
|
@@ -127,6 +149,167 @@ const HUMAN_DECISION_SINKS = Object.freeze({
|
|
|
127
149
|
* individually, which is the whole argument for the suite below. */
|
|
128
150
|
const DISPOSITION_RE = /^\s*"?\s*(PROMOTE(D)?|ITERATE(D)?|KILL(ED)?)\b/i;
|
|
129
151
|
|
|
152
|
+
/**
|
|
153
|
+
* The leading-token rule above is correct and insufficient, and the measurement says so.
|
|
154
|
+
*
|
|
155
|
+
* Anchoring to the value's first token bought the false positive back (`derived from the promoted
|
|
156
|
+
* prototype` is permitted, rightly) and paid for it in false NEGATIVES nobody counted. Swept over
|
|
157
|
+
* every disposition-bearing `status:` line in this repo's own planning/initiatives/ on 2026-08-04:
|
|
158
|
+
* **26 of 30 were permitted**, including four real recorded PM dispositions —
|
|
159
|
+
*
|
|
160
|
+
* status: "CLOSED — disposed PROMOTE by PM wawan 2026-07-28…"
|
|
161
|
+
* status: "CLOSED — PROMOTED 2026-07-28…"
|
|
162
|
+
* status: "…PM disposition 2026-07-29: ITERATE A…"
|
|
163
|
+
* status: "**PM disposition 2026-07-30: ITERATE C**"
|
|
164
|
+
*
|
|
165
|
+
* — because each leads with CLOSED, or with the word "PM". A guard validated against six test
|
|
166
|
+
* strings and never against the corpus it protects. So the leading-token rule stays, and these
|
|
167
|
+
* value-INTERNAL forms join it: a disposition announced as a disposition, wherever it sits in the
|
|
168
|
+
* line. Prose that merely mentions a verb still passes, because these require the announcing noun.
|
|
169
|
+
*/
|
|
170
|
+
const DISPOSITION_INTERNAL_RE = new RegExp(
|
|
171
|
+
'(?:' +
|
|
172
|
+
'dispositions?\\s*[:=]\\s*|' + // "disposition: PROMOTE", "PM disposition: ITERATE A"
|
|
173
|
+
'dispos(?:ed|ition)\\s+(?:to\\s+)?|' + // "disposed PROMOTE", "disposed to KILL"
|
|
174
|
+
'disposition\\s+\\S{0,24}?\\s*[:=]\\s*' + // "PM disposition 2026-07-30: ITERATE C"
|
|
175
|
+
')' +
|
|
176
|
+
'\\W{0,4}(PROMOTE(?:D)?|ITERATE(?:D)?|KILL(?:ED)?)\\b',
|
|
177
|
+
'i');
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* The third rule, rewritten after independent verification refuted its first form.
|
|
181
|
+
*
|
|
182
|
+
* The first attempt keyed on CASE: dispositions are written `PROMOTED`, prose writes "the promoted
|
|
183
|
+
* prototype". Swept over the corpus it looked perfect — 7 all-caps lines, all 7 genuine, zero false
|
|
184
|
+
* positives. An independent verifier then pointed out the obvious: **a driver forging a disposition
|
|
185
|
+
* defeats a case-sensitive guard by not pressing shift.**
|
|
186
|
+
*
|
|
187
|
+
* status: CLOSED — promoted 2026-07-28 by PM wawan prev=refused → mine=PERMITTED
|
|
188
|
+
*
|
|
189
|
+
* That is `workspace-doc-editing.md`'s real recorded disposition in lower case, and it was a strict
|
|
190
|
+
* regression against the module I was fixing — the same veto I wrote the rubric to catch, in the fix
|
|
191
|
+
* for the round that failed it. It also carried the SAME `KILLED?` typo this file documents itself
|
|
192
|
+
* catching twenty lines up ("KILLE" plus an optional D never matches a bare `KILL`).
|
|
193
|
+
*
|
|
194
|
+
* So case is not the signal. The signal is GRAMMAR: a disposition is the verb used as a verb, and
|
|
195
|
+
* the false positive this round must preserve is the verb used as an ADJECTIVE or in a PASSIVE
|
|
196
|
+
* clause — "derived from *the promoted* prototype", "M3 *was killed* off upstream". Both are marked
|
|
197
|
+
* by a small closed set of words immediately before the verb. Everything else is a disposition,
|
|
198
|
+
* whatever its case.
|
|
199
|
+
*
|
|
200
|
+
* Default refuse, carve out narrowly, and enumerate the carve-out — rather than default permit and
|
|
201
|
+
* enumerate the catches, which is the denylist shape this whole round exists to remove.
|
|
202
|
+
*/
|
|
203
|
+
const DISPOSITION_VERB = '(?:PROMOTE[SD]?|ITERATE[SD]?|KILL(?:ED|S)?)';
|
|
204
|
+
|
|
205
|
+
/** Immediately-preceding words that make the verb a modifier or a passive, not an act. */
|
|
206
|
+
const PROSE_LEAD = new Set([
|
|
207
|
+
'the', 'a', 'an', 'this', 'that', 'these', 'those', 'its', 'their', 'our', 'his', 'her',
|
|
208
|
+
'was', 'were', 'been', 'being', 'is', 'are', 'be', 'get', 'gets', 'got', 'gotten',
|
|
209
|
+
'not', 'never', 'unless', 'if', 'when', 'would', 'could', 'should', 'may', 'might',
|
|
210
|
+
]);
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* True when a status VALUE announces a PM disposition. Case-insensitive by construction; the
|
|
214
|
+
* carve-out, not the catch, is what is enumerated.
|
|
215
|
+
*/
|
|
216
|
+
function isDisposition(value) {
|
|
217
|
+
const v = String(value);
|
|
218
|
+
if (DISPOSITION_RE.test(v) || DISPOSITION_INTERNAL_RE.test(v)) return true;
|
|
219
|
+
const re = new RegExp(`(^|[^A-Za-z])(${DISPOSITION_VERB})\\b`, 'gi');
|
|
220
|
+
let m;
|
|
221
|
+
while ((m = re.exec(v)) !== null) {
|
|
222
|
+
const before = v.slice(0, m.index + m[1].length);
|
|
223
|
+
const prior = (before.match(/([A-Za-z']+)[^A-Za-z']*$/) || [])[1];
|
|
224
|
+
if (prior && PROSE_LEAD.has(prior.toLowerCase())) continue; // adjectival or passive — prose
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* NORMALIZE BEFORE MATCHING — and the reason this rule is written as a rule.
|
|
232
|
+
*
|
|
233
|
+
* The predecessor tested `/PM_approved\s*[:=]\s*true/i`: unbounded whitespace. Its replacement — the
|
|
234
|
+
* "hardening" — tested `/PM_approved\W{0,4}(true|1|yes)/i`, which widened the VALUE side and, without
|
|
235
|
+
* anyone noticing, capped the SEPARATOR side at four non-word characters. An independent evaluator
|
|
236
|
+
* measured both modules side by side on 2026-08-04:
|
|
237
|
+
*
|
|
238
|
+
* PM_approved : true old=refused new=PERMITTED
|
|
239
|
+
* PM_approved: true old=refused new=PERMITTED
|
|
240
|
+
*
|
|
241
|
+
* A hardening round that shipped a strict regression, on the one predicate whose whole job is to stop
|
|
242
|
+
* a driver forging the PM's approval. `{"pre_run_contract" : {"PM_approved" : true}}` written
|
|
243
|
+
* as text to the allowlisted agent-runs.jsonl JSON.parses to `PM_approved === true`, which
|
|
244
|
+
* scripts/workflow-state.js accepts as O5 satisfaction.
|
|
245
|
+
*
|
|
246
|
+
* The lesson is not "use a bigger budget". Any budget is a denylist of the gaps you thought of, and
|
|
247
|
+
* the next one is defeated by a form nobody enumerated (the double-stringified payload defeated four
|
|
248
|
+
* with five escape characters). So: collapse the input to a canonical form FIRST — drop the
|
|
249
|
+
* characters an encoding can multiply, squeeze whitespace — and only then match, with unbounded
|
|
250
|
+
* separators. There is no budget to overrun because there is nothing left to pad with.
|
|
251
|
+
*/
|
|
252
|
+
/** Returned when normalization will not settle — the call site treats it as refusal, never as text. */
|
|
253
|
+
const NON_CONVERGENT = '\u0000NON_CONVERGENT\u0000';
|
|
254
|
+
|
|
255
|
+
function normalizeForMatch(s) {
|
|
256
|
+
let out = String(s);
|
|
257
|
+
/* DECODE, do not delete. The first attempt at this stripped backslashes outright — and a literal
|
|
258
|
+
* tab, which JSON.stringify writes as the two characters `\t`, became the letter `t`:
|
|
259
|
+
*
|
|
260
|
+
* PM_approved<TAB>:<TAB>true → "PM_approved\t:\ttrue" → PM_approvedt:ttrue
|
|
261
|
+
*
|
|
262
|
+
* …which matches nothing, so a tab-separated approval walked straight through the "fix". Caught by
|
|
263
|
+
* this round's own N-R1 assertion, which is the argument for attempting each form individually.
|
|
264
|
+
* Escape sequences are DATA and must be decoded to what they denote, never erased.
|
|
265
|
+
*
|
|
266
|
+
* Applied repeatedly so that multiply-stringified payloads (the ninth bypass) unwrap layer by
|
|
267
|
+
* layer rather than one level deep. Bounded, because a fixed point is not guaranteed on hostile
|
|
268
|
+
* input and an unbounded loop here is a denial of service. */
|
|
269
|
+
/* ONE left-to-right pass per round, not four independent global replaces.
|
|
270
|
+
*
|
|
271
|
+
* The four-replace form was itself defective, and an independent verifier found it: because
|
|
272
|
+
* `JSON.stringify` DOUBLES backslashes, the `\\u....` replace consumed the second backslash and
|
|
273
|
+
* left the first dangling in front of the decoded letter, which the `\\[tnrfvb0]` replace then
|
|
274
|
+
* ATE as a control char — turning `PM_approved` into `PM_app o ed` and permitting the write. The
|
|
275
|
+
* same `\t` → `t` corruption as the first attempt, one rule to the right, because the rules could
|
|
276
|
+
* see each other's output mid-pass. A single scanner consumes each escape exactly once.
|
|
277
|
+
*
|
|
278
|
+
* Loops to a FIXED POINT rather than a fixed count. The earlier `i < 4` was a budget — and this
|
|
279
|
+
* file argues at length that a budget is a denylist of the gaps you thought of; five layers of
|
|
280
|
+
* stringification walked past it. The cap here is a non-termination guard, not a budget, and
|
|
281
|
+
* hitting it means the input is still changing after 16 rounds, which is not something a
|
|
282
|
+
* legitimate write does: it fails CLOSED at the call site. */
|
|
283
|
+
const decodeOnce = (s) => {
|
|
284
|
+
let r = '';
|
|
285
|
+
for (let i = 0; i < s.length; i++) {
|
|
286
|
+
if (s[i] !== '\\') { r += s[i]; continue; }
|
|
287
|
+
const n = s[i + 1];
|
|
288
|
+
if (n === undefined) { r += ' '; break; }
|
|
289
|
+
if (n === 'u' && /^[0-9a-fA-F]{4}$/.test(s.slice(i + 2, i + 6))) {
|
|
290
|
+
r += String.fromCharCode(parseInt(s.slice(i + 2, i + 6), 16)); i += 5;
|
|
291
|
+
} else if (n === 'x' && /^[0-9a-fA-F]{2}$/.test(s.slice(i + 2, i + 4))) {
|
|
292
|
+
r += String.fromCharCode(parseInt(s.slice(i + 2, i + 4), 16)); i += 3;
|
|
293
|
+
} else if ('tnrfvb0'.includes(n)) { r += ' '; i += 1; }
|
|
294
|
+
else { r += n; i += 1; }
|
|
295
|
+
}
|
|
296
|
+
return r;
|
|
297
|
+
};
|
|
298
|
+
let converged = false;
|
|
299
|
+
for (let i = 0; i < 16; i++) {
|
|
300
|
+
const before = out;
|
|
301
|
+
out = decodeOnce(out);
|
|
302
|
+
if (out === before) { converged = true; break; }
|
|
303
|
+
}
|
|
304
|
+
if (!converged) return NON_CONVERGENT;
|
|
305
|
+
return out
|
|
306
|
+
.replace(/["'`]/g, '') // quoting is packaging, not content
|
|
307
|
+
.replace(/\s+/g, ' '); // any run of whitespace is one space
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** Unbounded separators, by construction. Applied only to normalized text. */
|
|
311
|
+
const APPROVAL_RE = /PM_approved\s*[:=]\s*(true|1|yes)\b/i;
|
|
312
|
+
|
|
130
313
|
/** Truthy-tolerant: an approval the PM never gave is what is being refused; its JSON type is not the
|
|
131
314
|
* point. Closes bypass 2. */
|
|
132
315
|
const TRUTHY = new Set([true, 1, 'true', 'TRUE', 'True', 'yes', 'YES', '1']);
|
|
@@ -145,22 +328,53 @@ function carriesApproval(value, seen) {
|
|
|
145
328
|
return false;
|
|
146
329
|
}
|
|
147
330
|
|
|
148
|
-
/**
|
|
331
|
+
/**
|
|
332
|
+
* True when a `status`/`disposition` key ANYWHERE carries a disposition (bypass 5).
|
|
333
|
+
*
|
|
334
|
+
* Three type-loose gaps closed here, each measured on 2026-08-04: the key test was case-SENSITIVE
|
|
335
|
+
* (`Status` walked past), the value test required `typeof v === 'string'` (so `status: ['PROMOTED']`
|
|
336
|
+
* walked past), and a Buffer serialized to a byte array invisible to every scan.
|
|
337
|
+
*/
|
|
149
338
|
function carriesDisposition(value, seen) {
|
|
150
339
|
const s = seen || new Set();
|
|
151
340
|
if (!isObj(value) || s.has(value)) return false;
|
|
152
341
|
s.add(value);
|
|
153
342
|
for (const [k, v] of Object.entries(value)) {
|
|
154
|
-
|
|
343
|
+
const isStatusKey = k.toLowerCase() === 'status' || k.toLowerCase() === 'disposition';
|
|
344
|
+
if (isStatusKey) {
|
|
345
|
+
// A status value may be a string, a Buffer, or a list of candidate values. All are inspected.
|
|
346
|
+
const candidates = Array.isArray(v) ? v : [v];
|
|
347
|
+
for (const c of candidates) {
|
|
348
|
+
if (typeof c === 'string' && isDisposition(c)) return true;
|
|
349
|
+
if (Buffer.isBuffer(c) && isDisposition(c.toString('utf8'))) return true;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
if (Buffer.isBuffer(v)) continue; // a Buffer's numeric indices are not an object graph
|
|
155
353
|
if (carriesDisposition(v, s)) return true;
|
|
156
354
|
}
|
|
157
355
|
return false;
|
|
158
356
|
}
|
|
159
357
|
|
|
358
|
+
/**
|
|
359
|
+
* Repo-relative, canonical, or nothing.
|
|
360
|
+
*
|
|
361
|
+
* `path.posix.normalize` collapses `.` and `..` properly instead of the old substring test, which
|
|
362
|
+
* both over-refused (a legitimate `draft..v2.md`) and under-thought the problem. The backslash
|
|
363
|
+
* rewrite is gone: on POSIX a backslash is an ordinary filename character, and rewriting it let the
|
|
364
|
+
* single root-level file `planning\prd\x.md` pose as a file inside `planning/prd/`.
|
|
365
|
+
*/
|
|
160
366
|
function onAllowlist(file) {
|
|
161
|
-
const
|
|
162
|
-
if (
|
|
163
|
-
|
|
367
|
+
const raw = String(file);
|
|
368
|
+
if (path.posix.isAbsolute(raw) || path.win32.isAbsolute(raw)) return false; // never outside the repo
|
|
369
|
+
const norm = path.posix.normalize(raw.replace(/^\.\//, ''));
|
|
370
|
+
if (norm === '.' || norm.startsWith('../')) return false; // escaped the repo root
|
|
371
|
+
if (norm.split('/').includes('..')) return false;
|
|
372
|
+
if (ALLOWED_FILES.includes(norm)) return true;
|
|
373
|
+
/* The prefix half admits DRAFT ARTIFACTS, never ledgers. Without this the module's own
|
|
374
|
+
* fourth-ledger argument fails against its own second directory: `planning/evals/anything.jsonl`
|
|
375
|
+
* was permitted, so the next append-only ledger anyone puts there is admitted by default —
|
|
376
|
+
* "exactly the class-failure this rewrite exists to close", as the comment above says. */
|
|
377
|
+
if (/\.jsonl$/i.test(norm)) return false;
|
|
164
378
|
return ALLOWED_DIR_PREFIXES.some((p) => norm.startsWith(p));
|
|
165
379
|
}
|
|
166
380
|
|
|
@@ -179,6 +393,20 @@ function assertWritable(req) {
|
|
|
179
393
|
throw new ForbiddenWrite('a request with no target', 'a write request must name its `file` — refused (fail-closed)');
|
|
180
394
|
}
|
|
181
395
|
|
|
396
|
+
/* A Buffer or TypedArray supplied AS the payload is bytes the writer will put on disk verbatim,
|
|
397
|
+
* and JSON.stringify renders it `{"type":"Buffer","data":[80,77,…]}` — invisible to the text scan
|
|
398
|
+
* and to the object walk alike. Both were handled NESTED and neither at the top level, which an
|
|
399
|
+
* independent verifier walked straight through with
|
|
400
|
+
* `payload: Buffer.from(JSON.stringify({pre_run_contract:{PM_approved:true}}))`. Decode to text
|
|
401
|
+
* and adjudicate it as text. */
|
|
402
|
+
if (Buffer.isBuffer(req.payload) || ArrayBuffer.isView(req.payload)) {
|
|
403
|
+
const asText = Buffer.from(
|
|
404
|
+
ArrayBuffer.isView(req.payload) && !Buffer.isBuffer(req.payload)
|
|
405
|
+
? Buffer.from(req.payload.buffer, req.payload.byteOffset, req.payload.byteLength)
|
|
406
|
+
: req.payload).toString('utf8');
|
|
407
|
+
return assertWritable({ file: req.file, text: `${req.text || ''}\n${asText}` });
|
|
408
|
+
}
|
|
409
|
+
|
|
182
410
|
const why = HUMAN_DECISION_SINKS[path.basename(req.file)];
|
|
183
411
|
if (why) throw new ForbiddenWrite(path.basename(req.file), why);
|
|
184
412
|
|
|
@@ -197,7 +425,12 @@ function assertWritable(req) {
|
|
|
197
425
|
throw new ForbiddenWrite('an unserialisable payload',
|
|
198
426
|
'a payload that cannot be inspected cannot be permitted — refused (fail-closed)');
|
|
199
427
|
}
|
|
200
|
-
|
|
428
|
+
const normalized = normalizeForMatch(serialized);
|
|
429
|
+
if (normalized === NON_CONVERGENT) {
|
|
430
|
+
throw new ForbiddenWrite('an input that will not normalize',
|
|
431
|
+
'a payload whose escaping does not settle cannot be inspected — refused (fail-closed)');
|
|
432
|
+
}
|
|
433
|
+
if (APPROVAL_RE.test(normalized) || carriesApproval(req.payload)) {
|
|
201
434
|
throw new ForbiddenWrite('pre_run_contract.PM_approved',
|
|
202
435
|
'the contract is the PM\'s to approve before any build (D16, AGENTS.md §4)');
|
|
203
436
|
}
|
|
@@ -209,11 +442,34 @@ function assertWritable(req) {
|
|
|
209
442
|
throw new ForbiddenWrite('a disposition', 'promote/iterate/kill is the PM\'s (D60 step 7)');
|
|
210
443
|
}
|
|
211
444
|
if (typeof req.text === 'string') {
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
445
|
+
// Split on /\r?\n/, not '\n'. The old split left a trailing '\r' on every CRLF line, and the
|
|
446
|
+
// matcher's `$` anchor (without the `m` flag) then matched nothing at all — a CRLF document
|
|
447
|
+
// sailed through the raw-text check entirely.
|
|
448
|
+
for (const line of req.text.split(/\r?\n/)) {
|
|
449
|
+
/* Markdown prefixes are how a status line is ACTUALLY written in the driver's own sinks —
|
|
450
|
+
* `- status:`, `**status:**`, `> status:`, `| status |` — and the old anchor allowed only
|
|
451
|
+
* leading whitespace, so every one of them walked past the check. */
|
|
452
|
+
const m = /^[\s>*\-+#|]*\**\s*(?:status|disposition)\**\s*[:|]\s*(.*?)\s*\|?\s*$/i.exec(line);
|
|
453
|
+
if (m && isDisposition(m[1])) {
|
|
215
454
|
throw new ForbiddenWrite('a disposition', 'promote/iterate/kill is the PM\'s (D60 step 7)');
|
|
216
455
|
}
|
|
456
|
+
// The raw and structured paths must agree. A JSONL line supplied as `text` was permitted while
|
|
457
|
+
// the identical object supplied as `payload` was refused — the same content, two verdicts,
|
|
458
|
+
// decided by which field it arrived in.
|
|
459
|
+
const t = line.trim();
|
|
460
|
+
if (t.startsWith('{') || t.startsWith('[')) {
|
|
461
|
+
let parsed;
|
|
462
|
+
try { parsed = JSON.parse(t); } catch { parsed = undefined; }
|
|
463
|
+
if (parsed !== undefined) {
|
|
464
|
+
if (carriesDisposition(parsed)) {
|
|
465
|
+
throw new ForbiddenWrite('a disposition', 'promote/iterate/kill is the PM\'s (D60 step 7)');
|
|
466
|
+
}
|
|
467
|
+
if (carriesApproval(parsed)) {
|
|
468
|
+
throw new ForbiddenWrite('pre_run_contract.PM_approved',
|
|
469
|
+
'the contract is the PM\'s to approve before any build (D16, AGENTS.md §4)');
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
}
|
|
217
473
|
}
|
|
218
474
|
}
|
|
219
475
|
return true;
|
|
@@ -859,14 +859,48 @@ function computeWorkflow(id, opts) {
|
|
|
859
859
|
// ── review ─ the Review Gate's verdict, where the lane has a rubric ──────────────────────────
|
|
860
860
|
const evals = readJsonl(path.join(L.runsDir, 'evals.jsonl')).filter((e) => e && e.initiative_id === id);
|
|
861
861
|
{
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
862
|
+
/* The rung reads the RUBRIC, not the lane.
|
|
863
|
+
*
|
|
864
|
+
* It keyed on `lane` alone, so a chore that deliberately carries a rubric could never surface
|
|
865
|
+
* its Review Gate — the ladder said "chore lane carries no rubric (D56)" while a SKIPPED verdict
|
|
866
|
+
* sat in evals.jsonl unread. That is how `guard-and-ledger-hardening`, a chore whose own record
|
|
867
|
+
* argues "blast radius overrides lane" and which carried a rubric on purpose, reached `accept`
|
|
868
|
+
* with the gate never run. The board could not show what the ledger already knew.
|
|
869
|
+
*
|
|
870
|
+
* A prototype is still never scored — D60 grills leg A as dialogue, and that is a property of
|
|
871
|
+
* the lane, not of whether someone attached a rubric. Every other lane defers to the artifact. */
|
|
872
|
+
const declaresRubric = !!String(fm.rubric_path || '').trim();
|
|
873
|
+
/* EVERY lane defers to the artifact — the first version said so in the comment and applied it
|
|
874
|
+
* only to `chore`, so a discovery or incident initiative with no rubric (legitimate under D56)
|
|
875
|
+
* showed a permanent `no Review-Gate verdict recorded`: a rung it could never satisfy. The lanes
|
|
876
|
+
* that carry no rubric by design are named once, here. */
|
|
877
|
+
const RUBRIC_OPTIONAL = ['chore', 'discovery', 'incident', 'content'];
|
|
878
|
+
if (effLane === 'prototype') {
|
|
879
|
+
steps.push(step('review', 'review', 'na', 'leg A is grilled as dialogue, never scored (D60)'));
|
|
880
|
+
} else if (RUBRIC_OPTIONAL.includes(effLane) && !declaresRubric && !evals.length) {
|
|
881
|
+
steps.push(step('review', 'review', 'na', `${effLane} lane carries no rubric (D56)`));
|
|
866
882
|
} else if (evals.length) {
|
|
867
883
|
const last = evals[evals.length - 1];
|
|
868
|
-
|
|
869
|
-
|
|
884
|
+
const verdict = String(last.verdict || '').toUpperCase();
|
|
885
|
+
const label = `${last.verdict}${last.score != null ? ` (${last.score})` : ''} — advisory until calibrated per evaluator model (D40)`;
|
|
886
|
+
/* A SKIPPED gate is NOT a satisfied gate, and a FAIL is emphatically not one.
|
|
887
|
+
*
|
|
888
|
+
* This rung rendered both as a green ✓, which is the watermelon this repo names as its own
|
|
889
|
+
* failure mode: the board said the Review Gate was done over a row that says judging never
|
|
890
|
+
* happened. Thirteen initiatives currently render `✓ review` over a SKIPPED verdict, and the
|
|
891
|
+
* fix that made a chore's gate visible ADDED one. The module's own rule, repeated in three
|
|
892
|
+
* files — "a skip is an honest RECORD that verification did not happen; it is not
|
|
893
|
+
* verification" — now applies to the surface that reports it. */
|
|
894
|
+
if (verdict === 'SKIPPED') {
|
|
895
|
+
steps.push(step('review', 'review', 'pending', label,
|
|
896
|
+
'the gate was skipped, so it is still owed — commission an independent evaluator and record a real verdict',
|
|
897
|
+
{ rule: 'R3' }));
|
|
898
|
+
} else if (verdict === 'FAIL') {
|
|
899
|
+
steps.push(step('review', 'review', 'blocked', label,
|
|
900
|
+
'the Review Gate FAILED — remediate the findings and record a new verdict', { rule: 'R3' }));
|
|
901
|
+
} else {
|
|
902
|
+
steps.push(step('review', 'review', 'done', label));
|
|
903
|
+
}
|
|
870
904
|
} else {
|
|
871
905
|
steps.push(step('review', 'review', 'pending', 'no Review-Gate verdict recorded',
|
|
872
906
|
'score the build against the rubric (build-skills/evaluator.skill), then append to evals.jsonl',
|
package/workspace/server.js
CHANGED
|
@@ -830,9 +830,20 @@ function searchIndex(root) {
|
|
|
830
830
|
const hit = searchIndexCache.get(root);
|
|
831
831
|
if (hit && now - hit.at < SEARCH_INDEX_TTL_MS) return hit;
|
|
832
832
|
const files = [];
|
|
833
|
-
let
|
|
833
|
+
let depthCut = false; // some branch was too deep — a LOCAL fact
|
|
834
|
+
let budgetCut = false; // the file budget ran out — a GLOBAL stop
|
|
835
|
+
/* Two truncation causes, two flags, and only ONE of them stops the walk.
|
|
836
|
+
*
|
|
837
|
+
* They used to share `truncated`, and the guard at the top of `walk` read it — so the first branch
|
|
838
|
+
* that hit SEARCH_MAX_DEPTH aborted every not-yet-visited subdirectory in the whole tree. Measured
|
|
839
|
+
* by an independent evaluator on 2026-08-04: 51 files at depth <= 1, `scanned: 50`, an entire
|
|
840
|
+
* in-bound sibling directory lost because an unrelated deep branch tripped the cap first — and
|
|
841
|
+
* which sibling survived depended on readdir order.
|
|
842
|
+
*
|
|
843
|
+
* Depth is a bound on ONE path. The file budget is a bound on the WHOLE walk. Only the second is
|
|
844
|
+
* a reason to stop walking. */
|
|
834
845
|
(function walk(dir, rel, depth) {
|
|
835
|
-
if (
|
|
846
|
+
if (budgetCut) return;
|
|
836
847
|
let ents = [];
|
|
837
848
|
try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
838
849
|
for (const e of ents) {
|
|
@@ -840,16 +851,17 @@ function searchIndex(root) {
|
|
|
840
851
|
const full = path.join(dir, e.name);
|
|
841
852
|
const r = rel ? rel + '/' + e.name : e.name;
|
|
842
853
|
if (e.isDirectory()) {
|
|
843
|
-
// Depth is a bound like any other: reaching it means files below were NOT looked at
|
|
854
|
+
// Depth is a bound like any other: reaching it means files below were NOT looked at —
|
|
855
|
+
// but only BELOW HERE. Siblings elsewhere in the tree are still in bounds.
|
|
844
856
|
if (depth < SEARCH_MAX_DEPTH) walk(full, r, depth + 1);
|
|
845
|
-
else
|
|
857
|
+
else depthCut = true;
|
|
846
858
|
} else if (e.isFile()) {
|
|
847
|
-
if (files.length >= SEARCH_MAX_FILES) {
|
|
859
|
+
if (files.length >= SEARCH_MAX_FILES) { budgetCut = true; return; }
|
|
848
860
|
files.push({ path: r, name: e.name, dir: rel, kind: preview.previewKind(e.name) });
|
|
849
861
|
}
|
|
850
862
|
}
|
|
851
863
|
})(root, '', 0);
|
|
852
|
-
const entry = { at: now, files, truncated, scanned: files.length };
|
|
864
|
+
const entry = { at: now, files, truncated: depthCut || budgetCut, scanned: files.length };
|
|
853
865
|
searchIndexCache.set(root, entry);
|
|
854
866
|
return entry;
|
|
855
867
|
}
|
|
@@ -882,15 +894,51 @@ function searchWorkspace(root, query, limit) {
|
|
|
882
894
|
const seen = new Set();
|
|
883
895
|
let overflow = false; // more matched than `cap` — a REAL cut, not a coincidence
|
|
884
896
|
|
|
897
|
+
/* `openable` means ONE thing: /api/doc will serve this. So it asks /api/doc's own reader rather
|
|
898
|
+
* than re-deciding from the extension.
|
|
899
|
+
*
|
|
900
|
+
* It used to be `SEARCH_OPENABLE.has(f.kind)` — extension only — while `readTextDoc` applies a
|
|
901
|
+
* SECOND refusal the search never replicated: a NUL byte anywhere in the file. A `.txt` holding
|
|
902
|
+
* binary therefore rendered as a clickable hit that opened a tab printing a read failure, which
|
|
903
|
+
* is the PRD's own constraint 6 violated ("the result says so rather than letting the PM click
|
|
904
|
+
* into a predictable failure"). An independent evaluator reproduced it on 2026-08-04:
|
|
905
|
+
*
|
|
906
|
+
* hit masquerade.txt kind=text openable=true -> /api/doc status=404
|
|
907
|
+
*
|
|
908
|
+
* Two notions of "servable" that can disagree is the defect; one definition, consulted, is the
|
|
909
|
+
* fix. Bounded by `cap` (<= 200 hits), and only for path matches — a content match already read
|
|
910
|
+
* the body and proved it NUL-free. */
|
|
911
|
+
const servable = (rel) => {
|
|
912
|
+
try {
|
|
913
|
+
/* SIZE-BOUNDED. `readTextDoc` has no size limit, so consulting it for every path hit read up
|
|
914
|
+
* to `cap` (200) WHOLE files on the server's only thread — measured by an independent verifier
|
|
915
|
+
* at 285 ms for 200 × 1 MB, and 5.7 s plus a 600 MB read for one oversized file. The comment
|
|
916
|
+
* twelve lines above SEARCH_CONTENT_BYTES says a bigger file is "path-matched only, never read
|
|
917
|
+
* whole"; the fix quietly broke that. A file past the content budget reports NOT openable —
|
|
918
|
+
* the honest answer is "we did not look", and the safe rendering of that is a hit the PM
|
|
919
|
+
* cannot click into an unknown. */
|
|
920
|
+
const st = fs.statSync(path.join(root, rel));
|
|
921
|
+
if (!st.isFile() || st.size > SEARCH_CONTENT_BYTES) return false;
|
|
922
|
+
return readTextDoc(root, rel) !== null;
|
|
923
|
+
} catch { return false; }
|
|
924
|
+
};
|
|
925
|
+
|
|
885
926
|
for (const f of idx.files) {
|
|
886
927
|
if (!f.path.toLowerCase().includes(needle)) continue;
|
|
887
928
|
seen.add(f.path);
|
|
888
929
|
if (hits.length >= cap) { overflow = true; break; }
|
|
889
|
-
hits.push({ ...f, match: 'path', line: null, snippet: null,
|
|
930
|
+
hits.push({ ...f, match: 'path', line: null, snippet: null,
|
|
931
|
+
openable: SEARCH_OPENABLE.has(f.kind) && servable(f.path) });
|
|
890
932
|
}
|
|
891
933
|
|
|
892
934
|
let read = 0;
|
|
893
935
|
let contentCut = false;
|
|
936
|
+
/* When path matches overflow the cap, the ENTIRE content pass is skipped — so every body in the
|
|
937
|
+
* workspace went unread. `content_truncated` reported `false` alongside `content_scanned: 0`,
|
|
938
|
+
* contradicting the flag's own documented meaning ("every file was named, but not every body was
|
|
939
|
+
* read"). Masked by `truncated: true`, so the PM was warned — but a flag that disagrees with
|
|
940
|
+
* reality is the failure the two-flag split exists to prevent. */
|
|
941
|
+
if (overflow) contentCut = true;
|
|
894
942
|
if (!overflow) {
|
|
895
943
|
for (const f of idx.files) {
|
|
896
944
|
if (seen.has(f.path) || !SEARCH_OPENABLE.has(f.kind)) continue;
|
|
@@ -906,7 +954,9 @@ function searchWorkspace(root, query, limit) {
|
|
|
906
954
|
if (text.indexOf(String.fromCharCode(0)) !== -1) continue; // binary masquerading as text — never scanned
|
|
907
955
|
const at = text.toLowerCase().indexOf(needle);
|
|
908
956
|
if (at === -1) continue;
|
|
909
|
-
|
|
957
|
+
// The cap biting HERE also leaves bodies unread — the sibling branch of the same defect, and
|
|
958
|
+
// the half the first fix missed.
|
|
959
|
+
if (hits.length >= cap) { overflow = true; contentCut = true; break; }
|
|
910
960
|
const before = text.slice(0, at);
|
|
911
961
|
const lineNo = before.split('\n').length;
|
|
912
962
|
const lineStart = before.lastIndexOf('\n') + 1;
|