@tduyng/pi-devos 0.13.6 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +26 -33
- package/package.json +1 -2
- package/plugins/devos/agents/verifier.md +4 -30
- package/plugins/devos/pi/extension/devos.ts +16 -26
- package/plugins/devos/scripts/check-evidence-router.sh +7 -1
- package/plugins/devos/scripts/evidence-router.py +5 -3
- package/plugins/devos/scripts/gates/check-dod-core.sh +5 -3
- package/plugins/devos/scripts/gates/check-port-source.sh +1 -1
- package/plugins/devos/scripts/gates/check-scenario-coverage.sh +10 -8
- package/plugins/devos/scripts/gates/phase-gate.sh +6 -6
- package/plugins/devos/scripts/gates/plan-template-check.py +6 -12
- package/plugins/devos/scripts/gates/spec-contract.sh +19 -0
- package/plugins/devos/scripts/gates/spec-template-check.py +8 -21
- package/plugins/devos/scripts/gates/suite/01-dod.sh +485 -0
- package/plugins/devos/scripts/gates/suite/02-integrity.sh +310 -0
- package/plugins/devos/scripts/gates/suite/03-archetype.sh +255 -0
- package/plugins/devos/scripts/gates/suite/04-spec-validator.sh +183 -0
- package/plugins/devos/scripts/gates/suite/05-requires.sh +360 -0
- package/plugins/devos/scripts/gates/suite/06-clarify-adapters.sh +435 -0
- package/plugins/devos/scripts/gates/suite/07-lint-evals.sh +350 -0
- package/plugins/devos/scripts/gates/suite/08-converge.sh +591 -0
- package/plugins/devos/scripts/gates/suite/09-quality.sh +516 -0
- package/plugins/devos/scripts/gates/suite/10-eval-harness.sh +679 -0
- package/plugins/devos/scripts/gates/test-gates.sh +23 -4253
- package/plugins/devos/scripts/gates/test-integrity-rubric.sh +1 -1
- package/plugins/devos/scripts/gates/test-phase-gate.sh +5 -5
- package/plugins/devos/skills/build/SKILL.md +3 -3
- package/plugins/devos/skills/build/references/refactor.md +1 -1
- package/plugins/devos/skills/build/references/routes.md +4 -4
- package/plugins/devos/skills/frame/SKILL.md +22 -39
- package/plugins/devos/skills/frame/references/template.md +115 -0
- package/plugins/devos/skills/implement/SKILL.md +4 -4
- package/plugins/devos/skills/implement/references/dod.md +6 -1
- package/plugins/devos/skills/implement/references/implement.md +3 -1
- package/plugins/devos/skills/implement/references/plan.md +9 -27
- package/plugins/devos/skills/implement/references/resume.md +1 -1
- package/plugins/devos/skills/implement/references/verify-close.md +4 -3
- package/plugins/devos/skills/implement/scripts/ops-log.sh +2 -2
- package/plugins/devos/skills/implement/scripts/validate-spec.sh +17 -65
- package/plugins/devos/skills/init/references/context.md +1 -1
- package/plugins/devos/skills/review/SKILL.md +24 -47
- package/plugins/devos/skills/review/references/comments.md +13 -0
- package/plugins/devos/skills/review/references/reviewer.md +124 -0
- package/plugins/devos/skills/review/references/routing.md +35 -0
- package/plugins/devos/skills/review/scripts/__pycache__/prepare.cpython-313.pyc +0 -0
- package/plugins/devos/skills/review/scripts/prepare.py +763 -0
- package/plugins/devos/skills/spec/SKILL.md +5 -6
- package/plugins/devos/skills/spec/references/write-spec.md +36 -51
- package/plugins/devos/skills/spec/scripts/clarify.sh +4 -4
- package/plugins/devos/skills/status/SKILL.md +4 -4
- package/plugins/devos/skills/status/scripts/check-requires.py +4 -4
- package/plugins/devos/skills/status/scripts/convention-usage.sh +1 -1
- package/plugins/devos/skills/status/scripts/quality-correlation.sh +2 -25
- package/plugins/devos/scripts/gates/check-handoff.sh +0 -215
- package/plugins/devos/skills/design-taste/SKILL.md +0 -55
- package/plugins/devos/skills/design-taste/references/tells.md +0 -85
- package/plugins/devos/skills/design-taste/scripts/slop-scan.py +0 -305
- package/plugins/devos/skills/docs/SKILL.md +0 -22
- package/plugins/devos/skills/docs/references/backfill.md +0 -54
- package/plugins/devos/skills/docs/references/init.md +0 -43
- package/plugins/devos/skills/docs/references/status.md +0 -71
- package/plugins/devos/skills/docs/references/update.md +0 -80
- package/plugins/devos/skills/explain/SKILL.md +0 -45
- package/plugins/devos/skills/explain/references/quiz.md +0 -21
- package/plugins/devos/skills/frame/references/boundaries.md +0 -53
- package/plugins/devos/skills/frame/references/decompose.md +0 -96
- package/plugins/devos/skills/frame/references/impact.md +0 -43
- package/plugins/devos/skills/frame/references/interrogate.md +0 -68
- package/plugins/devos/skills/product/SKILL.md +0 -40
- package/plugins/devos/skills/product/references/planner.md +0 -54
- package/plugins/devos/skills/surgical-diff/SKILL.md +0 -23
- package/plugins/devos/skills/test-integrity/SKILL.md +0 -26
- package/plugins/devos/skills/verification-evidence/SKILL.md +0 -17
- package/plugins/devos/workflows/common/backlog-file-structure.md +0 -44
- package/plugins/devos/workflows/common/category-file-structure.md +0 -41
- package/plugins/devos/workflows/common/feature-line-format.md +0 -10
- package/plugins/devos/workflows/common/roadmap-file-structure.md +0 -51
- package/plugins/devos/workflows/common/suggest-categories.md +0 -76
- package/plugins/devos/workflows/common/suggest-milestones.md +0 -92
- package/plugins/devos/workflows/doc/doc-health.md +0 -137
- package/plugins/devos/workflows/doc/doc-writer.md +0 -44
- package/plugins/devos/workflows/doc/init-documentation.md +0 -196
- package/plugins/devos/workflows/doc/retro-documentation.md +0 -304
- package/plugins/devos/workflows/doc/templates/library.md +0 -58
- package/plugins/devos/workflows/doc/templates/service.md +0 -61
- package/plugins/devos/workflows/doc/update-documentation.md +0 -194
- package/plugins/devos/workflows/doc/write-documentation.md +0 -137
- package/plugins/devos/workflows/product/create-product-mission.md +0 -45
- package/plugins/devos/workflows/product/create-product-roadmap.md +0 -106
- package/plugins/devos/workflows/product/create-product-tech-stack.md +0 -19
- package/plugins/devos/workflows/product/gather-product-info.md +0 -25
- package/plugins/devos/workflows/product/resume-detection.md +0 -70
- package/plugins/devos/workflows/product/reverse-backlog.md +0 -118
- package/plugins/devos/workflows/product/reverse-domain-model.md +0 -86
- package/plugins/devos/workflows/product/reverse-product-plan.md +0 -81
- package/plugins/devos/workflows/product/reverse-tech-stack.md +0 -73
- package/plugins/devos/workflows/product/scan-deep.md +0 -113
- package/plugins/devos/workflows/product/scan-git.md +0 -122
- package/plugins/devos/workflows/product/scan-light.md +0 -154
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
# --- clarify.sh ----------------------------------------
|
|
2
|
+
CLARIFY="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/spec/scripts/clarify.sh"
|
|
3
|
+
FIXTURES="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/fixtures"
|
|
4
|
+
|
|
5
|
+
dirty="$tmp/devos/specs/2026-01-01-dirty"
|
|
6
|
+
mkdir -p "$dirty/planning"
|
|
7
|
+
cp "$FIXTURES/clarify-dirty/spec.md" "$dirty/spec.md"
|
|
8
|
+
cp "$FIXTURES/clarify-dirty/planning/notes.md" "$dirty/planning/notes.md"
|
|
9
|
+
out=$(bash "$CLARIFY" "$dirty")
|
|
10
|
+
[[ "$out" == *"Vague quantifier"* ]]; check "clarify: vague quantifier detected" 0 $?
|
|
11
|
+
[[ "$out" == *"3 clarification(s) found, 3 suppressed"* ]]; check "clarify: cap at 3, rest suppressed" 0 $?
|
|
12
|
+
grep -q "^## Clarifications Needed" "$dirty/spec.md"; check "clarify: section appended to dirty spec" 0 $?
|
|
13
|
+
|
|
14
|
+
# The appended blocks quote the lines that triggered them, so a rerun used to
|
|
15
|
+
# read its own output back as fresh ambiguity: the same spec reported 3 found /
|
|
16
|
+
# 3 suppressed once and 3 found / 6 suppressed after.
|
|
17
|
+
out=$(bash "$CLARIFY" "$dirty" | head -1)
|
|
18
|
+
[[ "$out" == "clarify: 3 clarification(s) found, 3 suppressed (cap: 3)" ]]; check "clarify: a rerun does not count its own output as ambiguity" 0 $?
|
|
19
|
+
out=$(bash "$CLARIFY" "$dirty" | head -1)
|
|
20
|
+
[[ "$out" == "clarify: 3 clarification(s) found, 3 suppressed (cap: 3)" ]]; check "clarify: repeated reruns stay stable" 0 $?
|
|
21
|
+
[[ "$(grep -c '^## Clarifications Needed' "$dirty/spec.md")" == 1 ]]; check "clarify: reruns do not append a second section" 0 $?
|
|
22
|
+
|
|
23
|
+
clean="$tmp/devos/specs/2026-01-02-clean"
|
|
24
|
+
mkdir -p "$clean"
|
|
25
|
+
cp "$FIXTURES/clarify-clean/spec.md" "$clean/spec.md"
|
|
26
|
+
out=$(bash "$CLARIFY" "$clean")
|
|
27
|
+
[[ "$out" == "clarify: 0 clarification(s) found, 0 suppressed (cap: 3)" ]]; check "clarify: zero false positives on clean spec" 0 $?
|
|
28
|
+
! grep -q "^## Clarifications Needed" "$clean/spec.md"; check "clarify: no section appended to clean spec" 0 $?
|
|
29
|
+
|
|
30
|
+
answered="$tmp/devos/specs/2026-01-03-answered"
|
|
31
|
+
mkdir -p "$answered/planning"
|
|
32
|
+
printf '%s\n' '- Option A: Use CSV format' '- Option B: Use JSON format' \
|
|
33
|
+
> "$answered/planning/notes.md"
|
|
34
|
+
cat > "$answered/spec.md" <<'MD'
|
|
35
|
+
# Answered spec
|
|
36
|
+
|
|
37
|
+
## 1. Goal
|
|
38
|
+
Users can export data.
|
|
39
|
+
|
|
40
|
+
## 2. User Stories
|
|
41
|
+
### US-1: Export data
|
|
42
|
+
As a user, I want to export data.
|
|
43
|
+
|
|
44
|
+
**Acceptance criteria:**
|
|
45
|
+
- [ ] Export completes within 1 second
|
|
46
|
+
|
|
47
|
+
## 3. Functionality
|
|
48
|
+
**Business rules:**
|
|
49
|
+
- Export must use CSV
|
|
50
|
+
|
|
51
|
+
**Answer: A**
|
|
52
|
+
|
|
53
|
+
## Out of Scope
|
|
54
|
+
- Import
|
|
55
|
+
MD
|
|
56
|
+
out=$(bash "$CLARIFY" "$answered")
|
|
57
|
+
[[ "$out" == "clarify: 0 clarification(s) found, 0 suppressed (cap: 3)" ]]; check "clarify: explicit Answer resolves a planning fork" 0 $?
|
|
58
|
+
! grep -q "Unresolved fork" <<<"$out"; check "clarify: answered fork is not reported" 0 $?
|
|
59
|
+
|
|
60
|
+
note_answered="$tmp/devos/specs/2026-01-04-note-answered"
|
|
61
|
+
mkdir -p "$note_answered/planning"
|
|
62
|
+
printf '%s\n' '- Option A: Use CSV format' '- Option B: Use JSON format' '**Answer: A**' \
|
|
63
|
+
> "$note_answered/planning/notes.md"
|
|
64
|
+
cat > "$note_answered/spec.md" <<'MD'
|
|
65
|
+
# Planning-note answer
|
|
66
|
+
|
|
67
|
+
## 1. Goal
|
|
68
|
+
Users can export data.
|
|
69
|
+
|
|
70
|
+
## 2. User Stories
|
|
71
|
+
### US-1: Export data
|
|
72
|
+
As a user, I want to export data.
|
|
73
|
+
|
|
74
|
+
**Acceptance criteria:**
|
|
75
|
+
- [ ] Export completes within 1 second
|
|
76
|
+
|
|
77
|
+
## 3. Functionality
|
|
78
|
+
**Business rules:**
|
|
79
|
+
- Export must use CSV
|
|
80
|
+
|
|
81
|
+
## Out of Scope
|
|
82
|
+
- Import
|
|
83
|
+
MD
|
|
84
|
+
out=$(bash "$CLARIFY" "$note_answered")
|
|
85
|
+
[[ "$out" == "clarify: 0 clarification(s) found, 0 suppressed (cap: 3)" ]]; check "clarify: planning-note Answer resolves a planning fork" 0 $?
|
|
86
|
+
! grep -q "Unresolved fork" <<<"$out"; check "clarify: planning-note answered fork is not reported" 0 $?
|
|
87
|
+
|
|
88
|
+
# --- doctor.sh: stale spec workspaces ---------------------------------------
|
|
89
|
+
DOCTOR="$(cd "$(dirname "$DOD")/../../../.." && pwd)/scripts/doctor.sh"
|
|
90
|
+
|
|
91
|
+
repo="$tmp/wtrepo"
|
|
92
|
+
git init -q -b main "$repo"
|
|
93
|
+
mkdir -p "$repo/devos/specs/2026-01-01-merged" "$repo/devos/specs/2026-01-01-open"
|
|
94
|
+
printf "# s\n" > "$repo/devos/specs/2026-01-01-merged/spec.md"
|
|
95
|
+
printf "# s\n" > "$repo/devos/specs/2026-01-01-open/spec.md"
|
|
96
|
+
git -C "$repo" -c user.email=t@t -c user.name=t add -A
|
|
97
|
+
git -C "$repo" -c user.email=t@t -c user.name=t commit -q -m init
|
|
98
|
+
git -C "$repo" branch 2026-01-01-merged
|
|
99
|
+
git -C "$repo" worktree add -q "$tmp/wt-merged" 2026-01-01-merged
|
|
100
|
+
git -C "$repo" branch 2026-01-01-open
|
|
101
|
+
git -C "$repo" worktree add -q "$tmp/wt-open" 2026-01-01-open
|
|
102
|
+
git -C "$tmp/wt-open" -c user.email=t@t -c user.name=t commit -q --allow-empty -m "unmerged work"
|
|
103
|
+
|
|
104
|
+
out=$(cd "$repo" && HOME="$tmp" bash "$DOCTOR" 2>/dev/null)
|
|
105
|
+
[[ "$out" == *"STALE worktree"*"2026-01-01-merged"* ]]; check "doctor: merged-branch worktree flagged" 0 $?
|
|
106
|
+
[[ "$out" != *"2026-01-01-open is fully merged"* ]]; check "doctor: unmerged worktree not flagged" 0 $?
|
|
107
|
+
|
|
108
|
+
dneg="$tmp/doctor-negated"
|
|
109
|
+
mkdir -p "$dneg/home" "$dneg/devos/specs/2026-01-01-active/verifications"
|
|
110
|
+
printf 'Completeness: NOT CLEAN\nQuality: NOT PASS\n' \
|
|
111
|
+
> "$dneg/devos/specs/2026-01-01-active/verifications/verification.md"
|
|
112
|
+
out=$(cd "$dneg" && HOME="$dneg/home" bash "$DOCTOR" 2>&1)
|
|
113
|
+
if grep -Fq 'complete but still active' <<<"$out"; then
|
|
114
|
+
check "doctor: negated verdict is not reported complete" 1 0
|
|
115
|
+
else
|
|
116
|
+
check "doctor: negated verdict is not reported complete" 0 0
|
|
117
|
+
fi
|
|
118
|
+
|
|
119
|
+
# doctor.sh: same version string, different commit. `claude plugin install`
|
|
120
|
+
# copies a snapshot, so editing the clone without bumping the version leaves
|
|
121
|
+
# Claude Code running the old files while a version-only check reports OK. That
|
|
122
|
+
# is the silent case, and it is the one that makes a fixed skill look unfixed.
|
|
123
|
+
DREPO="$(dirname "$(dirname "$DOCTOR")")"
|
|
124
|
+
dver=$(python3 -c "import json;print(json.load(open('$DREPO/plugins/devos/.claude-plugin/plugin.json'))['version'])")
|
|
125
|
+
fakereg() { # fakereg <sha>
|
|
126
|
+
mkdir -p "$tmp/dhome/.claude/plugins"
|
|
127
|
+
printf '{"version":2,"plugins":{"devos@devos":[{"version":"%s","gitCommitSha":"%s"}]}}\n' \
|
|
128
|
+
"$dver" "$1" > "$tmp/dhome/.claude/plugins/installed_plugins.json"
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
fakereg "$(git -C "$DREPO" rev-parse HEAD~1 2>/dev/null || echo deadbee)"
|
|
132
|
+
out=$(cd "$DREPO" && HOME="$tmp/dhome" bash "$DOCTOR" 2>/dev/null)
|
|
133
|
+
grep -q "DRIFT devos: both at $dver" <<<"$out"
|
|
134
|
+
check "doctor: a stale snapshot at the same version is drift" 0 $?
|
|
135
|
+
|
|
136
|
+
fakereg "$(git -C "$DREPO" rev-parse HEAD)"
|
|
137
|
+
out=$(cd "$DREPO" && HOME="$tmp/dhome" bash "$DOCTOR" 2>/dev/null)
|
|
138
|
+
grep -q "OK devos: clone and installed snapshot both at $dver" <<<"$out"
|
|
139
|
+
check "doctor: snapshot at the clone's own commit is not drift" 0 $?
|
|
140
|
+
|
|
141
|
+
# doctor.sh: broken Requires: edge
|
|
142
|
+
reqdoc="$tmp/reqdoc"
|
|
143
|
+
mkdir -p "$reqdoc/devos/specs/2026-01-04-broken"
|
|
144
|
+
printf '# Broken\nRequires: [ghost-spec]\n' > "$reqdoc/devos/specs/2026-01-04-broken/spec.md"
|
|
145
|
+
out=$(cd "$reqdoc" && HOME="$tmp" bash "$DOCTOR" 2>/dev/null || true)
|
|
146
|
+
[[ "$out" == *"BROKEN Requires: ghost-spec"* ]]
|
|
147
|
+
check "doctor: surfaces broken Requires edge" 0 $?
|
|
148
|
+
|
|
149
|
+
# Pi extension installed state: dangling symlink fails loudly; a settings.json
|
|
150
|
+
# registration resolving into the clone passes clean.
|
|
151
|
+
dangle="$tmp/pi-dangle"
|
|
152
|
+
mkdir -p "$dangle/.pi/extensions"
|
|
153
|
+
ln -s "$dangle/nowhere/devos.ts" "$dangle/.pi/extensions/devos.ts"
|
|
154
|
+
# Project scope resolves .pi/extensions relative to the project cwd, so the
|
|
155
|
+
# probe must run from inside the dangle project to see its broken link.
|
|
156
|
+
out=$(cd "$dangle" && HOME="$dangle" bash "$DOCTOR" 2>/dev/null || true)
|
|
157
|
+
grep -q "BROKEN pi extension symlink" <<<"$out"; check "doctor: dangling pi extension symlink caught" 0 $?
|
|
158
|
+
|
|
159
|
+
reg="$tmp/pi-reg"
|
|
160
|
+
mkdir -p "$reg/.pi/agent"
|
|
161
|
+
printf '{"extensions": ["%s/plugins/devos/pi/extension/devos.ts"]}' "$(cd "$(dirname "$DOCTOR")/.." && pwd)" > "$reg/.pi/agent/settings.json"
|
|
162
|
+
out=$(cd "$reqdoc" && HOME="$reg" bash "$DOCTOR" 2>/dev/null)
|
|
163
|
+
grep -q "OK pi extension (settings.json) resolves into this clone" <<<"$out"; check "doctor: settings.json pi extension into clone passes" 0 $?
|
|
164
|
+
|
|
165
|
+
# --- hook wiring resolves ---------------------------------------------------
|
|
166
|
+
# Every ${CLAUDE_PLUGIN_ROOT} command-hook path must point at a file inside the
|
|
167
|
+
# plugin (a hook may never climb out: install copies only the plugin dir).
|
|
168
|
+
# Resolve each and assert it exists. Swept over the plugin-root hooks.json and
|
|
169
|
+
# every skill body, since skills also invoke gate scripts inline in prose.
|
|
170
|
+
PLUGIN_ROOT="$(cd "$(dirname "$DOD")/../.." && pwd)"
|
|
171
|
+
SKILL="$PLUGIN_ROOT/skills/implement/SKILL.md"
|
|
172
|
+
CLAUDE_HOOKS_JSON="$PLUGIN_ROOT/hooks/hooks.json"
|
|
173
|
+
needle='${CLAUDE_PLUGIN_ROOT}'
|
|
174
|
+
while IFS= read -r cmd; do
|
|
175
|
+
resolved="${cmd/"$needle"/$PLUGIN_ROOT}"
|
|
176
|
+
case "$resolved" in
|
|
177
|
+
"$PLUGIN_ROOT"/*) : ;;
|
|
178
|
+
*) echo "FAIL hook wiring: $cmd climbs out of the plugin root" >&2; fail=$((fail+1)); continue ;;
|
|
179
|
+
esac
|
|
180
|
+
[[ -f "$resolved" ]]; check "hook wiring: ${cmd##*/} resolves inside plugin" 0 $?
|
|
181
|
+
done < <(grep -hoE '\$\{CLAUDE_PLUGIN_ROOT\}[^"]*\.sh' \
|
|
182
|
+
"$CLAUDE_HOOKS_JSON" "$PLUGIN_ROOT"/skills/*/SKILL.md "$PLUGIN_ROOT"/skills/*/references/*.md 2>/dev/null | sort -u)
|
|
183
|
+
|
|
184
|
+
# --- gate scope parity across tools -----------------------------------------
|
|
185
|
+
# Until 0.10.10 Claude's three gates lived in implement/SKILL.md and
|
|
186
|
+
# spec/SKILL.md frontmatter, which scopes a hook to sessions where that
|
|
187
|
+
# skill is loaded. Every other tool registers the same three at install,
|
|
188
|
+
# session-wide. So a test-file edit from /devos:spec, /devos:review or plain
|
|
189
|
+
# chat was gated on some adapters and ungated on Claude, and nothing anywhere compared
|
|
190
|
+
# the two: each side was individually correct, which is what made the gap
|
|
191
|
+
# invisible. Pin both halves — the plugin root registers all of them, and no
|
|
192
|
+
# skill quietly takes one back by re-declaring it in its own frontmatter.
|
|
193
|
+
[[ -f "$CLAUDE_HOOKS_JSON" ]]
|
|
194
|
+
check "plugin root has a session-scoped hooks.json" 0 $?
|
|
195
|
+
for gate in run-gate.sh check-dod.sh; do
|
|
196
|
+
grep -q "$gate" "$CLAUDE_HOOKS_JSON" 2>/dev/null
|
|
197
|
+
check "session-scoped hooks.json registers $gate" 0 $?
|
|
198
|
+
done
|
|
199
|
+
skill_hooks=$(grep -l '^hooks:' "$PLUGIN_ROOT"/skills/*/SKILL.md 2>/dev/null | tr '\n' ' ')
|
|
200
|
+
[[ -z "$skill_hooks" ]]
|
|
201
|
+
check "no skill re-scopes a gate to its own frontmatter ($skill_hooks)" 0 $?
|
|
202
|
+
|
|
203
|
+
# No hook may be model-judged (`type: prompt`). It has no fail-open path: when
|
|
204
|
+
# the judge answers in prose instead of the required JSON the harness reads
|
|
205
|
+
# that as a hook error and blocks the write. One was live in 0.7.3 and blocked
|
|
206
|
+
# two non-test writes during this project's own dogfooding (devos/deferred.md,
|
|
207
|
+
# gate-hardening-2026-07-27). Deleted in 0.8.4; this pins it, now also over the
|
|
208
|
+
# plugin-root registration the gates moved to.
|
|
209
|
+
prompt_hooks=$(grep -l 'type: prompt\|"type": *"prompt"' \
|
|
210
|
+
"$CLAUDE_HOOKS_JSON" "$PLUGIN_ROOT"/skills/*/SKILL.md 2>/dev/null | tr '\n' ' ')
|
|
211
|
+
[[ -z "$prompt_hooks" ]]
|
|
212
|
+
check "no model-judged prompt hook is registered ($prompt_hooks)" 0 $?
|
|
213
|
+
|
|
214
|
+
# A valueless --agent option is a malformed dispatcher invocation. It must fail
|
|
215
|
+
# open with the usage decision, not die on an unbound $2 and become a silent
|
|
216
|
+
# hook error that blocks every edit.
|
|
217
|
+
printf '{}' | bash "$PLUGIN_ROOT/scripts/gates/run-gate.sh" --agent >/dev/null 2>&1
|
|
218
|
+
check "run-gate: valueless --agent fails open" 0 $?
|
|
219
|
+
|
|
220
|
+
# --- copy-install self-containment ------------------------------------------
|
|
221
|
+
# The guard that was missing while the DoD gate silently no-op'd on real
|
|
222
|
+
# installs for months: `claude plugin install` copies ONLY the plugin dir into
|
|
223
|
+
# a version cache; repo-root files are never shipped. Reproduce that exactly,
|
|
224
|
+
# copy plugins/devos alone, then drive the hooks from the copy. If a hook
|
|
225
|
+
# resolved its decision core by climbing out of the plugin, that core would be
|
|
226
|
+
# absent here and the gate would silently exit 0 (the bug). These assert the
|
|
227
|
+
# gates still ENFORCE from an isolated copy.
|
|
228
|
+
inst="$tmp/install/devos"
|
|
229
|
+
mkdir -p "$inst"
|
|
230
|
+
cp -R "$PLUGIN_ROOT/." "$inst/"
|
|
231
|
+
|
|
232
|
+
tj="$tmp/inst-transcript.jsonl"
|
|
233
|
+
printf '{"type":"assistant","message":{"content":[{"type":"text","text":"Verdict: DONE: devos/specs/absent-spec"}]}}\n' > "$tj"
|
|
234
|
+
printf '{"transcript_path":"%s","stop_hook_active":false}' "$tj" | bash "$inst/scripts/gates/check-dod.sh" 2>/dev/null
|
|
235
|
+
check "copy-install: DoD gate still blocks a bogus DONE from the copied plugin" 2 $?
|
|
236
|
+
|
|
237
|
+
payload='{"tool_input":{"file_path":"src/main.test.ts","old_string":"it(\"a\",()=>{expect(x).toBe(1)})","new_string":"it.skip(\"a\",()=>{expect(x).toBe(1)})"}}'
|
|
238
|
+
printf '%s' "$payload" | bash "$inst/scripts/gates/run-gate.sh" --agent claude test-integrity 2>/dev/null
|
|
239
|
+
check "copy-install: test-integrity floor still denies .skip from the copied plugin" 2 $?
|
|
240
|
+
|
|
241
|
+
payload='{"tool_input":{"file_path":"src/auth.test.ts","content":"it(\"SC-3: returns 401\",()=>{})"}}'
|
|
242
|
+
printf '%s' "$payload" | bash "$inst/scripts/gates/run-gate.sh" --agent claude test-integrity 2>/dev/null
|
|
243
|
+
check "copy-install: output boundary still denies an SC-labeled test from the copied plugin" 2 $?
|
|
244
|
+
|
|
245
|
+
# The gates now enforce only because the copy carries their registration too:
|
|
246
|
+
# a hooks.json left outside the plugin dir would ship nothing and the scripts
|
|
247
|
+
# above would sit there unwired, which is the state this whole section exists
|
|
248
|
+
# to catch.
|
|
249
|
+
[[ -f "$inst/hooks/hooks.json" ]]
|
|
250
|
+
check "copy-install: session-scoped hooks.json ships with the plugin" 0 $?
|
|
251
|
+
|
|
252
|
+
# --- every adapter's hooks.json routes its gates through run-gate.sh -------
|
|
253
|
+
# The six per-tool shim scripts are gone; one dispatcher (run-gate.sh) serves
|
|
254
|
+
# every tool, selected by --agent. Pin that each tool's hook config actually
|
|
255
|
+
# invokes it with the right args, so a typo here fails loud, not as a silent
|
|
256
|
+
# dead gate.
|
|
257
|
+
REPO="$(cd "$PLUGIN_ROOT/../.." && pwd)"
|
|
258
|
+
gatedir="$PLUGIN_ROOT/scripts/gates"
|
|
259
|
+
[[ -x "$gatedir/run-gate.sh" ]]; check "run-gate.sh dispatcher exists and is executable" 0 $?
|
|
260
|
+
route() { grep -q "$2" "$1" 2>/dev/null; }
|
|
261
|
+
route "$REPO/plugins/devos/hooks/hooks.json" 'run-gate.sh --agent claude test-integrity'
|
|
262
|
+
check "claude hooks.json routes test-integrity through run-gate.sh" 0 $?
|
|
263
|
+
route "$REPO/plugins/devos/copilot/hooks.json" 'run-gate.sh --agent copilot test-integrity'
|
|
264
|
+
check "copilot hooks.json routes test-integrity through run-gate.sh" 0 $?
|
|
265
|
+
route "$REPO/plugins/devos/copilot/hooks.json" 'run-gate.sh --agent copilot dod'
|
|
266
|
+
check "copilot hooks.json routes DoD through run-gate.sh" 0 $?
|
|
267
|
+
route "$REPO/plugins/devos/codex/hooks/hooks.json" 'run-gate.sh --agent codex test-integrity'
|
|
268
|
+
check "codex hooks.json routes test-integrity through run-gate.sh" 0 $?
|
|
269
|
+
route "$REPO/plugins/devos/codex/hooks/hooks.json" 'run-gate.sh --agent codex dod'
|
|
270
|
+
check "codex hooks.json routes DoD through run-gate.sh" 0 $?
|
|
271
|
+
route "$REPO/plugins/devos/cursor/hooks.json" 'run-gate.sh --agent cursor test-integrity'
|
|
272
|
+
check "cursor hooks.json routes test-integrity through run-gate.sh" 0 $?
|
|
273
|
+
|
|
274
|
+
# --- Cursor permission contract ----------------------------------------------
|
|
275
|
+
printf 'it("works", () => { expect(a).toBe(1) })\n' > "$old"
|
|
276
|
+
printf 'it("works", () => { })\n' > "$new"
|
|
277
|
+
out=$(bash "$RUBRIC" --format cursor --file src/main.test.ts --old "$old" --new "$new")
|
|
278
|
+
[[ "$out" == '{"permission":"deny"'* ]]; check "rubric: cursor weakened test denies in cursor's permission shape" 0 $?
|
|
279
|
+
out=$(bash "$RUBRIC" --format cursor --file src/main.ts --old "$old" --new "$new")
|
|
280
|
+
[[ "$out" == '{"permission":"allow"}' ]]; check "rubric: cursor non-test file allows in cursor's permission shape" 0 $?
|
|
281
|
+
|
|
282
|
+
# --- adapter DoD gates emit a real decision on both paths -------------------
|
|
283
|
+
# Run the dispatcher for each tool that has a DoD gate; assert both the allow
|
|
284
|
+
# and deny paths emit parseable JSON with the gate's actual reason, not a
|
|
285
|
+
# placeholder. (Replaces the old per-tool shim execution checks.)
|
|
286
|
+
dodtj="$tmp/adapter-dod-transcript.txt"
|
|
287
|
+
printf 'Verdict: DONE: devos/specs/absent-spec\n' > "$dodtj"
|
|
288
|
+
for tool in copilot codex; do
|
|
289
|
+
sname="run-gate.sh --agent $tool dod"
|
|
290
|
+
if [[ "$tool" == "copilot" ]]; then tf='transcriptPath'; else tf='transcript_path'; fi
|
|
291
|
+
aout=$(printf '{}' | bash "$gatedir/run-gate.sh" --agent "$tool" dod 2>"$tmp/dod-err")
|
|
292
|
+
printf '%s' "$aout" | jq -e . >/dev/null 2>&1
|
|
293
|
+
check "$sname: allow path emits parseable JSON" 0 $?
|
|
294
|
+
[[ ! -s "$tmp/dod-err" ]]; check "$sname: allow path is silent on stderr" 0 $?
|
|
295
|
+
dout=$(printf '{"%s":"%s"}' "$tf" "$dodtj" | bash "$gatedir/run-gate.sh" --agent "$tool" dod 2>/dev/null)
|
|
296
|
+
printf '%s' "$dout" | jq -e . >/dev/null 2>&1
|
|
297
|
+
check "$sname: deny path emits parseable JSON" 0 $?
|
|
298
|
+
[[ "$dout" == *"verification.md"* ]]
|
|
299
|
+
check "$sname: deny path carries the gate's reason, not a placeholder" 0 $?
|
|
300
|
+
done
|
|
301
|
+
|
|
302
|
+
# --- ops-log.sh --------------------------------------------
|
|
303
|
+
OPS="$EVAL_DIR/../plugins/devos/skills/implement/scripts/ops-log.sh"
|
|
304
|
+
spec="devos/specs/2026-01-01-demo"
|
|
305
|
+
bash "$OPS" "$spec" started "route L"; check "ops-log: first append exits 0" 0 $?
|
|
306
|
+
grep -q '^{"at":"[0-9T:-]*Z","event":"started","detail":"route L"}$' "$spec/ops.jsonl"; check "ops-log: line is timestamped JSON with detail" 0 $?
|
|
307
|
+
bash "$OPS" "$spec" plan-approved; check "ops-log: second append exits 0" 0 $?
|
|
308
|
+
[[ "$(wc -l < "$spec/ops.jsonl" | tr -d ' ')" == 2 ]]; check "ops-log: appends preserve prior lines" 0 $?
|
|
309
|
+
bash "$OPS" "$spec" bogus 2>/dev/null; check "ops-log: unknown event is usage error" 2 $?
|
|
310
|
+
bash "$OPS" 2>/dev/null; check "ops-log: missing args is usage error" 2 $?
|
|
311
|
+
bash "$OPS" "$spec" $'two\nlines' 2>/dev/null; check "ops-log: multiline detail rejected" 2 $?
|
|
312
|
+
bash "$OPS" "$spec" note 'say "no"'; check "ops-log: quoted detail accepted" 0 $?
|
|
313
|
+
n=$(grep -c '"event":"note","detail":"say \\\"no\\\""}' "$spec/ops.jsonl"); [[ "$n" == 1 ]]; check "ops-log: embedded quotes escaped, still one JSON line" 0 $?
|
|
314
|
+
tab_detail=$'before\tmiddle'
|
|
315
|
+
bash "$OPS" "$spec" note "$tab_detail"; check "ops-log: tab detail accepted" 0 $?
|
|
316
|
+
python3 - "$spec/ops.jsonl" "$tab_detail" <<'PY'
|
|
317
|
+
import json
|
|
318
|
+
import sys
|
|
319
|
+
rows = [json.loads(line) for line in open(sys.argv[1])]
|
|
320
|
+
assert any(row.get("event") == "note" and row.get("detail") == sys.argv[2] for row in rows)
|
|
321
|
+
PY
|
|
322
|
+
check "ops-log: control characters are escaped and round-trip" 0 $?
|
|
323
|
+
for i in $(seq 1 20); do bash "$OPS" "$spec" note "n$i" & done
|
|
324
|
+
wait
|
|
325
|
+
[[ "$(wc -l < "$spec/ops.jsonl" | tr -d ' ')" == 24 ]]; check "ops-log: 20 concurrent appends all land" 0 $?
|
|
326
|
+
[[ -d "$spec/.ops.lock" ]]; check "ops-log: lock removed after runs" 1 $?
|
|
327
|
+
rm -rf "$spec"
|
|
328
|
+
|
|
329
|
+
# --- check-claims.py check 3: per-command invoke cost --
|
|
330
|
+
# Measurement comes from the real plugin; only the documentation tree is a
|
|
331
|
+
# fixture, so these assert the comparison logic without editing docs/ in place.
|
|
332
|
+
CLAIMS="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/check-claims.py"
|
|
333
|
+
cfix="$tmp/claimsfix"
|
|
334
|
+
mkdir -p "$cfix/docs"
|
|
335
|
+
crow() { printf '%s\n' "$1" > "$cfix/docs/a.md"; python3 "$CLAIMS" "$cfix" >/dev/null 2>&1; }
|
|
336
|
+
|
|
337
|
+
# Read build's cost from the same source check-claims.py uses, rather than
|
|
338
|
+
# pinning a literal: this fixture asserts that a *matching* row passes, and a
|
|
339
|
+
# hardcoded number turns every edit to build/SKILL.md into a false failure
|
|
340
|
+
# here. The wrong-number direction below still exercises the real comparison.
|
|
341
|
+
bcost=$(bash "$(dirname "$DOD")/../../skills/status/scripts/context-cost.sh" \
|
|
342
|
+
| awk -F'|' '$2 ~ /^ *build *$/ { gsub(/ /, "", $4); print $4 }')
|
|
343
|
+
crow "| \`build\` | ~$bcost | ok |"
|
|
344
|
+
check "claims3: row matching measurement passes" 0 $?
|
|
345
|
+
crow '| `build` | ~99 | wrong |'
|
|
346
|
+
check "claims3: row off by >15% fails" 1 $?
|
|
347
|
+
crow '| `nosuchskill` | ~100 | x |'
|
|
348
|
+
check "claims3: row naming a non-existent skill fails" 1 $?
|
|
349
|
+
# Same reason as bcost above: derived, not pinned. The wrong-sum row is the
|
|
350
|
+
# two-skill subset measured against double its own sum, which no edit to a
|
|
351
|
+
# skill can drag back inside the 15% tolerance (a literal can, and did).
|
|
352
|
+
dcost() { bash "$(dirname "$DOD")/../../skills/status/scripts/context-cost.sh" \
|
|
353
|
+
| awk -F'|' -v s="$1" '$2 ~ "^ *"s" *$" { gsub(/ /, "", $4); print $4 }'; }
|
|
354
|
+
sd=$(dcost learn); ti=$(dcost implement); ve=$(dcost spec)
|
|
355
|
+
crow "| \`learn\` + \`implement\` + \`spec\` | ~$((sd + ti + ve)) | combined |"
|
|
356
|
+
check "claims3: multi-skill row compares against the sum" 0 $?
|
|
357
|
+
crow "| \`learn\` + \`implement\` | ~$(((sd + ti) * 2)) | combined |"
|
|
358
|
+
check "claims3: multi-skill row with a wrong sum fails" 1 $?
|
|
359
|
+
crow 'The `build` skill costs roughly ~900 tokens, in prose.'
|
|
360
|
+
check "claims3: prose naming a skill and a number is not a claim row" 0 $?
|
|
361
|
+
crow '| Refresh | Re-run of `init` on unchanged code: ~0 (stops) | x |'
|
|
362
|
+
check "claims3: a ~N inside a prose cell is not a claim row" 0 $?
|
|
363
|
+
|
|
364
|
+
# --- check-claims.py check 4: per-moment load cost -------------------------
|
|
365
|
+
# Same fixture harness. Derived, not pinned, for the same reason as bcost.
|
|
366
|
+
fcost() { echo $(( $(LC_ALL=C wc -c < "$SKILLS/$1") / 4 )); }
|
|
367
|
+
plan_sum=$(( $(fcost implement/SKILL.md) + $(fcost implement/references/plan.md) ))
|
|
368
|
+
|
|
369
|
+
crow "| Fresh M/L run, planning | \`implement/SKILL.md\` + \`implement/references/plan.md\` | ~$plan_sum |"
|
|
370
|
+
check "claims4: moment row matching the measured sum passes" 0 $?
|
|
371
|
+
# The historical drift: this row carried ~3,786 against a real ~4,976, and
|
|
372
|
+
# nothing failed. The figure is expressed relative to the current measurement
|
|
373
|
+
# rather than pinned, because a pinned number silently becomes correct again
|
|
374
|
+
# the moment the skills shrink, which is exactly when the gate stops testing
|
|
375
|
+
# what it was written for. Half the measured sum is off by 50%, outside the
|
|
376
|
+
# 15% tolerance from either side, at any size these skills ever take.
|
|
377
|
+
crow "| Fresh M/L run, planning | \`implement/SKILL.md\` + \`implement/references/plan.md\` | ~$((plan_sum / 2)) |"
|
|
378
|
+
check "claims4: a stale figure fails" 1 $?
|
|
379
|
+
crow '| Verify and close | `implement/references/no-such-phase.md` | ~1,431 |'
|
|
380
|
+
check "claims4: row naming a non-existent reference file fails" 1 $?
|
|
381
|
+
crow "| Verify and close | \`implement/references/verify-close.md\` | ~$(fcost implement/references/verify-close.md) |"
|
|
382
|
+
check "claims4: single-file moment row passes" 0 $?
|
|
383
|
+
# The pre-0.9.9 shape: no resolvable path, so nothing to measure. Unmatched
|
|
384
|
+
# rather than failed, or every prose table in docs/ becomes a build failure.
|
|
385
|
+
crow '| Fresh M/L run, planning | SKILL.md + references/plan.md | ~1 |'
|
|
386
|
+
check "claims4: a moment row naming no resolvable path is not a claim row" 0 $?
|
|
387
|
+
crow "| \`build\` | ~$bcost | check 3 territory, not check 4 |"
|
|
388
|
+
check "claims4: does not double-report check 3's rows" 0 $?
|
|
389
|
+
|
|
390
|
+
# --- check-repo-hygiene.sh --------------------------
|
|
391
|
+
HYGIENE="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/check-repo-hygiene.sh"
|
|
392
|
+
|
|
393
|
+
hfix="$tmp/hygiene-clean"
|
|
394
|
+
mkdir -p "$hfix/src"
|
|
395
|
+
(
|
|
396
|
+
cd "$hfix" && git init -q . && git config user.email t@t && git config user.name t
|
|
397
|
+
echo "hi" > src/main.js
|
|
398
|
+
git add -A && git commit -qm init
|
|
399
|
+
)
|
|
400
|
+
bash "$HYGIENE" "$hfix" >/dev/null; check "hygiene: clean repo passes" 0 $?
|
|
401
|
+
|
|
402
|
+
# Planted gitlink with no .gitmodules: the exact shape v0.8.8 shipped.
|
|
403
|
+
hbad="$tmp/hygiene-gitlink"
|
|
404
|
+
inner="$tmp/hygiene-inner"
|
|
405
|
+
mkdir -p "$inner" && (cd "$inner" && git init -q . && git config user.email t@t && git config user.name t && git commit -q --allow-empty -m init)
|
|
406
|
+
mkdir -p "$hbad"
|
|
407
|
+
(
|
|
408
|
+
cd "$hbad" && git init -q . && git config user.email t@t && git config user.name t
|
|
409
|
+
git -c protocol.file.allow=always submodule add -q "$inner" 06c-fabricated-data >/dev/null 2>&1
|
|
410
|
+
rm -f .gitmodules
|
|
411
|
+
git rm -r --cached --ignore-unmatch .gitmodules >/dev/null 2>&1
|
|
412
|
+
git add -A && git commit -qm "plant unmapped gitlink" >/dev/null 2>&1
|
|
413
|
+
)
|
|
414
|
+
bash "$HYGIENE" "$hbad" >/dev/null 2>&1; check "hygiene: unmapped gitlink fails" 1 $?
|
|
415
|
+
|
|
416
|
+
# Same shape, but mapped in .gitmodules: a legitimate submodule passes.
|
|
417
|
+
hgood="$tmp/hygiene-submodule"
|
|
418
|
+
mkdir -p "$hgood"
|
|
419
|
+
(
|
|
420
|
+
cd "$hgood" && git init -q . && git config user.email t@t && git config user.name t
|
|
421
|
+
git -c protocol.file.allow=always submodule add -q "$inner" vendor/lib >/dev/null 2>&1
|
|
422
|
+
git add -A && git commit -qm "add real submodule" >/dev/null 2>&1
|
|
423
|
+
)
|
|
424
|
+
bash "$HYGIENE" "$hgood" >/dev/null 2>&1; check "hygiene: mapped submodule passes" 0 $?
|
|
425
|
+
|
|
426
|
+
# Eval-workspace-shaped directory tracked at repo root.
|
|
427
|
+
hws="$tmp/hygiene-workspace"
|
|
428
|
+
mkdir -p "$hws/06c-fabricated-data"
|
|
429
|
+
(
|
|
430
|
+
cd "$hws" && git init -q . && git config user.email t@t && git config user.name t
|
|
431
|
+
echo "x" > 06c-fabricated-data/file.txt
|
|
432
|
+
git add -A && git commit -qm "accidental workspace add"
|
|
433
|
+
)
|
|
434
|
+
bash "$HYGIENE" "$hws" >/dev/null 2>&1; check "hygiene: eval-workspace-shaped root dir fails" 1 $?
|
|
435
|
+
|