@tduyng/pi-devos 0.13.6 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/README.md +26 -33
  3. package/package.json +1 -2
  4. package/plugins/devos/agents/verifier.md +4 -30
  5. package/plugins/devos/pi/extension/devos.ts +16 -26
  6. package/plugins/devos/scripts/check-evidence-router.sh +7 -1
  7. package/plugins/devos/scripts/evidence-router.py +5 -3
  8. package/plugins/devos/scripts/gates/check-dod-core.sh +5 -3
  9. package/plugins/devos/scripts/gates/check-port-source.sh +1 -1
  10. package/plugins/devos/scripts/gates/check-scenario-coverage.sh +10 -8
  11. package/plugins/devos/scripts/gates/phase-gate.sh +6 -6
  12. package/plugins/devos/scripts/gates/plan-template-check.py +6 -12
  13. package/plugins/devos/scripts/gates/spec-contract.sh +19 -0
  14. package/plugins/devos/scripts/gates/spec-template-check.py +8 -21
  15. package/plugins/devos/scripts/gates/suite/01-dod.sh +485 -0
  16. package/plugins/devos/scripts/gates/suite/02-integrity.sh +310 -0
  17. package/plugins/devos/scripts/gates/suite/03-archetype.sh +255 -0
  18. package/plugins/devos/scripts/gates/suite/04-spec-validator.sh +183 -0
  19. package/plugins/devos/scripts/gates/suite/05-requires.sh +360 -0
  20. package/plugins/devos/scripts/gates/suite/06-clarify-adapters.sh +435 -0
  21. package/plugins/devos/scripts/gates/suite/07-lint-evals.sh +350 -0
  22. package/plugins/devos/scripts/gates/suite/08-converge.sh +591 -0
  23. package/plugins/devos/scripts/gates/suite/09-quality.sh +516 -0
  24. package/plugins/devos/scripts/gates/suite/10-eval-harness.sh +679 -0
  25. package/plugins/devos/scripts/gates/test-gates.sh +23 -4253
  26. package/plugins/devos/scripts/gates/test-integrity-rubric.sh +1 -1
  27. package/plugins/devos/scripts/gates/test-phase-gate.sh +5 -5
  28. package/plugins/devos/skills/build/SKILL.md +3 -3
  29. package/plugins/devos/skills/build/references/refactor.md +1 -1
  30. package/plugins/devos/skills/build/references/routes.md +4 -4
  31. package/plugins/devos/skills/frame/SKILL.md +22 -39
  32. package/plugins/devos/skills/frame/references/template.md +115 -0
  33. package/plugins/devos/skills/implement/SKILL.md +4 -4
  34. package/plugins/devos/skills/implement/references/dod.md +6 -1
  35. package/plugins/devos/skills/implement/references/implement.md +3 -1
  36. package/plugins/devos/skills/implement/references/plan.md +9 -27
  37. package/plugins/devos/skills/implement/references/resume.md +1 -1
  38. package/plugins/devos/skills/implement/references/verify-close.md +4 -3
  39. package/plugins/devos/skills/implement/scripts/ops-log.sh +2 -2
  40. package/plugins/devos/skills/implement/scripts/validate-spec.sh +17 -65
  41. package/plugins/devos/skills/init/references/context.md +1 -1
  42. package/plugins/devos/skills/review/SKILL.md +24 -47
  43. package/plugins/devos/skills/review/references/comments.md +13 -0
  44. package/plugins/devos/skills/review/references/reviewer.md +124 -0
  45. package/plugins/devos/skills/review/references/routing.md +35 -0
  46. package/plugins/devos/skills/review/scripts/__pycache__/prepare.cpython-313.pyc +0 -0
  47. package/plugins/devos/skills/review/scripts/prepare.py +763 -0
  48. package/plugins/devos/skills/spec/SKILL.md +5 -6
  49. package/plugins/devos/skills/spec/references/write-spec.md +36 -51
  50. package/plugins/devos/skills/spec/scripts/clarify.sh +4 -4
  51. package/plugins/devos/skills/status/SKILL.md +4 -4
  52. package/plugins/devos/skills/status/scripts/check-requires.py +4 -4
  53. package/plugins/devos/skills/status/scripts/convention-usage.sh +1 -1
  54. package/plugins/devos/skills/status/scripts/quality-correlation.sh +2 -25
  55. package/plugins/devos/scripts/gates/check-handoff.sh +0 -215
  56. package/plugins/devos/skills/design-taste/SKILL.md +0 -55
  57. package/plugins/devos/skills/design-taste/references/tells.md +0 -85
  58. package/plugins/devos/skills/design-taste/scripts/slop-scan.py +0 -305
  59. package/plugins/devos/skills/docs/SKILL.md +0 -22
  60. package/plugins/devos/skills/docs/references/backfill.md +0 -54
  61. package/plugins/devos/skills/docs/references/init.md +0 -43
  62. package/plugins/devos/skills/docs/references/status.md +0 -71
  63. package/plugins/devos/skills/docs/references/update.md +0 -80
  64. package/plugins/devos/skills/explain/SKILL.md +0 -45
  65. package/plugins/devos/skills/explain/references/quiz.md +0 -21
  66. package/plugins/devos/skills/frame/references/boundaries.md +0 -53
  67. package/plugins/devos/skills/frame/references/decompose.md +0 -96
  68. package/plugins/devos/skills/frame/references/impact.md +0 -43
  69. package/plugins/devos/skills/frame/references/interrogate.md +0 -68
  70. package/plugins/devos/skills/product/SKILL.md +0 -40
  71. package/plugins/devos/skills/product/references/planner.md +0 -54
  72. package/plugins/devos/skills/surgical-diff/SKILL.md +0 -23
  73. package/plugins/devos/skills/test-integrity/SKILL.md +0 -26
  74. package/plugins/devos/skills/verification-evidence/SKILL.md +0 -17
  75. package/plugins/devos/workflows/common/backlog-file-structure.md +0 -44
  76. package/plugins/devos/workflows/common/category-file-structure.md +0 -41
  77. package/plugins/devos/workflows/common/feature-line-format.md +0 -10
  78. package/plugins/devos/workflows/common/roadmap-file-structure.md +0 -51
  79. package/plugins/devos/workflows/common/suggest-categories.md +0 -76
  80. package/plugins/devos/workflows/common/suggest-milestones.md +0 -92
  81. package/plugins/devos/workflows/doc/doc-health.md +0 -137
  82. package/plugins/devos/workflows/doc/doc-writer.md +0 -44
  83. package/plugins/devos/workflows/doc/init-documentation.md +0 -196
  84. package/plugins/devos/workflows/doc/retro-documentation.md +0 -304
  85. package/plugins/devos/workflows/doc/templates/library.md +0 -58
  86. package/plugins/devos/workflows/doc/templates/service.md +0 -61
  87. package/plugins/devos/workflows/doc/update-documentation.md +0 -194
  88. package/plugins/devos/workflows/doc/write-documentation.md +0 -137
  89. package/plugins/devos/workflows/product/create-product-mission.md +0 -45
  90. package/plugins/devos/workflows/product/create-product-roadmap.md +0 -106
  91. package/plugins/devos/workflows/product/create-product-tech-stack.md +0 -19
  92. package/plugins/devos/workflows/product/gather-product-info.md +0 -25
  93. package/plugins/devos/workflows/product/resume-detection.md +0 -70
  94. package/plugins/devos/workflows/product/reverse-backlog.md +0 -118
  95. package/plugins/devos/workflows/product/reverse-domain-model.md +0 -86
  96. package/plugins/devos/workflows/product/reverse-product-plan.md +0 -81
  97. package/plugins/devos/workflows/product/reverse-tech-stack.md +0 -73
  98. package/plugins/devos/workflows/product/scan-deep.md +0 -113
  99. package/plugins/devos/workflows/product/scan-git.md +0 -122
  100. package/plugins/devos/workflows/product/scan-light.md +0 -154
@@ -3,6 +3,7 @@
3
3
  # no model calls. Run via `just verify`.
4
4
  set -u
5
5
  cd "$(dirname "${BASH_SOURCE[0]}")"
6
+ SUITE="$PWD/suite"
6
7
  DOD="$PWD/check-dod-core.sh"
7
8
  RUBRIC="$PWD/test-integrity-rubric.sh"
8
9
  BOUNDARY="$PWD/check-implementation-boundary.sh"
@@ -12,6 +13,11 @@ pass=0 fail=0
12
13
  # After the cd at the top, $PWD is the gates directory (plugins/devos/scripts/gates/).
13
14
  # EVAL_DIR is four levels up then into evals.
14
15
  EVAL_DIR="$(cd ../../../../evals && pwd)"
16
+ REPO_ROOT="$(cd ../../../.. && pwd)"
17
+ EVALS="$REPO_ROOT/evals"
18
+ SKILLS="$PWD/../../skills"
19
+ QC="$SKILLS/status/scripts/quality-correlation.sh"
20
+ VSPEC="$SKILLS/implement/scripts/validate-spec.sh"
15
21
 
16
22
  check() { # check <name> <expected-exit> <actual-exit>
17
23
  if [[ "$2" == "$3" ]]; then
@@ -29,4264 +35,28 @@ tmp=$(mktemp -d)
29
35
  # $tmp (this script's cwd) under every test that came after it. Only the main
30
36
  # shell cleans up. BASH_SUBSHELL is 0 there and non-zero everywhere else, and
31
37
  # it exists in bash 3.2, which is what darwin ships.
32
- trap '[[ $BASH_SUBSHELL -eq 0 ]] && rm -rf "$tmp"' EXIT
38
+ trap '[[ $BASH_SUBSHELL -eq 0 ]] && rm -rf "$tmp" "${results:-}"' EXIT
33
39
  cd "$tmp"
34
40
 
35
- # --- check-dod-core.sh ---------------------------------------------------
36
- spec="devos/specs/2026-01-01-demo"
37
- mkdir -p "$spec/verifications"
38
-
39
- t="$tmp/transcript.txt"
40
-
41
- echo "working on it, no verdict yet" > "$t"
42
- bash "$DOD" "$t"; check "dod: no DONE claim allows" 0 $?
43
-
44
- echo "Verdict: DONE" > "$t"
45
- bash "$DOD" "$t" 2>/dev/null; check "dod: DONE without spec name blocks" 2 $?
46
-
47
- echo "Verdict: DONE: $spec" > "$t"
48
- bash "$DOD" "$t" 2>/dev/null; check "dod: DONE with missing report blocks" 2 $?
49
-
50
- printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$spec/verifications/verification.md"
51
- bash "$DOD" "$t"; check "dod: CLEAN+PASS report allows" 0 $?
52
-
53
- # This gate's own message names the format as `Verdict: DONE: devos/specs/[name]`.
54
- # A model that echoes the reminder after the real verdict used to have the echo
55
- # read as the verdict and get blocked for repeating the gate's words; the last
56
- # matching line is not necessarily the one that names a spec.
57
- printf 'Verdict: DONE: %s\n\nFormat reminder: `Verdict: DONE: devos/specs/[name]`\n' "$spec" > "$t"
58
- bash "$DOD" "$t"; check "dod: format reminder after a valid verdict allows" 0 $?
59
-
60
- echo "Verdict: DONE" > "$t"
61
- bash "$DOD" "$t" 2>/dev/null; check "dod: no line names a spec still blocks" 2 $?
62
-
63
- # Restore the naming verdict for the report-state tests below, which read the
64
- # report, not the transcript.
65
- echo "Verdict: DONE: $spec" > "$t"
66
-
67
- printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **FAIL**\n' > "$spec/verifications/verification.md"
68
- bash "$DOD" "$t" 2>/dev/null; check "dod: FAIL report blocks" 2 $?
69
-
70
- printf '# Verification\n> 2026-01-01 | Completeness: **HAS ISSUES** | Quality: **PASS**\n' > "$spec/verifications/verification.md"
71
- bash "$DOD" "$t" 2>/dev/null; check "dod: HAS ISSUES report blocks" 2 $?
72
-
73
- printf '# Verification\n> 2026-01-01 | Completeness: NOT CLEAN | Quality: NOT PASS\n' > "$spec/verifications/verification.md"
74
- bash "$DOD" "$t" 2>/dev/null; check "dod: negated CLEAN and PASS report blocks" 2 $?
75
-
76
- printf 'looks great, ship it\n' > "$spec/verifications/verification.md"
77
- bash "$DOD" "$t" 2>/dev/null; check "dod: malformed report (no verdict line) blocks" 2 $?
78
-
79
- bash "$DOD" "$tmp/nonexistent.txt"; check "dod: missing transcript allows" 0 $?
80
-
81
- arch="devos/specs/archive/2026-01-01-demo"
82
- mkdir -p "$arch/verifications"
83
- printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$arch/verifications/verification.md"
84
- echo "Verdict: DONE: $arch" > "$t"
85
- bash "$DOD" "$t"; check "dod: archived spec path allows" 0 $?
86
-
87
- echo "Verdict: DONE: devos/specs/archive/absent-spec" > "$t"
88
- bash "$DOD" "$t" 2>/dev/null; check "dod: archived path without report blocks" 2 $?
89
-
90
- echo "Verdict: DONE: devos/specs/archive/..." > "$t"
91
- bash "$DOD" "$t" 2>/dev/null; check "dod: ellipsis placeholder is the missing-spec case, not a report check" 2 $?
92
- echo "Verdict: DONE: devos/specs/archive/verifications/verification.md" > "$t"
93
- bash "$DOD" "$t" 2>/dev/null; check "dod: report path pasted as spec blocks as missing-spec (no verifications/verifications)" 2 $?
94
- [[ "$(bash "$DOD" "$t" 2>&1 >/dev/null)" != *"verifications/verifications"* ]]
95
- check "dod: report-path-as-spec message never doubles the verifications dir" 0 $?
96
-
97
- # --- check-dod-core.sh: a pre-existing failure has to be shown --------------
98
- # The DoD's rationalization table answers "the 2 failures are unrelated" with
99
- # "then prove it". Until Phase B recorded a baseline there was nothing to point
100
- # at, so the claim was accepted on the implementer's word.
101
- pe="$tmp/preexisting"
102
- mkdir -p "$pe/$spec/verifications"
103
- (cd "$pe" && git init -q . && git config user.email t@t && git config user.name t)
104
- pet="$pe/transcript.txt"
105
- echo "Verdict: DONE: $spec" >"$pet"
106
- clean_report='# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n'
107
-
108
- printf "$clean_report" >"$pe/$spec/verifications/verification.md"
109
- (cd "$pe" && bash "$DOD" "$pet"); check "dod: a report making no such claim needs no baseline" 0 $?
110
-
111
- printf "${clean_report}No pre-existing failures.\n" >"$pe/$spec/verifications/verification.md"
112
- (cd "$pe" && bash "$DOD" "$pet"); check "dod: a negated pre-existing claim needs no baseline" 0 $?
113
-
114
- printf "${clean_report}No pre-existing failures. 2 failures are pre-existing.\n" >"$pe/$spec/verifications/verification.md"
115
- (cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: mixed negated and affirmative claims block" 2 $?
116
-
117
- printf "${clean_report}2 failures are pre-existing on this branch.\n" >"$pe/$spec/verifications/verification.md"
118
- (cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: 'pre-existing' with no baseline.md blocks" 2 $?
119
-
120
- printf 'node test.js -> exit 1 (2 failed)\n' >"$pe/$spec/verifications/baseline.md"
121
- (cd "$pe" && bash "$DOD" "$pet"); check "dod: the same claim with a baseline allows" 0 $?
122
-
123
- rm -f "$pe/$spec/verifications/baseline.md"
124
- printf "${clean_report}The suite was already failing before this work.\n" >"$pe/$spec/verifications/verification.md"
125
- (cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: 'already failing' is the same claim in other words" 2 $?
126
-
127
- # --- check-dod-core.sh: a verdict is about the code it judged ----------------
128
- # The report records the commit it read. Nothing re-runs the verifier when a
129
- # later edit lands, so without that field a CLEAN report outlives the code it
130
- # was true of, and every green report reads the same whether it still holds.
131
- #
132
- # Scoped to the plan's file list on purpose: the close protocol writes the
133
- # behavior files, the archive, the changelog and the roadmap after the verifier
134
- # ran, so a whole-tree rule would block every spec that ships its paperwork.
135
- vc="$tmp/verdict-commit"
136
- mkdir -p "$vc/$spec/verifications" "$vc/docs"
137
- (cd "$vc" && git init -q . && git config user.email t@t && git config user.name t)
138
- vct="$vc/transcript.txt"
139
- echo "Verdict: DONE: $spec" >"$vct"
140
- printf 'module.exports = 1\n' >"$vc/shipping.js"
141
- printf '# Demo\n\n## Implementation Plan\n\n### Task Group 1: fee\n**Files:** `shipping.js`\n' >"$vc/$spec/spec.md"
142
- printf '# Changelog\n' >"$vc/CHANGELOG.md"
143
- printf '# Roadmap\n' >"$vc/docs/roadmap.md"
144
- printf 'module.exports = 0\n' >"$vc/unrelated.js"
145
- (cd "$vc" && git add -A && git commit -qm init)
146
- vsha=$(cd "$vc" && git rev-parse --short HEAD)
147
- printf '# Verification\n> 2026-01-01 | commit: %s | Completeness: **CLEAN** | Quality: **PASS**\n' "$vsha" >"$vc/$spec/verifications/verification.md"
148
- (cd "$vc" && bash "$DOD" "$vct"); check "dod: a report bound to the tree it read allows" 0 $?
149
-
150
- printf 'module.exports = 2\n' >"$vc/shipping.js"
151
- (cd "$vc" && bash "$DOD" "$vct" 2>/dev/null); check "dod: a delivered file changed after the judged commit blocks" 2 $?
152
-
153
- (cd "$vc" && git checkout -q -- shipping.js)
154
- printf '# Changelog\n- shipped the fee\n' >"$vc/CHANGELOG.md"
155
- printf '# Roadmap\n- fee: done\n' >"$vc/docs/roadmap.md"
156
- printf 'module.exports = 0\n' >"$vc/unrelated.js"
157
- (cd "$vc" && bash "$DOD" "$vct"); check "dod: the close protocol's paperwork after the verdict allows" 0 $?
158
-
159
- printf '# Verification\n> 2026-01-01 | commit: 0000000 | Completeness: **CLEAN** | Quality: **PASS**\n' >"$vc/$spec/verifications/verification.md"
160
- (cd "$vc" && bash "$DOD" "$vct" 2>/dev/null); check "dod: a marker naming no commit blocks" 2 $?
161
-
162
- printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n' >"$vc/$spec/verifications/verification.md"
163
- (cd "$vc" && bash "$DOD" "$vct"); check "dod: a report with no marker is the pre-marker format, allowed" 0 $?
164
-
165
- printf 'module.exports = 3\n' >"$vc/shipping.js"
166
- printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n\nSee commit: %s for the history.\n' "$vsha" >"$vc/$spec/verifications/verification.md"
167
- (cd "$vc" && bash "$DOD" "$vct"); check "dod: a commit mentioned in the body is not the header's marker" 0 $?
168
- (cd "$vc" && git checkout -q -- shipping.js)
169
-
170
- printf '# Demo\n\n## Implementation Plan\n\nAn inline spec naming no files.\n' >"$vc/$spec/spec.md"
171
- printf '# Verification\n> 2026-01-01 | commit: 0000000 | Completeness: **CLEAN** | Quality: **PASS**\n' >"$vc/$spec/verifications/verification.md"
172
- (cd "$vc" && bash "$DOD" "$vct" 2>/dev/null); check "dod: a plan naming no files skips only the diff, not the marker" 2 $?
173
-
174
- # Paths are passed one by one, so a name git would split is still one file, and
175
- # a name git cannot resolve is the gate's limitation rather than a stale verdict.
176
- printf '# Demo\n\n## Implementation Plan\n**Files:** `sp ace.js`\n' >"$vc/$spec/spec.md"
177
- printf 'module.exports = 1\n' >"$vc/sp ace.js"
178
- (cd "$vc" && git add -A && git commit -qm spacey)
179
- ssha=$(cd "$vc" && git rev-parse --short HEAD)
180
- printf '# Verification\n> 2026-01-01 | commit: %s | Completeness: **CLEAN** | Quality: **PASS**\n' "$ssha" >"$vc/$spec/verifications/verification.md"
181
- (cd "$vc" && bash "$DOD" "$vct"); check "dod: a planned path with a space allows while unchanged" 0 $?
182
- printf 'module.exports = 2\n' >"$vc/sp ace.js"
183
- (cd "$vc" && bash "$DOD" "$vct" 2>/dev/null); check "dod: a changed planned path with a space still blocks" 2 $?
184
- (cd "$vc" && git checkout -q -- 'sp ace.js')
185
-
186
- : >"$tmp/outside.js"
187
- printf '# Demo\n\n## Implementation Plan\n**Files:** `../outside.js`\n' >"$vc/$spec/spec.md"
188
- printf '# Verification\n> 2026-01-01 | commit: %s | Completeness: **CLEAN** | Quality: **PASS**\n' "$ssha" >"$vc/$spec/verifications/verification.md"
189
- (cd "$vc" && bash "$DOD" "$vct"); check "dod: a planned path git cannot resolve fails open" 0 $?
190
-
191
- # --- check-dod-core.sh: an empty delivery is not done -----------------------
192
- # The verifier scopes to the diff, so an implementation that wrote nothing has
193
- # no placeholders in it and verifies CLEAN. The plan's own **Files:** lines say
194
- # what was meant to exist.
195
- printf "$clean_report" >"$pe/$spec/verifications/verification.md"
196
- printf '# Demo\n\n## Implementation Plan\n\n### Task Group 1: thing\n**Files:** `src/thing.js`, `src/thing.test.js`\n' >"$pe/$spec/spec.md"
197
- (cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: no planned file exists, nothing was delivered, blocks" 2 $?
198
-
199
- mkdir -p "$pe/src" && : >"$pe/src/thing.js"
200
- (cd "$pe" && bash "$DOD" "$pet"); check "dod: one planned file delivered allows (an amended list is normal)" 0 $?
201
-
202
- printf '# Demo\n\n## Implementation Plan\n\n### Task Group 1: source\n**Files:** `src/auth.ts`\n' >"$pe/$spec/spec.md"
203
- printf '// Implemented by DevOS\n' >"$pe/src/auth.ts"
204
- (cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: process metadata in a planned source file blocks" 2 $?
205
- printf 'export const status = 401\n' >"$pe/src/auth.ts"
206
- (cd "$pe" && bash "$DOD" "$pet"); check "dod: clean planned source allows" 0 $?
207
-
208
- rm -rf "$pe/src"
209
- printf '# Demo\n\n## Implementation Plan\n\nNo file list, an inline mini-spec.\n' >"$pe/$spec/spec.md"
210
- (cd "$pe" && bash "$DOD" "$pet"); check "dod: a plan naming no files is silent, not blocking" 0 $?
211
-
212
- # A spec whose whole job is to write a standards file delivered what it planned.
213
- # Blocking it would be the false positive; the check is "nothing exists", not
214
- # "nothing outside devos/ exists".
215
- printf '# Demo\n\n### Task Group 1: standards\n**Files:** `devos/standards/conventions.md`\n' >"$pe/$spec/spec.md"
216
- (cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: a devos-only plan with nothing written still blocks" 2 $?
217
- mkdir -p "$pe/devos/standards" && : >"$pe/devos/standards/conventions.md"
218
- (cd "$pe" && bash "$DOD" "$pet"); check "dod: a devos-only plan that wrote its file allows" 0 $?
219
-
220
- # --- check-port-source.sh: the tree a port reads is not the tree it writes ---
221
- # A port writes a new tree and leaves the old one alone, and the old one may
222
- # be running in production. The manifest names the sources, and the check
223
- # compares them against the branch, so an edit to a source is a finding.
224
- ps="$tmp/port-source"
225
- mkdir -p "$ps/legacy/pkg" "$ps/modernized/pkg" "$ps/devos/specs/2026-10-20-port"
226
- (cd "$ps" && git init -q && git config user.email t@t && git config user.name t)
227
- printf 'x\n' >"$ps/legacy/pkg/a.py"
228
- printf 'y\n' >"$ps/modernized/pkg/a.py"
229
- printf 'source\ttarget\nlegacy/pkg/a.py\tmodernized/pkg/a.py\n' >"$ps/devos/specs/2026-10-20-port/manifest.tsv"
230
- printf '# spec\n' >"$ps/devos/specs/2026-10-20-port/spec-functional.md"
231
- (cd "$ps" && git add -A && git commit -qm base)
232
-
233
- bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" >/dev/null 2>&1
234
- check "port-source: a port that leaves the tree it reads alone allows" 0 $?
235
-
236
- printf 'z\n' >>"$ps/legacy/pkg/a.py"
237
- out=$(bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" 2>&1); rc=$?
238
- [ "$rc" -eq 1 ] && echo "$out" | grep -q "the tree it reads is not the tree it writes"
239
- check "port-source: a port that edits the tree it reads blocks" 0 $?
240
-
241
- out=$(bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" 2>&1); rc=$?
242
- [ "$rc" -eq 1 ] && echo "$out" | grep -q "fix: revert that change"
243
- check "port-source: the finding says what to do" 0 $?
244
-
245
- printf 'w\n' >>"$ps/modernized/pkg/a.py"
246
- (cd "$ps" && git checkout -q -- legacy)
247
- bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" >/dev/null 2>&1
248
- check "port-source: a port that writes the tree it builds allows" 0 $?
249
-
250
- rm "$ps/devos/specs/2026-10-20-port/manifest.tsv"
251
- bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" >/dev/null 2>&1
252
- check "port-source: a spec with no manifest is not a port, and is silent" 0 $?
253
-
254
- # --- check-dod-core.sh: a port that edits the tree it reads is not done -----
255
- # The port's manifest names the sources, and a DONE claim on a port spec is
256
- # checked against them: the tree the port reads keeps running while the port
257
- # lands.
258
- port="$tmp/portrepo"
259
- mkdir -p "$port/legacy/pkg" "$port/devos/specs/2026-10-20-port/verifications"
260
- (cd "$port" && git init -q && git config user.email t@t && git config user.name t)
261
- printf 'x\n' >"$port/legacy/pkg/a.py"
262
- printf 'source\ttarget\nlegacy/pkg/a.py\tmodernized/pkg/a.py\n' >"$port/devos/specs/2026-10-20-port/manifest.tsv"
263
- printf '# Verification\n> 2026-10-20 | Completeness: **CLEAN** | Quality: **PASS**\n' >"$port/devos/specs/2026-10-20-port/verifications/verification.md"
264
- (cd "$port" && git add -A && git commit -qm base)
265
- echo "Verdict: DONE: devos/specs/2026-10-20-port" > "$tmp/port.txt"
266
- (cd "$port" && bash "$DOD" "$tmp/port.txt"); check "dod: a port that left its source tree alone allows" 0 $?
267
- printf 'y\n' >>"$port/legacy/pkg/a.py"
268
- (cd "$port" && bash "$DOD" "$tmp/port.txt") 2>/dev/null; check "dod: a port that edited its source tree blocks" 2 $?
269
-
270
- # --- check-dod-core.sh: behavior-merge check --------
271
- # Against the real corpus this repo ships: all three archived specs must
272
- # pass unchanged, run from the actual repo (a regression test that only
273
- # ever sees synthetic input has never seen a true negative).
274
- REPO_ROOT="$(cd "$(dirname "$DOD")/../../../.." && pwd)"
275
- for real in \
276
- 2026-07-27-pressure-fixture-time-pressure \
277
- 2026-07-27-reverse-engineer-spec-stubs \
278
- 2026-07-27-run-sh-results-suffix; do
279
- rt="$tmp/real-transcript-$real.txt"
280
- echo "Verdict: DONE: devos/specs/archive/$real" > "$rt"
281
- (cd "$REPO_ROOT" && bash "$DOD" "$rt") >/dev/null 2>&1
282
- check "dod: real archived spec '$real' behavior delta passes unchanged" 0 $?
283
- done
284
-
285
- # Planted miss: delta names a capability with no behavior file at all.
286
- ghost="$tmp/devos/specs/2026-01-01-ghost-capability"
287
- mkdir -p "$ghost/verifications"
288
- cat > "$ghost/spec-functional.md" <<'MD'
289
- # Ghost Capability
290
- ## Goal
291
- x
292
- ## 3bis. Behavior delta
293
-
294
- **Added** (capability: `ghost-capability`):
295
- - Something new that never got merged.
296
- ## Out of Scope
297
- - y
298
- MD
299
- printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$ghost/verifications/verification.md"
300
- echo "Verdict: DONE: devos/specs/2026-01-01-ghost-capability" > "$t"
301
- bash "$DOD" "$t" 2>/dev/null; check "dod: delta names capability with no behavior file blocks" 2 $?
302
-
303
- # Fixed: behavior file exists with a matching last-spec header.
304
- mkdir -p "$tmp/devos/product/behaviors"
305
- printf '# Ghost Capability\n\n> updated: 2026-01-01 | last spec: 2026-01-01-ghost-capability\n\nSomething new that never got merged.\n' \
306
- > "$tmp/devos/product/behaviors/ghost-capability.md"
307
- bash "$DOD" "$t"; check "dod: matching behavior file with last-spec header allows" 0 $?
308
-
309
- # Removed entry: needs a History tombstone naming this spec, not a fresh header.
310
- removed="$tmp/devos/specs/2026-01-02-drop-capability"
311
- mkdir -p "$removed/verifications"
312
- cat > "$removed/spec-functional.md" <<'MD'
313
- # Drop Capability
314
- ## Goal
315
- x
316
- ## 3bis. Behavior delta
317
-
318
- **Removed** (capability: `ghost-capability`):
319
- - Dropped on purpose, replaced by nothing.
320
- ## Out of Scope
321
- - y
322
- MD
323
- printf '# Verification\n> 2026-01-02 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$removed/verifications/verification.md"
324
- rt2="$tmp/removed-transcript.txt"
325
- echo "Verdict: DONE: devos/specs/2026-01-02-drop-capability" > "$rt2"
326
- bash "$DOD" "$rt2" 2>/dev/null; check "dod: removed capability with no History tombstone blocks" 2 $?
327
-
328
- printf '# Ghost Capability\n\n> updated: 2026-01-01 | last spec: 2026-01-01-ghost-capability\n\n## History\n- removed ghost-capability: dropped on purpose, per 2026-01-02-drop-capability\n' \
329
- > "$tmp/devos/product/behaviors/ghost-capability.md"
330
- bash "$DOD" "$rt2"; check "dod: removed capability with a matching History tombstone allows" 0 $?
331
-
332
- # No Behavior delta section at all: silent, same posture as validate-spec.sh.
333
- nodelta="$tmp/devos/specs/2026-01-03-no-delta"
334
- mkdir -p "$nodelta/verifications"
335
- printf '# No Delta\n## Goal\nx\n## Out of Scope\n- y\n' > "$nodelta/spec-functional.md"
336
- printf '# Verification\n> 2026-01-03 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$nodelta/verifications/verification.md"
337
- echo "Verdict: DONE: devos/specs/2026-01-03-no-delta" > "$t"
338
- bash "$DOD" "$t"; check "dod: spec with no Behavior delta section is silent" 0 $?
339
-
340
- # --- check-dod-core.sh: leftover debug instrumentation ----------------------
341
- # The tag is assembled with printf, never written literally, so this very file
342
- # stays clean: it is tracked, and the real-corpus assertions above run the gate
343
- # from REPO_ROOT, where a literal tag here would block every one of them.
344
- dbg="$tmp/debugtag"
345
- mkdir -p "$dbg/devos/specs/2026-01-04-dbg/verifications"
346
- (cd "$dbg" && git init -q .) 2>/dev/null
347
- printf '# Verification\n> 2026-01-04 | Completeness: **CLEAN** | Quality: **PASS**\n' \
348
- > "$dbg/devos/specs/2026-01-04-dbg/verifications/verification.md"
349
- dt="$tmp/dbg-transcript.txt"
350
- echo "Verdict: DONE: devos/specs/2026-01-04-dbg" > "$dt"
351
-
352
- printf 'console.log("[DEBUG-%s] here")\n' a4f2 > "$dbg/src.js"
353
- (cd "$dbg" && git add -A) 2>/dev/null
354
- (cd "$dbg" && bash "$DOD" "$dt") 2>/dev/null
355
- check "dod: leftover [DEBUG-hex] tag in a tracked file blocks" 2 $?
356
-
357
- # The placeholder form the docs use is not the tag: it must not match itself.
358
- printf 'tag every probe [DEBUG-xxxx] so cleanup is one grep\n' > "$dbg/src.js"
359
- (cd "$dbg" && git add -A) 2>/dev/null
360
- (cd "$dbg" && bash "$DOD" "$dt")
361
- check "dod: [DEBUG-xxxx] placeholder in prose allows" 0 $?
362
-
363
- # Untracked scratch file was never shipped: not the gate's business.
364
- printf 'console.log("[DEBUG-%s]")\n' 9bee > "$dbg/scratch.js"
365
- (cd "$dbg" && bash "$DOD" "$dt")
366
- check "dod: untracked file with a tag allows" 0 $?
367
-
368
- rm -f "$dbg/src.js" "$dbg/scratch.js"
369
- (cd "$dbg" && git add -A) 2>/dev/null
370
- (cd "$dbg" && bash "$DOD" "$dt")
371
- check "dod: clean tree allows" 0 $?
372
-
373
- # --- check-report-locations.sh: the verdict's citations are re-derived ------
374
- # A `file:line` in the report is the part a candidate gets right in prose and
375
- # wrong in fact. Before this pass nothing read it back, so a citation past the
376
- # end of its file blocked nothing and read like every other line.
377
- LOC="$(dirname "$DOD")/check-report-locations.sh"
378
- loc="$tmp/loc"
379
- mkdir -p "$loc/devos/specs/2026-01-05-loc/verifications"
380
- printf 'one\ntwo\nthree\n' > "$loc/src.js"
381
- LR="$loc/devos/specs/2026-01-05-loc/verifications/verification.md"
382
- lt="$tmp/loc-transcript.txt"
383
- echo "Verdict: DONE: devos/specs/2026-01-05-loc" > "$lt"
384
-
385
- printf '# Verification\n> 2026-01-05 | Completeness: **CLEAN** | Quality: **PASS**\n| `src.js:2` | code | problem |\n' > "$LR"
386
- (cd "$loc" && bash "$DOD" "$lt") 2>/dev/null
387
- check "dod: a resolvable citation allows" 0 $?
388
-
389
- printf '# Verification\n> 2026-01-05 | Completeness: **CLEAN** | Quality: **PASS**\n| `src.js:99` | code | problem |\n' > "$LR"
390
- (cd "$loc" && bash "$DOD" "$lt") 2>/dev/null
391
- check "dod: a citation past the end of its file blocks" 2 $?
392
-
393
- printf '# Verification\n> 2026-01-05 | Completeness: **CLEAN** | Quality: **PASS**\n| `src/gone.js:3` | code | problem |\n' > "$LR"
394
- (cd "$loc" && bash "$DOD" "$lt") 2>/dev/null
395
- check "dod: a citation to a nonexistent file blocks" 2 $?
396
-
397
- # A URL carries a colon-number too; it is not a citation.
398
- printf '# Verification\n> 2026-01-05 | Completeness: **CLEAN** | Quality: **PASS**\nsee https://example.com:8080/a.js:9\n' > "$LR"
399
- (cd "$loc" && bash "$DOD" "$lt")
400
- check "dod: a URL colon-number is not a citation" 0 $?
401
-
402
- # The reflection pass on its own, out of the hook: same exit contract.
403
- bash "$LOC" "$loc/devos/specs/2026-01-05-loc" >/dev/null 2>&1
404
- check "report-locations: a clean report exits 0" 0 $?
405
- printf '# V\n| `src.js:99` | code |\n' > "$LR"
406
- bash "$LOC" "$loc/devos/specs/2026-01-05-loc" >/dev/null 2>&1
407
- check "report-locations: a drifted citation exits 1" 1 $?
408
- printf '# V\n| `src.js:3-2` | reversed |\n' > "$LR"
409
- report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
410
- check "report-locations: a reversed range exits 1" 1 $report_rc
411
- printf '%s\n' "$report_out" | grep -Fq 'range end 2 is before start 3'; check "report-locations: reversed range names the bounds" 0 $?
412
- printf '# V\n| `src.js:0` | zero |\n' > "$LR"
413
- report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
414
- check "report-locations: line zero exits 1" 1 $report_rc
415
- printf '%s\n' "$report_out" | grep -Fq 'source line numbers start at 1'; check "report-locations: line zero names the one-based rule" 0 $?
416
- printf 'KEY=value\n' > "$loc/.env"
417
- printf '# V\n| `.env:99` | drifted |\n' > "$LR"
418
- report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
419
- check "report-locations: dotfile citation exits 1" 1 $report_rc
420
- printf '%s\n' "$report_out" | grep -Fq '.env:99'; check "report-locations: dotfile citation is named" 0 $?
421
- printf 'outside\n' > "$tmp/report-outside.txt"
422
- printf '# V\n| `../report-outside.txt:1` | escaped |\n' > "$LR"
423
- report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
424
- check "report-locations: a citation outside the project root exits 1" 1 $report_rc
425
- printf '%s\n' "$report_out" | grep -Fq 'outside the project root'; check "report-locations: path escape names the boundary" 0 $?
426
- ln -s "$tmp/report-outside.txt" "$loc/escape.txt"
427
- printf '# V\n| `escape.txt:1` | symlink escape |\n' > "$LR"
428
- report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
429
- check "report-locations: a symlink outside the project root exits 1" 1 $report_rc
430
- printf '%s\n' "$report_out" | grep -Fq 'outside the project root'; check "report-locations: symlink escape names the boundary" 0 $?
431
- printf 'spec-local\n' > "$loc/devos/specs/2026-01-05-loc/notes.md"
432
- printf '# V\n| `notes.md:1` | spec-relative |\n' > "$LR"
433
- bash "$LOC" "$loc/devos/specs/2026-01-05-loc" --root "$loc" >/dev/null 2>&1
434
- check "report-locations: a spec-relative in-root citation allows" 0 $?
435
- bash "$LOC" >/dev/null 2>&1
436
- check "report-locations: no argument is a usage error" 2 $?
437
-
438
- # --- check-scenario-coverage.sh: a scenario recorded nowhere was not verified --
439
- # The verifier is told to record every SC-n from spec-functional.md in its
440
- # report, and nothing checked it did. A missing row is the one failure a
441
- # clean-looking report cannot show: every row it carries resolves, the verdict
442
- # reads CLEAN + PASS, and the scenario nobody ran leaves no trace.
443
- COV="$(dirname "$DOD")/check-scenario-coverage.sh"
444
- cov="$tmp/cov"
445
- COVS="$cov/devos/specs/2026-01-06-cov"
446
- mkdir -p "$COVS/verifications"
447
- CR="$COVS/verifications/verification.md"
448
- covspec="$COVS/spec-functional.md"
449
- ct="$tmp/cov-transcript.txt"
450
- printf 'x\nx\nx\nx\nx\nx\nx\nx\nx\nx\n' > "$cov/one.test.js"
451
- echo "Verdict: DONE: devos/specs/2026-01-06-cov" > "$ct"
452
- printf '# Demo\n\n## 3. Functionality\n\n**Scenarios:**\n- Scenario SC-1: one / WHEN a / THEN b\n- Scenario SC-2: two / WHEN c / THEN d\n' > "$covspec"
453
-
454
- printf '# Verification\n> 2026-01-06 | Completeness: **CLEAN** | Quality: **PASS**\n| SC-1 | pass | `one.test.js:3` |\n| SC-2 | pass | `one.test.js:9` |\n' > "$CR"
455
- (cd "$cov" && bash "$DOD" "$ct"); check "coverage: DONE with every scenario recorded allows" 0 $?
456
-
457
- printf '# Verification\n> 2026-01-06 | Completeness: **CLEAN** | Quality: **PASS**\n| SC-1 | pass | `one.test.js:3` |\n' > "$CR"
458
- cov_out=$(cd "$cov" && bash "$DOD" "$ct" 2>&1); cov_rc=$?
459
- check "coverage: a scenario the report never records blocks DONE" 2 $cov_rc
460
- printf '%s\n' "$cov_out" | grep -Fq 'SC-2'; check "coverage: the missing ID is named" 0 $?
461
-
462
- # Presence, not format: prose that records each ID is as valid as a table row.
463
- printf '# V\nSC-1 pass (`one.test.js:3`), SC-2 AWAITING HUMAN.\n' > "$CR"
464
- bash "$COV" "$COVS" >/dev/null 2>&1; check "coverage: a mapping line records the scenarios" 0 $?
465
-
466
- # A scenario the spec dropped during revision leaves the report's old row stale.
467
- printf '# V\nSC-1 pass, SC-2 pass, SC-3 pass\n' > "$CR"
468
- cov_out=$(bash "$COV" "$COVS" 2>&1); cov_rc=$?
469
- check "coverage: a report citing an undeclared ID exits 1" 1 $cov_rc
470
- printf '%s\n' "$cov_out" | grep -Fq 'SC-3'; check "coverage: the undeclared ID is named" 0 $?
471
-
472
- # SC-10 is a different ID from SC-1, so a prefix match would let an omitted
473
- # scenario pass on its neighbour's row.
474
- printf '# Demo\n- Scenario SC-1: one / WHEN a / THEN b\n- Scenario SC-10: ten / WHEN c / THEN d\n' > "$covspec"
475
- printf '# V\nSC-10 pass\n' > "$CR"
476
- cov_out=$(bash "$COV" "$COVS" 2>&1); cov_rc=$?
477
- check "coverage: SC-1 missing while SC-10 is recorded exits 1" 1 $cov_rc
478
- printf '%s\n' "$cov_out" | grep -Fq 'SC-10'; check "coverage: a recorded SC-10 does not satisfy SC-1" 1 $?
479
-
480
- # Exemptions: a C-route spec declares no scenarios by design, and a pre-SC spec
481
- # has no IDs to trace. Both are silence, never a block.
482
- printf '# Demo\n\n## Invariant\n- npm test exits 0 with the same count\n' > "$covspec"
483
- bash "$COV" "$COVS" >/dev/null 2>&1; check "coverage: an invariant spec exits 0" 0 $?
484
- printf '# Demo\n\n- Scenario: one / WHEN a / THEN b\n' > "$covspec"
485
- bash "$COV" "$COVS" >/dev/null 2>&1; check "coverage: an unnumbered legacy spec exits 0" 0 $?
486
- rm -f "$covspec"
487
- bash "$COV" "$COVS" >/dev/null 2>&1; check "coverage: a spec with no functional doc exits 0" 0 $?
488
- # The report path is accepted directly, since a caller holding one should not
489
- # have to reconstruct the spec folder.
490
- printf '# Demo\n- Scenario SC-1: one / WHEN a / THEN b\n' > "$covspec"
491
- printf '# V\nSC-1 pass\n' > "$CR"
492
- bash "$COV" "$CR" >/dev/null 2>&1; check "coverage: a report path records the scenarios" 0 $?
493
- printf '# V\nnothing\n' > "$CR"
494
- bash "$COV" "$CR" >/dev/null 2>&1; check "coverage: a report path with a missing ID exits 1" 1 $?
495
- bash "$COV" >/dev/null 2>&1; check "coverage: no argument is a usage error" 2 $?
496
-
497
- # --- check-dod-core.sh: the branch resolves the spec --------------
498
- # The pipeline creates a `[spec-name]` branch before the first edit, so the gate
499
- # derives the spec instead of trusting a path the model typed. A placeholder
500
- # like `devos/specs/x` must not bounce the run into a retry loop when the
501
- # branch's own report is there.
502
- br="$tmp/branch"
503
- mkdir -p "$br/devos/specs/2026-01-02-real/verifications"
504
- (cd "$br" && git init -q . && git checkout -q -b 2026-01-02-real) 2>/dev/null
505
- printf '# Verification\n> 2026-01-02 | Completeness: **CLEAN** | Quality: **PASS**\n' \
506
- > "$br/devos/specs/2026-01-02-real/verifications/verification.md"
507
- bt="$tmp/branch-transcript.txt"
508
-
509
- echo "Verdict: DONE: devos/specs/x" > "$bt"
510
- (cd "$br" && bash "$DOD" "$bt"); check "dod: a placeholder spec resolves to the branch's spec" 0 $?
511
-
512
- echo "Verdict: DONE" > "$bt"
513
- (cd "$br" && bash "$DOD" "$bt"); check "dod: a bare verdict resolves on a spec branch" 0 $?
514
-
515
- # The fallback is not a bypass: with no report on the branch's spec, naming is
516
- # still required and a placeholder still blocks.
517
- rm -rf "$br/devos/specs/2026-01-02-real/verifications"
518
- echo "Verdict: DONE: devos/specs/x" > "$bt"
519
- (cd "$br" && bash "$DOD" "$bt" 2>/dev/null); check "dod: branch fallback needs a real report" 2 $?
520
-
521
- # --- test-integrity-rubric.sh ---------------------------------------------
522
- old="$tmp/old.txt" new="$tmp/new.txt"
523
-
524
- printf 'expect(a).toBe(1)\n' > "$old"
525
- printf 'const x = 2\n' > "$new"
526
- out=$(bash "$RUBRIC" --format copilot --file src/main.ts --old "$old" --new "$new")
527
- [[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: non-test file allows" 0 $?
528
-
529
- printf 'it("works", () => { expect(a).toBe(1) })\n' > "$old"
530
- printf 'it.skip("works", () => { expect(a).toBe(1) })\n' > "$new"
531
- out=$(bash "$RUBRIC" --format copilot --file src/main.test.ts --old "$old" --new "$new")
532
- [[ "$out" == *'"deny"'* ]]; check "rubric: added .skip denies" 0 $?
533
-
534
- printf 'expect(a).toBe(1)\nexpect(b).toBe(2)\n' > "$old"
535
- printf 'expect(a).toBe(1)\n' > "$new"
536
- out=$(bash "$RUBRIC" --format copilot --file src/main.test.ts --old "$old" --new "$new")
537
- [[ "$out" == *'"deny"'* ]]; check "rubric: removed assertion denies" 0 $?
538
-
539
- printf 'expect(a).toBe(1)\n' > "$old"
540
- printf 'expect(a).toBe(1)\nexpect(b).toBe(2)\n' > "$new"
541
- out=$(bash "$RUBRIC" --format copilot --file src/main.test.ts --old "$old" --new "$new")
542
- [[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: added assertion allows" 0 $?
543
-
544
- printf 'expect(a).toBe(1)\n' > "$new"
545
- out=$(bash "$RUBRIC" --format copilot --file src/new.test.ts --old "$tmp/absent.txt" --new "$new")
546
- [[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: brand-new test file allows" 0 $?
547
-
548
- # Regression: bare "spec" substring in a path (DevOS's own devos/specs/*.md
549
- # artifacts) must never be treated as a test file, even if the prose content
550
- # happens to contain words like "expect"/"assert".
551
- printf 'Expect the API to return 200.\n' > "$old"
552
- printf 'A change to what we expect from the API.\n' > "$new"
553
- out=$(bash "$RUBRIC" --format copilot --file devos/specs/2026-01-01-foo/spec.md --old "$old" --new "$new")
554
- [[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: devos/specs/*.md is never a test file" 0 $?
555
-
556
- printf 'assert.strictEqual(a, 1)\n' > "$old"
557
- printf 'assert.strictEqual(a, 1)\nassert.strictEqual(b, 2)\n' > "$new"
558
- out=$(bash "$RUBRIC" --format copilot --file devos/specs/2026-01-01-foo/decisions.md --old "$old" --new "$new")
559
- [[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: decisions.md is never a test file" 0 $?
560
-
561
- # Bare "test.js" (no dot-test-dot, this project's own eval fixtures' shape) still counts as a test file.
562
- printf 'assert.strictEqual(a, 1)\nassert.strictEqual(b, 2)\n' > "$old"
563
- printf 'assert.strictEqual(a, 1)\n' > "$new"
564
- out=$(bash "$RUBRIC" --format copilot --file test.js --old "$old" --new "$new")
565
- [[ "$out" == *'"deny"'* ]]; check "rubric: bare test.js still detects removed assertion" 0 $?
566
-
567
- # Work-tree-relative test directories: what every hook actually passes. A
568
- # leading-slash-only pattern waved all three of these straight through.
569
- for relpath in tests/login.js test/helpers.js __tests__/setup.js; do
570
- out=$(bash "$RUBRIC" --format copilot --file "$relpath" --old "$old" --new "$new")
571
- [[ "$out" == *'"deny"'* ]]; check "rubric: relative $relpath is gated" 0 $?
572
- done
573
-
574
- # The neighbouring word that must stay allowed: DevOS's own devos/specs/ tree.
575
- out=$(bash "$RUBRIC" --format copilot --file specs/login.md --old "$old" --new "$new")
576
- [[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: relative specs/ is not a test dir" 0 $?
577
-
578
- # No --format: the default exit-code contract, used by Claude Code's own
579
- # wrapper (test-integrity-hook-claude.sh) and by any future adapter with a
580
- # plain allow/deny hook.
581
- printf 'it("works", () => { expect(a).toBe(1) })\n' > "$old"
582
- printf 'it.skip("works", () => { expect(a).toBe(1) })\n' > "$new"
583
- bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
584
- check "rubric: no --format, added .skip denies (exit 2)" 2 $?
585
-
586
- # A value-bearing option with nothing after it is a malformed invocation, not a
587
- # policy decision. Reading $2 anyway died under `set -u` with exit 1, which
588
- # run-gate.sh forwards to the harness as a block carrying a shell error instead
589
- # of a reason, so a typo in a hook's arguments bricked every edit.
590
- for opt in --format --file --old --new; do
591
- bash "$RUBRIC" "$opt" >/dev/null 2>&1 </dev/null
592
- check "rubric: valueless $opt allows rather than dying" 0 $?
593
- done
594
- [[ "$(bash "$RUBRIC" --format copilot 2>/dev/null)" == '{"permissionDecision":"allow"}' ]]
595
- check "rubric: a valueless option still emits the adapter's allow decision" 0 $?
596
- bash "$RUBRIC" --write --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
597
- check "rubric: a well-formed --write invocation still denies (exit 2)" 2 $?
598
-
599
- printf 'expect(a).toBe(1)\n' > "$old"
600
- printf 'expect(a).toBe(1)\nexpect(b).toBe(2)\n' > "$new"
601
- bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new"
602
- check "rubric: no --format, added assertion allows (exit 0)" 0 $?
603
-
604
- # --- implementation output boundary -----------------------------------------
605
- # SC IDs and framework metadata belong in devos/ artifacts. Project code and
606
- # tests use behavior-facing names and carry no trace of the process that wrote
607
- # them. The edit gate applies this to source as well as tests.
608
- printf "it('SC-3: rejects an expired token', () => {})\n" > "$new"
609
- bash "$RUBRIC" --file src/auth.test.ts --old "$tmp/absent.txt" --new "$new" 2>/dev/null
610
- check "boundary: SC ID in a new test name denies" 2 $?
611
-
612
- printf 'def test_sc_3_rejects_expired_token():\n pass\n' > "$new"
613
- bash "$RUBRIC" --file tests/auth_test.py --old "$tmp/absent.txt" --new "$new" 2>/dev/null
614
- check "boundary: SC ID in a snake-case test name denies" 2 $?
615
-
616
- printf 'it("returns 401 for an expired token", () => {})\n' > "$new"
617
- bash "$RUBRIC" --file src/auth.test.ts --old "$tmp/absent.txt" --new "$new" 2>/dev/null
618
- check "boundary: natural behavior test name allows" 0 $?
619
-
620
- printf '// Implemented by DevOS\n' > "$new"
621
- bash "$RUBRIC" --file src/auth.ts --old "$tmp/absent.txt" --new "$new" 2>/dev/null
622
- check "boundary: framework attribution in source denies" 2 $?
623
-
624
- printf 'The contract is devos-contract: 1\n' > "$new"
625
- bash "$RUBRIC" --file src/contract.ts --old "$tmp/absent.txt" --new "$new" 2>/dev/null
626
- check "boundary: framework contract marker in a string denies" 2 $?
627
-
628
- printf 'Scenario SC-3: rejects an expired token\n' > "$new"
629
- bash "$RUBRIC" --file devos/specs/auth/spec.md --old "$tmp/absent.txt" --new "$new" 2>/dev/null
630
- check "boundary: devos artifact keeps internal trace metadata" 0 $?
631
-
632
- mkdir -p "$tmp/devos/src"
633
- printf '// Implemented by DevOS\n' > "$tmp/devos/src/trace.ts"
634
- bash "$BOUNDARY" --scan --file devos/src/trace.ts 2>/dev/null
635
- check "boundary: code under devos keeps internal trace metadata" 0 $?
636
-
637
- mkdir -p "$tmp/plugins/devos"
638
- printf '// Implemented by DevOS\n' > "$tmp/plugins/devos/trace.ts"
639
- bash "$BOUNDARY" --scan --file plugins/devos/trace.ts 2>/dev/null
640
- check "boundary: a project plugins/devos directory is still project code" 2 $?
641
-
642
- mkdir -p "$tmp/src"
643
- printf 'const value = 1\n' > "$old"
644
- printf '// Implemented by DevOS\nconst value = 1\n' > "$new"
645
- bash "$BOUNDARY" --file devos/../src/traversal.ts --old "$old" --new "$new" 2>/dev/null
646
- check "boundary: traversal cannot disguise project code as a devos artifact" 2 $?
647
-
648
- printf 'Project tools may mention DevOS in onboarding docs.\n' > "$new"
649
- bash "$RUBRIC" --file AGENTS.md --old "$tmp/absent.txt" --new "$new" 2>/dev/null
650
- check "boundary: non-code onboarding file is outside this gate" 0 $?
651
-
652
- printf '// Implemented by DevOS\nconst status = 401\n' > "$old"
653
- printf '// Implemented by DevOS\nconst status = 401\nconst role = "user"\n' > "$new"
654
- bash "$RUBRIC" --file src/auth.ts --old "$old" --new "$new" 2>/dev/null
655
- check "boundary: unrelated edit around pre-existing metadata allows" 0 $?
656
-
657
- mkdir -p "$tmp/src"
658
- printf '// Implemented by DevOS\nconst status = 401\n' > "$tmp/src/auth.ts"
659
- bash "$BOUNDARY" --scan --file src/auth.ts 2>/dev/null
660
- check "boundary: scan mode rejects framework metadata already in source" 2 $?
661
-
662
- # --- run-gate.sh (claude test-integrity) -----------------------------------
663
- # Claude's test-integrity gate is now the single dispatcher: run-gate.sh
664
- # --agent claude test-integrity. The old per-tool shim is gone, so a
665
- # behavioral regression here surfaces as a changed decision below.
666
- claude_ti() { bash "$(dirname "$DOD")/run-gate.sh" --agent claude test-integrity; }
667
- CLAUDE_HOOK="$(dirname "$DOD")/run-gate.sh"
668
-
669
- payload=$(jq -n --arg f src/main.test.ts --arg c 'it("works", () => { expect(a).toBe(1) })' '{tool_input:{file_path:$f,content:$c}}')
670
- echo "$payload" | claude_ti
671
- check "claude hook: Write adding an assertion allows" 0 $?
672
-
673
- payload=$(jq -n --arg f src/main.test.ts --arg c 'it.skip("works", () => { expect(a).toBe(1) })' '{tool_input:{file_path:$f,content:$c}}')
674
- echo "$payload" | claude_ti 2>/dev/null
675
- check "claude hook: Write adding .skip denies" 2 $?
676
-
677
- mkdir -p "$tmp/src"
678
- printf 'it("works", () => { expect(a).toBe(1) })\n' > "$tmp/src/write.test.ts"
679
- payload=$(jq -n --arg f "$tmp/src/write.test.ts" --arg c 'it.skip("works", () => { expect(a).toBe(1) })' '{tool_input:{file_path:$f,content:$c}}')
680
- echo "$payload" | claude_ti 2>/dev/null
681
- check "claude hook: Write to an existing test file adding .skip denies" 2 $?
682
-
683
- payload=$(jq -n --arg f src/main.ts --arg c 'const x = 2' '{tool_input:{file_path:$f,content:$c}}')
684
- echo "$payload" | claude_ti
685
- check "claude hook: non-test file allows" 0 $?
686
-
687
- payload=$(jq -n --arg f src/auth.ts --arg c '// Implemented by DevOS' '{tool_input:{file_path:$f,content:$c}}')
688
- echo "$payload" | claude_ti 2>/dev/null
689
- check "claude hook: process metadata in source denies" 2 $?
690
-
691
- for tool in copilot codex; do
692
- if [[ "$tool" == "copilot" ]]; then
693
- payload=$(jq -n --arg f src/auth.ts --arg c '// Implemented by DevOS' '{toolArgs:{path:$f,content:$c}}')
694
- else
695
- payload=$(jq -n --arg f src/auth.ts --arg c '// Implemented by DevOS' '{tool_input:{path:$f,content:$c}}')
696
- fi
697
- out=$(printf '%s' "$payload" | bash "$CLAUDE_HOOK" --agent "$tool" test-integrity)
698
- [[ "$out" == *'"permissionDecision":"deny"'* ]]
699
- check "$tool hook: process metadata in source denies through the adapter contract" 0 $?
700
- done
701
-
702
- payload=$(jq -n --arg f src/auth.ts --arg c '// Implemented by DevOS' '{tool_input:{path:$f,content:$c}}')
703
- out=$(printf '%s' "$payload" | bash "$CLAUDE_HOOK" --agent cursor test-integrity)
704
- [[ "$out" == '{"permission":"deny"'* ]]
705
- check "cursor hook: process metadata in source denies through the adapter contract" 0 $?
706
-
707
- # --- payload-field contract -------------------------------------------------
708
- # Every case above builds its payload from the same field names the hook reads,
709
- # so the pair agrees with itself whichever names it uses. Rename a field on
710
- # both sides and all of them still pass while the live gate reads `empty` from
711
- # a real payload and exits 0: enforcement gone, suite green. Two checks break
712
- # that circle.
713
- #
714
- # 1. Pin the exact jq paths the hook may read, against the published PreToolUse
715
- # schema (code.claude.com/docs/en/hooks: tool_input carries the tool's own
716
- # arguments, so Edit gives file_path/old_string/new_string and Write gives
717
- # file_path/content). Editing the hook's field names now fails here until
718
- # someone re-reads the doc and updates this list deliberately.
719
- pinned=$(printf '%s\n' \
720
- '.tool_input.content' \
721
- '.tool_input.file_path' \
722
- '.tool_input.new_string' \
723
- '.tool_input.old_string' | sort | tr '\n' ' ')
724
- actual=$(sed -n '/^claude)/,/;;/p' "$CLAUDE_HOOK" | grep -oE "TI_(FILE|CONTENT|NEW|OLD)='\.tool_input\.[a-z_]+" | sed -E "s/^TI_[A-Z]+='//" | sort -u | tr '\n' ' ')
725
- [[ "$actual" == "$pinned" ]]
726
- check "claude hook reads exactly the pinned PreToolUse fields (got: $actual)" 0 $?
727
-
728
- # 2. A full payload as the harness actually sends it, every documented
729
- # top-level field present, not the four-key stub the cases above use. Catches
730
- # a hook that only works on the shape its own tests hand it.
731
- payload='{"session_id":"abc123","prompt_id":"550e8400-e29b-41d4-a716-446655440000","transcript_path":"/tmp/devos-nonexistent.jsonl","cwd":"/repo","permission_mode":"default","hook_event_name":"PreToolUse","tool_name":"Edit","tool_use_id":"toolu_01ABC","tool_input":{"file_path":"src/main.test.ts","old_string":"expect(a).toBe(1)\nexpect(b).toBe(2)","new_string":"expect(a).toBe(1)"}}'
732
- printf '%s' "$payload" | bash "$CLAUDE_HOOK" --agent claude test-integrity 2>/dev/null
733
- check "claude hook: full documented PreToolUse payload still denies" 2 $?
734
-
735
- # --- false-positive regressions: the gate must not block legitimate work ----
736
- # The rubric used to set-difference assertion LINES, so any edit that changed
737
- # the text of a line without removing an assertion was denied: renames,
738
- # expected-value fixes, and every edit to a file that already had a .skip.
739
- # 69 tests passed while that was true, because none of them covered it. A gate
740
- # that blocks legitimate work trains the engineer to bypass it, which is worse
741
- # than no gate. These pin the counting contract.
742
-
743
- printf 'const foo = calc(1)\nexpect(foo).toBe(2)\n' > "$old"
744
- printf 'const result = calc(1)\nexpect(result).toBe(2)\n' > "$new"
745
- bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
746
- check "rubric: rename inside an assertion allows" 0 $?
747
-
748
- printf 'expect(foo).toBe(2)\n' > "$old"
749
- printf 'expect(foo).toBe(3)\n' > "$new"
750
- bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
751
- check "rubric: expected-value fix allows" 0 $?
752
-
753
- printf 'describe.skip("legacy", () => {})\nexpect(a).toBe(1)\n' > "$old"
754
- printf 'describe.skip("legacy", () => {})\nexpect(b).toBe(1)\n' > "$new"
755
- bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
756
- check "rubric: pre-existing .skip, unrelated edit allows" 0 $?
757
-
758
- printf 'expect(o.a).toBe(1)\nexpect(o.b).toBe(2)\nexpect(o.c).toBe(3)\n' > "$old"
759
- printf 'expect(o).toMatchObject({a:1,b:2,c:3})\n' > "$new"
760
- bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
761
- check "rubric: genuine consolidation allows" 0 $?
762
-
763
- # Table-driven consolidation: N copy-pasted `it` blocks collapsed into one
764
- # it.each table legitimately drops the raw match count (the expect line is
765
- # now written once, not once per case).
766
- printf 'it("f(1)", () => { expect(f(1)).toBe(2) })\nit("f(2)", () => { expect(f(2)).toBe(4) })\n' > "$old"
767
- printf 'it.each([[1,2],[2,4]])("f(%%d)", (a, b) => { expect(f(a)).toBe(b) })\n' > "$new"
768
- bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
769
- check "rubric: consolidation into it.each allows" 0 $?
770
-
771
- # ...but a pre-existing it.each is not a licence to delete a case from it,
772
- # same rule as pre-existing toMatchObject above.
773
- printf 'it.each([[1,2]])("f(%%d)", (a, b) => { expect(f(a)).toBe(b) })\nexpect(g()).toBe(3)\n' > "$old"
774
- printf 'it.each([[1,2]])("f(%%d)", (a, b) => { expect(f(a)).toBe(b) })\n' > "$new"
775
- bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
776
- check "rubric: pre-existing it.each does not excuse deletion" 2 $?
777
-
778
- # ...but consolidation markers already present in the OLD content are not a
779
- # licence to delete assertions in the new one.
780
- printf 'expect(o).toMatchObject({a:1})\nexpect(p).toBe(2)\nexpect(q).toBe(3)\n' > "$old"
781
- printf 'expect(o).toMatchObject({a:1})\n' > "$new"
782
- bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
783
- check "rubric: pre-existing toMatchObject does not excuse deletion" 2 $?
784
-
785
- payload=$(printf '{"tool_input":{"file_path":"src/main.test.ts","old_string":"expect(foo).toBe(2)","new_string":"expect(renamed).toBe(2)"}}')
786
- echo "$payload" | bash "$CLAUDE_HOOK" --agent claude test-integrity 2>/dev/null
787
- check "claude hook: Edit renaming inside an assertion allows" 0 $?
788
-
789
- # --- hooks fail open on their own errors, never on policy -------------------
790
- # A hook may exit non-zero only for a deliberate policy decision. Its own
791
- # internal failure (garbage stdin, unreachable rubric, missing transcript)
792
- # must allow: a broken gate must not brick every edit in the session.
793
- #
794
- # Discovered, not enumerated. Listing hooks by name meant a newly added one
795
- # silently had no fail-open coverage until someone remembered to add it — and
796
- # the one hook that got it wrong (check-handoff.sh, which fell THROUGH to
797
- # blocking on its own arithmetic error) was exactly the one nobody had listed.
798
- # Every stdin-reading hook under the plugin and the adapters is swept here, so
799
- # the next one inherits the check for free. The name pattern is deliberately
800
- # wide (any check-*.sh or *hook*.sh); membership is decided by whether the
801
- # script reads a payload from stdin, not by what it is called.
802
- repo_for_hooks="$(cd "$(dirname "$DOD")/../../../.." && pwd)"
803
- hook_count=0
804
- while IFS= read -r hook; do
805
- [[ -f "$hook" ]] || continue
806
- # Only hooks that read a payload from stdin; check-dod-core.sh and the
807
- # rubric take argv and are covered by their own cases above.
808
- grep -q 'input=$(cat' "$hook" || continue
809
- hook_count=$((hook_count + 1))
810
- hname=$(basename "$hook")
811
- for bad_input in 'garbage not json' '' '{}' '{"tool_input":{}}'; do
812
- printf '%s' "$bad_input" | bash "$hook" >/dev/null 2>&1
813
- check "$hname: fails open on [${bad_input:-<empty>}]" 0 $?
814
- done
815
- done < <(find "$repo_for_hooks/plugins" \
816
- \( -name '*hook*.sh' -o -name 'check-*.sh' -o -name 'run-gate.sh' \) 2>/dev/null | sort)
817
-
818
- # The sweep is only meaningful if it actually found hooks. A find that matches
819
- # nothing would otherwise report a silent pass.
820
- (( hook_count >= 3 ))
821
- check "fail-open sweep discovered the hooks (found $hook_count, expect >=5)" 0 $?
822
-
823
- # check-handoff.sh takes a project root on argv rather than stdin, so the
824
- # sweep above skips it. Its own no-op cases are covered below; this pins the
825
- # specific regression: an unreadable root must not reach the blocking path.
826
- bash "$(dirname "$DOD")/check-handoff.sh" /nonexistent-root-xyz >/dev/null 2>&1
827
- check "check-handoff.sh: fails open on an unreadable project root" 0 $?
828
-
829
- # An unreachable rubric (moved clone, broken adapter symlink) must allow.
830
- # The DoD shim always guarded this; the test-integrity shim did not and
831
- # exited 127 with no decision emitted.
832
- repo_root="$(cd "$(dirname "$DOD")/../../../.." && pwd)"
833
- cp "$CLAUDE_HOOK" "$tmp/broken-shim.sh"
834
- sed -i.bak 's#^GATES=.*#GATES="/nonexistent/gates"#' "$tmp/broken-shim.sh"
835
- out=$(echo '{"tool_input":{"path":"a.test.js","content":"x"}}' | bash "$tmp/broken-shim.sh" --agent copilot test-integrity 2>/dev/null)
836
- [[ "$out" == '{"permissionDecision":"allow"}' ]]; check "shim: unreachable rubric allows, emits a decision" 0 $?
837
-
838
- # --- check-handoff.sh ------------
839
- # Solo regression is the important half: `grep -c ... || echo 0` printed
840
- # "0\n0" on zero matches, making the arithmetic test error out and fall
841
- # THROUGH to blocking. And the template documents its own format in a fenced
842
- # block holding a literal `- @name: role:` line, so one real member counted
843
- # as two and armed the team gate for a solo user.
844
- #
845
- # The rebuild replaced a bare `grep '^## Handoff'` presence check with
846
- # the gate docs/team-workflow.md had been describing all along: phase detected
847
- # from artifacts, the phase's own artifact named by a handoff line, every named
848
- # artifact existing on disk, every role resolving to a member. Every state
849
- # below is run against a real fixture project, never asserted from reading.
850
- HANDOFF="$(dirname "$DOD")/check-handoff.sh"
851
- hp="$tmp/handoff-proj"
852
- S="$hp/devos/specs/2026-07-01-alpha"
853
- mkdir -p "$hp/devos/team" "$S"
854
- hrun() { bash "$HANDOFF" "$hp" </dev/null >/dev/null 2>&1; }
855
- herr() { bash "$HANDOFF" "$hp" </dev/null 2>&1 >/dev/null; }
856
-
857
- printf '# spec\n' > "$S/spec-functional.md"
858
- hrun; check "handoff: absent members.md no-ops" 0 $?
859
-
860
- printf '## Format\n\n```\n- @name: role: po|dev, skills: [spec]\n```\n\n## Members\n' > "$hp/devos/team/members.md"
861
- hrun; check "handoff: template-only members.md no-ops" 0 $?
862
-
863
- printf -- '- @alice: role: dev, skills: [spec]\n' >> "$hp/devos/team/members.md"
864
- hrun; check "handoff: single real member no-ops" 0 $?
865
-
866
- # A role outside po|architect|dev|qa does not count as a member: a typo in the
867
- # team file must fall back to solo, never arm a gate naming a role that isn't real.
868
- printf -- '- @typo: role: developer, skills: [spec]\n' >> "$hp/devos/team/members.md"
869
- hrun; check "handoff: unrecognized role does not arm the gate" 0 $?
870
-
871
- printf -- '- @alice: role: dev, skills: [spec]\n- @bob: role: qa, skills: [verify]\n' > "$hp/devos/team/members.md"
872
- hrun; check "handoff: team, requirements phase, no section, blocks" 2 $?
873
- [[ "$(herr)" == *"no ## Handoff section"* ]]
874
- check "handoff: block reason goes to stderr, not stdout" 0 $?
875
-
876
- # Requirements phase: the section lives in spec-functional.md until a plan exists.
877
- printf '# spec\n\n## Handoff\n\n- @po → @dev | spec-functional.md | Requirements confirmed\n' > "$S/spec-functional.md"
878
- hrun; check "handoff: unknown role in a line blocks (@po is not a member)" 2 $?
879
- [[ "$(herr)" == *"not a member or role"* ]]; check "handoff: unknown-role reason names the problem" 0 $?
880
-
881
- printf '# spec\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | Requirements confirmed\n' > "$S/spec-functional.md"
882
- hrun; check "handoff: requirements phase satisfied by a line naming spec-functional.md" 0 $?
883
-
884
- # A handoff may not promise an artifact that does not exist (the DoD gate's
885
- # rule, applied to handoffs).
886
- printf '# spec\n\n## Handoff\n\n- @dev → @qa | verifications/verification.md | ok\n' > "$S/spec-functional.md"
887
- hrun; check "handoff: line naming a nonexistent artifact blocks" 2 $?
888
- [[ "$(herr)" == *"does not exist"* ]]; check "handoff: missing-artifact reason names the path" 0 $?
889
-
890
- printf 'outside\n' > "$tmp/handoff-outside.md"
891
- printf '# spec\n\n## Handoff\n\n- @dev → @qa | ../../../../handoff-outside.md | ok\n' > "$S/spec-functional.md"
892
- hrun; check "handoff: traversal artifact outside the project blocks" 2 $?
893
- [[ "$(herr)" == *"outside the project root"* ]]; check "handoff: outside-artifact reason names the boundary" 0 $?
894
-
895
- ln -s "$tmp/handoff-outside.md" "$hp/linked-outside.md"
896
- printf '# spec\n\n## Handoff\n\n- @dev → @qa | linked-outside.md | ok\n' > "$S/spec-functional.md"
897
- hrun; check "handoff: symlinked artifact outside the project blocks" 2 $?
898
- rm -f "$hp/linked-outside.md" "$tmp/handoff-outside.md"
899
-
900
- printf '# spec\n\n## Handoff\n\n- @dev @qa | spec-functional.md | ok\n' > "$S/spec-functional.md"
901
- hrun; check "handoff: line with no → is not a transition, blocks" 2 $?
902
-
903
- printf '# spec\n\n## Handoff\n\n- @dev → @qa | spec-functional.md\n' > "$S/spec-functional.md"
904
- hrun; check "handoff: line missing its acceptance field blocks" 2 $?
905
-
906
- # Planning phase: spec.md exists, so the section moves there and the
907
- # requirements line alone no longer satisfies the gate.
908
- printf '# spec\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | ok\n' > "$S/spec-functional.md"
909
- printf '# plan\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | ok\n' > "$S/spec.md"
910
- hrun; check "handoff: planning phase, only the stale requirements line, blocks" 2 $?
911
- [[ "$(herr)" == *"produced spec.md"* ]]; check "handoff: phase reason names the artifact produced" 0 $?
912
-
913
- printf -- '- @dev → @qa | spec.md | Plan approved\n' >> "$S/spec.md"
914
- hrun; check "handoff: planning phase satisfied" 0 $?
915
-
916
- # A textual prefix is not the phase artifact. The misleading file exists, so
917
- # this isolates the identity check from the missing-file check.
918
- printf '# plan\n\n## Handoff\n\n- @dev → @qa | not-spec.md | misleading\n' > "$S/spec.md"
919
- printf 'not the plan\n' > "$S/not-spec.md"
920
- hrun; check "handoff: phase artifact matching is exact" 2 $?
921
- [[ "$(herr)" == *"produced spec.md"* ]]; check "handoff: exact-artifact block names the phase requirement" 0 $?
922
- rm -f "$S/not-spec.md"
923
- printf '# plan\n\n## Handoff\n\n- @dev → @qa | spec.md | Plan approved\n' > "$S/spec.md"
924
-
925
- # Verification phase: the report exists, so its own handoff is now required.
926
- mkdir -p "$S/verifications"
927
- printf 'Completeness: CLEAN\nQuality: PASS\n' > "$S/verifications/verification.md"
928
- hrun; check "handoff: verification phase, no line for the report, blocks" 2 $?
929
-
930
- printf -- '- @qa → @dev | verifications/verification.md | Issues found\n' >> "$S/spec.md"
931
- hrun; check "handoff: verification phase satisfied" 0 $?
932
-
933
- # Loop guard: a stop-gate that keeps blocking a turn the model cannot fix would
934
- # block every stop forever. The harness says it already blocked once; honor it.
935
- printf '# plan\n' > "$S/spec.md"
936
- hrun; check "handoff: control case for the loop guard blocks without a payload" 2 $?
937
- for payload in '{"stop_hook_active":true}' '{"stopHookActive":true}' '{"loop_count":1}' 'garbage not json'; do
938
- printf '%s' "$payload" | bash "$HANDOFF" "$hp" >/dev/null 2>&1
939
- check "handoff: allows on [$payload]" 0 $?
940
- done
941
- printf '{"stop_hook_active":false}' | bash "$HANDOFF" "$hp" >/dev/null 2>&1
942
- check "handoff: still blocks when the payload says it has not blocked yet" 2 $?
943
-
944
- # An archived spec is finished work: it must not gate anything.
945
- rm -rf "$S"
946
- mkdir -p "$hp/devos/specs/archive/2026-01-01-old"
947
- printf '# old\n' > "$hp/devos/specs/archive/2026-01-01-old/spec.md"
948
- hrun; check "handoff: only archived specs no-ops" 0 $?
949
-
950
- bash "$HANDOFF" "$tmp/no-such-project-dir" </dev/null >/dev/null 2>&1
951
- check "handoff: unreadable project root no-ops" 0 $?
952
-
953
- # Two specs open at once is the normal team case, and newest-wins alone got it
954
- # wrong: the older spec could reach any phase with no handoff recorded. The
955
- # active spec now comes from the branch, which Phase
956
- # B creates before the first edit.
957
- hb="$tmp/handoff-branch"
958
- mkdir -p "$hb/devos/team"
959
- printf -- '- @alice: role: dev, skills: [spec]\n- @bob: role: qa, skills: [verify]\n' > "$hb/devos/team/members.md"
960
- OLD="$hb/devos/specs/2026-07-01-older"
961
- NEW="$hb/devos/specs/2026-07-09-newer"
962
- mkdir -p "$OLD" "$NEW"
963
- # The newer spec is fully compliant, so anything that blocks is about the older.
964
- printf '# newer\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | ok\n' > "$NEW/spec-functional.md"
965
- printf '# older\n' > "$OLD/spec-functional.md"
966
- (cd "$hb" && git init -q . && git config user.email t@t && git config user.name t && git add -A && git commit -qm init)
967
- hbrun() { (cd "$hb" && git checkout -q -B "$1" 2>/dev/null; bash "$HANDOFF" "$hb" </dev/null >/dev/null 2>&1); }
968
-
969
- hbrun 2026-07-01-older
970
- check "handoff: on a spec branch, gates that spec even when a newer one is clean" 2 $?
971
- printf '# older\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | ok\n' > "$OLD/spec-functional.md"
972
- hbrun 2026-07-01-older
973
- check "handoff: on a spec branch, a satisfied older spec passes" 0 $?
974
-
975
- # Regression: everything not on a spec branch keeps the old newest-wins path.
976
- printf '# older\n' > "$OLD/spec-functional.md"
977
- hbrun some-feature-branch
978
- check "handoff: off a spec branch, newest still wins (older stays ungated)" 0 $?
979
- hbrun 2026-07-01-does-not-exist
980
- check "handoff: a branch naming no spec folder falls back to newest" 0 $?
981
- printf '# newer\n' > "$NEW/spec-functional.md"
982
- hbrun some-feature-branch
983
- check "handoff: off a spec branch, the newest spec is the one gated" 2 $?
984
-
985
- # --- detect-archetype.py ------------------------------
986
- # Success criterion 15 named three repos: React/Tailwind/Storybook, Terraform,
987
- # and a Go CLI that must gain NO frontend sections. The first version detected
988
- # none of the Terraform signals and none of the @scoped packages, because
989
- # quoted globs never expand and `p.deps.@storybook/react` is a JS syntax error.
990
- # Both classes are pinned here so they cannot silently die again.
991
- ARCH="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/init/scripts/detect-archetype.py"
992
- af="$tmp/arch"
993
-
994
- mkdir -p "$af/react/src" "$af/react/.storybook"
995
- cat > "$af/react/package.json" <<'J'
996
- {"name":"web","dependencies":{"react":"^19","next":"^15","@radix-ui/react-dialog":"^1","@reduxjs/toolkit":"^2"},
997
- "devDependencies":{"tailwindcss":"^4","@storybook/react":"^8"}}
998
- J
999
- touch "$af/react/src/Button.module.css"
1000
- out=$(python3 "$ARCH" "$af/react")
1001
- for tag in ui:react ui:next design:storybook design:radix style:tailwind style:css-modules state:redux; do
1002
- [[ "$out" == *"\"$tag\""* ]]; check "archetype: React repo detects $tag" 0 $?
1003
- done
1004
-
1005
- # Terraform: glob-only detection, entirely dead before 0.8.4.
1006
- mkdir -p "$af/tf/modules/vpc"
1007
- printf 'resource "aws_s3_bucket" "b" {}\n' > "$af/tf/modules/vpc/main.tf"
1008
- out=$(python3 "$ARCH" "$af/tf")
1009
- [[ "$out" == *'"infra:terraform"'* ]]; check "archetype: Terraform repo detects infra:terraform" 0 $?
1010
-
1011
- # Go CLI: the zero-cost half of criterion 15. No ui:/design:/style:/state: tag.
1012
- mkdir -p "$af/gocli"
1013
- printf 'module example.com/cli\n\ngo 1.23\n' > "$af/gocli/go.mod"
1014
- printf 'all:\n\tgo build ./...\n' > "$af/gocli/Makefile"
1015
- out=$(python3 "$ARCH" "$af/gocli")
1016
- [[ "$out" == *'"cli:go"'* ]]; check "archetype: Go CLI detects cli:go" 0 $?
1017
-
1018
- # The only table row whose glob names a directory is the GitHub Actions one,
1019
- # and the glob reader used `find -name`, which matches a basename: the pattern
1020
- # `.github/workflows/*.y*ml` could not match any repository, so a repo with
1021
- # real workflows was never detected as ci:github-actions.
1022
- mkdir -p "$af/gha/.github/workflows" "$af/gha/elsewhere/workflows"
1023
- printf 'name: ci\n' > "$af/gha/.github/workflows/build.yml"
1024
- out=$(python3 "$ARCH" "$af/gha")
1025
- [[ "$out" == *'"ci:github-actions"'* ]]; check "archetype: a GitHub Actions workflow is detected" 0 $?
1026
- # Widening the match must not turn every workflow-shaped file into the signal.
1027
- printf 'name: ci\n' > "$af/gha/elsewhere/workflows/build.yml"
1028
- out=$(python3 "$ARCH" "$af/gha/elsewhere")
1029
- [[ "$out" != *'"ci:github-actions"'* && "$out" == *'"detected":false'* ]] \
1030
- && check "archetype: the same file outside .github is not the CI signal" 0 $? \
1031
- || { echo "FAIL archetype: a workflow outside .github was read as GitHub Actions: $out"; fail=1; }
1032
- mkdir -p "$af/gha-root"
1033
- printf 'name: ci\n' > "$af/gha-root/ci.yml"
1034
- out=$(python3 "$ARCH" "$af/gha-root")
1035
- [[ "$out" != *'"ci:github-actions"'* && "$out" == *'"detected":false'* ]] \
1036
- && check "archetype: a root-level workflow file is not the CI signal" 0 $? \
1037
- || { echo "FAIL archetype: a root-level ci.yml was read as GitHub Actions: $out"; fail=1; }
1038
-
1039
- # The TABLE reader is `IFS='|' read -r tag kind arg arg2`, so a row with five
1040
- # fields silently folds the fifth into arg2 and the rule tests a filename that
1041
- # cannot exist. One row per library keeps every field count at four.
1042
- for lib in click typer; do
1043
- mkdir -p "$af/py-$lib"
1044
- printf 'dependencies = ["%s"]\n' "$lib" > "$af/py-$lib/pyproject.toml"
1045
- out=$(python3 "$ARCH" "$af/py-$lib")
1046
- [[ "$out" == *'"cli:python"'* ]]; check "archetype: Python $lib CLI detects cli:python" 0 $?
1047
- done
1048
- [[ "$out" != *'"ui:'* && "$out" != *'"design:'* && "$out" != *'"style:'* && "$out" != *'"state:'* ]]
1049
- check "archetype: Go CLI gains no frontend sections" 0 $?
1050
-
1051
- mkdir -p "$af/bare"
1052
- out=$(python3 "$ARCH" "$af/bare")
1053
- [[ "$out" == '{"archetypes":[],"detected":false}' ]]; check "archetype: bare dir reports undetected" 0 $?
1054
-
1055
- python3 "$ARCH" "$af/no-such-dir" >/dev/null 2>&1
1056
- check "archetype: missing dir exits 1, does not hang" 1 $?
1057
-
1058
- # No node on PATH must not change the answer: detection is pure POSIX tools.
1059
- out=$(PATH="/usr/bin:/bin" python3 "$ARCH" "$af/react" 2>/dev/null)
1060
- [[ "$out" == *'"design:storybook"'* ]]; check "archetype: works without node on PATH" 0 $?
1061
-
1062
- # --- detect-archetype.py --facts -------------
1063
- # The same stack at two different majors must produce two different fact
1064
- # blocks: that difference is the entire value of the mode, so it is the test.
1065
- # Both bugs pinned below were found by running these fixtures before the
1066
- # roadmap box was ticked, not after.
1067
- mkdir -p "$af/v18/node_modules/react" "$af/v18/node_modules/next"
1068
- cat > "$af/v18/package.json" <<'J'
1069
- {
1070
- "dependencies": { "react": "^18.2.0", "next": "^14.1.0" },
1071
- "devDependencies": { "tailwindcss": "^3.4.1", "storybook": "^8.0.0" },
1072
- "scripts": {
1073
- "dev": "next dev",
1074
- "build": "next build",
1075
- "typecheck": "tsc --noEmit",
1076
- "test:a11y": "axe ./dist"
1077
- }
1078
- }
1079
- J
1080
- echo '{"name":"react","version":"18.3.1"}' > "$af/v18/node_modules/react/package.json"
1081
- echo '{"name":"next","version":"14.2.5"}' > "$af/v18/node_modules/next/package.json"
1082
- out=$(python3 "$ARCH" --facts "$af/v18")
1083
- [[ "$out" == *"| react | 18.3.1 | node_modules (installed) |"* ]]
1084
- check "facts: installed version wins over the declared range" 0 $?
1085
- [[ "$out" == *"| tailwindcss | ^3.4.1 |"* ]]
1086
- check "facts: range read from a one-line object is the right key's value" 0 $?
1087
- [[ "$out" == *"range, NOT resolved"* ]]
1088
- check "facts: an unresolved range is labelled, never passed off as installed" 0 $?
1089
-
1090
- # Same stack, next major, nothing installed. Single-line manifest, and a
1091
- # `storybook` script whose value must never be reported as a version.
1092
- mkdir -p "$af/v19"
1093
- cat > "$af/v19/package.json" <<'J'
1094
- {"dependencies":{"react":"^19.0.0","next":"^15.3.0","@mui/material":"^6.1.0"},
1095
- "devDependencies":{"tailwindcss":"^4.0.9"},
1096
- "scripts":{"build":"next build","storybook":"storybook dev"}}
1097
- J
1098
- out19=$(python3 "$ARCH" --facts "$af/v19")
1099
- [[ "$out19" == *"| react | ^19.0.0 |"* && "$out19" == *"| tailwindcss | ^4.0.9 |"* ]]
1100
- check "facts: second major reports its own versions" 0 $?
1101
- [[ "$out19" == *"| @mui/material | ^6.1.0 |"* ]]
1102
- check "facts: a scoped package resolves" 0 $?
1103
- # Assert on the version table alone: `storybook` is a legitimate row in the
1104
- # checks table below it, so a whole-output grep would pass no matter what.
1105
- vtable19=${out19%%Declared checks*}
1106
- [[ "$vtable19" != *"| storybook |"* ]]
1107
- check "facts: a script named like a package is not reported as its version" 0 $?
1108
- [[ "$out" != "$out19" ]]
1109
- check "facts: two majors of one stack produce different blocks" 0 $?
1110
-
1111
- # Declared checks: only the keys that prove something, `dev` excluded.
1112
- [[ "$out" == *"npm run typecheck\` | tsc --noEmit |"* && "$out" == *"npm run test:a11y\` | axe ./dist |"* ]]
1113
- check "facts: suffixed check scripts are declared" 0 $?
1114
- [[ "$out" != *"run dev\` | next dev |"* ]]
1115
- check "facts: a server-starting script is not a check" 0 $?
1116
-
1117
- # Tier 41: exactly the keys that prove something, no triage. Eight scripts in,
1118
- # six out: `dev` starts a server and `clean` deletes a directory; neither can
1119
- # fail in a way that means the code is wrong.
1120
- mkdir -p "$af/checks"
1121
- cat > "$af/checks/package.json" <<'J'
1122
- {"dependencies":{"react":"^19"},
1123
- "scripts":{"dev":"vite","clean":"rm -rf dist","build":"vite build","lint":"eslint .",
1124
- "typecheck":"tsc --noEmit","test":"vitest run","test:e2e":"playwright test",
1125
- "test:a11y":"axe ./dist"}}
1126
- J
1127
- outc=$(python3 "$ARCH" --facts "$af/checks")
1128
- n=$(printf '%s\n' "$outc" | sed -n '/| Source | Command | Runs |/,$p' | grep -c '^| package.json |')
1129
- [[ "$n" == 6 ]]
1130
- check "checks: six qualifying scripts of eight are declared" 0 $?
1131
- for k in build lint typecheck test test:e2e test:a11y; do
1132
- [[ "$outc" == *"run $k\` |"* ]]; check "checks: $k is declared" 0 $?
1133
- done
1134
- [[ "$outc" != *"run dev\` |"* && "$outc" != *"run clean\` |"* ]]
1135
- check "checks: dev and clean are not proof of anything" 0 $?
1136
-
1137
- # The check table decides what the verifier runs, so a name filter narrow
1138
- # enough to drop a functional suite silently decides that suite never runs.
1139
- # The lockfile names the runner: `npm run` on a pnpm repo is a command the
1140
- # project does not use.
1141
- mkdir -p "$af/wide"
1142
- touch "$af/wide/pnpm-lock.yaml"
1143
- cat > "$af/wide/package.json" <<'J'
1144
- {"dependencies":{"react":"^19"},
1145
- "scripts":{"test-func":"node ./scripts/test.mjs func","coverage-unit":"c8 mocha",
1146
- "format:check":"prettier --check .","lint:check":"eslint .",
1147
- "test:watch":"vitest","deploy-prod":"./k8s.mjs deploy"}}
1148
- J
1149
- outw=$(python3 "$ARCH" --facts "$af/wide")
1150
- for k in test-func coverage-unit format:check lint:check; do
1151
- [[ "$outw" == *"pnpm $k\` |"* ]]; check "checks: $k survives the name filter" 0 $?
1152
- done
1153
- [[ "$outw" != *"npm run "* ]]
1154
- check "checks: the lockfile picks the package manager" 0 $?
1155
- [[ "$outw" != *"test:watch"* ]]
1156
- check "checks: a watch script is not a check (it never exits)" 0 $?
1157
- [[ "$outw" != *"deploy-prod"* ]]
1158
- check "checks: an operational script is still excluded" 0 $?
1159
-
1160
- # Bun writes bun.lock, the text lockfile that has been its default since
1161
- # v1.2. A repo carrying only that file was reported as npm, and the table is
1162
- # what the verifier runs.
1163
- mkdir -p "$af/bun-text" "$af/bun-binary"
1164
- cat > "$af/bun-text/package.json" <<'J'
1165
- {"scripts":{"test":"bun test","lint":"eslint ."}}
1166
- J
1167
- printf '{}\n' > "$af/bun-text/bun.lock"
1168
- outbt=$(python3 "$ARCH" --facts "$af/bun-text")
1169
- [[ "$outbt" == *'| `bun run test` |'* && "$outbt" != *"npm run"* ]]
1170
- check "checks: the bun text lockfile picks bun run" 0 $?
1171
- cp "$af/bun-text/package.json" "$af/bun-binary/package.json"
1172
- printf 'binary\n' > "$af/bun-binary/bun.lockb"
1173
- outbb=$(python3 "$ARCH" --facts "$af/bun-binary")
1174
- [[ "$outbb" == *'| `bun run test` |'* ]]
1175
- check "checks: the legacy bun binary lockfile still picks bun run" 0 $?
1176
-
1177
- # A task runner is a manifest too. Reading package.json alone reported "no
1178
- # declared checks" for every Make/just project, DevOS's own repo included.
1179
- mkdir -p "$af/runner"
1180
- cat > "$af/runner/justfile" <<'J'
1181
- PLUGIN_JSONS := "a b"
1182
-
1183
- default:
1184
- @just --list
1185
-
1186
- verify:
1187
- ./run-checks.sh
1188
-
1189
- lint target="all":
1190
- eslint {{target}}
1191
-
1192
- deploy:
1193
- ./ship.sh
1194
- J
1195
- outr=$(python3 "$ARCH" --facts "$af/runner")
1196
- [[ "$outr" == *'`just verify`'* && "$outr" == *'`just lint`'* ]]
1197
- check "checks: justfile recipes are declared checks" 0 $?
1198
- [[ "$outr" != *"just deploy"* && "$outr" != *"just default"* && "$outr" != *"PLUGIN_JSONS"* ]]
1199
- check "checks: recipes that prove nothing, and assignments, stay out" 0 $?
1200
-
1201
- mkdir -p "$af/mk"
1202
- printf '.PHONY: test\ntest:\n\tgo test ./...\n\nrelease:\n\t./ship.sh\n' > "$af/mk/Makefile"
1203
- outm=$(python3 "$ARCH" --facts "$af/mk")
1204
- [[ "$outm" == *'`make test`'* && "$outm" != *"make release"* ]]
1205
- check "checks: Makefile targets are declared checks" 0 $?
1206
-
1207
- # Where a command runs is part of the command: a suite that only passes inside
1208
- # the app container fails on the host for reasons the diff did not cause.
1209
- mkdir -p "$af/ctx"
1210
- printf 'services:\n app:\n build: .\n redis:\n image: redis\nvolumes:\n data:\n' > "$af/ctx/docker-compose.yml"
1211
- outx=$(python3 "$ARCH" --facts "$af/ctx")
1212
- [[ "$outx" == *"Execution context:"* && "$outx" == *"app, redis"* ]]
1213
- check "checks: compose services are reported as the execution context" 0 $?
1214
- [[ "$outx" != *"app, redis, data"* ]]
1215
- check "checks: a sibling top-level key is not read as a service" 0 $?
1216
- [[ "$(python3 "$ARCH" --facts "$af/mk")" != *"Execution context:"* ]]
1217
- check "checks: no compose file means no context line" 0 $?
1218
-
1219
- # Tier 42: the same `dev` script that cannot be a check is exactly what a
1220
- # browser needs, so it surfaces as a preview candidate instead of vanishing.
1221
- [[ "$outc" == *'Preview candidates: `dev` (`vite`)'* ]]
1222
- check "preview: the dev script surfaces as a preview candidate" 0 $?
1223
- out=$(python3 "$ARCH" --facts "$af/gocli")
1224
- [[ "$out" == *"Preview candidates: none found"* ]]
1225
- check "preview: a repo with nothing to serve says so" 0 $?
1226
-
1227
- # Non-JS and empty repos degrade to an honest answer, never a guessed one.
1228
- out=$(python3 "$ARCH" --facts "$af/gocli")
1229
- [[ "$out" == *"| go (toolchain) | 1.23 | go.mod |"* ]]
1230
- check "facts: go toolchain pin is read" 0 $?
1231
- out=$(python3 "$ARCH" --facts "$af/bare")
1232
- [[ "$out" == *"No resolvable versions found"* ]]
1233
- check "facts: bare dir says so instead of inventing versions" 0 $?
1234
-
1235
- # The default mode is untouched: every existing caller still gets JSON.
1236
- out=$(python3 "$ARCH" "$af/v18")
1237
- [[ "$out" == '{"archetypes":['* ]]
1238
- check "facts: default mode still prints JSON" 0 $?
1239
-
1240
- # --- validate-spec.sh -------------------------------------------------------
1241
- VSPEC="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/implement/scripts/validate-spec.sh"
1242
-
1243
- good="$tmp/devos/specs/2026-01-01-good"
1244
- mkdir -p "$good"
1245
- cat > "$good/spec-functional.md" <<'MD'
1246
- # Good Feature
1247
- ## 1. Goal
1248
- Users MUST be able to export their data.
1249
- ## 3. Functionality
1250
- **Scenarios:**
1251
- - Scenario SC-1: happy export / WHEN the user clicks export THEN a CSV downloads
1252
- ## 5. Out of Scope
1253
- - PDF export
1254
- MD
1255
- bash "$VSPEC" "$good" >/dev/null; check "vspec: well-formed spec passes" 0 $?
1256
-
1257
- historical="$tmp/devos/specs/archive/2026-01-01-historical"
1258
- mkdir -p "$historical"
1259
- cp "$good/spec-functional.md" "$historical/spec-functional.md"
1260
- printf '# Historical capsule\n' > "$historical/spec.md"
1261
- bash "$VSPEC" "$historical" >/dev/null; check "vspec: archived plan skips current plan gates" 0 $?
1262
-
1263
- anchors="$tmp/devos/specs/2026-01-01-anchors"
1264
- mkdir -p "$anchors/planning"
1265
- cp "$good/spec-functional.md" "$anchors/spec-functional.md"
1266
- cat > "$anchors/planning/notes.md" <<'MD'
1267
- # Planning notes
1268
-
1269
- ## Explicit contract anchors
1270
- - Function `exportUsers()` in `src/users.js`
1271
- MD
1272
- out=$(bash "$VSPEC" "$anchors" 2>&1); rc=$?
1273
- grep -q "spec-functional.md drops explicit contract anchors from planning/notes.md" <<<"$out" && [[ $rc -eq 1 ]] \
1274
- && check "vspec: dropped explicit contract anchors fail with the gap named" 0 $? \
1275
- || { echo "FAIL vspec: dropped explicit contract anchors not caught"; echo "$out"; fail=1; }
1276
-
1277
- cat "$good/spec-functional.md" > "$anchors/spec-functional.md"
1278
- printf '\n## Appendix\n**Explicit contract anchors:** `None supplied`\n' >> "$anchors/spec-functional.md"
1279
- out=$(bash "$VSPEC" "$anchors" 2>&1); rc=$?
1280
- grep -q "spec-functional.md records None supplied despite explicit contract anchors" <<<"$out" && [[ $rc -eq 1 ]] \
1281
- && check "vspec: negated explicit contract anchors fail with the gap named" 0 $? \
1282
- || { echo "FAIL vspec: negated explicit contract anchors not caught"; echo "$out"; fail=1; }
1283
-
1284
- cat "$good/spec-functional.md" > "$anchors/spec-functional.md"
1285
- printf '\n## Appendix\n**Explicit contract anchors:** Function `exportUsers()` in `src/users.js`\n' >> "$anchors/spec-functional.md"
1286
- bash "$VSPEC" "$anchors" >/dev/null; check "vspec: carried explicit contract anchors pass" 0 $?
1287
-
1288
- cat "$good/spec-functional.md" > "$anchors/spec-functional.md"
1289
- printf '\n## Appendix\n**Explicit contract anchors:** Function `exportUsers()` in `src/users.js`\n' >> "$anchors/spec-functional.md"
1290
- cat > "$anchors/spec.md" <<'MD'
1291
- ## Local Craft
1292
- None
1293
- ## Goal / Technical Context / Architecture / Technical Requirements / Existing Code to Leverage
1294
- The implementation MUST call `exportUsers()`.
1295
- ## Design Convergence
1296
- - **Direct MVP**: call the existing function
1297
- - **Failed Items**: None
1298
- - **Adopted Additions**: None
1299
- - **Rejected Additions**: None
1300
- ## Code Map
1301
- - `src/users.js`: update the export
1302
- ## Implementation Plan
1303
- ### Task Group 1: update export
1304
- **Files:** `src/users.js`
1305
- **Scenarios covered:** SC-1
1306
- MD
1307
- bash "$VSPEC" "$anchors" >/dev/null; check "vspec: plan carries explicit contract anchors" 0 $?
1308
-
1309
- cat "$good/spec-functional.md" > "$anchors/spec-functional.md"
1310
- printf '\n## Appendix\n**Explicit contract anchors:** Function `exportUsers()`\n' >> "$anchors/spec-functional.md"
1311
- out=$(bash "$VSPEC" "$anchors" 2>&1); rc=$?
1312
- grep -q "spec-functional.md drops explicit contract anchor: \`src/users.js\`" <<<"$out" && [[ $rc -eq 1 ]] \
1313
- && check "vspec: a dropped explicit contract token fails with the token named" 0 $? \
1314
- || { echo "FAIL vspec: a dropped explicit contract token not caught"; echo "$out"; fail=1; }
1315
-
1316
- cat > "$anchors/spec.md" <<'MD'
1317
- ## Local Craft
1318
- None
1319
- ## Goal / Technical Context / Architecture / Technical Requirements / Existing Code to Leverage
1320
- The implementation MUST preserve the supplied behavior.
1321
- ## Design Convergence
1322
- - **Direct MVP**: update the existing implementation
1323
- - **Failed Items**: None
1324
- - **Adopted Additions**: None
1325
- - **Rejected Additions**: None
1326
- ## Code Map
1327
- - update the implementation file
1328
- ## Implementation Plan
1329
- ### Task Group 1: update export
1330
- **Files:** `src/users.js`
1331
- **Scenarios covered:** SC-1
1332
- MD
1333
- out=$(bash "$VSPEC" "$anchors" 2>&1); rc=$?
1334
- grep -q "spec.md drops explicit contract anchor from Technical Requirements: \`exportUsers()\`" <<<"$out" \
1335
- && grep -q "spec.md drops explicit contract anchor from Code Map: \`src/users.js\`" <<<"$out" \
1336
- && [[ $rc -eq 1 ]] \
1337
- && check "vspec: a plan that drops contract anchors fails with both locations named" 0 $? \
1338
- || { echo "FAIL vspec: dropped plan contract anchors not caught"; echo "$out"; fail=1; }
1339
-
1340
- bad="$tmp/devos/specs/2026-01-01-bad"
1341
- mkdir -p "$bad"
1342
- printf '# Bad\n## Goal\nx\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1343
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: no scenarios fails" 1 $?
1344
-
1345
- printf '# Bad\n## Goal\nx\n- Scenario SC-1: broken, no when-then\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1346
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: malformed scenario fails" 1 $?
1347
-
1348
- # SC-n IDs form the internal plan index. Source tests use behavior-facing
1349
- # names, and the output-boundary gate keeps the IDs out of implementation files.
1350
- printf '# Bad\n## Goal\nx\n- Scenario: unnumbered / WHEN a THEN b\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1351
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: scenario without SC-n ID fails" 1 $?
1352
-
1353
- printf '# Bad\n## Goal\nx\n- Scenario SC-1: a / WHEN a THEN b\n- Scenario SC-1: b / WHEN c THEN d\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1354
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: duplicate SC-n IDs fail" 1 $?
1355
-
1356
- printf '# Bad\n## Goal\nx\n- Scenario SC-1: a / WHEN a THEN b\n- Scenario SC-2: b / WHEN c THEN d\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1357
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: sequential SC-n IDs pass" 0 $?
1358
-
1359
- cp "$good/spec-functional.md" "$bad/spec-functional.md"
1360
- seq 1 301 | sed 's/^/line /' > "$bad/spec.md"
1361
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: oversized capsule fails" 1 $?
1362
-
1363
- bash "$VSPEC" "$tmp/devos/specs/absent" >/dev/null 2>&1; check "vspec: missing folder exits 2" 2 $?
1364
-
1365
- # Scenario traceability across artifacts: the plan's prose rule (every SC-n
1366
- # covered somewhere, every citation real) as a gate. Both directions matter:
1367
- # an uncovered scenario is a planning gap; a ghost citation verifies nothing.
1368
- mkdir -p "$bad"
1369
- cp "$good/spec-functional.md" "$bad/spec-functional.md"
1370
- cat > "$bad/spec.md" <<'MD'
1371
- ## Local Craft
1372
- None
1373
- ## Design Convergence
1374
- - **Direct MVP**: call it directly
1375
- ## Implementation Plan
1376
- ### Task Group 1
1377
- **Files:** `a.ts`
1378
- **Scenarios covered:** SC-1
1379
- MD
1380
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: fully covered scenarios pass" 0 $?
1381
-
1382
- cat > "$bad/spec.md" <<'MD'
1383
- ## Implementation Plan
1384
- ### Task Group 1
1385
- **Scenarios covered:** none
1386
- MD
1387
- out=$(bash "$VSPEC" "$bad" 2>&1); echo "$out" | grep -q "SC-1 is declared but covered nowhere" && check "vspec: uncovered scenario fails with the gap named" 0 $? || { echo "FAIL vspec: uncovered scenario not caught"; fail=1; }
1388
-
1389
- cat > "$bad/spec.md" <<'MD'
1390
- ## Implementation Plan
1391
- ### Task Group 1
1392
- **Scenarios covered:** SC-9
1393
- MD
1394
- out=$(bash "$VSPEC" "$bad" 2>&1); rc=$?
1395
- grep -q "cites SC-9 which spec-functional.md does not declare" <<<"$out" && [[ $rc -eq 1 ]] \
1396
- && check "vspec: ghost citation fails" 0 $? || { echo "FAIL vspec: ghost citation not caught"; fail=1; }
1397
-
1398
- # C route exemption: Invariant specs declare no scenarios, so nothing to trace
1399
- printf '# C\n## Goal\nx\n## Invariant\n`./run` -> exit 0\n**Surface:** none, internal only\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1400
- cat > "$bad/spec.md" <<'MD'
1401
- ## Local Craft
1402
- None
1403
- ## Design Convergence
1404
- - **Direct MVP**: call it directly
1405
- ## Implementation Plan
1406
- ### Task Group 1
1407
- **Files:** `a.ts`
1408
- MD
1409
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: C route exempt from scenario tracing" 0 $?
1410
-
1411
- # The oversized capsule above is still on disk and would fail every spec here
1412
- # for the wrong reason. These cases are about spec-functional.md alone.
1413
- rm -f "$bad/spec.md"
1414
-
1415
- # Normative wording. These moved out of write-spec.md's self-check, where they
1416
- # were prose asking the writer to grade itself. The negative cases matter most:
1417
- # an over-eager matcher that flags "shoulder" or a deliberately deferred item
1418
- # would make the gate unusable, and the word-boundary bug it guards against
1419
- # (gawk's \y, silent on darwin's awk) passes every positive test by matching
1420
- # nothing at all.
1421
- printf '# H\n## Goal\nx\n## 3. Functionality\n- The export should include archived rows\n- Scenario SC-1: s / WHEN a THEN b\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1422
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge word in a requirement fails" 1 $?
1423
-
1424
- printf '# H\n## Goal\nx\n## 3. Functionality\n- Scenario SC-1: s / WHEN a THEN it may download\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1425
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge word inside a scenario fails" 1 $?
1426
-
1427
- printf '# H\n## Goal\nx\n## 3. Functionality\n- The shoulder strap MUST be listed\n- Scenario SC-1: s / WHEN a THEN b\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1428
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge substring in a word does not fail" 0 $?
1429
-
1430
- printf '# H\n## Goal\nx\n## 3. Functionality\n- Scenario SC-1: s / WHEN a THEN b\n## Out of Scope\n- PDF export, may come later\n' > "$bad/spec-functional.md"
1431
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge in Out of Scope is allowed" 0 $?
1432
-
1433
- printf '# H\n## Goal\nx\n## 3. Functionality\n- Scenario SC-1: s / WHEN a THEN b\n## 3bis. Behavior delta\n**Modified:** "users should wait" -> instant\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1434
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge in a quoted Behavior delta is allowed" 0 $?
1435
-
1436
- # A functional spec describes behavior; a fence is implementation leaking in.
1437
- printf '# C\n## Goal\nx\n## 3. Functionality\n- Scenario SC-1: s / WHEN a THEN b\n```ts\ninterface X {}\n```\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1438
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: code block in a functional spec fails" 1 $?
1439
-
1440
- printf '# D\n## Goal\nx\n- Scenario SC-1: s / WHEN a THEN b\n## Behavior delta\n**Removed:** old login flow\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1441
- rm -f "$bad/spec.md"
1442
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: removal without reason fails" 1 $?
1443
-
1444
- printf '# D\n## Goal\nx\n- Scenario SC-1: s / WHEN a THEN b\n## Behavior delta\n**Removed:** (none)\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
1445
- bash "$VSPEC" "$bad" >/dev/null; check "vspec: empty removed marker passes" 0 $?
1446
-
1447
- # --- check-requires.py ----------------------------------------
1448
- CREQ="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/status/scripts/check-requires.py"
1449
- SDIFF="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/status/scripts/spec-diff.sh"
1450
-
1451
- # A comparison name is one directory name, not a path expression. Without
1452
- # this boundary, ../ can resolve a sibling directory and make status compare
1453
- # an artifact outside devos/specs.
1454
- escape_root="$tmp/spec-diff-escape"
1455
- mkdir -p "$escape_root/devos/specs" "$tmp/spec-diff-outside"
1456
- printf '# Outside plan\n' > "$tmp/spec-diff-outside/spec.md"
1457
- out=$(bash "$SDIFF" '../../../spec-diff-outside' '../../../spec-diff-outside' "$escape_root" 2>&1); rc=$?
1458
- grep -q "spec names must be single directory names without path separators" <<<"$out" \
1459
- && [[ $rc -eq 2 ]] \
1460
- && check "spec-diff: traversal spec names are rejected" 0 $? \
1461
- || { echo "FAIL spec-diff: traversal spec name was not rejected"; echo "$out"; fail=1; }
1462
-
1463
- # A same-named active folder is not interchangeable with an archived record.
1464
- # If the active plan is missing, comparing the historical plan silently hides
1465
- # the active artifact failure and reports a clean self-comparison instead.
1466
- sdf="$tmp/spec-diff-resolution/devos/specs"
1467
- mkdir -p "$sdf/active-only" "$sdf/archive/active-only"
1468
- printf '# Archived plan\n## Implementation Plan\n### Task Group 1: archived\n' \
1469
- > "$sdf/archive/active-only/spec.md"
1470
- out=$(bash "$SDIFF" active-only active-only "$tmp/spec-diff-resolution" 2>&1); rc=$?
1471
- grep -q "active spec 'active-only' exists but its spec.md is missing" <<<"$out" \
1472
- && [[ $rc -eq 1 ]] \
1473
- && check "spec-diff: an active folder without spec.md does not fall back to archive" 0 $? \
1474
- || { echo "FAIL spec-diff: active missing-plan fallback not caught"; echo "$out"; fail=1; }
1475
-
1476
- # The normal archived lookup remains available when no active folder exists.
1477
- mkdir -p "$sdf/archive/archive-only"
1478
- printf '# Archived plan\n## Implementation Plan\n### Task Group 1: archived\n' \
1479
- > "$sdf/archive/archive-only/spec.md"
1480
- bash "$SDIFF" archive-only archive-only "$tmp/spec-diff-resolution" >/dev/null
1481
- check "spec-diff: an archive-only spec still resolves" 0 $?
1482
-
1483
- # When both records exist and both are complete, the active plan is the one
1484
- # being worked on and must win over the historical copy.
1485
- printf '# Active plan\n## Implementation Plan\n### Task Group 1: active\n' \
1486
- > "$sdf/active-only/spec.md"
1487
- out=$(bash "$SDIFF" active-only archive-only "$tmp/spec-diff-resolution" 2>&1)
1488
- grep -q "Task groups differ" <<<"$out" \
1489
- && check "spec-diff: a complete active plan wins over an archived namesake" 0 $? \
1490
- || { echo "FAIL spec-diff: active plan did not win over archive"; echo "$out"; fail=1; }
1491
-
1492
- # Every other plan reader (validate-spec, check-converge,
1493
- # sizing-calibration) accepts any run of spaces around
1494
- # the words in a group header. spec-diff read the literal '### Task Group', so
1495
- # a plan written '### Task Group 2: gamma' had no groups at all and the
1496
- # comparison called two different plans identical.
1497
- sdg="$tmp/spec-diff-groups/devos/specs"
1498
- mkdir -p "$sdg/wide" "$sdg/loose"
1499
- printf '# Wide\n## Implementation Plan\n### Task Group 1: alpha\n### Task Group 2: beta\n' \
1500
- > "$sdg/wide/spec.md"
1501
- printf '# Loose\n## Implementation Plan\n### Task Group 1: alpha\n### Task Group 2: gamma\n' \
1502
- > "$sdg/loose/spec.md"
1503
- out=$(bash "$SDIFF" wide loose "$tmp/spec-diff-groups" 2>&1)
1504
- grep -qx "Task groups differ:" <<<"$out" \
1505
- && grep -q "Task Group 2: gamma" <<<"$out" \
1506
- && check "spec-diff: a group header with extra spacing is still compared" 0 $? \
1507
- || { echo "FAIL spec-diff: extra spacing hid a group from the comparison"; echo "$out"; fail=1; }
1508
-
1509
- # The other direction: widening the pattern must not promote a fourth-level
1510
- # heading to a task group, or any deeply nested note named like a group would
1511
- # start showing up in the structural diff.
1512
- sdg4="$tmp/spec-diff-level/devos/specs"
1513
- mkdir -p "$sdg4/deep" "$sdg4/single"
1514
- printf '# Deep\n## Implementation Plan\n#### Task Group 1: alpha\n### Task Group 2: beta\n' \
1515
- > "$sdg4/deep/spec.md"
1516
- printf '# Single\n## Implementation Plan\n### Task Group 2: beta\n' > "$sdg4/single/spec.md"
1517
- out=$(bash "$SDIFF" deep single "$tmp/spec-diff-level" 2>&1)
1518
- grep -qx "Task groups are identical." <<<"$out" \
1519
- && check "spec-diff: a fourth-level heading is not counted as a task group" 0 $? \
1520
- || { echo "FAIL spec-diff: a fourth-level heading was read as a task group"; echo "$out"; fail=1; }
1521
-
1522
- # The colon separator is the other half of the same heading. check-converge
1523
- # reads it and the plan gates read it, so a plan written '### Task Group: alpha'
1524
- # had no groups here: both sides of the comparison came out empty and two plans
1525
- # with different groups were reported identical.
1526
- sdgc="$tmp/spec-diff-colon/devos/specs"
1527
- mkdir -p "$sdgc/colon-a" "$sdgc/colon-b" "$sdgc/numbered-a" "$sdgc/numbered-b"
1528
- printf '# Colon A\n## Implementation Plan\n### Task Group: alpha\n' > "$sdgc/colon-a/spec.md"
1529
- printf '# Colon B\n## Implementation Plan\n### Task Group: beta\n' > "$sdgc/colon-b/spec.md"
1530
- out=$(bash "$SDIFF" colon-a colon-b "$tmp/spec-diff-colon" 2>&1)
1531
- grep -qx "Task groups differ:" <<<"$out" \
1532
- && grep -q "Task Group: beta" <<<"$out" \
1533
- && check "spec-diff: a colon-form group header is still compared" 0 $? \
1534
- || { echo "FAIL spec-diff: a colon-form group header was hidden from the comparison"; echo "$out"; fail=1; }
1535
- # The same two plans written in the numbered form have to keep reporting the
1536
- # same difference, so the fix cannot be a pattern that simply matches more.
1537
- printf '# Numbered A\n## Implementation Plan\n### Task Group 1: alpha\n' > "$sdgc/numbered-a/spec.md"
1538
- printf '# Numbered B\n## Implementation Plan\n### Task Group 2: beta\n' > "$sdgc/numbered-b/spec.md"
1539
- out=$(bash "$SDIFF" numbered-a numbered-b "$tmp/spec-diff-colon" 2>&1)
1540
- grep -qx "Task groups differ:" <<<"$out" \
1541
- && grep -q "Task Group 2: beta" <<<"$out" \
1542
- && check "spec-diff: a numbered group header still reports the same difference" 0 $? \
1543
- || { echo "FAIL spec-diff: numbered group comparison changed"; echo "$out"; fail=1; }
1544
- # A heading that only starts like a group is still not one.
1545
- printf '# Deep colon\n## Implementation Plan\n#### Task Group: alpha\n### Task Group: beta\n' \
1546
- > "$sdgc/colon-a/spec.md"
1547
- out=$(bash "$SDIFF" colon-a colon-b "$tmp/spec-diff-colon" 2>&1)
1548
- grep -qx "Task groups are identical." <<<"$out" \
1549
- && check "spec-diff: a fourth-level colon-form heading is not a task group" 0 $? \
1550
- || { echo "FAIL spec-diff: a fourth-level colon-form heading was read as a task group"; echo "$out"; fail=1; }
1551
-
1552
- reqfix="$tmp/reqproj"
1553
- mkdir -p "$reqfix/devos/specs/archive/2026-01-01-auth"
1554
- mkdir -p "$reqfix/devos/specs/2026-01-02-export"
1555
- mkdir -p "$reqfix/devos/specs/2026-01-03-billing"
1556
- mkdir -p "$reqfix/devos/specs/2026-01-04-broken"
1557
- mkdir -p "$reqfix/devos/specs/2026-01-05-a"
1558
- mkdir -p "$reqfix/devos/specs/2026-01-06-b"
1559
- printf '# Auth\n' > "$reqfix/devos/specs/archive/2026-01-01-auth/spec-functional.md"
1560
- printf '# Export\nRequires: [2026-01-01-auth]\n' > "$reqfix/devos/specs/2026-01-02-export/spec-functional.md"
1561
- printf '# Billing\nRequires: [export]\n' > "$reqfix/devos/specs/2026-01-03-billing/spec-functional.md"
1562
- printf '# Broken\nRequires: [ghost-spec]\n' > "$reqfix/devos/specs/2026-01-04-broken/spec-functional.md"
1563
- printf '# A\nRequires: [2026-01-06-b]\n' > "$reqfix/devos/specs/2026-01-05-a/spec-functional.md"
1564
- printf '# B\nRequires: [2026-01-05-a]\n' > "$reqfix/devos/specs/2026-01-06-b/spec-functional.md"
1565
- mkdir -p "$reqfix/devos/specs/2026-01-07-c" "$reqfix/devos/specs/2026-01-08-d"
1566
- printf '# C\nRequires: [2026-01-08-d]\n' > "$reqfix/devos/specs/2026-01-07-c/spec-functional.md"
1567
- printf '# D\nRequires: [2026-01-07-c]\n' > "$reqfix/devos/specs/2026-01-08-d/spec-functional.md"
1568
-
1569
- out=$(python3 "$CREQ" "$reqfix")
1570
- [[ "$out" == *"2026-01-03-billing blocked-by: 2026-01-02-export"* ]]
1571
- check "creq: unmet dep shows blocked-by (suffix resolve)" 0 $?
1572
- [[ "$out" == *"2026-01-02-export blocks: 2026-01-03-billing"* ]]
1573
- check "creq: reverse blocks: annotation" 0 $?
1574
- echo "$out" | grep -E '^2026-01-02-export ' | grep -qv 'blocked-by'
1575
- check "creq: archived dep does not block" 0 $?
1576
-
1577
- shadow="$tmp/active-archive-shadow"
1578
- mkdir -p "$shadow/devos/specs/a" "$shadow/devos/specs/archive/a" "$shadow/devos/specs/b"
1579
- printf '# A active\n' > "$shadow/devos/specs/a/spec-functional.md"
1580
- printf '# A archived\n' > "$shadow/devos/specs/archive/a/spec-functional.md"
1581
- printf '# B\nRequires: [a]\n' > "$shadow/devos/specs/b/spec-functional.md"
1582
- out=$(python3 "$CREQ" "$shadow")
1583
- grep -Fqx 'b blocked-by: a' <<<"$out"; check "creq: active namesake shadows archived dependency" 0 $?
1584
- out=$(python3 "$CREQ" "$reqfix")
1585
-
1586
- [[ "$out" == *"BROKEN"*"ghost-spec"* ]]
1587
- check "creq: unknown target is BROKEN" 0 $?
1588
- [[ "$out" == *"CYCLE"*"2026-01-05-a"*"2026-01-06-b"* ]]
1589
- check "creq: cycle detected" 0 $?
1590
-
1591
- fout=$(python3 "$CREQ" --spec 2026-01-05-a "$reqfix")
1592
- grep -Fqx '2026-01-05-a blocked-by: 2026-01-06-b blocks: 2026-01-06-b' <<<"$fout" \
1593
- && grep -Fqx 'CYCLE 2026-01-05-a -> 2026-01-06-b -> 2026-01-05-a' <<<"$fout" \
1594
- && check "creq: --spec reports status and a cycle through another spec" 0 $? \
1595
- || { echo "FAIL creq: --spec omitted its cross-spec cycle"; echo "$fout"; fail=1; }
1596
- [[ "$fout" != *"2026-01-07-c"* ]]
1597
- check "creq: --spec omits unrelated cycle nodes" 0 $?
1598
-
1599
- # A short suffix matching two specs resolves to neither, and reporting that as
1600
- # a missing folder sends the reader looking for a typo instead of at the name
1601
- # they have to spell out. The candidates are the whole diagnosis.
1602
- amb="$tmp/req-ambiguous"
1603
- mkdir -p "$amb/devos/specs/archive/2026-01-01-a-export" \
1604
- "$amb/devos/specs/archive/2026-01-02-b-export" \
1605
- "$amb/devos/specs/2026-01-03-billing"
1606
- printf '# Billing\nRequires: [export]\n' \
1607
- > "$amb/devos/specs/2026-01-03-billing/spec-functional.md"
1608
- aout=$(python3 "$CREQ" --spec 2026-01-03-billing "$amb" 2>&1)
1609
- grep -Fq 'AMBIGUOUS' <<<"$aout" && grep -Fq '2026-01-01-a-export' <<<"$aout" \
1610
- && grep -Fq '2026-01-02-b-export' <<<"$aout" \
1611
- && check "creq: an ambiguous short suffix names every candidate" 0 $? \
1612
- || { echo "FAIL creq: ambiguous suffix not diagnosed"; echo "$aout"; fail=1; }
1613
- ! grep -Fq 'no folder under specs/ or archive/' <<<"$aout"
1614
- check "creq: an ambiguous suffix is not reported as a missing folder" 0 $?
1615
- dout=$(python3 "$CREQ" --doctor "$amb" 2>&1)
1616
- grep -Fq 'AMBIGUOUS Requires: export in 2026-01-03-billing' <<<"$dout" \
1617
- && check "creq: doctor reports the ambiguous suffix with its candidates" 0 $? \
1618
- || { echo "FAIL creq: doctor did not report the ambiguous suffix"; echo "$dout"; fail=1; }
1619
- vamb=$(python3 "$CREQ" --verify "$amb/devos/specs/2026-01-03-billing" "$amb" 2>&1)
1620
- grep -Fq 'AMBIGUOUS short suffix' <<<"$vamb" \
1621
- && grep -Fq '2026-01-01-a-export' <<<"$vamb" \
1622
- && grep -Fq '2026-01-02-b-export' <<<"$vamb" \
1623
- && ! grep -Fq 'unknown target' <<<"$vamb" \
1624
- && check "creq: verify preserves the ambiguous classification and candidates" 0 $? \
1625
- || { echo "FAIL creq: verify misdiagnosed an ambiguous suffix"; echo "$vamb"; fail=1; }
1626
- printf '# Billing\nRequires: [ghost]\n' > "$amb/devos/specs/2026-01-03-billing/spec-functional.md"
1627
- gout=$(python3 "$CREQ" --spec 2026-01-03-billing "$amb" 2>&1)
1628
- grep -Fq 'no folder under specs/ or archive/' <<<"$gout" && ! grep -Fq 'AMBIGUOUS' <<<"$gout" \
1629
- && check "creq: a genuinely missing target is still BROKEN" 0 $? \
1630
- || { echo "FAIL creq: missing target misdiagnosed"; echo "$gout"; fail=1; }
1631
- printf '# Billing\nRequires: [2026-01-02-b-export]\n' \
1632
- > "$amb/devos/specs/2026-01-03-billing/spec-functional.md"
1633
- sout=$(python3 "$CREQ" --spec 2026-01-03-billing "$amb" 2>&1)
1634
- ! grep -Fq 'AMBIGUOUS' <<<"$sout" && ! grep -Fq 'BROKEN' <<<"$sout" \
1635
- && check "creq: the full name of a shared suffix resolves" 0 $? \
1636
- || { echo "FAIL creq: full name did not resolve"; echo "$sout"; fail=1; }
1637
-
1638
- # A Requires heading followed by the numbered sections of the write-spec.md
1639
- # skeleton. The old terminator required a letter after the hashes, so
1640
- # `## 5. Out of Scope` never closed the list and `- none` became a
1641
- # dependency named "none".
1642
- reqsec="$tmp/req-section"
1643
- mkdir -p "$reqsec/devos/specs/2026-01-04-a" "$reqsec/devos/specs/2026-01-05-b" \
1644
- "$reqsec/devos/specs/2026-01-05-c" "$reqsec/devos/specs/2026-01-06-d"
1645
- printf '# A\n## 1. Goal\nx\n## 4. Functional Constraints\n### Requires:\n- 2026-01-05-b\n## 5. Out of Scope\n- none\n## Appendix\n' \
1646
- > "$reqsec/devos/specs/2026-01-04-a/spec-functional.md"
1647
- printf '# B\n## 1. Goal\nx\n' > "$reqsec/devos/specs/2026-01-05-b/spec-functional.md"
1648
- printf '# C\n## 1. Goal\nx\n### Requires:\n- 2026-01-06-d\n### Rationale\n- 2026-01-05-b\n' \
1649
- > "$reqsec/devos/specs/2026-01-05-c/spec-functional.md"
1650
- printf '# D\n## 1. Goal\nx\n' > "$reqsec/devos/specs/2026-01-06-d/spec-functional.md"
1651
- rsout=$(python3 "$CREQ" "$reqsec" 2>&1)
1652
- ! grep -Fq 'BROKEN' <<<"$rsout" \
1653
- && check "creq: a numbered section after Requires does not leak its bullets into the graph" 0 $? \
1654
- || { echo "FAIL creq: a numbered section leaked a dependency"; echo "$rsout"; fail=1; }
1655
- grep -Fqx '2026-01-04-a blocked-by: 2026-01-05-b' <<<"$rsout" \
1656
- && grep -Fqx '2026-01-05-b blocks: 2026-01-04-a' <<<"$rsout" \
1657
- && check "creq: the declared dependency is still collected under a Requires heading" 0 $? \
1658
- || { echo "FAIL creq: the Requires bullet was dropped"; echo "$rsout"; fail=1; }
1659
- grep -Fqx '2026-01-05-c blocked-by: 2026-01-06-d' <<<"$rsout" \
1660
- && check "creq: a lettered heading after Requires still ends the section" 0 $? \
1661
- || { echo "FAIL creq: a lettered heading no longer ends the Requires section"; echo "$rsout"; fail=1; }
1662
-
1663
- # The reader finds the declaration with grep -i but used to strip the label
1664
- # with a case-sensitive `s/^[Rr]equires:/`, so `REQUIRES: [x]` was accepted as
1665
- # a declaration and then parsed as a dependency literally named
1666
- # "REQUIRES: [x" and reported BROKEN. The heading branch matched one spelling.
1667
- reqcase="$tmp/req-case"
1668
- mkdir -p "$reqcase/devos/specs/2026-01-05-a" "$reqcase/devos/specs/2026-01-06-b"
1669
- printf '# B\n' > "$reqcase/devos/specs/2026-01-06-b/spec-functional.md"
1670
- reqcase_ok=1
1671
- for spelling in 'Requires:' 'requires:' 'REQUIRES:' 'ReQuIrEs:'; do
1672
- printf '# A\n\n%s [2026-01-06-b]\n' "$spelling" \
1673
- > "$reqcase/devos/specs/2026-01-05-a/spec-functional.md"
1674
- cout=$(python3 "$CREQ" "$reqcase" 2>&1)
1675
- { grep -Fqx '2026-01-05-a blocked-by: 2026-01-06-b' <<<"$cout" \
1676
- && ! grep -Fq 'BROKEN' <<<"$cout"; } || { reqcase_ok=0; echo "--- inline '$spelling'"; echo "$cout"; }
1677
- done
1678
- for spelling in Requires REQUIRES; do
1679
- printf '# A\n\n### %s:\n- 2026-01-06-b\n## 5. Out of Scope\n- none\n' "$spelling" \
1680
- > "$reqcase/devos/specs/2026-01-05-a/spec-functional.md"
1681
- cout=$(python3 "$CREQ" "$reqcase" 2>&1)
1682
- { grep -Fqx '2026-01-05-a blocked-by: 2026-01-06-b' <<<"$cout" \
1683
- && ! grep -Fq 'BROKEN' <<<"$cout"; } || { reqcase_ok=0; echo "--- heading '$spelling'"; echo "$cout"; }
1684
- done
1685
- [[ "$reqcase_ok" -eq 1 ]] \
1686
- && check "creq: every accepted case of the Requires label resolves the dependency" 0 $? \
1687
- || { echo "FAIL creq: a case-folded Requires label did not resolve"; fail=1; }
1688
- printf '# A\n\nREQUIRES: [2026-02-01-ghost]\n' \
1689
- > "$reqcase/devos/specs/2026-01-05-a/spec-functional.md"
1690
- cout=$(python3 "$CREQ" "$reqcase" 2>&1)
1691
- grep -Fqx 'BROKEN 2026-01-05-a Requires: 2026-02-01-ghost (no folder under specs/ or archive/)' <<<"$cout" \
1692
- && check "creq: a truly missing target is still BROKEN, naming the target alone" 0 $? \
1693
- || { echo "FAIL creq: case folding turned a missing target into a resolved one"; echo "$cout"; fail=1; }
1694
-
1695
- vout=$(python3 "$CREQ" --verify "$reqfix/devos/specs/2026-01-03-billing" "$reqfix")
1696
- [[ "$vout" == *"INFO depends on 2026-01-02-export, not yet shipped"* ]]
1697
- check "creq: --verify unmet is INFO" 0 $?
1698
- vout=$(python3 "$CREQ" --verify "$reqfix/devos/specs/2026-01-02-export" "$reqfix")
1699
- [[ -z "$vout" ]]
1700
- check "creq: --verify met dep is silent" 0 $?
1701
-
1702
- dout=$(python3 "$CREQ" --doctor "$reqfix")
1703
- [[ "$dout" == *"BROKEN Requires: ghost-spec"* ]]
1704
- check "creq: --doctor flags broken edge" 0 $?
1705
- [[ "$dout" == *"CYCLE"* ]]
1706
- check "creq: --doctor flags cycle" 0 $?
1707
- [[ "$dout" != *"blocked-by"* ]]
1708
- check "creq: --doctor omits status blocked-by" 0 $?
1709
-
1710
- # --transitive: the A->B->C chain.
1711
- tout=$(python3 "$CREQ" --transitive "$reqfix")
1712
- [[ "$tout" == *"2026-01-03-billing -> 2026-01-02-export"* ]]
1713
- check "creq: --transitive resolves a two-hop chain" 0 $?
1714
-
1715
- # --timeline: the same T25 claimed verification "against a capability with 3 or
1716
- # more merges" — every behavior file in this repo has exactly one. It used
1717
- # `git log --diff-filter=M`, which excludes the A commit that CREATES the file,
1718
- # so a capability with only its creating commit printed a bare header. And
1719
- # `git log` exits 0 while printing nothing, so the `|| echo` fallback it carried
1720
- # never fired. Both pinned here: a real file must yield its diff, an
1721
- # empty-history file must say so.
1722
- tlfix="$tmp/timeline-fix"
1723
- mkdir -p "$tlfix/devos/product/behaviors"
1724
- (
1725
- cd "$tlfix" && git init -q . && git config user.email t@t && git config user.name t
1726
- printf '# Search\n\n## Added\n- filters by tag\n' > devos/product/behaviors/search.md
1727
- git add -A && git commit -qm 'chore(spec): complete 2026-01-01-search, merge behavior delta'
1728
- )
1729
- tout=$(cd "$tlfix" && python3 "$CREQ" --timeline search .)
1730
- [[ "$tout" == *"filters by tag"* ]]
1731
- check "creq: --timeline shows the creating commit's diff" 0 $?
1732
-
1733
- # The explicit project root must control both capability lookup and git history.
1734
- # The caller may be outside the project, as the gate test process is.
1735
- tout=$(python3 "$CREQ" --timeline search "$tlfix")
1736
- [[ "$tout" == *"filters by tag"* ]]
1737
- check "creq: --timeline resolves project root from outside" 0 $?
1738
- [[ "$tout" == *"2026-01-01-search"* ]]
1739
- check "creq: --timeline carries the spec name from the commit subject" 0 $?
1740
-
1741
- touch "$tlfix/devos/product/behaviors/untracked.md"
1742
- tout=$(cd "$tlfix" && python3 "$CREQ" --timeline untracked .)
1743
- [[ "$tout" == *"no git history"* ]]
1744
- check "creq: --timeline reports empty history instead of a bare header" 0 $?
1745
-
1746
- python3 "$CREQ" --timeline no-such-capability "$tlfix" >/dev/null 2>&1
1747
- check "creq: --timeline missing capability exits 1" 1 $?
1748
-
1749
- printf '# Outside behavior\n' > "$tlfix/devos/product/outside.md"
1750
- (cd "$tlfix" && git add -A && git commit -qm 'chore: outside behavior')
1751
- out=$(python3 "$CREQ" --timeline ../outside "$tlfix" 2>&1); rc=$?
1752
- grep -Fq 'capability must be a single name without path separators' <<<"$out" && [[ $rc -eq 2 ]] \
1753
- && check "creq: --timeline rejects capability path traversal" 0 $? \
1754
- || { echo "FAIL creq: --timeline accepted capability path traversal"; echo "$out"; fail=1; }
1755
-
1756
- # --- render-rule.py ----------------------------------------------------------
1757
- RENDER="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/init/scripts/render-rule.py"
1758
-
1759
- rproj="$tmp/rule-project"
1760
- mkdir -p "$rproj/.github" "$rproj/devos/standards/rules/02-languages"
1761
- cat > "$rproj/devos/standards/rules/02-languages/ts-strict.md" <<'MD'
1762
- ---
1763
- globs: src/**/*.ts, src/**/*.tsx
1764
- ---
1765
- # TypeScript strict mode
1766
- - No `any` without a comment justifying it
1767
- MD
1768
- (cd "$rproj" && python3 "$RENDER" devos/standards/rules/02-languages/ts-strict.md >/dev/null)
1769
- check "render-rule: copilot instructions written" 0 $?
1770
- [[ -f "$rproj/.github/instructions/languages-ts-strict.instructions.md" ]]
1771
- check "render-rule: copilot output exists" 0 $?
1772
- grep -q 'applyTo: "src/\*\*/\*.ts,src/\*\*/\*.tsx"' "$rproj/.github/instructions/languages-ts-strict.instructions.md"
1773
- check "render-rule: copilot applyTo is comma-separated" 0 $?
1774
-
1775
- mkdir -p "$rproj/devos/standards/rules/00-architecture"
1776
- cat > "$rproj/devos/standards/rules/00-architecture/layering.md" <<'MD'
1777
- ---
1778
- ---
1779
- # Layering
1780
- - Presentation never imports Infrastructure directly
1781
- MD
1782
- (cd "$rproj" && python3 "$RENDER" devos/standards/rules/00-architecture/layering.md >/dev/null)
1783
- grep -q 'applyTo: "\*\*"' "$rproj/.github/instructions/architecture-layering.instructions.md"
1784
- check "render-rule: all-files rule sets applyTo ** on copilot" 0 $?
1785
-
1786
- noinstall="$tmp/no-install-project"
1787
- mkdir -p "$noinstall/devos/standards/rules/09-other"
1788
- cp "$rproj/devos/standards/rules/00-architecture/layering.md" "$noinstall/devos/standards/rules/09-other/layering.md"
1789
- out=$(cd "$noinstall" && python3 "$RENDER" devos/standards/rules/09-other/layering.md)
1790
- [[ "$out" == *"nothing to render"* ]]
1791
- check "render-rule: no Copilot install renders nothing, says so" 0 $?
1792
-
1793
- python3 "$RENDER" /nonexistent/rule.md >/dev/null 2>&1
1794
- check "render-rule: missing file exits 2" 2 $?
1795
-
1796
- # Frontmatter that opens but never closes: the old guard only checked the
1797
- # opener, so the rule text was swallowed as frontmatter and an applyTo-only
1798
- # instruction file was written and reported as a success.
1799
- printf '%s\n' '---' 'globs: src/**/*.ts' '' '# Truncated' '' '- Body that lost its fence.' \
1800
- > "$rproj/devos/standards/rules/02-languages/truncated.md"
1801
- out=$(cd "$rproj" && python3 "$RENDER" devos/standards/rules/02-languages/truncated.md 2>&1); rc=$?
1802
- grep -Fq 'unterminated frontmatter' <<<"$out" && [[ $rc -eq 1 ]]
1803
- check "render-rule: unterminated frontmatter is malformed input" 0 $?
1804
- [[ ! -e "$rproj/.github/instructions/languages-truncated.instructions.md" ]] && [[ "$out" != *"wrote "* ]]
1805
- check "render-rule: unterminated frontmatter writes no instruction file" 0 $?
1806
-
1807
- # --- clarify.sh ----------------------------------------
1808
- CLARIFY="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/spec/scripts/clarify.sh"
1809
- FIXTURES="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/fixtures"
1810
-
1811
- dirty="$tmp/devos/specs/2026-01-01-dirty"
1812
- mkdir -p "$dirty/planning"
1813
- cp "$FIXTURES/clarify-dirty/spec-functional.md" "$dirty/spec-functional.md"
1814
- cp "$FIXTURES/clarify-dirty/planning/notes.md" "$dirty/planning/notes.md"
1815
- out=$(bash "$CLARIFY" "$dirty")
1816
- [[ "$out" == *"Vague quantifier"* ]]; check "clarify: vague quantifier detected" 0 $?
1817
- [[ "$out" == *"3 clarification(s) found, 3 suppressed"* ]]; check "clarify: cap at 3, rest suppressed" 0 $?
1818
- grep -q "^## Clarifications Needed" "$dirty/spec-functional.md"; check "clarify: section appended to dirty spec" 0 $?
1819
-
1820
- # The appended blocks quote the lines that triggered them, so a rerun used to
1821
- # read its own output back as fresh ambiguity: the same spec reported 3 found /
1822
- # 3 suppressed once and 3 found / 6 suppressed after.
1823
- out=$(bash "$CLARIFY" "$dirty" | head -1)
1824
- [[ "$out" == "clarify: 3 clarification(s) found, 3 suppressed (cap: 3)" ]]; check "clarify: a rerun does not count its own output as ambiguity" 0 $?
1825
- out=$(bash "$CLARIFY" "$dirty" | head -1)
1826
- [[ "$out" == "clarify: 3 clarification(s) found, 3 suppressed (cap: 3)" ]]; check "clarify: repeated reruns stay stable" 0 $?
1827
- [[ "$(grep -c '^## Clarifications Needed' "$dirty/spec-functional.md")" == 1 ]]; check "clarify: reruns do not append a second section" 0 $?
1828
-
1829
- clean="$tmp/devos/specs/2026-01-02-clean"
1830
- mkdir -p "$clean"
1831
- cp "$FIXTURES/clarify-clean/spec-functional.md" "$clean/spec-functional.md"
1832
- out=$(bash "$CLARIFY" "$clean")
1833
- [[ "$out" == "clarify: 0 clarification(s) found, 0 suppressed (cap: 3)" ]]; check "clarify: zero false positives on clean spec" 0 $?
1834
- ! grep -q "^## Clarifications Needed" "$clean/spec-functional.md"; check "clarify: no section appended to clean spec" 0 $?
1835
-
1836
- answered="$tmp/devos/specs/2026-01-03-answered"
1837
- mkdir -p "$answered/planning"
1838
- printf '%s\n' '- Option A: Use CSV format' '- Option B: Use JSON format' \
1839
- > "$answered/planning/notes.md"
1840
- cat > "$answered/spec-functional.md" <<'MD'
1841
- # Answered spec
1842
-
1843
- ## 1. Goal
1844
- Users can export data.
1845
-
1846
- ## 2. User Stories
1847
- ### US-1: Export data
1848
- As a user, I want to export data.
1849
-
1850
- **Acceptance criteria:**
1851
- - [ ] Export completes within 1 second
1852
-
1853
- ## 3. Functionality
1854
- **Business rules:**
1855
- - Export must use CSV
1856
-
1857
- **Answer: A**
1858
-
1859
- ## Out of Scope
1860
- - Import
1861
- MD
1862
- out=$(bash "$CLARIFY" "$answered")
1863
- [[ "$out" == "clarify: 0 clarification(s) found, 0 suppressed (cap: 3)" ]]; check "clarify: explicit Answer resolves a planning fork" 0 $?
1864
- ! grep -q "Unresolved fork" <<<"$out"; check "clarify: answered fork is not reported" 0 $?
1865
-
1866
- note_answered="$tmp/devos/specs/2026-01-04-note-answered"
1867
- mkdir -p "$note_answered/planning"
1868
- printf '%s\n' '- Option A: Use CSV format' '- Option B: Use JSON format' '**Answer: A**' \
1869
- > "$note_answered/planning/notes.md"
1870
- cat > "$note_answered/spec-functional.md" <<'MD'
1871
- # Planning-note answer
1872
-
1873
- ## 1. Goal
1874
- Users can export data.
1875
-
1876
- ## 2. User Stories
1877
- ### US-1: Export data
1878
- As a user, I want to export data.
1879
-
1880
- **Acceptance criteria:**
1881
- - [ ] Export completes within 1 second
1882
-
1883
- ## 3. Functionality
1884
- **Business rules:**
1885
- - Export must use CSV
1886
-
1887
- ## Out of Scope
1888
- - Import
1889
- MD
1890
- out=$(bash "$CLARIFY" "$note_answered")
1891
- [[ "$out" == "clarify: 0 clarification(s) found, 0 suppressed (cap: 3)" ]]; check "clarify: planning-note Answer resolves a planning fork" 0 $?
1892
- ! grep -q "Unresolved fork" <<<"$out"; check "clarify: planning-note answered fork is not reported" 0 $?
1893
-
1894
- # --- doctor.sh: stale spec workspaces ---------------------------------------
1895
- DOCTOR="$(cd "$(dirname "$DOD")/../../../.." && pwd)/scripts/doctor.sh"
1896
-
1897
- repo="$tmp/wtrepo"
1898
- git init -q -b main "$repo"
1899
- mkdir -p "$repo/devos/specs/2026-01-01-merged" "$repo/devos/specs/2026-01-01-open"
1900
- printf "# s\n" > "$repo/devos/specs/2026-01-01-merged/spec.md"
1901
- printf "# s\n" > "$repo/devos/specs/2026-01-01-open/spec.md"
1902
- git -C "$repo" -c user.email=t@t -c user.name=t add -A
1903
- git -C "$repo" -c user.email=t@t -c user.name=t commit -q -m init
1904
- git -C "$repo" branch 2026-01-01-merged
1905
- git -C "$repo" worktree add -q "$tmp/wt-merged" 2026-01-01-merged
1906
- git -C "$repo" branch 2026-01-01-open
1907
- git -C "$repo" worktree add -q "$tmp/wt-open" 2026-01-01-open
1908
- git -C "$tmp/wt-open" -c user.email=t@t -c user.name=t commit -q --allow-empty -m "unmerged work"
1909
-
1910
- out=$(cd "$repo" && HOME="$tmp" bash "$DOCTOR" 2>/dev/null)
1911
- [[ "$out" == *"STALE worktree"*"2026-01-01-merged"* ]]; check "doctor: merged-branch worktree flagged" 0 $?
1912
- [[ "$out" != *"2026-01-01-open is fully merged"* ]]; check "doctor: unmerged worktree not flagged" 0 $?
1913
-
1914
- dneg="$tmp/doctor-negated"
1915
- mkdir -p "$dneg/home" "$dneg/devos/specs/2026-01-01-active/verifications"
1916
- printf 'Completeness: NOT CLEAN\nQuality: NOT PASS\n' \
1917
- > "$dneg/devos/specs/2026-01-01-active/verifications/verification.md"
1918
- out=$(cd "$dneg" && HOME="$dneg/home" bash "$DOCTOR" 2>&1)
1919
- if grep -Fq 'complete but still active' <<<"$out"; then
1920
- check "doctor: negated verdict is not reported complete" 1 0
1921
- else
1922
- check "doctor: negated verdict is not reported complete" 0 0
1923
- fi
1924
-
1925
- # doctor.sh: same version string, different commit. `claude plugin install`
1926
- # copies a snapshot, so editing the clone without bumping the version leaves
1927
- # Claude Code running the old files while a version-only check reports OK. That
1928
- # is the silent case, and it is the one that makes a fixed skill look unfixed.
1929
- DREPO="$(dirname "$(dirname "$DOCTOR")")"
1930
- dver=$(python3 -c "import json;print(json.load(open('$DREPO/plugins/devos/.claude-plugin/plugin.json'))['version'])")
1931
- fakereg() { # fakereg <sha>
1932
- mkdir -p "$tmp/dhome/.claude/plugins"
1933
- printf '{"version":2,"plugins":{"devos@devos":[{"version":"%s","gitCommitSha":"%s"}]}}\n' \
1934
- "$dver" "$1" > "$tmp/dhome/.claude/plugins/installed_plugins.json"
1935
- }
1936
-
1937
- fakereg "$(git -C "$DREPO" rev-parse HEAD~1 2>/dev/null || echo deadbee)"
1938
- out=$(cd "$DREPO" && HOME="$tmp/dhome" bash "$DOCTOR" 2>/dev/null)
1939
- grep -q "DRIFT devos: both at $dver" <<<"$out"
1940
- check "doctor: a stale snapshot at the same version is drift" 0 $?
1941
-
1942
- fakereg "$(git -C "$DREPO" rev-parse HEAD)"
1943
- out=$(cd "$DREPO" && HOME="$tmp/dhome" bash "$DOCTOR" 2>/dev/null)
1944
- grep -q "OK devos: clone and installed snapshot both at $dver" <<<"$out"
1945
- check "doctor: snapshot at the clone's own commit is not drift" 0 $?
1946
-
1947
- # doctor.sh: broken Requires: edge
1948
- reqdoc="$tmp/reqdoc"
1949
- mkdir -p "$reqdoc/devos/specs/2026-01-04-broken"
1950
- printf '# Broken\nRequires: [ghost-spec]\n' > "$reqdoc/devos/specs/2026-01-04-broken/spec-functional.md"
1951
- out=$(cd "$reqdoc" && HOME="$tmp" bash "$DOCTOR" 2>/dev/null || true)
1952
- [[ "$out" == *"BROKEN Requires: ghost-spec"* ]]
1953
- check "doctor: surfaces broken Requires edge" 0 $?
1954
-
1955
- # Pi extension installed state: dangling symlink fails loudly; a settings.json
1956
- # registration resolving into the clone passes clean.
1957
- dangle="$tmp/pi-dangle"
1958
- mkdir -p "$dangle/.pi/extensions"
1959
- ln -s "$dangle/nowhere/devos.ts" "$dangle/.pi/extensions/devos.ts"
1960
- # Project scope resolves .pi/extensions relative to the project cwd, so the
1961
- # probe must run from inside the dangle project to see its broken link.
1962
- out=$(cd "$dangle" && HOME="$dangle" bash "$DOCTOR" 2>/dev/null || true)
1963
- grep -q "BROKEN pi extension symlink" <<<"$out"; check "doctor: dangling pi extension symlink caught" 0 $?
1964
-
1965
- reg="$tmp/pi-reg"
1966
- mkdir -p "$reg/.pi/agent"
1967
- printf '{"extensions": ["%s/plugins/devos/pi/extension/devos.ts"]}' "$(cd "$(dirname "$DOCTOR")/.." && pwd)" > "$reg/.pi/agent/settings.json"
1968
- out=$(cd "$reqdoc" && HOME="$reg" bash "$DOCTOR" 2>/dev/null)
1969
- grep -q "OK pi extension (settings.json) resolves into this clone" <<<"$out"; check "doctor: settings.json pi extension into clone passes" 0 $?
1970
-
1971
- # --- hook wiring resolves ---------------------------------------------------
1972
- # Every ${CLAUDE_PLUGIN_ROOT} command-hook path must point at a file inside the
1973
- # plugin (a hook may never climb out: install copies only the plugin dir).
1974
- # Resolve each and assert it exists. Swept over the plugin-root hooks.json and
1975
- # every skill body, since skills also invoke gate scripts inline in prose.
1976
- PLUGIN_ROOT="$(cd "$(dirname "$DOD")/../.." && pwd)"
1977
- SKILL="$PLUGIN_ROOT/skills/implement/SKILL.md"
1978
- CLAUDE_HOOKS_JSON="$PLUGIN_ROOT/hooks/hooks.json"
1979
- needle='${CLAUDE_PLUGIN_ROOT}'
1980
- while IFS= read -r cmd; do
1981
- resolved="${cmd/"$needle"/$PLUGIN_ROOT}"
1982
- case "$resolved" in
1983
- "$PLUGIN_ROOT"/*) : ;;
1984
- *) echo "FAIL hook wiring: $cmd climbs out of the plugin root" >&2; fail=$((fail+1)); continue ;;
1985
- esac
1986
- [[ -f "$resolved" ]]; check "hook wiring: ${cmd##*/} resolves inside plugin" 0 $?
1987
- done < <(grep -hoE '\$\{CLAUDE_PLUGIN_ROOT\}[^"]*\.sh' \
1988
- "$CLAUDE_HOOKS_JSON" "$PLUGIN_ROOT"/skills/*/SKILL.md "$PLUGIN_ROOT"/skills/*/references/*.md 2>/dev/null | sort -u)
1989
-
1990
- # --- gate scope parity across tools -----------------------------------------
1991
- # Until 0.10.10 Claude's three gates lived in implement/SKILL.md and
1992
- # spec/SKILL.md frontmatter, which scopes a hook to sessions where that
1993
- # skill is loaded. Every other tool registers the same three at install,
1994
- # session-wide. So a test-file edit from /devos:spec, /devos:review or plain
1995
- # chat was gated on some adapters and ungated on Claude, and nothing anywhere compared
1996
- # the two: each side was individually correct, which is what made the gap
1997
- # invisible. Pin both halves — the plugin root registers all three, and no
1998
- # skill quietly takes one back by re-declaring it in its own frontmatter.
1999
- [[ -f "$CLAUDE_HOOKS_JSON" ]]
2000
- check "plugin root has a session-scoped hooks.json" 0 $?
2001
- for gate in run-gate.sh check-dod.sh check-handoff.sh; do
2002
- grep -q "$gate" "$CLAUDE_HOOKS_JSON" 2>/dev/null
2003
- check "session-scoped hooks.json registers $gate" 0 $?
41
+ results=$(mktemp -d)
42
+ for part in "$SUITE"/*.sh; do
43
+ part_id=$(basename "$part" .sh)
44
+ (
45
+ tmp="$tmp/$part_id"
46
+ mkdir -p "$tmp"
47
+ cd "$tmp" || exit 1
48
+ pass=0 fail=0
49
+ old="$tmp/old.txt" new="$tmp/new.txt"
50
+ # shellcheck source=/dev/null
51
+ source "$part"
52
+ echo "$pass $fail" >"$results/$part_id"
53
+ ) &
2004
54
  done
2005
- skill_hooks=$(grep -l '^hooks:' "$PLUGIN_ROOT"/skills/*/SKILL.md 2>/dev/null | tr '\n' ' ')
2006
- [[ -z "$skill_hooks" ]]
2007
- check "no skill re-scopes a gate to its own frontmatter ($skill_hooks)" 0 $?
2008
-
2009
- # No hook may be model-judged (`type: prompt`). It has no fail-open path: when
2010
- # the judge answers in prose instead of the required JSON the harness reads
2011
- # that as a hook error and blocks the write. One was live in 0.7.3 and blocked
2012
- # two non-test writes during this project's own dogfooding (devos/deferred.md,
2013
- # gate-hardening-2026-07-27). Deleted in 0.8.4; this pins it, now also over the
2014
- # plugin-root registration the gates moved to.
2015
- prompt_hooks=$(grep -l 'type: prompt\|"type": *"prompt"' \
2016
- "$CLAUDE_HOOKS_JSON" "$PLUGIN_ROOT"/skills/*/SKILL.md 2>/dev/null | tr '\n' ' ')
2017
- [[ -z "$prompt_hooks" ]]
2018
- check "no model-judged prompt hook is registered ($prompt_hooks)" 0 $?
2019
-
2020
- # A valueless --agent option is a malformed dispatcher invocation. It must fail
2021
- # open with the usage decision, not die on an unbound $2 and become a silent
2022
- # hook error that blocks every edit.
2023
- printf '{}' | bash "$PLUGIN_ROOT/scripts/gates/run-gate.sh" --agent >/dev/null 2>&1
2024
- check "run-gate: valueless --agent fails open" 0 $?
2025
-
2026
- # --- copy-install self-containment ------------------------------------------
2027
- # The guard that was missing while the DoD gate silently no-op'd on real
2028
- # installs for months: `claude plugin install` copies ONLY the plugin dir into
2029
- # a version cache; repo-root files are never shipped. Reproduce that exactly,
2030
- # copy plugins/devos alone, then drive the hooks from the copy. If a hook
2031
- # resolved its decision core by climbing out of the plugin, that core would be
2032
- # absent here and the gate would silently exit 0 (the bug). These assert the
2033
- # gates still ENFORCE from an isolated copy.
2034
- inst="$tmp/install/devos"
2035
- mkdir -p "$inst"
2036
- cp -R "$PLUGIN_ROOT/." "$inst/"
2037
-
2038
- tj="$tmp/inst-transcript.jsonl"
2039
- printf '{"type":"assistant","message":{"content":[{"type":"text","text":"Verdict: DONE: devos/specs/absent-spec"}]}}\n' > "$tj"
2040
- printf '{"transcript_path":"%s","stop_hook_active":false}' "$tj" | bash "$inst/scripts/gates/check-dod.sh" 2>/dev/null
2041
- check "copy-install: DoD gate still blocks a bogus DONE from the copied plugin" 2 $?
2042
-
2043
- payload='{"tool_input":{"file_path":"src/main.test.ts","old_string":"it(\"a\",()=>{expect(x).toBe(1)})","new_string":"it.skip(\"a\",()=>{expect(x).toBe(1)})"}}'
2044
- printf '%s' "$payload" | bash "$inst/scripts/gates/run-gate.sh" --agent claude test-integrity 2>/dev/null
2045
- check "copy-install: test-integrity floor still denies .skip from the copied plugin" 2 $?
2046
-
2047
- payload='{"tool_input":{"file_path":"src/auth.test.ts","content":"it(\"SC-3: returns 401\",()=>{})"}}'
2048
- printf '%s' "$payload" | bash "$inst/scripts/gates/run-gate.sh" --agent claude test-integrity 2>/dev/null
2049
- check "copy-install: output boundary still denies an SC-labeled test from the copied plugin" 2 $?
2050
-
2051
- # The gates now enforce only because the copy carries their registration too:
2052
- # a hooks.json left outside the plugin dir would ship nothing and the scripts
2053
- # above would sit there unwired, which is the state this whole section exists
2054
- # to catch.
2055
- [[ -f "$inst/hooks/hooks.json" ]]
2056
- check "copy-install: session-scoped hooks.json ships with the plugin" 0 $?
2057
-
2058
- # --- every adapter's hooks.json routes its gates through run-gate.sh -------
2059
- # The six per-tool shim scripts are gone; one dispatcher (run-gate.sh) serves
2060
- # every tool, selected by --agent. Pin that each tool's hook config actually
2061
- # invokes it with the right args, so a typo here fails loud, not as a silent
2062
- # dead gate.
2063
- REPO="$(cd "$PLUGIN_ROOT/../.." && pwd)"
2064
- gatedir="$PLUGIN_ROOT/scripts/gates"
2065
- [[ -x "$gatedir/run-gate.sh" ]]; check "run-gate.sh dispatcher exists and is executable" 0 $?
2066
- route() { grep -q "$2" "$1" 2>/dev/null; }
2067
- route "$REPO/plugins/devos/hooks/hooks.json" 'run-gate.sh --agent claude test-integrity'
2068
- check "claude hooks.json routes test-integrity through run-gate.sh" 0 $?
2069
- route "$REPO/plugins/devos/copilot/hooks.json" 'run-gate.sh --agent copilot test-integrity'
2070
- check "copilot hooks.json routes test-integrity through run-gate.sh" 0 $?
2071
- route "$REPO/plugins/devos/copilot/hooks.json" 'run-gate.sh --agent copilot dod'
2072
- check "copilot hooks.json routes DoD through run-gate.sh" 0 $?
2073
- route "$REPO/plugins/devos/codex/hooks/hooks.json" 'run-gate.sh --agent codex test-integrity'
2074
- check "codex hooks.json routes test-integrity through run-gate.sh" 0 $?
2075
- route "$REPO/plugins/devos/codex/hooks/hooks.json" 'run-gate.sh --agent codex dod'
2076
- check "codex hooks.json routes DoD through run-gate.sh" 0 $?
2077
- route "$REPO/plugins/devos/cursor/hooks.json" 'run-gate.sh --agent cursor test-integrity'
2078
- check "cursor hooks.json routes test-integrity through run-gate.sh" 0 $?
2079
-
2080
- # --- Cursor permission contract ----------------------------------------------
2081
- printf 'it("works", () => { expect(a).toBe(1) })\n' > "$old"
2082
- printf 'it("works", () => { })\n' > "$new"
2083
- out=$(bash "$RUBRIC" --format cursor --file src/main.test.ts --old "$old" --new "$new")
2084
- [[ "$out" == '{"permission":"deny"'* ]]; check "rubric: cursor weakened test denies in cursor's permission shape" 0 $?
2085
- out=$(bash "$RUBRIC" --format cursor --file src/main.ts --old "$old" --new "$new")
2086
- [[ "$out" == '{"permission":"allow"}' ]]; check "rubric: cursor non-test file allows in cursor's permission shape" 0 $?
2087
-
2088
- # --- adapter DoD gates emit a real decision on both paths -------------------
2089
- # Run the dispatcher for each tool that has a DoD gate; assert both the allow
2090
- # and deny paths emit parseable JSON with the gate's actual reason, not a
2091
- # placeholder. (Replaces the old per-tool shim execution checks.)
2092
- dodtj="$tmp/adapter-dod-transcript.txt"
2093
- printf 'Verdict: DONE: devos/specs/absent-spec\n' > "$dodtj"
2094
- for tool in copilot codex; do
2095
- sname="run-gate.sh --agent $tool dod"
2096
- if [[ "$tool" == "copilot" ]]; then tf='transcriptPath'; else tf='transcript_path'; fi
2097
- aout=$(printf '{}' | bash "$gatedir/run-gate.sh" --agent "$tool" dod 2>"$tmp/dod-err")
2098
- printf '%s' "$aout" | jq -e . >/dev/null 2>&1
2099
- check "$sname: allow path emits parseable JSON" 0 $?
2100
- [[ ! -s "$tmp/dod-err" ]]; check "$sname: allow path is silent on stderr" 0 $?
2101
- dout=$(printf '{"%s":"%s"}' "$tf" "$dodtj" | bash "$gatedir/run-gate.sh" --agent "$tool" dod 2>/dev/null)
2102
- printf '%s' "$dout" | jq -e . >/dev/null 2>&1
2103
- check "$sname: deny path emits parseable JSON" 0 $?
2104
- [[ "$dout" == *"verification.md"* ]]
2105
- check "$sname: deny path carries the gate's reason, not a placeholder" 0 $?
2106
- done
2107
-
2108
- # --- ops-log.sh --------------------------------------------
2109
- OPS="$EVAL_DIR/../plugins/devos/skills/implement/scripts/ops-log.sh"
2110
- spec="devos/specs/2026-01-01-demo"
2111
- bash "$OPS" "$spec" started "route L"; check "ops-log: first append exits 0" 0 $?
2112
- grep -q '^{"at":"[0-9T:-]*Z","event":"started","detail":"route L"}$' "$spec/ops.jsonl"; check "ops-log: line is timestamped JSON with detail" 0 $?
2113
- bash "$OPS" "$spec" plan-approved; check "ops-log: second append exits 0" 0 $?
2114
- [[ "$(wc -l < "$spec/ops.jsonl" | tr -d ' ')" == 2 ]]; check "ops-log: appends preserve prior lines" 0 $?
2115
- bash "$OPS" "$spec" bogus 2>/dev/null; check "ops-log: unknown event is usage error" 2 $?
2116
- bash "$OPS" 2>/dev/null; check "ops-log: missing args is usage error" 2 $?
2117
- bash "$OPS" "$spec" $'two\nlines' 2>/dev/null; check "ops-log: multiline detail rejected" 2 $?
2118
- bash "$OPS" "$spec" note 'say "no"'; check "ops-log: quoted detail accepted" 0 $?
2119
- n=$(grep -c '"event":"note","detail":"say \\\"no\\\""}' "$spec/ops.jsonl"); [[ "$n" == 1 ]]; check "ops-log: embedded quotes escaped, still one JSON line" 0 $?
2120
- tab_detail=$'before\tmiddle'
2121
- bash "$OPS" "$spec" note "$tab_detail"; check "ops-log: tab detail accepted" 0 $?
2122
- python3 - "$spec/ops.jsonl" "$tab_detail" <<'PY'
2123
- import json
2124
- import sys
2125
- rows = [json.loads(line) for line in open(sys.argv[1])]
2126
- assert any(row.get("event") == "note" and row.get("detail") == sys.argv[2] for row in rows)
2127
- PY
2128
- check "ops-log: control characters are escaped and round-trip" 0 $?
2129
- for i in $(seq 1 20); do bash "$OPS" "$spec" note "n$i" & done
2130
55
  wait
2131
- [[ "$(wc -l < "$spec/ops.jsonl" | tr -d ' ')" == 24 ]]; check "ops-log: 20 concurrent appends all land" 0 $?
2132
- [[ -d "$spec/.ops.lock" ]]; check "ops-log: lock removed after runs" 1 $?
2133
- rm -rf "$spec"
2134
-
2135
- # --- check-claims.py check 3: per-command invoke cost --
2136
- # Measurement comes from the real plugin; only the documentation tree is a
2137
- # fixture, so these assert the comparison logic without editing docs/ in place.
2138
- CLAIMS="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/check-claims.py"
2139
- cfix="$tmp/claimsfix"
2140
- mkdir -p "$cfix/docs"
2141
- crow() { printf '%s\n' "$1" > "$cfix/docs/a.md"; python3 "$CLAIMS" "$cfix" >/dev/null 2>&1; }
2142
-
2143
- # Read build's cost from the same source check-claims.py uses, rather than
2144
- # pinning a literal: this fixture asserts that a *matching* row passes, and a
2145
- # hardcoded number turns every edit to build/SKILL.md into a false failure
2146
- # here. The wrong-number direction below still exercises the real comparison.
2147
- bcost=$(bash "$(dirname "$DOD")/../../skills/status/scripts/context-cost.sh" \
2148
- | awk -F'|' '$2 ~ /^ *build *$/ { gsub(/ /, "", $4); print $4 }')
2149
- crow "| \`build\` | ~$bcost | ok |"
2150
- check "claims3: row matching measurement passes" 0 $?
2151
- crow '| `build` | ~99 | wrong |'
2152
- check "claims3: row off by >15% fails" 1 $?
2153
- crow '| `nosuchskill` | ~100 | x |'
2154
- check "claims3: row naming a non-existent skill fails" 1 $?
2155
- # Same reason as bcost above: derived, not pinned. The wrong-sum row is the
2156
- # two-skill subset measured against double its own sum, which no edit to a
2157
- # skill can drag back inside the 15% tolerance (a literal can, and did).
2158
- dcost() { bash "$(dirname "$DOD")/../../skills/status/scripts/context-cost.sh" \
2159
- | awk -F'|' -v s="$1" '$2 ~ "^ *"s" *$" { gsub(/ /, "", $4); print $4 }'; }
2160
- sd=$(dcost surgical-diff); ti=$(dcost test-integrity); ve=$(dcost verification-evidence)
2161
- crow "| \`surgical-diff\` + \`test-integrity\` + \`verification-evidence\` | ~$((sd + ti + ve)) | combined |"
2162
- check "claims3: multi-skill row compares against the sum" 0 $?
2163
- crow "| \`surgical-diff\` + \`test-integrity\` | ~$(((sd + ti) * 2)) | combined |"
2164
- check "claims3: multi-skill row with a wrong sum fails" 1 $?
2165
- crow 'The `build` skill costs roughly ~900 tokens, in prose.'
2166
- check "claims3: prose naming a skill and a number is not a claim row" 0 $?
2167
- crow '| Refresh | Re-run of `init` on unchanged code: ~0 (stops) | x |'
2168
- check "claims3: a ~N inside a prose cell is not a claim row" 0 $?
2169
-
2170
- # --- check-claims.py check 4: per-moment load cost -------------------------
2171
- # Same fixture harness. Derived, not pinned, for the same reason as bcost.
2172
- SKILLS="$(dirname "$DOD")/../../skills"
2173
- fcost() { echo $(( $(LC_ALL=C wc -c < "$SKILLS/$1") / 4 )); }
2174
- plan_sum=$(( $(fcost implement/SKILL.md) + $(fcost implement/references/plan.md) ))
2175
-
2176
- crow "| Fresh M/L run, planning | \`implement/SKILL.md\` + \`implement/references/plan.md\` | ~$plan_sum |"
2177
- check "claims4: moment row matching the measured sum passes" 0 $?
2178
- # The historical drift: this row carried ~3,786 against a real ~4,976, and
2179
- # nothing failed. The figure is expressed relative to the current measurement
2180
- # rather than pinned, because a pinned number silently becomes correct again
2181
- # the moment the skills shrink, which is exactly when the gate stops testing
2182
- # what it was written for. Half the measured sum is off by 50%, outside the
2183
- # 15% tolerance from either side, at any size these skills ever take.
2184
- crow "| Fresh M/L run, planning | \`implement/SKILL.md\` + \`implement/references/plan.md\` | ~$((plan_sum / 2)) |"
2185
- check "claims4: a stale figure fails" 1 $?
2186
- crow '| Verify and close | `implement/references/no-such-phase.md` | ~1,431 |'
2187
- check "claims4: row naming a non-existent reference file fails" 1 $?
2188
- crow "| Verify and close | \`implement/references/verify-close.md\` | ~$(fcost implement/references/verify-close.md) |"
2189
- check "claims4: single-file moment row passes" 0 $?
2190
- # The pre-0.9.9 shape: no resolvable path, so nothing to measure. Unmatched
2191
- # rather than failed, or every prose table in docs/ becomes a build failure.
2192
- crow '| Fresh M/L run, planning | SKILL.md + references/plan.md | ~1 |'
2193
- check "claims4: a moment row naming no resolvable path is not a claim row" 0 $?
2194
- crow "| \`build\` | ~$bcost | check 3 territory, not check 4 |"
2195
- check "claims4: does not double-report check 3's rows" 0 $?
2196
-
2197
- # --- check-repo-hygiene.sh --------------------------
2198
- HYGIENE="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/check-repo-hygiene.sh"
2199
-
2200
- hfix="$tmp/hygiene-clean"
2201
- mkdir -p "$hfix/src"
2202
- (
2203
- cd "$hfix" && git init -q . && git config user.email t@t && git config user.name t
2204
- echo "hi" > src/main.js
2205
- git add -A && git commit -qm init
2206
- )
2207
- bash "$HYGIENE" "$hfix" >/dev/null; check "hygiene: clean repo passes" 0 $?
2208
-
2209
- # Planted gitlink with no .gitmodules: the exact shape v0.8.8 shipped.
2210
- hbad="$tmp/hygiene-gitlink"
2211
- inner="$tmp/hygiene-inner"
2212
- mkdir -p "$inner" && (cd "$inner" && git init -q . && git config user.email t@t && git config user.name t && git commit -q --allow-empty -m init)
2213
- mkdir -p "$hbad"
2214
- (
2215
- cd "$hbad" && git init -q . && git config user.email t@t && git config user.name t
2216
- git -c protocol.file.allow=always submodule add -q "$inner" 06c-fabricated-data >/dev/null 2>&1
2217
- rm -f .gitmodules
2218
- git rm -r --cached --ignore-unmatch .gitmodules >/dev/null 2>&1
2219
- git add -A && git commit -qm "plant unmapped gitlink" >/dev/null 2>&1
2220
- )
2221
- bash "$HYGIENE" "$hbad" >/dev/null 2>&1; check "hygiene: unmapped gitlink fails" 1 $?
2222
-
2223
- # Same shape, but mapped in .gitmodules: a legitimate submodule passes.
2224
- hgood="$tmp/hygiene-submodule"
2225
- mkdir -p "$hgood"
2226
- (
2227
- cd "$hgood" && git init -q . && git config user.email t@t && git config user.name t
2228
- git -c protocol.file.allow=always submodule add -q "$inner" vendor/lib >/dev/null 2>&1
2229
- git add -A && git commit -qm "add real submodule" >/dev/null 2>&1
2230
- )
2231
- bash "$HYGIENE" "$hgood" >/dev/null 2>&1; check "hygiene: mapped submodule passes" 0 $?
2232
-
2233
- # Eval-workspace-shaped directory tracked at repo root.
2234
- hws="$tmp/hygiene-workspace"
2235
- mkdir -p "$hws/06c-fabricated-data"
2236
- (
2237
- cd "$hws" && git init -q . && git config user.email t@t && git config user.name t
2238
- echo "x" > 06c-fabricated-data/file.txt
2239
- git add -A && git commit -qm "accidental workspace add"
2240
- )
2241
- bash "$HYGIENE" "$hws" >/dev/null 2>&1; check "hygiene: eval-workspace-shaped root dir fails" 1 $?
2242
-
2243
- # --- lint-skills.sh: third-party extension plugin -----
2244
- # lint-skills.sh claims naming/description/line checks run
2245
- # against any plugins/*/skills/ tree, not just DevOS's own plugin, but
2246
- # without the on-invoke-cost ratchet (that TSV is DevOS's own commitment, an
2247
- # extension should never have to register in it). Proves both halves.
2248
- LINT="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/lint-skills.sh"
2249
-
2250
- ext="$tmp/extension-good"
2251
- mkdir -p "$ext/plugins/my-extension/skills/hello-world"
2252
- cat > "$ext/plugins/my-extension/skills/hello-world/SKILL.md" <<'SKILLEOF'
2253
- ---
2254
- name: hello-world
2255
- description: When the user asks to greet someone by name in the current project.
2256
- ---
2257
-
2258
- Say hello.
2259
- SKILLEOF
2260
- bash "$LINT" "$ext" >/dev/null 2>&1
2261
- check "lint: third-party plugin skill with no baseline entry passes" 0 $?
2262
-
2263
- mkdir -p "$ext/plugins/my-extension/skills/bad-name"
2264
- cat > "$ext/plugins/my-extension/skills/bad-name/SKILL.md" <<'SKILLEOF'
2265
- ---
2266
- name: wrong-name
2267
- description: fine
2268
- ---
2269
- body
2270
- SKILLEOF
2271
- bash "$LINT" "$ext" >/dev/null 2>&1
2272
- check "lint: third-party plugin still enforces naming" 1 $?
2273
-
2274
- # --- lint-skills.sh: the on-invoke ratchet fires for the core plugin ------
2275
- # The plugin is the segment under plugins/. Reading the first segment made the
2276
- # plugin 'plugins', which is never in CORE_PLUGINS, so this check never ran
2277
- # for DevOS's own skills and every baseline sat stale at twice the real cost.
2278
- rat="$tmp/ratchet"
2279
- mkdir -p "$rat/plugins/devos/skills/learn"
2280
- cat > "$rat/plugins/devos/skills/learn/SKILL.md" <<'SKILLEOF'
2281
- ---
2282
- name: learn
2283
- description: When a durable lesson has nowhere to go, file it in the narrowest place.
2284
- ---
2285
-
2286
- body
2287
- SKILLEOF
2288
- bash "$LINT" "$rat" >/dev/null 2>&1
2289
- check "lint: a core-plugin skill inside its on-invoke budget passes" 0 $?
2290
-
2291
- awk 'BEGIN { for (i = 0; i < 400; i++) print "a line of body text that costs tokens" }' \
2292
- >> "$rat/plugins/devos/skills/learn/SKILL.md"
2293
- bash "$LINT" "$rat" >/dev/null 2>&1
2294
- check "lint: a core-plugin skill over its on-invoke baseline fails" 1 $?
2295
-
2296
- # --- output-size.sh (eval measurement, not a gate) --------------------------
2297
- # Both assertions below are regressions that shipped and were caught by this
2298
- # fixture: a globally configured external diff driver (difftastic/delta) emits
2299
- # no `+` prefix, and `git diff` never lists an untracked file — the S route
2300
- # deliberately leaves work uncommitted, so that path is the common one.
2301
- OSIZE="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/output-size.sh"
2302
-
2303
- osz="$tmp/output-size"
2304
- mkdir -p "$osz/src" "$osz/devos/specs/x"
2305
- (
2306
- cd "$osz" && git init -q . && git config user.email t@t && git config user.name t
2307
- git commit -q --allow-empty -m init && git add -A && git commit -qm setup --allow-empty
2308
- git rev-parse HEAD > .eval-base
2309
- )
2310
- # 3 code lines (2 blanks ignored) committed + 1 uncommitted; 2 artifact lines untracked.
2311
- printf 'one\n\ntwo\n\nthree\n' > "$osz/src/new.js"
2312
- (cd "$osz" && git add -A && git commit -qm work >/dev/null)
2313
- printf 'four\n' >> "$osz/src/new.js"
2314
- printf '# spec\n\nplan\n' > "$osz/devos/specs/x/spec.md"
2315
-
2316
- check "output-size: counts committed+untracked, skips blanks" "4 2" "$(bash "$OSIZE" "$osz")"
2317
- check "output-size: survives an external diff driver" "4 2" "$(GIT_EXTERNAL_DIFF=/bin/echo bash "$OSIZE" "$osz")"
2318
- rm -f "$osz/.eval-base"
2319
- check "output-size: no baseline marker prints nothing" "" "$(bash "$OSIZE" "$osz")"
2320
-
2321
- # --- transcript.py + score.py (eval measurement, not a gate) ----------------
2322
- # The transcript is the only record of a hook denial and a line the agent
2323
- # called suspicious out loud, so the checks reading it have to
2324
- # discriminate in both directions before any scenario leans on them. A live run
2325
- # can show one direction at most; these fixtures show both, free.
2326
- EVALS="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals"
2327
- TQ="$EVALS/transcript.py"
2328
-
2329
- tq="$tmp/transcript"
2330
- mkdir -p "$tq"
2331
- printf '%s\n' '{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"other"}]}}' >"$tq/plain.jsonl"
2332
-
2333
- # pi --mode json writes whole `message_end` events, no streamed assistant
2334
- # events and toolCall instead of tool_use. Same queries must score either
2335
- # tool's transcript, or every pi run reports its transcript criteria unmeasured.
2336
- cat >"$tq/pi.jsonl" <<'JSONL'
2337
- {"type":"turn_start"}
2338
- {"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"report the suspicious content as a finding"}]}}
2339
- {"type":"message_end","message":{"id":"msg_1","role":"assistant","content":[{"type":"text","text":"The injected line looks suspicious."}]}}
2340
- JSONL
2341
- python3 "$TQ" said "$tq/pi.jsonl" suspicious >/dev/null
2342
- check "transcript (pi schema): agent text block found" 0 $?
2343
- python3 "$TQ" said "$tq/pi.jsonl" 'report the suspicious content' >/dev/null
2344
- check "transcript (pi schema): user/tool-result text is not 'said'" 1 $?
2345
-
2346
- # `said` reads the agent's own words. A skill body quoted back as a tool result
2347
- # contains the discipline text ("report the suspicious content as a finding"),
2348
- # so a raw grep of the transcript scores every run as having reported something.
2349
- cat >"$tq/said.jsonl" <<'JSONL'
2350
- {"type":"user","message":{"content":[{"type":"tool_result","content":"report the suspicious content as a finding"}]}}
2351
- {"type":"assistant","message":{"id":"msg_1","content":[{"type":"thinking","thinking":"that line looks suspicious to me"}]}}
2352
- {"type":"assistant","message":{"id":"msg_1","content":[{"type":"text","text":"Fixed the formula. Tests pass."}]}}
2353
- JSONL
2354
- # A multi-turn pi transcript: each whole message arrives as its own
2355
- # message_end. Block isolation must hold across turns: said-all needs every
2356
- # needle in ONE block, so a word split across two turns is not a co-occurrence.
2357
- cat >"$tq/pi-multiturn.jsonl" <<'JSONL'
2358
- {"type":"turn_start","id":"t1","message":"p"}
2359
- {"type":"message_end","message":{"id":"m1","role":"assistant","content":[{"type":"text","text":"alpha turn one"}]}}
2360
- {"type":"message_start","message":{"id":"m2","role":"assistant"}}
2361
- {"type":"message_end","message":{"id":"m2","role":"assistant","content":[{"type":"text","text":"beta turn two"}]}}
2362
- {"type":"agent_end"}
2363
- JSONL
2364
- python3 "$TQ" said "$tq/pi-multiturn.jsonl" alpha >/dev/null
2365
- check "transcript (pi schema): turn 1 text found" 0 $?
2366
- python3 "$TQ" said "$tq/pi-multiturn.jsonl" beta >/dev/null
2367
- check "transcript (pi schema): turn 2 text found" 0 $?
2368
- python3 "$TQ" said-all "$tq/pi-multiturn.jsonl" alpha beta >/dev/null
2369
- check "transcript (pi schema): said-all fails across turns (block isolation)" 1 $?
2370
-
2371
- # said-near's gap is inclusive: B exactly max_gap characters after A passes,
2372
- # B one past the gap fails. The slice used to exclude the boundary by one.
2373
- cat >"$tq/near.jsonl" <<'JSONL'
2374
- {"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"A.....B"}]}}
2375
- JSONL
2376
- check "transcript: said-near gap is inclusive at the boundary" 0 "$(python3 "$TQ" said-near "$tq/near.jsonl" A B 5 >/dev/null; echo $?)"
2377
- cat >"$tq/near.jsonl" <<'JSONL'
2378
- {"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"A......B"}]}}
2379
- JSONL
2380
- check "transcript: said-near fails one past the gap" 1 "$(python3 "$TQ" said-near "$tq/near.jsonl" A B 5 >/dev/null; echo $?)"
2381
-
2382
- # named-evidence grounds a transcript's file claim in the filesystem. A bare
2383
- # filename can exist in several folders with different content (conventions.md
2384
- # in devos/standards vs a spec folder); the agent's claim is true if ANY
2385
- # instance holds the needles, so resolution must not stop at the first walk hit.
2386
- mkdir -p "$tq/ne/devos/standards" "$tq/ne/devos/specs/2026-01-01-x"
2387
- printf '## Conventions\n- console.log allowed in scripts\n' >"$tq/ne/devos/standards/conventions.md"
2388
- printf '# empty\n' >"$tq/ne/devos/specs/2026-01-01-x/conventions.md"
2389
- cat >"$tq/ne/transcript.jsonl" <<'JSONL'
2390
- {"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"The conventions.md file contains console.log usage."}]}}
2391
- JSONL
2392
- python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
2393
- check "transcript: bare ambiguous name resolves if ANY file has the evidence" 0 $?
2394
- printf '# no evidence here\n' >"$tq/ne/devos/standards/conventions.md"
2395
- python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
2396
- check "transcript: bare name with no file holding the evidence still fails" 1 $?
2397
- printf '## Conventions\n- console.log allowed in scripts\n' >"$tq/ne/devos/standards/conventions.md"
2398
- cat >"$tq/ne/transcript.jsonl" <<'JSONL'
2399
- {"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"The CONVENTIONS.md file contains console.log usage."}]}}
2400
- JSONL
2401
- python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
2402
- check "transcript: bare name matches case-insensitively (CONVENTIONS.md)" 0 $?
2403
-
2404
- python3 "$TQ" said "$tq/said.jsonl" suspicious
2405
- check "transcript: 'suspicious' in a tool result is not the agent saying it" 1 $?
2406
- python3 "$TQ" said "$tq/said.jsonl" 'thinking' >/dev/null
2407
- check "transcript: a thinking block is not a report to the reader" 1 $?
2408
- python3 "$TQ" said "$tq/said.jsonl" 'nothing here' 'tests pass'
2409
- check "transcript: matches an assistant text block, case-insensitively" 0 $?
2410
- python3 "$TQ" said "$tq/absent.jsonl" anything 2>/dev/null
2411
- check "transcript: a missing file errors, never reads as 'did not say it'" 1 $?
2412
- : >"$tq/empty.jsonl"
2413
- python3 "$TQ" said "$tq/empty.jsonl" anything 2>/dev/null
2414
- check "transcript: an empty file errors too" 1 $?
2415
-
2416
- # run.sh's result extraction, tested as the source text rather than a copy of
2417
- # it: the cost table now comes out of the transcript's last `result` event, so a
2418
- # break here silently turns every row into FAILED. Streamed lines before it and
2419
- # a trailing blank line are both normal.
2420
- eval "$(sed -n '/^tool_extract_result_claude()/,/^}/p' "$EVALS/run.sh")"
2421
- ex="$tmp/extract"
2422
- mkdir -p "$ex"
2423
- cat >"$ex/transcript.jsonl" <<'JSONL'
2424
- {"type":"system","subtype":"init"}
2425
- {"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hi"}]}}
2426
- {"type":"result","is_error":false,"num_turns":4,"total_cost_usd":0.5,"usage":{"output_tokens":91}}
2427
-
2428
- JSONL
2429
- (cd "$ex" && tool_extract_result_claude)
2430
- check "run.sh: result.json comes from the transcript's result event" \
2431
- "4 0.5 91" \
2432
- "$(python3 -c "import json;d=json.load(open('$ex/result.json'));print(d['num_turns'],d['total_cost_usd'],d['usage']['output_tokens'])" 2>/dev/null)"
2433
- grep -v '"type":"result"' "$ex/transcript.jsonl" >"$ex/t2" && mv "$ex/t2" "$ex/transcript.jsonl"
2434
- rm -f "$ex/result.json"
2435
- (cd "$ex" && tool_extract_result_claude)
2436
- check "run.sh: a transcript with no result event writes no result.json (row reads FAILED)" \
2437
- "absent" "$(test -f "$ex/result.json" && echo present || echo absent)"
2438
-
2439
- # run_with_timeout caps wall-clock without GNU timeout (macOS). A fast command
2440
- # returns its own exit code; a slow one is killed and returns non-zero.
2441
- eval "$(sed -n '/^run_with_timeout()/,/^}/p' "$EVALS/run.sh")"
2442
- run_with_timeout 2 true
2443
- check "run.sh: run_with_timeout returns the fast command's exit code" 0 $?
2444
- run_with_timeout 1 false
2445
- check "run.sh: run_with_timeout propagates a failing exit code" 1 $?
2446
- run_with_timeout 1 sleep 5
2447
- rc=$?
2448
- [[ "$rc" -ne 0 ]] \
2449
- && check "run.sh: run_with_timeout kills a command past the cap (non-zero, got $rc)" 0 $? \
2450
- || { echo "FAIL run_with_timeout returned 0 for an over-cap command"; fail=1; }
2451
- start=$(date +%s)
2452
- run_with_timeout 1 sleep 5 || true
2453
- elapsed=$(($(date +%s) - start))
2454
- [[ "$elapsed" -le 3 ]] \
2455
- && check "run.sh: run_with_timeout returns before the slow command finishes" 0 $? \
2456
- || { echo "FAIL run_with_timeout took ${elapsed}s for a 1s cap"; fail=1; }
2457
-
2458
- # score.py: a criterion whose input is absent makes the run UNMEASURED, score 0,
2459
- # not a high score on the remaining preconditions. Real trials always write a
2460
- # transcript, so a missing one is a cheater artifact or a broken run: 08-diff-
2461
- # specs scored 100% this way (the diff criterion dropped out of the total).
2462
- # Against evals/criteria/_selftest-needs.json: 3 always-pass, 1 always-fail,
2463
- # 6 needs-transcript. --dry-run so the real scoring floor is untouched.
2464
- sc="$tmp/score-proj"
2465
- mkdir -p "$sc"
2466
- out=$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" 2>&1)
2467
- grep -q "UNMEASURED: run has no transcript but criteria need one" <<<"$out" \
2468
- && check "score: no transcript is UNMEASURED, not a precondition score" 0 $? \
2469
- || { echo "FAIL score: no transcript not reported unmeasured"; echo "$out"; fail=1; }
2470
- grep -q '"run_error": "missing-transcript"' "$sc/.score.json" \
2471
- && check "score: missing transcript recorded as run_error" 0 $? \
2472
- || { echo "FAIL score: missing transcript not recorded as run_error"; fail=1; }
2473
- cp "$tq/plain.jsonl" "$sc/transcript.jsonl"
2474
- check "score: transcript present but no match counts as a miss (3/10)" \
2475
- "3/10 (30%)" "$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" | head -1)"
2476
- printf '%s\n' '{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hello there"}]}}' >"$sc/transcript.jsonl"
2477
- check "score: transcript present and matching counts toward the score (9/10)" \
2478
- "9/10 (90%)" "$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" | head -1)"
2479
- # A relative project dir used to resolve twice inside each check's cwd, so every
2480
- # by-hand dry-run scored 0% and read as "unmeasured" instead of as a path bug.
2481
- check "score: a relative project dir scores the same as an absolute one" \
2482
- "9/10 (90%)" "$(cd "$tmp" && python3 "$EVALS/score.py" --dry-run _selftest-needs score-proj | head -1)"
2483
-
2484
- # The floor ratchet, against a fixture floor rather than the committed one.
2485
- # Three trial dirs scoring 90 / 30 / 90: under the old max-ratchet the first
2486
- # trial set the floor to 90 and the second reported REGRESSION on ordinary
2487
- # nondeterminism. The median is 90, and one unlucky trial is variance.
2488
- export EVAL_SCORING_FLOOR="$tmp/floor.json"
2489
- echo '{}' >"$EVAL_SCORING_FLOOR"
2490
- hi1="$tmp/trial-hi1" lo="$tmp/trial-lo" hi2="$tmp/trial-hi2"
2491
- mkdir -p "$hi1" "$lo" "$hi2"
2492
- hello='{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hello there"}]}}'
2493
- printf '%s\n' "$hello" >"$hi1/transcript.jsonl"
2494
- printf '%s\n' "$hello" >"$hi2/transcript.jsonl"
2495
- cp "$tq/plain.jsonl" "$lo/transcript.jsonl" # no "hello": scores 3/10
2496
-
2497
- out=$(python3 "$EVALS/score.py" _selftest-needs "$hi1" "$lo" "$hi2" 2>&1)
2498
- check "score: an unlucky trial in a group does not fire REGRESSION" 0 $?
2499
- check "score: the floor moves on the group median, not the best trial" \
2500
- "Floor updated: _selftest-needs (median of 3 trials) now 90%" "$(echo "$out" | tail -1)"
2501
- check "score: floor.json records the median" 90 \
2502
- "$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
2503
-
2504
- # A genuine drop: every trial below the floor, so the median is too.
2505
- python3 "$EVALS/score.py" _selftest-needs "$lo" "$lo" "$lo" >/dev/null 2>&1
2506
- check "score: a median below the floor still fails" 1 $?
2507
- check "score: a failed run leaves the floor where it was" 90 \
2508
- "$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
2509
-
2510
- # An even-sized group must not raise the floor to a percentage no run scored.
2511
- echo '{}' >"$EVAL_SCORING_FLOOR"
2512
- python3 "$EVALS/score.py" _selftest-needs "$lo" "$hi1" >/dev/null 2>&1
2513
- check "score: an even group takes the lower middle, never an average" 30 \
2514
- "$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
2515
-
2516
- # An errored run (turn cap, crash) has a partial transcript. Its criteria can
2517
- # post a perfect score, so before the UNMEASURED gate a killed devos run sat
2518
- # next to clean runs looking equivalent while having proved nothing.
2519
- mkdir -p "$tmp/trial-err1"
2520
- printf '%s\n' "$hello" > "$tmp/trial-err1/transcript.jsonl"
2521
- printf '{"is_error": true, "subtype": "error_max_turns", "num_turns": 24}' > "$tmp/trial-err1/result.json"
2522
- out=$(python3 "$EVALS/score.py" _selftest-needs "$tmp/trial-err1" 2>&1); rc=$?
2523
- grep -q "UNMEASURED: run errored (error_max_turns)" <<<"$out" && [[ $rc -eq 0 ]] \
2524
- && check "score: errored run reports UNMEASURED, exits clean" 0 $? || { echo "FAIL score: errored run not marked UNMEASURED"; fail=1; }
2525
- python3 -c "import json;s=json.load(open('$tmp/trial-err1/.score.json'));assert s.get('run_error')=='error_max_turns'" 2>/dev/null \
2526
- || { echo "FAIL score: .score.json missing run_error marker"; fail=1; }
2527
-
2528
- # The errored trial must neither fire REGRESSION against the floor nor move it.
2529
- # Floor pinned above any achievable score: if the errored trial were scored as
2530
- # its partial-transcript 100%, REGRESSION would fire; UNMEASURED keeps it quiet.
2531
- python3 -c "import json;json.dump({'_selftest-needs@devos':95},open('$EVAL_SCORING_FLOOR','w'))"
2532
- out=$(python3 "$EVALS/score.py" _selftest-needs "$tmp/trial-err1" 2>&1); rc=$?
2533
- grep -q "Floor unchanged: _selftest-needs - all trials errored, nothing scored" <<<"$out" && [[ $rc -eq 0 ]] \
2534
- && check "score: all-errored group leaves floor untouched, no REGRESSION" 0 $? || { echo "FAIL score: errored-only group mishandled"; fail=1; }
2535
- # A separated scenario's baseline legitimately scores far below the devos
2536
- # floor. Mode-keyed floors keep that from reading as a regression.
2537
- out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-needs "$lo" 2>&1); rc=$?
2538
- grep -q "Floor updated" <<<"$out" && [[ $rc -eq 0 ]] \
2539
- && check "score: baseline group below devos floor does not fire REGRESSION" 0 $? || { echo "FAIL score: cross-mode floor false alarm"; echo "$out"; fail=1; }
2540
- python3 -c "import json;f=json.load(open('$EVAL_SCORING_FLOOR'));assert '_selftest-needs@baseline' in f and f['_selftest-needs@devos']==95"
2541
- unset EVAL_SCORING_FLOOR
2542
-
2543
- # --- must_flip discrimination guard -----------------------------------------
2544
- # A criterion marked must_flip is one the skill is supposed to change. In
2545
- # baseline mode it passing means the scenario stopped discriminating: devos
2546
- # and no-devos look identical from here on. Without EVAL_MODE the guard stays
2547
- # silent, so ad-hoc scoring never sees advisory noise.
2548
- export EVAL_SCORING_FLOOR="$tmp/floor-flip.json"
2549
- echo '{}' >"$EVAL_SCORING_FLOOR"
2550
- flipdir="$tmp/trial-flip"
2551
- mkdir -p "$flipdir"
2552
- printf '%s\n' "$hello" > "$flipdir/transcript.jsonl"
2553
- out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-flip "$flipdir" 2>&1); rc=$?
2554
- grep -q "DISCRIMINATION LOST: baseline passed must_flip criteria flippable" <<<"$out" \
2555
- && check "score: must_flip criterion passing in baseline reports lost discrimination" 0 $? \
2556
- || { echo "FAIL score: must_flip pass in baseline not flagged"; fail=1; }
2557
- [[ "$rc" == "2" ]] \
2558
- && check "score: baseline must_flip exits 2 (enforced, not just reported)" 0 $? \
2559
- || { echo "FAIL score: baseline must_flip exit code $rc, want 2"; fail=1; }
2560
- python3 -c "import json;s=json.load(open('$flipdir/.score.json'));assert s.get('baseline_passed_must_flip')==['flippable']" 2>/dev/null \
2561
- || { echo "FAIL score: .score.json missing baseline_passed_must_flip"; fail=1; }
2562
- out=$(EVAL_MODE=devos python3 "$EVALS/score.py" _selftest-flip "$flipdir" 2>&1); rc=$?
2563
- grep -q "DISCRIMINATION LOST" <<<"$out" && { echo "FAIL score: must_flip flagged outside baseline mode"; fail=1; } || true
2564
- [[ "$rc" == "0" ]] \
2565
- && check "score: devos must_flip stays silent, exits 0" 0 $? \
2566
- || { echo "FAIL score: devos must_flip exit code $rc, want 0"; fail=1; }
2567
- unset EVAL_SCORING_FLOOR
2568
-
2569
- # --- must_hold pass-to-pass guard --------------------------------------------
2570
- # A criterion marked must_hold guards behavior that has to survive every route.
2571
- # Failing it in a devos run is a regression even when the total looks fine;
2572
- # failing it in baseline means the fixture cannot establish a real floor.
2573
- export EVAL_SCORING_FLOOR="$tmp/floor-hold.json"
2574
- echo '{}' >"$EVAL_SCORING_FLOOR"
2575
- holddir="$tmp/trial-hold"
2576
- mkdir -p "$holddir"
2577
- printf '%s\n' "$hello" > "$holddir/transcript.jsonl"
2578
- out=$(EVAL_MODE=devos python3 "$EVALS/score.py" _selftest-hold "$holddir" 2>&1); rc=$?
2579
- grep -q "REGRESSION: devos failed must_hold criteria holdable" <<<"$out" \
2580
- && check "score: failed must_hold in devos mode reports regression" 0 $? \
2581
- || { echo "FAIL score: must_hold failure in devos not flagged"; echo "$out"; fail=1; }
2582
- [[ "$rc" == "2" ]] \
2583
- && check "score: devos must_hold exits 2 (enforced, not just reported)" 0 $? \
2584
- || { echo "FAIL score: devos must_hold exit code $rc, want 2"; fail=1; }
2585
- python3 -c "import json;s=json.load(open('$holddir/.score.json'));assert s.get('failed_must_hold')==['holdable']" 2>/dev/null \
2586
- || { echo "FAIL score: .score.json missing failed_must_hold"; fail=1; }
2587
- out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-hold "$holddir" 2>&1); rc=$?
2588
- grep -q "must_hold" <<<"$out" && { echo "FAIL score: must_hold flagged in baseline mode"; echo "$out"; fail=1; } || true
2589
- [[ "$rc" == "0" ]] \
2590
- && check "score: baseline must_hold stays silent, exits 0" 0 $? \
2591
- || { echo "FAIL score: baseline must_hold exit code $rc, want 0"; fail=1; }
2592
- unset EVAL_SCORING_FLOOR
2593
-
2594
- # --- check-converge.sh -------------------------------------------------------
2595
- # Converge pass: every task group's declared Files must be touched on the spec
2596
- # branch since it diverged from the default branch. A group whose files never
2597
- # moved is done-drift (marked complete, work never landed).
2598
- CCV="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/implement/scripts/check-converge.sh"
2599
- # Own repo root: the shared $tmp accumulates stray files and git state from
2600
- # other sections, which leaks into diff/untracked output.
2601
- cvgroot="$tmp/cvgrepo"
2602
- cvg="$cvgroot/devos/specs/cvg"
2603
- mkdir -p "$cvg" "$cvgroot/src"
2604
- printf '%s\n' '## Implementation Plan' '### Task Group 1: cache' '**Files:** `src/cache.ts`' '### Task Group 2: export' '**Files:** `src/export.ts`' > "$cvg/spec.md"
2605
- git -C "$cvgroot" init -q -b main
2606
- git -C "$cvgroot" config user.email t@t && git -C "$cvgroot" config user.name t
2607
- echo base > "$cvgroot/src/base.txt"
2608
- git -C "$cvgroot" add -A && git -C "$cvgroot" commit -qm base
2609
- git -C "$cvgroot" checkout -qb feature
2610
- echo e > "$cvgroot/src/export.ts"
2611
- out=$(cd "$cvgroot" && bash "$CCV" devos/specs/cvg 2>&1); rc=$?
2612
- [[ $rc -eq 1 ]] || { echo "FAIL converge: expected findings, got rc=$rc"; echo "$out"; fail=1; }
2613
- echo "$out" | grep -q "Task Group 1" || { echo "FAIL converge: untouched group 1 not named"; fail=1; }
2614
- echo "$out" | grep -q "Task Group 2" && { echo "FAIL converge: touched group 2 wrongly flagged"; fail=1; }
2615
- # Landing group 1's work clears the finding.
2616
- echo c > "$cvgroot/src/cache.ts"
2617
- out=$(cd "$cvgroot" && bash "$CCV" devos/specs/cvg 2>&1); rc=$?
2618
- [[ $rc -eq 0 ]] || { echo "FAIL converge: converged repo still flagged ($out)"; fail=1; }
2619
- # Outside a git repo: advisory skip, never a brick.
2620
- mkdir -p "$tmp/nogit/devos/specs/cvg2" && cp "$cvg/spec.md" "$tmp/nogit/devos/specs/cvg2/"
2621
- out=$(bash "$CCV" "$tmp/nogit/devos/specs/cvg2" 2>&1); rc=$?
2622
- [[ $rc -eq 0 ]] || { echo "FAIL converge: non-git dir must exit 0 (fail open), got $rc"; fail=1; }
2623
- # Nothing touched at all: the branch is even with the default branch and the
2624
- # tree is clean, so the touched-file list is empty. An FNR == NR reader sees the
2625
- # plan as its first file and calls every declared path touched, which passes the
2626
- # one state the gate exists to catch.
2627
- cvgempty="$tmp/cvgempty"
2628
- mkdir -p "$cvgempty/devos/specs/cvg3"
2629
- cp "$cvg/spec.md" "$cvgempty/devos/specs/cvg3/"
2630
- git -C "$cvgempty" init -q -b main
2631
- git -C "$cvgempty" config user.email t@t && git -C "$cvgempty" config user.name t
2632
- git -C "$cvgempty" add -A && git -C "$cvgempty" commit -qm base
2633
- out=$(cd "$cvgempty" && bash "$CCV" devos/specs/cvg3 2>&1); rc=$?
2634
- [[ $rc -eq 1 ]] || { echo "FAIL converge: empty touched list must still flag both groups, got rc=$rc"; echo "$out"; fail=1; }
2635
- echo "$out" | grep -q "Task Group 1" || { echo "FAIL converge: empty touched list did not name group 1"; fail=1; }
2636
- echo "$out" | grep -q "Task Group 2" || { echo "FAIL converge: empty touched list did not name group 2"; fail=1; }
2637
-
2638
- # A directory declaration must match descendants, not a sibling with the same
2639
- # prefix. `src` owns `src/file.ts`, but not `src2/file.ts`.
2640
- cvgprefix="$tmp/cvgprefix"
2641
- mkdir -p "$cvgprefix/devos/specs/cvg4"
2642
- printf '%s\n' '## Implementation Plan' '### Task Group 1: source' '**Files:** `src`' > "$cvgprefix/devos/specs/cvg4/spec.md"
2643
- git -C "$cvgprefix" init -q -b main
2644
- git -C "$cvgprefix" config user.email t@t && git -C "$cvgprefix" config user.name t
2645
- mkdir -p "$cvgprefix/src2"
2646
- git -C "$cvgprefix" add -A && git -C "$cvgprefix" commit -qm base
2647
- git -C "$cvgprefix" checkout -qb feature
2648
- echo touched > "$cvgprefix/src2/file.ts"
2649
- git -C "$cvgprefix" add -A && git -C "$cvgprefix" commit -qm touch
2650
- out=$(cd "$cvgprefix" && bash "$CCV" devos/specs/cvg4 2>&1); rc=$?
2651
- [[ $rc -eq 1 ]] || { echo "FAIL converge: a sibling prefix must not satisfy a directory declaration, got rc=$rc ($out)"; fail=1; }
2652
- echo "$out" | grep -q "Task Group 1" || { echo "FAIL converge: sibling prefix wrongly treated as the declared directory"; fail=1; }
2653
-
2654
- # A group the gate cannot identify is not checkable, and a group that cannot be
2655
- # checked used to be the one case this gate passed. The header pattern required
2656
- # whitespace after "Task Group", so "### Task Group: Parser" never even matched:
2657
- # a plan whose only group declared an untouched file exited 0.
2658
- cvgunnum="$tmp/cvgunnum"
2659
- mkdir -p "$cvgunnum/devos/specs/cvg5" "$cvgunnum/src"
2660
- printf '%s\n' '## Implementation Plan' '### Task Group: Parser' '**Files:** `src/never-touched.ts`' \
2661
- > "$cvgunnum/devos/specs/cvg5/spec.md"
2662
- echo base > "$cvgunnum/src/other.ts"
2663
- git -C "$cvgunnum" init -q -b main
2664
- git -C "$cvgunnum" config user.email t@t && git -C "$cvgunnum" config user.name t
2665
- git -C "$cvgunnum" add -A && git -C "$cvgunnum" commit -qm base
2666
- git -C "$cvgunnum" checkout -qb feature
2667
- out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
2668
- [[ $rc -eq 1 ]] || { echo "FAIL converge: an unidentifiable group must be reported, got rc=$rc ($out)"; fail=1; }
2669
- echo "$out" | grep -q "has no resolvable group number" || { echo "FAIL converge: unidentifiable group not diagnosed ($out)"; fail=1; }
2670
- echo "$out" | grep -q "Task Group: Parser" || { echo "FAIL converge: diagnostic does not name the header"; fail=1; }
2671
- # It stays reported even when the declared file did land: the point is that the
2672
- # gate could not attribute the files to a group, not that a file went missing.
2673
- echo landed > "$cvgunnum/src/never-touched.ts"
2674
- out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
2675
- [[ $rc -eq 1 ]] && echo "$out" | grep -q "has no resolvable group number" \
2676
- || { echo "FAIL converge: an unidentifiable group must stay reported when files land ($out)"; fail=1; }
2677
- # The documented and legacy numbered forms must be unaffected, and a heading
2678
- # that merely contains the words "Task Groups" is not a group.
2679
- printf '%s\n' '## Task Groups Overview' '### Task Group 1: Parser' '**Files:** `src/absent.ts`' \
2680
- > "$cvgunnum/devos/specs/cvg5/spec.md"
2681
- out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
2682
- [[ $rc -eq 1 ]] && echo "$out" | grep -q "Task Group 1" \
2683
- && ! echo "$out" | grep -q "no resolvable group number" \
2684
- || { echo "FAIL converge: numbered group misparsed after the change ($out)"; fail=1; }
2685
- printf '%s\n' '## Task Groups Overview' '### Task Group 1: Parser' '**Files:** `src/never-touched.ts`' \
2686
- > "$cvgunnum/devos/specs/cvg5/spec.md"
2687
- echo more >> "$cvgunnum/src/never-touched.ts"
2688
- out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
2689
- [[ $rc -eq 0 ]] || { echo "FAIL converge: a 'Task Groups' heading was read as a group ($out)"; fail=1; }
2690
- printf '%s\n' '## Implementation Plan' '### Task Group A: Parser' '**Files:** `src/never-touched.ts`' \
2691
- > "$cvgunnum/devos/specs/cvg5/spec.md"
2692
- out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
2693
- [[ $rc -eq 0 ]] || { echo "FAIL converge: the pre-0.8.6 letter form must still resolve ($out)"; fail=1; }
2694
-
2695
- # The declared file list is stored newline-joined, and interpolating it directly
2696
- # broke the one-line envelope: two untouched files printed three lines with a
2697
- # bare ") declares files" tail. A reader taking one line per finding saw a
2698
- # truncated message.
2699
- cvgmulti="$tmp/cvgmulti"
2700
- mkdir -p "$cvgmulti/devos/specs/cvg6" "$cvgmulti/src"
2701
- printf '%s\n' '## Implementation Plan' '### Task Group 1: Parser' '**Files:** `src/one.ts`, `src/two.ts`' \
2702
- > "$cvgmulti/devos/specs/cvg6/spec.md"
2703
- echo base > "$cvgmulti/src/base.ts"
2704
- git -C "$cvgmulti" init -q -b main
2705
- git -C "$cvgmulti" config user.email t@t && git -C "$cvgmulti" config user.name t
2706
- git -C "$cvgmulti" add -A && git -C "$cvgmulti" commit -qm base
2707
- git -C "$cvgmulti" checkout -qb feature
2708
- echo work > "$cvgmulti/src/work.ts"
2709
- git -C "$cvgmulti" add -A && git -C "$cvgmulti" commit -qm work
2710
- out=$(cd "$cvgmulti" && bash "$CCV" devos/specs/cvg6 2>&1); rc=$?
2711
- [[ $rc -eq 1 ]] || { echo "FAIL converge: a multi-file untouched group must fail, got rc=$rc ($out)"; fail=1; }
2712
- echo "$out" | grep -Fq 'Task Group 1 (src/one.ts, src/two.ts) declares files' \
2713
- || { echo "FAIL converge: multi-file finding is not one line: $out"; fail=1; }
2714
- # The stray tail lines that broke the envelope must be gone.
2715
- [[ "$(printf '%s\n' "$out" | grep -c '^src/')" -eq 0 ]] \
2716
- || { echo "FAIL converge: file list still leaks onto its own line: $out"; fail=1; }
2717
- [[ "$(printf '%s\n' "$out" | grep -c '^)')" -eq 0 ]] \
2718
- || { echo "FAIL converge: a bare closing paren line remains: $out"; fail=1; }
2719
- # The single-file form is unchanged: one file, no trailing separator.
2720
- printf '%s\n' '## Implementation Plan' '### Task Group 1: Parser' '**Files:** `src/absent.ts`' \
2721
- > "$cvgmulti/devos/specs/cvg6/spec.md"
2722
- out=$(cd "$cvgmulti" && bash "$CCV" devos/specs/cvg6 2>&1); rc=$?
2723
- [[ $rc -eq 1 ]] || { echo "FAIL converge: single-file untouched group must fail, got rc=$rc ($out)"; fail=1; }
2724
- echo "$out" | grep -Fq 'Task Group 1 (src/absent.ts) declares files' \
2725
- || { echo "FAIL converge: single-file finding changed shape: $out"; fail=1; }
2726
-
2727
- # --- C route: validate-spec accepts an Invariant in place of scenarios ------
2728
-
2729
- # A behavior-preserving spec declares what must NOT change, so
2730
- # it has no scenarios and no SC-n to trace. Every case here is a shape the old
2731
- # script either rejected (a valid C spec) or waved through (a confused one).
2732
- VS="$SKILLS/implement/scripts/validate-spec.sh"
2733
- cdir="$tmp/croute"
2734
- mkspec() { mkdir -p "$cdir/$1" && cat >"$cdir/$1/spec-functional.md"; }
2735
-
2736
- mkspec ok <<'EOF'
2737
- # Migrate the runner
2738
- ## 1. Goal
2739
- Config is duplicated across four packages.
2740
- ## 2. Invariant
2741
- **Commands that MUST still pass, with no fewer tests executed:**
2742
- - `npm test` -> baseline: exit 0, 42 passed
2743
- **Public surface that MUST NOT move:**
2744
- - none, internal only
2745
- ## 3. Out of Scope
2746
- - Behavior change of any kind.
2747
- EOF
2748
- bash "$VS" "$cdir/ok" >/dev/null 2>&1
2749
- check "validate-spec: an Invariant spec with no scenarios passes" 0 $?
2750
-
2751
- mkspec nocmd <<'EOF'
2752
- # Migrate
2753
- ## 1. Goal
2754
- x
2755
- ## 2. Invariant
2756
- **Commands that MUST still pass:**
2757
- - the existing suite
2758
- **Public surface that MUST NOT move:**
2759
- - none, internal only
2760
- ## 3. Out of Scope
2761
- - none
2762
- EOF
2763
- bash "$VS" "$cdir/nocmd" >/dev/null 2>&1
2764
- check "validate-spec: an Invariant naming no command fails" 1 $?
2765
-
2766
- mkspec nosurface <<'EOF'
2767
- # Migrate
2768
- ## 1. Goal
2769
- x
2770
- ## 2. Invariant
2771
- **Commands that MUST still pass:**
2772
- - `npm test` -> baseline: exit 0, 42 passed
2773
- ## 3. Out of Scope
2774
- - none
2775
- EOF
2776
- bash "$VS" "$cdir/nosurface" >/dev/null 2>&1
2777
- check "validate-spec: an Invariant with no public-surface line fails" 1 $?
2778
-
2779
- mkspec both <<'EOF'
2780
- # Migrate
2781
- ## 1. Goal
2782
- x
2783
- ## 2. Invariant
2784
- **Commands that MUST still pass:**
2785
- - `npm test` -> baseline: exit 0, 42 passed
2786
- **Public surface that MUST NOT move:**
2787
- - none, internal only
2788
- ## 3bis. Behavior delta
2789
- **Added** (capability: `runner`):
2790
- - reports coverage inline
2791
- ## 4. Out of Scope
2792
- - none
2793
- EOF
2794
- bash "$VS" "$cdir/both" >/dev/null 2>&1
2795
- check "validate-spec: Invariant plus a non-empty Behavior delta fails" 1 $?
2796
-
2797
- mkspec nonedelta <<'EOF'
2798
- # Migrate
2799
- ## 1. Goal
2800
- x
2801
- ## 2. Invariant
2802
- **Commands that MUST still pass:**
2803
- - `npm test` -> baseline: exit 0, 42 passed
2804
- **Public surface that MUST NOT move:**
2805
- - none, internal only
2806
- ## 3bis. Behavior delta
2807
- **Added:** (none)
2808
- **Removed:** (none)
2809
- ## 4. Out of Scope
2810
- - none
2811
- EOF
2812
- bash "$VS" "$cdir/nonedelta" >/dev/null 2>&1
2813
- check "validate-spec: Invariant plus an empty Behavior delta passes" 0 $?
2814
-
2815
- # The heading is matched exactly, never as a substring: a feature spec is free
2816
- # to have a "Business invariants" section, and reading that as the C route
2817
- # would silently switch off the scenario checks it still needs.
2818
- mkspec businv <<'EOF'
2819
- # Feature
2820
- ## 1. Goal
2821
- x
2822
- ## 2. Business invariants
2823
- - balance MUST never go below 0
2824
- ## 3. Out of Scope
2825
- - none
2826
- EOF
2827
- bash "$VS" "$cdir/businv" >/dev/null 2>&1
2828
- check "validate-spec: a 'Business invariants' heading is not the C route" 1 $?
2829
-
2830
- mkspec feature <<'EOF'
2831
- # Feature
2832
- ## 1. Goal
2833
- x
2834
- ## 2. Functionality
2835
- - Scenario SC-1: happy / WHEN a user submits THEN the record MUST persist
2836
- ## 3. Out of Scope
2837
- - none
2838
- EOF
2839
- bash "$VS" "$cdir/feature" >/dev/null 2>&1
2840
- check "validate-spec: an ordinary scenario spec is unaffected" 0 $?
2841
-
2842
- # The plan capsule's own structural checks. Both run at Phase A step 5b now,
2843
- # before the human ever reads the plan, so a defect here costs one edit rather
2844
- # than a re-derivation after the code was written against it.
2845
- pdir="$cdir/plan"; mkdir -p "$pdir"
2846
- printf '# P\n## 1. Goal\nx\n## 2. Functionality\n- Scenario SC-1: a / WHEN x THEN y\n## 3. Out of Scope\n- none\n' > "$pdir/spec-functional.md"
2847
- mkplan() { { printf '## Local Craft\nNone\n'; cat; } > "$pdir/spec.md"; }
2848
-
2849
- mkplan <<'MD'
2850
- ## Design Convergence
2851
- - **Direct MVP**: call it directly
2852
- - **Failed Items**: None
2853
- ## Implementation Plan
2854
- ### Task Group 1: A
2855
- **Files:** `a.ts`
2856
- **Scenarios covered:** SC-1
2857
- MD
2858
- bash "$VS" "$pdir" >/dev/null 2>&1
2859
- check "validate-spec: a well-formed capsule passes" 0 $?
2860
-
2861
- # "None" with a reason is still None. The fix line says to state None, and a
2862
- # capsule that writes "None: the tree is new" was failing the gate that told
2863
- # it to write None.
2864
- { printf '## Local Craft\nNone: the project has nothing to reuse here\n'; cat <<'MD'
2865
- ## Design Convergence
2866
- - **Direct MVP**: call it directly
2867
- - **Failed Items**: None
2868
- ## Implementation Plan
2869
- ### Task Group 1: A
2870
- **Files:** `a.ts`
2871
- **Scenarios covered:** SC-1
2872
- MD
2873
- } > "$pdir/spec.md"
2874
- bash "$VS" "$pdir" >/dev/null 2>&1
2875
- check "validate-spec: Local Craft may state None with a reason" 0 $?
2876
-
2877
- # A change that spans many units holds one group for the whole change, and
2878
- # the group's files are the target tree. The units live in the manifest, so
2879
- # the capsule does not carry them.
2880
- { printf '## Local Craft\nNone\n'; cat <<'MD'
2881
- ## Design Convergence
2882
- - **Direct MVP**: one group, the units live in the manifest
2883
- - **Failed Items**: None
2884
- ## Implementation Plan
2885
- ### Task Group 1: the port
2886
- **Files:** `modernized/`
2887
- **Scenarios covered:** SC-1
2888
- MD
2889
- } > "$pdir/spec.md"
2890
- bash "$VS" "$pdir" >/dev/null 2>&1
2891
- check "validate-spec: a one-group capsule naming a directory passes" 0 $?
2892
-
2893
- write_capsule() {
2894
- local blank_lines="$1" trailing_newline="$2"
2895
- {
2896
- printf '%s\n' \
2897
- '## Local Craft' \
2898
- 'None' \
2899
- '## Design Convergence' \
2900
- '- **Direct MVP**: call it directly' \
2901
- '- **Failed Items**: None' \
2902
- '## Implementation Plan' \
2903
- '### Task Group 1: A' \
2904
- '**Files:** `a.ts`' \
2905
- '**Scenarios covered:** SC-1'
2906
- for ((i = 0; i < blank_lines; i++)); do printf '\n'; done
2907
- if [[ "$trailing_newline" -eq 1 ]]; then
2908
- printf 'final logical line\n'
2909
- else
2910
- printf 'final logical line'
2911
- fi
2912
- } > "$pdir/spec.md"
2913
- }
2914
-
2915
- write_capsule 290 1
2916
- bash "$VS" "$pdir" >/dev/null 2>&1
2917
- check "validate-spec: a 300-line capsule with a trailing newline passes" 0 $?
2918
-
2919
- write_capsule 290 0
2920
- bash "$VS" "$pdir" >/dev/null 2>&1
2921
- check "validate-spec: a 300-line capsule without a trailing newline passes" 0 $?
2922
-
2923
- write_capsule 291 0
2924
- out=$(bash "$VS" "$pdir" 2>&1); rc=$?
2925
- grep -q 'spec.md capsule is 301 lines (budget 300)' <<<"$out" && [[ $rc -eq 1 ]] \
2926
- && check "validate-spec: an unterminated 301st line is counted" 0 $? \
2927
- || { echo "FAIL validate-spec: unterminated final line not counted"; echo "$out"; fail=1; }
2928
-
2929
- cat > "$pdir/spec.md" <<'MD'
2930
- ## Design Convergence
2931
- - **Direct MVP**: call it directly
2932
- ## Implementation Plan
2933
- ### Task Group 1: A
2934
- **Files:** `a.ts`
2935
- **Scenarios covered:** SC-1
2936
- MD
2937
- out=$(bash "$VS" "$pdir" 2>&1); rc=$?
2938
- grep -q 'spec.md has no Local Craft section' <<<"$out" && [[ $rc -eq 1 ]] \
2939
- && check "validate-spec: a capsule with no Local Craft section fails" 0 $? \
2940
- || { echo "FAIL validate-spec: missing Local Craft not caught"; echo "$out"; fail=1; }
2941
-
2942
- cat > "$pdir/spec.md" <<'MD'
2943
- ## Local Craft
2944
- - Reuse the existing helper.
2945
- ## Design Convergence
2946
- - **Direct MVP**: call it directly
2947
- ## Implementation Plan
2948
- ### Task Group 1: A
2949
- **Files:** `a.ts`
2950
- **Scenarios covered:** SC-1
2951
- MD
2952
- out=$(bash "$VS" "$pdir" 2>&1); rc=$?
2953
- grep -q 'spec.md Local Craft has no file:line or URL evidence' <<<"$out" && [[ $rc -eq 1 ]] \
2954
- && check "validate-spec: ungrounded Local Craft evidence fails" 0 $? \
2955
- || { echo "FAIL validate-spec: ungrounded Local Craft not caught"; echo "$out"; fail=1; }
2956
-
2957
- cat > "$pdir/spec.md" <<'MD'
2958
- ## Local Craft
2959
- - `src/a.ts:1` uses the existing helper.
2960
- ## Design Convergence
2961
- - **Direct MVP**: call it directly
2962
- ## Implementation Plan
2963
- ### Task Group 1: A
2964
- **Files:** `a.ts`
2965
- **Scenarios covered:** SC-1
2966
- MD
2967
- bash "$VS" "$pdir" >/dev/null 2>&1; check "validate-spec: grounded Local Craft evidence passes" 0 $?
2968
-
2969
- # A group with no Files list is silently skipped by check-converge.sh, so the
2970
- # omission switches off an existing gate instead of merely thinning the plan.
2971
- mkplan <<'MD'
2972
- ## Design Convergence
2973
- - **Direct MVP**: call it directly
2974
- ## Implementation Plan
2975
- ### Task Group 1: A
2976
- **Scenarios covered:** SC-1
2977
- MD
2978
- out=$(bash "$VS" "$pdir" 2>&1)
2979
- grep -q 'has no \*\*Files:\*\* line' <<<"$out" \
2980
- && check "validate-spec: a task group with no Files line fails" 0 $? \
2981
- || { echo "FAIL validate-spec: missing Files line not caught"; echo "$out"; fail=1; }
2982
-
2983
- # The section a plan skips exactly when it is adding the most.
2984
- mkplan <<'MD'
2985
- ## Implementation Plan
2986
- ### Task Group 1: A
2987
- **Files:** `a.ts`
2988
- **Scenarios covered:** SC-1
2989
- MD
2990
- out=$(bash "$VS" "$pdir" 2>&1)
2991
- grep -q 'no Design Convergence section' <<<"$out" \
2992
- && check "validate-spec: a capsule with no Design Convergence fails" 0 $? \
2993
- || { echo "FAIL validate-spec: missing Design Convergence not caught"; echo "$out"; fail=1; }
2994
-
2995
- # A `## ` heading after the last group must not be read as a group missing its
2996
- # Files line: Change Log and Implementation Notes always follow the plan.
2997
- mkplan <<'MD'
2998
- ## Design Convergence
2999
- - **Direct MVP**: call it directly
3000
- ## Implementation Plan
3001
- ### Task Group 1: A
3002
- **Files:** `a.ts`
3003
- **Scenarios covered:** SC-1
3004
- ## Implementation Notes
3005
- - none
3006
- ## Change Log
3007
- - none
3008
- MD
3009
- bash "$VS" "$pdir" >/dev/null 2>&1
3010
- check "validate-spec: trailing sections after the last group are not a gap" 0 $?
3011
-
3012
- # Presence alone passes with four empty labels. Every Adopted Addition answers
3013
- # a Failed Item, so additions against `Failed Items: None` is the section
3014
- # contradicting itself, which is the shape an unjustified abstraction takes.
3015
- mkconv() { printf '## Local Craft\nNone\n## Design Convergence\n%s\n## Implementation Plan\n### Task Group 1: A\n**Files:** `a.ts`\n**Scenarios covered:** SC-1\n' "$1" > "$pdir/spec.md"; }
3016
-
3017
- mkconv '- **Direct MVP**: call it directly
3018
- - **Failed Items**: None
3019
- - **Adopted Additions**: None'
3020
- bash "$VS" "$pdir" >/dev/null 2>&1
3021
- check "validate-spec: a converged plan with no additions passes" 0 $?
3022
-
3023
- mkconv '- **Direct MVP**: call it directly
3024
- - **Failed Items**: None
3025
- - **Adopted Additions**: a registry layer -> nothing -> it is more flexible'
3026
- out=$(bash "$VS" "$pdir" 2>&1)
3027
- grep -q 'adopts additions while Failed Items is None' <<<"$out" \
3028
- && check "validate-spec: an addition with no Failed Item fails" 0 $? \
3029
- || { echo "FAIL validate-spec: unjustified addition not caught"; echo "$out"; fail=1; }
3030
-
3031
- mkconv '- **Direct MVP**: call it directly
3032
- - **Failed Items**: the MVP cannot express per-tenant limits, evidence in the scenario
3033
- - **Adopted Additions**: a per-tenant table -> that item -> nothing smaller holds the state'
3034
- bash "$VS" "$pdir" >/dev/null 2>&1
3035
- check "validate-spec: an addition answering a Failed Item passes" 0 $?
3036
- rm -rf "$pdir"
3037
-
3038
- # --- Concurrent grouping across specs ----------------------
3039
- # Advisory only. The failure that matters is a false CONCURRENT: two specs
3040
- # grouped when a dependency or a shared file makes them unsafe.
3041
- CR="$SKILLS/status/scripts/check-requires.py"
3042
- gdir="$tmp/concurrent"
3043
- mkopen() { # mkopen <name> <requires|-> <files|->
3044
- mkdir -p "$gdir/devos/specs/$1"
3045
- { echo "# $1"; echo "## Goal"; echo "x"
3046
- [[ "$2" != "-" ]] && echo "Requires: [$2]"
3047
- echo "## Out of Scope"; echo "- none"; } >"$gdir/devos/specs/$1/spec-functional.md"
3048
- [[ "$3" != "-" ]] && printf '### Task Group 1: t\n**Files:** %s\n' "$3" >"$gdir/devos/specs/$1/spec.md"
3049
- return 0
3050
- }
3051
- mkopen alpha - '`src/a.ts`, `src/shared.ts`'
3052
- mkopen beta - '`src/b.ts`'
3053
- mkopen gamma beta '`src/g.ts`'
3054
- mkopen delta - '`src/shared.ts`, `src/d.ts`'
3055
- mkopen epsilon - -
3056
- gout=$(python3 "$CR" --concurrent "$gdir" 2>&1)
3057
-
3058
- check "concurrent: disjoint independent specs group together" \
3059
- "CONCURRENT group 1: alpha beta" "$(grep '^CONCURRENT group 1' <<<"$gout")"
3060
- check "concurrent: a shared planned file separates two specs" 1 \
3061
- "$(grep -c 'OVERLAP alpha + delta: both plan to touch src/shared.ts' <<<"$gout")"
3062
- check "concurrent: a dependency edge separates two specs" 1 \
3063
- "$(grep -c 'BLOCKED beta + gamma' <<<"$gout")"
3064
- check "concurrent: a spec with no plan is UNPLANNED, never grouped" 1 \
3065
- "$(grep -c '^UNPLANNED epsilon' <<<"$gout")"
3066
- check "concurrent: an unplanned spec appears in no CONCURRENT group" 0 \
3067
- "$(grep '^CONCURRENT group' <<<"$gout" | grep -c epsilon || true)"
3068
- check "concurrent: dependent specs never share a group" 0 \
3069
- "$(grep '^CONCURRENT group' <<<"$gout" | grep -c 'beta.*gamma\|gamma.*beta' || true)"
3070
-
3071
- # Directory declarations and descendants are not exact-string matches. Keep
3072
- # this separate from task-group safety so both consumers stay covered.
3073
- pdir="$tmp/concurrent-path"
3074
- mkdir -p "$pdir/devos/specs/alpha" "$pdir/devos/specs/beta"
3075
- for name in alpha beta; do
3076
- printf '# %s\n\n## Goal\nx\n\n## Out of Scope\n- none\n' "$name" \
3077
- > "$pdir/devos/specs/$name/spec-functional.md"
3078
- printf '## Implementation Plan\n### Task Group 1: work\n**Files:** `%s`\n' \
3079
- "$([[ "$name" == alpha ]] && echo src || echo src/render.ts)" \
3080
- > "$pdir/devos/specs/$name/spec.md"
3081
- done
3082
- pout=$(python3 "$CR" --concurrent "$pdir" 2>&1)
3083
- check "concurrent: directory and descendant specs are separated" 1 \
3084
- "$(grep -c '^OVERLAP alpha + beta: both plan to touch src$' <<<"$pout")"
3085
- check "concurrent: sibling prefix specs remain groupable" 1 \
3086
- "$(sed -i.bak 's|src/render.ts|src2/render.ts|' "$pdir/devos/specs/beta/spec.md"; rm -f "$pdir/devos/specs/beta/spec.md.bak"; python3 "$CR" --concurrent "$pdir" 2>&1 | grep -c '^CONCURRENT group 1: alpha beta$')"
3087
-
3088
- rm -rf "$gdir/devos/specs/beta" "$gdir/devos/specs/gamma" "$gdir/devos/specs/delta" "$gdir/devos/specs/epsilon"
3089
- check "concurrent: a single open spec reports nothing to parallelize" \
3090
- "CONCURRENT 1 open spec(s): nothing to run in parallel" \
3091
- "$(python3 "$CR" --concurrent "$gdir" 2>&1)"
3092
-
3093
- # --- findings-summary.sh: count records, not ledger lines -----------------
3094
- FS="$SKILLS/status/scripts/findings-summary.sh"
3095
- fdir="$tmp/findings/devos"
3096
- mkdir -p "$fdir"
3097
- cat > "$fdir/deferred.md" <<'MD'
3098
- # Deferred Findings
3099
-
3100
- Severity and resolution are not records by themselves.
3101
-
3102
- - [2026-01-01] [warning] defer: first finding
3103
- * [2026-01-02] [critical] [spec-amendment]: second finding
3104
- - [2026-01-03] [suggestion] [reject]: third finding
3105
- - [2026-01-04] [warning] defer: patch the critical path
3106
- - [2026-01-05] [suggestion] reject: avoid patch and spec-amendment in the message
3107
- [not-a-record] this prose must not be counted
3108
- MD
3109
- fout=$(bash "$FS" "$tmp/findings" 2>&1)
3110
- check "findings: headings, blank lines, and prose are not findings" 1 \
3111
- "$(grep -c '^| Total findings | 5 |$' <<<"$fout")"
3112
- if grep -Fqx '| By severity: critical | 1 |' <<<"$fout" \
3113
- && grep -Fqx '| By severity: warning | 2 |' <<<"$fout" \
3114
- && grep -Fqx '| By resolution: patch | 0 |' <<<"$fout" \
3115
- && grep -Fqx '| By resolution: spec-amendment | 1 |' <<<"$fout" \
3116
- && grep -Fqx '| By resolution: defer | 2 |' <<<"$fout" \
3117
- && grep -Fqx '| By resolution: reject | 2 |' <<<"$fout"; then
3118
- check "findings: severity and resolution counts come from record fields" 0 0
3119
- else
3120
- check "findings: severity and resolution counts come from record fields" 1 0
3121
- fi
3122
- check "findings: trend counts record dates only" 1 \
3123
- "$(grep -c '^ 2026-01: 5 findings$' <<<"$fout")"
3124
-
3125
- # The month belongs to the record's leading bracketed field. A month named in
3126
- # the message is prose, not a second date, and counting it reported one finding
3127
- # under two months.
3128
- fmonth="$tmp/findings-month/devos"
3129
- mkdir -p "$fmonth/specs/archive/example"
3130
- printf -- '- [2026-01-01] [warning] defer: regression was seen in 2025-12\n' \
3131
- > "$fmonth/deferred.md"
3132
- fmout=$(bash "$FS" "$tmp/findings-month" 2>&1)
3133
- check "findings: a month named in the message is not a second date" 1 \
3134
- "$(grep -c '^ 2026-01: 1 findings$' <<<"$fmout")"
3135
- ! grep -Fq ' 2025-12:' <<<"$fmout"
3136
- check "findings: trend never invents a month from message text" 0 $?
3137
- rm -rf "$tmp/findings-month"
3138
-
3139
- # The zero-findings rule counts specs, not ledger lines, and only L-route ones.
3140
- # Counting records against a completed-spec denominator printed "3 of 2", a
3141
- # ratio no set of specs can have, and an M-route entry inflated an L-only rule.
3142
- zroute="$tmp/findings-zero-route/devos/specs/archive"
3143
- mkdir -p "$zroute/large" "$zroute/small"
3144
- cat > "$tmp/findings-zero-route/devos/deferred.md" <<'MD'
3145
- - [large] [warning] defer: zero findings on first run
3146
- - [large] [suggestion] defer: zero findings on first run
3147
- - [small] [warning] defer: no findings on first run
3148
- MD
3149
- printf 'large\tL\tyes\t0\tnone\tyes\t?\t-/0\t-\n' > "$zroute/large.quality.md"
3150
- printf 'small\tM\tyes\t0\tnone\tyes\t?\t-/0\t-\n' > "$zroute/small.quality.md"
3151
- zout=$(bash "$FS" "$tmp/findings-zero-route" 2>&1)
3152
- check "findings: a spec counts once however many records mention it" 1 \
3153
- "$(grep -c '^Zero-findings rule: 1 of 2 completed specs' <<<"$zout")"
3154
- ! grep -Fq ' 2 of ' <<<"$zout" && ! grep -Fq ' 3 of ' <<<"$zout"
3155
- check "findings: the numerator can never exceed the completed specs" 0 $?
3156
- sed -i.bak 's/^small\tM\t/small\tL\t/' "$zroute/small.quality.md" && rm -f "$zroute/small.quality.md.bak"
3157
- zout2=$(bash "$FS" "$tmp/findings-zero-route" 2>&1)
3158
- check "findings: an L-route spec does count toward the rule" 1 \
3159
- "$(grep -c '^Zero-findings rule: 2 of 2 completed specs' <<<"$zout2")"
3160
- rm -rf "$tmp/findings-zero-route"
3161
-
3162
- # The rule measures a clean first pass, so the record has to claim one. Prose
3163
- # that mentions an absence is not a measurement, and crediting it inflated the
3164
- # numerator with a spec whose first pass was never established.
3165
- fpass="$tmp/findings-first-pass/devos/specs/archive"
3166
- mkdir -p "$fpass/prose" "$fpass/measured"
3167
- printf -- '- [prose] [warning] defer: no findings were observed\n' \
3168
- > "$tmp/findings-first-pass/devos/deferred.md"
3169
- printf 'prose\tL\tyes\t0\tnone\tyes\t?\t-/0\t-\n' > "$fpass/prose.quality.md"
3170
- printf 'measured\tL\tyes\t0\tnone\tyes\t?\t-/0\t-\n' > "$fpass/measured.quality.md"
3171
- fpout=$(bash "$FS" "$tmp/findings-first-pass" 2>&1)
3172
- check "findings: prose mentioning no findings is not a first-pass claim" 1 \
3173
- "$(grep -c '^Zero-findings rule: 0 of 2 completed specs' <<<"$fpout")"
3174
- printf -- '- [measured] [warning] defer: zero findings on first run\n' \
3175
- >> "$tmp/findings-first-pass/devos/deferred.md"
3176
- fpout2=$(bash "$FS" "$tmp/findings-first-pass" 2>&1)
3177
- check "findings: an explicit first-run claim is still counted" 1 \
3178
- "$(grep -c '^Zero-findings rule: 1 of 2 completed specs' <<<"$fpout2")"
3179
- rm -rf "$tmp/findings-first-pass"
3180
-
3181
- # --- .quality.md: C route and the two derived columns ------
3182
- QC="$SKILLS/status/scripts/quality-correlation.sh"
3183
- qdir="$tmp/qual/devos/specs/archive"
3184
- mkdir -p "$qdir"
3185
- printf 'a\tL\tyes\t1\tsuggestion\tyes\t2\t2/5\n' >"$qdir/a.quality.md"
3186
- printf 'b\tM\tyes\t0\tnone\tyes\t1\t-/0\n' >"$qdir/b.quality.md"
3187
- printf 'c\tC\tyes\t0\tnone\tyes\t1\t0/2\n' >"$qdir/c.quality.md"
3188
- printf 'd\tC\tno\t3\tcritical\tno\t4\t3/3\n' >"$qdir/d.quality.md"
3189
- printf 'e\tL\tyes\t0\tnone\tyes\t?\t?/4\n' >"$qdir/e.quality.md"
3190
- printf 'f\tL\tyes\t2\twarning\tyes\n' >"$qdir/f.quality.md"
3191
- qout=$(bash "$QC" "$tmp/qual" 2>&1)
3192
-
3193
- check "quality: the C route is counted, not folded into ?" 1 \
3194
- "$(grep -cE '^C +2 +3 +1\.5$' <<<"$qout")"
3195
- check "quality: a 6-field row still counts toward findings" 1 \
3196
- "$(grep -cE '^TOTAL +6 +6 +1\.0$' <<<"$qout")"
3197
- check "quality: an unknown triage count is excluded from the average" 1 \
3198
- "$(grep -cE '^L +1 +2\.0' <<<"$qout")"
3199
- check "quality: the audit hit rate ignores unrecorded hits" 1 \
3200
- "$(grep -c '5 of 10 audit findings confirmed downstream (50%), across 4 spec(s)' <<<"$qout")"
3201
- check "quality: an excluded row is reported, not silently dropped" 1 \
3202
- "$(grep -c '1 spec(s) excluded: audit findings recorded, hits never were' <<<"$qout")"
3203
-
3204
- qbad="$tmp/qual-bad/devos/specs/archive"
3205
- mkdir -p "$qbad"
3206
- for name in a b c d e; do
3207
- printf '%s\tL\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qbad/$name.quality.md"
56
+ for part in "$SUITE"/*.sh; do
57
+ read -r p f <"$results/$(basename "$part" .sh)" || { echo "FAIL suite $(basename "$part") produced no result" >&2; f=1 p=0; }
58
+ pass=$((pass + p)) fail=$((fail + f))
3208
59
  done
3209
- printf 'malformed row\n' > "$qbad/f.quality.md"
3210
- qbadout=$(bash "$QC" "$tmp/qual-bad" 2>&1)
3211
- check "quality: malformed rows do not inflate analyzed specs" 1 \
3212
- "$(grep -c '^Specs analyzed: 5$' <<<"$qbadout")"
3213
- check "quality: malformed rows are surfaced" 1 \
3214
- "$(grep -c '^Skipped quality rows: 1 (fewer than 6 tab-separated fields)$' <<<"$qbadout")"
3215
-
3216
- qblank="$tmp/qual-blank-audit/devos/specs/archive"
3217
- mkdir -p "$qblank"
3218
- for name in a b c d e; do
3219
- printf '%s\tS\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qblank/$name.quality.md"
3220
- done
3221
- printf 'blank-audit\tS\tyes\t0\tnone\tyes\t1\t\t-\n' > "$qblank/blank-audit.quality.md"
3222
- qblankout=$(bash "$QC" "$tmp/qual-blank-audit" 2>&1)
3223
- check "quality: a present blank audit field is surfaced" 1 \
3224
- "$(grep -c '^Invalid audit-hits values: 1 ' <<<"$qblankout")"
3225
- check "quality: a blank audit field is excluded only from audit metrics" 1 \
3226
- "$(grep -c 'no audit findings to confirm across 5 spec(s)' <<<"$qblankout")"
3227
- check "quality: a blank audit row remains in general aggregates" 1 \
3228
- "$(grep -cE '^TOTAL +6 +0 +0\.0$' <<<"$qblankout")"
3229
-
3230
- qabsent="$tmp/qual-absent-audit/devos/specs/archive"
3231
- mkdir -p "$qabsent"
3232
- for name in a b c d e; do
3233
- printf '%s\tS\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qabsent/$name.quality.md"
3234
- done
3235
- printf 'absent-audit\tS\tyes\t0\tnone\tyes\t1\t-\n' > "$qabsent/absent-audit.quality.md"
3236
- qabsentout=$(bash "$QC" "$tmp/qual-absent-audit" 2>&1)
3237
- check "quality: an absent audit column is not invalid" 1 \
3238
- "$(grep -c '^Invalid audit-hits values:' <<<"$qabsentout" || true)"
3239
- check "quality: an absent audit column remains analyzable" 1 \
3240
- "$(grep -cE '^TOTAL +6 +0 +0\.0$' <<<"$qabsentout")"
3241
-
3242
- qmeta="$tmp/qual-audit-metadata/devos/specs/archive"
3243
- mkdir -p "$qmeta"
3244
- for name in a b c d e; do
3245
- printf '%s\tS\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qmeta/$name.quality.md"
3246
- done
3247
- printf 'bad-denominator\tS\tyes\t0\tnone\tyes\t1\t-/x\t-\n' > "$qmeta/bad-denominator.quality.md"
3248
- printf 'bad-components\tS\tyes\t0\tnone\tyes\t1\t2/5/9\t-\n' > "$qmeta/bad-components.quality.md"
3249
- qmetaout=$(bash "$QC" "$tmp/qual-audit-metadata" 2>&1)
3250
- check "quality: nonnumeric audit denominators are surfaced" 1 \
3251
- "$(grep -c '^Invalid audit-hits values: 2 ' <<<"$qmetaout")"
3252
- check "quality: extra audit components are not accepted" 1 \
3253
- "$(grep -cE '^TOTAL +7 +0 +0\.0$' <<<"$qmetaout")"
3254
-
3255
- # Every per-route table walked the route map with a bare for-in, so the rows
3256
- # came out in awk hash order: C, L, M, S on this awk, and no guarantee at all
3257
- # on another, which is the opposite of the S, M, L, C the script declares in its
3258
- # own route list. A reader comparing routes by size needs a stable order.
3259
- qorder="$tmp/qual-route-order/devos/specs/archive"
3260
- mkdir -p "$qorder"
3261
- i=0
3262
- for route in L S M C; do
3263
- for n in 1 2; do
3264
- i=$((i + 1))
3265
- printf 'spec%s\t%s\tyes\t2\twarning\tyes\t1\t-/0\t-\n' "$i" "$route" \
3266
- > "$qorder/spec$i.quality.md"
3267
- done
3268
- done
3269
- qorderout=$(bash "$QC" "$tmp/qual-route-order" 2>&1)
3270
- # Control: each route's own numbers are right, so a report that printed no
3271
- # per-route table could not satisfy the ordering checks.
3272
- check "quality: every route reports its own specs and findings" 4 \
3273
- "$(grep -cE '^(S|M|L|C) +2 +4 +2\.0$' <<<"$qorderout")"
3274
- qtable=$(sed -n '/^## Findings per Route/,/^## Routing/p' <<<"$qorderout" |
3275
- grep -oE '^[SMLC]([[:space:]]|$)' | tr -d '[:space:]' | tr -d '\n')
3276
- check "quality: the per-route tables read in declared S/M/L/C order" 0 \
3277
- "$([[ "$qtable" == 'SMLC' ]] && echo 0 || echo 1)"
3278
- # All four tables walk the same map, so all four must be ordered.
3279
- qmatrix=$(sed -n '/^## Routing Confusion Matrix/,/^## Triage/p' <<<"$qorderout" |
3280
- grep -oE '^[SMLC]([[:space:]]|$)' | tr -d '[:space:]' | tr -d '\n')
3281
- check "quality: the confusion matrix reads in the same declared order" 0 \
3282
- "$([[ "$qmatrix" == 'SMLC' ]] && echo 0 || echo 1)"
3283
- qloops=$(sed -n '/^## Triage Loops per Route/,/^## Spec Audit/p' <<<"$qorderout" |
3284
- grep -oE '^[SMLC]([[:space:]]|$)' | tr -d '[:space:]' | tr -d '\n')
3285
- check "quality: the triage table reads in the same declared order" 0 \
3286
- "$([[ "$qloops" == 'SMLC' ]] && echo 0 || echo 1)"
3287
- # The unrecognized-route bucket belongs after the recognized ones, not wherever
3288
- # the hash put it.
3289
- qbucket="$tmp/qual-route-bucket/devos/specs/archive"
3290
- mkdir -p "$qbucket"
3291
- i=0
3292
- for route in C L S M banana banana; do
3293
- i=$((i + 1))
3294
- printf 'spec%s\t%s\tyes\t2\twarning\tyes\t1\t-/0\t-\n' "$i" "$route" \
3295
- > "$qbucket/spec$i.quality.md"
3296
- done
3297
- qbucketout=$(bash "$QC" "$tmp/qual-route-bucket" 2>&1)
3298
- qbucketfirst=$(sed -n '/^## Findings per Route/,/^## Routing/p' <<<"$qbucketout" |
3299
- grep -oE '^[SMLC?]' | tr -d '\n')
3300
- check "quality: an unrecognized route is bucketed after the real routes" 0 \
3301
- "$([[ "$qbucketfirst" == 'SMLC?' ]] && echo 0 || echo 1)"
3302
- check "quality: the ? bucket still counts its specs" 1 \
3303
- "$(grep -cE '^\? +2 +4 +2\.0$' <<<"$qbucketout")"
3304
- rm -rf "$tmp/qual-route-order" "$tmp/qual-route-bucket"
3305
-
3306
- qvalidate="$tmp/qual-validate/devos/specs/archive"
3307
- mkdir -p "$qvalidate"
3308
- for name in a b c d e; do
3309
- printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qvalidate/$name.quality.md"
3310
- done
3311
- printf 'bad-validate\tS\tmaybe\t2\tnone\tyes\t1\t-/0\t-\n' > "$qvalidate/bad-validate.quality.md"
3312
- qvalidateout=$(bash "$QC" "$tmp/qual-validate" 2>&1)
3313
- check "quality: invalid validate-pass values are excluded" 1 \
3314
- "$(grep -c '^Specs analyzed: 5$' <<<"$qvalidateout")"
3315
- check "quality: invalid validate-pass values are surfaced" 1 \
3316
- "$(grep -c '^Invalid validate-pass values: 1 ' <<<"$qvalidateout")"
3317
- check "quality: invalid validate-pass rows do not dilute totals" 1 \
3318
- "$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qvalidateout")"
3319
-
3320
- qtriage="$tmp/qual-triage/devos/specs/archive"
3321
- mkdir -p "$qtriage"
3322
- for name in a b c d e; do
3323
- printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qtriage/$name.quality.md"
3324
- done
3325
- printf 'bad-triage\tS\tyes\t2\tnone\tyes\tmany\t-/0\t-\n' > "$qtriage/bad-triage.quality.md"
3326
- qtriageout=$(bash "$QC" "$tmp/qual-triage" 2>&1)
3327
- check "quality: malformed triage-loop values stay out of the average" 1 \
3328
- "$(grep -c '^Invalid triage-loop values: 1 ' <<<"$qtriageout")"
3329
- check "quality: malformed triage-loop rows remain available for other metrics" 1 \
3330
- "$(grep -cE '^TOTAL +6 +12 +2\.0$' <<<"$qtriageout")"
3331
- printf 'unknown-triage\tS\tyes\t2\tnone\tyes\t?\t-/0\t-\n' > "$qtriage/unknown-triage.quality.md"
3332
- qtriageout=$(bash "$QC" "$tmp/qual-triage" 2>&1)
3333
- check "quality: unknown triage-loop remains valid but does not enter the average" 1 \
3334
- "$(grep -cE '^S +5 +1\.0$' <<<"$qtriageout")"
3335
-
3336
- qdod="$tmp/qual-dod/devos/specs/archive"
3337
- mkdir -p "$qdod"
3338
- for name in a b c d e; do
3339
- printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qdod/$name.quality.md"
3340
- done
3341
- printf 'bad-dod\tS\tyes\t2\tnone\tmaybe\t1\t-/0\t-\n' > "$qdod/bad-dod.quality.md"
3342
- qdodout=$(bash "$QC" "$tmp/qual-dod" 2>&1)
3343
- check "quality: invalid dod-pass values are excluded" 1 \
3344
- "$(grep -c '^Specs analyzed: 5$' <<<"$qdodout")"
3345
- check "quality: invalid dod-pass values are surfaced" 1 \
3346
- "$(grep -c '^Invalid dod-pass values: 1 ' <<<"$qdodout")"
3347
- check "quality: invalid dod-pass rows do not dilute totals" 1 \
3348
- "$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qdodout")"
3349
-
3350
- qseverity="$tmp/qual-severity/devos/specs/archive"
3351
- mkdir -p "$qseverity"
3352
- for name in a b c d e; do
3353
- printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qseverity/$name.quality.md"
3354
- done
3355
- printf 'bad-severity\tS\tyes\t2\tbanana\tyes\t1\t-/0\t-\n' > "$qseverity/bad-severity.quality.md"
3356
- qseverityout=$(bash "$QC" "$tmp/qual-severity" 2>&1)
3357
- check "quality: invalid severity values are excluded" 1 \
3358
- "$(grep -c '^Specs analyzed: 5$' <<<"$qseverityout")"
3359
- check "quality: invalid severity values are surfaced" 1 \
3360
- "$(grep -c '^Invalid severity values: 1 ' <<<"$qseverityout")"
3361
- check "quality: invalid severity rows do not dilute totals" 1 \
3362
- "$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qseverityout")"
3363
-
3364
- qunderstanding="$tmp/qual-understanding/devos/specs/archive"
3365
- mkdir -p "$qunderstanding"
3366
- for name in a b c d e; do
3367
- printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qunderstanding/$name.quality.md"
3368
- done
3369
- printf 'bad-understanding\tS\tyes\t2\tnone\tyes\t1\t-/0\tbanana\n' > "$qunderstanding/bad-understanding.quality.md"
3370
- qunderstandingout=$(bash "$QC" "$tmp/qual-understanding" 2>&1)
3371
- check "quality: invalid understanding values are excluded" 1 \
3372
- "$(grep -c '^Specs analyzed: 5$' <<<"$qunderstandingout")"
3373
- check "quality: invalid understanding values are surfaced" 1 \
3374
- "$(grep -c '^Invalid understanding values: 1 ' <<<"$qunderstandingout")"
3375
- check "quality: invalid understanding rows do not dilute totals" 1 \
3376
- "$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qunderstandingout")"
3377
-
3378
- qdiag="$tmp/qual-diagnostic/devos/specs/archive"
3379
- mkdir -p "$qdiag"
3380
- for name in a b c d e; do
3381
- printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qdiag/$name.quality.md"
3382
- done
3383
- printf 'bad-count\tS\tyes\tbogus\tnone\tyes\t1\t-/0\t-\n' > "$qdiag/bad-count.quality.md"
3384
- printf 'short row\n' > "$qdiag/short.quality.md"
3385
- qdiagout=$(bash "$QC" "$tmp/qual-diagnostic" 2>&1)
3386
- check "quality: short rows and invalid counts have separate diagnostics" 1 \
3387
- "$(grep -c '^Specs analyzed: 5$' <<<"$qdiagout")"
3388
- check "quality: only the short row is reported as skipped" 1 \
3389
- "$(grep -c '^Skipped quality rows: 1 (fewer than 6 tab-separated fields)$' <<<"$qdiagout")"
3390
- check "quality: invalid finding counts are not mislabeled as short rows" 1 \
3391
- "$(grep -c '^Invalid finding-count values: 1 ' <<<"$qdiagout")"
3392
-
3393
- qfindings="$tmp/qual-findings/devos/specs/archive"
3394
- mkdir -p "$qfindings"
3395
- for name in a b c d e; do
3396
- printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qfindings/$name.quality.md"
3397
- done
3398
- printf 'bad\tS\tyes\tbogus\tnone\tyes\t1\t-/0\t-\n' > "$qfindings/bad.quality.md"
3399
- qfindingsout=$(bash "$QC" "$tmp/qual-findings" 2>&1)
3400
- check "quality: non-integer finding counts are excluded" 1 \
3401
- "$(grep -c '^Specs analyzed: 5$' <<<"$qfindingsout")"
3402
- check "quality: non-integer finding counts are surfaced" 1 \
3403
- "$(grep -c '^Invalid finding-count values: 1 ' <<<"$qfindingsout")"
3404
- check "quality: invalid finding counts do not dilute averages" 1 \
3405
- "$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qfindingsout")"
3406
-
3407
- qaudit="$tmp/qual-audit/devos/specs/archive"
3408
- mkdir -p "$qaudit"
3409
- for name in a b c d; do
3410
- printf '%s\tL\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qaudit/$name.quality.md"
3411
- done
3412
- printf 'e\tL\tyes\t0\tnone\tyes\t1\t-/5\t-\n' > "$qaudit/e.quality.md"
3413
- qaout=$(bash "$QC" "$tmp/qual-audit" 2>&1)
3414
- check "quality: malformed audit metadata is not clean" 1 \
3415
- "$(grep -c 'no audit findings to confirm across 4 spec(s)' <<<"$qaout")"
3416
- check "quality: malformed audit metadata is surfaced" 1 \
3417
- "$(grep -c '^Invalid audit-hits values: 1 ' <<<"$qaout")"
3418
-
3419
- qratio="$tmp/qual-audit-ratio/devos/specs/archive"
3420
- mkdir -p "$qratio"
3421
- for name in a b c d; do
3422
- printf '%s\tL\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qratio/$name.quality.md"
3423
- done
3424
- printf 'e\tL\tyes\t0\tnone\tyes\t1\t3/2\t-\n' > "$qratio/e.quality.md"
3425
- qrout=$(bash "$QC" "$tmp/qual-audit-ratio" 2>&1)
3426
- check "quality: an impossible audit ratio is not included" 1 \
3427
- "$(grep -c 'no audit findings to confirm across 4 spec(s)' <<<"$qrout")"
3428
- check "quality: an impossible audit ratio is surfaced" 1 \
3429
- "$(grep -c '^Invalid audit-hits values: 1 ' <<<"$qrout")"
3430
-
3431
- qsentinel="$tmp/qual-audit-sentinel/devos/specs/archive"
3432
- mkdir -p "$qsentinel"
3433
- for name in a b c d; do
3434
- printf '%s\tL\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qsentinel/$name.quality.md"
3435
- done
3436
- printf 'e\tL\tyes\t0\tnone\tyes\t1\t0/0\t-\n' > "$qsentinel/e.quality.md"
3437
- qsout=$(bash "$QC" "$tmp/qual-audit-sentinel" 2>&1)
3438
- check "quality: numeric 0/0 is not counted as clean" 1 \
3439
- "$(grep -c 'no audit findings to confirm across 4 spec(s)' <<<"$qsout")"
3440
- check "quality: numeric 0/0 is surfaced as invalid" 1 \
3441
- "$(grep -c '^Invalid audit-hits values: 1 ' <<<"$qsout")"
3442
-
3443
- # --- artifact-budgets.sh: the three bands ----------------------------------
3444
- AB="$SKILLS/status/scripts/artifact-budgets.sh"
3445
- bdir="$tmp/budget"
3446
-
3447
- mkdir -p "$bdir"
3448
- check "budget: no onboarded artifacts is not a failure" 0 \
3449
- "$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
3450
- check "budget: no artifacts says so rather than printing an empty table" 1 \
3451
- "$(bash "$AB" "$bdir" 2>&1 | grep -c 'No budgeted artifacts found')"
3452
-
3453
- mkdir -p "$bdir/devos/product"
3454
- # Cap 60: 53 lines is OK (below the 54-line 90% mark), 54 is NEAR, 61 is OVER.
3455
- seq 1 53 >"$bdir/devos/product/domain-model.md"
3456
- check "budget: below 90% of cap is OK" 1 \
3457
- "$(bash "$AB" "$bdir" 2>&1 | grep -c '^| OK | devos/product/domain-model.md | 53 | 60 |')"
3458
- check "budget: an OK-only run exits 0" 0 \
3459
- "$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
3460
-
3461
- seq 1 54 >"$bdir/devos/product/domain-model.md"
3462
- check "budget: 90% of cap is NEAR" 1 \
3463
- "$(bash "$AB" "$bdir" 2>&1 | grep -c '^| NEAR | devos/product/domain-model.md | 54 | 60 |')"
3464
- check "budget: NEAR is reported but never fatal" 0 \
3465
- "$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
3466
-
3467
- seq 1 60 >"$bdir/devos/product/domain-model.md"
3468
- check "budget: exactly at cap is NEAR, not OVER" 1 \
3469
- "$(bash "$AB" "$bdir" 2>&1 | grep -c '^| NEAR | devos/product/domain-model.md | 60 | 60 |')"
3470
-
3471
- seq 1 61 >"$bdir/devos/product/domain-model.md"
3472
- check "budget: one line past cap is OVER" 1 \
3473
- "$(bash "$AB" "$bdir" 2>&1 | grep -c '^| OVER | devos/product/domain-model.md | 61 | 60 |')"
3474
- check "budget: an OVER artifact exits 1" 1 \
3475
- "$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
3476
-
3477
- { seq 1 60; printf '61'; } >"$bdir/devos/product/domain-model.md"
3478
- check "budget: an unterminated final line is counted" 1 \
3479
- "$(bash "$AB" "$bdir" 2>&1 | grep -c '^| OVER | devos/product/domain-model.md | 61 | 60 |')"
3480
- check "budget: an unterminated final OVER line exits 1" 1 \
3481
- "$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
3482
-
3483
- # Row order, not just row content. Lexical order over the band names is
3484
- # NEAR, OK, OVER, which puts the only actionable row last in a long table.
3485
- mkdir -p "$bdir/devos/product/behaviors"
3486
- seq 1 54 >"$bdir/devos/product/behaviors/near.md"
3487
- seq 1 10 >"$bdir/devos/product/behaviors/fine.md"
3488
- bandorder=$(bash "$AB" "$bdir" 2>&1 | awk -F'|' '/^\| (OK|NEAR|OVER) \|/ { gsub(/ /, "", $2); printf "%s ", $2 }')
3489
- check "budget: OVER leads, then NEAR, then OK" "OVER NEAR OK " "$bandorder"
3490
- rm -f "$bdir/devos/product/behaviors/near.md" "$bdir/devos/product/behaviors/fine.md"
3491
-
3492
- # Each artifact class carries its own cap: 121 lines is OVER for the map (120)
3493
- # but well inside a capsule's 300.
3494
- mkdir -p "$bdir/devos/specs/archive/demo"
3495
- seq 1 121 >"$bdir/devos/product/codebase-map.md"
3496
- seq 1 121 >"$bdir/devos/specs/archive/demo/spec.md"
3497
- bout=$(bash "$AB" "$bdir" 2>&1)
3498
- check "budget: the map's cap is 120, not the capsule's 300" 1 \
3499
- "$(grep -c '^| OVER | devos/product/codebase-map.md | 121 | 120 |' <<<"$bout")"
3500
- check "budget: the same line count is OK inside a capsule's larger cap" 1 \
3501
- "$(grep -c '^| OK | devos/specs/archive/demo/spec.md | 121 | 300 |' <<<"$bout")"
3502
- check "budget: OVER rows lead the table so the actionable ones are read first" "OVER" \
3503
- "$(grep '^| ' <<<"$bout" | sed -n '3p' | awk -F'|' '{gsub(/ /,"",$2); print $2}')"
3504
- check "budget: every OVER artifact is counted in the failure line" 1 \
3505
- "$(grep -c '^2 artifact(s) OVER budget' <<<"$bout")"
3506
-
3507
- # A behaviors file shares the domain model's 60-line page budget.
3508
- mkdir -p "$bdir/devos/product/behaviors"
3509
- seq 1 61 >"$bdir/devos/product/behaviors/thing.md"
3510
- check "budget: a behaviors capability page shares the 60-line cap" 1 \
3511
- "$(bash "$AB" "$bdir" 2>&1 | grep -c '^| OVER | devos/product/behaviors/thing.md | 61 | 60 |')"
3512
-
3513
- check "budget: a missing repo directory fails loudly" 1 \
3514
- "$(bash "$AB" "$tmp/no-such-dir" >/dev/null 2>&1; echo $?)"
3515
-
3516
- # --- convention-usage.sh: which conventions capsules actually exercised -----
3517
- CU="$SKILLS/status/scripts/convention-usage.sh"
3518
- cdir="$tmp/convuse"
3519
- cbdir="$tmp/convuse-bullets"
3520
-
3521
- mkdir -p "$cdir"
3522
- check "convuse: no conventions.md is not a failure" 0 \
3523
- "$(bash "$CU" "$cdir" >/dev/null 2>&1; echo $?)"
3524
- check "convuse: no conventions.md says so" 1 \
3525
- "$(bash "$CU" "$cdir" 2>&1 | grep -c 'nothing onboarded')"
3526
-
3527
- mkdir -p "$cdir/devos/standards" "$cdir/devos/specs/archive"
3528
- {
3529
- echo "# Project Conventions"
3530
- echo ""
3531
- echo "## Testing"
3532
- echo ""
3533
- echo "- Asserts live in \`plugins/devos/scripts/gates/test-gates.sh\`, no framework."
3534
- echo "- Fixtures under \`evals/fixtures/\` are copied to a temp dir first."
3535
- echo "- One logical change per commit."
3536
- echo "- Never invoke \`legacy/old-runner.sh\`; it was replaced."
3537
- echo "- Each spec writes a \`devos/specs/<name>/spec.md\` capsule."
3538
- } >"$cdir/devos/standards/conventions.md"
3539
-
3540
- # Below the floor the ratio is meaningless, so it must refuse rather than guess.
3541
- for n in a b c; do
3542
- mkdir -p "$cdir/devos/specs/archive/spec-$n"
3543
- echo "ran test-gates.sh over evals/fixtures/thing" >"$cdir/devos/specs/archive/spec-$n/spec.md"
3544
- done
3545
- check "convuse: below 5 capsules reports INSUFFICIENT DATA" 1 \
3546
- "$(bash "$CU" "$cdir" 2>&1 | grep -c 'INSUFFICIENT DATA: only 3 archived capsule')"
3547
- check "convuse: INSUFFICIENT DATA still exits 0 (a report, not a gate)" 0 \
3548
- "$(bash "$CU" "$cdir" >/dev/null 2>&1; echo $?)"
3549
-
3550
- for n in d e; do
3551
- mkdir -p "$cdir/devos/specs/archive/spec-$n"
3552
- echo "ran test-gates.sh over evals/fixtures/thing" >"$cdir/devos/specs/archive/spec-$n/spec.md"
3553
- done
3554
- cout=$(bash "$CU" "$cdir" 2>&1)
3555
-
3556
- check "convuse: at the floor it reports instead of refusing" 1 \
3557
- "$(grep -c '^Source: 5 archived capsule' <<<"$cout")"
3558
- check "convuse: an exact path match counts as CITED" 1 \
3559
- "$(grep -c '^| CITED | 2 |' <<<"$cout")"
3560
- check "convuse: a full-path convention is CITED when the capsule names just the file" 1 \
3561
- "$(grep -c '| test-gates.sh |' <<<"$cout")"
3562
- check "convuse: a basename-only match is flagged for a second look" 1 \
3563
- "$(grep -c '^## CITED by filename only' <<<"$cout")"
3564
- check "convuse: an unambiguous full-path match is not listed as needing one" 0 \
3565
- "$(sed -n '/^## CITED by filename only/,/^$/p' <<<"$cout" | grep -c 'evals/fixtures')"
3566
- check "convuse: an identifier no capsule mentions is UNCITED" 1 \
3567
- "$(grep -c 'Never invoke legacy/old-runner.sh' <<<"$cout")"
3568
- check "convuse: a prose-only entry is UNMEASURABLE, never UNCITED" 1 \
3569
- "$(grep -c '^| UNMEASURABLE | 1 |' <<<"$cout")"
3570
- check "convuse: a generic basename never produces a false CITED" 1 \
3571
- "$(grep -c 'devos/specs/<name>/spec.md' <<<"$cout")"
3572
- check "convuse: every entry lands in exactly one class" "5" \
3573
- "$(awk -F'|' '/^\| (CITED|UNCITED|UNMEASURABLE) \|/ {s+=$3} END{print s}' <<<"$cout")"
3574
- mkdir -p "$cdir/devos/specs/archive/spec-f"
3575
- printf '%s\n' "ran test-gates.sh over evals/fixtures/thing" > "$cdir/devos/specs/archive/spec-f/spec.md"
3576
- printf '%s' '- Never invoke `legacy/final-runner.sh`; it was replaced.' > "$cdir/devos/standards/conventions.md"
3577
- cfinal=$(bash "$CU" "$cdir" 2>&1)
3578
- check "convuse: a final unterminated bullet is counted" 1 \
3579
- "$(grep -c '^| UNCITED | 1 |$' <<<"$cfinal")"
3580
- check "convuse: uncited entries never change the exit code" 0 \
3581
- "$(bash "$CU" "$cdir" >/dev/null 2>&1; echo $?)"
3582
- check "convuse: a missing repo directory fails loudly" 1 \
3583
- "$(bash "$CU" "$tmp/no-such-dir" >/dev/null 2>&1; echo $?)"
3584
-
3585
- # Both markdown bullet markers make an entry. Only the dash form was
3586
- # recognized, so a convention written `* foo` was dropped from every class and
3587
- # from the total: a file with two entries reported one.
3588
- mkdir -p "$cbdir/devos/specs/archive" "$cbdir/devos/standards"
3589
- for n in one two three four five; do
3590
- mkdir -p "$cbdir/devos/specs/archive/spec-$n"
3591
- echo "uses src/index.js" >"$cbdir/devos/specs/archive/spec-$n/spec.md"
3592
- done
3593
- {
3594
- echo "# Project Conventions"
3595
- echo ""
3596
- echo "## Testing"
3597
- echo ""
3598
- echo "- \`src/index.js\`: keep the entry point thin."
3599
- echo "* \`lib/unused.ts\`: written with a star bullet."
3600
- } >"$cbdir/devos/standards/conventions.md"
3601
- cbout=$(bash "$CU" "$cbdir" 2>&1)
3602
- check "convuse: a star-bulleted entry is counted" 1 \
3603
- "$(grep -c '^| Total entries | 2 |' <<<"$cbout")"
3604
- check "convuse: an uncited star-bulleted entry is reported, not dropped" 1 \
3605
- "$(grep -c 'lib/unused.ts' <<<"$cbout")"
3606
- check "convuse: both bullet forms land in exactly one class" "2" \
3607
- "$(awk -F'|' '/^\| (CITED|UNCITED|UNMEASURABLE) \|/ {s+=$3} END{print s}' <<<"$cbout")"
3608
- printf 'uses lib/unused.ts\n' >>"$cbdir/devos/specs/archive/spec-one/spec.md"
3609
- cbcited=$(bash "$CU" "$cbdir" 2>&1)
3610
- check "convuse: a cited star-bulleted entry is CITED" 1 \
3611
- "$(grep -c '^| CITED | 2 |' <<<"$cbcited")"
3612
- check "convuse: a cited star-bulleted entry is not left in UNCITED" 1 \
3613
- "$(grep -c '^| UNCITED | 0 |' <<<"$cbcited")"
3614
- # A star-bulleted prose-only entry is still unmeasurable, not cited by accident.
3615
- {
3616
- echo "# Project Conventions"
3617
- echo ""
3618
- echo "- one logical change per commit."
3619
- echo "* \`legacy/old-runner.sh\` was replaced."
3620
- } >"$cbdir/devos/standards/conventions.md"
3621
- cbprose=$(bash "$CU" "$cbdir" 2>&1)
3622
- check "convuse: an uncited star-bulleted entry is UNCITED by identifier" 1 \
3623
- "$(grep -c '^| UNCITED | 1 |' <<<"$cbprose")"
3624
- check "convuse: a prose-only star-bulleted entry is UNMEASURABLE" 1 \
3625
- "$(grep -c '^| UNMEASURABLE | 1 |' <<<"$cbprose")"
3626
-
3627
- # A spec directory name is a free-form slug, so it can contain a space. The
3628
- # haystack was built with cat over an unquoted expansion, so such a path split
3629
- # into two arguments that both failed to open, stderr was discarded, and the
3630
- # capsule vanished from the search. Every entry only that capsule mentioned was
3631
- # then listed UNCITED, which is the list a reader is told to consider deleting.
3632
- csdir="$tmp/convuse-spaced"
3633
- mkdir -p "$csdir/devos/standards" "$csdir/devos/specs/archive/alpha feature"
3634
- printf 'uses lib/spaced.ts\n' > "$csdir/devos/specs/archive/alpha feature/spec.md"
3635
- for n in beta gamma delta epsilon; do
3636
- mkdir -p "$csdir/devos/specs/archive/spec-$n"
3637
- echo "ran test-gates.sh over evals/fixtures/thing" >"$csdir/devos/specs/archive/spec-$n/spec.md"
3638
- done
3639
- {
3640
- echo "# Project Conventions"
3641
- echo ""
3642
- echo "- \`lib/spaced.ts\`: keep the archive loader in sync."
3643
- echo "- \`lib/absent.ts\`: delete once it is dead."
3644
- } >"$csdir/devos/standards/conventions.md"
3645
- csout=$(bash "$CU" "$csdir" 2>&1)
3646
- # Control: the identifier no capsule mentions must still be UNCITED, so a
3647
- # script that classified nothing at all cannot satisfy the next check.
3648
- check "convuse: an identifier no capsule mentions is still UNCITED" 1 \
3649
- "$(grep -c 'lib/absent.ts' <<<"$csout")"
3650
- check "convuse: a capsule under a spaced directory is searched" 1 \
3651
- "$(grep -c '^| CITED | 1 |' <<<"$csout")"
3652
- check "convuse: a spaced-capsule entry is not put on the delete list" 0 \
3653
- "$(grep -c 'lib/spaced.ts' <<<"$csout")"
3654
- check "convuse: a spaced spec directory still counts toward the capsule floor" 1 \
3655
- "$(grep -c '^Source: 5 archived capsule' <<<"$csout")"
3656
-
3657
- # --- Sizing calibration ------------------------------------------------------
3658
- # The router's size letter lands in the started event's detail; the capsule's
3659
- # task groups are reality. M with many groups is under-sized, L with one is
3660
- # over-sized, and a started detail without an S/M/L (resume notes) is skipped.
3661
- SC="$SKILLS/status/scripts/sizing-calibration.sh"
3662
- sz="$tmp/sizing/devos/specs"
3663
- mkdir -p "$sz/a" "$sz/b" "$sz/archive/c" "$sz/archive/d" "$sz/archive/e"
3664
- printf '{"at":"t","event":"started","detail":"M"}\n' > "$sz/a/ops.jsonl"
3665
- for i in 1 2 3 4 5 6; do printf '### Task Group %d: g\n' "$i"; done > "$sz/a/spec.md"
3666
- printf '{"at":"t","event":"started","detail":"L"}\n' > "$sz/b/ops.jsonl"
3667
- printf '### Task Group 1: only\n' > "$sz/b/spec.md"
3668
- printf '{"at":"t","event":"started","detail":"resuming at Phase C"}\n' > "$sz/archive/c/ops.jsonl"
3669
- printf '### Task Group 1: g\n' > "$sz/archive/c/spec.md"
3670
- # A capital in a path is not a sizing decision. d names a file and nothing
3671
- # else, so the router never sized it and there is nothing to calibrate; e
3672
- # names the same file and then carries the real size call, which must still
3673
- # be read.
3674
- printf '{"at":"t","event":"started","detail":"resuming: rewrite src/L.ts parser"}\n' > "$sz/archive/d/ops.jsonl"
3675
- printf '### Task Group 1: g\n' > "$sz/archive/d/spec.md"
3676
- printf '{"at":"t","event":"started","detail":"resuming: rewrite src/L.ts parser, routed L"}\n' > "$sz/archive/e/ops.jsonl"
3677
- printf '### Task Group 1: g\n' > "$sz/archive/e/spec.md"
3678
- out=$(bash "$SC" "$tmp/sizing" 2>&1)
3679
- grep -q "UNDERSIZED a: sized M, capsule has 6 task groups" <<<"$out" \
3680
- && check "sizing: M with 6 groups reported under-sized" 0 $? \
3681
- || { echo "FAIL sizing: under-sized M not flagged"; echo "$out"; fail=1; }
3682
- grep -q "OVERSIZED b: sized L, capsule has 1 task group" <<<"$out" \
3683
- && check "sizing: L with 1 group reported over-sized" 0 $? \
3684
- || { echo "FAIL sizing: over-sized L not flagged"; fail=1; }
3685
- grep -q "UNDERSIZED c\|OVERSIZED c" <<<"$out" \
3686
- && { echo "FAIL sizing: resume detail without size letter was scored"; fail=1; } || true
3687
- grep -q "UNDERSIZED d\|OVERSIZED d" <<<"$out" \
3688
- && { echo "FAIL sizing: a detail naming a file was read as a size call"; echo "$out"; fail=1; } || true
3689
- grep -q "OVERSIZED e: sized L, capsule has 1 task group" <<<"$out" \
3690
- && check "sizing: a size call after a file name is still read" 0 $? \
3691
- || { echo "FAIL sizing: size call after a file name was lost"; echo "$out"; fail=1; }
3692
- grep -q "calibrated: 0 ok, 1 under-sized M, 2 over-sized L" <<<"$out" \
3693
- && check "sizing: summary counts mismatches and skips resume details" 0 $? \
3694
- || { echo "FAIL sizing: wrong summary counts"; echo "$out"; fail=1; }
3695
- out=$(bash "$SC" "$tmp/no-such-project" 2>&1); rc=$?
3696
- [[ $rc -eq 0 && -n "$out" ]] || { echo "FAIL sizing: missing specs dir must advise, got rc=$rc"; fail=1; }
3697
-
3698
- # --- Scenario <-> criteria consistency -------------------------------------
3699
- # Every scenario must have a criteria file and vice versa. A scenario without
3700
- # criteria can never be scored; criteria without a scenario is dead config.
3701
- missing=0
3702
- for s in "$EVALS"/scenarios/*.md; do
3703
- name=$(basename "$s" .md)
3704
- [[ -f "$EVALS/criteria/${name}.json" ]] || { echo "FAIL scenario $name: no criteria.json" >&2; missing=1; }
3705
- done
3706
- for c in "$EVALS"/criteria/*.json; do
3707
- name=$(basename "$c" .json)
3708
- [[ "$name" == "_selftest-needs" || "$name" == "_selftest-flip" || "$name" == "_selftest-hold" ]] && continue
3709
- [[ -f "$EVALS/scenarios/${name}.md" ]] || { echo "FAIL criteria $name: no scenario.md" >&2; missing=1; }
3710
- done
3711
- check "every scenario has criteria and every criteria has a scenario" 0 "$missing"
3712
-
3713
- # --- Concurrent template-cache invalidation ---------------------------------
3714
- # no-op-gate and oracle-gate start together under `just verify`. If both see
3715
- # a stale fingerprint before either takes the lock, the second process must
3716
- # re-check after acquisition. Otherwise it deletes templates the first process
3717
- # is already building. Hold the lock, advance the fingerprint, then call ensure.
3718
- (
3719
- export EVAL_DIR="$EVALS"
3720
- TEMPLATES="$tmp/template-cache"
3721
- source "$EVALS/build-templates.sh" || exit 1
3722
- mkdir -p "$TEMPLATES"
3723
- printf 'stale\n' >"$TEMPLATES/.fingerprint"
3724
- mkdir "$TEMPLATES.lock"
3725
- printf 'keep\n' >"$TEMPLATES/sentinel"
3726
- want=$(templates_fingerprint)
3727
- (
3728
- sleep 0.2
3729
- printf '%s\n' "$want" >"$TEMPLATES/.fingerprint"
3730
- rmdir "$TEMPLATES.lock"
3731
- ) &
3732
- ensure_templates_fresh
3733
- wait
3734
- test -f "$TEMPLATES/sentinel"
3735
- )
3736
- check "concurrent cache invalidation preserves templates built by the lock holder" 0 "$?"
3737
-
3738
- # --- Every scenario has a template build case --------------------------------
3739
- # A scenario with no build_template case in run.sh silently runs against an
3740
- # empty repo, producing garbage scores. EVAL_TEMPLATES_ONLY=1 reports EMPTY,
3741
- # but `just evals` doesn't set it. Catch the gap in `just verify` by dry-running
3742
- # the same template build and flagging any EMPTY result.
3743
- tmpl_out=$(EVAL_TEMPLATES_ONLY=1 bash "$EVALS/run.sh" 2>&1)
3744
- empty_count=$(grep -c 'EMPTY' <<<"$tmpl_out" || true)
3745
- check "no scenario has an empty template build" 0 "$empty_count"
3746
-
3747
- # --- Criteria must discriminate (at least one non-transcript criterion fails) ---
3748
- # A criterion that passes against the pre-fix template is dead weight: it adds
3749
- # to the total but can never catch a regression. For each scenario, at least
3750
- # one filesystem-checkable criterion MUST fail against the untouched template.
3751
- # Pure-analysis scenarios (all criteria need a transcript) are validated by the
3752
- # transcript-dependency check instead — they need at least one non-transcript
3753
- # pre-condition criterion to be scored at all.
3754
- disc_fail=0
3755
- for s in "$EVALS"/scenarios/*.md; do
3756
- name=$(basename "$s" .md)
3757
- [[ "$name" == "_selftest-needs" || "$name" == "_selftest-flip" || "$name" == "_selftest-hold" ]] && continue
3758
- crit="$EVALS/criteria/${name}.json"
3759
- [[ -f "$crit" ]] || continue
3760
- tmpl="$EVALS/templates/$name"
3761
- if [[ ! -d "$tmpl" ]]; then
3762
- ( cd "$EVALS" && EVAL_TEMPLATES_ONLY=1 bash run.sh >/dev/null 2>&1 )
3763
- fi
3764
- [[ -d "$tmpl" ]] || continue
3765
- # Count filesystem vs transcript criteria.
3766
- total=$(python3 -c "import json;print(len(json.load(open('$crit'))))")
3767
- total_fs=$(python3 -c "import json;print(sum(1 for x in json.load(open('$crit')) if not x.get('needs')))")
3768
- total_tx=$((total - total_fs))
3769
- # Pure-analysis (all need transcript): validated by transcript-dependency check.
3770
- [[ "$total_fs" -eq 0 ]] && continue
3771
- # If there are transcript criteria AND filesystem criteria are pre-conditions
3772
- # (pass against template), that's OK — the transcript criteria do the real testing.
3773
- # Only flag if there are NO transcript criteria AND all filesystem criteria pass.
3774
- if [[ "$total_tx" -gt 0 ]]; then
3775
- # Has transcript criteria: filesystem criteria are pre-conditions. Skip discrimination check.
3776
- continue
3777
- fi
3778
- # Run each filesystem criterion against the template and count failures.
3779
- # Each check is designed to run with cwd=PROJ, so cd there first.
3780
- # The Python script prints the count of failing criteria to stdout.
3781
- fails=$(python3 - "$crit" "$tmpl" "$EVAL_DIR" <<'PYEOF'
3782
- import json, os, subprocess, sys
3783
- crit, proj, eval_dir = sys.argv[1], sys.argv[2], sys.argv[3]
3784
- with open(crit) as f:
3785
- criteria = json.load(f)
3786
- fs = [c for c in criteria if not c.get('needs')]
3787
- env = {**os.environ, 'PROJ': proj, 'EVAL_DIR': eval_dir, 'TRANSCRIPT': proj + '/transcript.jsonl'}
3788
- fails = 0
3789
- for c in fs:
3790
- try:
3791
- cmd = 'cd "$PROJ" && ' + c['check'] if not c['check'].strip().startswith('cd ') and 'cd "$PROJ"' not in c['check'] else c['check']
3792
- r = subprocess.run(['bash', '-c', cmd], capture_output=True, text=True, timeout=10, env=env)
3793
- if r.returncode != 0:
3794
- fails += 1
3795
- except Exception:
3796
- fails += 1
3797
- print(fails)
3798
- PYEOF
3799
- )
3800
- # fails is the count of failing criteria (captured from stdout).
3801
- if [[ "$fails" -eq 0 ]]; then
3802
- echo "FAIL $name: all $total_fs filesystem criteria pass against pre-fix template (none discriminate)" >&2
3803
- disc_fail=1
3804
- fi
3805
- done
3806
- check "every scenario has at least one discriminating filesystem criterion" 0 "$disc_fail"
3807
-
3808
- # --- No scenario is entirely transcript-dependent --------------------------------
3809
- # If every criterion in a scenario needs a transcript, the scenario can never be
3810
- # scored without one — it's a manual-read item in disguise, not an eval. Flag it.
3811
- all_transcript=0
3812
- for c in "$EVALS"/criteria/*.json; do
3813
- name=$(basename "$c" .json)
3814
- [[ "$name" == "_selftest-needs" || "$name" == "_selftest-flip" || "$name" == "_selftest-hold" ]] && continue
3815
- total=$(python3 -c "import json;print(len(json.load(open('$c'))))")
3816
- needs_tx=$(python3 -c "import json;print(sum(1 for x in json.load(open('$c')) if x.get('needs')))")
3817
- if [[ "$total" -gt 0 && "$needs_tx" -eq "$total" ]]; then
3818
- echo "FAIL $name: all $total criteria need a transcript (scores 0 without one)" >&2
3819
- all_transcript=1
3820
- fi
3821
- done
3822
- check "no scenario is entirely transcript-dependent" 0 "$all_transcript"
3823
-
3824
- # --- gate-check.mjs ---------------------------------------------------------
3825
- GC="$(dirname "$DOD")/gate-check.mjs"
3826
- g="$tmp/gates.md"
3827
- cat > "$g" <<'EOF'
3828
- # Gates: demo
3829
-
3830
- - [ ] G1: passes via exit code
3831
- CHECK: echo ok
3832
- EVIDENCE: pending
3833
-
3834
- - [ ] G2: passes via EXPECT regex
3835
- CHECK: echo "count: 42"
3836
- EXPECT: /count: 42/
3837
- EVIDENCE: pending
3838
-
3839
- - [ ] G3: failing check stays unchecked
3840
- CHECK: false
3841
- EVIDENCE: pending
3842
-
3843
- - [x] G4: checked box with pending evidence counts UNMET
3844
- EVIDENCE: pending
3845
-
3846
- - [ ] G5: honestly abandoned
3847
- EVIDENCE: pending
3848
- ABANDON: G5 needs prod access this machine lacks
3849
- EOF
3850
- node "$GC" "$g" >/dev/null; check "gate-check: unmet gates exit 1" 1 $?
3851
- grep -q '^- \[x\] G1' "$g"; check "gate-check: passing gate flipped" 0 $?
3852
- grep -q '^- \[ \] G3' "$g"; check "gate-check: failing gate stays unchecked" 0 $?
3853
- grep -qE '^ EVIDENCE: ok' "$g"; check "gate-check: evidence recorded from output" 0 $?
3854
- grep -qE '^ EVIDENCE: count: 42' "$g"; check "gate-check: regex EXPECT recorded evidence" 0 $?
3855
- node "$GC" --status "$g" >/dev/null; check "gate-check: --status reports remaining unmet as exit 1" 1 $?
3856
-
3857
- # A gate file may omit the EVIDENCE placeholder. The pass path used to flip the
3858
- # box and report ALL MET while writing nothing, so the next --status run on the
3859
- # same bytes read the checked box as an unproven claim and exited 1: one run's
3860
- # result contradicted the next on an unchanged file.
3861
- gnoev="$tmp/gates-no-evidence.md"
3862
- printf -- '- [ ] G1: no evidence line\n CHECK: echo ok\n EXPECT: ok\n- [ ] G2: second gate\n CHECK: echo world\n EXPECT: world\n' > "$gnoev"
3863
- node "$GC" "$gnoev" >/dev/null; check "gate-check: gates without an EVIDENCE line can pass" 0 $?
3864
- grep -qE '^ EVIDENCE: ok$' "$gnoev"; check "gate-check: a missing EVIDENCE line is written, not skipped" 0 $?
3865
- grep -qE '^ EVIDENCE: world$' "$gnoev"; check "gate-check: each gate keeps its own evidence after a line insert" 0 $?
3866
- [[ "$(grep -cE '^ EVIDENCE: ' "$gnoev")" == 2 ]]; check "gate-check: exactly one evidence line per gate" 0 $?
3867
- node "$GC" --status "$gnoev" >/dev/null; check "gate-check: the recorded evidence satisfies a later --status run" 0 $?
3868
-
3869
- gok="$tmp/gates-ok.md"
3870
- printf -- '- [x] G1: done\n EVIDENCE: 3 rows rendered\n' > "$gok"
3871
- node "$GC" "$gok" >/dev/null; check "gate-check: all met exits 0" 0 $?
3872
-
3873
- gdupe="$tmp/gates-duplicate.md"
3874
- cat > "$gdupe" <<'EOF'
3875
- - [ ] G1: first outcome
3876
- CHECK: false
3877
- EVIDENCE: pending
3878
- - [ ] G1: second outcome
3879
- CHECK: false
3880
- EVIDENCE: pending
3881
- ABANDON: G1 only one outcome was abandoned
3882
- EOF
3883
- out=$(node "$GC" --status "$gdupe" 2>&1); rc=$?
3884
- check "gate-check: duplicate identifiers are a parse error" 2 $rc
3885
- printf '%s\n' "$out" | grep -Fq 'duplicate gate identifier G1'; check "gate-check: duplicate diagnostic names the identifier" 0 $?
3886
-
3887
- # The gates.md format is enforced by gate-check.mjs alone. check-dod-core.sh
3888
- # deliberately does not parse it, so there is no DoD
3889
- # blocking case to test here until a real spec uses gates and earns enforcement.
3890
-
3891
- # --- evals/run.sh isolation and timeout -------------------------------------
3892
- # A machine-local pi extension can fail before an eval starts. Both DevOS and
3893
- # baseline runs must disable those extensions. A child that ignores TERM must
3894
- # also not leave the eval harness waiting forever.
3895
- run_sh=$(cat "$EVALS/run.sh")
3896
- if grep -q 'local ext_args=(-ne)' <<<"$run_sh" && grep -q 'pi -p "$prompt" ${PROVIDER:+--provider "$PROVIDER"}' <<<"$run_sh"; then isolation_ok=0; else isolation_ok=1; fi
3897
- check "eval pi invocations disable machine-local extensions" 0 "$isolation_ok"
3898
-
3899
- timeout_fn=$(awk '/^run_with_timeout\(\)/,/^}/' "$EVALS/run.sh")
3900
- timeout_rc=0
3901
- timeout_start=$(date +%s)
3902
- (eval "$timeout_fn"; run_with_timeout 1 bash -c 'trap "" TERM; sleep 5') >/dev/null 2>&1 || timeout_rc=$?
3903
- timeout_elapsed=$(( $(date +%s) - timeout_start ))
3904
- if [[ "$timeout_rc" -ne 0 && "$timeout_elapsed" -le 3 ]]; then timeout_ok=0; else timeout_ok=1; fi
3905
- check "run timeout kills a TERM-ignoring child" 0 "$timeout_ok"
3906
- (eval "$timeout_fn"; run_with_timeout 5 bash -c 'exit 0') >/dev/null 2>&1
3907
- check "run timeout preserves a successful child" 0 "$?"
3908
-
3909
- # --- build/scripts/ledger.sh (H route) --------------------------------------
3910
- # The H route's two evidence-backed decisions live in the script, so they get a
3911
- # real test instead of a prose promise: `next` must flag a reissued task, and
3912
- # `notes` must replace and cap instead of growing. Every action runs against a
3913
- # throwaway run dir, the same way the route runs it.
3914
- LEDGER="$REPO_ROOT/plugins/devos/skills/build/scripts/ledger.sh"
3915
-
3916
- lr="$tmp/hard-run"
3917
- bash "$LEDGER" init --dir "$lr" --task "make the widget pass" --check "true" >/dev/null
3918
- check "ledger: init creates the run" 0 $?
3919
- grep -q 'make the widget pass' "$lr/task.md"; check "ledger: init writes task.md" 0 $?
3920
-
3921
- lpa="$tmp/ledger-path-artifact"
3922
- out=$(bash "$LEDGER" init --dir "$lpa" --task "make a path artifact" --artifact ../escaped.txt 2>&1); rc=$?
3923
- check "ledger: rejects a path artifact" 2 "$rc"
3924
- grep -Fq 'artifact must be a file name' <<<"$out"; check "ledger: names the artifact path contract" 0 $?
3925
- [[ ! -e "$tmp/escaped.txt" ]]; check "ledger: does not create an escaped artifact" 0 $?
3926
-
3927
- out=$(bash "$LEDGER" read --dir "$lr")
3928
- printf '%s' "$out" | grep -q 'round: 0/10'; check "ledger: read shows the round budget" 0 $?
3929
- printf '%s' "$out" | grep -q 'artifact: solution.py (empty)'; check "ledger: read reports an empty artifact" 0 $?
3930
-
3931
- # Plan is capped, not trusted whole.
3932
- out=$(head -c 5000 /dev/zero | tr '\0' 'p' | bash "$LEDGER" plan --dir "$lr")
3933
- [ "$(wc -c < "$lr/plan.md" | tr -d ' ')" -le 4000 ]; check "ledger: plan capped at 4000" 0 $?
3934
- # Capping without saying so reads as a plan that landed whole, and the worker
3935
- # is handed the cut half.
3936
- printf '%s' "$out" | grep -q 'TRUNCATED'; check "ledger: an over-cap plan says it was cut" 0 $?
3937
- printf 'short plan\n' | bash "$LEDGER" plan --dir "$lr" | grep -q 'TRUNCATED'
3938
- [[ $? -ne 0 ]]; check "ledger: a plan that fits is not reported as cut" 0 $?
3939
-
3940
- printf -- '- [todo] first\n- [todo] second\n' | bash "$LEDGER" tasks --dir "$lr" >/dev/null
3941
- [ "$(grep -c '^- \[' "$lr/tasks.md" || true)" = "2" ]; check "ledger: tasks keeps only list lines" 0 $?
3942
-
3943
- # The guard: the same task twice is the no-progress signal the paper relies on.
3944
- bash "$LEDGER" next --dir "$lr" --task "patch the parser" >/dev/null
3945
- out=$(bash "$LEDGER" next --dir "$lr" --task "Patch the parser")
3946
- printf '%s' "$out" | grep -q 'GUARD:'; check "ledger: reissued task raises the guard" 0 $?
3947
- printf '%s' "$out" | grep -q 'round 2/10'; check "ledger: next increments the round" 0 $?
3948
-
3949
- # The round budget is a guard, not display text. Exhaustion must stop before
3950
- # the task or round mutates, or the manager can run forever past its cap.
3951
- lbr="$tmp/hard-run-budget"
3952
- bash "$LEDGER" init --dir "$lbr" --task "initial task" --check "true" --max-iters 1 >/dev/null
3953
- bash "$LEDGER" next --dir "$lbr" --task "allowed task" >/dev/null
3954
- out=$(bash "$LEDGER" next --dir "$lbr" --task "over-budget task" 2>&1); rc=$?
3955
- check "ledger: exhausted round budget refuses another task" 1 "$rc"
3956
- printf '%s' "$out" | grep -q 'STOP: iteration budget exhausted (1/1)'; check "ledger: exhausted budget names the limit" 0 $?
3957
- [[ "$(awk -F= '$1=="round" {print $2}' "$lbr/meta")" == "1" ]]; check "ledger: refused task does not increment the round" 0 $?
3958
- [[ "$(awk -F= '$1=="last_task" {sub(/^[^=]*=/, ""); print}' "$lbr/meta")" == "allowed task" ]]; check "ledger: refused task does not replace the last task" 0 $?
3959
-
3960
- # Budget validation must happen before init creates any run state. A valid
3961
- # positive integer remains a usable control for the same command.
3962
- for invalid_budget in nope 0 -1; do
3963
- invalid_dir="$tmp/invalid-budget-$invalid_budget"
3964
- out=$(bash "$LEDGER" init --dir "$invalid_dir" --task "invalid budget" --check true --max-iters "$invalid_budget" 2>&1); rc=$?
3965
- check "ledger: rejects invalid budget $invalid_budget" 2 "$rc"
3966
- grep -Fq 'max-iters must be a positive integer' <<<"$out"; check "ledger: names invalid budget $invalid_budget" 0 $?
3967
- [[ ! -e "$invalid_dir" ]]; check "ledger: does not create state for invalid budget $invalid_budget" 0 $?
3968
- done
3969
- valid_budget_dir="$tmp/valid-budget"
3970
- bash "$LEDGER" init --dir "$valid_budget_dir" --task "valid budget" --check true --max-iters 2 >/dev/null
3971
- check "ledger: accepts a positive budget" 0 $?
3972
- grep -q '^max_iters=2$' "$valid_budget_dir/meta"; check "ledger: stores the validated positive budget" 0 $?
3973
-
3974
- # Notes replace by default and cap; --append adds without unbounded growth.
3975
- printf 'first finding\n' | bash "$LEDGER" notes --dir "$lr" >/dev/null
3976
- printf 'second finding\n' | bash "$LEDGER" notes --dir "$lr" >/dev/null
3977
- [ "$(cat "$lr/notes.md")" = "second finding" ]; check "ledger: notes replace, old content gone" 0 $?
3978
- head -c 9000 /dev/zero | tr '\0' 'n' | bash "$LEDGER" notes --dir "$lr" --append >/dev/null
3979
- [ "$(wc -c < "$lr/notes.md" | tr -d ' ')" -le 8000 ]; check "ledger: notes capped at 8000" 0 $?
3980
- out=$(head -c 9000 /dev/zero | tr '\0' 'n' | bash "$LEDGER" notes --dir "$lr")
3981
- printf '%s' "$out" | grep -q 'TRUNCATED'; check "ledger: over-cap notes say they were cut" 0 $?
3982
-
3983
- # Artifact write and read.
3984
- printf 'print("ok")\n' | bash "$LEDGER" artifact --dir "$lr" >/dev/null
3985
- bash "$LEDGER" artifact --dir "$lr" --read | grep -q 'print("ok")'; check "ledger: artifact round-trips" 0 $?
3986
-
3987
- # Check is ground truth: exit code and recorded verdict.
3988
- bash "$LEDGER" check --dir "$lr" --cmd "grep -q ok solution.py" >/dev/null; check "ledger: check passes on the artifact" 0 $?
3989
- grep -q '^verdict=pass' "$lr/meta"; check "ledger: a pass is recorded" 0 $?
3990
- bash "$LEDGER" check --dir "$lr" --cmd "grep -q missing solution.py" >/dev/null 2>&1; check "ledger: a failing check exits non-zero" 1 $?
3991
- grep -q '^verdict=fail' "$lr/meta"; check "ledger: a fail overrides the recorded pass" 0 $?
3992
-
3993
- # Refuses to clobber an initialized run, and errors on an uninitialized dir.
3994
- bash "$LEDGER" init --dir "$lr" --task x >/dev/null 2>&1; check "ledger: init will not clobber a run" 2 $?
3995
- bash "$LEDGER" read --dir "$tmp/no-such-run" >/dev/null 2>&1; check "ledger: read on an uninitialized dir errors" 2 $?
3996
-
3997
- # --- design-taste/scripts/slop-scan.py: what a skipped file is ----------------
3998
- # The skipped tally says binary/generated. A file was skipped when grep found no
3999
- # line of text in it, which is also true of an empty or blank file, so an empty
4000
- # .css was described as binary and a project whose only frontend file is empty
4001
- # was told it had no frontend files. The rules cannot fire on a file with no
4002
- # content, but the file is still an authored frontend file and belongs in the
4003
- # count.
4004
- SLOP="$REPO_ROOT/plugins/devos/skills/design-taste/scripts/slop-scan.py"
4005
- slopd="$tmp/slop-empty"
4006
- mkdir -p "$slopd"
4007
- : > "$slopd/app.css"
4008
- out=$(python3 "$SLOP" "$slopd")
4009
- ! printf '%s' "$out" | grep -q 'no frontend files'; check "slop: an empty-only tree is not called frontend-free" 0 $?
4010
- printf '%s' "$out" | grep -q 'across 1 files'; check "slop: an empty frontend file is still counted" 0 $?
4011
-
4012
- printf 'body { color: #16110d; }\n' > "$slopd/real.css"
4013
- printf 'GITCRYPT\000\314"\332\376' > "$slopd/locked.css"
4014
- : > "$slopd/blank.css"
4015
- out=$(python3 "$SLOP" "$slopd")
4016
- printf '%s' "$out" | grep -q 'across 3 files (1 binary/generated skipped)'; check "slop: only the binary file is skipped" 0 $?
4017
-
4018
- # A blank file, not just a zero-byte one, is the same case.
4019
- printf '\n\n' > "$slopd/blank.css"
4020
- out=$(python3 "$SLOP" "$slopd")
4021
- printf '%s' "$out" | grep -q 'across 3 files (1 binary/generated skipped)'; check "slop: a whitespace-only file is not skipped" 0 $?
4022
-
4023
- # The skip path itself still works, so keeping every file cannot pass the
4024
- # checks above.
4025
- slopb="$tmp/slop-binary"
4026
- mkdir -p "$slopb"
4027
- printf 'GITCRYPT\000\314"\332\376' > "$slopb/locked.css"
4028
- out=$(python3 "$SLOP" "$slopb")
4029
- printf '%s' "$out" | grep -q 'no frontend files'; check "slop: a binary-only tree has nothing to scan" 0 $?
4030
- python3 "$SLOP" --selftest >/dev/null; check "slop: selftest passes" 0 $?
4031
-
4032
- # --- init/scripts/reverse-engineer.sh: a non-repo is not an empty history ---
4033
- # The git log ran with stderr sent to /dev/null, so a directory that is not a
4034
- # repository produced no output, fell into the same empty branch as a repo
4035
- # whose last 6 months hold nothing, and printed "no clusters found in the last
4036
- # 6 months". The reference tells the operator a greenfield repo is the case to
4037
- # skip, so a directory with no history to read was green-lit on a claim about
4038
- # history that was never read.
4039
- REVENG="$REPO_ROOT/plugins/devos/skills/init/scripts/reverse-engineer.sh"
4040
- reveng_notgit="$tmp/reveng-not-git"
4041
- mkdir -p "$reveng_notgit"
4042
- out=$(cd "$reveng_notgit" && bash "$REVENG" 2>&1); rc=$?
4043
- [[ $rc -ne 0 ]] || { echo "FAIL reveng: a non-repo must not exit 0"; fail=1; }
4044
- [[ "$out" == *"not a git repository"* ]] || { echo "FAIL reveng: a non-repo must name the cause: $out"; fail=1; }
4045
- [[ "$out" != *"no clusters found"* ]] || { echo "FAIL reveng: a non-repo must not claim an empty history: $out"; fail=1; }
4046
-
4047
- # Control: a real repository with no commits is the genuine zero-cluster case
4048
- # and keeps the documented line and exit 0. Without it, "always exit non-zero"
4049
- # would pass the checks above.
4050
- reveng_empty="$tmp/reveng-empty"
4051
- mkdir -p "$reveng_empty"
4052
- git -C "$reveng_empty" init -q -b main
4053
- out=$(cd "$reveng_empty" && bash "$REVENG" 2>&1); rc=$?
4054
- check "reveng: an empty repository still reports no clusters" 0 "$rc"
4055
- [[ "$out" == *"no clusters found in the last 6 months"* ]] || { echo "FAIL reveng: an empty repo lost its line: $out"; fail=1; }
4056
-
4057
- # A missing directory is still its own error, not the repository one.
4058
- out=$(cd "$reveng_empty" && bash "$REVENG" "$tmp/reveng-absent" 2>&1); rc=$?
4059
- [[ $rc -ne 0 ]] || { echo "FAIL reveng: a missing directory must exit non-zero"; fail=1; }
4060
- [[ "$out" == *"no such directory"* ]] || { echo "FAIL reveng: a missing directory must name itself: $out"; fail=1; }
4061
-
4062
- # Control: a real repo with one cluster commit still writes its stub, so the
4063
- # repository check cannot pass by refusing all work.
4064
- reveng_real="$tmp/reveng-real"
4065
- mkdir -p "$reveng_real/plugins/demo/skills/thing"
4066
- git -C "$reveng_real" init -q -b main
4067
- git -C "$reveng_real" config user.email t@t && git -C "$reveng_real" config user.name t
4068
- echo x > "$reveng_real/plugins/demo/skills/thing/run.sh"
4069
- git -C "$reveng_real" add -A && git -C "$reveng_real" commit -qm 'feat: thing'
4070
- out=$(cd "$reveng_real" && bash "$REVENG" 2>&1); rc=$?
4071
- check "reveng: a repository with history writes stubs" 0 "$rc"
4072
- ls "$reveng_real/devos/specs/archive/"*-plugins-demo-skills-thing.spec.md >/dev/null 2>&1; check "reveng: the cluster stub lands on disk" 0 $?
4073
-
4074
- # --------------------------------------------------------------------------
4075
- # The smallest spec folder an M change is told to write, end to end.
4076
- #
4077
- # The router sends an M change through the same spec folder and the same plan
4078
- # capsule as an L feature, so what a small change costs is set by what the
4079
- # templates demand. write-spec.md names a one-page shape for M and a 300-line
4080
- # ceiling for the capsule, and the claim that the short one works is only true
4081
- # if every gate accepts it. It does not stop at validate-spec: the capsule is
4082
- # archived and a verdict claimed, because a spec that validates but cannot
4083
- # close the pipeline is not a shape the framework supports.
4084
- mspec=$(mktemp -d)
4085
- mspec_name=2026-01-01-invoice-pdf
4086
- mkdir -p "$mspec/devos/specs/$mspec_name/verifications"
4087
- cat > "$mspec/devos/specs/$mspec_name/spec-functional.md" <<'EOF'
4088
- # Export invoices
4089
-
4090
- ## 1. Goal
4091
-
4092
- A user can download a PDF of their invoices without asking support.
4093
-
4094
- ## 3. Functionality
4095
-
4096
- ### 3.1 Export
4097
- **Scenarios:**
4098
- - Scenario SC-1: plain export / WHEN an account has 3 invoices / THEN a PDF with all 3 is downloaded
4099
- - Scenario SC-2: empty account / WHEN an account has no invoices / THEN the export is refused with one line naming why
4100
-
4101
- ## 5. Out of Scope
4102
-
4103
- - Scheduled or recurring exports
4104
- - Choosing a paper size or locale
4105
-
4106
- ## Appendix
4107
-
4108
- **Explicit contract anchors:** `GET /invoices/:id/pdf`
4109
- EOF
4110
- cat > "$mspec/devos/specs/$mspec_name/spec.md" <<'EOF'
4111
- ---
4112
- devos-contract: 1
4113
- ---
4114
- # Export invoices
4115
-
4116
- ## Goal
4117
-
4118
- A user downloads a PDF of their invoices.
4119
-
4120
- ## Local Craft
4121
-
4122
- None
4123
-
4124
- ## Design Convergence
4125
- - **Direct MVP**: the existing invoice list route renders a print view
4126
- - **Failed Items**: None
4127
- - **Adopted Additions**: None
4128
- - **Rejected Additions**: None
4129
-
4130
- ## Boundaries
4131
- - **Always** (in scope): the export route and its test
4132
- - **Ask First** (none)
4133
- - **Never** (out of scope): scheduled exports
4134
-
4135
- ## Code Map
4136
-
4137
- - `src/invoices/pdf.ts`: the renderer
4138
-
4139
- ## Implementation Plan
4140
-
4141
- ### Overview
4142
- Total: 1 task group.
4143
-
4144
- ### Task Group 1: Export route
4145
- **Files:** `src/invoices/pdf.ts`
4146
- - Render and test
4147
- **Scenarios covered:** SC-1, SC-2
4148
- **Acceptance Criteria:** None
4149
- EOF
4150
- printf 'Completeness: CLEAN\nQuality: PASS\nNo pre-existing failures.\n\nSC-1 pass: export test, 3 invoices.\nSC-2 pass: export test, empty account refused.\n' \
4151
- > "$mspec/devos/specs/$mspec_name/verifications/verification.md"
4152
- mkdir -p "$mspec/src/invoices"
4153
- printf 'export const render = (invoices) => invoices.join(",");\n' \
4154
- > "$mspec/src/invoices/pdf.ts"
4155
- out=$(bash "$VSPEC" "$mspec/devos/specs/$mspec_name" 2>&1); rc=$?
4156
- check "m-spec: the one-page functional spec validates" 0 "$rc"
4157
- # The negative control, because a guard that has only ever passed is not a
4158
- # guard. Deleting the one delivered source file has to fail the DoD gate, or
4159
- # this block is asserting nothing.
4160
- git -C "$mspec" init -q -b "$mspec_name"
4161
- mv "$mspec/devos/specs/$mspec_name" "$mspec/devos/specs/archive/$mspec_name" 2>/dev/null \
4162
- || { mkdir -p "$mspec/devos/specs/archive" && mv "$mspec/devos/specs/$mspec_name" "$mspec/devos/specs/archive/$mspec_name"; }
4163
- printf 'Implemented the export route.\nVerdict: DONE: devos/specs/archive/%s\n' "$mspec_name" \
4164
- > "$mspec/transcript.txt"
4165
- out=$(cd "$mspec" && bash "$DOD" transcript.txt 2>&1); rc=$?
4166
- check "m-spec: the one-page folder closes the pipeline" 0 "$rc"
4167
- rm "$mspec/src/invoices/pdf.ts"
4168
- out=$(cd "$mspec" && bash "$DOD" transcript.txt 2>&1); rc=$?
4169
- check "m-spec: the same folder fails once the delivered file is gone" 2 "$rc"
4170
-
4171
- # The quality row the close path writes. verify-close.md 4.3 specifies six
4172
- # columns, after dropping the three that no row in this repo has ever filled.
4173
- # The contract that matters is not the column count but that the parser counts
4174
- # the row: an instruction and a status script that disagree is how a metric
4175
- # silently starts reporting on nothing. A legacy nine-column row has to keep
4176
- # parsing too, because the archive is full of them.
4177
- qrow=$(mktemp -d)
4178
- mkdir -p "$qrow/devos/specs/archive"
4179
- # Five rows minimum: quality-correlation.sh refuses to report below that, and a
4180
- # refusal would mask the thing this check is about.
4181
- for n in a b c d; do
4182
- printf '%s\tL\tyes\t0\tnone\tyes\n' "$n" > "$qrow/devos/specs/archive/$n.quality.md"
4183
- done
4184
- printf 'legacy\tL\tyes\t2\twarning\tyes\t3\t1/4\tdebt\n' > "$qrow/devos/specs/archive/legacy.quality.md"
4185
- out=$(bash "$QC" "$qrow" 2>&1); rc=$?
4186
- check "quality-row: a six-column corpus does not error" 0 "$rc"
4187
- # The real contract: both row shapes are counted, so the metric does not
4188
- # silently start reporting on nothing once old rows age out.
4189
- if grep -q 'Quality files: 5' <<<"$out" && grep -q 'Specs analyzed: 5' <<<"$out"; then
4190
- pass=$((pass + 1))
4191
- else
4192
- echo "FAIL quality-row: six-column and legacy rows not both counted" >&2
4193
- printf '%s\n' "$out" >&2
4194
- fail=$((fail + 1))
4195
- fi
4196
-
4197
- # --------------------------------------------------------------------------
4198
- # knowledge.py: the absorbed-source manifest. Its contract is that it needs no
4199
- # network and no model, so drift is a diff between two files on disk, and that
4200
- # a re-read never loses the date a collection was last distilled.
4201
- KNOW="$REPO_ROOT/plugins/devos/skills/absorb/scripts/knowledge.py"
4202
- kdir="$tmp/knowledge/devos/product/knowledge"
4203
- know() { python3 -I "$KNOW" --dir "$kdir" "$@" 2>&1; }
4204
- cat >"$tmp/sitemap.xml" <<'XML'
4205
- <urlset><url><loc>https://help.example.com/fr/articles/111-statuses</loc><lastmod>2026-01-02T00:00:00Z</lastmod></url>
4206
- <url><loc>https://help.example.com/fr/articles/222-quotes</loc><lastmod>2026-01-03T00:00:00Z</lastmod></url></urlset>
4207
- XML
4208
- cat >"$tmp/collection.md" <<'MD'
4209
- - [Invoice statuses](https://help.example.com/fr/articles/111-statuses)
4210
- - [Quotes](https://help.example.com/fr/articles/222-quotes)
4211
- MD
4212
- know init >/dev/null
4213
- grep -q 'cache/' "$kdir/.gitignore" 2>/dev/null
4214
- check "knowledge: init ignores the fetch cache" 0 $?
4215
- out=$(know import --collection sales --listing "$tmp/collection.md" --lastmod "$tmp/sitemap.xml")
4216
- check "knowledge: import counts the rows" "import sales: 2 new, 0 updated, 2 rows total" "$out"
4217
- grep -q $'111\thttps://help.example.com/fr/articles/111-statuses\tInvoice statuses\tsales\tin\t2026-01-02' "$kdir/sources.tsv"
4218
- check "knowledge: a row carries url, title, collection, scope and source date" 0 $?
4219
- out=$(know check --lastmod "$tmp/sitemap.xml" | tail -1)
4220
- check "knowledge: unread rows are reported before any read date" "check: 0 changed, 0 new, 0 gone, 2 unread" "$out"
4221
- know read --collection sales --date 2026-02-01 >/dev/null
4222
- out=$(know check --lastmod "$tmp/sitemap.xml" | tail -1)
4223
- check "knowledge: no drift once read after the source date" "check: no drift, 2 in-scope articles current" "$out"
4224
- cat >"$tmp/sitemap.xml" <<'XML'
4225
- <urlset><url><loc>https://help.example.com/fr/articles/111-statuses</loc><lastmod>2026-03-09T00:00:00Z</lastmod></url>
4226
- <url><loc>https://help.example.com/fr/articles/333-dunning</loc><lastmod>2026-03-01T00:00:00Z</lastmod></url></urlset>
4227
- XML
4228
- out=$(know check --lastmod "$tmp/sitemap.xml")
4229
- grep -q '^changed sales/111 read 2026-02-01, source 2026-03-09' <<<"$out"; check "knowledge: a moved source is changed" 0 $?
4230
- grep -q '^new 333' <<<"$out"; check "knowledge: an unlisted article is new" 0 $?
4231
- grep -q '^gone sales/222' <<<"$out"; check "knowledge: a withdrawn article is gone" 0 $?
4232
- know scope --collection purchases --reason "owned by the purchases service" >/dev/null
4233
- grep -q $'collection:purchases\t\tcollection purchases\tpurchases\tout\t\t\t\towned by the purchases service' "$kdir/sources.tsv"
4234
- check "knowledge: an out-of-scope collection keeps its reason" 0 $?
4235
- out=$(know check --lastmod "$tmp/sitemap.xml" | tail -1)
4236
- grep -qv 'purchases' <<<"$out"; check "knowledge: out-of-scope rows never drift" 0 $?
4237
- before=$(cat "$kdir/sources.tsv")
4238
- know check --lastmod "$tmp/sitemap.xml" >/dev/null
4239
- check "knowledge: check writes nothing" "$before" "$(cat "$kdir/sources.tsv")"
4240
- know import --collection sales --listing "$tmp/collection.md" --lastmod "$tmp/sitemap.xml" >/dev/null
4241
- grep -q $'111\t.*\t2026-02-01' "$kdir/sources.tsv"; check "knowledge: re-import keeps the read date" 0 $?
4242
- # A sitemap fetched through a markdown converter keeps the dates and loses the tags.
4243
- cat >"$tmp/sitemap.md" <<'MD'
4244
- [https://help.example.com/fr/articles/111-statuses](https://help.example.com/fr/articles/111-statuses)
4245
- 2026-04-04T00:00:00Z
4246
- MD
4247
- out=$(know check --lastmod "$tmp/sitemap.md")
4248
- grep -q '^changed sales/111 read 2026-02-01, source 2026-04-04' <<<"$out"
4249
- check "knowledge: a markdown-rendered sitemap still carries dates" 0 $?
4250
-
4251
- # --------------------------------------------------------------------------
4252
- # frame survey: where the project already touches a vocabulary
4253
- SV="$SKILLS/frame/scripts/survey.sh"
4254
- sv="$tmp/survey"
4255
- mkdir -p "$sv/src/api/handlers" "$sv/src/jobs"
4256
- printf 'void a payment\n' >"$sv/src/api/handlers/void.js"
4257
- printf 'void a payment in a job\n' >"$sv/src/jobs/void.job.js"
4258
- out=$(cd "$sv" && bash "$SV" void)
4259
- grep -q 'code: 2 files' <<<"$out"
4260
- check "survey: counts the files that touch a term" 0 $?
4261
- grep -q 'src/api/handlers' <<<"$out"
4262
- check "survey: groups matches by area" 0 $?
4263
- out=$(cd "$sv" && bash "$SV" nothing-here)
4264
- grep -q 'code: none under src' <<<"$out"
4265
- check "survey: a term with no code says so" 0 $?
4266
- out=$(cd "$sv" && bash "$SV")
4267
- check "survey: no term is a usage error" 2 $?
4268
-
4269
- CR="$SKILLS/absorb/scripts/check-refs.sh"
4270
- cr="$tmp/checkrefs"
4271
- mkdir -p "$cr/docs" "$cr/src/app" "$cr/src/db"
4272
- printf 'model X {\n id String\n}\n' >/dev/null
4273
- printf 'enum InvoiceStatus {\n PAID\n}\n' >"$cr/src/db/schema.prisma"
4274
- printf 'export const kinds = [\x27DELETE\x27] as const\n' >"$cr/src/app/kinds.ts"
4275
- printf 'page names `src/app/kinds.ts` and `InvoiceStatus.PAID` and `kinds.DELETE`\n' >"$cr/docs/ok.md"
4276
- out=$(bash "$CR" --docs "$cr/docs" --src "$cr/src" 2>&1)
4277
- check "check-refs: a page that resolves passes" 0 $?
4278
- grep -q 'missing refs: 0' <<<"$out"
4279
- check "check-refs: reports zero missing" 0 $?
4280
- printf 'page names `src/app/gone.ts` and `InvoiceStatus.TERMINATED`\n' >"$cr/docs/bad.md"
4281
- out=$(bash "$CR" --docs "$cr/docs" --src "$cr/src" 2>&1) && rc=0 || rc=$?
4282
- check "check-refs: a missing path fails" 1 $rc
4283
- grep -q 'MISSING src/app/gone.ts' <<<"$out"
4284
- check "check-refs: names the missing path" 0 $?
4285
- grep -q 'InvoiceStatus.TERMINATED' <<<"$out"
4286
- check "check-refs: names the unknown member" 0 $?
4287
- rm "$cr/docs/bad.md"
4288
- out=$(bash "$CR" --docs "$cr/docs" 2>&1) && rc=0 || rc=$?
4289
- check "check-refs: no source root is a usage error" 2 $rc
4290
60
 
4291
61
  # --------------------------------------------------------------------------
4292
62
  # The suite runs entirely inside $tmp. If that directory disappears mid-run,