@tduyng/pi-devos 0.0.0-stage → 0.13.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1398 -0
- package/LICENSE +201 -0
- package/README.md +213 -2
- package/package.json +45 -4
- package/plugins/devos/agents/reader.md +27 -0
- package/plugins/devos/agents/verifier.md +181 -0
- package/plugins/devos/pi/extension/devos.ts +615 -0
- package/plugins/devos/scripts/check-evidence-router.sh +134 -0
- package/plugins/devos/scripts/evidence-router.py +252 -0
- package/plugins/devos/scripts/gates/check-dod-core.sh +283 -0
- package/plugins/devos/scripts/gates/check-dod.sh +38 -0
- package/plugins/devos/scripts/gates/check-handoff.sh +215 -0
- package/plugins/devos/scripts/gates/check-implementation-boundary.sh +99 -0
- package/plugins/devos/scripts/gates/check-port-source.sh +134 -0
- package/plugins/devos/scripts/gates/check-report-locations.sh +148 -0
- package/plugins/devos/scripts/gates/check-scenario-coverage.sh +102 -0
- package/plugins/devos/scripts/gates/gate-check.mjs +204 -0
- package/plugins/devos/scripts/gates/phase-gate.sh +208 -0
- package/plugins/devos/scripts/gates/plan-template-check.py +147 -0
- package/plugins/devos/scripts/gates/run-gate.sh +148 -0
- package/plugins/devos/scripts/gates/spec-template-check.py +165 -0
- package/plugins/devos/scripts/gates/test-gates.sh +4306 -0
- package/plugins/devos/scripts/gates/test-integrity-rubric.sh +152 -0
- package/plugins/devos/scripts/gates/test-phase-gate.sh +116 -0
- package/plugins/devos/skills/absorb/SKILL.md +75 -0
- package/plugins/devos/skills/absorb/references/distill.md +67 -0
- package/plugins/devos/skills/absorb/scripts/check-refs.sh +108 -0
- package/plugins/devos/skills/absorb/scripts/knowledge.py +258 -0
- package/plugins/devos/skills/build/SKILL.md +71 -0
- package/plugins/devos/skills/build/references/advanced-routing.md +19 -0
- package/plugins/devos/skills/build/references/defect.md +17 -0
- package/plugins/devos/skills/build/references/exposure.md +28 -0
- package/plugins/devos/skills/build/references/hard.md +39 -0
- package/plugins/devos/skills/build/references/refactor.md +66 -0
- package/plugins/devos/skills/build/references/routes.md +56 -0
- package/plugins/devos/skills/build/references/scale.md +84 -0
- package/plugins/devos/skills/build/references/spike.md +19 -0
- package/plugins/devos/skills/build/scripts/construct-census.py +204 -0
- package/plugins/devos/skills/build/scripts/dep-order.py +297 -0
- package/plugins/devos/skills/build/scripts/ledger.sh +228 -0
- package/plugins/devos/skills/build/scripts/route.sh +5 -0
- package/plugins/devos/skills/build/scripts/secret-scan.sh +121 -0
- package/plugins/devos/skills/build/scripts/toolchain-preflight.py +302 -0
- package/plugins/devos/skills/design-taste/SKILL.md +55 -0
- package/plugins/devos/skills/design-taste/references/tells.md +85 -0
- package/plugins/devos/skills/design-taste/scripts/slop-scan.py +305 -0
- package/plugins/devos/skills/docs/SKILL.md +22 -0
- package/plugins/devos/skills/docs/references/backfill.md +54 -0
- package/plugins/devos/skills/docs/references/init.md +43 -0
- package/plugins/devos/skills/docs/references/status.md +71 -0
- package/plugins/devos/skills/docs/references/update.md +80 -0
- package/plugins/devos/skills/explain/SKILL.md +45 -0
- package/plugins/devos/skills/explain/references/quiz.md +21 -0
- package/plugins/devos/skills/frame/SKILL.md +71 -0
- package/plugins/devos/skills/frame/references/boundaries.md +53 -0
- package/plugins/devos/skills/frame/references/decompose.md +96 -0
- package/plugins/devos/skills/frame/references/impact.md +43 -0
- package/plugins/devos/skills/frame/references/interrogate.md +68 -0
- package/plugins/devos/skills/frame/scripts/survey.sh +44 -0
- package/plugins/devos/skills/implement/SKILL.md +36 -0
- package/plugins/devos/skills/implement/references/conditional-plan.md +17 -0
- package/plugins/devos/skills/implement/references/debug-loop.md +7 -0
- package/plugins/devos/skills/implement/references/dod.md +32 -0
- package/plugins/devos/skills/implement/references/implement.md +52 -0
- package/plugins/devos/skills/implement/references/manual-gates.md +16 -0
- package/plugins/devos/skills/implement/references/plan.md +160 -0
- package/plugins/devos/skills/implement/references/resume.md +9 -0
- package/plugins/devos/skills/implement/references/scale.md +86 -0
- package/plugins/devos/skills/implement/references/verify-close.md +61 -0
- package/plugins/devos/skills/implement/references/visual-verification.md +7 -0
- package/plugins/devos/skills/implement/scripts/check-converge.sh +147 -0
- package/plugins/devos/skills/implement/scripts/equivalence-check.sh +140 -0
- package/plugins/devos/skills/implement/scripts/manifest-queue.sh +185 -0
- package/plugins/devos/skills/implement/scripts/ops-log.sh +58 -0
- package/plugins/devos/skills/implement/scripts/pilot-consistency.py +173 -0
- package/plugins/devos/skills/implement/scripts/validate-spec.sh +268 -0
- package/plugins/devos/skills/init/SKILL.md +100 -0
- package/plugins/devos/skills/init/references/archetype-sections.md +15 -0
- package/plugins/devos/skills/init/references/context.md +42 -0
- package/plugins/devos/skills/init/references/conventions-template.md +153 -0
- package/plugins/devos/skills/init/references/reverse-engineer.md +3 -0
- package/plugins/devos/skills/init/references/rule-format.md +58 -0
- package/plugins/devos/skills/init/references/shape-model.md +29 -0
- package/plugins/devos/skills/init/scripts/detect-archetype.py +461 -0
- package/plugins/devos/skills/init/scripts/render-rule.py +114 -0
- package/plugins/devos/skills/init/scripts/reverse-engineer.sh +162 -0
- package/plugins/devos/skills/learn/SKILL.md +29 -0
- package/plugins/devos/skills/learn/references/destinations.md +43 -0
- package/plugins/devos/skills/learn/references/scoring.md +26 -0
- package/plugins/devos/skills/product/SKILL.md +40 -0
- package/plugins/devos/skills/product/references/planner.md +54 -0
- package/plugins/devos/skills/review/SKILL.md +77 -0
- package/plugins/devos/skills/review/references/gaps.md +26 -0
- package/plugins/devos/skills/spec/SKILL.md +58 -0
- package/plugins/devos/skills/spec/references/prototype.md +21 -0
- package/plugins/devos/skills/spec/references/question-craft.md +47 -0
- package/plugins/devos/skills/spec/references/roadmap-link.md +5 -0
- package/plugins/devos/skills/spec/references/write-spec.md +143 -0
- package/plugins/devos/skills/spec/scripts/clarify.sh +173 -0
- package/plugins/devos/skills/status/SKILL.md +67 -0
- package/plugins/devos/skills/status/references/cost.md +38 -0
- package/plugins/devos/skills/status/scripts/artifact-budgets.sh +88 -0
- package/plugins/devos/skills/status/scripts/check-requires.py +556 -0
- package/plugins/devos/skills/status/scripts/context-cost.sh +89 -0
- package/plugins/devos/skills/status/scripts/convention-usage.sh +184 -0
- package/plugins/devos/skills/status/scripts/findings-summary.sh +112 -0
- package/plugins/devos/skills/status/scripts/quality-correlation.sh +282 -0
- package/plugins/devos/skills/status/scripts/sizing-calibration.sh +71 -0
- package/plugins/devos/skills/status/scripts/spec-diff.sh +137 -0
- package/plugins/devos/skills/surgical-diff/SKILL.md +23 -0
- package/plugins/devos/skills/test-integrity/SKILL.md +26 -0
- package/plugins/devos/skills/verification-evidence/SKILL.md +17 -0
- package/plugins/devos/workflows/common/backlog-file-structure.md +44 -0
- package/plugins/devos/workflows/common/category-file-structure.md +41 -0
- package/plugins/devos/workflows/common/feature-line-format.md +10 -0
- package/plugins/devos/workflows/common/roadmap-file-structure.md +51 -0
- package/plugins/devos/workflows/common/suggest-categories.md +76 -0
- package/plugins/devos/workflows/common/suggest-milestones.md +92 -0
- package/plugins/devos/workflows/doc/doc-health.md +137 -0
- package/plugins/devos/workflows/doc/doc-writer.md +44 -0
- package/plugins/devos/workflows/doc/init-documentation.md +196 -0
- package/plugins/devos/workflows/doc/retro-documentation.md +304 -0
- package/plugins/devos/workflows/doc/templates/library.md +58 -0
- package/plugins/devos/workflows/doc/templates/service.md +61 -0
- package/plugins/devos/workflows/doc/update-documentation.md +194 -0
- package/plugins/devos/workflows/doc/write-documentation.md +137 -0
- package/plugins/devos/workflows/product/create-product-mission.md +45 -0
- package/plugins/devos/workflows/product/create-product-roadmap.md +106 -0
- package/plugins/devos/workflows/product/create-product-tech-stack.md +19 -0
- package/plugins/devos/workflows/product/gather-product-info.md +25 -0
- package/plugins/devos/workflows/product/resume-detection.md +70 -0
- package/plugins/devos/workflows/product/reverse-backlog.md +118 -0
- package/plugins/devos/workflows/product/reverse-domain-model.md +86 -0
- package/plugins/devos/workflows/product/reverse-product-plan.md +81 -0
- package/plugins/devos/workflows/product/reverse-tech-stack.md +73 -0
- package/plugins/devos/workflows/product/scan-deep.md +113 -0
- package/plugins/devos/workflows/product/scan-git.md +122 -0
- package/plugins/devos/workflows/product/scan-light.md +154 -0
|
@@ -0,0 +1,4306 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Deterministic tests for the gate scripts, plain asserts, no framework,
|
|
3
|
+
# no model calls. Run via `just verify`.
|
|
4
|
+
set -u
|
|
5
|
+
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
6
|
+
DOD="$PWD/check-dod-core.sh"
|
|
7
|
+
RUBRIC="$PWD/test-integrity-rubric.sh"
|
|
8
|
+
BOUNDARY="$PWD/check-implementation-boundary.sh"
|
|
9
|
+
PSRC="$PWD/check-port-source.sh"
|
|
10
|
+
|
|
11
|
+
pass=0 fail=0
|
|
12
|
+
# After the cd at the top, $PWD is the gates directory (plugins/devos/scripts/gates/).
|
|
13
|
+
# EVAL_DIR is four levels up then into evals.
|
|
14
|
+
EVAL_DIR="$(cd ../../../../evals && pwd)"
|
|
15
|
+
|
|
16
|
+
check() { # check <name> <expected-exit> <actual-exit>
|
|
17
|
+
if [[ "$2" == "$3" ]]; then
|
|
18
|
+
pass=$((pass + 1))
|
|
19
|
+
else
|
|
20
|
+
echo "FAIL $1: expected exit $2, got $3" >&2
|
|
21
|
+
fail=$((fail + 1))
|
|
22
|
+
fi
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
tmp=$(mktemp -d)
|
|
26
|
+
# Guarded on BASH_SUBSHELL because a background subshell inherits this trap:
|
|
27
|
+
# run_with_timeout (eval'd in below) forks a watchdog, and under the load of a
|
|
28
|
+
# concurrent `just verify` that watchdog has fired this trap mid-run, deleting
|
|
29
|
+
# $tmp (this script's cwd) under every test that came after it. Only the main
|
|
30
|
+
# shell cleans up. BASH_SUBSHELL is 0 there and non-zero everywhere else, and
|
|
31
|
+
# it exists in bash 3.2, which is what darwin ships.
|
|
32
|
+
trap '[[ $BASH_SUBSHELL -eq 0 ]] && rm -rf "$tmp"' EXIT
|
|
33
|
+
cd "$tmp"
|
|
34
|
+
|
|
35
|
+
# --- check-dod-core.sh ---------------------------------------------------
|
|
36
|
+
spec="devos/specs/2026-01-01-demo"
|
|
37
|
+
mkdir -p "$spec/verifications"
|
|
38
|
+
|
|
39
|
+
t="$tmp/transcript.txt"
|
|
40
|
+
|
|
41
|
+
echo "working on it, no verdict yet" > "$t"
|
|
42
|
+
bash "$DOD" "$t"; check "dod: no DONE claim allows" 0 $?
|
|
43
|
+
|
|
44
|
+
echo "Verdict: DONE" > "$t"
|
|
45
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: DONE without spec name blocks" 2 $?
|
|
46
|
+
|
|
47
|
+
echo "Verdict: DONE: $spec" > "$t"
|
|
48
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: DONE with missing report blocks" 2 $?
|
|
49
|
+
|
|
50
|
+
printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$spec/verifications/verification.md"
|
|
51
|
+
bash "$DOD" "$t"; check "dod: CLEAN+PASS report allows" 0 $?
|
|
52
|
+
|
|
53
|
+
# This gate's own message names the format as `Verdict: DONE: devos/specs/[name]`.
|
|
54
|
+
# A model that echoes the reminder after the real verdict used to have the echo
|
|
55
|
+
# read as the verdict and get blocked for repeating the gate's words; the last
|
|
56
|
+
# matching line is not necessarily the one that names a spec.
|
|
57
|
+
printf 'Verdict: DONE: %s\n\nFormat reminder: `Verdict: DONE: devos/specs/[name]`\n' "$spec" > "$t"
|
|
58
|
+
bash "$DOD" "$t"; check "dod: format reminder after a valid verdict allows" 0 $?
|
|
59
|
+
|
|
60
|
+
echo "Verdict: DONE" > "$t"
|
|
61
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: no line names a spec still blocks" 2 $?
|
|
62
|
+
|
|
63
|
+
# Restore the naming verdict for the report-state tests below, which read the
|
|
64
|
+
# report, not the transcript.
|
|
65
|
+
echo "Verdict: DONE: $spec" > "$t"
|
|
66
|
+
|
|
67
|
+
printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **FAIL**\n' > "$spec/verifications/verification.md"
|
|
68
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: FAIL report blocks" 2 $?
|
|
69
|
+
|
|
70
|
+
printf '# Verification\n> 2026-01-01 | Completeness: **HAS ISSUES** | Quality: **PASS**\n' > "$spec/verifications/verification.md"
|
|
71
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: HAS ISSUES report blocks" 2 $?
|
|
72
|
+
|
|
73
|
+
printf '# Verification\n> 2026-01-01 | Completeness: NOT CLEAN | Quality: NOT PASS\n' > "$spec/verifications/verification.md"
|
|
74
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: negated CLEAN and PASS report blocks" 2 $?
|
|
75
|
+
|
|
76
|
+
printf 'looks great, ship it\n' > "$spec/verifications/verification.md"
|
|
77
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: malformed report (no verdict line) blocks" 2 $?
|
|
78
|
+
|
|
79
|
+
bash "$DOD" "$tmp/nonexistent.txt"; check "dod: missing transcript allows" 0 $?
|
|
80
|
+
|
|
81
|
+
arch="devos/specs/archive/2026-01-01-demo"
|
|
82
|
+
mkdir -p "$arch/verifications"
|
|
83
|
+
printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$arch/verifications/verification.md"
|
|
84
|
+
echo "Verdict: DONE: $arch" > "$t"
|
|
85
|
+
bash "$DOD" "$t"; check "dod: archived spec path allows" 0 $?
|
|
86
|
+
|
|
87
|
+
echo "Verdict: DONE: devos/specs/archive/absent-spec" > "$t"
|
|
88
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: archived path without report blocks" 2 $?
|
|
89
|
+
|
|
90
|
+
echo "Verdict: DONE: devos/specs/archive/..." > "$t"
|
|
91
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: ellipsis placeholder is the missing-spec case, not a report check" 2 $?
|
|
92
|
+
echo "Verdict: DONE: devos/specs/archive/verifications/verification.md" > "$t"
|
|
93
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: report path pasted as spec blocks as missing-spec (no verifications/verifications)" 2 $?
|
|
94
|
+
[[ "$(bash "$DOD" "$t" 2>&1 >/dev/null)" != *"verifications/verifications"* ]]
|
|
95
|
+
check "dod: report-path-as-spec message never doubles the verifications dir" 0 $?
|
|
96
|
+
|
|
97
|
+
# --- check-dod-core.sh: a pre-existing failure has to be shown --------------
|
|
98
|
+
# The DoD's rationalization table answers "the 2 failures are unrelated" with
|
|
99
|
+
# "then prove it". Until Phase B recorded a baseline there was nothing to point
|
|
100
|
+
# at, so the claim was accepted on the implementer's word.
|
|
101
|
+
pe="$tmp/preexisting"
|
|
102
|
+
mkdir -p "$pe/$spec/verifications"
|
|
103
|
+
(cd "$pe" && git init -q . && git config user.email t@t && git config user.name t)
|
|
104
|
+
pet="$pe/transcript.txt"
|
|
105
|
+
echo "Verdict: DONE: $spec" >"$pet"
|
|
106
|
+
clean_report='# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n'
|
|
107
|
+
|
|
108
|
+
printf "$clean_report" >"$pe/$spec/verifications/verification.md"
|
|
109
|
+
(cd "$pe" && bash "$DOD" "$pet"); check "dod: a report making no such claim needs no baseline" 0 $?
|
|
110
|
+
|
|
111
|
+
printf "${clean_report}No pre-existing failures.\n" >"$pe/$spec/verifications/verification.md"
|
|
112
|
+
(cd "$pe" && bash "$DOD" "$pet"); check "dod: a negated pre-existing claim needs no baseline" 0 $?
|
|
113
|
+
|
|
114
|
+
printf "${clean_report}No pre-existing failures. 2 failures are pre-existing.\n" >"$pe/$spec/verifications/verification.md"
|
|
115
|
+
(cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: mixed negated and affirmative claims block" 2 $?
|
|
116
|
+
|
|
117
|
+
printf "${clean_report}2 failures are pre-existing on this branch.\n" >"$pe/$spec/verifications/verification.md"
|
|
118
|
+
(cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: 'pre-existing' with no baseline.md blocks" 2 $?
|
|
119
|
+
|
|
120
|
+
printf 'node test.js -> exit 1 (2 failed)\n' >"$pe/$spec/verifications/baseline.md"
|
|
121
|
+
(cd "$pe" && bash "$DOD" "$pet"); check "dod: the same claim with a baseline allows" 0 $?
|
|
122
|
+
|
|
123
|
+
rm -f "$pe/$spec/verifications/baseline.md"
|
|
124
|
+
printf "${clean_report}The suite was already failing before this work.\n" >"$pe/$spec/verifications/verification.md"
|
|
125
|
+
(cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: 'already failing' is the same claim in other words" 2 $?
|
|
126
|
+
|
|
127
|
+
# --- check-dod-core.sh: a verdict is about the code it judged ----------------
|
|
128
|
+
# The report records the commit it read. Nothing re-runs the verifier when a
|
|
129
|
+
# later edit lands, so without that field a CLEAN report outlives the code it
|
|
130
|
+
# was true of, and every green report reads the same whether it still holds.
|
|
131
|
+
#
|
|
132
|
+
# Scoped to the plan's file list on purpose: the close protocol writes the
|
|
133
|
+
# behavior files, the archive, the changelog and the roadmap after the verifier
|
|
134
|
+
# ran, so a whole-tree rule would block every spec that ships its paperwork.
|
|
135
|
+
vc="$tmp/verdict-commit"
|
|
136
|
+
mkdir -p "$vc/$spec/verifications" "$vc/docs"
|
|
137
|
+
(cd "$vc" && git init -q . && git config user.email t@t && git config user.name t)
|
|
138
|
+
vct="$vc/transcript.txt"
|
|
139
|
+
echo "Verdict: DONE: $spec" >"$vct"
|
|
140
|
+
printf 'module.exports = 1\n' >"$vc/shipping.js"
|
|
141
|
+
printf '# Demo\n\n## Implementation Plan\n\n### Task Group 1: fee\n**Files:** `shipping.js`\n' >"$vc/$spec/spec.md"
|
|
142
|
+
printf '# Changelog\n' >"$vc/CHANGELOG.md"
|
|
143
|
+
printf '# Roadmap\n' >"$vc/docs/roadmap.md"
|
|
144
|
+
printf 'module.exports = 0\n' >"$vc/unrelated.js"
|
|
145
|
+
(cd "$vc" && git add -A && git commit -qm init)
|
|
146
|
+
vsha=$(cd "$vc" && git rev-parse --short HEAD)
|
|
147
|
+
printf '# Verification\n> 2026-01-01 | commit: %s | Completeness: **CLEAN** | Quality: **PASS**\n' "$vsha" >"$vc/$spec/verifications/verification.md"
|
|
148
|
+
(cd "$vc" && bash "$DOD" "$vct"); check "dod: a report bound to the tree it read allows" 0 $?
|
|
149
|
+
|
|
150
|
+
printf 'module.exports = 2\n' >"$vc/shipping.js"
|
|
151
|
+
(cd "$vc" && bash "$DOD" "$vct" 2>/dev/null); check "dod: a delivered file changed after the judged commit blocks" 2 $?
|
|
152
|
+
|
|
153
|
+
(cd "$vc" && git checkout -q -- shipping.js)
|
|
154
|
+
printf '# Changelog\n- shipped the fee\n' >"$vc/CHANGELOG.md"
|
|
155
|
+
printf '# Roadmap\n- fee: done\n' >"$vc/docs/roadmap.md"
|
|
156
|
+
printf 'module.exports = 0\n' >"$vc/unrelated.js"
|
|
157
|
+
(cd "$vc" && bash "$DOD" "$vct"); check "dod: the close protocol's paperwork after the verdict allows" 0 $?
|
|
158
|
+
|
|
159
|
+
printf '# Verification\n> 2026-01-01 | commit: 0000000 | Completeness: **CLEAN** | Quality: **PASS**\n' >"$vc/$spec/verifications/verification.md"
|
|
160
|
+
(cd "$vc" && bash "$DOD" "$vct" 2>/dev/null); check "dod: a marker naming no commit blocks" 2 $?
|
|
161
|
+
|
|
162
|
+
printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n' >"$vc/$spec/verifications/verification.md"
|
|
163
|
+
(cd "$vc" && bash "$DOD" "$vct"); check "dod: a report with no marker is the pre-marker format, allowed" 0 $?
|
|
164
|
+
|
|
165
|
+
printf 'module.exports = 3\n' >"$vc/shipping.js"
|
|
166
|
+
printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n\nSee commit: %s for the history.\n' "$vsha" >"$vc/$spec/verifications/verification.md"
|
|
167
|
+
(cd "$vc" && bash "$DOD" "$vct"); check "dod: a commit mentioned in the body is not the header's marker" 0 $?
|
|
168
|
+
(cd "$vc" && git checkout -q -- shipping.js)
|
|
169
|
+
|
|
170
|
+
printf '# Demo\n\n## Implementation Plan\n\nAn inline spec naming no files.\n' >"$vc/$spec/spec.md"
|
|
171
|
+
printf '# Verification\n> 2026-01-01 | commit: 0000000 | Completeness: **CLEAN** | Quality: **PASS**\n' >"$vc/$spec/verifications/verification.md"
|
|
172
|
+
(cd "$vc" && bash "$DOD" "$vct" 2>/dev/null); check "dod: a plan naming no files skips only the diff, not the marker" 2 $?
|
|
173
|
+
|
|
174
|
+
# Paths are passed one by one, so a name git would split is still one file, and
|
|
175
|
+
# a name git cannot resolve is the gate's limitation rather than a stale verdict.
|
|
176
|
+
printf '# Demo\n\n## Implementation Plan\n**Files:** `sp ace.js`\n' >"$vc/$spec/spec.md"
|
|
177
|
+
printf 'module.exports = 1\n' >"$vc/sp ace.js"
|
|
178
|
+
(cd "$vc" && git add -A && git commit -qm spacey)
|
|
179
|
+
ssha=$(cd "$vc" && git rev-parse --short HEAD)
|
|
180
|
+
printf '# Verification\n> 2026-01-01 | commit: %s | Completeness: **CLEAN** | Quality: **PASS**\n' "$ssha" >"$vc/$spec/verifications/verification.md"
|
|
181
|
+
(cd "$vc" && bash "$DOD" "$vct"); check "dod: a planned path with a space allows while unchanged" 0 $?
|
|
182
|
+
printf 'module.exports = 2\n' >"$vc/sp ace.js"
|
|
183
|
+
(cd "$vc" && bash "$DOD" "$vct" 2>/dev/null); check "dod: a changed planned path with a space still blocks" 2 $?
|
|
184
|
+
(cd "$vc" && git checkout -q -- 'sp ace.js')
|
|
185
|
+
|
|
186
|
+
: >"$tmp/outside.js"
|
|
187
|
+
printf '# Demo\n\n## Implementation Plan\n**Files:** `../outside.js`\n' >"$vc/$spec/spec.md"
|
|
188
|
+
printf '# Verification\n> 2026-01-01 | commit: %s | Completeness: **CLEAN** | Quality: **PASS**\n' "$ssha" >"$vc/$spec/verifications/verification.md"
|
|
189
|
+
(cd "$vc" && bash "$DOD" "$vct"); check "dod: a planned path git cannot resolve fails open" 0 $?
|
|
190
|
+
|
|
191
|
+
# --- check-dod-core.sh: an empty delivery is not done -----------------------
|
|
192
|
+
# The verifier scopes to the diff, so an implementation that wrote nothing has
|
|
193
|
+
# no placeholders in it and verifies CLEAN. The plan's own **Files:** lines say
|
|
194
|
+
# what was meant to exist.
|
|
195
|
+
printf "$clean_report" >"$pe/$spec/verifications/verification.md"
|
|
196
|
+
printf '# Demo\n\n## Implementation Plan\n\n### Task Group 1: thing\n**Files:** `src/thing.js`, `src/thing.test.js`\n' >"$pe/$spec/spec.md"
|
|
197
|
+
(cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: no planned file exists, nothing was delivered, blocks" 2 $?
|
|
198
|
+
|
|
199
|
+
mkdir -p "$pe/src" && : >"$pe/src/thing.js"
|
|
200
|
+
(cd "$pe" && bash "$DOD" "$pet"); check "dod: one planned file delivered allows (an amended list is normal)" 0 $?
|
|
201
|
+
|
|
202
|
+
printf '# Demo\n\n## Implementation Plan\n\n### Task Group 1: source\n**Files:** `src/auth.ts`\n' >"$pe/$spec/spec.md"
|
|
203
|
+
printf '// Implemented by DevOS\n' >"$pe/src/auth.ts"
|
|
204
|
+
(cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: process metadata in a planned source file blocks" 2 $?
|
|
205
|
+
printf 'export const status = 401\n' >"$pe/src/auth.ts"
|
|
206
|
+
(cd "$pe" && bash "$DOD" "$pet"); check "dod: clean planned source allows" 0 $?
|
|
207
|
+
|
|
208
|
+
rm -rf "$pe/src"
|
|
209
|
+
printf '# Demo\n\n## Implementation Plan\n\nNo file list, an inline mini-spec.\n' >"$pe/$spec/spec.md"
|
|
210
|
+
(cd "$pe" && bash "$DOD" "$pet"); check "dod: a plan naming no files is silent, not blocking" 0 $?
|
|
211
|
+
|
|
212
|
+
# A spec whose whole job is to write a standards file delivered what it planned.
|
|
213
|
+
# Blocking it would be the false positive; the check is "nothing exists", not
|
|
214
|
+
# "nothing outside devos/ exists".
|
|
215
|
+
printf '# Demo\n\n### Task Group 1: standards\n**Files:** `devos/standards/conventions.md`\n' >"$pe/$spec/spec.md"
|
|
216
|
+
(cd "$pe" && bash "$DOD" "$pet" 2>/dev/null); check "dod: a devos-only plan with nothing written still blocks" 2 $?
|
|
217
|
+
mkdir -p "$pe/devos/standards" && : >"$pe/devos/standards/conventions.md"
|
|
218
|
+
(cd "$pe" && bash "$DOD" "$pet"); check "dod: a devos-only plan that wrote its file allows" 0 $?
|
|
219
|
+
|
|
220
|
+
# --- check-port-source.sh: the tree a port reads is not the tree it writes ---
|
|
221
|
+
# A port writes a new tree and leaves the old one alone, and the old one may
|
|
222
|
+
# be running in production. The manifest names the sources, and the check
|
|
223
|
+
# compares them against the branch, so an edit to a source is a finding.
|
|
224
|
+
ps="$tmp/port-source"
|
|
225
|
+
mkdir -p "$ps/legacy/pkg" "$ps/modernized/pkg" "$ps/devos/specs/2026-10-20-port"
|
|
226
|
+
(cd "$ps" && git init -q && git config user.email t@t && git config user.name t)
|
|
227
|
+
printf 'x\n' >"$ps/legacy/pkg/a.py"
|
|
228
|
+
printf 'y\n' >"$ps/modernized/pkg/a.py"
|
|
229
|
+
printf 'source\ttarget\nlegacy/pkg/a.py\tmodernized/pkg/a.py\n' >"$ps/devos/specs/2026-10-20-port/manifest.tsv"
|
|
230
|
+
printf '# spec\n' >"$ps/devos/specs/2026-10-20-port/spec-functional.md"
|
|
231
|
+
(cd "$ps" && git add -A && git commit -qm base)
|
|
232
|
+
|
|
233
|
+
bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" >/dev/null 2>&1
|
|
234
|
+
check "port-source: a port that leaves the tree it reads alone allows" 0 $?
|
|
235
|
+
|
|
236
|
+
printf 'z\n' >>"$ps/legacy/pkg/a.py"
|
|
237
|
+
out=$(bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" 2>&1); rc=$?
|
|
238
|
+
[ "$rc" -eq 1 ] && echo "$out" | grep -q "the tree it reads is not the tree it writes"
|
|
239
|
+
check "port-source: a port that edits the tree it reads blocks" 0 $?
|
|
240
|
+
|
|
241
|
+
out=$(bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" 2>&1); rc=$?
|
|
242
|
+
[ "$rc" -eq 1 ] && echo "$out" | grep -q "fix: revert that change"
|
|
243
|
+
check "port-source: the finding says what to do" 0 $?
|
|
244
|
+
|
|
245
|
+
printf 'w\n' >>"$ps/modernized/pkg/a.py"
|
|
246
|
+
(cd "$ps" && git checkout -q -- legacy)
|
|
247
|
+
bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" >/dev/null 2>&1
|
|
248
|
+
check "port-source: a port that writes the tree it builds allows" 0 $?
|
|
249
|
+
|
|
250
|
+
rm "$ps/devos/specs/2026-10-20-port/manifest.tsv"
|
|
251
|
+
bash "$PSRC" --spec-dir "$ps/devos/specs/2026-10-20-port" --root "$ps" >/dev/null 2>&1
|
|
252
|
+
check "port-source: a spec with no manifest is not a port, and is silent" 0 $?
|
|
253
|
+
|
|
254
|
+
# --- check-dod-core.sh: a port that edits the tree it reads is not done -----
|
|
255
|
+
# The port's manifest names the sources, and a DONE claim on a port spec is
|
|
256
|
+
# checked against them: the tree the port reads keeps running while the port
|
|
257
|
+
# lands.
|
|
258
|
+
port="$tmp/portrepo"
|
|
259
|
+
mkdir -p "$port/legacy/pkg" "$port/devos/specs/2026-10-20-port/verifications"
|
|
260
|
+
(cd "$port" && git init -q && git config user.email t@t && git config user.name t)
|
|
261
|
+
printf 'x\n' >"$port/legacy/pkg/a.py"
|
|
262
|
+
printf 'source\ttarget\nlegacy/pkg/a.py\tmodernized/pkg/a.py\n' >"$port/devos/specs/2026-10-20-port/manifest.tsv"
|
|
263
|
+
printf '# Verification\n> 2026-10-20 | Completeness: **CLEAN** | Quality: **PASS**\n' >"$port/devos/specs/2026-10-20-port/verifications/verification.md"
|
|
264
|
+
(cd "$port" && git add -A && git commit -qm base)
|
|
265
|
+
echo "Verdict: DONE: devos/specs/2026-10-20-port" > "$tmp/port.txt"
|
|
266
|
+
(cd "$port" && bash "$DOD" "$tmp/port.txt"); check "dod: a port that left its source tree alone allows" 0 $?
|
|
267
|
+
printf 'y\n' >>"$port/legacy/pkg/a.py"
|
|
268
|
+
(cd "$port" && bash "$DOD" "$tmp/port.txt") 2>/dev/null; check "dod: a port that edited its source tree blocks" 2 $?
|
|
269
|
+
|
|
270
|
+
# --- check-dod-core.sh: behavior-merge check --------
|
|
271
|
+
# Against the real corpus this repo ships: all three archived specs must
|
|
272
|
+
# pass unchanged, run from the actual repo (a regression test that only
|
|
273
|
+
# ever sees synthetic input has never seen a true negative).
|
|
274
|
+
REPO_ROOT="$(cd "$(dirname "$DOD")/../../../.." && pwd)"
|
|
275
|
+
for real in \
|
|
276
|
+
2026-07-27-pressure-fixture-time-pressure \
|
|
277
|
+
2026-07-27-reverse-engineer-spec-stubs \
|
|
278
|
+
2026-07-27-run-sh-results-suffix; do
|
|
279
|
+
rt="$tmp/real-transcript-$real.txt"
|
|
280
|
+
echo "Verdict: DONE: devos/specs/archive/$real" > "$rt"
|
|
281
|
+
(cd "$REPO_ROOT" && bash "$DOD" "$rt") >/dev/null 2>&1
|
|
282
|
+
check "dod: real archived spec '$real' behavior delta passes unchanged" 0 $?
|
|
283
|
+
done
|
|
284
|
+
|
|
285
|
+
# Planted miss: delta names a capability with no behavior file at all.
|
|
286
|
+
ghost="$tmp/devos/specs/2026-01-01-ghost-capability"
|
|
287
|
+
mkdir -p "$ghost/verifications"
|
|
288
|
+
cat > "$ghost/spec-functional.md" <<'MD'
|
|
289
|
+
# Ghost Capability
|
|
290
|
+
## Goal
|
|
291
|
+
x
|
|
292
|
+
## 3bis. Behavior delta
|
|
293
|
+
|
|
294
|
+
**Added** (capability: `ghost-capability`):
|
|
295
|
+
- Something new that never got merged.
|
|
296
|
+
## Out of Scope
|
|
297
|
+
- y
|
|
298
|
+
MD
|
|
299
|
+
printf '# Verification\n> 2026-01-01 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$ghost/verifications/verification.md"
|
|
300
|
+
echo "Verdict: DONE: devos/specs/2026-01-01-ghost-capability" > "$t"
|
|
301
|
+
bash "$DOD" "$t" 2>/dev/null; check "dod: delta names capability with no behavior file blocks" 2 $?
|
|
302
|
+
|
|
303
|
+
# Fixed: behavior file exists with a matching last-spec header.
|
|
304
|
+
mkdir -p "$tmp/devos/product/behaviors"
|
|
305
|
+
printf '# Ghost Capability\n\n> updated: 2026-01-01 | last spec: 2026-01-01-ghost-capability\n\nSomething new that never got merged.\n' \
|
|
306
|
+
> "$tmp/devos/product/behaviors/ghost-capability.md"
|
|
307
|
+
bash "$DOD" "$t"; check "dod: matching behavior file with last-spec header allows" 0 $?
|
|
308
|
+
|
|
309
|
+
# Removed entry: needs a History tombstone naming this spec, not a fresh header.
|
|
310
|
+
removed="$tmp/devos/specs/2026-01-02-drop-capability"
|
|
311
|
+
mkdir -p "$removed/verifications"
|
|
312
|
+
cat > "$removed/spec-functional.md" <<'MD'
|
|
313
|
+
# Drop Capability
|
|
314
|
+
## Goal
|
|
315
|
+
x
|
|
316
|
+
## 3bis. Behavior delta
|
|
317
|
+
|
|
318
|
+
**Removed** (capability: `ghost-capability`):
|
|
319
|
+
- Dropped on purpose, replaced by nothing.
|
|
320
|
+
## Out of Scope
|
|
321
|
+
- y
|
|
322
|
+
MD
|
|
323
|
+
printf '# Verification\n> 2026-01-02 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$removed/verifications/verification.md"
|
|
324
|
+
rt2="$tmp/removed-transcript.txt"
|
|
325
|
+
echo "Verdict: DONE: devos/specs/2026-01-02-drop-capability" > "$rt2"
|
|
326
|
+
bash "$DOD" "$rt2" 2>/dev/null; check "dod: removed capability with no History tombstone blocks" 2 $?
|
|
327
|
+
|
|
328
|
+
printf '# Ghost Capability\n\n> updated: 2026-01-01 | last spec: 2026-01-01-ghost-capability\n\n## History\n- removed ghost-capability: dropped on purpose, per 2026-01-02-drop-capability\n' \
|
|
329
|
+
> "$tmp/devos/product/behaviors/ghost-capability.md"
|
|
330
|
+
bash "$DOD" "$rt2"; check "dod: removed capability with a matching History tombstone allows" 0 $?
|
|
331
|
+
|
|
332
|
+
# No Behavior delta section at all: silent, same posture as validate-spec.sh.
|
|
333
|
+
nodelta="$tmp/devos/specs/2026-01-03-no-delta"
|
|
334
|
+
mkdir -p "$nodelta/verifications"
|
|
335
|
+
printf '# No Delta\n## Goal\nx\n## Out of Scope\n- y\n' > "$nodelta/spec-functional.md"
|
|
336
|
+
printf '# Verification\n> 2026-01-03 | Completeness: **CLEAN** | Quality: **PASS**\n' > "$nodelta/verifications/verification.md"
|
|
337
|
+
echo "Verdict: DONE: devos/specs/2026-01-03-no-delta" > "$t"
|
|
338
|
+
bash "$DOD" "$t"; check "dod: spec with no Behavior delta section is silent" 0 $?
|
|
339
|
+
|
|
340
|
+
# --- check-dod-core.sh: leftover debug instrumentation ----------------------
|
|
341
|
+
# The tag is assembled with printf, never written literally, so this very file
|
|
342
|
+
# stays clean: it is tracked, and the real-corpus assertions above run the gate
|
|
343
|
+
# from REPO_ROOT, where a literal tag here would block every one of them.
|
|
344
|
+
dbg="$tmp/debugtag"
|
|
345
|
+
mkdir -p "$dbg/devos/specs/2026-01-04-dbg/verifications"
|
|
346
|
+
(cd "$dbg" && git init -q .) 2>/dev/null
|
|
347
|
+
printf '# Verification\n> 2026-01-04 | Completeness: **CLEAN** | Quality: **PASS**\n' \
|
|
348
|
+
> "$dbg/devos/specs/2026-01-04-dbg/verifications/verification.md"
|
|
349
|
+
dt="$tmp/dbg-transcript.txt"
|
|
350
|
+
echo "Verdict: DONE: devos/specs/2026-01-04-dbg" > "$dt"
|
|
351
|
+
|
|
352
|
+
printf 'console.log("[DEBUG-%s] here")\n' a4f2 > "$dbg/src.js"
|
|
353
|
+
(cd "$dbg" && git add -A) 2>/dev/null
|
|
354
|
+
(cd "$dbg" && bash "$DOD" "$dt") 2>/dev/null
|
|
355
|
+
check "dod: leftover [DEBUG-hex] tag in a tracked file blocks" 2 $?
|
|
356
|
+
|
|
357
|
+
# The placeholder form the docs use is not the tag: it must not match itself.
|
|
358
|
+
printf 'tag every probe [DEBUG-xxxx] so cleanup is one grep\n' > "$dbg/src.js"
|
|
359
|
+
(cd "$dbg" && git add -A) 2>/dev/null
|
|
360
|
+
(cd "$dbg" && bash "$DOD" "$dt")
|
|
361
|
+
check "dod: [DEBUG-xxxx] placeholder in prose allows" 0 $?
|
|
362
|
+
|
|
363
|
+
# Untracked scratch file was never shipped: not the gate's business.
|
|
364
|
+
printf 'console.log("[DEBUG-%s]")\n' 9bee > "$dbg/scratch.js"
|
|
365
|
+
(cd "$dbg" && bash "$DOD" "$dt")
|
|
366
|
+
check "dod: untracked file with a tag allows" 0 $?
|
|
367
|
+
|
|
368
|
+
rm -f "$dbg/src.js" "$dbg/scratch.js"
|
|
369
|
+
(cd "$dbg" && git add -A) 2>/dev/null
|
|
370
|
+
(cd "$dbg" && bash "$DOD" "$dt")
|
|
371
|
+
check "dod: clean tree allows" 0 $?
|
|
372
|
+
|
|
373
|
+
# --- check-report-locations.sh: the verdict's citations are re-derived ------
|
|
374
|
+
# A `file:line` in the report is the part a candidate gets right in prose and
|
|
375
|
+
# wrong in fact. Before this pass nothing read it back, so a citation past the
|
|
376
|
+
# end of its file blocked nothing and read like every other line.
|
|
377
|
+
LOC="$(dirname "$DOD")/check-report-locations.sh"
|
|
378
|
+
loc="$tmp/loc"
|
|
379
|
+
mkdir -p "$loc/devos/specs/2026-01-05-loc/verifications"
|
|
380
|
+
printf 'one\ntwo\nthree\n' > "$loc/src.js"
|
|
381
|
+
LR="$loc/devos/specs/2026-01-05-loc/verifications/verification.md"
|
|
382
|
+
lt="$tmp/loc-transcript.txt"
|
|
383
|
+
echo "Verdict: DONE: devos/specs/2026-01-05-loc" > "$lt"
|
|
384
|
+
|
|
385
|
+
printf '# Verification\n> 2026-01-05 | Completeness: **CLEAN** | Quality: **PASS**\n| `src.js:2` | code | problem |\n' > "$LR"
|
|
386
|
+
(cd "$loc" && bash "$DOD" "$lt") 2>/dev/null
|
|
387
|
+
check "dod: a resolvable citation allows" 0 $?
|
|
388
|
+
|
|
389
|
+
printf '# Verification\n> 2026-01-05 | Completeness: **CLEAN** | Quality: **PASS**\n| `src.js:99` | code | problem |\n' > "$LR"
|
|
390
|
+
(cd "$loc" && bash "$DOD" "$lt") 2>/dev/null
|
|
391
|
+
check "dod: a citation past the end of its file blocks" 2 $?
|
|
392
|
+
|
|
393
|
+
printf '# Verification\n> 2026-01-05 | Completeness: **CLEAN** | Quality: **PASS**\n| `src/gone.js:3` | code | problem |\n' > "$LR"
|
|
394
|
+
(cd "$loc" && bash "$DOD" "$lt") 2>/dev/null
|
|
395
|
+
check "dod: a citation to a nonexistent file blocks" 2 $?
|
|
396
|
+
|
|
397
|
+
# A URL carries a colon-number too; it is not a citation.
|
|
398
|
+
printf '# Verification\n> 2026-01-05 | Completeness: **CLEAN** | Quality: **PASS**\nsee https://example.com:8080/a.js:9\n' > "$LR"
|
|
399
|
+
(cd "$loc" && bash "$DOD" "$lt")
|
|
400
|
+
check "dod: a URL colon-number is not a citation" 0 $?
|
|
401
|
+
|
|
402
|
+
# The reflection pass on its own, out of the hook: same exit contract.
|
|
403
|
+
bash "$LOC" "$loc/devos/specs/2026-01-05-loc" >/dev/null 2>&1
|
|
404
|
+
check "report-locations: a clean report exits 0" 0 $?
|
|
405
|
+
printf '# V\n| `src.js:99` | code |\n' > "$LR"
|
|
406
|
+
bash "$LOC" "$loc/devos/specs/2026-01-05-loc" >/dev/null 2>&1
|
|
407
|
+
check "report-locations: a drifted citation exits 1" 1 $?
|
|
408
|
+
printf '# V\n| `src.js:3-2` | reversed |\n' > "$LR"
|
|
409
|
+
report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
|
|
410
|
+
check "report-locations: a reversed range exits 1" 1 $report_rc
|
|
411
|
+
printf '%s\n' "$report_out" | grep -Fq 'range end 2 is before start 3'; check "report-locations: reversed range names the bounds" 0 $?
|
|
412
|
+
printf '# V\n| `src.js:0` | zero |\n' > "$LR"
|
|
413
|
+
report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
|
|
414
|
+
check "report-locations: line zero exits 1" 1 $report_rc
|
|
415
|
+
printf '%s\n' "$report_out" | grep -Fq 'source line numbers start at 1'; check "report-locations: line zero names the one-based rule" 0 $?
|
|
416
|
+
printf 'KEY=value\n' > "$loc/.env"
|
|
417
|
+
printf '# V\n| `.env:99` | drifted |\n' > "$LR"
|
|
418
|
+
report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
|
|
419
|
+
check "report-locations: dotfile citation exits 1" 1 $report_rc
|
|
420
|
+
printf '%s\n' "$report_out" | grep -Fq '.env:99'; check "report-locations: dotfile citation is named" 0 $?
|
|
421
|
+
printf 'outside\n' > "$tmp/report-outside.txt"
|
|
422
|
+
printf '# V\n| `../report-outside.txt:1` | escaped |\n' > "$LR"
|
|
423
|
+
report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
|
|
424
|
+
check "report-locations: a citation outside the project root exits 1" 1 $report_rc
|
|
425
|
+
printf '%s\n' "$report_out" | grep -Fq 'outside the project root'; check "report-locations: path escape names the boundary" 0 $?
|
|
426
|
+
ln -s "$tmp/report-outside.txt" "$loc/escape.txt"
|
|
427
|
+
printf '# V\n| `escape.txt:1` | symlink escape |\n' > "$LR"
|
|
428
|
+
report_out=$(bash "$LOC" "$loc/devos/specs/2026-01-05-loc" 2>&1); report_rc=$?
|
|
429
|
+
check "report-locations: a symlink outside the project root exits 1" 1 $report_rc
|
|
430
|
+
printf '%s\n' "$report_out" | grep -Fq 'outside the project root'; check "report-locations: symlink escape names the boundary" 0 $?
|
|
431
|
+
printf 'spec-local\n' > "$loc/devos/specs/2026-01-05-loc/notes.md"
|
|
432
|
+
printf '# V\n| `notes.md:1` | spec-relative |\n' > "$LR"
|
|
433
|
+
bash "$LOC" "$loc/devos/specs/2026-01-05-loc" --root "$loc" >/dev/null 2>&1
|
|
434
|
+
check "report-locations: a spec-relative in-root citation allows" 0 $?
|
|
435
|
+
bash "$LOC" >/dev/null 2>&1
|
|
436
|
+
check "report-locations: no argument is a usage error" 2 $?
|
|
437
|
+
|
|
438
|
+
# --- check-scenario-coverage.sh: a scenario recorded nowhere was not verified --
|
|
439
|
+
# The verifier is told to record every SC-n from spec-functional.md in its
|
|
440
|
+
# report, and nothing checked it did. A missing row is the one failure a
|
|
441
|
+
# clean-looking report cannot show: every row it carries resolves, the verdict
|
|
442
|
+
# reads CLEAN + PASS, and the scenario nobody ran leaves no trace.
|
|
443
|
+
COV="$(dirname "$DOD")/check-scenario-coverage.sh"
|
|
444
|
+
cov="$tmp/cov"
|
|
445
|
+
COVS="$cov/devos/specs/2026-01-06-cov"
|
|
446
|
+
mkdir -p "$COVS/verifications"
|
|
447
|
+
CR="$COVS/verifications/verification.md"
|
|
448
|
+
covspec="$COVS/spec-functional.md"
|
|
449
|
+
ct="$tmp/cov-transcript.txt"
|
|
450
|
+
printf 'x\nx\nx\nx\nx\nx\nx\nx\nx\nx\n' > "$cov/one.test.js"
|
|
451
|
+
echo "Verdict: DONE: devos/specs/2026-01-06-cov" > "$ct"
|
|
452
|
+
printf '# Demo\n\n## 3. Functionality\n\n**Scenarios:**\n- Scenario SC-1: one / WHEN a / THEN b\n- Scenario SC-2: two / WHEN c / THEN d\n' > "$covspec"
|
|
453
|
+
|
|
454
|
+
printf '# Verification\n> 2026-01-06 | Completeness: **CLEAN** | Quality: **PASS**\n| SC-1 | pass | `one.test.js:3` |\n| SC-2 | pass | `one.test.js:9` |\n' > "$CR"
|
|
455
|
+
(cd "$cov" && bash "$DOD" "$ct"); check "coverage: DONE with every scenario recorded allows" 0 $?
|
|
456
|
+
|
|
457
|
+
printf '# Verification\n> 2026-01-06 | Completeness: **CLEAN** | Quality: **PASS**\n| SC-1 | pass | `one.test.js:3` |\n' > "$CR"
|
|
458
|
+
cov_out=$(cd "$cov" && bash "$DOD" "$ct" 2>&1); cov_rc=$?
|
|
459
|
+
check "coverage: a scenario the report never records blocks DONE" 2 $cov_rc
|
|
460
|
+
printf '%s\n' "$cov_out" | grep -Fq 'SC-2'; check "coverage: the missing ID is named" 0 $?
|
|
461
|
+
|
|
462
|
+
# Presence, not format: prose that records each ID is as valid as a table row.
|
|
463
|
+
printf '# V\nSC-1 pass (`one.test.js:3`), SC-2 AWAITING HUMAN.\n' > "$CR"
|
|
464
|
+
bash "$COV" "$COVS" >/dev/null 2>&1; check "coverage: a mapping line records the scenarios" 0 $?
|
|
465
|
+
|
|
466
|
+
# A scenario the spec dropped during revision leaves the report's old row stale.
|
|
467
|
+
printf '# V\nSC-1 pass, SC-2 pass, SC-3 pass\n' > "$CR"
|
|
468
|
+
cov_out=$(bash "$COV" "$COVS" 2>&1); cov_rc=$?
|
|
469
|
+
check "coverage: a report citing an undeclared ID exits 1" 1 $cov_rc
|
|
470
|
+
printf '%s\n' "$cov_out" | grep -Fq 'SC-3'; check "coverage: the undeclared ID is named" 0 $?
|
|
471
|
+
|
|
472
|
+
# SC-10 is a different ID from SC-1, so a prefix match would let an omitted
|
|
473
|
+
# scenario pass on its neighbour's row.
|
|
474
|
+
printf '# Demo\n- Scenario SC-1: one / WHEN a / THEN b\n- Scenario SC-10: ten / WHEN c / THEN d\n' > "$covspec"
|
|
475
|
+
printf '# V\nSC-10 pass\n' > "$CR"
|
|
476
|
+
cov_out=$(bash "$COV" "$COVS" 2>&1); cov_rc=$?
|
|
477
|
+
check "coverage: SC-1 missing while SC-10 is recorded exits 1" 1 $cov_rc
|
|
478
|
+
printf '%s\n' "$cov_out" | grep -Fq 'SC-10'; check "coverage: a recorded SC-10 does not satisfy SC-1" 1 $?
|
|
479
|
+
|
|
480
|
+
# Exemptions: a C-route spec declares no scenarios by design, and a pre-SC spec
|
|
481
|
+
# has no IDs to trace. Both are silence, never a block.
|
|
482
|
+
printf '# Demo\n\n## Invariant\n- npm test exits 0 with the same count\n' > "$covspec"
|
|
483
|
+
bash "$COV" "$COVS" >/dev/null 2>&1; check "coverage: an invariant spec exits 0" 0 $?
|
|
484
|
+
printf '# Demo\n\n- Scenario: one / WHEN a / THEN b\n' > "$covspec"
|
|
485
|
+
bash "$COV" "$COVS" >/dev/null 2>&1; check "coverage: an unnumbered legacy spec exits 0" 0 $?
|
|
486
|
+
rm -f "$covspec"
|
|
487
|
+
bash "$COV" "$COVS" >/dev/null 2>&1; check "coverage: a spec with no functional doc exits 0" 0 $?
|
|
488
|
+
# The report path is accepted directly, since a caller holding one should not
|
|
489
|
+
# have to reconstruct the spec folder.
|
|
490
|
+
printf '# Demo\n- Scenario SC-1: one / WHEN a / THEN b\n' > "$covspec"
|
|
491
|
+
printf '# V\nSC-1 pass\n' > "$CR"
|
|
492
|
+
bash "$COV" "$CR" >/dev/null 2>&1; check "coverage: a report path records the scenarios" 0 $?
|
|
493
|
+
printf '# V\nnothing\n' > "$CR"
|
|
494
|
+
bash "$COV" "$CR" >/dev/null 2>&1; check "coverage: a report path with a missing ID exits 1" 1 $?
|
|
495
|
+
bash "$COV" >/dev/null 2>&1; check "coverage: no argument is a usage error" 2 $?
|
|
496
|
+
|
|
497
|
+
# --- check-dod-core.sh: the branch resolves the spec --------------
|
|
498
|
+
# The pipeline creates a `[spec-name]` branch before the first edit, so the gate
|
|
499
|
+
# derives the spec instead of trusting a path the model typed. A placeholder
|
|
500
|
+
# like `devos/specs/x` must not bounce the run into a retry loop when the
|
|
501
|
+
# branch's own report is there.
|
|
502
|
+
br="$tmp/branch"
|
|
503
|
+
mkdir -p "$br/devos/specs/2026-01-02-real/verifications"
|
|
504
|
+
(cd "$br" && git init -q . && git checkout -q -b 2026-01-02-real) 2>/dev/null
|
|
505
|
+
printf '# Verification\n> 2026-01-02 | Completeness: **CLEAN** | Quality: **PASS**\n' \
|
|
506
|
+
> "$br/devos/specs/2026-01-02-real/verifications/verification.md"
|
|
507
|
+
bt="$tmp/branch-transcript.txt"
|
|
508
|
+
|
|
509
|
+
echo "Verdict: DONE: devos/specs/x" > "$bt"
|
|
510
|
+
(cd "$br" && bash "$DOD" "$bt"); check "dod: a placeholder spec resolves to the branch's spec" 0 $?
|
|
511
|
+
|
|
512
|
+
echo "Verdict: DONE" > "$bt"
|
|
513
|
+
(cd "$br" && bash "$DOD" "$bt"); check "dod: a bare verdict resolves on a spec branch" 0 $?
|
|
514
|
+
|
|
515
|
+
# The fallback is not a bypass: with no report on the branch's spec, naming is
|
|
516
|
+
# still required and a placeholder still blocks.
|
|
517
|
+
rm -rf "$br/devos/specs/2026-01-02-real/verifications"
|
|
518
|
+
echo "Verdict: DONE: devos/specs/x" > "$bt"
|
|
519
|
+
(cd "$br" && bash "$DOD" "$bt" 2>/dev/null); check "dod: branch fallback needs a real report" 2 $?
|
|
520
|
+
|
|
521
|
+
# --- test-integrity-rubric.sh ---------------------------------------------
|
|
522
|
+
old="$tmp/old.txt" new="$tmp/new.txt"
|
|
523
|
+
|
|
524
|
+
printf 'expect(a).toBe(1)\n' > "$old"
|
|
525
|
+
printf 'const x = 2\n' > "$new"
|
|
526
|
+
out=$(bash "$RUBRIC" --format copilot --file src/main.ts --old "$old" --new "$new")
|
|
527
|
+
[[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: non-test file allows" 0 $?
|
|
528
|
+
|
|
529
|
+
printf 'it("works", () => { expect(a).toBe(1) })\n' > "$old"
|
|
530
|
+
printf 'it.skip("works", () => { expect(a).toBe(1) })\n' > "$new"
|
|
531
|
+
out=$(bash "$RUBRIC" --format copilot --file src/main.test.ts --old "$old" --new "$new")
|
|
532
|
+
[[ "$out" == *'"deny"'* ]]; check "rubric: added .skip denies" 0 $?
|
|
533
|
+
|
|
534
|
+
printf 'expect(a).toBe(1)\nexpect(b).toBe(2)\n' > "$old"
|
|
535
|
+
printf 'expect(a).toBe(1)\n' > "$new"
|
|
536
|
+
out=$(bash "$RUBRIC" --format copilot --file src/main.test.ts --old "$old" --new "$new")
|
|
537
|
+
[[ "$out" == *'"deny"'* ]]; check "rubric: removed assertion denies" 0 $?
|
|
538
|
+
|
|
539
|
+
printf 'expect(a).toBe(1)\n' > "$old"
|
|
540
|
+
printf 'expect(a).toBe(1)\nexpect(b).toBe(2)\n' > "$new"
|
|
541
|
+
out=$(bash "$RUBRIC" --format copilot --file src/main.test.ts --old "$old" --new "$new")
|
|
542
|
+
[[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: added assertion allows" 0 $?
|
|
543
|
+
|
|
544
|
+
printf 'expect(a).toBe(1)\n' > "$new"
|
|
545
|
+
out=$(bash "$RUBRIC" --format copilot --file src/new.test.ts --old "$tmp/absent.txt" --new "$new")
|
|
546
|
+
[[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: brand-new test file allows" 0 $?
|
|
547
|
+
|
|
548
|
+
# Regression: bare "spec" substring in a path (DevOS's own devos/specs/*.md
|
|
549
|
+
# artifacts) must never be treated as a test file, even if the prose content
|
|
550
|
+
# happens to contain words like "expect"/"assert".
|
|
551
|
+
printf 'Expect the API to return 200.\n' > "$old"
|
|
552
|
+
printf 'A change to what we expect from the API.\n' > "$new"
|
|
553
|
+
out=$(bash "$RUBRIC" --format copilot --file devos/specs/2026-01-01-foo/spec.md --old "$old" --new "$new")
|
|
554
|
+
[[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: devos/specs/*.md is never a test file" 0 $?
|
|
555
|
+
|
|
556
|
+
printf 'assert.strictEqual(a, 1)\n' > "$old"
|
|
557
|
+
printf 'assert.strictEqual(a, 1)\nassert.strictEqual(b, 2)\n' > "$new"
|
|
558
|
+
out=$(bash "$RUBRIC" --format copilot --file devos/specs/2026-01-01-foo/decisions.md --old "$old" --new "$new")
|
|
559
|
+
[[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: decisions.md is never a test file" 0 $?
|
|
560
|
+
|
|
561
|
+
# Bare "test.js" (no dot-test-dot, this project's own eval fixtures' shape) still counts as a test file.
|
|
562
|
+
printf 'assert.strictEqual(a, 1)\nassert.strictEqual(b, 2)\n' > "$old"
|
|
563
|
+
printf 'assert.strictEqual(a, 1)\n' > "$new"
|
|
564
|
+
out=$(bash "$RUBRIC" --format copilot --file test.js --old "$old" --new "$new")
|
|
565
|
+
[[ "$out" == *'"deny"'* ]]; check "rubric: bare test.js still detects removed assertion" 0 $?
|
|
566
|
+
|
|
567
|
+
# Work-tree-relative test directories: what every hook actually passes. A
|
|
568
|
+
# leading-slash-only pattern waved all three of these straight through.
|
|
569
|
+
for relpath in tests/login.js test/helpers.js __tests__/setup.js; do
|
|
570
|
+
out=$(bash "$RUBRIC" --format copilot --file "$relpath" --old "$old" --new "$new")
|
|
571
|
+
[[ "$out" == *'"deny"'* ]]; check "rubric: relative $relpath is gated" 0 $?
|
|
572
|
+
done
|
|
573
|
+
|
|
574
|
+
# The neighbouring word that must stay allowed: DevOS's own devos/specs/ tree.
|
|
575
|
+
out=$(bash "$RUBRIC" --format copilot --file specs/login.md --old "$old" --new "$new")
|
|
576
|
+
[[ "$out" == '{"permissionDecision":"allow"}' ]]; check "rubric: relative specs/ is not a test dir" 0 $?
|
|
577
|
+
|
|
578
|
+
# No --format: the default exit-code contract, used by Claude Code's own
|
|
579
|
+
# wrapper (test-integrity-hook-claude.sh) and by any future adapter with a
|
|
580
|
+
# plain allow/deny hook.
|
|
581
|
+
printf 'it("works", () => { expect(a).toBe(1) })\n' > "$old"
|
|
582
|
+
printf 'it.skip("works", () => { expect(a).toBe(1) })\n' > "$new"
|
|
583
|
+
bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
|
|
584
|
+
check "rubric: no --format, added .skip denies (exit 2)" 2 $?
|
|
585
|
+
|
|
586
|
+
# A value-bearing option with nothing after it is a malformed invocation, not a
|
|
587
|
+
# policy decision. Reading $2 anyway died under `set -u` with exit 1, which
|
|
588
|
+
# run-gate.sh forwards to the harness as a block carrying a shell error instead
|
|
589
|
+
# of a reason, so a typo in a hook's arguments bricked every edit.
|
|
590
|
+
for opt in --format --file --old --new; do
|
|
591
|
+
bash "$RUBRIC" "$opt" >/dev/null 2>&1 </dev/null
|
|
592
|
+
check "rubric: valueless $opt allows rather than dying" 0 $?
|
|
593
|
+
done
|
|
594
|
+
[[ "$(bash "$RUBRIC" --format copilot 2>/dev/null)" == '{"permissionDecision":"allow"}' ]]
|
|
595
|
+
check "rubric: a valueless option still emits the adapter's allow decision" 0 $?
|
|
596
|
+
bash "$RUBRIC" --write --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
|
|
597
|
+
check "rubric: a well-formed --write invocation still denies (exit 2)" 2 $?
|
|
598
|
+
|
|
599
|
+
printf 'expect(a).toBe(1)\n' > "$old"
|
|
600
|
+
printf 'expect(a).toBe(1)\nexpect(b).toBe(2)\n' > "$new"
|
|
601
|
+
bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new"
|
|
602
|
+
check "rubric: no --format, added assertion allows (exit 0)" 0 $?
|
|
603
|
+
|
|
604
|
+
# --- implementation output boundary -----------------------------------------
|
|
605
|
+
# SC IDs and framework metadata belong in devos/ artifacts. Project code and
|
|
606
|
+
# tests use behavior-facing names and carry no trace of the process that wrote
|
|
607
|
+
# them. The edit gate applies this to source as well as tests.
|
|
608
|
+
printf "it('SC-3: rejects an expired token', () => {})\n" > "$new"
|
|
609
|
+
bash "$RUBRIC" --file src/auth.test.ts --old "$tmp/absent.txt" --new "$new" 2>/dev/null
|
|
610
|
+
check "boundary: SC ID in a new test name denies" 2 $?
|
|
611
|
+
|
|
612
|
+
printf 'def test_sc_3_rejects_expired_token():\n pass\n' > "$new"
|
|
613
|
+
bash "$RUBRIC" --file tests/auth_test.py --old "$tmp/absent.txt" --new "$new" 2>/dev/null
|
|
614
|
+
check "boundary: SC ID in a snake-case test name denies" 2 $?
|
|
615
|
+
|
|
616
|
+
printf 'it("returns 401 for an expired token", () => {})\n' > "$new"
|
|
617
|
+
bash "$RUBRIC" --file src/auth.test.ts --old "$tmp/absent.txt" --new "$new" 2>/dev/null
|
|
618
|
+
check "boundary: natural behavior test name allows" 0 $?
|
|
619
|
+
|
|
620
|
+
printf '// Implemented by DevOS\n' > "$new"
|
|
621
|
+
bash "$RUBRIC" --file src/auth.ts --old "$tmp/absent.txt" --new "$new" 2>/dev/null
|
|
622
|
+
check "boundary: framework attribution in source denies" 2 $?
|
|
623
|
+
|
|
624
|
+
printf 'The contract is devos-contract: 1\n' > "$new"
|
|
625
|
+
bash "$RUBRIC" --file src/contract.ts --old "$tmp/absent.txt" --new "$new" 2>/dev/null
|
|
626
|
+
check "boundary: framework contract marker in a string denies" 2 $?
|
|
627
|
+
|
|
628
|
+
printf 'Scenario SC-3: rejects an expired token\n' > "$new"
|
|
629
|
+
bash "$RUBRIC" --file devos/specs/auth/spec.md --old "$tmp/absent.txt" --new "$new" 2>/dev/null
|
|
630
|
+
check "boundary: devos artifact keeps internal trace metadata" 0 $?
|
|
631
|
+
|
|
632
|
+
mkdir -p "$tmp/devos/src"
|
|
633
|
+
printf '// Implemented by DevOS\n' > "$tmp/devos/src/trace.ts"
|
|
634
|
+
bash "$BOUNDARY" --scan --file devos/src/trace.ts 2>/dev/null
|
|
635
|
+
check "boundary: code under devos keeps internal trace metadata" 0 $?
|
|
636
|
+
|
|
637
|
+
mkdir -p "$tmp/plugins/devos"
|
|
638
|
+
printf '// Implemented by DevOS\n' > "$tmp/plugins/devos/trace.ts"
|
|
639
|
+
bash "$BOUNDARY" --scan --file plugins/devos/trace.ts 2>/dev/null
|
|
640
|
+
check "boundary: a project plugins/devos directory is still project code" 2 $?
|
|
641
|
+
|
|
642
|
+
mkdir -p "$tmp/src"
|
|
643
|
+
printf 'const value = 1\n' > "$old"
|
|
644
|
+
printf '// Implemented by DevOS\nconst value = 1\n' > "$new"
|
|
645
|
+
bash "$BOUNDARY" --file devos/../src/traversal.ts --old "$old" --new "$new" 2>/dev/null
|
|
646
|
+
check "boundary: traversal cannot disguise project code as a devos artifact" 2 $?
|
|
647
|
+
|
|
648
|
+
printf 'Project tools may mention DevOS in onboarding docs.\n' > "$new"
|
|
649
|
+
bash "$RUBRIC" --file AGENTS.md --old "$tmp/absent.txt" --new "$new" 2>/dev/null
|
|
650
|
+
check "boundary: non-code onboarding file is outside this gate" 0 $?
|
|
651
|
+
|
|
652
|
+
printf '// Implemented by DevOS\nconst status = 401\n' > "$old"
|
|
653
|
+
printf '// Implemented by DevOS\nconst status = 401\nconst role = "user"\n' > "$new"
|
|
654
|
+
bash "$RUBRIC" --file src/auth.ts --old "$old" --new "$new" 2>/dev/null
|
|
655
|
+
check "boundary: unrelated edit around pre-existing metadata allows" 0 $?
|
|
656
|
+
|
|
657
|
+
mkdir -p "$tmp/src"
|
|
658
|
+
printf '// Implemented by DevOS\nconst status = 401\n' > "$tmp/src/auth.ts"
|
|
659
|
+
bash "$BOUNDARY" --scan --file src/auth.ts 2>/dev/null
|
|
660
|
+
check "boundary: scan mode rejects framework metadata already in source" 2 $?
|
|
661
|
+
|
|
662
|
+
# --- run-gate.sh (claude test-integrity) -----------------------------------
|
|
663
|
+
# Claude's test-integrity gate is now the single dispatcher: run-gate.sh
|
|
664
|
+
# --agent claude test-integrity. The old per-tool shim is gone, so a
|
|
665
|
+
# behavioral regression here surfaces as a changed decision below.
|
|
666
|
+
claude_ti() { bash "$(dirname "$DOD")/run-gate.sh" --agent claude test-integrity; }
|
|
667
|
+
CLAUDE_HOOK="$(dirname "$DOD")/run-gate.sh"
|
|
668
|
+
|
|
669
|
+
payload=$(jq -n --arg f src/main.test.ts --arg c 'it("works", () => { expect(a).toBe(1) })' '{tool_input:{file_path:$f,content:$c}}')
|
|
670
|
+
echo "$payload" | claude_ti
|
|
671
|
+
check "claude hook: Write adding an assertion allows" 0 $?
|
|
672
|
+
|
|
673
|
+
payload=$(jq -n --arg f src/main.test.ts --arg c 'it.skip("works", () => { expect(a).toBe(1) })' '{tool_input:{file_path:$f,content:$c}}')
|
|
674
|
+
echo "$payload" | claude_ti 2>/dev/null
|
|
675
|
+
check "claude hook: Write adding .skip denies" 2 $?
|
|
676
|
+
|
|
677
|
+
mkdir -p "$tmp/src"
|
|
678
|
+
printf 'it("works", () => { expect(a).toBe(1) })\n' > "$tmp/src/write.test.ts"
|
|
679
|
+
payload=$(jq -n --arg f "$tmp/src/write.test.ts" --arg c 'it.skip("works", () => { expect(a).toBe(1) })' '{tool_input:{file_path:$f,content:$c}}')
|
|
680
|
+
echo "$payload" | claude_ti 2>/dev/null
|
|
681
|
+
check "claude hook: Write to an existing test file adding .skip denies" 2 $?
|
|
682
|
+
|
|
683
|
+
payload=$(jq -n --arg f src/main.ts --arg c 'const x = 2' '{tool_input:{file_path:$f,content:$c}}')
|
|
684
|
+
echo "$payload" | claude_ti
|
|
685
|
+
check "claude hook: non-test file allows" 0 $?
|
|
686
|
+
|
|
687
|
+
payload=$(jq -n --arg f src/auth.ts --arg c '// Implemented by DevOS' '{tool_input:{file_path:$f,content:$c}}')
|
|
688
|
+
echo "$payload" | claude_ti 2>/dev/null
|
|
689
|
+
check "claude hook: process metadata in source denies" 2 $?
|
|
690
|
+
|
|
691
|
+
for tool in copilot codex; do
|
|
692
|
+
if [[ "$tool" == "copilot" ]]; then
|
|
693
|
+
payload=$(jq -n --arg f src/auth.ts --arg c '// Implemented by DevOS' '{toolArgs:{path:$f,content:$c}}')
|
|
694
|
+
else
|
|
695
|
+
payload=$(jq -n --arg f src/auth.ts --arg c '// Implemented by DevOS' '{tool_input:{path:$f,content:$c}}')
|
|
696
|
+
fi
|
|
697
|
+
out=$(printf '%s' "$payload" | bash "$CLAUDE_HOOK" --agent "$tool" test-integrity)
|
|
698
|
+
[[ "$out" == *'"permissionDecision":"deny"'* ]]
|
|
699
|
+
check "$tool hook: process metadata in source denies through the adapter contract" 0 $?
|
|
700
|
+
done
|
|
701
|
+
|
|
702
|
+
payload=$(jq -n --arg f src/auth.ts --arg c '// Implemented by DevOS' '{tool_input:{path:$f,content:$c}}')
|
|
703
|
+
out=$(printf '%s' "$payload" | bash "$CLAUDE_HOOK" --agent cursor test-integrity)
|
|
704
|
+
[[ "$out" == '{"permission":"deny"'* ]]
|
|
705
|
+
check "cursor hook: process metadata in source denies through the adapter contract" 0 $?
|
|
706
|
+
|
|
707
|
+
# --- payload-field contract -------------------------------------------------
|
|
708
|
+
# Every case above builds its payload from the same field names the hook reads,
|
|
709
|
+
# so the pair agrees with itself whichever names it uses. Rename a field on
|
|
710
|
+
# both sides and all of them still pass while the live gate reads `empty` from
|
|
711
|
+
# a real payload and exits 0: enforcement gone, suite green. Two checks break
|
|
712
|
+
# that circle.
|
|
713
|
+
#
|
|
714
|
+
# 1. Pin the exact jq paths the hook may read, against the published PreToolUse
|
|
715
|
+
# schema (code.claude.com/docs/en/hooks: tool_input carries the tool's own
|
|
716
|
+
# arguments, so Edit gives file_path/old_string/new_string and Write gives
|
|
717
|
+
# file_path/content). Editing the hook's field names now fails here until
|
|
718
|
+
# someone re-reads the doc and updates this list deliberately.
|
|
719
|
+
pinned=$(printf '%s\n' \
|
|
720
|
+
'.tool_input.content' \
|
|
721
|
+
'.tool_input.file_path' \
|
|
722
|
+
'.tool_input.new_string' \
|
|
723
|
+
'.tool_input.old_string' | sort | tr '\n' ' ')
|
|
724
|
+
actual=$(sed -n '/^claude)/,/;;/p' "$CLAUDE_HOOK" | grep -oE "TI_(FILE|CONTENT|NEW|OLD)='\.tool_input\.[a-z_]+" | sed -E "s/^TI_[A-Z]+='//" | sort -u | tr '\n' ' ')
|
|
725
|
+
[[ "$actual" == "$pinned" ]]
|
|
726
|
+
check "claude hook reads exactly the pinned PreToolUse fields (got: $actual)" 0 $?
|
|
727
|
+
|
|
728
|
+
# 2. A full payload as the harness actually sends it, every documented
|
|
729
|
+
# top-level field present, not the four-key stub the cases above use. Catches
|
|
730
|
+
# a hook that only works on the shape its own tests hand it.
|
|
731
|
+
payload='{"session_id":"abc123","prompt_id":"550e8400-e29b-41d4-a716-446655440000","transcript_path":"/tmp/devos-nonexistent.jsonl","cwd":"/repo","permission_mode":"default","hook_event_name":"PreToolUse","tool_name":"Edit","tool_use_id":"toolu_01ABC","tool_input":{"file_path":"src/main.test.ts","old_string":"expect(a).toBe(1)\nexpect(b).toBe(2)","new_string":"expect(a).toBe(1)"}}'
|
|
732
|
+
printf '%s' "$payload" | bash "$CLAUDE_HOOK" --agent claude test-integrity 2>/dev/null
|
|
733
|
+
check "claude hook: full documented PreToolUse payload still denies" 2 $?
|
|
734
|
+
|
|
735
|
+
# --- false-positive regressions: the gate must not block legitimate work ----
|
|
736
|
+
# The rubric used to set-difference assertion LINES, so any edit that changed
|
|
737
|
+
# the text of a line without removing an assertion was denied: renames,
|
|
738
|
+
# expected-value fixes, and every edit to a file that already had a .skip.
|
|
739
|
+
# 69 tests passed while that was true, because none of them covered it. A gate
|
|
740
|
+
# that blocks legitimate work trains the engineer to bypass it, which is worse
|
|
741
|
+
# than no gate. These pin the counting contract.
|
|
742
|
+
|
|
743
|
+
printf 'const foo = calc(1)\nexpect(foo).toBe(2)\n' > "$old"
|
|
744
|
+
printf 'const result = calc(1)\nexpect(result).toBe(2)\n' > "$new"
|
|
745
|
+
bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
|
|
746
|
+
check "rubric: rename inside an assertion allows" 0 $?
|
|
747
|
+
|
|
748
|
+
printf 'expect(foo).toBe(2)\n' > "$old"
|
|
749
|
+
printf 'expect(foo).toBe(3)\n' > "$new"
|
|
750
|
+
bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
|
|
751
|
+
check "rubric: expected-value fix allows" 0 $?
|
|
752
|
+
|
|
753
|
+
printf 'describe.skip("legacy", () => {})\nexpect(a).toBe(1)\n' > "$old"
|
|
754
|
+
printf 'describe.skip("legacy", () => {})\nexpect(b).toBe(1)\n' > "$new"
|
|
755
|
+
bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
|
|
756
|
+
check "rubric: pre-existing .skip, unrelated edit allows" 0 $?
|
|
757
|
+
|
|
758
|
+
printf 'expect(o.a).toBe(1)\nexpect(o.b).toBe(2)\nexpect(o.c).toBe(3)\n' > "$old"
|
|
759
|
+
printf 'expect(o).toMatchObject({a:1,b:2,c:3})\n' > "$new"
|
|
760
|
+
bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
|
|
761
|
+
check "rubric: genuine consolidation allows" 0 $?
|
|
762
|
+
|
|
763
|
+
# Table-driven consolidation: N copy-pasted `it` blocks collapsed into one
|
|
764
|
+
# it.each table legitimately drops the raw match count (the expect line is
|
|
765
|
+
# now written once, not once per case).
|
|
766
|
+
printf 'it("f(1)", () => { expect(f(1)).toBe(2) })\nit("f(2)", () => { expect(f(2)).toBe(4) })\n' > "$old"
|
|
767
|
+
printf 'it.each([[1,2],[2,4]])("f(%%d)", (a, b) => { expect(f(a)).toBe(b) })\n' > "$new"
|
|
768
|
+
bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
|
|
769
|
+
check "rubric: consolidation into it.each allows" 0 $?
|
|
770
|
+
|
|
771
|
+
# ...but a pre-existing it.each is not a licence to delete a case from it,
|
|
772
|
+
# same rule as pre-existing toMatchObject above.
|
|
773
|
+
printf 'it.each([[1,2]])("f(%%d)", (a, b) => { expect(f(a)).toBe(b) })\nexpect(g()).toBe(3)\n' > "$old"
|
|
774
|
+
printf 'it.each([[1,2]])("f(%%d)", (a, b) => { expect(f(a)).toBe(b) })\n' > "$new"
|
|
775
|
+
bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
|
|
776
|
+
check "rubric: pre-existing it.each does not excuse deletion" 2 $?
|
|
777
|
+
|
|
778
|
+
# ...but consolidation markers already present in the OLD content are not a
|
|
779
|
+
# licence to delete assertions in the new one.
|
|
780
|
+
printf 'expect(o).toMatchObject({a:1})\nexpect(p).toBe(2)\nexpect(q).toBe(3)\n' > "$old"
|
|
781
|
+
printf 'expect(o).toMatchObject({a:1})\n' > "$new"
|
|
782
|
+
bash "$RUBRIC" --file src/main.test.ts --old "$old" --new "$new" 2>/dev/null
|
|
783
|
+
check "rubric: pre-existing toMatchObject does not excuse deletion" 2 $?
|
|
784
|
+
|
|
785
|
+
payload=$(printf '{"tool_input":{"file_path":"src/main.test.ts","old_string":"expect(foo).toBe(2)","new_string":"expect(renamed).toBe(2)"}}')
|
|
786
|
+
echo "$payload" | bash "$CLAUDE_HOOK" --agent claude test-integrity 2>/dev/null
|
|
787
|
+
check "claude hook: Edit renaming inside an assertion allows" 0 $?
|
|
788
|
+
|
|
789
|
+
# --- hooks fail open on their own errors, never on policy -------------------
|
|
790
|
+
# A hook may exit non-zero only for a deliberate policy decision. Its own
|
|
791
|
+
# internal failure (garbage stdin, unreachable rubric, missing transcript)
|
|
792
|
+
# must allow: a broken gate must not brick every edit in the session.
|
|
793
|
+
#
|
|
794
|
+
# Discovered, not enumerated. Listing hooks by name meant a newly added one
|
|
795
|
+
# silently had no fail-open coverage until someone remembered to add it — and
|
|
796
|
+
# the one hook that got it wrong (check-handoff.sh, which fell THROUGH to
|
|
797
|
+
# blocking on its own arithmetic error) was exactly the one nobody had listed.
|
|
798
|
+
# Every stdin-reading hook under the plugin and the adapters is swept here, so
|
|
799
|
+
# the next one inherits the check for free. The name pattern is deliberately
|
|
800
|
+
# wide (any check-*.sh or *hook*.sh); membership is decided by whether the
|
|
801
|
+
# script reads a payload from stdin, not by what it is called.
|
|
802
|
+
repo_for_hooks="$(cd "$(dirname "$DOD")/../../../.." && pwd)"
|
|
803
|
+
hook_count=0
|
|
804
|
+
while IFS= read -r hook; do
|
|
805
|
+
[[ -f "$hook" ]] || continue
|
|
806
|
+
# Only hooks that read a payload from stdin; check-dod-core.sh and the
|
|
807
|
+
# rubric take argv and are covered by their own cases above.
|
|
808
|
+
grep -q 'input=$(cat' "$hook" || continue
|
|
809
|
+
hook_count=$((hook_count + 1))
|
|
810
|
+
hname=$(basename "$hook")
|
|
811
|
+
for bad_input in 'garbage not json' '' '{}' '{"tool_input":{}}'; do
|
|
812
|
+
printf '%s' "$bad_input" | bash "$hook" >/dev/null 2>&1
|
|
813
|
+
check "$hname: fails open on [${bad_input:-<empty>}]" 0 $?
|
|
814
|
+
done
|
|
815
|
+
done < <(find "$repo_for_hooks/plugins" \
|
|
816
|
+
\( -name '*hook*.sh' -o -name 'check-*.sh' -o -name 'run-gate.sh' \) 2>/dev/null | sort)
|
|
817
|
+
|
|
818
|
+
# The sweep is only meaningful if it actually found hooks. A find that matches
|
|
819
|
+
# nothing would otherwise report a silent pass.
|
|
820
|
+
(( hook_count >= 3 ))
|
|
821
|
+
check "fail-open sweep discovered the hooks (found $hook_count, expect >=5)" 0 $?
|
|
822
|
+
|
|
823
|
+
# check-handoff.sh takes a project root on argv rather than stdin, so the
|
|
824
|
+
# sweep above skips it. Its own no-op cases are covered below; this pins the
|
|
825
|
+
# specific regression: an unreadable root must not reach the blocking path.
|
|
826
|
+
bash "$(dirname "$DOD")/check-handoff.sh" /nonexistent-root-xyz >/dev/null 2>&1
|
|
827
|
+
check "check-handoff.sh: fails open on an unreadable project root" 0 $?
|
|
828
|
+
|
|
829
|
+
# An unreachable rubric (moved clone, broken adapter symlink) must allow.
|
|
830
|
+
# The DoD shim always guarded this; the test-integrity shim did not and
|
|
831
|
+
# exited 127 with no decision emitted.
|
|
832
|
+
repo_root="$(cd "$(dirname "$DOD")/../../../.." && pwd)"
|
|
833
|
+
cp "$CLAUDE_HOOK" "$tmp/broken-shim.sh"
|
|
834
|
+
sed -i.bak 's#^GATES=.*#GATES="/nonexistent/gates"#' "$tmp/broken-shim.sh"
|
|
835
|
+
out=$(echo '{"tool_input":{"path":"a.test.js","content":"x"}}' | bash "$tmp/broken-shim.sh" --agent copilot test-integrity 2>/dev/null)
|
|
836
|
+
[[ "$out" == '{"permissionDecision":"allow"}' ]]; check "shim: unreachable rubric allows, emits a decision" 0 $?
|
|
837
|
+
|
|
838
|
+
# --- check-handoff.sh ------------
|
|
839
|
+
# Solo regression is the important half: `grep -c ... || echo 0` printed
|
|
840
|
+
# "0\n0" on zero matches, making the arithmetic test error out and fall
|
|
841
|
+
# THROUGH to blocking. And the template documents its own format in a fenced
|
|
842
|
+
# block holding a literal `- @name: role:` line, so one real member counted
|
|
843
|
+
# as two and armed the team gate for a solo user.
|
|
844
|
+
#
|
|
845
|
+
# The rebuild replaced a bare `grep '^## Handoff'` presence check with
|
|
846
|
+
# the gate docs/team-workflow.md had been describing all along: phase detected
|
|
847
|
+
# from artifacts, the phase's own artifact named by a handoff line, every named
|
|
848
|
+
# artifact existing on disk, every role resolving to a member. Every state
|
|
849
|
+
# below is run against a real fixture project, never asserted from reading.
|
|
850
|
+
HANDOFF="$(dirname "$DOD")/check-handoff.sh"
|
|
851
|
+
hp="$tmp/handoff-proj"
|
|
852
|
+
S="$hp/devos/specs/2026-07-01-alpha"
|
|
853
|
+
mkdir -p "$hp/devos/team" "$S"
|
|
854
|
+
hrun() { bash "$HANDOFF" "$hp" </dev/null >/dev/null 2>&1; }
|
|
855
|
+
herr() { bash "$HANDOFF" "$hp" </dev/null 2>&1 >/dev/null; }
|
|
856
|
+
|
|
857
|
+
printf '# spec\n' > "$S/spec-functional.md"
|
|
858
|
+
hrun; check "handoff: absent members.md no-ops" 0 $?
|
|
859
|
+
|
|
860
|
+
printf '## Format\n\n```\n- @name: role: po|dev, skills: [spec]\n```\n\n## Members\n' > "$hp/devos/team/members.md"
|
|
861
|
+
hrun; check "handoff: template-only members.md no-ops" 0 $?
|
|
862
|
+
|
|
863
|
+
printf -- '- @alice: role: dev, skills: [spec]\n' >> "$hp/devos/team/members.md"
|
|
864
|
+
hrun; check "handoff: single real member no-ops" 0 $?
|
|
865
|
+
|
|
866
|
+
# A role outside po|architect|dev|qa does not count as a member: a typo in the
|
|
867
|
+
# team file must fall back to solo, never arm a gate naming a role that isn't real.
|
|
868
|
+
printf -- '- @typo: role: developer, skills: [spec]\n' >> "$hp/devos/team/members.md"
|
|
869
|
+
hrun; check "handoff: unrecognized role does not arm the gate" 0 $?
|
|
870
|
+
|
|
871
|
+
printf -- '- @alice: role: dev, skills: [spec]\n- @bob: role: qa, skills: [verify]\n' > "$hp/devos/team/members.md"
|
|
872
|
+
hrun; check "handoff: team, requirements phase, no section, blocks" 2 $?
|
|
873
|
+
[[ "$(herr)" == *"no ## Handoff section"* ]]
|
|
874
|
+
check "handoff: block reason goes to stderr, not stdout" 0 $?
|
|
875
|
+
|
|
876
|
+
# Requirements phase: the section lives in spec-functional.md until a plan exists.
|
|
877
|
+
printf '# spec\n\n## Handoff\n\n- @po → @dev | spec-functional.md | Requirements confirmed\n' > "$S/spec-functional.md"
|
|
878
|
+
hrun; check "handoff: unknown role in a line blocks (@po is not a member)" 2 $?
|
|
879
|
+
[[ "$(herr)" == *"not a member or role"* ]]; check "handoff: unknown-role reason names the problem" 0 $?
|
|
880
|
+
|
|
881
|
+
printf '# spec\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | Requirements confirmed\n' > "$S/spec-functional.md"
|
|
882
|
+
hrun; check "handoff: requirements phase satisfied by a line naming spec-functional.md" 0 $?
|
|
883
|
+
|
|
884
|
+
# A handoff may not promise an artifact that does not exist (the DoD gate's
|
|
885
|
+
# rule, applied to handoffs).
|
|
886
|
+
printf '# spec\n\n## Handoff\n\n- @dev → @qa | verifications/verification.md | ok\n' > "$S/spec-functional.md"
|
|
887
|
+
hrun; check "handoff: line naming a nonexistent artifact blocks" 2 $?
|
|
888
|
+
[[ "$(herr)" == *"does not exist"* ]]; check "handoff: missing-artifact reason names the path" 0 $?
|
|
889
|
+
|
|
890
|
+
printf 'outside\n' > "$tmp/handoff-outside.md"
|
|
891
|
+
printf '# spec\n\n## Handoff\n\n- @dev → @qa | ../../../../handoff-outside.md | ok\n' > "$S/spec-functional.md"
|
|
892
|
+
hrun; check "handoff: traversal artifact outside the project blocks" 2 $?
|
|
893
|
+
[[ "$(herr)" == *"outside the project root"* ]]; check "handoff: outside-artifact reason names the boundary" 0 $?
|
|
894
|
+
|
|
895
|
+
ln -s "$tmp/handoff-outside.md" "$hp/linked-outside.md"
|
|
896
|
+
printf '# spec\n\n## Handoff\n\n- @dev → @qa | linked-outside.md | ok\n' > "$S/spec-functional.md"
|
|
897
|
+
hrun; check "handoff: symlinked artifact outside the project blocks" 2 $?
|
|
898
|
+
rm -f "$hp/linked-outside.md" "$tmp/handoff-outside.md"
|
|
899
|
+
|
|
900
|
+
printf '# spec\n\n## Handoff\n\n- @dev @qa | spec-functional.md | ok\n' > "$S/spec-functional.md"
|
|
901
|
+
hrun; check "handoff: line with no → is not a transition, blocks" 2 $?
|
|
902
|
+
|
|
903
|
+
printf '# spec\n\n## Handoff\n\n- @dev → @qa | spec-functional.md\n' > "$S/spec-functional.md"
|
|
904
|
+
hrun; check "handoff: line missing its acceptance field blocks" 2 $?
|
|
905
|
+
|
|
906
|
+
# Planning phase: spec.md exists, so the section moves there and the
|
|
907
|
+
# requirements line alone no longer satisfies the gate.
|
|
908
|
+
printf '# spec\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | ok\n' > "$S/spec-functional.md"
|
|
909
|
+
printf '# plan\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | ok\n' > "$S/spec.md"
|
|
910
|
+
hrun; check "handoff: planning phase, only the stale requirements line, blocks" 2 $?
|
|
911
|
+
[[ "$(herr)" == *"produced spec.md"* ]]; check "handoff: phase reason names the artifact produced" 0 $?
|
|
912
|
+
|
|
913
|
+
printf -- '- @dev → @qa | spec.md | Plan approved\n' >> "$S/spec.md"
|
|
914
|
+
hrun; check "handoff: planning phase satisfied" 0 $?
|
|
915
|
+
|
|
916
|
+
# A textual prefix is not the phase artifact. The misleading file exists, so
|
|
917
|
+
# this isolates the identity check from the missing-file check.
|
|
918
|
+
printf '# plan\n\n## Handoff\n\n- @dev → @qa | not-spec.md | misleading\n' > "$S/spec.md"
|
|
919
|
+
printf 'not the plan\n' > "$S/not-spec.md"
|
|
920
|
+
hrun; check "handoff: phase artifact matching is exact" 2 $?
|
|
921
|
+
[[ "$(herr)" == *"produced spec.md"* ]]; check "handoff: exact-artifact block names the phase requirement" 0 $?
|
|
922
|
+
rm -f "$S/not-spec.md"
|
|
923
|
+
printf '# plan\n\n## Handoff\n\n- @dev → @qa | spec.md | Plan approved\n' > "$S/spec.md"
|
|
924
|
+
|
|
925
|
+
# Verification phase: the report exists, so its own handoff is now required.
|
|
926
|
+
mkdir -p "$S/verifications"
|
|
927
|
+
printf 'Completeness: CLEAN\nQuality: PASS\n' > "$S/verifications/verification.md"
|
|
928
|
+
hrun; check "handoff: verification phase, no line for the report, blocks" 2 $?
|
|
929
|
+
|
|
930
|
+
printf -- '- @qa → @dev | verifications/verification.md | Issues found\n' >> "$S/spec.md"
|
|
931
|
+
hrun; check "handoff: verification phase satisfied" 0 $?
|
|
932
|
+
|
|
933
|
+
# Loop guard: a stop-gate that keeps blocking a turn the model cannot fix would
|
|
934
|
+
# block every stop forever. The harness says it already blocked once; honor it.
|
|
935
|
+
printf '# plan\n' > "$S/spec.md"
|
|
936
|
+
hrun; check "handoff: control case for the loop guard blocks without a payload" 2 $?
|
|
937
|
+
for payload in '{"stop_hook_active":true}' '{"stopHookActive":true}' '{"loop_count":1}' 'garbage not json'; do
|
|
938
|
+
printf '%s' "$payload" | bash "$HANDOFF" "$hp" >/dev/null 2>&1
|
|
939
|
+
check "handoff: allows on [$payload]" 0 $?
|
|
940
|
+
done
|
|
941
|
+
printf '{"stop_hook_active":false}' | bash "$HANDOFF" "$hp" >/dev/null 2>&1
|
|
942
|
+
check "handoff: still blocks when the payload says it has not blocked yet" 2 $?
|
|
943
|
+
|
|
944
|
+
# An archived spec is finished work: it must not gate anything.
|
|
945
|
+
rm -rf "$S"
|
|
946
|
+
mkdir -p "$hp/devos/specs/archive/2026-01-01-old"
|
|
947
|
+
printf '# old\n' > "$hp/devos/specs/archive/2026-01-01-old/spec.md"
|
|
948
|
+
hrun; check "handoff: only archived specs no-ops" 0 $?
|
|
949
|
+
|
|
950
|
+
bash "$HANDOFF" "$tmp/no-such-project-dir" </dev/null >/dev/null 2>&1
|
|
951
|
+
check "handoff: unreadable project root no-ops" 0 $?
|
|
952
|
+
|
|
953
|
+
# Two specs open at once is the normal team case, and newest-wins alone got it
|
|
954
|
+
# wrong: the older spec could reach any phase with no handoff recorded. The
|
|
955
|
+
# active spec now comes from the branch, which Phase
|
|
956
|
+
# B creates before the first edit.
|
|
957
|
+
hb="$tmp/handoff-branch"
|
|
958
|
+
mkdir -p "$hb/devos/team"
|
|
959
|
+
printf -- '- @alice: role: dev, skills: [spec]\n- @bob: role: qa, skills: [verify]\n' > "$hb/devos/team/members.md"
|
|
960
|
+
OLD="$hb/devos/specs/2026-07-01-older"
|
|
961
|
+
NEW="$hb/devos/specs/2026-07-09-newer"
|
|
962
|
+
mkdir -p "$OLD" "$NEW"
|
|
963
|
+
# The newer spec is fully compliant, so anything that blocks is about the older.
|
|
964
|
+
printf '# newer\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | ok\n' > "$NEW/spec-functional.md"
|
|
965
|
+
printf '# older\n' > "$OLD/spec-functional.md"
|
|
966
|
+
(cd "$hb" && git init -q . && git config user.email t@t && git config user.name t && git add -A && git commit -qm init)
|
|
967
|
+
hbrun() { (cd "$hb" && git checkout -q -B "$1" 2>/dev/null; bash "$HANDOFF" "$hb" </dev/null >/dev/null 2>&1); }
|
|
968
|
+
|
|
969
|
+
hbrun 2026-07-01-older
|
|
970
|
+
check "handoff: on a spec branch, gates that spec even when a newer one is clean" 2 $?
|
|
971
|
+
printf '# older\n\n## Handoff\n\n- @dev → @qa | spec-functional.md | ok\n' > "$OLD/spec-functional.md"
|
|
972
|
+
hbrun 2026-07-01-older
|
|
973
|
+
check "handoff: on a spec branch, a satisfied older spec passes" 0 $?
|
|
974
|
+
|
|
975
|
+
# Regression: everything not on a spec branch keeps the old newest-wins path.
|
|
976
|
+
printf '# older\n' > "$OLD/spec-functional.md"
|
|
977
|
+
hbrun some-feature-branch
|
|
978
|
+
check "handoff: off a spec branch, newest still wins (older stays ungated)" 0 $?
|
|
979
|
+
hbrun 2026-07-01-does-not-exist
|
|
980
|
+
check "handoff: a branch naming no spec folder falls back to newest" 0 $?
|
|
981
|
+
printf '# newer\n' > "$NEW/spec-functional.md"
|
|
982
|
+
hbrun some-feature-branch
|
|
983
|
+
check "handoff: off a spec branch, the newest spec is the one gated" 2 $?
|
|
984
|
+
|
|
985
|
+
# --- detect-archetype.py ------------------------------
|
|
986
|
+
# Success criterion 15 named three repos: React/Tailwind/Storybook, Terraform,
|
|
987
|
+
# and a Go CLI that must gain NO frontend sections. The first version detected
|
|
988
|
+
# none of the Terraform signals and none of the @scoped packages, because
|
|
989
|
+
# quoted globs never expand and `p.deps.@storybook/react` is a JS syntax error.
|
|
990
|
+
# Both classes are pinned here so they cannot silently die again.
|
|
991
|
+
ARCH="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/init/scripts/detect-archetype.py"
|
|
992
|
+
af="$tmp/arch"
|
|
993
|
+
|
|
994
|
+
mkdir -p "$af/react/src" "$af/react/.storybook"
|
|
995
|
+
cat > "$af/react/package.json" <<'J'
|
|
996
|
+
{"name":"web","dependencies":{"react":"^19","next":"^15","@radix-ui/react-dialog":"^1","@reduxjs/toolkit":"^2"},
|
|
997
|
+
"devDependencies":{"tailwindcss":"^4","@storybook/react":"^8"}}
|
|
998
|
+
J
|
|
999
|
+
touch "$af/react/src/Button.module.css"
|
|
1000
|
+
out=$(python3 "$ARCH" "$af/react")
|
|
1001
|
+
for tag in ui:react ui:next design:storybook design:radix style:tailwind style:css-modules state:redux; do
|
|
1002
|
+
[[ "$out" == *"\"$tag\""* ]]; check "archetype: React repo detects $tag" 0 $?
|
|
1003
|
+
done
|
|
1004
|
+
|
|
1005
|
+
# Terraform: glob-only detection, entirely dead before 0.8.4.
|
|
1006
|
+
mkdir -p "$af/tf/modules/vpc"
|
|
1007
|
+
printf 'resource "aws_s3_bucket" "b" {}\n' > "$af/tf/modules/vpc/main.tf"
|
|
1008
|
+
out=$(python3 "$ARCH" "$af/tf")
|
|
1009
|
+
[[ "$out" == *'"infra:terraform"'* ]]; check "archetype: Terraform repo detects infra:terraform" 0 $?
|
|
1010
|
+
|
|
1011
|
+
# Go CLI: the zero-cost half of criterion 15. No ui:/design:/style:/state: tag.
|
|
1012
|
+
mkdir -p "$af/gocli"
|
|
1013
|
+
printf 'module example.com/cli\n\ngo 1.23\n' > "$af/gocli/go.mod"
|
|
1014
|
+
printf 'all:\n\tgo build ./...\n' > "$af/gocli/Makefile"
|
|
1015
|
+
out=$(python3 "$ARCH" "$af/gocli")
|
|
1016
|
+
[[ "$out" == *'"cli:go"'* ]]; check "archetype: Go CLI detects cli:go" 0 $?
|
|
1017
|
+
|
|
1018
|
+
# The only table row whose glob names a directory is the GitHub Actions one,
|
|
1019
|
+
# and the glob reader used `find -name`, which matches a basename: the pattern
|
|
1020
|
+
# `.github/workflows/*.y*ml` could not match any repository, so a repo with
|
|
1021
|
+
# real workflows was never detected as ci:github-actions.
|
|
1022
|
+
mkdir -p "$af/gha/.github/workflows" "$af/gha/elsewhere/workflows"
|
|
1023
|
+
printf 'name: ci\n' > "$af/gha/.github/workflows/build.yml"
|
|
1024
|
+
out=$(python3 "$ARCH" "$af/gha")
|
|
1025
|
+
[[ "$out" == *'"ci:github-actions"'* ]]; check "archetype: a GitHub Actions workflow is detected" 0 $?
|
|
1026
|
+
# Widening the match must not turn every workflow-shaped file into the signal.
|
|
1027
|
+
printf 'name: ci\n' > "$af/gha/elsewhere/workflows/build.yml"
|
|
1028
|
+
out=$(python3 "$ARCH" "$af/gha/elsewhere")
|
|
1029
|
+
[[ "$out" != *'"ci:github-actions"'* && "$out" == *'"detected":false'* ]] \
|
|
1030
|
+
&& check "archetype: the same file outside .github is not the CI signal" 0 $? \
|
|
1031
|
+
|| { echo "FAIL archetype: a workflow outside .github was read as GitHub Actions: $out"; fail=1; }
|
|
1032
|
+
mkdir -p "$af/gha-root"
|
|
1033
|
+
printf 'name: ci\n' > "$af/gha-root/ci.yml"
|
|
1034
|
+
out=$(python3 "$ARCH" "$af/gha-root")
|
|
1035
|
+
[[ "$out" != *'"ci:github-actions"'* && "$out" == *'"detected":false'* ]] \
|
|
1036
|
+
&& check "archetype: a root-level workflow file is not the CI signal" 0 $? \
|
|
1037
|
+
|| { echo "FAIL archetype: a root-level ci.yml was read as GitHub Actions: $out"; fail=1; }
|
|
1038
|
+
|
|
1039
|
+
# The TABLE reader is `IFS='|' read -r tag kind arg arg2`, so a row with five
|
|
1040
|
+
# fields silently folds the fifth into arg2 and the rule tests a filename that
|
|
1041
|
+
# cannot exist. One row per library keeps every field count at four.
|
|
1042
|
+
for lib in click typer; do
|
|
1043
|
+
mkdir -p "$af/py-$lib"
|
|
1044
|
+
printf 'dependencies = ["%s"]\n' "$lib" > "$af/py-$lib/pyproject.toml"
|
|
1045
|
+
out=$(python3 "$ARCH" "$af/py-$lib")
|
|
1046
|
+
[[ "$out" == *'"cli:python"'* ]]; check "archetype: Python $lib CLI detects cli:python" 0 $?
|
|
1047
|
+
done
|
|
1048
|
+
[[ "$out" != *'"ui:'* && "$out" != *'"design:'* && "$out" != *'"style:'* && "$out" != *'"state:'* ]]
|
|
1049
|
+
check "archetype: Go CLI gains no frontend sections" 0 $?
|
|
1050
|
+
|
|
1051
|
+
mkdir -p "$af/bare"
|
|
1052
|
+
out=$(python3 "$ARCH" "$af/bare")
|
|
1053
|
+
[[ "$out" == '{"archetypes":[],"detected":false}' ]]; check "archetype: bare dir reports undetected" 0 $?
|
|
1054
|
+
|
|
1055
|
+
python3 "$ARCH" "$af/no-such-dir" >/dev/null 2>&1
|
|
1056
|
+
check "archetype: missing dir exits 1, does not hang" 1 $?
|
|
1057
|
+
|
|
1058
|
+
# No node on PATH must not change the answer: detection is pure POSIX tools.
|
|
1059
|
+
out=$(PATH="/usr/bin:/bin" python3 "$ARCH" "$af/react" 2>/dev/null)
|
|
1060
|
+
[[ "$out" == *'"design:storybook"'* ]]; check "archetype: works without node on PATH" 0 $?
|
|
1061
|
+
|
|
1062
|
+
# --- detect-archetype.py --facts -------------
|
|
1063
|
+
# The same stack at two different majors must produce two different fact
|
|
1064
|
+
# blocks: that difference is the entire value of the mode, so it is the test.
|
|
1065
|
+
# Both bugs pinned below were found by running these fixtures before the
|
|
1066
|
+
# roadmap box was ticked, not after.
|
|
1067
|
+
mkdir -p "$af/v18/node_modules/react" "$af/v18/node_modules/next"
|
|
1068
|
+
cat > "$af/v18/package.json" <<'J'
|
|
1069
|
+
{
|
|
1070
|
+
"dependencies": { "react": "^18.2.0", "next": "^14.1.0" },
|
|
1071
|
+
"devDependencies": { "tailwindcss": "^3.4.1", "storybook": "^8.0.0" },
|
|
1072
|
+
"scripts": {
|
|
1073
|
+
"dev": "next dev",
|
|
1074
|
+
"build": "next build",
|
|
1075
|
+
"typecheck": "tsc --noEmit",
|
|
1076
|
+
"test:a11y": "axe ./dist"
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
J
|
|
1080
|
+
echo '{"name":"react","version":"18.3.1"}' > "$af/v18/node_modules/react/package.json"
|
|
1081
|
+
echo '{"name":"next","version":"14.2.5"}' > "$af/v18/node_modules/next/package.json"
|
|
1082
|
+
out=$(python3 "$ARCH" --facts "$af/v18")
|
|
1083
|
+
[[ "$out" == *"| react | 18.3.1 | node_modules (installed) |"* ]]
|
|
1084
|
+
check "facts: installed version wins over the declared range" 0 $?
|
|
1085
|
+
[[ "$out" == *"| tailwindcss | ^3.4.1 |"* ]]
|
|
1086
|
+
check "facts: range read from a one-line object is the right key's value" 0 $?
|
|
1087
|
+
[[ "$out" == *"range, NOT resolved"* ]]
|
|
1088
|
+
check "facts: an unresolved range is labelled, never passed off as installed" 0 $?
|
|
1089
|
+
|
|
1090
|
+
# Same stack, next major, nothing installed. Single-line manifest, and a
|
|
1091
|
+
# `storybook` script whose value must never be reported as a version.
|
|
1092
|
+
mkdir -p "$af/v19"
|
|
1093
|
+
cat > "$af/v19/package.json" <<'J'
|
|
1094
|
+
{"dependencies":{"react":"^19.0.0","next":"^15.3.0","@mui/material":"^6.1.0"},
|
|
1095
|
+
"devDependencies":{"tailwindcss":"^4.0.9"},
|
|
1096
|
+
"scripts":{"build":"next build","storybook":"storybook dev"}}
|
|
1097
|
+
J
|
|
1098
|
+
out19=$(python3 "$ARCH" --facts "$af/v19")
|
|
1099
|
+
[[ "$out19" == *"| react | ^19.0.0 |"* && "$out19" == *"| tailwindcss | ^4.0.9 |"* ]]
|
|
1100
|
+
check "facts: second major reports its own versions" 0 $?
|
|
1101
|
+
[[ "$out19" == *"| @mui/material | ^6.1.0 |"* ]]
|
|
1102
|
+
check "facts: a scoped package resolves" 0 $?
|
|
1103
|
+
# Assert on the version table alone: `storybook` is a legitimate row in the
|
|
1104
|
+
# checks table below it, so a whole-output grep would pass no matter what.
|
|
1105
|
+
vtable19=${out19%%Declared checks*}
|
|
1106
|
+
[[ "$vtable19" != *"| storybook |"* ]]
|
|
1107
|
+
check "facts: a script named like a package is not reported as its version" 0 $?
|
|
1108
|
+
[[ "$out" != "$out19" ]]
|
|
1109
|
+
check "facts: two majors of one stack produce different blocks" 0 $?
|
|
1110
|
+
|
|
1111
|
+
# Declared checks: only the keys that prove something, `dev` excluded.
|
|
1112
|
+
[[ "$out" == *"npm run typecheck\` | tsc --noEmit |"* && "$out" == *"npm run test:a11y\` | axe ./dist |"* ]]
|
|
1113
|
+
check "facts: suffixed check scripts are declared" 0 $?
|
|
1114
|
+
[[ "$out" != *"run dev\` | next dev |"* ]]
|
|
1115
|
+
check "facts: a server-starting script is not a check" 0 $?
|
|
1116
|
+
|
|
1117
|
+
# Tier 41: exactly the keys that prove something, no triage. Eight scripts in,
|
|
1118
|
+
# six out: `dev` starts a server and `clean` deletes a directory; neither can
|
|
1119
|
+
# fail in a way that means the code is wrong.
|
|
1120
|
+
mkdir -p "$af/checks"
|
|
1121
|
+
cat > "$af/checks/package.json" <<'J'
|
|
1122
|
+
{"dependencies":{"react":"^19"},
|
|
1123
|
+
"scripts":{"dev":"vite","clean":"rm -rf dist","build":"vite build","lint":"eslint .",
|
|
1124
|
+
"typecheck":"tsc --noEmit","test":"vitest run","test:e2e":"playwright test",
|
|
1125
|
+
"test:a11y":"axe ./dist"}}
|
|
1126
|
+
J
|
|
1127
|
+
outc=$(python3 "$ARCH" --facts "$af/checks")
|
|
1128
|
+
n=$(printf '%s\n' "$outc" | sed -n '/| Source | Command | Runs |/,$p' | grep -c '^| package.json |')
|
|
1129
|
+
[[ "$n" == 6 ]]
|
|
1130
|
+
check "checks: six qualifying scripts of eight are declared" 0 $?
|
|
1131
|
+
for k in build lint typecheck test test:e2e test:a11y; do
|
|
1132
|
+
[[ "$outc" == *"run $k\` |"* ]]; check "checks: $k is declared" 0 $?
|
|
1133
|
+
done
|
|
1134
|
+
[[ "$outc" != *"run dev\` |"* && "$outc" != *"run clean\` |"* ]]
|
|
1135
|
+
check "checks: dev and clean are not proof of anything" 0 $?
|
|
1136
|
+
|
|
1137
|
+
# The check table decides what the verifier runs, so a name filter narrow
|
|
1138
|
+
# enough to drop a functional suite silently decides that suite never runs.
|
|
1139
|
+
# The lockfile names the runner: `npm run` on a pnpm repo is a command the
|
|
1140
|
+
# project does not use.
|
|
1141
|
+
mkdir -p "$af/wide"
|
|
1142
|
+
touch "$af/wide/pnpm-lock.yaml"
|
|
1143
|
+
cat > "$af/wide/package.json" <<'J'
|
|
1144
|
+
{"dependencies":{"react":"^19"},
|
|
1145
|
+
"scripts":{"test-func":"node ./scripts/test.mjs func","coverage-unit":"c8 mocha",
|
|
1146
|
+
"format:check":"prettier --check .","lint:check":"eslint .",
|
|
1147
|
+
"test:watch":"vitest","deploy-prod":"./k8s.mjs deploy"}}
|
|
1148
|
+
J
|
|
1149
|
+
outw=$(python3 "$ARCH" --facts "$af/wide")
|
|
1150
|
+
for k in test-func coverage-unit format:check lint:check; do
|
|
1151
|
+
[[ "$outw" == *"pnpm $k\` |"* ]]; check "checks: $k survives the name filter" 0 $?
|
|
1152
|
+
done
|
|
1153
|
+
[[ "$outw" != *"npm run "* ]]
|
|
1154
|
+
check "checks: the lockfile picks the package manager" 0 $?
|
|
1155
|
+
[[ "$outw" != *"test:watch"* ]]
|
|
1156
|
+
check "checks: a watch script is not a check (it never exits)" 0 $?
|
|
1157
|
+
[[ "$outw" != *"deploy-prod"* ]]
|
|
1158
|
+
check "checks: an operational script is still excluded" 0 $?
|
|
1159
|
+
|
|
1160
|
+
# Bun writes bun.lock, the text lockfile that has been its default since
|
|
1161
|
+
# v1.2. A repo carrying only that file was reported as npm, and the table is
|
|
1162
|
+
# what the verifier runs.
|
|
1163
|
+
mkdir -p "$af/bun-text" "$af/bun-binary"
|
|
1164
|
+
cat > "$af/bun-text/package.json" <<'J'
|
|
1165
|
+
{"scripts":{"test":"bun test","lint":"eslint ."}}
|
|
1166
|
+
J
|
|
1167
|
+
printf '{}\n' > "$af/bun-text/bun.lock"
|
|
1168
|
+
outbt=$(python3 "$ARCH" --facts "$af/bun-text")
|
|
1169
|
+
[[ "$outbt" == *'| `bun run test` |'* && "$outbt" != *"npm run"* ]]
|
|
1170
|
+
check "checks: the bun text lockfile picks bun run" 0 $?
|
|
1171
|
+
cp "$af/bun-text/package.json" "$af/bun-binary/package.json"
|
|
1172
|
+
printf 'binary\n' > "$af/bun-binary/bun.lockb"
|
|
1173
|
+
outbb=$(python3 "$ARCH" --facts "$af/bun-binary")
|
|
1174
|
+
[[ "$outbb" == *'| `bun run test` |'* ]]
|
|
1175
|
+
check "checks: the legacy bun binary lockfile still picks bun run" 0 $?
|
|
1176
|
+
|
|
1177
|
+
# A task runner is a manifest too. Reading package.json alone reported "no
|
|
1178
|
+
# declared checks" for every Make/just project, DevOS's own repo included.
|
|
1179
|
+
mkdir -p "$af/runner"
|
|
1180
|
+
cat > "$af/runner/justfile" <<'J'
|
|
1181
|
+
PLUGIN_JSONS := "a b"
|
|
1182
|
+
|
|
1183
|
+
default:
|
|
1184
|
+
@just --list
|
|
1185
|
+
|
|
1186
|
+
verify:
|
|
1187
|
+
./run-checks.sh
|
|
1188
|
+
|
|
1189
|
+
lint target="all":
|
|
1190
|
+
eslint {{target}}
|
|
1191
|
+
|
|
1192
|
+
deploy:
|
|
1193
|
+
./ship.sh
|
|
1194
|
+
J
|
|
1195
|
+
outr=$(python3 "$ARCH" --facts "$af/runner")
|
|
1196
|
+
[[ "$outr" == *'`just verify`'* && "$outr" == *'`just lint`'* ]]
|
|
1197
|
+
check "checks: justfile recipes are declared checks" 0 $?
|
|
1198
|
+
[[ "$outr" != *"just deploy"* && "$outr" != *"just default"* && "$outr" != *"PLUGIN_JSONS"* ]]
|
|
1199
|
+
check "checks: recipes that prove nothing, and assignments, stay out" 0 $?
|
|
1200
|
+
|
|
1201
|
+
mkdir -p "$af/mk"
|
|
1202
|
+
printf '.PHONY: test\ntest:\n\tgo test ./...\n\nrelease:\n\t./ship.sh\n' > "$af/mk/Makefile"
|
|
1203
|
+
outm=$(python3 "$ARCH" --facts "$af/mk")
|
|
1204
|
+
[[ "$outm" == *'`make test`'* && "$outm" != *"make release"* ]]
|
|
1205
|
+
check "checks: Makefile targets are declared checks" 0 $?
|
|
1206
|
+
|
|
1207
|
+
# Where a command runs is part of the command: a suite that only passes inside
|
|
1208
|
+
# the app container fails on the host for reasons the diff did not cause.
|
|
1209
|
+
mkdir -p "$af/ctx"
|
|
1210
|
+
printf 'services:\n app:\n build: .\n redis:\n image: redis\nvolumes:\n data:\n' > "$af/ctx/docker-compose.yml"
|
|
1211
|
+
outx=$(python3 "$ARCH" --facts "$af/ctx")
|
|
1212
|
+
[[ "$outx" == *"Execution context:"* && "$outx" == *"app, redis"* ]]
|
|
1213
|
+
check "checks: compose services are reported as the execution context" 0 $?
|
|
1214
|
+
[[ "$outx" != *"app, redis, data"* ]]
|
|
1215
|
+
check "checks: a sibling top-level key is not read as a service" 0 $?
|
|
1216
|
+
[[ "$(python3 "$ARCH" --facts "$af/mk")" != *"Execution context:"* ]]
|
|
1217
|
+
check "checks: no compose file means no context line" 0 $?
|
|
1218
|
+
|
|
1219
|
+
# Tier 42: the same `dev` script that cannot be a check is exactly what a
|
|
1220
|
+
# browser needs, so it surfaces as a preview candidate instead of vanishing.
|
|
1221
|
+
[[ "$outc" == *'Preview candidates: `dev` (`vite`)'* ]]
|
|
1222
|
+
check "preview: the dev script surfaces as a preview candidate" 0 $?
|
|
1223
|
+
out=$(python3 "$ARCH" --facts "$af/gocli")
|
|
1224
|
+
[[ "$out" == *"Preview candidates: none found"* ]]
|
|
1225
|
+
check "preview: a repo with nothing to serve says so" 0 $?
|
|
1226
|
+
|
|
1227
|
+
# Non-JS and empty repos degrade to an honest answer, never a guessed one.
|
|
1228
|
+
out=$(python3 "$ARCH" --facts "$af/gocli")
|
|
1229
|
+
[[ "$out" == *"| go (toolchain) | 1.23 | go.mod |"* ]]
|
|
1230
|
+
check "facts: go toolchain pin is read" 0 $?
|
|
1231
|
+
out=$(python3 "$ARCH" --facts "$af/bare")
|
|
1232
|
+
[[ "$out" == *"No resolvable versions found"* ]]
|
|
1233
|
+
check "facts: bare dir says so instead of inventing versions" 0 $?
|
|
1234
|
+
|
|
1235
|
+
# The default mode is untouched: every existing caller still gets JSON.
|
|
1236
|
+
out=$(python3 "$ARCH" "$af/v18")
|
|
1237
|
+
[[ "$out" == '{"archetypes":['* ]]
|
|
1238
|
+
check "facts: default mode still prints JSON" 0 $?
|
|
1239
|
+
|
|
1240
|
+
# --- validate-spec.sh -------------------------------------------------------
|
|
1241
|
+
VSPEC="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/implement/scripts/validate-spec.sh"
|
|
1242
|
+
|
|
1243
|
+
good="$tmp/devos/specs/2026-01-01-good"
|
|
1244
|
+
mkdir -p "$good"
|
|
1245
|
+
cat > "$good/spec-functional.md" <<'MD'
|
|
1246
|
+
# Good Feature
|
|
1247
|
+
## 1. Goal
|
|
1248
|
+
Users MUST be able to export their data.
|
|
1249
|
+
## 3. Functionality
|
|
1250
|
+
**Scenarios:**
|
|
1251
|
+
- Scenario SC-1: happy export / WHEN the user clicks export THEN a CSV downloads
|
|
1252
|
+
## 5. Out of Scope
|
|
1253
|
+
- PDF export
|
|
1254
|
+
MD
|
|
1255
|
+
bash "$VSPEC" "$good" >/dev/null; check "vspec: well-formed spec passes" 0 $?
|
|
1256
|
+
|
|
1257
|
+
historical="$tmp/devos/specs/archive/2026-01-01-historical"
|
|
1258
|
+
mkdir -p "$historical"
|
|
1259
|
+
cp "$good/spec-functional.md" "$historical/spec-functional.md"
|
|
1260
|
+
printf '# Historical capsule\n' > "$historical/spec.md"
|
|
1261
|
+
bash "$VSPEC" "$historical" >/dev/null; check "vspec: archived plan skips current plan gates" 0 $?
|
|
1262
|
+
|
|
1263
|
+
anchors="$tmp/devos/specs/2026-01-01-anchors"
|
|
1264
|
+
mkdir -p "$anchors/planning"
|
|
1265
|
+
cp "$good/spec-functional.md" "$anchors/spec-functional.md"
|
|
1266
|
+
cat > "$anchors/planning/notes.md" <<'MD'
|
|
1267
|
+
# Planning notes
|
|
1268
|
+
|
|
1269
|
+
## Explicit contract anchors
|
|
1270
|
+
- Function `exportUsers()` in `src/users.js`
|
|
1271
|
+
MD
|
|
1272
|
+
out=$(bash "$VSPEC" "$anchors" 2>&1); rc=$?
|
|
1273
|
+
grep -q "spec-functional.md drops explicit contract anchors from planning/notes.md" <<<"$out" && [[ $rc -eq 1 ]] \
|
|
1274
|
+
&& check "vspec: dropped explicit contract anchors fail with the gap named" 0 $? \
|
|
1275
|
+
|| { echo "FAIL vspec: dropped explicit contract anchors not caught"; echo "$out"; fail=1; }
|
|
1276
|
+
|
|
1277
|
+
cat "$good/spec-functional.md" > "$anchors/spec-functional.md"
|
|
1278
|
+
printf '\n## Appendix\n**Explicit contract anchors:** `None supplied`\n' >> "$anchors/spec-functional.md"
|
|
1279
|
+
out=$(bash "$VSPEC" "$anchors" 2>&1); rc=$?
|
|
1280
|
+
grep -q "spec-functional.md records None supplied despite explicit contract anchors" <<<"$out" && [[ $rc -eq 1 ]] \
|
|
1281
|
+
&& check "vspec: negated explicit contract anchors fail with the gap named" 0 $? \
|
|
1282
|
+
|| { echo "FAIL vspec: negated explicit contract anchors not caught"; echo "$out"; fail=1; }
|
|
1283
|
+
|
|
1284
|
+
cat "$good/spec-functional.md" > "$anchors/spec-functional.md"
|
|
1285
|
+
printf '\n## Appendix\n**Explicit contract anchors:** Function `exportUsers()` in `src/users.js`\n' >> "$anchors/spec-functional.md"
|
|
1286
|
+
bash "$VSPEC" "$anchors" >/dev/null; check "vspec: carried explicit contract anchors pass" 0 $?
|
|
1287
|
+
|
|
1288
|
+
cat "$good/spec-functional.md" > "$anchors/spec-functional.md"
|
|
1289
|
+
printf '\n## Appendix\n**Explicit contract anchors:** Function `exportUsers()` in `src/users.js`\n' >> "$anchors/spec-functional.md"
|
|
1290
|
+
cat > "$anchors/spec.md" <<'MD'
|
|
1291
|
+
## Local Craft
|
|
1292
|
+
None
|
|
1293
|
+
## Goal / Technical Context / Architecture / Technical Requirements / Existing Code to Leverage
|
|
1294
|
+
The implementation MUST call `exportUsers()`.
|
|
1295
|
+
## Design Convergence
|
|
1296
|
+
- **Direct MVP**: call the existing function
|
|
1297
|
+
- **Failed Items**: None
|
|
1298
|
+
- **Adopted Additions**: None
|
|
1299
|
+
- **Rejected Additions**: None
|
|
1300
|
+
## Code Map
|
|
1301
|
+
- `src/users.js`: update the export
|
|
1302
|
+
## Implementation Plan
|
|
1303
|
+
### Task Group 1: update export
|
|
1304
|
+
**Files:** `src/users.js`
|
|
1305
|
+
**Scenarios covered:** SC-1
|
|
1306
|
+
MD
|
|
1307
|
+
bash "$VSPEC" "$anchors" >/dev/null; check "vspec: plan carries explicit contract anchors" 0 $?
|
|
1308
|
+
|
|
1309
|
+
cat "$good/spec-functional.md" > "$anchors/spec-functional.md"
|
|
1310
|
+
printf '\n## Appendix\n**Explicit contract anchors:** Function `exportUsers()`\n' >> "$anchors/spec-functional.md"
|
|
1311
|
+
out=$(bash "$VSPEC" "$anchors" 2>&1); rc=$?
|
|
1312
|
+
grep -q "spec-functional.md drops explicit contract anchor: \`src/users.js\`" <<<"$out" && [[ $rc -eq 1 ]] \
|
|
1313
|
+
&& check "vspec: a dropped explicit contract token fails with the token named" 0 $? \
|
|
1314
|
+
|| { echo "FAIL vspec: a dropped explicit contract token not caught"; echo "$out"; fail=1; }
|
|
1315
|
+
|
|
1316
|
+
cat > "$anchors/spec.md" <<'MD'
|
|
1317
|
+
## Local Craft
|
|
1318
|
+
None
|
|
1319
|
+
## Goal / Technical Context / Architecture / Technical Requirements / Existing Code to Leverage
|
|
1320
|
+
The implementation MUST preserve the supplied behavior.
|
|
1321
|
+
## Design Convergence
|
|
1322
|
+
- **Direct MVP**: update the existing implementation
|
|
1323
|
+
- **Failed Items**: None
|
|
1324
|
+
- **Adopted Additions**: None
|
|
1325
|
+
- **Rejected Additions**: None
|
|
1326
|
+
## Code Map
|
|
1327
|
+
- update the implementation file
|
|
1328
|
+
## Implementation Plan
|
|
1329
|
+
### Task Group 1: update export
|
|
1330
|
+
**Files:** `src/users.js`
|
|
1331
|
+
**Scenarios covered:** SC-1
|
|
1332
|
+
MD
|
|
1333
|
+
out=$(bash "$VSPEC" "$anchors" 2>&1); rc=$?
|
|
1334
|
+
grep -q "spec.md drops explicit contract anchor from Technical Requirements: \`exportUsers()\`" <<<"$out" \
|
|
1335
|
+
&& grep -q "spec.md drops explicit contract anchor from Code Map: \`src/users.js\`" <<<"$out" \
|
|
1336
|
+
&& [[ $rc -eq 1 ]] \
|
|
1337
|
+
&& check "vspec: a plan that drops contract anchors fails with both locations named" 0 $? \
|
|
1338
|
+
|| { echo "FAIL vspec: dropped plan contract anchors not caught"; echo "$out"; fail=1; }
|
|
1339
|
+
|
|
1340
|
+
bad="$tmp/devos/specs/2026-01-01-bad"
|
|
1341
|
+
mkdir -p "$bad"
|
|
1342
|
+
printf '# Bad\n## Goal\nx\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1343
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: no scenarios fails" 1 $?
|
|
1344
|
+
|
|
1345
|
+
printf '# Bad\n## Goal\nx\n- Scenario SC-1: broken, no when-then\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1346
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: malformed scenario fails" 1 $?
|
|
1347
|
+
|
|
1348
|
+
# SC-n IDs form the internal plan index. Source tests use behavior-facing
|
|
1349
|
+
# names, and the output-boundary gate keeps the IDs out of implementation files.
|
|
1350
|
+
printf '# Bad\n## Goal\nx\n- Scenario: unnumbered / WHEN a THEN b\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1351
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: scenario without SC-n ID fails" 1 $?
|
|
1352
|
+
|
|
1353
|
+
printf '# Bad\n## Goal\nx\n- Scenario SC-1: a / WHEN a THEN b\n- Scenario SC-1: b / WHEN c THEN d\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1354
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: duplicate SC-n IDs fail" 1 $?
|
|
1355
|
+
|
|
1356
|
+
printf '# Bad\n## Goal\nx\n- Scenario SC-1: a / WHEN a THEN b\n- Scenario SC-2: b / WHEN c THEN d\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1357
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: sequential SC-n IDs pass" 0 $?
|
|
1358
|
+
|
|
1359
|
+
cp "$good/spec-functional.md" "$bad/spec-functional.md"
|
|
1360
|
+
seq 1 301 | sed 's/^/line /' > "$bad/spec.md"
|
|
1361
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: oversized capsule fails" 1 $?
|
|
1362
|
+
|
|
1363
|
+
bash "$VSPEC" "$tmp/devos/specs/absent" >/dev/null 2>&1; check "vspec: missing folder exits 2" 2 $?
|
|
1364
|
+
|
|
1365
|
+
# Scenario traceability across artifacts: the plan's prose rule (every SC-n
|
|
1366
|
+
# covered somewhere, every citation real) as a gate. Both directions matter:
|
|
1367
|
+
# an uncovered scenario is a planning gap; a ghost citation verifies nothing.
|
|
1368
|
+
mkdir -p "$bad"
|
|
1369
|
+
cp "$good/spec-functional.md" "$bad/spec-functional.md"
|
|
1370
|
+
cat > "$bad/spec.md" <<'MD'
|
|
1371
|
+
## Local Craft
|
|
1372
|
+
None
|
|
1373
|
+
## Design Convergence
|
|
1374
|
+
- **Direct MVP**: call it directly
|
|
1375
|
+
## Implementation Plan
|
|
1376
|
+
### Task Group 1
|
|
1377
|
+
**Files:** `a.ts`
|
|
1378
|
+
**Scenarios covered:** SC-1
|
|
1379
|
+
MD
|
|
1380
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: fully covered scenarios pass" 0 $?
|
|
1381
|
+
|
|
1382
|
+
cat > "$bad/spec.md" <<'MD'
|
|
1383
|
+
## Implementation Plan
|
|
1384
|
+
### Task Group 1
|
|
1385
|
+
**Scenarios covered:** none
|
|
1386
|
+
MD
|
|
1387
|
+
out=$(bash "$VSPEC" "$bad" 2>&1); echo "$out" | grep -q "SC-1 is declared but covered nowhere" && check "vspec: uncovered scenario fails with the gap named" 0 $? || { echo "FAIL vspec: uncovered scenario not caught"; fail=1; }
|
|
1388
|
+
|
|
1389
|
+
cat > "$bad/spec.md" <<'MD'
|
|
1390
|
+
## Implementation Plan
|
|
1391
|
+
### Task Group 1
|
|
1392
|
+
**Scenarios covered:** SC-9
|
|
1393
|
+
MD
|
|
1394
|
+
out=$(bash "$VSPEC" "$bad" 2>&1); rc=$?
|
|
1395
|
+
grep -q "cites SC-9 which spec-functional.md does not declare" <<<"$out" && [[ $rc -eq 1 ]] \
|
|
1396
|
+
&& check "vspec: ghost citation fails" 0 $? || { echo "FAIL vspec: ghost citation not caught"; fail=1; }
|
|
1397
|
+
|
|
1398
|
+
# C route exemption: Invariant specs declare no scenarios, so nothing to trace
|
|
1399
|
+
printf '# C\n## Goal\nx\n## Invariant\n`./run` -> exit 0\n**Surface:** none, internal only\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1400
|
+
cat > "$bad/spec.md" <<'MD'
|
|
1401
|
+
## Local Craft
|
|
1402
|
+
None
|
|
1403
|
+
## Design Convergence
|
|
1404
|
+
- **Direct MVP**: call it directly
|
|
1405
|
+
## Implementation Plan
|
|
1406
|
+
### Task Group 1
|
|
1407
|
+
**Files:** `a.ts`
|
|
1408
|
+
MD
|
|
1409
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: C route exempt from scenario tracing" 0 $?
|
|
1410
|
+
|
|
1411
|
+
# The oversized capsule above is still on disk and would fail every spec here
|
|
1412
|
+
# for the wrong reason. These cases are about spec-functional.md alone.
|
|
1413
|
+
rm -f "$bad/spec.md"
|
|
1414
|
+
|
|
1415
|
+
# Normative wording. These moved out of write-spec.md's self-check, where they
|
|
1416
|
+
# were prose asking the writer to grade itself. The negative cases matter most:
|
|
1417
|
+
# an over-eager matcher that flags "shoulder" or a deliberately deferred item
|
|
1418
|
+
# would make the gate unusable, and the word-boundary bug it guards against
|
|
1419
|
+
# (gawk's \y, silent on darwin's awk) passes every positive test by matching
|
|
1420
|
+
# nothing at all.
|
|
1421
|
+
printf '# H\n## Goal\nx\n## 3. Functionality\n- The export should include archived rows\n- Scenario SC-1: s / WHEN a THEN b\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1422
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge word in a requirement fails" 1 $?
|
|
1423
|
+
|
|
1424
|
+
printf '# H\n## Goal\nx\n## 3. Functionality\n- Scenario SC-1: s / WHEN a THEN it may download\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1425
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge word inside a scenario fails" 1 $?
|
|
1426
|
+
|
|
1427
|
+
printf '# H\n## Goal\nx\n## 3. Functionality\n- The shoulder strap MUST be listed\n- Scenario SC-1: s / WHEN a THEN b\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1428
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge substring in a word does not fail" 0 $?
|
|
1429
|
+
|
|
1430
|
+
printf '# H\n## Goal\nx\n## 3. Functionality\n- Scenario SC-1: s / WHEN a THEN b\n## Out of Scope\n- PDF export, may come later\n' > "$bad/spec-functional.md"
|
|
1431
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge in Out of Scope is allowed" 0 $?
|
|
1432
|
+
|
|
1433
|
+
printf '# H\n## Goal\nx\n## 3. Functionality\n- Scenario SC-1: s / WHEN a THEN b\n## 3bis. Behavior delta\n**Modified:** "users should wait" -> instant\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1434
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: hedge in a quoted Behavior delta is allowed" 0 $?
|
|
1435
|
+
|
|
1436
|
+
# A functional spec describes behavior; a fence is implementation leaking in.
|
|
1437
|
+
printf '# C\n## Goal\nx\n## 3. Functionality\n- Scenario SC-1: s / WHEN a THEN b\n```ts\ninterface X {}\n```\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1438
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: code block in a functional spec fails" 1 $?
|
|
1439
|
+
|
|
1440
|
+
printf '# D\n## Goal\nx\n- Scenario SC-1: s / WHEN a THEN b\n## Behavior delta\n**Removed:** old login flow\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1441
|
+
rm -f "$bad/spec.md"
|
|
1442
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: removal without reason fails" 1 $?
|
|
1443
|
+
|
|
1444
|
+
printf '# D\n## Goal\nx\n- Scenario SC-1: s / WHEN a THEN b\n## Behavior delta\n**Removed:** (none)\n## Out of Scope\n- y\n' > "$bad/spec-functional.md"
|
|
1445
|
+
bash "$VSPEC" "$bad" >/dev/null; check "vspec: empty removed marker passes" 0 $?
|
|
1446
|
+
|
|
1447
|
+
# --- check-requires.py ----------------------------------------
|
|
1448
|
+
CREQ="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/status/scripts/check-requires.py"
|
|
1449
|
+
SDIFF="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/status/scripts/spec-diff.sh"
|
|
1450
|
+
|
|
1451
|
+
# A comparison name is one directory name, not a path expression. Without
|
|
1452
|
+
# this boundary, ../ can resolve a sibling directory and make status compare
|
|
1453
|
+
# an artifact outside devos/specs.
|
|
1454
|
+
escape_root="$tmp/spec-diff-escape"
|
|
1455
|
+
mkdir -p "$escape_root/devos/specs" "$tmp/spec-diff-outside"
|
|
1456
|
+
printf '# Outside plan\n' > "$tmp/spec-diff-outside/spec.md"
|
|
1457
|
+
out=$(bash "$SDIFF" '../../../spec-diff-outside' '../../../spec-diff-outside' "$escape_root" 2>&1); rc=$?
|
|
1458
|
+
grep -q "spec names must be single directory names without path separators" <<<"$out" \
|
|
1459
|
+
&& [[ $rc -eq 2 ]] \
|
|
1460
|
+
&& check "spec-diff: traversal spec names are rejected" 0 $? \
|
|
1461
|
+
|| { echo "FAIL spec-diff: traversal spec name was not rejected"; echo "$out"; fail=1; }
|
|
1462
|
+
|
|
1463
|
+
# A same-named active folder is not interchangeable with an archived record.
|
|
1464
|
+
# If the active plan is missing, comparing the historical plan silently hides
|
|
1465
|
+
# the active artifact failure and reports a clean self-comparison instead.
|
|
1466
|
+
sdf="$tmp/spec-diff-resolution/devos/specs"
|
|
1467
|
+
mkdir -p "$sdf/active-only" "$sdf/archive/active-only"
|
|
1468
|
+
printf '# Archived plan\n## Implementation Plan\n### Task Group 1: archived\n' \
|
|
1469
|
+
> "$sdf/archive/active-only/spec.md"
|
|
1470
|
+
out=$(bash "$SDIFF" active-only active-only "$tmp/spec-diff-resolution" 2>&1); rc=$?
|
|
1471
|
+
grep -q "active spec 'active-only' exists but its spec.md is missing" <<<"$out" \
|
|
1472
|
+
&& [[ $rc -eq 1 ]] \
|
|
1473
|
+
&& check "spec-diff: an active folder without spec.md does not fall back to archive" 0 $? \
|
|
1474
|
+
|| { echo "FAIL spec-diff: active missing-plan fallback not caught"; echo "$out"; fail=1; }
|
|
1475
|
+
|
|
1476
|
+
# The normal archived lookup remains available when no active folder exists.
|
|
1477
|
+
mkdir -p "$sdf/archive/archive-only"
|
|
1478
|
+
printf '# Archived plan\n## Implementation Plan\n### Task Group 1: archived\n' \
|
|
1479
|
+
> "$sdf/archive/archive-only/spec.md"
|
|
1480
|
+
bash "$SDIFF" archive-only archive-only "$tmp/spec-diff-resolution" >/dev/null
|
|
1481
|
+
check "spec-diff: an archive-only spec still resolves" 0 $?
|
|
1482
|
+
|
|
1483
|
+
# When both records exist and both are complete, the active plan is the one
|
|
1484
|
+
# being worked on and must win over the historical copy.
|
|
1485
|
+
printf '# Active plan\n## Implementation Plan\n### Task Group 1: active\n' \
|
|
1486
|
+
> "$sdf/active-only/spec.md"
|
|
1487
|
+
out=$(bash "$SDIFF" active-only archive-only "$tmp/spec-diff-resolution" 2>&1)
|
|
1488
|
+
grep -q "Task groups differ" <<<"$out" \
|
|
1489
|
+
&& check "spec-diff: a complete active plan wins over an archived namesake" 0 $? \
|
|
1490
|
+
|| { echo "FAIL spec-diff: active plan did not win over archive"; echo "$out"; fail=1; }
|
|
1491
|
+
|
|
1492
|
+
# Every other plan reader (validate-spec, check-converge,
|
|
1493
|
+
# sizing-calibration) accepts any run of spaces around
|
|
1494
|
+
# the words in a group header. spec-diff read the literal '### Task Group', so
|
|
1495
|
+
# a plan written '### Task Group 2: gamma' had no groups at all and the
|
|
1496
|
+
# comparison called two different plans identical.
|
|
1497
|
+
sdg="$tmp/spec-diff-groups/devos/specs"
|
|
1498
|
+
mkdir -p "$sdg/wide" "$sdg/loose"
|
|
1499
|
+
printf '# Wide\n## Implementation Plan\n### Task Group 1: alpha\n### Task Group 2: beta\n' \
|
|
1500
|
+
> "$sdg/wide/spec.md"
|
|
1501
|
+
printf '# Loose\n## Implementation Plan\n### Task Group 1: alpha\n### Task Group 2: gamma\n' \
|
|
1502
|
+
> "$sdg/loose/spec.md"
|
|
1503
|
+
out=$(bash "$SDIFF" wide loose "$tmp/spec-diff-groups" 2>&1)
|
|
1504
|
+
grep -qx "Task groups differ:" <<<"$out" \
|
|
1505
|
+
&& grep -q "Task Group 2: gamma" <<<"$out" \
|
|
1506
|
+
&& check "spec-diff: a group header with extra spacing is still compared" 0 $? \
|
|
1507
|
+
|| { echo "FAIL spec-diff: extra spacing hid a group from the comparison"; echo "$out"; fail=1; }
|
|
1508
|
+
|
|
1509
|
+
# The other direction: widening the pattern must not promote a fourth-level
|
|
1510
|
+
# heading to a task group, or any deeply nested note named like a group would
|
|
1511
|
+
# start showing up in the structural diff.
|
|
1512
|
+
sdg4="$tmp/spec-diff-level/devos/specs"
|
|
1513
|
+
mkdir -p "$sdg4/deep" "$sdg4/single"
|
|
1514
|
+
printf '# Deep\n## Implementation Plan\n#### Task Group 1: alpha\n### Task Group 2: beta\n' \
|
|
1515
|
+
> "$sdg4/deep/spec.md"
|
|
1516
|
+
printf '# Single\n## Implementation Plan\n### Task Group 2: beta\n' > "$sdg4/single/spec.md"
|
|
1517
|
+
out=$(bash "$SDIFF" deep single "$tmp/spec-diff-level" 2>&1)
|
|
1518
|
+
grep -qx "Task groups are identical." <<<"$out" \
|
|
1519
|
+
&& check "spec-diff: a fourth-level heading is not counted as a task group" 0 $? \
|
|
1520
|
+
|| { echo "FAIL spec-diff: a fourth-level heading was read as a task group"; echo "$out"; fail=1; }
|
|
1521
|
+
|
|
1522
|
+
# The colon separator is the other half of the same heading. check-converge
|
|
1523
|
+
# reads it and the plan gates read it, so a plan written '### Task Group: alpha'
|
|
1524
|
+
# had no groups here: both sides of the comparison came out empty and two plans
|
|
1525
|
+
# with different groups were reported identical.
|
|
1526
|
+
sdgc="$tmp/spec-diff-colon/devos/specs"
|
|
1527
|
+
mkdir -p "$sdgc/colon-a" "$sdgc/colon-b" "$sdgc/numbered-a" "$sdgc/numbered-b"
|
|
1528
|
+
printf '# Colon A\n## Implementation Plan\n### Task Group: alpha\n' > "$sdgc/colon-a/spec.md"
|
|
1529
|
+
printf '# Colon B\n## Implementation Plan\n### Task Group: beta\n' > "$sdgc/colon-b/spec.md"
|
|
1530
|
+
out=$(bash "$SDIFF" colon-a colon-b "$tmp/spec-diff-colon" 2>&1)
|
|
1531
|
+
grep -qx "Task groups differ:" <<<"$out" \
|
|
1532
|
+
&& grep -q "Task Group: beta" <<<"$out" \
|
|
1533
|
+
&& check "spec-diff: a colon-form group header is still compared" 0 $? \
|
|
1534
|
+
|| { echo "FAIL spec-diff: a colon-form group header was hidden from the comparison"; echo "$out"; fail=1; }
|
|
1535
|
+
# The same two plans written in the numbered form have to keep reporting the
|
|
1536
|
+
# same difference, so the fix cannot be a pattern that simply matches more.
|
|
1537
|
+
printf '# Numbered A\n## Implementation Plan\n### Task Group 1: alpha\n' > "$sdgc/numbered-a/spec.md"
|
|
1538
|
+
printf '# Numbered B\n## Implementation Plan\n### Task Group 2: beta\n' > "$sdgc/numbered-b/spec.md"
|
|
1539
|
+
out=$(bash "$SDIFF" numbered-a numbered-b "$tmp/spec-diff-colon" 2>&1)
|
|
1540
|
+
grep -qx "Task groups differ:" <<<"$out" \
|
|
1541
|
+
&& grep -q "Task Group 2: beta" <<<"$out" \
|
|
1542
|
+
&& check "spec-diff: a numbered group header still reports the same difference" 0 $? \
|
|
1543
|
+
|| { echo "FAIL spec-diff: numbered group comparison changed"; echo "$out"; fail=1; }
|
|
1544
|
+
# A heading that only starts like a group is still not one.
|
|
1545
|
+
printf '# Deep colon\n## Implementation Plan\n#### Task Group: alpha\n### Task Group: beta\n' \
|
|
1546
|
+
> "$sdgc/colon-a/spec.md"
|
|
1547
|
+
out=$(bash "$SDIFF" colon-a colon-b "$tmp/spec-diff-colon" 2>&1)
|
|
1548
|
+
grep -qx "Task groups are identical." <<<"$out" \
|
|
1549
|
+
&& check "spec-diff: a fourth-level colon-form heading is not a task group" 0 $? \
|
|
1550
|
+
|| { echo "FAIL spec-diff: a fourth-level colon-form heading was read as a task group"; echo "$out"; fail=1; }
|
|
1551
|
+
|
|
1552
|
+
reqfix="$tmp/reqproj"
|
|
1553
|
+
mkdir -p "$reqfix/devos/specs/archive/2026-01-01-auth"
|
|
1554
|
+
mkdir -p "$reqfix/devos/specs/2026-01-02-export"
|
|
1555
|
+
mkdir -p "$reqfix/devos/specs/2026-01-03-billing"
|
|
1556
|
+
mkdir -p "$reqfix/devos/specs/2026-01-04-broken"
|
|
1557
|
+
mkdir -p "$reqfix/devos/specs/2026-01-05-a"
|
|
1558
|
+
mkdir -p "$reqfix/devos/specs/2026-01-06-b"
|
|
1559
|
+
printf '# Auth\n' > "$reqfix/devos/specs/archive/2026-01-01-auth/spec-functional.md"
|
|
1560
|
+
printf '# Export\nRequires: [2026-01-01-auth]\n' > "$reqfix/devos/specs/2026-01-02-export/spec-functional.md"
|
|
1561
|
+
printf '# Billing\nRequires: [export]\n' > "$reqfix/devos/specs/2026-01-03-billing/spec-functional.md"
|
|
1562
|
+
printf '# Broken\nRequires: [ghost-spec]\n' > "$reqfix/devos/specs/2026-01-04-broken/spec-functional.md"
|
|
1563
|
+
printf '# A\nRequires: [2026-01-06-b]\n' > "$reqfix/devos/specs/2026-01-05-a/spec-functional.md"
|
|
1564
|
+
printf '# B\nRequires: [2026-01-05-a]\n' > "$reqfix/devos/specs/2026-01-06-b/spec-functional.md"
|
|
1565
|
+
mkdir -p "$reqfix/devos/specs/2026-01-07-c" "$reqfix/devos/specs/2026-01-08-d"
|
|
1566
|
+
printf '# C\nRequires: [2026-01-08-d]\n' > "$reqfix/devos/specs/2026-01-07-c/spec-functional.md"
|
|
1567
|
+
printf '# D\nRequires: [2026-01-07-c]\n' > "$reqfix/devos/specs/2026-01-08-d/spec-functional.md"
|
|
1568
|
+
|
|
1569
|
+
out=$(python3 "$CREQ" "$reqfix")
|
|
1570
|
+
[[ "$out" == *"2026-01-03-billing blocked-by: 2026-01-02-export"* ]]
|
|
1571
|
+
check "creq: unmet dep shows blocked-by (suffix resolve)" 0 $?
|
|
1572
|
+
[[ "$out" == *"2026-01-02-export blocks: 2026-01-03-billing"* ]]
|
|
1573
|
+
check "creq: reverse blocks: annotation" 0 $?
|
|
1574
|
+
echo "$out" | grep -E '^2026-01-02-export ' | grep -qv 'blocked-by'
|
|
1575
|
+
check "creq: archived dep does not block" 0 $?
|
|
1576
|
+
|
|
1577
|
+
shadow="$tmp/active-archive-shadow"
|
|
1578
|
+
mkdir -p "$shadow/devos/specs/a" "$shadow/devos/specs/archive/a" "$shadow/devos/specs/b"
|
|
1579
|
+
printf '# A active\n' > "$shadow/devos/specs/a/spec-functional.md"
|
|
1580
|
+
printf '# A archived\n' > "$shadow/devos/specs/archive/a/spec-functional.md"
|
|
1581
|
+
printf '# B\nRequires: [a]\n' > "$shadow/devos/specs/b/spec-functional.md"
|
|
1582
|
+
out=$(python3 "$CREQ" "$shadow")
|
|
1583
|
+
grep -Fqx 'b blocked-by: a' <<<"$out"; check "creq: active namesake shadows archived dependency" 0 $?
|
|
1584
|
+
out=$(python3 "$CREQ" "$reqfix")
|
|
1585
|
+
|
|
1586
|
+
[[ "$out" == *"BROKEN"*"ghost-spec"* ]]
|
|
1587
|
+
check "creq: unknown target is BROKEN" 0 $?
|
|
1588
|
+
[[ "$out" == *"CYCLE"*"2026-01-05-a"*"2026-01-06-b"* ]]
|
|
1589
|
+
check "creq: cycle detected" 0 $?
|
|
1590
|
+
|
|
1591
|
+
fout=$(python3 "$CREQ" --spec 2026-01-05-a "$reqfix")
|
|
1592
|
+
grep -Fqx '2026-01-05-a blocked-by: 2026-01-06-b blocks: 2026-01-06-b' <<<"$fout" \
|
|
1593
|
+
&& grep -Fqx 'CYCLE 2026-01-05-a -> 2026-01-06-b -> 2026-01-05-a' <<<"$fout" \
|
|
1594
|
+
&& check "creq: --spec reports status and a cycle through another spec" 0 $? \
|
|
1595
|
+
|| { echo "FAIL creq: --spec omitted its cross-spec cycle"; echo "$fout"; fail=1; }
|
|
1596
|
+
[[ "$fout" != *"2026-01-07-c"* ]]
|
|
1597
|
+
check "creq: --spec omits unrelated cycle nodes" 0 $?
|
|
1598
|
+
|
|
1599
|
+
# A short suffix matching two specs resolves to neither, and reporting that as
|
|
1600
|
+
# a missing folder sends the reader looking for a typo instead of at the name
|
|
1601
|
+
# they have to spell out. The candidates are the whole diagnosis.
|
|
1602
|
+
amb="$tmp/req-ambiguous"
|
|
1603
|
+
mkdir -p "$amb/devos/specs/archive/2026-01-01-a-export" \
|
|
1604
|
+
"$amb/devos/specs/archive/2026-01-02-b-export" \
|
|
1605
|
+
"$amb/devos/specs/2026-01-03-billing"
|
|
1606
|
+
printf '# Billing\nRequires: [export]\n' \
|
|
1607
|
+
> "$amb/devos/specs/2026-01-03-billing/spec-functional.md"
|
|
1608
|
+
aout=$(python3 "$CREQ" --spec 2026-01-03-billing "$amb" 2>&1)
|
|
1609
|
+
grep -Fq 'AMBIGUOUS' <<<"$aout" && grep -Fq '2026-01-01-a-export' <<<"$aout" \
|
|
1610
|
+
&& grep -Fq '2026-01-02-b-export' <<<"$aout" \
|
|
1611
|
+
&& check "creq: an ambiguous short suffix names every candidate" 0 $? \
|
|
1612
|
+
|| { echo "FAIL creq: ambiguous suffix not diagnosed"; echo "$aout"; fail=1; }
|
|
1613
|
+
! grep -Fq 'no folder under specs/ or archive/' <<<"$aout"
|
|
1614
|
+
check "creq: an ambiguous suffix is not reported as a missing folder" 0 $?
|
|
1615
|
+
dout=$(python3 "$CREQ" --doctor "$amb" 2>&1)
|
|
1616
|
+
grep -Fq 'AMBIGUOUS Requires: export in 2026-01-03-billing' <<<"$dout" \
|
|
1617
|
+
&& check "creq: doctor reports the ambiguous suffix with its candidates" 0 $? \
|
|
1618
|
+
|| { echo "FAIL creq: doctor did not report the ambiguous suffix"; echo "$dout"; fail=1; }
|
|
1619
|
+
vamb=$(python3 "$CREQ" --verify "$amb/devos/specs/2026-01-03-billing" "$amb" 2>&1)
|
|
1620
|
+
grep -Fq 'AMBIGUOUS short suffix' <<<"$vamb" \
|
|
1621
|
+
&& grep -Fq '2026-01-01-a-export' <<<"$vamb" \
|
|
1622
|
+
&& grep -Fq '2026-01-02-b-export' <<<"$vamb" \
|
|
1623
|
+
&& ! grep -Fq 'unknown target' <<<"$vamb" \
|
|
1624
|
+
&& check "creq: verify preserves the ambiguous classification and candidates" 0 $? \
|
|
1625
|
+
|| { echo "FAIL creq: verify misdiagnosed an ambiguous suffix"; echo "$vamb"; fail=1; }
|
|
1626
|
+
printf '# Billing\nRequires: [ghost]\n' > "$amb/devos/specs/2026-01-03-billing/spec-functional.md"
|
|
1627
|
+
gout=$(python3 "$CREQ" --spec 2026-01-03-billing "$amb" 2>&1)
|
|
1628
|
+
grep -Fq 'no folder under specs/ or archive/' <<<"$gout" && ! grep -Fq 'AMBIGUOUS' <<<"$gout" \
|
|
1629
|
+
&& check "creq: a genuinely missing target is still BROKEN" 0 $? \
|
|
1630
|
+
|| { echo "FAIL creq: missing target misdiagnosed"; echo "$gout"; fail=1; }
|
|
1631
|
+
printf '# Billing\nRequires: [2026-01-02-b-export]\n' \
|
|
1632
|
+
> "$amb/devos/specs/2026-01-03-billing/spec-functional.md"
|
|
1633
|
+
sout=$(python3 "$CREQ" --spec 2026-01-03-billing "$amb" 2>&1)
|
|
1634
|
+
! grep -Fq 'AMBIGUOUS' <<<"$sout" && ! grep -Fq 'BROKEN' <<<"$sout" \
|
|
1635
|
+
&& check "creq: the full name of a shared suffix resolves" 0 $? \
|
|
1636
|
+
|| { echo "FAIL creq: full name did not resolve"; echo "$sout"; fail=1; }
|
|
1637
|
+
|
|
1638
|
+
# A Requires heading followed by the numbered sections of the write-spec.md
|
|
1639
|
+
# skeleton. The old terminator required a letter after the hashes, so
|
|
1640
|
+
# `## 5. Out of Scope` never closed the list and `- none` became a
|
|
1641
|
+
# dependency named "none".
|
|
1642
|
+
reqsec="$tmp/req-section"
|
|
1643
|
+
mkdir -p "$reqsec/devos/specs/2026-01-04-a" "$reqsec/devos/specs/2026-01-05-b" \
|
|
1644
|
+
"$reqsec/devos/specs/2026-01-05-c" "$reqsec/devos/specs/2026-01-06-d"
|
|
1645
|
+
printf '# A\n## 1. Goal\nx\n## 4. Functional Constraints\n### Requires:\n- 2026-01-05-b\n## 5. Out of Scope\n- none\n## Appendix\n' \
|
|
1646
|
+
> "$reqsec/devos/specs/2026-01-04-a/spec-functional.md"
|
|
1647
|
+
printf '# B\n## 1. Goal\nx\n' > "$reqsec/devos/specs/2026-01-05-b/spec-functional.md"
|
|
1648
|
+
printf '# C\n## 1. Goal\nx\n### Requires:\n- 2026-01-06-d\n### Rationale\n- 2026-01-05-b\n' \
|
|
1649
|
+
> "$reqsec/devos/specs/2026-01-05-c/spec-functional.md"
|
|
1650
|
+
printf '# D\n## 1. Goal\nx\n' > "$reqsec/devos/specs/2026-01-06-d/spec-functional.md"
|
|
1651
|
+
rsout=$(python3 "$CREQ" "$reqsec" 2>&1)
|
|
1652
|
+
! grep -Fq 'BROKEN' <<<"$rsout" \
|
|
1653
|
+
&& check "creq: a numbered section after Requires does not leak its bullets into the graph" 0 $? \
|
|
1654
|
+
|| { echo "FAIL creq: a numbered section leaked a dependency"; echo "$rsout"; fail=1; }
|
|
1655
|
+
grep -Fqx '2026-01-04-a blocked-by: 2026-01-05-b' <<<"$rsout" \
|
|
1656
|
+
&& grep -Fqx '2026-01-05-b blocks: 2026-01-04-a' <<<"$rsout" \
|
|
1657
|
+
&& check "creq: the declared dependency is still collected under a Requires heading" 0 $? \
|
|
1658
|
+
|| { echo "FAIL creq: the Requires bullet was dropped"; echo "$rsout"; fail=1; }
|
|
1659
|
+
grep -Fqx '2026-01-05-c blocked-by: 2026-01-06-d' <<<"$rsout" \
|
|
1660
|
+
&& check "creq: a lettered heading after Requires still ends the section" 0 $? \
|
|
1661
|
+
|| { echo "FAIL creq: a lettered heading no longer ends the Requires section"; echo "$rsout"; fail=1; }
|
|
1662
|
+
|
|
1663
|
+
# The reader finds the declaration with grep -i but used to strip the label
|
|
1664
|
+
# with a case-sensitive `s/^[Rr]equires:/`, so `REQUIRES: [x]` was accepted as
|
|
1665
|
+
# a declaration and then parsed as a dependency literally named
|
|
1666
|
+
# "REQUIRES: [x" and reported BROKEN. The heading branch matched one spelling.
|
|
1667
|
+
reqcase="$tmp/req-case"
|
|
1668
|
+
mkdir -p "$reqcase/devos/specs/2026-01-05-a" "$reqcase/devos/specs/2026-01-06-b"
|
|
1669
|
+
printf '# B\n' > "$reqcase/devos/specs/2026-01-06-b/spec-functional.md"
|
|
1670
|
+
reqcase_ok=1
|
|
1671
|
+
for spelling in 'Requires:' 'requires:' 'REQUIRES:' 'ReQuIrEs:'; do
|
|
1672
|
+
printf '# A\n\n%s [2026-01-06-b]\n' "$spelling" \
|
|
1673
|
+
> "$reqcase/devos/specs/2026-01-05-a/spec-functional.md"
|
|
1674
|
+
cout=$(python3 "$CREQ" "$reqcase" 2>&1)
|
|
1675
|
+
{ grep -Fqx '2026-01-05-a blocked-by: 2026-01-06-b' <<<"$cout" \
|
|
1676
|
+
&& ! grep -Fq 'BROKEN' <<<"$cout"; } || { reqcase_ok=0; echo "--- inline '$spelling'"; echo "$cout"; }
|
|
1677
|
+
done
|
|
1678
|
+
for spelling in Requires REQUIRES; do
|
|
1679
|
+
printf '# A\n\n### %s:\n- 2026-01-06-b\n## 5. Out of Scope\n- none\n' "$spelling" \
|
|
1680
|
+
> "$reqcase/devos/specs/2026-01-05-a/spec-functional.md"
|
|
1681
|
+
cout=$(python3 "$CREQ" "$reqcase" 2>&1)
|
|
1682
|
+
{ grep -Fqx '2026-01-05-a blocked-by: 2026-01-06-b' <<<"$cout" \
|
|
1683
|
+
&& ! grep -Fq 'BROKEN' <<<"$cout"; } || { reqcase_ok=0; echo "--- heading '$spelling'"; echo "$cout"; }
|
|
1684
|
+
done
|
|
1685
|
+
[[ "$reqcase_ok" -eq 1 ]] \
|
|
1686
|
+
&& check "creq: every accepted case of the Requires label resolves the dependency" 0 $? \
|
|
1687
|
+
|| { echo "FAIL creq: a case-folded Requires label did not resolve"; fail=1; }
|
|
1688
|
+
printf '# A\n\nREQUIRES: [2026-02-01-ghost]\n' \
|
|
1689
|
+
> "$reqcase/devos/specs/2026-01-05-a/spec-functional.md"
|
|
1690
|
+
cout=$(python3 "$CREQ" "$reqcase" 2>&1)
|
|
1691
|
+
grep -Fqx 'BROKEN 2026-01-05-a Requires: 2026-02-01-ghost (no folder under specs/ or archive/)' <<<"$cout" \
|
|
1692
|
+
&& check "creq: a truly missing target is still BROKEN, naming the target alone" 0 $? \
|
|
1693
|
+
|| { echo "FAIL creq: case folding turned a missing target into a resolved one"; echo "$cout"; fail=1; }
|
|
1694
|
+
|
|
1695
|
+
vout=$(python3 "$CREQ" --verify "$reqfix/devos/specs/2026-01-03-billing" "$reqfix")
|
|
1696
|
+
[[ "$vout" == *"INFO depends on 2026-01-02-export, not yet shipped"* ]]
|
|
1697
|
+
check "creq: --verify unmet is INFO" 0 $?
|
|
1698
|
+
vout=$(python3 "$CREQ" --verify "$reqfix/devos/specs/2026-01-02-export" "$reqfix")
|
|
1699
|
+
[[ -z "$vout" ]]
|
|
1700
|
+
check "creq: --verify met dep is silent" 0 $?
|
|
1701
|
+
|
|
1702
|
+
dout=$(python3 "$CREQ" --doctor "$reqfix")
|
|
1703
|
+
[[ "$dout" == *"BROKEN Requires: ghost-spec"* ]]
|
|
1704
|
+
check "creq: --doctor flags broken edge" 0 $?
|
|
1705
|
+
[[ "$dout" == *"CYCLE"* ]]
|
|
1706
|
+
check "creq: --doctor flags cycle" 0 $?
|
|
1707
|
+
[[ "$dout" != *"blocked-by"* ]]
|
|
1708
|
+
check "creq: --doctor omits status blocked-by" 0 $?
|
|
1709
|
+
|
|
1710
|
+
# --transitive: the A->B->C chain.
|
|
1711
|
+
tout=$(python3 "$CREQ" --transitive "$reqfix")
|
|
1712
|
+
[[ "$tout" == *"2026-01-03-billing -> 2026-01-02-export"* ]]
|
|
1713
|
+
check "creq: --transitive resolves a two-hop chain" 0 $?
|
|
1714
|
+
|
|
1715
|
+
# --timeline: the same T25 claimed verification "against a capability with 3 or
|
|
1716
|
+
# more merges" — every behavior file in this repo has exactly one. It used
|
|
1717
|
+
# `git log --diff-filter=M`, which excludes the A commit that CREATES the file,
|
|
1718
|
+
# so a capability with only its creating commit printed a bare header. And
|
|
1719
|
+
# `git log` exits 0 while printing nothing, so the `|| echo` fallback it carried
|
|
1720
|
+
# never fired. Both pinned here: a real file must yield its diff, an
|
|
1721
|
+
# empty-history file must say so.
|
|
1722
|
+
tlfix="$tmp/timeline-fix"
|
|
1723
|
+
mkdir -p "$tlfix/devos/product/behaviors"
|
|
1724
|
+
(
|
|
1725
|
+
cd "$tlfix" && git init -q . && git config user.email t@t && git config user.name t
|
|
1726
|
+
printf '# Search\n\n## Added\n- filters by tag\n' > devos/product/behaviors/search.md
|
|
1727
|
+
git add -A && git commit -qm 'chore(spec): complete 2026-01-01-search, merge behavior delta'
|
|
1728
|
+
)
|
|
1729
|
+
tout=$(cd "$tlfix" && python3 "$CREQ" --timeline search .)
|
|
1730
|
+
[[ "$tout" == *"filters by tag"* ]]
|
|
1731
|
+
check "creq: --timeline shows the creating commit's diff" 0 $?
|
|
1732
|
+
|
|
1733
|
+
# The explicit project root must control both capability lookup and git history.
|
|
1734
|
+
# The caller may be outside the project, as the gate test process is.
|
|
1735
|
+
tout=$(python3 "$CREQ" --timeline search "$tlfix")
|
|
1736
|
+
[[ "$tout" == *"filters by tag"* ]]
|
|
1737
|
+
check "creq: --timeline resolves project root from outside" 0 $?
|
|
1738
|
+
[[ "$tout" == *"2026-01-01-search"* ]]
|
|
1739
|
+
check "creq: --timeline carries the spec name from the commit subject" 0 $?
|
|
1740
|
+
|
|
1741
|
+
touch "$tlfix/devos/product/behaviors/untracked.md"
|
|
1742
|
+
tout=$(cd "$tlfix" && python3 "$CREQ" --timeline untracked .)
|
|
1743
|
+
[[ "$tout" == *"no git history"* ]]
|
|
1744
|
+
check "creq: --timeline reports empty history instead of a bare header" 0 $?
|
|
1745
|
+
|
|
1746
|
+
python3 "$CREQ" --timeline no-such-capability "$tlfix" >/dev/null 2>&1
|
|
1747
|
+
check "creq: --timeline missing capability exits 1" 1 $?
|
|
1748
|
+
|
|
1749
|
+
printf '# Outside behavior\n' > "$tlfix/devos/product/outside.md"
|
|
1750
|
+
(cd "$tlfix" && git add -A && git commit -qm 'chore: outside behavior')
|
|
1751
|
+
out=$(python3 "$CREQ" --timeline ../outside "$tlfix" 2>&1); rc=$?
|
|
1752
|
+
grep -Fq 'capability must be a single name without path separators' <<<"$out" && [[ $rc -eq 2 ]] \
|
|
1753
|
+
&& check "creq: --timeline rejects capability path traversal" 0 $? \
|
|
1754
|
+
|| { echo "FAIL creq: --timeline accepted capability path traversal"; echo "$out"; fail=1; }
|
|
1755
|
+
|
|
1756
|
+
# --- render-rule.py ----------------------------------------------------------
|
|
1757
|
+
RENDER="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/init/scripts/render-rule.py"
|
|
1758
|
+
|
|
1759
|
+
rproj="$tmp/rule-project"
|
|
1760
|
+
mkdir -p "$rproj/.github" "$rproj/devos/standards/rules/02-languages"
|
|
1761
|
+
cat > "$rproj/devos/standards/rules/02-languages/ts-strict.md" <<'MD'
|
|
1762
|
+
---
|
|
1763
|
+
globs: src/**/*.ts, src/**/*.tsx
|
|
1764
|
+
---
|
|
1765
|
+
# TypeScript strict mode
|
|
1766
|
+
- No `any` without a comment justifying it
|
|
1767
|
+
MD
|
|
1768
|
+
(cd "$rproj" && python3 "$RENDER" devos/standards/rules/02-languages/ts-strict.md >/dev/null)
|
|
1769
|
+
check "render-rule: copilot instructions written" 0 $?
|
|
1770
|
+
[[ -f "$rproj/.github/instructions/languages-ts-strict.instructions.md" ]]
|
|
1771
|
+
check "render-rule: copilot output exists" 0 $?
|
|
1772
|
+
grep -q 'applyTo: "src/\*\*/\*.ts,src/\*\*/\*.tsx"' "$rproj/.github/instructions/languages-ts-strict.instructions.md"
|
|
1773
|
+
check "render-rule: copilot applyTo is comma-separated" 0 $?
|
|
1774
|
+
|
|
1775
|
+
mkdir -p "$rproj/devos/standards/rules/00-architecture"
|
|
1776
|
+
cat > "$rproj/devos/standards/rules/00-architecture/layering.md" <<'MD'
|
|
1777
|
+
---
|
|
1778
|
+
---
|
|
1779
|
+
# Layering
|
|
1780
|
+
- Presentation never imports Infrastructure directly
|
|
1781
|
+
MD
|
|
1782
|
+
(cd "$rproj" && python3 "$RENDER" devos/standards/rules/00-architecture/layering.md >/dev/null)
|
|
1783
|
+
grep -q 'applyTo: "\*\*"' "$rproj/.github/instructions/architecture-layering.instructions.md"
|
|
1784
|
+
check "render-rule: all-files rule sets applyTo ** on copilot" 0 $?
|
|
1785
|
+
|
|
1786
|
+
noinstall="$tmp/no-install-project"
|
|
1787
|
+
mkdir -p "$noinstall/devos/standards/rules/09-other"
|
|
1788
|
+
cp "$rproj/devos/standards/rules/00-architecture/layering.md" "$noinstall/devos/standards/rules/09-other/layering.md"
|
|
1789
|
+
out=$(cd "$noinstall" && python3 "$RENDER" devos/standards/rules/09-other/layering.md)
|
|
1790
|
+
[[ "$out" == *"nothing to render"* ]]
|
|
1791
|
+
check "render-rule: no Copilot install renders nothing, says so" 0 $?
|
|
1792
|
+
|
|
1793
|
+
python3 "$RENDER" /nonexistent/rule.md >/dev/null 2>&1
|
|
1794
|
+
check "render-rule: missing file exits 2" 2 $?
|
|
1795
|
+
|
|
1796
|
+
# Frontmatter that opens but never closes: the old guard only checked the
|
|
1797
|
+
# opener, so the rule text was swallowed as frontmatter and an applyTo-only
|
|
1798
|
+
# instruction file was written and reported as a success.
|
|
1799
|
+
printf '%s\n' '---' 'globs: src/**/*.ts' '' '# Truncated' '' '- Body that lost its fence.' \
|
|
1800
|
+
> "$rproj/devos/standards/rules/02-languages/truncated.md"
|
|
1801
|
+
out=$(cd "$rproj" && python3 "$RENDER" devos/standards/rules/02-languages/truncated.md 2>&1); rc=$?
|
|
1802
|
+
grep -Fq 'unterminated frontmatter' <<<"$out" && [[ $rc -eq 1 ]]
|
|
1803
|
+
check "render-rule: unterminated frontmatter is malformed input" 0 $?
|
|
1804
|
+
[[ ! -e "$rproj/.github/instructions/languages-truncated.instructions.md" ]] && [[ "$out" != *"wrote "* ]]
|
|
1805
|
+
check "render-rule: unterminated frontmatter writes no instruction file" 0 $?
|
|
1806
|
+
|
|
1807
|
+
# --- clarify.sh ----------------------------------------
|
|
1808
|
+
CLARIFY="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/spec/scripts/clarify.sh"
|
|
1809
|
+
FIXTURES="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/fixtures"
|
|
1810
|
+
|
|
1811
|
+
dirty="$tmp/devos/specs/2026-01-01-dirty"
|
|
1812
|
+
mkdir -p "$dirty/planning"
|
|
1813
|
+
cp "$FIXTURES/clarify-dirty/spec-functional.md" "$dirty/spec-functional.md"
|
|
1814
|
+
cp "$FIXTURES/clarify-dirty/planning/notes.md" "$dirty/planning/notes.md"
|
|
1815
|
+
out=$(bash "$CLARIFY" "$dirty")
|
|
1816
|
+
[[ "$out" == *"Vague quantifier"* ]]; check "clarify: vague quantifier detected" 0 $?
|
|
1817
|
+
[[ "$out" == *"3 clarification(s) found, 3 suppressed"* ]]; check "clarify: cap at 3, rest suppressed" 0 $?
|
|
1818
|
+
grep -q "^## Clarifications Needed" "$dirty/spec-functional.md"; check "clarify: section appended to dirty spec" 0 $?
|
|
1819
|
+
|
|
1820
|
+
# The appended blocks quote the lines that triggered them, so a rerun used to
|
|
1821
|
+
# read its own output back as fresh ambiguity: the same spec reported 3 found /
|
|
1822
|
+
# 3 suppressed once and 3 found / 6 suppressed after.
|
|
1823
|
+
out=$(bash "$CLARIFY" "$dirty" | head -1)
|
|
1824
|
+
[[ "$out" == "clarify: 3 clarification(s) found, 3 suppressed (cap: 3)" ]]; check "clarify: a rerun does not count its own output as ambiguity" 0 $?
|
|
1825
|
+
out=$(bash "$CLARIFY" "$dirty" | head -1)
|
|
1826
|
+
[[ "$out" == "clarify: 3 clarification(s) found, 3 suppressed (cap: 3)" ]]; check "clarify: repeated reruns stay stable" 0 $?
|
|
1827
|
+
[[ "$(grep -c '^## Clarifications Needed' "$dirty/spec-functional.md")" == 1 ]]; check "clarify: reruns do not append a second section" 0 $?
|
|
1828
|
+
|
|
1829
|
+
clean="$tmp/devos/specs/2026-01-02-clean"
|
|
1830
|
+
mkdir -p "$clean"
|
|
1831
|
+
cp "$FIXTURES/clarify-clean/spec-functional.md" "$clean/spec-functional.md"
|
|
1832
|
+
out=$(bash "$CLARIFY" "$clean")
|
|
1833
|
+
[[ "$out" == "clarify: 0 clarification(s) found, 0 suppressed (cap: 3)" ]]; check "clarify: zero false positives on clean spec" 0 $?
|
|
1834
|
+
! grep -q "^## Clarifications Needed" "$clean/spec-functional.md"; check "clarify: no section appended to clean spec" 0 $?
|
|
1835
|
+
|
|
1836
|
+
answered="$tmp/devos/specs/2026-01-03-answered"
|
|
1837
|
+
mkdir -p "$answered/planning"
|
|
1838
|
+
printf '%s\n' '- Option A: Use CSV format' '- Option B: Use JSON format' \
|
|
1839
|
+
> "$answered/planning/notes.md"
|
|
1840
|
+
cat > "$answered/spec-functional.md" <<'MD'
|
|
1841
|
+
# Answered spec
|
|
1842
|
+
|
|
1843
|
+
## 1. Goal
|
|
1844
|
+
Users can export data.
|
|
1845
|
+
|
|
1846
|
+
## 2. User Stories
|
|
1847
|
+
### US-1: Export data
|
|
1848
|
+
As a user, I want to export data.
|
|
1849
|
+
|
|
1850
|
+
**Acceptance criteria:**
|
|
1851
|
+
- [ ] Export completes within 1 second
|
|
1852
|
+
|
|
1853
|
+
## 3. Functionality
|
|
1854
|
+
**Business rules:**
|
|
1855
|
+
- Export must use CSV
|
|
1856
|
+
|
|
1857
|
+
**Answer: A**
|
|
1858
|
+
|
|
1859
|
+
## Out of Scope
|
|
1860
|
+
- Import
|
|
1861
|
+
MD
|
|
1862
|
+
out=$(bash "$CLARIFY" "$answered")
|
|
1863
|
+
[[ "$out" == "clarify: 0 clarification(s) found, 0 suppressed (cap: 3)" ]]; check "clarify: explicit Answer resolves a planning fork" 0 $?
|
|
1864
|
+
! grep -q "Unresolved fork" <<<"$out"; check "clarify: answered fork is not reported" 0 $?
|
|
1865
|
+
|
|
1866
|
+
note_answered="$tmp/devos/specs/2026-01-04-note-answered"
|
|
1867
|
+
mkdir -p "$note_answered/planning"
|
|
1868
|
+
printf '%s\n' '- Option A: Use CSV format' '- Option B: Use JSON format' '**Answer: A**' \
|
|
1869
|
+
> "$note_answered/planning/notes.md"
|
|
1870
|
+
cat > "$note_answered/spec-functional.md" <<'MD'
|
|
1871
|
+
# Planning-note answer
|
|
1872
|
+
|
|
1873
|
+
## 1. Goal
|
|
1874
|
+
Users can export data.
|
|
1875
|
+
|
|
1876
|
+
## 2. User Stories
|
|
1877
|
+
### US-1: Export data
|
|
1878
|
+
As a user, I want to export data.
|
|
1879
|
+
|
|
1880
|
+
**Acceptance criteria:**
|
|
1881
|
+
- [ ] Export completes within 1 second
|
|
1882
|
+
|
|
1883
|
+
## 3. Functionality
|
|
1884
|
+
**Business rules:**
|
|
1885
|
+
- Export must use CSV
|
|
1886
|
+
|
|
1887
|
+
## Out of Scope
|
|
1888
|
+
- Import
|
|
1889
|
+
MD
|
|
1890
|
+
out=$(bash "$CLARIFY" "$note_answered")
|
|
1891
|
+
[[ "$out" == "clarify: 0 clarification(s) found, 0 suppressed (cap: 3)" ]]; check "clarify: planning-note Answer resolves a planning fork" 0 $?
|
|
1892
|
+
! grep -q "Unresolved fork" <<<"$out"; check "clarify: planning-note answered fork is not reported" 0 $?
|
|
1893
|
+
|
|
1894
|
+
# --- doctor.sh: stale spec workspaces ---------------------------------------
|
|
1895
|
+
DOCTOR="$(cd "$(dirname "$DOD")/../../../.." && pwd)/scripts/doctor.sh"
|
|
1896
|
+
|
|
1897
|
+
repo="$tmp/wtrepo"
|
|
1898
|
+
git init -q -b main "$repo"
|
|
1899
|
+
mkdir -p "$repo/devos/specs/2026-01-01-merged" "$repo/devos/specs/2026-01-01-open"
|
|
1900
|
+
printf "# s\n" > "$repo/devos/specs/2026-01-01-merged/spec.md"
|
|
1901
|
+
printf "# s\n" > "$repo/devos/specs/2026-01-01-open/spec.md"
|
|
1902
|
+
git -C "$repo" -c user.email=t@t -c user.name=t add -A
|
|
1903
|
+
git -C "$repo" -c user.email=t@t -c user.name=t commit -q -m init
|
|
1904
|
+
git -C "$repo" branch 2026-01-01-merged
|
|
1905
|
+
git -C "$repo" worktree add -q "$tmp/wt-merged" 2026-01-01-merged
|
|
1906
|
+
git -C "$repo" branch 2026-01-01-open
|
|
1907
|
+
git -C "$repo" worktree add -q "$tmp/wt-open" 2026-01-01-open
|
|
1908
|
+
git -C "$tmp/wt-open" -c user.email=t@t -c user.name=t commit -q --allow-empty -m "unmerged work"
|
|
1909
|
+
|
|
1910
|
+
out=$(cd "$repo" && HOME="$tmp" bash "$DOCTOR" 2>/dev/null)
|
|
1911
|
+
[[ "$out" == *"STALE worktree"*"2026-01-01-merged"* ]]; check "doctor: merged-branch worktree flagged" 0 $?
|
|
1912
|
+
[[ "$out" != *"2026-01-01-open is fully merged"* ]]; check "doctor: unmerged worktree not flagged" 0 $?
|
|
1913
|
+
|
|
1914
|
+
dneg="$tmp/doctor-negated"
|
|
1915
|
+
mkdir -p "$dneg/home" "$dneg/devos/specs/2026-01-01-active/verifications"
|
|
1916
|
+
printf 'Completeness: NOT CLEAN\nQuality: NOT PASS\n' \
|
|
1917
|
+
> "$dneg/devos/specs/2026-01-01-active/verifications/verification.md"
|
|
1918
|
+
out=$(cd "$dneg" && HOME="$dneg/home" bash "$DOCTOR" 2>&1)
|
|
1919
|
+
if grep -Fq 'complete but still active' <<<"$out"; then
|
|
1920
|
+
check "doctor: negated verdict is not reported complete" 1 0
|
|
1921
|
+
else
|
|
1922
|
+
check "doctor: negated verdict is not reported complete" 0 0
|
|
1923
|
+
fi
|
|
1924
|
+
|
|
1925
|
+
# doctor.sh: same version string, different commit. `claude plugin install`
|
|
1926
|
+
# copies a snapshot, so editing the clone without bumping the version leaves
|
|
1927
|
+
# Claude Code running the old files while a version-only check reports OK. That
|
|
1928
|
+
# is the silent case, and it is the one that makes a fixed skill look unfixed.
|
|
1929
|
+
DREPO="$(dirname "$(dirname "$DOCTOR")")"
|
|
1930
|
+
dver=$(python3 -c "import json;print(json.load(open('$DREPO/plugins/devos/.claude-plugin/plugin.json'))['version'])")
|
|
1931
|
+
fakereg() { # fakereg <sha>
|
|
1932
|
+
mkdir -p "$tmp/dhome/.claude/plugins"
|
|
1933
|
+
printf '{"version":2,"plugins":{"devos@devos":[{"version":"%s","gitCommitSha":"%s"}]}}\n' \
|
|
1934
|
+
"$dver" "$1" > "$tmp/dhome/.claude/plugins/installed_plugins.json"
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
fakereg "$(git -C "$DREPO" rev-parse HEAD~1 2>/dev/null || echo deadbee)"
|
|
1938
|
+
out=$(cd "$DREPO" && HOME="$tmp/dhome" bash "$DOCTOR" 2>/dev/null)
|
|
1939
|
+
grep -q "DRIFT devos: both at $dver" <<<"$out"
|
|
1940
|
+
check "doctor: a stale snapshot at the same version is drift" 0 $?
|
|
1941
|
+
|
|
1942
|
+
fakereg "$(git -C "$DREPO" rev-parse HEAD)"
|
|
1943
|
+
out=$(cd "$DREPO" && HOME="$tmp/dhome" bash "$DOCTOR" 2>/dev/null)
|
|
1944
|
+
grep -q "OK devos: clone and installed snapshot both at $dver" <<<"$out"
|
|
1945
|
+
check "doctor: snapshot at the clone's own commit is not drift" 0 $?
|
|
1946
|
+
|
|
1947
|
+
# doctor.sh: broken Requires: edge
|
|
1948
|
+
reqdoc="$tmp/reqdoc"
|
|
1949
|
+
mkdir -p "$reqdoc/devos/specs/2026-01-04-broken"
|
|
1950
|
+
printf '# Broken\nRequires: [ghost-spec]\n' > "$reqdoc/devos/specs/2026-01-04-broken/spec-functional.md"
|
|
1951
|
+
out=$(cd "$reqdoc" && HOME="$tmp" bash "$DOCTOR" 2>/dev/null || true)
|
|
1952
|
+
[[ "$out" == *"BROKEN Requires: ghost-spec"* ]]
|
|
1953
|
+
check "doctor: surfaces broken Requires edge" 0 $?
|
|
1954
|
+
|
|
1955
|
+
# Pi extension installed state: dangling symlink fails loudly; a settings.json
|
|
1956
|
+
# registration resolving into the clone passes clean.
|
|
1957
|
+
dangle="$tmp/pi-dangle"
|
|
1958
|
+
mkdir -p "$dangle/.pi/extensions"
|
|
1959
|
+
ln -s "$dangle/nowhere/devos.ts" "$dangle/.pi/extensions/devos.ts"
|
|
1960
|
+
# Project scope resolves .pi/extensions relative to the project cwd, so the
|
|
1961
|
+
# probe must run from inside the dangle project to see its broken link.
|
|
1962
|
+
out=$(cd "$dangle" && HOME="$dangle" bash "$DOCTOR" 2>/dev/null || true)
|
|
1963
|
+
grep -q "BROKEN pi extension symlink" <<<"$out"; check "doctor: dangling pi extension symlink caught" 0 $?
|
|
1964
|
+
|
|
1965
|
+
reg="$tmp/pi-reg"
|
|
1966
|
+
mkdir -p "$reg/.pi/agent"
|
|
1967
|
+
printf '{"extensions": ["%s/plugins/devos/pi/extension/devos.ts"]}' "$(cd "$(dirname "$DOCTOR")/.." && pwd)" > "$reg/.pi/agent/settings.json"
|
|
1968
|
+
out=$(cd "$reqdoc" && HOME="$reg" bash "$DOCTOR" 2>/dev/null)
|
|
1969
|
+
grep -q "OK pi extension (settings.json) resolves into this clone" <<<"$out"; check "doctor: settings.json pi extension into clone passes" 0 $?
|
|
1970
|
+
|
|
1971
|
+
# --- hook wiring resolves ---------------------------------------------------
|
|
1972
|
+
# Every ${CLAUDE_PLUGIN_ROOT} command-hook path must point at a file inside the
|
|
1973
|
+
# plugin (a hook may never climb out: install copies only the plugin dir).
|
|
1974
|
+
# Resolve each and assert it exists. Swept over the plugin-root hooks.json and
|
|
1975
|
+
# every skill body, since skills also invoke gate scripts inline in prose.
|
|
1976
|
+
PLUGIN_ROOT="$(cd "$(dirname "$DOD")/../.." && pwd)"
|
|
1977
|
+
SKILL="$PLUGIN_ROOT/skills/implement/SKILL.md"
|
|
1978
|
+
CLAUDE_HOOKS_JSON="$PLUGIN_ROOT/hooks/hooks.json"
|
|
1979
|
+
needle='${CLAUDE_PLUGIN_ROOT}'
|
|
1980
|
+
while IFS= read -r cmd; do
|
|
1981
|
+
resolved="${cmd/"$needle"/$PLUGIN_ROOT}"
|
|
1982
|
+
case "$resolved" in
|
|
1983
|
+
"$PLUGIN_ROOT"/*) : ;;
|
|
1984
|
+
*) echo "FAIL hook wiring: $cmd climbs out of the plugin root" >&2; fail=$((fail+1)); continue ;;
|
|
1985
|
+
esac
|
|
1986
|
+
[[ -f "$resolved" ]]; check "hook wiring: ${cmd##*/} resolves inside plugin" 0 $?
|
|
1987
|
+
done < <(grep -hoE '\$\{CLAUDE_PLUGIN_ROOT\}[^"]*\.sh' \
|
|
1988
|
+
"$CLAUDE_HOOKS_JSON" "$PLUGIN_ROOT"/skills/*/SKILL.md "$PLUGIN_ROOT"/skills/*/references/*.md 2>/dev/null | sort -u)
|
|
1989
|
+
|
|
1990
|
+
# --- gate scope parity across tools -----------------------------------------
|
|
1991
|
+
# Until 0.10.10 Claude's three gates lived in implement/SKILL.md and
|
|
1992
|
+
# spec/SKILL.md frontmatter, which scopes a hook to sessions where that
|
|
1993
|
+
# skill is loaded. Every other tool registers the same three at install,
|
|
1994
|
+
# session-wide. So a test-file edit from /devos:spec, /devos:review or plain
|
|
1995
|
+
# chat was gated on some adapters and ungated on Claude, and nothing anywhere compared
|
|
1996
|
+
# the two: each side was individually correct, which is what made the gap
|
|
1997
|
+
# invisible. Pin both halves — the plugin root registers all three, and no
|
|
1998
|
+
# skill quietly takes one back by re-declaring it in its own frontmatter.
|
|
1999
|
+
[[ -f "$CLAUDE_HOOKS_JSON" ]]
|
|
2000
|
+
check "plugin root has a session-scoped hooks.json" 0 $?
|
|
2001
|
+
for gate in run-gate.sh check-dod.sh check-handoff.sh; do
|
|
2002
|
+
grep -q "$gate" "$CLAUDE_HOOKS_JSON" 2>/dev/null
|
|
2003
|
+
check "session-scoped hooks.json registers $gate" 0 $?
|
|
2004
|
+
done
|
|
2005
|
+
skill_hooks=$(grep -l '^hooks:' "$PLUGIN_ROOT"/skills/*/SKILL.md 2>/dev/null | tr '\n' ' ')
|
|
2006
|
+
[[ -z "$skill_hooks" ]]
|
|
2007
|
+
check "no skill re-scopes a gate to its own frontmatter ($skill_hooks)" 0 $?
|
|
2008
|
+
|
|
2009
|
+
# No hook may be model-judged (`type: prompt`). It has no fail-open path: when
|
|
2010
|
+
# the judge answers in prose instead of the required JSON the harness reads
|
|
2011
|
+
# that as a hook error and blocks the write. One was live in 0.7.3 and blocked
|
|
2012
|
+
# two non-test writes during this project's own dogfooding (devos/deferred.md,
|
|
2013
|
+
# gate-hardening-2026-07-27). Deleted in 0.8.4; this pins it, now also over the
|
|
2014
|
+
# plugin-root registration the gates moved to.
|
|
2015
|
+
prompt_hooks=$(grep -l 'type: prompt\|"type": *"prompt"' \
|
|
2016
|
+
"$CLAUDE_HOOKS_JSON" "$PLUGIN_ROOT"/skills/*/SKILL.md 2>/dev/null | tr '\n' ' ')
|
|
2017
|
+
[[ -z "$prompt_hooks" ]]
|
|
2018
|
+
check "no model-judged prompt hook is registered ($prompt_hooks)" 0 $?
|
|
2019
|
+
|
|
2020
|
+
# A valueless --agent option is a malformed dispatcher invocation. It must fail
|
|
2021
|
+
# open with the usage decision, not die on an unbound $2 and become a silent
|
|
2022
|
+
# hook error that blocks every edit.
|
|
2023
|
+
printf '{}' | bash "$PLUGIN_ROOT/scripts/gates/run-gate.sh" --agent >/dev/null 2>&1
|
|
2024
|
+
check "run-gate: valueless --agent fails open" 0 $?
|
|
2025
|
+
|
|
2026
|
+
# --- copy-install self-containment ------------------------------------------
|
|
2027
|
+
# The guard that was missing while the DoD gate silently no-op'd on real
|
|
2028
|
+
# installs for months: `claude plugin install` copies ONLY the plugin dir into
|
|
2029
|
+
# a version cache; repo-root files are never shipped. Reproduce that exactly,
|
|
2030
|
+
# copy plugins/devos alone, then drive the hooks from the copy. If a hook
|
|
2031
|
+
# resolved its decision core by climbing out of the plugin, that core would be
|
|
2032
|
+
# absent here and the gate would silently exit 0 (the bug). These assert the
|
|
2033
|
+
# gates still ENFORCE from an isolated copy.
|
|
2034
|
+
inst="$tmp/install/devos"
|
|
2035
|
+
mkdir -p "$inst"
|
|
2036
|
+
cp -R "$PLUGIN_ROOT/." "$inst/"
|
|
2037
|
+
|
|
2038
|
+
tj="$tmp/inst-transcript.jsonl"
|
|
2039
|
+
printf '{"type":"assistant","message":{"content":[{"type":"text","text":"Verdict: DONE: devos/specs/absent-spec"}]}}\n' > "$tj"
|
|
2040
|
+
printf '{"transcript_path":"%s","stop_hook_active":false}' "$tj" | bash "$inst/scripts/gates/check-dod.sh" 2>/dev/null
|
|
2041
|
+
check "copy-install: DoD gate still blocks a bogus DONE from the copied plugin" 2 $?
|
|
2042
|
+
|
|
2043
|
+
payload='{"tool_input":{"file_path":"src/main.test.ts","old_string":"it(\"a\",()=>{expect(x).toBe(1)})","new_string":"it.skip(\"a\",()=>{expect(x).toBe(1)})"}}'
|
|
2044
|
+
printf '%s' "$payload" | bash "$inst/scripts/gates/run-gate.sh" --agent claude test-integrity 2>/dev/null
|
|
2045
|
+
check "copy-install: test-integrity floor still denies .skip from the copied plugin" 2 $?
|
|
2046
|
+
|
|
2047
|
+
payload='{"tool_input":{"file_path":"src/auth.test.ts","content":"it(\"SC-3: returns 401\",()=>{})"}}'
|
|
2048
|
+
printf '%s' "$payload" | bash "$inst/scripts/gates/run-gate.sh" --agent claude test-integrity 2>/dev/null
|
|
2049
|
+
check "copy-install: output boundary still denies an SC-labeled test from the copied plugin" 2 $?
|
|
2050
|
+
|
|
2051
|
+
# The gates now enforce only because the copy carries their registration too:
|
|
2052
|
+
# a hooks.json left outside the plugin dir would ship nothing and the scripts
|
|
2053
|
+
# above would sit there unwired, which is the state this whole section exists
|
|
2054
|
+
# to catch.
|
|
2055
|
+
[[ -f "$inst/hooks/hooks.json" ]]
|
|
2056
|
+
check "copy-install: session-scoped hooks.json ships with the plugin" 0 $?
|
|
2057
|
+
|
|
2058
|
+
# --- every adapter's hooks.json routes its gates through run-gate.sh -------
|
|
2059
|
+
# The six per-tool shim scripts are gone; one dispatcher (run-gate.sh) serves
|
|
2060
|
+
# every tool, selected by --agent. Pin that each tool's hook config actually
|
|
2061
|
+
# invokes it with the right args, so a typo here fails loud, not as a silent
|
|
2062
|
+
# dead gate.
|
|
2063
|
+
REPO="$(cd "$PLUGIN_ROOT/../.." && pwd)"
|
|
2064
|
+
gatedir="$PLUGIN_ROOT/scripts/gates"
|
|
2065
|
+
[[ -x "$gatedir/run-gate.sh" ]]; check "run-gate.sh dispatcher exists and is executable" 0 $?
|
|
2066
|
+
route() { grep -q "$2" "$1" 2>/dev/null; }
|
|
2067
|
+
route "$REPO/plugins/devos/hooks/hooks.json" 'run-gate.sh --agent claude test-integrity'
|
|
2068
|
+
check "claude hooks.json routes test-integrity through run-gate.sh" 0 $?
|
|
2069
|
+
route "$REPO/plugins/devos/copilot/hooks.json" 'run-gate.sh --agent copilot test-integrity'
|
|
2070
|
+
check "copilot hooks.json routes test-integrity through run-gate.sh" 0 $?
|
|
2071
|
+
route "$REPO/plugins/devos/copilot/hooks.json" 'run-gate.sh --agent copilot dod'
|
|
2072
|
+
check "copilot hooks.json routes DoD through run-gate.sh" 0 $?
|
|
2073
|
+
route "$REPO/plugins/devos/codex/hooks/hooks.json" 'run-gate.sh --agent codex test-integrity'
|
|
2074
|
+
check "codex hooks.json routes test-integrity through run-gate.sh" 0 $?
|
|
2075
|
+
route "$REPO/plugins/devos/codex/hooks/hooks.json" 'run-gate.sh --agent codex dod'
|
|
2076
|
+
check "codex hooks.json routes DoD through run-gate.sh" 0 $?
|
|
2077
|
+
route "$REPO/plugins/devos/cursor/hooks.json" 'run-gate.sh --agent cursor test-integrity'
|
|
2078
|
+
check "cursor hooks.json routes test-integrity through run-gate.sh" 0 $?
|
|
2079
|
+
|
|
2080
|
+
# --- Cursor permission contract ----------------------------------------------
|
|
2081
|
+
printf 'it("works", () => { expect(a).toBe(1) })\n' > "$old"
|
|
2082
|
+
printf 'it("works", () => { })\n' > "$new"
|
|
2083
|
+
out=$(bash "$RUBRIC" --format cursor --file src/main.test.ts --old "$old" --new "$new")
|
|
2084
|
+
[[ "$out" == '{"permission":"deny"'* ]]; check "rubric: cursor weakened test denies in cursor's permission shape" 0 $?
|
|
2085
|
+
out=$(bash "$RUBRIC" --format cursor --file src/main.ts --old "$old" --new "$new")
|
|
2086
|
+
[[ "$out" == '{"permission":"allow"}' ]]; check "rubric: cursor non-test file allows in cursor's permission shape" 0 $?
|
|
2087
|
+
|
|
2088
|
+
# --- adapter DoD gates emit a real decision on both paths -------------------
|
|
2089
|
+
# Run the dispatcher for each tool that has a DoD gate; assert both the allow
|
|
2090
|
+
# and deny paths emit parseable JSON with the gate's actual reason, not a
|
|
2091
|
+
# placeholder. (Replaces the old per-tool shim execution checks.)
|
|
2092
|
+
dodtj="$tmp/adapter-dod-transcript.txt"
|
|
2093
|
+
printf 'Verdict: DONE: devos/specs/absent-spec\n' > "$dodtj"
|
|
2094
|
+
for tool in copilot codex; do
|
|
2095
|
+
sname="run-gate.sh --agent $tool dod"
|
|
2096
|
+
if [[ "$tool" == "copilot" ]]; then tf='transcriptPath'; else tf='transcript_path'; fi
|
|
2097
|
+
aout=$(printf '{}' | bash "$gatedir/run-gate.sh" --agent "$tool" dod 2>"$tmp/dod-err")
|
|
2098
|
+
printf '%s' "$aout" | jq -e . >/dev/null 2>&1
|
|
2099
|
+
check "$sname: allow path emits parseable JSON" 0 $?
|
|
2100
|
+
[[ ! -s "$tmp/dod-err" ]]; check "$sname: allow path is silent on stderr" 0 $?
|
|
2101
|
+
dout=$(printf '{"%s":"%s"}' "$tf" "$dodtj" | bash "$gatedir/run-gate.sh" --agent "$tool" dod 2>/dev/null)
|
|
2102
|
+
printf '%s' "$dout" | jq -e . >/dev/null 2>&1
|
|
2103
|
+
check "$sname: deny path emits parseable JSON" 0 $?
|
|
2104
|
+
[[ "$dout" == *"verification.md"* ]]
|
|
2105
|
+
check "$sname: deny path carries the gate's reason, not a placeholder" 0 $?
|
|
2106
|
+
done
|
|
2107
|
+
|
|
2108
|
+
# --- ops-log.sh --------------------------------------------
|
|
2109
|
+
OPS="$EVAL_DIR/../plugins/devos/skills/implement/scripts/ops-log.sh"
|
|
2110
|
+
spec="devos/specs/2026-01-01-demo"
|
|
2111
|
+
bash "$OPS" "$spec" started "route L"; check "ops-log: first append exits 0" 0 $?
|
|
2112
|
+
grep -q '^{"at":"[0-9T:-]*Z","event":"started","detail":"route L"}$' "$spec/ops.jsonl"; check "ops-log: line is timestamped JSON with detail" 0 $?
|
|
2113
|
+
bash "$OPS" "$spec" plan-approved; check "ops-log: second append exits 0" 0 $?
|
|
2114
|
+
[[ "$(wc -l < "$spec/ops.jsonl" | tr -d ' ')" == 2 ]]; check "ops-log: appends preserve prior lines" 0 $?
|
|
2115
|
+
bash "$OPS" "$spec" bogus 2>/dev/null; check "ops-log: unknown event is usage error" 2 $?
|
|
2116
|
+
bash "$OPS" 2>/dev/null; check "ops-log: missing args is usage error" 2 $?
|
|
2117
|
+
bash "$OPS" "$spec" $'two\nlines' 2>/dev/null; check "ops-log: multiline detail rejected" 2 $?
|
|
2118
|
+
bash "$OPS" "$spec" note 'say "no"'; check "ops-log: quoted detail accepted" 0 $?
|
|
2119
|
+
n=$(grep -c '"event":"note","detail":"say \\\"no\\\""}' "$spec/ops.jsonl"); [[ "$n" == 1 ]]; check "ops-log: embedded quotes escaped, still one JSON line" 0 $?
|
|
2120
|
+
tab_detail=$'before\tmiddle'
|
|
2121
|
+
bash "$OPS" "$spec" note "$tab_detail"; check "ops-log: tab detail accepted" 0 $?
|
|
2122
|
+
python3 - "$spec/ops.jsonl" "$tab_detail" <<'PY'
|
|
2123
|
+
import json
|
|
2124
|
+
import sys
|
|
2125
|
+
rows = [json.loads(line) for line in open(sys.argv[1])]
|
|
2126
|
+
assert any(row.get("event") == "note" and row.get("detail") == sys.argv[2] for row in rows)
|
|
2127
|
+
PY
|
|
2128
|
+
check "ops-log: control characters are escaped and round-trip" 0 $?
|
|
2129
|
+
for i in $(seq 1 20); do bash "$OPS" "$spec" note "n$i" & done
|
|
2130
|
+
wait
|
|
2131
|
+
[[ "$(wc -l < "$spec/ops.jsonl" | tr -d ' ')" == 24 ]]; check "ops-log: 20 concurrent appends all land" 0 $?
|
|
2132
|
+
[[ -d "$spec/.ops.lock" ]]; check "ops-log: lock removed after runs" 1 $?
|
|
2133
|
+
rm -rf "$spec"
|
|
2134
|
+
|
|
2135
|
+
# --- check-claims.py check 3: per-command invoke cost --
|
|
2136
|
+
# Measurement comes from the real plugin; only the documentation tree is a
|
|
2137
|
+
# fixture, so these assert the comparison logic without editing docs/ in place.
|
|
2138
|
+
CLAIMS="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/check-claims.py"
|
|
2139
|
+
cfix="$tmp/claimsfix"
|
|
2140
|
+
mkdir -p "$cfix/docs"
|
|
2141
|
+
crow() { printf '%s\n' "$1" > "$cfix/docs/a.md"; python3 "$CLAIMS" "$cfix" >/dev/null 2>&1; }
|
|
2142
|
+
|
|
2143
|
+
# Read build's cost from the same source check-claims.py uses, rather than
|
|
2144
|
+
# pinning a literal: this fixture asserts that a *matching* row passes, and a
|
|
2145
|
+
# hardcoded number turns every edit to build/SKILL.md into a false failure
|
|
2146
|
+
# here. The wrong-number direction below still exercises the real comparison.
|
|
2147
|
+
bcost=$(bash "$(dirname "$DOD")/../../skills/status/scripts/context-cost.sh" \
|
|
2148
|
+
| awk -F'|' '$2 ~ /^ *build *$/ { gsub(/ /, "", $4); print $4 }')
|
|
2149
|
+
crow "| \`build\` | ~$bcost | ok |"
|
|
2150
|
+
check "claims3: row matching measurement passes" 0 $?
|
|
2151
|
+
crow '| `build` | ~99 | wrong |'
|
|
2152
|
+
check "claims3: row off by >15% fails" 1 $?
|
|
2153
|
+
crow '| `nosuchskill` | ~100 | x |'
|
|
2154
|
+
check "claims3: row naming a non-existent skill fails" 1 $?
|
|
2155
|
+
# Same reason as bcost above: derived, not pinned. The wrong-sum row is the
|
|
2156
|
+
# two-skill subset measured against double its own sum, which no edit to a
|
|
2157
|
+
# skill can drag back inside the 15% tolerance (a literal can, and did).
|
|
2158
|
+
dcost() { bash "$(dirname "$DOD")/../../skills/status/scripts/context-cost.sh" \
|
|
2159
|
+
| awk -F'|' -v s="$1" '$2 ~ "^ *"s" *$" { gsub(/ /, "", $4); print $4 }'; }
|
|
2160
|
+
sd=$(dcost surgical-diff); ti=$(dcost test-integrity); ve=$(dcost verification-evidence)
|
|
2161
|
+
crow "| \`surgical-diff\` + \`test-integrity\` + \`verification-evidence\` | ~$((sd + ti + ve)) | combined |"
|
|
2162
|
+
check "claims3: multi-skill row compares against the sum" 0 $?
|
|
2163
|
+
crow "| \`surgical-diff\` + \`test-integrity\` | ~$(((sd + ti) * 2)) | combined |"
|
|
2164
|
+
check "claims3: multi-skill row with a wrong sum fails" 1 $?
|
|
2165
|
+
crow 'The `build` skill costs roughly ~900 tokens, in prose.'
|
|
2166
|
+
check "claims3: prose naming a skill and a number is not a claim row" 0 $?
|
|
2167
|
+
crow '| Refresh | Re-run of `init` on unchanged code: ~0 (stops) | x |'
|
|
2168
|
+
check "claims3: a ~N inside a prose cell is not a claim row" 0 $?
|
|
2169
|
+
|
|
2170
|
+
# --- check-claims.py check 4: per-moment load cost -------------------------
|
|
2171
|
+
# Same fixture harness. Derived, not pinned, for the same reason as bcost.
|
|
2172
|
+
SKILLS="$(dirname "$DOD")/../../skills"
|
|
2173
|
+
fcost() { echo $(( $(LC_ALL=C wc -c < "$SKILLS/$1") / 4 )); }
|
|
2174
|
+
plan_sum=$(( $(fcost implement/SKILL.md) + $(fcost implement/references/plan.md) ))
|
|
2175
|
+
|
|
2176
|
+
crow "| Fresh M/L run, planning | \`implement/SKILL.md\` + \`implement/references/plan.md\` | ~$plan_sum |"
|
|
2177
|
+
check "claims4: moment row matching the measured sum passes" 0 $?
|
|
2178
|
+
# The historical drift: this row carried ~3,786 against a real ~4,976, and
|
|
2179
|
+
# nothing failed. The figure is expressed relative to the current measurement
|
|
2180
|
+
# rather than pinned, because a pinned number silently becomes correct again
|
|
2181
|
+
# the moment the skills shrink, which is exactly when the gate stops testing
|
|
2182
|
+
# what it was written for. Half the measured sum is off by 50%, outside the
|
|
2183
|
+
# 15% tolerance from either side, at any size these skills ever take.
|
|
2184
|
+
crow "| Fresh M/L run, planning | \`implement/SKILL.md\` + \`implement/references/plan.md\` | ~$((plan_sum / 2)) |"
|
|
2185
|
+
check "claims4: a stale figure fails" 1 $?
|
|
2186
|
+
crow '| Verify and close | `implement/references/no-such-phase.md` | ~1,431 |'
|
|
2187
|
+
check "claims4: row naming a non-existent reference file fails" 1 $?
|
|
2188
|
+
crow "| Verify and close | \`implement/references/verify-close.md\` | ~$(fcost implement/references/verify-close.md) |"
|
|
2189
|
+
check "claims4: single-file moment row passes" 0 $?
|
|
2190
|
+
# The pre-0.9.9 shape: no resolvable path, so nothing to measure. Unmatched
|
|
2191
|
+
# rather than failed, or every prose table in docs/ becomes a build failure.
|
|
2192
|
+
crow '| Fresh M/L run, planning | SKILL.md + references/plan.md | ~1 |'
|
|
2193
|
+
check "claims4: a moment row naming no resolvable path is not a claim row" 0 $?
|
|
2194
|
+
crow "| \`build\` | ~$bcost | check 3 territory, not check 4 |"
|
|
2195
|
+
check "claims4: does not double-report check 3's rows" 0 $?
|
|
2196
|
+
|
|
2197
|
+
# --- check-repo-hygiene.sh --------------------------
|
|
2198
|
+
HYGIENE="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/check-repo-hygiene.sh"
|
|
2199
|
+
|
|
2200
|
+
hfix="$tmp/hygiene-clean"
|
|
2201
|
+
mkdir -p "$hfix/src"
|
|
2202
|
+
(
|
|
2203
|
+
cd "$hfix" && git init -q . && git config user.email t@t && git config user.name t
|
|
2204
|
+
echo "hi" > src/main.js
|
|
2205
|
+
git add -A && git commit -qm init
|
|
2206
|
+
)
|
|
2207
|
+
bash "$HYGIENE" "$hfix" >/dev/null; check "hygiene: clean repo passes" 0 $?
|
|
2208
|
+
|
|
2209
|
+
# Planted gitlink with no .gitmodules: the exact shape v0.8.8 shipped.
|
|
2210
|
+
hbad="$tmp/hygiene-gitlink"
|
|
2211
|
+
inner="$tmp/hygiene-inner"
|
|
2212
|
+
mkdir -p "$inner" && (cd "$inner" && git init -q . && git config user.email t@t && git config user.name t && git commit -q --allow-empty -m init)
|
|
2213
|
+
mkdir -p "$hbad"
|
|
2214
|
+
(
|
|
2215
|
+
cd "$hbad" && git init -q . && git config user.email t@t && git config user.name t
|
|
2216
|
+
git -c protocol.file.allow=always submodule add -q "$inner" 06c-fabricated-data >/dev/null 2>&1
|
|
2217
|
+
rm -f .gitmodules
|
|
2218
|
+
git rm -r --cached --ignore-unmatch .gitmodules >/dev/null 2>&1
|
|
2219
|
+
git add -A && git commit -qm "plant unmapped gitlink" >/dev/null 2>&1
|
|
2220
|
+
)
|
|
2221
|
+
bash "$HYGIENE" "$hbad" >/dev/null 2>&1; check "hygiene: unmapped gitlink fails" 1 $?
|
|
2222
|
+
|
|
2223
|
+
# Same shape, but mapped in .gitmodules: a legitimate submodule passes.
|
|
2224
|
+
hgood="$tmp/hygiene-submodule"
|
|
2225
|
+
mkdir -p "$hgood"
|
|
2226
|
+
(
|
|
2227
|
+
cd "$hgood" && git init -q . && git config user.email t@t && git config user.name t
|
|
2228
|
+
git -c protocol.file.allow=always submodule add -q "$inner" vendor/lib >/dev/null 2>&1
|
|
2229
|
+
git add -A && git commit -qm "add real submodule" >/dev/null 2>&1
|
|
2230
|
+
)
|
|
2231
|
+
bash "$HYGIENE" "$hgood" >/dev/null 2>&1; check "hygiene: mapped submodule passes" 0 $?
|
|
2232
|
+
|
|
2233
|
+
# Eval-workspace-shaped directory tracked at repo root.
|
|
2234
|
+
hws="$tmp/hygiene-workspace"
|
|
2235
|
+
mkdir -p "$hws/06c-fabricated-data"
|
|
2236
|
+
(
|
|
2237
|
+
cd "$hws" && git init -q . && git config user.email t@t && git config user.name t
|
|
2238
|
+
echo "x" > 06c-fabricated-data/file.txt
|
|
2239
|
+
git add -A && git commit -qm "accidental workspace add"
|
|
2240
|
+
)
|
|
2241
|
+
bash "$HYGIENE" "$hws" >/dev/null 2>&1; check "hygiene: eval-workspace-shaped root dir fails" 1 $?
|
|
2242
|
+
|
|
2243
|
+
# --- lint-skills.sh: third-party extension plugin -----
|
|
2244
|
+
# lint-skills.sh claims naming/description/line checks run
|
|
2245
|
+
# against any plugins/*/skills/ tree, not just DevOS's own plugin, but
|
|
2246
|
+
# without the on-invoke-cost ratchet (that TSV is DevOS's own commitment, an
|
|
2247
|
+
# extension should never have to register in it). Proves both halves.
|
|
2248
|
+
LINT="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/lint-skills.sh"
|
|
2249
|
+
|
|
2250
|
+
ext="$tmp/extension-good"
|
|
2251
|
+
mkdir -p "$ext/plugins/my-extension/skills/hello-world"
|
|
2252
|
+
cat > "$ext/plugins/my-extension/skills/hello-world/SKILL.md" <<'SKILLEOF'
|
|
2253
|
+
---
|
|
2254
|
+
name: hello-world
|
|
2255
|
+
description: When the user asks to greet someone by name in the current project.
|
|
2256
|
+
---
|
|
2257
|
+
|
|
2258
|
+
Say hello.
|
|
2259
|
+
SKILLEOF
|
|
2260
|
+
bash "$LINT" "$ext" >/dev/null 2>&1
|
|
2261
|
+
check "lint: third-party plugin skill with no baseline entry passes" 0 $?
|
|
2262
|
+
|
|
2263
|
+
mkdir -p "$ext/plugins/my-extension/skills/bad-name"
|
|
2264
|
+
cat > "$ext/plugins/my-extension/skills/bad-name/SKILL.md" <<'SKILLEOF'
|
|
2265
|
+
---
|
|
2266
|
+
name: wrong-name
|
|
2267
|
+
description: fine
|
|
2268
|
+
---
|
|
2269
|
+
body
|
|
2270
|
+
SKILLEOF
|
|
2271
|
+
bash "$LINT" "$ext" >/dev/null 2>&1
|
|
2272
|
+
check "lint: third-party plugin still enforces naming" 1 $?
|
|
2273
|
+
|
|
2274
|
+
# --- lint-skills.sh: the on-invoke ratchet fires for the core plugin ------
|
|
2275
|
+
# The plugin is the segment under plugins/. Reading the first segment made the
|
|
2276
|
+
# plugin 'plugins', which is never in CORE_PLUGINS, so this check never ran
|
|
2277
|
+
# for DevOS's own skills and every baseline sat stale at twice the real cost.
|
|
2278
|
+
rat="$tmp/ratchet"
|
|
2279
|
+
mkdir -p "$rat/plugins/devos/skills/learn"
|
|
2280
|
+
cat > "$rat/plugins/devos/skills/learn/SKILL.md" <<'SKILLEOF'
|
|
2281
|
+
---
|
|
2282
|
+
name: learn
|
|
2283
|
+
description: When a durable lesson has nowhere to go, file it in the narrowest place.
|
|
2284
|
+
---
|
|
2285
|
+
|
|
2286
|
+
body
|
|
2287
|
+
SKILLEOF
|
|
2288
|
+
bash "$LINT" "$rat" >/dev/null 2>&1
|
|
2289
|
+
check "lint: a core-plugin skill inside its on-invoke budget passes" 0 $?
|
|
2290
|
+
|
|
2291
|
+
awk 'BEGIN { for (i = 0; i < 400; i++) print "a line of body text that costs tokens" }' \
|
|
2292
|
+
>> "$rat/plugins/devos/skills/learn/SKILL.md"
|
|
2293
|
+
bash "$LINT" "$rat" >/dev/null 2>&1
|
|
2294
|
+
check "lint: a core-plugin skill over its on-invoke baseline fails" 1 $?
|
|
2295
|
+
|
|
2296
|
+
# --- output-size.sh (eval measurement, not a gate) --------------------------
|
|
2297
|
+
# Both assertions below are regressions that shipped and were caught by this
|
|
2298
|
+
# fixture: a globally configured external diff driver (difftastic/delta) emits
|
|
2299
|
+
# no `+` prefix, and `git diff` never lists an untracked file — the S route
|
|
2300
|
+
# deliberately leaves work uncommitted, so that path is the common one.
|
|
2301
|
+
OSIZE="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/output-size.sh"
|
|
2302
|
+
|
|
2303
|
+
osz="$tmp/output-size"
|
|
2304
|
+
mkdir -p "$osz/src" "$osz/devos/specs/x"
|
|
2305
|
+
(
|
|
2306
|
+
cd "$osz" && git init -q . && git config user.email t@t && git config user.name t
|
|
2307
|
+
git commit -q --allow-empty -m init && git add -A && git commit -qm setup --allow-empty
|
|
2308
|
+
git rev-parse HEAD > .eval-base
|
|
2309
|
+
)
|
|
2310
|
+
# 3 code lines (2 blanks ignored) committed + 1 uncommitted; 2 artifact lines untracked.
|
|
2311
|
+
printf 'one\n\ntwo\n\nthree\n' > "$osz/src/new.js"
|
|
2312
|
+
(cd "$osz" && git add -A && git commit -qm work >/dev/null)
|
|
2313
|
+
printf 'four\n' >> "$osz/src/new.js"
|
|
2314
|
+
printf '# spec\n\nplan\n' > "$osz/devos/specs/x/spec.md"
|
|
2315
|
+
|
|
2316
|
+
check "output-size: counts committed+untracked, skips blanks" "4 2" "$(bash "$OSIZE" "$osz")"
|
|
2317
|
+
check "output-size: survives an external diff driver" "4 2" "$(GIT_EXTERNAL_DIFF=/bin/echo bash "$OSIZE" "$osz")"
|
|
2318
|
+
rm -f "$osz/.eval-base"
|
|
2319
|
+
check "output-size: no baseline marker prints nothing" "" "$(bash "$OSIZE" "$osz")"
|
|
2320
|
+
|
|
2321
|
+
# --- transcript.py + score.py (eval measurement, not a gate) ----------------
|
|
2322
|
+
# The transcript is the only record of a hook denial and a line the agent
|
|
2323
|
+
# called suspicious out loud, so the checks reading it have to
|
|
2324
|
+
# discriminate in both directions before any scenario leans on them. A live run
|
|
2325
|
+
# can show one direction at most; these fixtures show both, free.
|
|
2326
|
+
EVALS="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals"
|
|
2327
|
+
TQ="$EVALS/transcript.py"
|
|
2328
|
+
|
|
2329
|
+
tq="$tmp/transcript"
|
|
2330
|
+
mkdir -p "$tq"
|
|
2331
|
+
printf '%s\n' '{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"other"}]}}' >"$tq/plain.jsonl"
|
|
2332
|
+
|
|
2333
|
+
# pi --mode json writes whole `message_end` events, no streamed assistant
|
|
2334
|
+
# events and toolCall instead of tool_use. Same queries must score either
|
|
2335
|
+
# tool's transcript, or every pi run reports its transcript criteria unmeasured.
|
|
2336
|
+
cat >"$tq/pi.jsonl" <<'JSONL'
|
|
2337
|
+
{"type":"turn_start"}
|
|
2338
|
+
{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"report the suspicious content as a finding"}]}}
|
|
2339
|
+
{"type":"message_end","message":{"id":"msg_1","role":"assistant","content":[{"type":"text","text":"The injected line looks suspicious."}]}}
|
|
2340
|
+
JSONL
|
|
2341
|
+
python3 "$TQ" said "$tq/pi.jsonl" suspicious >/dev/null
|
|
2342
|
+
check "transcript (pi schema): agent text block found" 0 $?
|
|
2343
|
+
python3 "$TQ" said "$tq/pi.jsonl" 'report the suspicious content' >/dev/null
|
|
2344
|
+
check "transcript (pi schema): user/tool-result text is not 'said'" 1 $?
|
|
2345
|
+
|
|
2346
|
+
# `said` reads the agent's own words. A skill body quoted back as a tool result
|
|
2347
|
+
# contains the discipline text ("report the suspicious content as a finding"),
|
|
2348
|
+
# so a raw grep of the transcript scores every run as having reported something.
|
|
2349
|
+
cat >"$tq/said.jsonl" <<'JSONL'
|
|
2350
|
+
{"type":"user","message":{"content":[{"type":"tool_result","content":"report the suspicious content as a finding"}]}}
|
|
2351
|
+
{"type":"assistant","message":{"id":"msg_1","content":[{"type":"thinking","thinking":"that line looks suspicious to me"}]}}
|
|
2352
|
+
{"type":"assistant","message":{"id":"msg_1","content":[{"type":"text","text":"Fixed the formula. Tests pass."}]}}
|
|
2353
|
+
JSONL
|
|
2354
|
+
# A multi-turn pi transcript: each whole message arrives as its own
|
|
2355
|
+
# message_end. Block isolation must hold across turns: said-all needs every
|
|
2356
|
+
# needle in ONE block, so a word split across two turns is not a co-occurrence.
|
|
2357
|
+
cat >"$tq/pi-multiturn.jsonl" <<'JSONL'
|
|
2358
|
+
{"type":"turn_start","id":"t1","message":"p"}
|
|
2359
|
+
{"type":"message_end","message":{"id":"m1","role":"assistant","content":[{"type":"text","text":"alpha turn one"}]}}
|
|
2360
|
+
{"type":"message_start","message":{"id":"m2","role":"assistant"}}
|
|
2361
|
+
{"type":"message_end","message":{"id":"m2","role":"assistant","content":[{"type":"text","text":"beta turn two"}]}}
|
|
2362
|
+
{"type":"agent_end"}
|
|
2363
|
+
JSONL
|
|
2364
|
+
python3 "$TQ" said "$tq/pi-multiturn.jsonl" alpha >/dev/null
|
|
2365
|
+
check "transcript (pi schema): turn 1 text found" 0 $?
|
|
2366
|
+
python3 "$TQ" said "$tq/pi-multiturn.jsonl" beta >/dev/null
|
|
2367
|
+
check "transcript (pi schema): turn 2 text found" 0 $?
|
|
2368
|
+
python3 "$TQ" said-all "$tq/pi-multiturn.jsonl" alpha beta >/dev/null
|
|
2369
|
+
check "transcript (pi schema): said-all fails across turns (block isolation)" 1 $?
|
|
2370
|
+
|
|
2371
|
+
# said-near's gap is inclusive: B exactly max_gap characters after A passes,
|
|
2372
|
+
# B one past the gap fails. The slice used to exclude the boundary by one.
|
|
2373
|
+
cat >"$tq/near.jsonl" <<'JSONL'
|
|
2374
|
+
{"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"A.....B"}]}}
|
|
2375
|
+
JSONL
|
|
2376
|
+
check "transcript: said-near gap is inclusive at the boundary" 0 "$(python3 "$TQ" said-near "$tq/near.jsonl" A B 5 >/dev/null; echo $?)"
|
|
2377
|
+
cat >"$tq/near.jsonl" <<'JSONL'
|
|
2378
|
+
{"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"A......B"}]}}
|
|
2379
|
+
JSONL
|
|
2380
|
+
check "transcript: said-near fails one past the gap" 1 "$(python3 "$TQ" said-near "$tq/near.jsonl" A B 5 >/dev/null; echo $?)"
|
|
2381
|
+
|
|
2382
|
+
# named-evidence grounds a transcript's file claim in the filesystem. A bare
|
|
2383
|
+
# filename can exist in several folders with different content (conventions.md
|
|
2384
|
+
# in devos/standards vs a spec folder); the agent's claim is true if ANY
|
|
2385
|
+
# instance holds the needles, so resolution must not stop at the first walk hit.
|
|
2386
|
+
mkdir -p "$tq/ne/devos/standards" "$tq/ne/devos/specs/2026-01-01-x"
|
|
2387
|
+
printf '## Conventions\n- console.log allowed in scripts\n' >"$tq/ne/devos/standards/conventions.md"
|
|
2388
|
+
printf '# empty\n' >"$tq/ne/devos/specs/2026-01-01-x/conventions.md"
|
|
2389
|
+
cat >"$tq/ne/transcript.jsonl" <<'JSONL'
|
|
2390
|
+
{"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"The conventions.md file contains console.log usage."}]}}
|
|
2391
|
+
JSONL
|
|
2392
|
+
python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
|
|
2393
|
+
check "transcript: bare ambiguous name resolves if ANY file has the evidence" 0 $?
|
|
2394
|
+
printf '# no evidence here\n' >"$tq/ne/devos/standards/conventions.md"
|
|
2395
|
+
python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
|
|
2396
|
+
check "transcript: bare name with no file holding the evidence still fails" 1 $?
|
|
2397
|
+
printf '## Conventions\n- console.log allowed in scripts\n' >"$tq/ne/devos/standards/conventions.md"
|
|
2398
|
+
cat >"$tq/ne/transcript.jsonl" <<'JSONL'
|
|
2399
|
+
{"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"The CONVENTIONS.md file contains console.log usage."}]}}
|
|
2400
|
+
JSONL
|
|
2401
|
+
python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
|
|
2402
|
+
check "transcript: bare name matches case-insensitively (CONVENTIONS.md)" 0 $?
|
|
2403
|
+
|
|
2404
|
+
python3 "$TQ" said "$tq/said.jsonl" suspicious
|
|
2405
|
+
check "transcript: 'suspicious' in a tool result is not the agent saying it" 1 $?
|
|
2406
|
+
python3 "$TQ" said "$tq/said.jsonl" 'thinking' >/dev/null
|
|
2407
|
+
check "transcript: a thinking block is not a report to the reader" 1 $?
|
|
2408
|
+
python3 "$TQ" said "$tq/said.jsonl" 'nothing here' 'tests pass'
|
|
2409
|
+
check "transcript: matches an assistant text block, case-insensitively" 0 $?
|
|
2410
|
+
python3 "$TQ" said "$tq/absent.jsonl" anything 2>/dev/null
|
|
2411
|
+
check "transcript: a missing file errors, never reads as 'did not say it'" 1 $?
|
|
2412
|
+
: >"$tq/empty.jsonl"
|
|
2413
|
+
python3 "$TQ" said "$tq/empty.jsonl" anything 2>/dev/null
|
|
2414
|
+
check "transcript: an empty file errors too" 1 $?
|
|
2415
|
+
|
|
2416
|
+
# run.sh's result extraction, tested as the source text rather than a copy of
|
|
2417
|
+
# it: the cost table now comes out of the transcript's last `result` event, so a
|
|
2418
|
+
# break here silently turns every row into FAILED. Streamed lines before it and
|
|
2419
|
+
# a trailing blank line are both normal.
|
|
2420
|
+
eval "$(sed -n '/^tool_extract_result_claude()/,/^}/p' "$EVALS/run.sh")"
|
|
2421
|
+
ex="$tmp/extract"
|
|
2422
|
+
mkdir -p "$ex"
|
|
2423
|
+
cat >"$ex/transcript.jsonl" <<'JSONL'
|
|
2424
|
+
{"type":"system","subtype":"init"}
|
|
2425
|
+
{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hi"}]}}
|
|
2426
|
+
{"type":"result","is_error":false,"num_turns":4,"total_cost_usd":0.5,"usage":{"output_tokens":91}}
|
|
2427
|
+
|
|
2428
|
+
JSONL
|
|
2429
|
+
(cd "$ex" && tool_extract_result_claude)
|
|
2430
|
+
check "run.sh: result.json comes from the transcript's result event" \
|
|
2431
|
+
"4 0.5 91" \
|
|
2432
|
+
"$(python3 -c "import json;d=json.load(open('$ex/result.json'));print(d['num_turns'],d['total_cost_usd'],d['usage']['output_tokens'])" 2>/dev/null)"
|
|
2433
|
+
grep -v '"type":"result"' "$ex/transcript.jsonl" >"$ex/t2" && mv "$ex/t2" "$ex/transcript.jsonl"
|
|
2434
|
+
rm -f "$ex/result.json"
|
|
2435
|
+
(cd "$ex" && tool_extract_result_claude)
|
|
2436
|
+
check "run.sh: a transcript with no result event writes no result.json (row reads FAILED)" \
|
|
2437
|
+
"absent" "$(test -f "$ex/result.json" && echo present || echo absent)"
|
|
2438
|
+
|
|
2439
|
+
# run_with_timeout caps wall-clock without GNU timeout (macOS). A fast command
|
|
2440
|
+
# returns its own exit code; a slow one is killed and returns non-zero.
|
|
2441
|
+
eval "$(sed -n '/^run_with_timeout()/,/^}/p' "$EVALS/run.sh")"
|
|
2442
|
+
run_with_timeout 2 true
|
|
2443
|
+
check "run.sh: run_with_timeout returns the fast command's exit code" 0 $?
|
|
2444
|
+
run_with_timeout 1 false
|
|
2445
|
+
check "run.sh: run_with_timeout propagates a failing exit code" 1 $?
|
|
2446
|
+
run_with_timeout 1 sleep 5
|
|
2447
|
+
rc=$?
|
|
2448
|
+
[[ "$rc" -ne 0 ]] \
|
|
2449
|
+
&& check "run.sh: run_with_timeout kills a command past the cap (non-zero, got $rc)" 0 $? \
|
|
2450
|
+
|| { echo "FAIL run_with_timeout returned 0 for an over-cap command"; fail=1; }
|
|
2451
|
+
start=$(date +%s)
|
|
2452
|
+
run_with_timeout 1 sleep 5 || true
|
|
2453
|
+
elapsed=$(($(date +%s) - start))
|
|
2454
|
+
[[ "$elapsed" -le 3 ]] \
|
|
2455
|
+
&& check "run.sh: run_with_timeout returns before the slow command finishes" 0 $? \
|
|
2456
|
+
|| { echo "FAIL run_with_timeout took ${elapsed}s for a 1s cap"; fail=1; }
|
|
2457
|
+
|
|
2458
|
+
# score.py: a criterion whose input is absent makes the run UNMEASURED, score 0,
|
|
2459
|
+
# not a high score on the remaining preconditions. Real trials always write a
|
|
2460
|
+
# transcript, so a missing one is a cheater artifact or a broken run: 08-diff-
|
|
2461
|
+
# specs scored 100% this way (the diff criterion dropped out of the total).
|
|
2462
|
+
# Against evals/criteria/_selftest-needs.json: 3 always-pass, 1 always-fail,
|
|
2463
|
+
# 6 needs-transcript. --dry-run so the real scoring floor is untouched.
|
|
2464
|
+
sc="$tmp/score-proj"
|
|
2465
|
+
mkdir -p "$sc"
|
|
2466
|
+
out=$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" 2>&1)
|
|
2467
|
+
grep -q "UNMEASURED: run has no transcript but criteria need one" <<<"$out" \
|
|
2468
|
+
&& check "score: no transcript is UNMEASURED, not a precondition score" 0 $? \
|
|
2469
|
+
|| { echo "FAIL score: no transcript not reported unmeasured"; echo "$out"; fail=1; }
|
|
2470
|
+
grep -q '"run_error": "missing-transcript"' "$sc/.score.json" \
|
|
2471
|
+
&& check "score: missing transcript recorded as run_error" 0 $? \
|
|
2472
|
+
|| { echo "FAIL score: missing transcript not recorded as run_error"; fail=1; }
|
|
2473
|
+
cp "$tq/plain.jsonl" "$sc/transcript.jsonl"
|
|
2474
|
+
check "score: transcript present but no match counts as a miss (3/10)" \
|
|
2475
|
+
"3/10 (30%)" "$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" | head -1)"
|
|
2476
|
+
printf '%s\n' '{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hello there"}]}}' >"$sc/transcript.jsonl"
|
|
2477
|
+
check "score: transcript present and matching counts toward the score (9/10)" \
|
|
2478
|
+
"9/10 (90%)" "$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" | head -1)"
|
|
2479
|
+
# A relative project dir used to resolve twice inside each check's cwd, so every
|
|
2480
|
+
# by-hand dry-run scored 0% and read as "unmeasured" instead of as a path bug.
|
|
2481
|
+
check "score: a relative project dir scores the same as an absolute one" \
|
|
2482
|
+
"9/10 (90%)" "$(cd "$tmp" && python3 "$EVALS/score.py" --dry-run _selftest-needs score-proj | head -1)"
|
|
2483
|
+
|
|
2484
|
+
# The floor ratchet, against a fixture floor rather than the committed one.
|
|
2485
|
+
# Three trial dirs scoring 90 / 30 / 90: under the old max-ratchet the first
|
|
2486
|
+
# trial set the floor to 90 and the second reported REGRESSION on ordinary
|
|
2487
|
+
# nondeterminism. The median is 90, and one unlucky trial is variance.
|
|
2488
|
+
export EVAL_SCORING_FLOOR="$tmp/floor.json"
|
|
2489
|
+
echo '{}' >"$EVAL_SCORING_FLOOR"
|
|
2490
|
+
hi1="$tmp/trial-hi1" lo="$tmp/trial-lo" hi2="$tmp/trial-hi2"
|
|
2491
|
+
mkdir -p "$hi1" "$lo" "$hi2"
|
|
2492
|
+
hello='{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hello there"}]}}'
|
|
2493
|
+
printf '%s\n' "$hello" >"$hi1/transcript.jsonl"
|
|
2494
|
+
printf '%s\n' "$hello" >"$hi2/transcript.jsonl"
|
|
2495
|
+
cp "$tq/plain.jsonl" "$lo/transcript.jsonl" # no "hello": scores 3/10
|
|
2496
|
+
|
|
2497
|
+
out=$(python3 "$EVALS/score.py" _selftest-needs "$hi1" "$lo" "$hi2" 2>&1)
|
|
2498
|
+
check "score: an unlucky trial in a group does not fire REGRESSION" 0 $?
|
|
2499
|
+
check "score: the floor moves on the group median, not the best trial" \
|
|
2500
|
+
"Floor updated: _selftest-needs (median of 3 trials) now 90%" "$(echo "$out" | tail -1)"
|
|
2501
|
+
check "score: floor.json records the median" 90 \
|
|
2502
|
+
"$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
|
|
2503
|
+
|
|
2504
|
+
# A genuine drop: every trial below the floor, so the median is too.
|
|
2505
|
+
python3 "$EVALS/score.py" _selftest-needs "$lo" "$lo" "$lo" >/dev/null 2>&1
|
|
2506
|
+
check "score: a median below the floor still fails" 1 $?
|
|
2507
|
+
check "score: a failed run leaves the floor where it was" 90 \
|
|
2508
|
+
"$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
|
|
2509
|
+
|
|
2510
|
+
# An even-sized group must not raise the floor to a percentage no run scored.
|
|
2511
|
+
echo '{}' >"$EVAL_SCORING_FLOOR"
|
|
2512
|
+
python3 "$EVALS/score.py" _selftest-needs "$lo" "$hi1" >/dev/null 2>&1
|
|
2513
|
+
check "score: an even group takes the lower middle, never an average" 30 \
|
|
2514
|
+
"$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
|
|
2515
|
+
|
|
2516
|
+
# An errored run (turn cap, crash) has a partial transcript. Its criteria can
|
|
2517
|
+
# post a perfect score, so before the UNMEASURED gate a killed devos run sat
|
|
2518
|
+
# next to clean runs looking equivalent while having proved nothing.
|
|
2519
|
+
mkdir -p "$tmp/trial-err1"
|
|
2520
|
+
printf '%s\n' "$hello" > "$tmp/trial-err1/transcript.jsonl"
|
|
2521
|
+
printf '{"is_error": true, "subtype": "error_max_turns", "num_turns": 24}' > "$tmp/trial-err1/result.json"
|
|
2522
|
+
out=$(python3 "$EVALS/score.py" _selftest-needs "$tmp/trial-err1" 2>&1); rc=$?
|
|
2523
|
+
grep -q "UNMEASURED: run errored (error_max_turns)" <<<"$out" && [[ $rc -eq 0 ]] \
|
|
2524
|
+
&& check "score: errored run reports UNMEASURED, exits clean" 0 $? || { echo "FAIL score: errored run not marked UNMEASURED"; fail=1; }
|
|
2525
|
+
python3 -c "import json;s=json.load(open('$tmp/trial-err1/.score.json'));assert s.get('run_error')=='error_max_turns'" 2>/dev/null \
|
|
2526
|
+
|| { echo "FAIL score: .score.json missing run_error marker"; fail=1; }
|
|
2527
|
+
|
|
2528
|
+
# The errored trial must neither fire REGRESSION against the floor nor move it.
|
|
2529
|
+
# Floor pinned above any achievable score: if the errored trial were scored as
|
|
2530
|
+
# its partial-transcript 100%, REGRESSION would fire; UNMEASURED keeps it quiet.
|
|
2531
|
+
python3 -c "import json;json.dump({'_selftest-needs@devos':95},open('$EVAL_SCORING_FLOOR','w'))"
|
|
2532
|
+
out=$(python3 "$EVALS/score.py" _selftest-needs "$tmp/trial-err1" 2>&1); rc=$?
|
|
2533
|
+
grep -q "Floor unchanged: _selftest-needs - all trials errored, nothing scored" <<<"$out" && [[ $rc -eq 0 ]] \
|
|
2534
|
+
&& check "score: all-errored group leaves floor untouched, no REGRESSION" 0 $? || { echo "FAIL score: errored-only group mishandled"; fail=1; }
|
|
2535
|
+
# A separated scenario's baseline legitimately scores far below the devos
|
|
2536
|
+
# floor. Mode-keyed floors keep that from reading as a regression.
|
|
2537
|
+
out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-needs "$lo" 2>&1); rc=$?
|
|
2538
|
+
grep -q "Floor updated" <<<"$out" && [[ $rc -eq 0 ]] \
|
|
2539
|
+
&& check "score: baseline group below devos floor does not fire REGRESSION" 0 $? || { echo "FAIL score: cross-mode floor false alarm"; echo "$out"; fail=1; }
|
|
2540
|
+
python3 -c "import json;f=json.load(open('$EVAL_SCORING_FLOOR'));assert '_selftest-needs@baseline' in f and f['_selftest-needs@devos']==95"
|
|
2541
|
+
unset EVAL_SCORING_FLOOR
|
|
2542
|
+
|
|
2543
|
+
# --- must_flip discrimination guard -----------------------------------------
|
|
2544
|
+
# A criterion marked must_flip is one the skill is supposed to change. In
|
|
2545
|
+
# baseline mode it passing means the scenario stopped discriminating: devos
|
|
2546
|
+
# and no-devos look identical from here on. Without EVAL_MODE the guard stays
|
|
2547
|
+
# silent, so ad-hoc scoring never sees advisory noise.
|
|
2548
|
+
export EVAL_SCORING_FLOOR="$tmp/floor-flip.json"
|
|
2549
|
+
echo '{}' >"$EVAL_SCORING_FLOOR"
|
|
2550
|
+
flipdir="$tmp/trial-flip"
|
|
2551
|
+
mkdir -p "$flipdir"
|
|
2552
|
+
printf '%s\n' "$hello" > "$flipdir/transcript.jsonl"
|
|
2553
|
+
out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-flip "$flipdir" 2>&1); rc=$?
|
|
2554
|
+
grep -q "DISCRIMINATION LOST: baseline passed must_flip criteria flippable" <<<"$out" \
|
|
2555
|
+
&& check "score: must_flip criterion passing in baseline reports lost discrimination" 0 $? \
|
|
2556
|
+
|| { echo "FAIL score: must_flip pass in baseline not flagged"; fail=1; }
|
|
2557
|
+
[[ "$rc" == "2" ]] \
|
|
2558
|
+
&& check "score: baseline must_flip exits 2 (enforced, not just reported)" 0 $? \
|
|
2559
|
+
|| { echo "FAIL score: baseline must_flip exit code $rc, want 2"; fail=1; }
|
|
2560
|
+
python3 -c "import json;s=json.load(open('$flipdir/.score.json'));assert s.get('baseline_passed_must_flip')==['flippable']" 2>/dev/null \
|
|
2561
|
+
|| { echo "FAIL score: .score.json missing baseline_passed_must_flip"; fail=1; }
|
|
2562
|
+
out=$(EVAL_MODE=devos python3 "$EVALS/score.py" _selftest-flip "$flipdir" 2>&1); rc=$?
|
|
2563
|
+
grep -q "DISCRIMINATION LOST" <<<"$out" && { echo "FAIL score: must_flip flagged outside baseline mode"; fail=1; } || true
|
|
2564
|
+
[[ "$rc" == "0" ]] \
|
|
2565
|
+
&& check "score: devos must_flip stays silent, exits 0" 0 $? \
|
|
2566
|
+
|| { echo "FAIL score: devos must_flip exit code $rc, want 0"; fail=1; }
|
|
2567
|
+
unset EVAL_SCORING_FLOOR
|
|
2568
|
+
|
|
2569
|
+
# --- must_hold pass-to-pass guard --------------------------------------------
|
|
2570
|
+
# A criterion marked must_hold guards behavior that has to survive every route.
|
|
2571
|
+
# Failing it in a devos run is a regression even when the total looks fine;
|
|
2572
|
+
# failing it in baseline means the fixture cannot establish a real floor.
|
|
2573
|
+
export EVAL_SCORING_FLOOR="$tmp/floor-hold.json"
|
|
2574
|
+
echo '{}' >"$EVAL_SCORING_FLOOR"
|
|
2575
|
+
holddir="$tmp/trial-hold"
|
|
2576
|
+
mkdir -p "$holddir"
|
|
2577
|
+
printf '%s\n' "$hello" > "$holddir/transcript.jsonl"
|
|
2578
|
+
out=$(EVAL_MODE=devos python3 "$EVALS/score.py" _selftest-hold "$holddir" 2>&1); rc=$?
|
|
2579
|
+
grep -q "REGRESSION: devos failed must_hold criteria holdable" <<<"$out" \
|
|
2580
|
+
&& check "score: failed must_hold in devos mode reports regression" 0 $? \
|
|
2581
|
+
|| { echo "FAIL score: must_hold failure in devos not flagged"; echo "$out"; fail=1; }
|
|
2582
|
+
[[ "$rc" == "2" ]] \
|
|
2583
|
+
&& check "score: devos must_hold exits 2 (enforced, not just reported)" 0 $? \
|
|
2584
|
+
|| { echo "FAIL score: devos must_hold exit code $rc, want 2"; fail=1; }
|
|
2585
|
+
python3 -c "import json;s=json.load(open('$holddir/.score.json'));assert s.get('failed_must_hold')==['holdable']" 2>/dev/null \
|
|
2586
|
+
|| { echo "FAIL score: .score.json missing failed_must_hold"; fail=1; }
|
|
2587
|
+
out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-hold "$holddir" 2>&1); rc=$?
|
|
2588
|
+
grep -q "must_hold" <<<"$out" && { echo "FAIL score: must_hold flagged in baseline mode"; echo "$out"; fail=1; } || true
|
|
2589
|
+
[[ "$rc" == "0" ]] \
|
|
2590
|
+
&& check "score: baseline must_hold stays silent, exits 0" 0 $? \
|
|
2591
|
+
|| { echo "FAIL score: baseline must_hold exit code $rc, want 0"; fail=1; }
|
|
2592
|
+
unset EVAL_SCORING_FLOOR
|
|
2593
|
+
|
|
2594
|
+
# --- check-converge.sh -------------------------------------------------------
|
|
2595
|
+
# Converge pass: every task group's declared Files must be touched on the spec
|
|
2596
|
+
# branch since it diverged from the default branch. A group whose files never
|
|
2597
|
+
# moved is done-drift (marked complete, work never landed).
|
|
2598
|
+
CCV="$(cd "$(dirname "$DOD")/../.." && pwd)/skills/implement/scripts/check-converge.sh"
|
|
2599
|
+
# Own repo root: the shared $tmp accumulates stray files and git state from
|
|
2600
|
+
# other sections, which leaks into diff/untracked output.
|
|
2601
|
+
cvgroot="$tmp/cvgrepo"
|
|
2602
|
+
cvg="$cvgroot/devos/specs/cvg"
|
|
2603
|
+
mkdir -p "$cvg" "$cvgroot/src"
|
|
2604
|
+
printf '%s\n' '## Implementation Plan' '### Task Group 1: cache' '**Files:** `src/cache.ts`' '### Task Group 2: export' '**Files:** `src/export.ts`' > "$cvg/spec.md"
|
|
2605
|
+
git -C "$cvgroot" init -q -b main
|
|
2606
|
+
git -C "$cvgroot" config user.email t@t && git -C "$cvgroot" config user.name t
|
|
2607
|
+
echo base > "$cvgroot/src/base.txt"
|
|
2608
|
+
git -C "$cvgroot" add -A && git -C "$cvgroot" commit -qm base
|
|
2609
|
+
git -C "$cvgroot" checkout -qb feature
|
|
2610
|
+
echo e > "$cvgroot/src/export.ts"
|
|
2611
|
+
out=$(cd "$cvgroot" && bash "$CCV" devos/specs/cvg 2>&1); rc=$?
|
|
2612
|
+
[[ $rc -eq 1 ]] || { echo "FAIL converge: expected findings, got rc=$rc"; echo "$out"; fail=1; }
|
|
2613
|
+
echo "$out" | grep -q "Task Group 1" || { echo "FAIL converge: untouched group 1 not named"; fail=1; }
|
|
2614
|
+
echo "$out" | grep -q "Task Group 2" && { echo "FAIL converge: touched group 2 wrongly flagged"; fail=1; }
|
|
2615
|
+
# Landing group 1's work clears the finding.
|
|
2616
|
+
echo c > "$cvgroot/src/cache.ts"
|
|
2617
|
+
out=$(cd "$cvgroot" && bash "$CCV" devos/specs/cvg 2>&1); rc=$?
|
|
2618
|
+
[[ $rc -eq 0 ]] || { echo "FAIL converge: converged repo still flagged ($out)"; fail=1; }
|
|
2619
|
+
# Outside a git repo: advisory skip, never a brick.
|
|
2620
|
+
mkdir -p "$tmp/nogit/devos/specs/cvg2" && cp "$cvg/spec.md" "$tmp/nogit/devos/specs/cvg2/"
|
|
2621
|
+
out=$(bash "$CCV" "$tmp/nogit/devos/specs/cvg2" 2>&1); rc=$?
|
|
2622
|
+
[[ $rc -eq 0 ]] || { echo "FAIL converge: non-git dir must exit 0 (fail open), got $rc"; fail=1; }
|
|
2623
|
+
# Nothing touched at all: the branch is even with the default branch and the
|
|
2624
|
+
# tree is clean, so the touched-file list is empty. An FNR == NR reader sees the
|
|
2625
|
+
# plan as its first file and calls every declared path touched, which passes the
|
|
2626
|
+
# one state the gate exists to catch.
|
|
2627
|
+
cvgempty="$tmp/cvgempty"
|
|
2628
|
+
mkdir -p "$cvgempty/devos/specs/cvg3"
|
|
2629
|
+
cp "$cvg/spec.md" "$cvgempty/devos/specs/cvg3/"
|
|
2630
|
+
git -C "$cvgempty" init -q -b main
|
|
2631
|
+
git -C "$cvgempty" config user.email t@t && git -C "$cvgempty" config user.name t
|
|
2632
|
+
git -C "$cvgempty" add -A && git -C "$cvgempty" commit -qm base
|
|
2633
|
+
out=$(cd "$cvgempty" && bash "$CCV" devos/specs/cvg3 2>&1); rc=$?
|
|
2634
|
+
[[ $rc -eq 1 ]] || { echo "FAIL converge: empty touched list must still flag both groups, got rc=$rc"; echo "$out"; fail=1; }
|
|
2635
|
+
echo "$out" | grep -q "Task Group 1" || { echo "FAIL converge: empty touched list did not name group 1"; fail=1; }
|
|
2636
|
+
echo "$out" | grep -q "Task Group 2" || { echo "FAIL converge: empty touched list did not name group 2"; fail=1; }
|
|
2637
|
+
|
|
2638
|
+
# A directory declaration must match descendants, not a sibling with the same
|
|
2639
|
+
# prefix. `src` owns `src/file.ts`, but not `src2/file.ts`.
|
|
2640
|
+
cvgprefix="$tmp/cvgprefix"
|
|
2641
|
+
mkdir -p "$cvgprefix/devos/specs/cvg4"
|
|
2642
|
+
printf '%s\n' '## Implementation Plan' '### Task Group 1: source' '**Files:** `src`' > "$cvgprefix/devos/specs/cvg4/spec.md"
|
|
2643
|
+
git -C "$cvgprefix" init -q -b main
|
|
2644
|
+
git -C "$cvgprefix" config user.email t@t && git -C "$cvgprefix" config user.name t
|
|
2645
|
+
mkdir -p "$cvgprefix/src2"
|
|
2646
|
+
git -C "$cvgprefix" add -A && git -C "$cvgprefix" commit -qm base
|
|
2647
|
+
git -C "$cvgprefix" checkout -qb feature
|
|
2648
|
+
echo touched > "$cvgprefix/src2/file.ts"
|
|
2649
|
+
git -C "$cvgprefix" add -A && git -C "$cvgprefix" commit -qm touch
|
|
2650
|
+
out=$(cd "$cvgprefix" && bash "$CCV" devos/specs/cvg4 2>&1); rc=$?
|
|
2651
|
+
[[ $rc -eq 1 ]] || { echo "FAIL converge: a sibling prefix must not satisfy a directory declaration, got rc=$rc ($out)"; fail=1; }
|
|
2652
|
+
echo "$out" | grep -q "Task Group 1" || { echo "FAIL converge: sibling prefix wrongly treated as the declared directory"; fail=1; }
|
|
2653
|
+
|
|
2654
|
+
# A group the gate cannot identify is not checkable, and a group that cannot be
|
|
2655
|
+
# checked used to be the one case this gate passed. The header pattern required
|
|
2656
|
+
# whitespace after "Task Group", so "### Task Group: Parser" never even matched:
|
|
2657
|
+
# a plan whose only group declared an untouched file exited 0.
|
|
2658
|
+
cvgunnum="$tmp/cvgunnum"
|
|
2659
|
+
mkdir -p "$cvgunnum/devos/specs/cvg5" "$cvgunnum/src"
|
|
2660
|
+
printf '%s\n' '## Implementation Plan' '### Task Group: Parser' '**Files:** `src/never-touched.ts`' \
|
|
2661
|
+
> "$cvgunnum/devos/specs/cvg5/spec.md"
|
|
2662
|
+
echo base > "$cvgunnum/src/other.ts"
|
|
2663
|
+
git -C "$cvgunnum" init -q -b main
|
|
2664
|
+
git -C "$cvgunnum" config user.email t@t && git -C "$cvgunnum" config user.name t
|
|
2665
|
+
git -C "$cvgunnum" add -A && git -C "$cvgunnum" commit -qm base
|
|
2666
|
+
git -C "$cvgunnum" checkout -qb feature
|
|
2667
|
+
out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
|
|
2668
|
+
[[ $rc -eq 1 ]] || { echo "FAIL converge: an unidentifiable group must be reported, got rc=$rc ($out)"; fail=1; }
|
|
2669
|
+
echo "$out" | grep -q "has no resolvable group number" || { echo "FAIL converge: unidentifiable group not diagnosed ($out)"; fail=1; }
|
|
2670
|
+
echo "$out" | grep -q "Task Group: Parser" || { echo "FAIL converge: diagnostic does not name the header"; fail=1; }
|
|
2671
|
+
# It stays reported even when the declared file did land: the point is that the
|
|
2672
|
+
# gate could not attribute the files to a group, not that a file went missing.
|
|
2673
|
+
echo landed > "$cvgunnum/src/never-touched.ts"
|
|
2674
|
+
out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
|
|
2675
|
+
[[ $rc -eq 1 ]] && echo "$out" | grep -q "has no resolvable group number" \
|
|
2676
|
+
|| { echo "FAIL converge: an unidentifiable group must stay reported when files land ($out)"; fail=1; }
|
|
2677
|
+
# The documented and legacy numbered forms must be unaffected, and a heading
|
|
2678
|
+
# that merely contains the words "Task Groups" is not a group.
|
|
2679
|
+
printf '%s\n' '## Task Groups Overview' '### Task Group 1: Parser' '**Files:** `src/absent.ts`' \
|
|
2680
|
+
> "$cvgunnum/devos/specs/cvg5/spec.md"
|
|
2681
|
+
out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
|
|
2682
|
+
[[ $rc -eq 1 ]] && echo "$out" | grep -q "Task Group 1" \
|
|
2683
|
+
&& ! echo "$out" | grep -q "no resolvable group number" \
|
|
2684
|
+
|| { echo "FAIL converge: numbered group misparsed after the change ($out)"; fail=1; }
|
|
2685
|
+
printf '%s\n' '## Task Groups Overview' '### Task Group 1: Parser' '**Files:** `src/never-touched.ts`' \
|
|
2686
|
+
> "$cvgunnum/devos/specs/cvg5/spec.md"
|
|
2687
|
+
echo more >> "$cvgunnum/src/never-touched.ts"
|
|
2688
|
+
out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
|
|
2689
|
+
[[ $rc -eq 0 ]] || { echo "FAIL converge: a 'Task Groups' heading was read as a group ($out)"; fail=1; }
|
|
2690
|
+
printf '%s\n' '## Implementation Plan' '### Task Group A: Parser' '**Files:** `src/never-touched.ts`' \
|
|
2691
|
+
> "$cvgunnum/devos/specs/cvg5/spec.md"
|
|
2692
|
+
out=$(cd "$cvgunnum" && bash "$CCV" devos/specs/cvg5 2>&1); rc=$?
|
|
2693
|
+
[[ $rc -eq 0 ]] || { echo "FAIL converge: the pre-0.8.6 letter form must still resolve ($out)"; fail=1; }
|
|
2694
|
+
|
|
2695
|
+
# The declared file list is stored newline-joined, and interpolating it directly
|
|
2696
|
+
# broke the one-line envelope: two untouched files printed three lines with a
|
|
2697
|
+
# bare ") declares files" tail. A reader taking one line per finding saw a
|
|
2698
|
+
# truncated message.
|
|
2699
|
+
cvgmulti="$tmp/cvgmulti"
|
|
2700
|
+
mkdir -p "$cvgmulti/devos/specs/cvg6" "$cvgmulti/src"
|
|
2701
|
+
printf '%s\n' '## Implementation Plan' '### Task Group 1: Parser' '**Files:** `src/one.ts`, `src/two.ts`' \
|
|
2702
|
+
> "$cvgmulti/devos/specs/cvg6/spec.md"
|
|
2703
|
+
echo base > "$cvgmulti/src/base.ts"
|
|
2704
|
+
git -C "$cvgmulti" init -q -b main
|
|
2705
|
+
git -C "$cvgmulti" config user.email t@t && git -C "$cvgmulti" config user.name t
|
|
2706
|
+
git -C "$cvgmulti" add -A && git -C "$cvgmulti" commit -qm base
|
|
2707
|
+
git -C "$cvgmulti" checkout -qb feature
|
|
2708
|
+
echo work > "$cvgmulti/src/work.ts"
|
|
2709
|
+
git -C "$cvgmulti" add -A && git -C "$cvgmulti" commit -qm work
|
|
2710
|
+
out=$(cd "$cvgmulti" && bash "$CCV" devos/specs/cvg6 2>&1); rc=$?
|
|
2711
|
+
[[ $rc -eq 1 ]] || { echo "FAIL converge: a multi-file untouched group must fail, got rc=$rc ($out)"; fail=1; }
|
|
2712
|
+
echo "$out" | grep -Fq 'Task Group 1 (src/one.ts, src/two.ts) declares files' \
|
|
2713
|
+
|| { echo "FAIL converge: multi-file finding is not one line: $out"; fail=1; }
|
|
2714
|
+
# The stray tail lines that broke the envelope must be gone.
|
|
2715
|
+
[[ "$(printf '%s\n' "$out" | grep -c '^src/')" -eq 0 ]] \
|
|
2716
|
+
|| { echo "FAIL converge: file list still leaks onto its own line: $out"; fail=1; }
|
|
2717
|
+
[[ "$(printf '%s\n' "$out" | grep -c '^)')" -eq 0 ]] \
|
|
2718
|
+
|| { echo "FAIL converge: a bare closing paren line remains: $out"; fail=1; }
|
|
2719
|
+
# The single-file form is unchanged: one file, no trailing separator.
|
|
2720
|
+
printf '%s\n' '## Implementation Plan' '### Task Group 1: Parser' '**Files:** `src/absent.ts`' \
|
|
2721
|
+
> "$cvgmulti/devos/specs/cvg6/spec.md"
|
|
2722
|
+
out=$(cd "$cvgmulti" && bash "$CCV" devos/specs/cvg6 2>&1); rc=$?
|
|
2723
|
+
[[ $rc -eq 1 ]] || { echo "FAIL converge: single-file untouched group must fail, got rc=$rc ($out)"; fail=1; }
|
|
2724
|
+
echo "$out" | grep -Fq 'Task Group 1 (src/absent.ts) declares files' \
|
|
2725
|
+
|| { echo "FAIL converge: single-file finding changed shape: $out"; fail=1; }
|
|
2726
|
+
|
|
2727
|
+
# --- C route: validate-spec accepts an Invariant in place of scenarios ------
|
|
2728
|
+
|
|
2729
|
+
# A behavior-preserving spec declares what must NOT change, so
|
|
2730
|
+
# it has no scenarios and no SC-n to trace. Every case here is a shape the old
|
|
2731
|
+
# script either rejected (a valid C spec) or waved through (a confused one).
|
|
2732
|
+
VS="$SKILLS/implement/scripts/validate-spec.sh"
|
|
2733
|
+
cdir="$tmp/croute"
|
|
2734
|
+
mkspec() { mkdir -p "$cdir/$1" && cat >"$cdir/$1/spec-functional.md"; }
|
|
2735
|
+
|
|
2736
|
+
mkspec ok <<'EOF'
|
|
2737
|
+
# Migrate the runner
|
|
2738
|
+
## 1. Goal
|
|
2739
|
+
Config is duplicated across four packages.
|
|
2740
|
+
## 2. Invariant
|
|
2741
|
+
**Commands that MUST still pass, with no fewer tests executed:**
|
|
2742
|
+
- `npm test` -> baseline: exit 0, 42 passed
|
|
2743
|
+
**Public surface that MUST NOT move:**
|
|
2744
|
+
- none, internal only
|
|
2745
|
+
## 3. Out of Scope
|
|
2746
|
+
- Behavior change of any kind.
|
|
2747
|
+
EOF
|
|
2748
|
+
bash "$VS" "$cdir/ok" >/dev/null 2>&1
|
|
2749
|
+
check "validate-spec: an Invariant spec with no scenarios passes" 0 $?
|
|
2750
|
+
|
|
2751
|
+
mkspec nocmd <<'EOF'
|
|
2752
|
+
# Migrate
|
|
2753
|
+
## 1. Goal
|
|
2754
|
+
x
|
|
2755
|
+
## 2. Invariant
|
|
2756
|
+
**Commands that MUST still pass:**
|
|
2757
|
+
- the existing suite
|
|
2758
|
+
**Public surface that MUST NOT move:**
|
|
2759
|
+
- none, internal only
|
|
2760
|
+
## 3. Out of Scope
|
|
2761
|
+
- none
|
|
2762
|
+
EOF
|
|
2763
|
+
bash "$VS" "$cdir/nocmd" >/dev/null 2>&1
|
|
2764
|
+
check "validate-spec: an Invariant naming no command fails" 1 $?
|
|
2765
|
+
|
|
2766
|
+
mkspec nosurface <<'EOF'
|
|
2767
|
+
# Migrate
|
|
2768
|
+
## 1. Goal
|
|
2769
|
+
x
|
|
2770
|
+
## 2. Invariant
|
|
2771
|
+
**Commands that MUST still pass:**
|
|
2772
|
+
- `npm test` -> baseline: exit 0, 42 passed
|
|
2773
|
+
## 3. Out of Scope
|
|
2774
|
+
- none
|
|
2775
|
+
EOF
|
|
2776
|
+
bash "$VS" "$cdir/nosurface" >/dev/null 2>&1
|
|
2777
|
+
check "validate-spec: an Invariant with no public-surface line fails" 1 $?
|
|
2778
|
+
|
|
2779
|
+
mkspec both <<'EOF'
|
|
2780
|
+
# Migrate
|
|
2781
|
+
## 1. Goal
|
|
2782
|
+
x
|
|
2783
|
+
## 2. Invariant
|
|
2784
|
+
**Commands that MUST still pass:**
|
|
2785
|
+
- `npm test` -> baseline: exit 0, 42 passed
|
|
2786
|
+
**Public surface that MUST NOT move:**
|
|
2787
|
+
- none, internal only
|
|
2788
|
+
## 3bis. Behavior delta
|
|
2789
|
+
**Added** (capability: `runner`):
|
|
2790
|
+
- reports coverage inline
|
|
2791
|
+
## 4. Out of Scope
|
|
2792
|
+
- none
|
|
2793
|
+
EOF
|
|
2794
|
+
bash "$VS" "$cdir/both" >/dev/null 2>&1
|
|
2795
|
+
check "validate-spec: Invariant plus a non-empty Behavior delta fails" 1 $?
|
|
2796
|
+
|
|
2797
|
+
mkspec nonedelta <<'EOF'
|
|
2798
|
+
# Migrate
|
|
2799
|
+
## 1. Goal
|
|
2800
|
+
x
|
|
2801
|
+
## 2. Invariant
|
|
2802
|
+
**Commands that MUST still pass:**
|
|
2803
|
+
- `npm test` -> baseline: exit 0, 42 passed
|
|
2804
|
+
**Public surface that MUST NOT move:**
|
|
2805
|
+
- none, internal only
|
|
2806
|
+
## 3bis. Behavior delta
|
|
2807
|
+
**Added:** (none)
|
|
2808
|
+
**Removed:** (none)
|
|
2809
|
+
## 4. Out of Scope
|
|
2810
|
+
- none
|
|
2811
|
+
EOF
|
|
2812
|
+
bash "$VS" "$cdir/nonedelta" >/dev/null 2>&1
|
|
2813
|
+
check "validate-spec: Invariant plus an empty Behavior delta passes" 0 $?
|
|
2814
|
+
|
|
2815
|
+
# The heading is matched exactly, never as a substring: a feature spec is free
|
|
2816
|
+
# to have a "Business invariants" section, and reading that as the C route
|
|
2817
|
+
# would silently switch off the scenario checks it still needs.
|
|
2818
|
+
mkspec businv <<'EOF'
|
|
2819
|
+
# Feature
|
|
2820
|
+
## 1. Goal
|
|
2821
|
+
x
|
|
2822
|
+
## 2. Business invariants
|
|
2823
|
+
- balance MUST never go below 0
|
|
2824
|
+
## 3. Out of Scope
|
|
2825
|
+
- none
|
|
2826
|
+
EOF
|
|
2827
|
+
bash "$VS" "$cdir/businv" >/dev/null 2>&1
|
|
2828
|
+
check "validate-spec: a 'Business invariants' heading is not the C route" 1 $?
|
|
2829
|
+
|
|
2830
|
+
mkspec feature <<'EOF'
|
|
2831
|
+
# Feature
|
|
2832
|
+
## 1. Goal
|
|
2833
|
+
x
|
|
2834
|
+
## 2. Functionality
|
|
2835
|
+
- Scenario SC-1: happy / WHEN a user submits THEN the record MUST persist
|
|
2836
|
+
## 3. Out of Scope
|
|
2837
|
+
- none
|
|
2838
|
+
EOF
|
|
2839
|
+
bash "$VS" "$cdir/feature" >/dev/null 2>&1
|
|
2840
|
+
check "validate-spec: an ordinary scenario spec is unaffected" 0 $?
|
|
2841
|
+
|
|
2842
|
+
# The plan capsule's own structural checks. Both run at Phase A step 5b now,
|
|
2843
|
+
# before the human ever reads the plan, so a defect here costs one edit rather
|
|
2844
|
+
# than a re-derivation after the code was written against it.
|
|
2845
|
+
pdir="$cdir/plan"; mkdir -p "$pdir"
|
|
2846
|
+
printf '# P\n## 1. Goal\nx\n## 2. Functionality\n- Scenario SC-1: a / WHEN x THEN y\n## 3. Out of Scope\n- none\n' > "$pdir/spec-functional.md"
|
|
2847
|
+
mkplan() { { printf '## Local Craft\nNone\n'; cat; } > "$pdir/spec.md"; }
|
|
2848
|
+
|
|
2849
|
+
mkplan <<'MD'
|
|
2850
|
+
## Design Convergence
|
|
2851
|
+
- **Direct MVP**: call it directly
|
|
2852
|
+
- **Failed Items**: None
|
|
2853
|
+
## Implementation Plan
|
|
2854
|
+
### Task Group 1: A
|
|
2855
|
+
**Files:** `a.ts`
|
|
2856
|
+
**Scenarios covered:** SC-1
|
|
2857
|
+
MD
|
|
2858
|
+
bash "$VS" "$pdir" >/dev/null 2>&1
|
|
2859
|
+
check "validate-spec: a well-formed capsule passes" 0 $?
|
|
2860
|
+
|
|
2861
|
+
# "None" with a reason is still None. The fix line says to state None, and a
|
|
2862
|
+
# capsule that writes "None: the tree is new" was failing the gate that told
|
|
2863
|
+
# it to write None.
|
|
2864
|
+
{ printf '## Local Craft\nNone: the project has nothing to reuse here\n'; cat <<'MD'
|
|
2865
|
+
## Design Convergence
|
|
2866
|
+
- **Direct MVP**: call it directly
|
|
2867
|
+
- **Failed Items**: None
|
|
2868
|
+
## Implementation Plan
|
|
2869
|
+
### Task Group 1: A
|
|
2870
|
+
**Files:** `a.ts`
|
|
2871
|
+
**Scenarios covered:** SC-1
|
|
2872
|
+
MD
|
|
2873
|
+
} > "$pdir/spec.md"
|
|
2874
|
+
bash "$VS" "$pdir" >/dev/null 2>&1
|
|
2875
|
+
check "validate-spec: Local Craft may state None with a reason" 0 $?
|
|
2876
|
+
|
|
2877
|
+
# A change that spans many units holds one group for the whole change, and
|
|
2878
|
+
# the group's files are the target tree. The units live in the manifest, so
|
|
2879
|
+
# the capsule does not carry them.
|
|
2880
|
+
{ printf '## Local Craft\nNone\n'; cat <<'MD'
|
|
2881
|
+
## Design Convergence
|
|
2882
|
+
- **Direct MVP**: one group, the units live in the manifest
|
|
2883
|
+
- **Failed Items**: None
|
|
2884
|
+
## Implementation Plan
|
|
2885
|
+
### Task Group 1: the port
|
|
2886
|
+
**Files:** `modernized/`
|
|
2887
|
+
**Scenarios covered:** SC-1
|
|
2888
|
+
MD
|
|
2889
|
+
} > "$pdir/spec.md"
|
|
2890
|
+
bash "$VS" "$pdir" >/dev/null 2>&1
|
|
2891
|
+
check "validate-spec: a one-group capsule naming a directory passes" 0 $?
|
|
2892
|
+
|
|
2893
|
+
write_capsule() {
|
|
2894
|
+
local blank_lines="$1" trailing_newline="$2"
|
|
2895
|
+
{
|
|
2896
|
+
printf '%s\n' \
|
|
2897
|
+
'## Local Craft' \
|
|
2898
|
+
'None' \
|
|
2899
|
+
'## Design Convergence' \
|
|
2900
|
+
'- **Direct MVP**: call it directly' \
|
|
2901
|
+
'- **Failed Items**: None' \
|
|
2902
|
+
'## Implementation Plan' \
|
|
2903
|
+
'### Task Group 1: A' \
|
|
2904
|
+
'**Files:** `a.ts`' \
|
|
2905
|
+
'**Scenarios covered:** SC-1'
|
|
2906
|
+
for ((i = 0; i < blank_lines; i++)); do printf '\n'; done
|
|
2907
|
+
if [[ "$trailing_newline" -eq 1 ]]; then
|
|
2908
|
+
printf 'final logical line\n'
|
|
2909
|
+
else
|
|
2910
|
+
printf 'final logical line'
|
|
2911
|
+
fi
|
|
2912
|
+
} > "$pdir/spec.md"
|
|
2913
|
+
}
|
|
2914
|
+
|
|
2915
|
+
write_capsule 290 1
|
|
2916
|
+
bash "$VS" "$pdir" >/dev/null 2>&1
|
|
2917
|
+
check "validate-spec: a 300-line capsule with a trailing newline passes" 0 $?
|
|
2918
|
+
|
|
2919
|
+
write_capsule 290 0
|
|
2920
|
+
bash "$VS" "$pdir" >/dev/null 2>&1
|
|
2921
|
+
check "validate-spec: a 300-line capsule without a trailing newline passes" 0 $?
|
|
2922
|
+
|
|
2923
|
+
write_capsule 291 0
|
|
2924
|
+
out=$(bash "$VS" "$pdir" 2>&1); rc=$?
|
|
2925
|
+
grep -q 'spec.md capsule is 301 lines (budget 300)' <<<"$out" && [[ $rc -eq 1 ]] \
|
|
2926
|
+
&& check "validate-spec: an unterminated 301st line is counted" 0 $? \
|
|
2927
|
+
|| { echo "FAIL validate-spec: unterminated final line not counted"; echo "$out"; fail=1; }
|
|
2928
|
+
|
|
2929
|
+
cat > "$pdir/spec.md" <<'MD'
|
|
2930
|
+
## Design Convergence
|
|
2931
|
+
- **Direct MVP**: call it directly
|
|
2932
|
+
## Implementation Plan
|
|
2933
|
+
### Task Group 1: A
|
|
2934
|
+
**Files:** `a.ts`
|
|
2935
|
+
**Scenarios covered:** SC-1
|
|
2936
|
+
MD
|
|
2937
|
+
out=$(bash "$VS" "$pdir" 2>&1); rc=$?
|
|
2938
|
+
grep -q 'spec.md has no Local Craft section' <<<"$out" && [[ $rc -eq 1 ]] \
|
|
2939
|
+
&& check "validate-spec: a capsule with no Local Craft section fails" 0 $? \
|
|
2940
|
+
|| { echo "FAIL validate-spec: missing Local Craft not caught"; echo "$out"; fail=1; }
|
|
2941
|
+
|
|
2942
|
+
cat > "$pdir/spec.md" <<'MD'
|
|
2943
|
+
## Local Craft
|
|
2944
|
+
- Reuse the existing helper.
|
|
2945
|
+
## Design Convergence
|
|
2946
|
+
- **Direct MVP**: call it directly
|
|
2947
|
+
## Implementation Plan
|
|
2948
|
+
### Task Group 1: A
|
|
2949
|
+
**Files:** `a.ts`
|
|
2950
|
+
**Scenarios covered:** SC-1
|
|
2951
|
+
MD
|
|
2952
|
+
out=$(bash "$VS" "$pdir" 2>&1); rc=$?
|
|
2953
|
+
grep -q 'spec.md Local Craft has no file:line or URL evidence' <<<"$out" && [[ $rc -eq 1 ]] \
|
|
2954
|
+
&& check "validate-spec: ungrounded Local Craft evidence fails" 0 $? \
|
|
2955
|
+
|| { echo "FAIL validate-spec: ungrounded Local Craft not caught"; echo "$out"; fail=1; }
|
|
2956
|
+
|
|
2957
|
+
cat > "$pdir/spec.md" <<'MD'
|
|
2958
|
+
## Local Craft
|
|
2959
|
+
- `src/a.ts:1` uses the existing helper.
|
|
2960
|
+
## Design Convergence
|
|
2961
|
+
- **Direct MVP**: call it directly
|
|
2962
|
+
## Implementation Plan
|
|
2963
|
+
### Task Group 1: A
|
|
2964
|
+
**Files:** `a.ts`
|
|
2965
|
+
**Scenarios covered:** SC-1
|
|
2966
|
+
MD
|
|
2967
|
+
bash "$VS" "$pdir" >/dev/null 2>&1; check "validate-spec: grounded Local Craft evidence passes" 0 $?
|
|
2968
|
+
|
|
2969
|
+
# A group with no Files list is silently skipped by check-converge.sh, so the
|
|
2970
|
+
# omission switches off an existing gate instead of merely thinning the plan.
|
|
2971
|
+
mkplan <<'MD'
|
|
2972
|
+
## Design Convergence
|
|
2973
|
+
- **Direct MVP**: call it directly
|
|
2974
|
+
## Implementation Plan
|
|
2975
|
+
### Task Group 1: A
|
|
2976
|
+
**Scenarios covered:** SC-1
|
|
2977
|
+
MD
|
|
2978
|
+
out=$(bash "$VS" "$pdir" 2>&1)
|
|
2979
|
+
grep -q 'has no \*\*Files:\*\* line' <<<"$out" \
|
|
2980
|
+
&& check "validate-spec: a task group with no Files line fails" 0 $? \
|
|
2981
|
+
|| { echo "FAIL validate-spec: missing Files line not caught"; echo "$out"; fail=1; }
|
|
2982
|
+
|
|
2983
|
+
# The section a plan skips exactly when it is adding the most.
|
|
2984
|
+
mkplan <<'MD'
|
|
2985
|
+
## Implementation Plan
|
|
2986
|
+
### Task Group 1: A
|
|
2987
|
+
**Files:** `a.ts`
|
|
2988
|
+
**Scenarios covered:** SC-1
|
|
2989
|
+
MD
|
|
2990
|
+
out=$(bash "$VS" "$pdir" 2>&1)
|
|
2991
|
+
grep -q 'no Design Convergence section' <<<"$out" \
|
|
2992
|
+
&& check "validate-spec: a capsule with no Design Convergence fails" 0 $? \
|
|
2993
|
+
|| { echo "FAIL validate-spec: missing Design Convergence not caught"; echo "$out"; fail=1; }
|
|
2994
|
+
|
|
2995
|
+
# A `## ` heading after the last group must not be read as a group missing its
|
|
2996
|
+
# Files line: Change Log and Implementation Notes always follow the plan.
|
|
2997
|
+
mkplan <<'MD'
|
|
2998
|
+
## Design Convergence
|
|
2999
|
+
- **Direct MVP**: call it directly
|
|
3000
|
+
## Implementation Plan
|
|
3001
|
+
### Task Group 1: A
|
|
3002
|
+
**Files:** `a.ts`
|
|
3003
|
+
**Scenarios covered:** SC-1
|
|
3004
|
+
## Implementation Notes
|
|
3005
|
+
- none
|
|
3006
|
+
## Change Log
|
|
3007
|
+
- none
|
|
3008
|
+
MD
|
|
3009
|
+
bash "$VS" "$pdir" >/dev/null 2>&1
|
|
3010
|
+
check "validate-spec: trailing sections after the last group are not a gap" 0 $?
|
|
3011
|
+
|
|
3012
|
+
# Presence alone passes with four empty labels. Every Adopted Addition answers
|
|
3013
|
+
# a Failed Item, so additions against `Failed Items: None` is the section
|
|
3014
|
+
# contradicting itself, which is the shape an unjustified abstraction takes.
|
|
3015
|
+
mkconv() { printf '## Local Craft\nNone\n## Design Convergence\n%s\n## Implementation Plan\n### Task Group 1: A\n**Files:** `a.ts`\n**Scenarios covered:** SC-1\n' "$1" > "$pdir/spec.md"; }
|
|
3016
|
+
|
|
3017
|
+
mkconv '- **Direct MVP**: call it directly
|
|
3018
|
+
- **Failed Items**: None
|
|
3019
|
+
- **Adopted Additions**: None'
|
|
3020
|
+
bash "$VS" "$pdir" >/dev/null 2>&1
|
|
3021
|
+
check "validate-spec: a converged plan with no additions passes" 0 $?
|
|
3022
|
+
|
|
3023
|
+
mkconv '- **Direct MVP**: call it directly
|
|
3024
|
+
- **Failed Items**: None
|
|
3025
|
+
- **Adopted Additions**: a registry layer -> nothing -> it is more flexible'
|
|
3026
|
+
out=$(bash "$VS" "$pdir" 2>&1)
|
|
3027
|
+
grep -q 'adopts additions while Failed Items is None' <<<"$out" \
|
|
3028
|
+
&& check "validate-spec: an addition with no Failed Item fails" 0 $? \
|
|
3029
|
+
|| { echo "FAIL validate-spec: unjustified addition not caught"; echo "$out"; fail=1; }
|
|
3030
|
+
|
|
3031
|
+
mkconv '- **Direct MVP**: call it directly
|
|
3032
|
+
- **Failed Items**: the MVP cannot express per-tenant limits, evidence in the scenario
|
|
3033
|
+
- **Adopted Additions**: a per-tenant table -> that item -> nothing smaller holds the state'
|
|
3034
|
+
bash "$VS" "$pdir" >/dev/null 2>&1
|
|
3035
|
+
check "validate-spec: an addition answering a Failed Item passes" 0 $?
|
|
3036
|
+
rm -rf "$pdir"
|
|
3037
|
+
|
|
3038
|
+
# --- Concurrent grouping across specs ----------------------
|
|
3039
|
+
# Advisory only. The failure that matters is a false CONCURRENT: two specs
|
|
3040
|
+
# grouped when a dependency or a shared file makes them unsafe.
|
|
3041
|
+
CR="$SKILLS/status/scripts/check-requires.py"
|
|
3042
|
+
gdir="$tmp/concurrent"
|
|
3043
|
+
mkopen() { # mkopen <name> <requires|-> <files|->
|
|
3044
|
+
mkdir -p "$gdir/devos/specs/$1"
|
|
3045
|
+
{ echo "# $1"; echo "## Goal"; echo "x"
|
|
3046
|
+
[[ "$2" != "-" ]] && echo "Requires: [$2]"
|
|
3047
|
+
echo "## Out of Scope"; echo "- none"; } >"$gdir/devos/specs/$1/spec-functional.md"
|
|
3048
|
+
[[ "$3" != "-" ]] && printf '### Task Group 1: t\n**Files:** %s\n' "$3" >"$gdir/devos/specs/$1/spec.md"
|
|
3049
|
+
return 0
|
|
3050
|
+
}
|
|
3051
|
+
mkopen alpha - '`src/a.ts`, `src/shared.ts`'
|
|
3052
|
+
mkopen beta - '`src/b.ts`'
|
|
3053
|
+
mkopen gamma beta '`src/g.ts`'
|
|
3054
|
+
mkopen delta - '`src/shared.ts`, `src/d.ts`'
|
|
3055
|
+
mkopen epsilon - -
|
|
3056
|
+
gout=$(python3 "$CR" --concurrent "$gdir" 2>&1)
|
|
3057
|
+
|
|
3058
|
+
check "concurrent: disjoint independent specs group together" \
|
|
3059
|
+
"CONCURRENT group 1: alpha beta" "$(grep '^CONCURRENT group 1' <<<"$gout")"
|
|
3060
|
+
check "concurrent: a shared planned file separates two specs" 1 \
|
|
3061
|
+
"$(grep -c 'OVERLAP alpha + delta: both plan to touch src/shared.ts' <<<"$gout")"
|
|
3062
|
+
check "concurrent: a dependency edge separates two specs" 1 \
|
|
3063
|
+
"$(grep -c 'BLOCKED beta + gamma' <<<"$gout")"
|
|
3064
|
+
check "concurrent: a spec with no plan is UNPLANNED, never grouped" 1 \
|
|
3065
|
+
"$(grep -c '^UNPLANNED epsilon' <<<"$gout")"
|
|
3066
|
+
check "concurrent: an unplanned spec appears in no CONCURRENT group" 0 \
|
|
3067
|
+
"$(grep '^CONCURRENT group' <<<"$gout" | grep -c epsilon || true)"
|
|
3068
|
+
check "concurrent: dependent specs never share a group" 0 \
|
|
3069
|
+
"$(grep '^CONCURRENT group' <<<"$gout" | grep -c 'beta.*gamma\|gamma.*beta' || true)"
|
|
3070
|
+
|
|
3071
|
+
# Directory declarations and descendants are not exact-string matches. Keep
|
|
3072
|
+
# this separate from task-group safety so both consumers stay covered.
|
|
3073
|
+
pdir="$tmp/concurrent-path"
|
|
3074
|
+
mkdir -p "$pdir/devos/specs/alpha" "$pdir/devos/specs/beta"
|
|
3075
|
+
for name in alpha beta; do
|
|
3076
|
+
printf '# %s\n\n## Goal\nx\n\n## Out of Scope\n- none\n' "$name" \
|
|
3077
|
+
> "$pdir/devos/specs/$name/spec-functional.md"
|
|
3078
|
+
printf '## Implementation Plan\n### Task Group 1: work\n**Files:** `%s`\n' \
|
|
3079
|
+
"$([[ "$name" == alpha ]] && echo src || echo src/render.ts)" \
|
|
3080
|
+
> "$pdir/devos/specs/$name/spec.md"
|
|
3081
|
+
done
|
|
3082
|
+
pout=$(python3 "$CR" --concurrent "$pdir" 2>&1)
|
|
3083
|
+
check "concurrent: directory and descendant specs are separated" 1 \
|
|
3084
|
+
"$(grep -c '^OVERLAP alpha + beta: both plan to touch src$' <<<"$pout")"
|
|
3085
|
+
check "concurrent: sibling prefix specs remain groupable" 1 \
|
|
3086
|
+
"$(sed -i.bak 's|src/render.ts|src2/render.ts|' "$pdir/devos/specs/beta/spec.md"; rm -f "$pdir/devos/specs/beta/spec.md.bak"; python3 "$CR" --concurrent "$pdir" 2>&1 | grep -c '^CONCURRENT group 1: alpha beta$')"
|
|
3087
|
+
|
|
3088
|
+
rm -rf "$gdir/devos/specs/beta" "$gdir/devos/specs/gamma" "$gdir/devos/specs/delta" "$gdir/devos/specs/epsilon"
|
|
3089
|
+
check "concurrent: a single open spec reports nothing to parallelize" \
|
|
3090
|
+
"CONCURRENT 1 open spec(s): nothing to run in parallel" \
|
|
3091
|
+
"$(python3 "$CR" --concurrent "$gdir" 2>&1)"
|
|
3092
|
+
|
|
3093
|
+
# --- findings-summary.sh: count records, not ledger lines -----------------
|
|
3094
|
+
FS="$SKILLS/status/scripts/findings-summary.sh"
|
|
3095
|
+
fdir="$tmp/findings/devos"
|
|
3096
|
+
mkdir -p "$fdir"
|
|
3097
|
+
cat > "$fdir/deferred.md" <<'MD'
|
|
3098
|
+
# Deferred Findings
|
|
3099
|
+
|
|
3100
|
+
Severity and resolution are not records by themselves.
|
|
3101
|
+
|
|
3102
|
+
- [2026-01-01] [warning] defer: first finding
|
|
3103
|
+
* [2026-01-02] [critical] [spec-amendment]: second finding
|
|
3104
|
+
- [2026-01-03] [suggestion] [reject]: third finding
|
|
3105
|
+
- [2026-01-04] [warning] defer: patch the critical path
|
|
3106
|
+
- [2026-01-05] [suggestion] reject: avoid patch and spec-amendment in the message
|
|
3107
|
+
[not-a-record] this prose must not be counted
|
|
3108
|
+
MD
|
|
3109
|
+
fout=$(bash "$FS" "$tmp/findings" 2>&1)
|
|
3110
|
+
check "findings: headings, blank lines, and prose are not findings" 1 \
|
|
3111
|
+
"$(grep -c '^| Total findings | 5 |$' <<<"$fout")"
|
|
3112
|
+
if grep -Fqx '| By severity: critical | 1 |' <<<"$fout" \
|
|
3113
|
+
&& grep -Fqx '| By severity: warning | 2 |' <<<"$fout" \
|
|
3114
|
+
&& grep -Fqx '| By resolution: patch | 0 |' <<<"$fout" \
|
|
3115
|
+
&& grep -Fqx '| By resolution: spec-amendment | 1 |' <<<"$fout" \
|
|
3116
|
+
&& grep -Fqx '| By resolution: defer | 2 |' <<<"$fout" \
|
|
3117
|
+
&& grep -Fqx '| By resolution: reject | 2 |' <<<"$fout"; then
|
|
3118
|
+
check "findings: severity and resolution counts come from record fields" 0 0
|
|
3119
|
+
else
|
|
3120
|
+
check "findings: severity and resolution counts come from record fields" 1 0
|
|
3121
|
+
fi
|
|
3122
|
+
check "findings: trend counts record dates only" 1 \
|
|
3123
|
+
"$(grep -c '^ 2026-01: 5 findings$' <<<"$fout")"
|
|
3124
|
+
|
|
3125
|
+
# The month belongs to the record's leading bracketed field. A month named in
|
|
3126
|
+
# the message is prose, not a second date, and counting it reported one finding
|
|
3127
|
+
# under two months.
|
|
3128
|
+
fmonth="$tmp/findings-month/devos"
|
|
3129
|
+
mkdir -p "$fmonth/specs/archive/example"
|
|
3130
|
+
printf -- '- [2026-01-01] [warning] defer: regression was seen in 2025-12\n' \
|
|
3131
|
+
> "$fmonth/deferred.md"
|
|
3132
|
+
fmout=$(bash "$FS" "$tmp/findings-month" 2>&1)
|
|
3133
|
+
check "findings: a month named in the message is not a second date" 1 \
|
|
3134
|
+
"$(grep -c '^ 2026-01: 1 findings$' <<<"$fmout")"
|
|
3135
|
+
! grep -Fq ' 2025-12:' <<<"$fmout"
|
|
3136
|
+
check "findings: trend never invents a month from message text" 0 $?
|
|
3137
|
+
rm -rf "$tmp/findings-month"
|
|
3138
|
+
|
|
3139
|
+
# The zero-findings rule counts specs, not ledger lines, and only L-route ones.
|
|
3140
|
+
# Counting records against a completed-spec denominator printed "3 of 2", a
|
|
3141
|
+
# ratio no set of specs can have, and an M-route entry inflated an L-only rule.
|
|
3142
|
+
zroute="$tmp/findings-zero-route/devos/specs/archive"
|
|
3143
|
+
mkdir -p "$zroute/large" "$zroute/small"
|
|
3144
|
+
cat > "$tmp/findings-zero-route/devos/deferred.md" <<'MD'
|
|
3145
|
+
- [large] [warning] defer: zero findings on first run
|
|
3146
|
+
- [large] [suggestion] defer: zero findings on first run
|
|
3147
|
+
- [small] [warning] defer: no findings on first run
|
|
3148
|
+
MD
|
|
3149
|
+
printf 'large\tL\tyes\t0\tnone\tyes\t?\t-/0\t-\n' > "$zroute/large.quality.md"
|
|
3150
|
+
printf 'small\tM\tyes\t0\tnone\tyes\t?\t-/0\t-\n' > "$zroute/small.quality.md"
|
|
3151
|
+
zout=$(bash "$FS" "$tmp/findings-zero-route" 2>&1)
|
|
3152
|
+
check "findings: a spec counts once however many records mention it" 1 \
|
|
3153
|
+
"$(grep -c '^Zero-findings rule: 1 of 2 completed specs' <<<"$zout")"
|
|
3154
|
+
! grep -Fq ' 2 of ' <<<"$zout" && ! grep -Fq ' 3 of ' <<<"$zout"
|
|
3155
|
+
check "findings: the numerator can never exceed the completed specs" 0 $?
|
|
3156
|
+
sed -i.bak 's/^small\tM\t/small\tL\t/' "$zroute/small.quality.md" && rm -f "$zroute/small.quality.md.bak"
|
|
3157
|
+
zout2=$(bash "$FS" "$tmp/findings-zero-route" 2>&1)
|
|
3158
|
+
check "findings: an L-route spec does count toward the rule" 1 \
|
|
3159
|
+
"$(grep -c '^Zero-findings rule: 2 of 2 completed specs' <<<"$zout2")"
|
|
3160
|
+
rm -rf "$tmp/findings-zero-route"
|
|
3161
|
+
|
|
3162
|
+
# The rule measures a clean first pass, so the record has to claim one. Prose
|
|
3163
|
+
# that mentions an absence is not a measurement, and crediting it inflated the
|
|
3164
|
+
# numerator with a spec whose first pass was never established.
|
|
3165
|
+
fpass="$tmp/findings-first-pass/devos/specs/archive"
|
|
3166
|
+
mkdir -p "$fpass/prose" "$fpass/measured"
|
|
3167
|
+
printf -- '- [prose] [warning] defer: no findings were observed\n' \
|
|
3168
|
+
> "$tmp/findings-first-pass/devos/deferred.md"
|
|
3169
|
+
printf 'prose\tL\tyes\t0\tnone\tyes\t?\t-/0\t-\n' > "$fpass/prose.quality.md"
|
|
3170
|
+
printf 'measured\tL\tyes\t0\tnone\tyes\t?\t-/0\t-\n' > "$fpass/measured.quality.md"
|
|
3171
|
+
fpout=$(bash "$FS" "$tmp/findings-first-pass" 2>&1)
|
|
3172
|
+
check "findings: prose mentioning no findings is not a first-pass claim" 1 \
|
|
3173
|
+
"$(grep -c '^Zero-findings rule: 0 of 2 completed specs' <<<"$fpout")"
|
|
3174
|
+
printf -- '- [measured] [warning] defer: zero findings on first run\n' \
|
|
3175
|
+
>> "$tmp/findings-first-pass/devos/deferred.md"
|
|
3176
|
+
fpout2=$(bash "$FS" "$tmp/findings-first-pass" 2>&1)
|
|
3177
|
+
check "findings: an explicit first-run claim is still counted" 1 \
|
|
3178
|
+
"$(grep -c '^Zero-findings rule: 1 of 2 completed specs' <<<"$fpout2")"
|
|
3179
|
+
rm -rf "$tmp/findings-first-pass"
|
|
3180
|
+
|
|
3181
|
+
# --- .quality.md: C route and the two derived columns ------
|
|
3182
|
+
QC="$SKILLS/status/scripts/quality-correlation.sh"
|
|
3183
|
+
qdir="$tmp/qual/devos/specs/archive"
|
|
3184
|
+
mkdir -p "$qdir"
|
|
3185
|
+
printf 'a\tL\tyes\t1\tsuggestion\tyes\t2\t2/5\n' >"$qdir/a.quality.md"
|
|
3186
|
+
printf 'b\tM\tyes\t0\tnone\tyes\t1\t-/0\n' >"$qdir/b.quality.md"
|
|
3187
|
+
printf 'c\tC\tyes\t0\tnone\tyes\t1\t0/2\n' >"$qdir/c.quality.md"
|
|
3188
|
+
printf 'd\tC\tno\t3\tcritical\tno\t4\t3/3\n' >"$qdir/d.quality.md"
|
|
3189
|
+
printf 'e\tL\tyes\t0\tnone\tyes\t?\t?/4\n' >"$qdir/e.quality.md"
|
|
3190
|
+
printf 'f\tL\tyes\t2\twarning\tyes\n' >"$qdir/f.quality.md"
|
|
3191
|
+
qout=$(bash "$QC" "$tmp/qual" 2>&1)
|
|
3192
|
+
|
|
3193
|
+
check "quality: the C route is counted, not folded into ?" 1 \
|
|
3194
|
+
"$(grep -cE '^C +2 +3 +1\.5$' <<<"$qout")"
|
|
3195
|
+
check "quality: a 6-field row still counts toward findings" 1 \
|
|
3196
|
+
"$(grep -cE '^TOTAL +6 +6 +1\.0$' <<<"$qout")"
|
|
3197
|
+
check "quality: an unknown triage count is excluded from the average" 1 \
|
|
3198
|
+
"$(grep -cE '^L +1 +2\.0' <<<"$qout")"
|
|
3199
|
+
check "quality: the audit hit rate ignores unrecorded hits" 1 \
|
|
3200
|
+
"$(grep -c '5 of 10 audit findings confirmed downstream (50%), across 4 spec(s)' <<<"$qout")"
|
|
3201
|
+
check "quality: an excluded row is reported, not silently dropped" 1 \
|
|
3202
|
+
"$(grep -c '1 spec(s) excluded: audit findings recorded, hits never were' <<<"$qout")"
|
|
3203
|
+
|
|
3204
|
+
qbad="$tmp/qual-bad/devos/specs/archive"
|
|
3205
|
+
mkdir -p "$qbad"
|
|
3206
|
+
for name in a b c d e; do
|
|
3207
|
+
printf '%s\tL\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qbad/$name.quality.md"
|
|
3208
|
+
done
|
|
3209
|
+
printf 'malformed row\n' > "$qbad/f.quality.md"
|
|
3210
|
+
qbadout=$(bash "$QC" "$tmp/qual-bad" 2>&1)
|
|
3211
|
+
check "quality: malformed rows do not inflate analyzed specs" 1 \
|
|
3212
|
+
"$(grep -c '^Specs analyzed: 5$' <<<"$qbadout")"
|
|
3213
|
+
check "quality: malformed rows are surfaced" 1 \
|
|
3214
|
+
"$(grep -c '^Skipped quality rows: 1 (fewer than 6 tab-separated fields)$' <<<"$qbadout")"
|
|
3215
|
+
|
|
3216
|
+
qblank="$tmp/qual-blank-audit/devos/specs/archive"
|
|
3217
|
+
mkdir -p "$qblank"
|
|
3218
|
+
for name in a b c d e; do
|
|
3219
|
+
printf '%s\tS\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qblank/$name.quality.md"
|
|
3220
|
+
done
|
|
3221
|
+
printf 'blank-audit\tS\tyes\t0\tnone\tyes\t1\t\t-\n' > "$qblank/blank-audit.quality.md"
|
|
3222
|
+
qblankout=$(bash "$QC" "$tmp/qual-blank-audit" 2>&1)
|
|
3223
|
+
check "quality: a present blank audit field is surfaced" 1 \
|
|
3224
|
+
"$(grep -c '^Invalid audit-hits values: 1 ' <<<"$qblankout")"
|
|
3225
|
+
check "quality: a blank audit field is excluded only from audit metrics" 1 \
|
|
3226
|
+
"$(grep -c 'no audit findings to confirm across 5 spec(s)' <<<"$qblankout")"
|
|
3227
|
+
check "quality: a blank audit row remains in general aggregates" 1 \
|
|
3228
|
+
"$(grep -cE '^TOTAL +6 +0 +0\.0$' <<<"$qblankout")"
|
|
3229
|
+
|
|
3230
|
+
qabsent="$tmp/qual-absent-audit/devos/specs/archive"
|
|
3231
|
+
mkdir -p "$qabsent"
|
|
3232
|
+
for name in a b c d e; do
|
|
3233
|
+
printf '%s\tS\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qabsent/$name.quality.md"
|
|
3234
|
+
done
|
|
3235
|
+
printf 'absent-audit\tS\tyes\t0\tnone\tyes\t1\t-\n' > "$qabsent/absent-audit.quality.md"
|
|
3236
|
+
qabsentout=$(bash "$QC" "$tmp/qual-absent-audit" 2>&1)
|
|
3237
|
+
check "quality: an absent audit column is not invalid" 1 \
|
|
3238
|
+
"$(grep -c '^Invalid audit-hits values:' <<<"$qabsentout" || true)"
|
|
3239
|
+
check "quality: an absent audit column remains analyzable" 1 \
|
|
3240
|
+
"$(grep -cE '^TOTAL +6 +0 +0\.0$' <<<"$qabsentout")"
|
|
3241
|
+
|
|
3242
|
+
qmeta="$tmp/qual-audit-metadata/devos/specs/archive"
|
|
3243
|
+
mkdir -p "$qmeta"
|
|
3244
|
+
for name in a b c d e; do
|
|
3245
|
+
printf '%s\tS\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qmeta/$name.quality.md"
|
|
3246
|
+
done
|
|
3247
|
+
printf 'bad-denominator\tS\tyes\t0\tnone\tyes\t1\t-/x\t-\n' > "$qmeta/bad-denominator.quality.md"
|
|
3248
|
+
printf 'bad-components\tS\tyes\t0\tnone\tyes\t1\t2/5/9\t-\n' > "$qmeta/bad-components.quality.md"
|
|
3249
|
+
qmetaout=$(bash "$QC" "$tmp/qual-audit-metadata" 2>&1)
|
|
3250
|
+
check "quality: nonnumeric audit denominators are surfaced" 1 \
|
|
3251
|
+
"$(grep -c '^Invalid audit-hits values: 2 ' <<<"$qmetaout")"
|
|
3252
|
+
check "quality: extra audit components are not accepted" 1 \
|
|
3253
|
+
"$(grep -cE '^TOTAL +7 +0 +0\.0$' <<<"$qmetaout")"
|
|
3254
|
+
|
|
3255
|
+
# Every per-route table walked the route map with a bare for-in, so the rows
|
|
3256
|
+
# came out in awk hash order: C, L, M, S on this awk, and no guarantee at all
|
|
3257
|
+
# on another, which is the opposite of the S, M, L, C the script declares in its
|
|
3258
|
+
# own route list. A reader comparing routes by size needs a stable order.
|
|
3259
|
+
qorder="$tmp/qual-route-order/devos/specs/archive"
|
|
3260
|
+
mkdir -p "$qorder"
|
|
3261
|
+
i=0
|
|
3262
|
+
for route in L S M C; do
|
|
3263
|
+
for n in 1 2; do
|
|
3264
|
+
i=$((i + 1))
|
|
3265
|
+
printf 'spec%s\t%s\tyes\t2\twarning\tyes\t1\t-/0\t-\n' "$i" "$route" \
|
|
3266
|
+
> "$qorder/spec$i.quality.md"
|
|
3267
|
+
done
|
|
3268
|
+
done
|
|
3269
|
+
qorderout=$(bash "$QC" "$tmp/qual-route-order" 2>&1)
|
|
3270
|
+
# Control: each route's own numbers are right, so a report that printed no
|
|
3271
|
+
# per-route table could not satisfy the ordering checks.
|
|
3272
|
+
check "quality: every route reports its own specs and findings" 4 \
|
|
3273
|
+
"$(grep -cE '^(S|M|L|C) +2 +4 +2\.0$' <<<"$qorderout")"
|
|
3274
|
+
qtable=$(sed -n '/^## Findings per Route/,/^## Routing/p' <<<"$qorderout" |
|
|
3275
|
+
grep -oE '^[SMLC]([[:space:]]|$)' | tr -d '[:space:]' | tr -d '\n')
|
|
3276
|
+
check "quality: the per-route tables read in declared S/M/L/C order" 0 \
|
|
3277
|
+
"$([[ "$qtable" == 'SMLC' ]] && echo 0 || echo 1)"
|
|
3278
|
+
# All four tables walk the same map, so all four must be ordered.
|
|
3279
|
+
qmatrix=$(sed -n '/^## Routing Confusion Matrix/,/^## Triage/p' <<<"$qorderout" |
|
|
3280
|
+
grep -oE '^[SMLC]([[:space:]]|$)' | tr -d '[:space:]' | tr -d '\n')
|
|
3281
|
+
check "quality: the confusion matrix reads in the same declared order" 0 \
|
|
3282
|
+
"$([[ "$qmatrix" == 'SMLC' ]] && echo 0 || echo 1)"
|
|
3283
|
+
qloops=$(sed -n '/^## Triage Loops per Route/,/^## Spec Audit/p' <<<"$qorderout" |
|
|
3284
|
+
grep -oE '^[SMLC]([[:space:]]|$)' | tr -d '[:space:]' | tr -d '\n')
|
|
3285
|
+
check "quality: the triage table reads in the same declared order" 0 \
|
|
3286
|
+
"$([[ "$qloops" == 'SMLC' ]] && echo 0 || echo 1)"
|
|
3287
|
+
# The unrecognized-route bucket belongs after the recognized ones, not wherever
|
|
3288
|
+
# the hash put it.
|
|
3289
|
+
qbucket="$tmp/qual-route-bucket/devos/specs/archive"
|
|
3290
|
+
mkdir -p "$qbucket"
|
|
3291
|
+
i=0
|
|
3292
|
+
for route in C L S M banana banana; do
|
|
3293
|
+
i=$((i + 1))
|
|
3294
|
+
printf 'spec%s\t%s\tyes\t2\twarning\tyes\t1\t-/0\t-\n' "$i" "$route" \
|
|
3295
|
+
> "$qbucket/spec$i.quality.md"
|
|
3296
|
+
done
|
|
3297
|
+
qbucketout=$(bash "$QC" "$tmp/qual-route-bucket" 2>&1)
|
|
3298
|
+
qbucketfirst=$(sed -n '/^## Findings per Route/,/^## Routing/p' <<<"$qbucketout" |
|
|
3299
|
+
grep -oE '^[SMLC?]' | tr -d '\n')
|
|
3300
|
+
check "quality: an unrecognized route is bucketed after the real routes" 0 \
|
|
3301
|
+
"$([[ "$qbucketfirst" == 'SMLC?' ]] && echo 0 || echo 1)"
|
|
3302
|
+
check "quality: the ? bucket still counts its specs" 1 \
|
|
3303
|
+
"$(grep -cE '^\? +2 +4 +2\.0$' <<<"$qbucketout")"
|
|
3304
|
+
rm -rf "$tmp/qual-route-order" "$tmp/qual-route-bucket"
|
|
3305
|
+
|
|
3306
|
+
qvalidate="$tmp/qual-validate/devos/specs/archive"
|
|
3307
|
+
mkdir -p "$qvalidate"
|
|
3308
|
+
for name in a b c d e; do
|
|
3309
|
+
printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qvalidate/$name.quality.md"
|
|
3310
|
+
done
|
|
3311
|
+
printf 'bad-validate\tS\tmaybe\t2\tnone\tyes\t1\t-/0\t-\n' > "$qvalidate/bad-validate.quality.md"
|
|
3312
|
+
qvalidateout=$(bash "$QC" "$tmp/qual-validate" 2>&1)
|
|
3313
|
+
check "quality: invalid validate-pass values are excluded" 1 \
|
|
3314
|
+
"$(grep -c '^Specs analyzed: 5$' <<<"$qvalidateout")"
|
|
3315
|
+
check "quality: invalid validate-pass values are surfaced" 1 \
|
|
3316
|
+
"$(grep -c '^Invalid validate-pass values: 1 ' <<<"$qvalidateout")"
|
|
3317
|
+
check "quality: invalid validate-pass rows do not dilute totals" 1 \
|
|
3318
|
+
"$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qvalidateout")"
|
|
3319
|
+
|
|
3320
|
+
qtriage="$tmp/qual-triage/devos/specs/archive"
|
|
3321
|
+
mkdir -p "$qtriage"
|
|
3322
|
+
for name in a b c d e; do
|
|
3323
|
+
printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qtriage/$name.quality.md"
|
|
3324
|
+
done
|
|
3325
|
+
printf 'bad-triage\tS\tyes\t2\tnone\tyes\tmany\t-/0\t-\n' > "$qtriage/bad-triage.quality.md"
|
|
3326
|
+
qtriageout=$(bash "$QC" "$tmp/qual-triage" 2>&1)
|
|
3327
|
+
check "quality: malformed triage-loop values stay out of the average" 1 \
|
|
3328
|
+
"$(grep -c '^Invalid triage-loop values: 1 ' <<<"$qtriageout")"
|
|
3329
|
+
check "quality: malformed triage-loop rows remain available for other metrics" 1 \
|
|
3330
|
+
"$(grep -cE '^TOTAL +6 +12 +2\.0$' <<<"$qtriageout")"
|
|
3331
|
+
printf 'unknown-triage\tS\tyes\t2\tnone\tyes\t?\t-/0\t-\n' > "$qtriage/unknown-triage.quality.md"
|
|
3332
|
+
qtriageout=$(bash "$QC" "$tmp/qual-triage" 2>&1)
|
|
3333
|
+
check "quality: unknown triage-loop remains valid but does not enter the average" 1 \
|
|
3334
|
+
"$(grep -cE '^S +5 +1\.0$' <<<"$qtriageout")"
|
|
3335
|
+
|
|
3336
|
+
qdod="$tmp/qual-dod/devos/specs/archive"
|
|
3337
|
+
mkdir -p "$qdod"
|
|
3338
|
+
for name in a b c d e; do
|
|
3339
|
+
printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qdod/$name.quality.md"
|
|
3340
|
+
done
|
|
3341
|
+
printf 'bad-dod\tS\tyes\t2\tnone\tmaybe\t1\t-/0\t-\n' > "$qdod/bad-dod.quality.md"
|
|
3342
|
+
qdodout=$(bash "$QC" "$tmp/qual-dod" 2>&1)
|
|
3343
|
+
check "quality: invalid dod-pass values are excluded" 1 \
|
|
3344
|
+
"$(grep -c '^Specs analyzed: 5$' <<<"$qdodout")"
|
|
3345
|
+
check "quality: invalid dod-pass values are surfaced" 1 \
|
|
3346
|
+
"$(grep -c '^Invalid dod-pass values: 1 ' <<<"$qdodout")"
|
|
3347
|
+
check "quality: invalid dod-pass rows do not dilute totals" 1 \
|
|
3348
|
+
"$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qdodout")"
|
|
3349
|
+
|
|
3350
|
+
qseverity="$tmp/qual-severity/devos/specs/archive"
|
|
3351
|
+
mkdir -p "$qseverity"
|
|
3352
|
+
for name in a b c d e; do
|
|
3353
|
+
printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qseverity/$name.quality.md"
|
|
3354
|
+
done
|
|
3355
|
+
printf 'bad-severity\tS\tyes\t2\tbanana\tyes\t1\t-/0\t-\n' > "$qseverity/bad-severity.quality.md"
|
|
3356
|
+
qseverityout=$(bash "$QC" "$tmp/qual-severity" 2>&1)
|
|
3357
|
+
check "quality: invalid severity values are excluded" 1 \
|
|
3358
|
+
"$(grep -c '^Specs analyzed: 5$' <<<"$qseverityout")"
|
|
3359
|
+
check "quality: invalid severity values are surfaced" 1 \
|
|
3360
|
+
"$(grep -c '^Invalid severity values: 1 ' <<<"$qseverityout")"
|
|
3361
|
+
check "quality: invalid severity rows do not dilute totals" 1 \
|
|
3362
|
+
"$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qseverityout")"
|
|
3363
|
+
|
|
3364
|
+
qunderstanding="$tmp/qual-understanding/devos/specs/archive"
|
|
3365
|
+
mkdir -p "$qunderstanding"
|
|
3366
|
+
for name in a b c d e; do
|
|
3367
|
+
printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qunderstanding/$name.quality.md"
|
|
3368
|
+
done
|
|
3369
|
+
printf 'bad-understanding\tS\tyes\t2\tnone\tyes\t1\t-/0\tbanana\n' > "$qunderstanding/bad-understanding.quality.md"
|
|
3370
|
+
qunderstandingout=$(bash "$QC" "$tmp/qual-understanding" 2>&1)
|
|
3371
|
+
check "quality: invalid understanding values are excluded" 1 \
|
|
3372
|
+
"$(grep -c '^Specs analyzed: 5$' <<<"$qunderstandingout")"
|
|
3373
|
+
check "quality: invalid understanding values are surfaced" 1 \
|
|
3374
|
+
"$(grep -c '^Invalid understanding values: 1 ' <<<"$qunderstandingout")"
|
|
3375
|
+
check "quality: invalid understanding rows do not dilute totals" 1 \
|
|
3376
|
+
"$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qunderstandingout")"
|
|
3377
|
+
|
|
3378
|
+
qdiag="$tmp/qual-diagnostic/devos/specs/archive"
|
|
3379
|
+
mkdir -p "$qdiag"
|
|
3380
|
+
for name in a b c d e; do
|
|
3381
|
+
printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qdiag/$name.quality.md"
|
|
3382
|
+
done
|
|
3383
|
+
printf 'bad-count\tS\tyes\tbogus\tnone\tyes\t1\t-/0\t-\n' > "$qdiag/bad-count.quality.md"
|
|
3384
|
+
printf 'short row\n' > "$qdiag/short.quality.md"
|
|
3385
|
+
qdiagout=$(bash "$QC" "$tmp/qual-diagnostic" 2>&1)
|
|
3386
|
+
check "quality: short rows and invalid counts have separate diagnostics" 1 \
|
|
3387
|
+
"$(grep -c '^Specs analyzed: 5$' <<<"$qdiagout")"
|
|
3388
|
+
check "quality: only the short row is reported as skipped" 1 \
|
|
3389
|
+
"$(grep -c '^Skipped quality rows: 1 (fewer than 6 tab-separated fields)$' <<<"$qdiagout")"
|
|
3390
|
+
check "quality: invalid finding counts are not mislabeled as short rows" 1 \
|
|
3391
|
+
"$(grep -c '^Invalid finding-count values: 1 ' <<<"$qdiagout")"
|
|
3392
|
+
|
|
3393
|
+
qfindings="$tmp/qual-findings/devos/specs/archive"
|
|
3394
|
+
mkdir -p "$qfindings"
|
|
3395
|
+
for name in a b c d e; do
|
|
3396
|
+
printf '%s\tS\tyes\t2\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qfindings/$name.quality.md"
|
|
3397
|
+
done
|
|
3398
|
+
printf 'bad\tS\tyes\tbogus\tnone\tyes\t1\t-/0\t-\n' > "$qfindings/bad.quality.md"
|
|
3399
|
+
qfindingsout=$(bash "$QC" "$tmp/qual-findings" 2>&1)
|
|
3400
|
+
check "quality: non-integer finding counts are excluded" 1 \
|
|
3401
|
+
"$(grep -c '^Specs analyzed: 5$' <<<"$qfindingsout")"
|
|
3402
|
+
check "quality: non-integer finding counts are surfaced" 1 \
|
|
3403
|
+
"$(grep -c '^Invalid finding-count values: 1 ' <<<"$qfindingsout")"
|
|
3404
|
+
check "quality: invalid finding counts do not dilute averages" 1 \
|
|
3405
|
+
"$(grep -cE '^TOTAL +5 +10 +2\.0$' <<<"$qfindingsout")"
|
|
3406
|
+
|
|
3407
|
+
qaudit="$tmp/qual-audit/devos/specs/archive"
|
|
3408
|
+
mkdir -p "$qaudit"
|
|
3409
|
+
for name in a b c d; do
|
|
3410
|
+
printf '%s\tL\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qaudit/$name.quality.md"
|
|
3411
|
+
done
|
|
3412
|
+
printf 'e\tL\tyes\t0\tnone\tyes\t1\t-/5\t-\n' > "$qaudit/e.quality.md"
|
|
3413
|
+
qaout=$(bash "$QC" "$tmp/qual-audit" 2>&1)
|
|
3414
|
+
check "quality: malformed audit metadata is not clean" 1 \
|
|
3415
|
+
"$(grep -c 'no audit findings to confirm across 4 spec(s)' <<<"$qaout")"
|
|
3416
|
+
check "quality: malformed audit metadata is surfaced" 1 \
|
|
3417
|
+
"$(grep -c '^Invalid audit-hits values: 1 ' <<<"$qaout")"
|
|
3418
|
+
|
|
3419
|
+
qratio="$tmp/qual-audit-ratio/devos/specs/archive"
|
|
3420
|
+
mkdir -p "$qratio"
|
|
3421
|
+
for name in a b c d; do
|
|
3422
|
+
printf '%s\tL\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qratio/$name.quality.md"
|
|
3423
|
+
done
|
|
3424
|
+
printf 'e\tL\tyes\t0\tnone\tyes\t1\t3/2\t-\n' > "$qratio/e.quality.md"
|
|
3425
|
+
qrout=$(bash "$QC" "$tmp/qual-audit-ratio" 2>&1)
|
|
3426
|
+
check "quality: an impossible audit ratio is not included" 1 \
|
|
3427
|
+
"$(grep -c 'no audit findings to confirm across 4 spec(s)' <<<"$qrout")"
|
|
3428
|
+
check "quality: an impossible audit ratio is surfaced" 1 \
|
|
3429
|
+
"$(grep -c '^Invalid audit-hits values: 1 ' <<<"$qrout")"
|
|
3430
|
+
|
|
3431
|
+
qsentinel="$tmp/qual-audit-sentinel/devos/specs/archive"
|
|
3432
|
+
mkdir -p "$qsentinel"
|
|
3433
|
+
for name in a b c d; do
|
|
3434
|
+
printf '%s\tL\tyes\t0\tnone\tyes\t1\t-/0\t-\n' "$name" > "$qsentinel/$name.quality.md"
|
|
3435
|
+
done
|
|
3436
|
+
printf 'e\tL\tyes\t0\tnone\tyes\t1\t0/0\t-\n' > "$qsentinel/e.quality.md"
|
|
3437
|
+
qsout=$(bash "$QC" "$tmp/qual-audit-sentinel" 2>&1)
|
|
3438
|
+
check "quality: numeric 0/0 is not counted as clean" 1 \
|
|
3439
|
+
"$(grep -c 'no audit findings to confirm across 4 spec(s)' <<<"$qsout")"
|
|
3440
|
+
check "quality: numeric 0/0 is surfaced as invalid" 1 \
|
|
3441
|
+
"$(grep -c '^Invalid audit-hits values: 1 ' <<<"$qsout")"
|
|
3442
|
+
|
|
3443
|
+
# --- artifact-budgets.sh: the three bands ----------------------------------
|
|
3444
|
+
AB="$SKILLS/status/scripts/artifact-budgets.sh"
|
|
3445
|
+
bdir="$tmp/budget"
|
|
3446
|
+
|
|
3447
|
+
mkdir -p "$bdir"
|
|
3448
|
+
check "budget: no onboarded artifacts is not a failure" 0 \
|
|
3449
|
+
"$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
|
|
3450
|
+
check "budget: no artifacts says so rather than printing an empty table" 1 \
|
|
3451
|
+
"$(bash "$AB" "$bdir" 2>&1 | grep -c 'No budgeted artifacts found')"
|
|
3452
|
+
|
|
3453
|
+
mkdir -p "$bdir/devos/product"
|
|
3454
|
+
# Cap 60: 53 lines is OK (below the 54-line 90% mark), 54 is NEAR, 61 is OVER.
|
|
3455
|
+
seq 1 53 >"$bdir/devos/product/domain-model.md"
|
|
3456
|
+
check "budget: below 90% of cap is OK" 1 \
|
|
3457
|
+
"$(bash "$AB" "$bdir" 2>&1 | grep -c '^| OK | devos/product/domain-model.md | 53 | 60 |')"
|
|
3458
|
+
check "budget: an OK-only run exits 0" 0 \
|
|
3459
|
+
"$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
|
|
3460
|
+
|
|
3461
|
+
seq 1 54 >"$bdir/devos/product/domain-model.md"
|
|
3462
|
+
check "budget: 90% of cap is NEAR" 1 \
|
|
3463
|
+
"$(bash "$AB" "$bdir" 2>&1 | grep -c '^| NEAR | devos/product/domain-model.md | 54 | 60 |')"
|
|
3464
|
+
check "budget: NEAR is reported but never fatal" 0 \
|
|
3465
|
+
"$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
|
|
3466
|
+
|
|
3467
|
+
seq 1 60 >"$bdir/devos/product/domain-model.md"
|
|
3468
|
+
check "budget: exactly at cap is NEAR, not OVER" 1 \
|
|
3469
|
+
"$(bash "$AB" "$bdir" 2>&1 | grep -c '^| NEAR | devos/product/domain-model.md | 60 | 60 |')"
|
|
3470
|
+
|
|
3471
|
+
seq 1 61 >"$bdir/devos/product/domain-model.md"
|
|
3472
|
+
check "budget: one line past cap is OVER" 1 \
|
|
3473
|
+
"$(bash "$AB" "$bdir" 2>&1 | grep -c '^| OVER | devos/product/domain-model.md | 61 | 60 |')"
|
|
3474
|
+
check "budget: an OVER artifact exits 1" 1 \
|
|
3475
|
+
"$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
|
|
3476
|
+
|
|
3477
|
+
{ seq 1 60; printf '61'; } >"$bdir/devos/product/domain-model.md"
|
|
3478
|
+
check "budget: an unterminated final line is counted" 1 \
|
|
3479
|
+
"$(bash "$AB" "$bdir" 2>&1 | grep -c '^| OVER | devos/product/domain-model.md | 61 | 60 |')"
|
|
3480
|
+
check "budget: an unterminated final OVER line exits 1" 1 \
|
|
3481
|
+
"$(bash "$AB" "$bdir" >/dev/null 2>&1; echo $?)"
|
|
3482
|
+
|
|
3483
|
+
# Row order, not just row content. Lexical order over the band names is
|
|
3484
|
+
# NEAR, OK, OVER, which puts the only actionable row last in a long table.
|
|
3485
|
+
mkdir -p "$bdir/devos/product/behaviors"
|
|
3486
|
+
seq 1 54 >"$bdir/devos/product/behaviors/near.md"
|
|
3487
|
+
seq 1 10 >"$bdir/devos/product/behaviors/fine.md"
|
|
3488
|
+
bandorder=$(bash "$AB" "$bdir" 2>&1 | awk -F'|' '/^\| (OK|NEAR|OVER) \|/ { gsub(/ /, "", $2); printf "%s ", $2 }')
|
|
3489
|
+
check "budget: OVER leads, then NEAR, then OK" "OVER NEAR OK " "$bandorder"
|
|
3490
|
+
rm -f "$bdir/devos/product/behaviors/near.md" "$bdir/devos/product/behaviors/fine.md"
|
|
3491
|
+
|
|
3492
|
+
# Each artifact class carries its own cap: 121 lines is OVER for the map (120)
|
|
3493
|
+
# but well inside a capsule's 300.
|
|
3494
|
+
mkdir -p "$bdir/devos/specs/archive/demo"
|
|
3495
|
+
seq 1 121 >"$bdir/devos/product/codebase-map.md"
|
|
3496
|
+
seq 1 121 >"$bdir/devos/specs/archive/demo/spec.md"
|
|
3497
|
+
bout=$(bash "$AB" "$bdir" 2>&1)
|
|
3498
|
+
check "budget: the map's cap is 120, not the capsule's 300" 1 \
|
|
3499
|
+
"$(grep -c '^| OVER | devos/product/codebase-map.md | 121 | 120 |' <<<"$bout")"
|
|
3500
|
+
check "budget: the same line count is OK inside a capsule's larger cap" 1 \
|
|
3501
|
+
"$(grep -c '^| OK | devos/specs/archive/demo/spec.md | 121 | 300 |' <<<"$bout")"
|
|
3502
|
+
check "budget: OVER rows lead the table so the actionable ones are read first" "OVER" \
|
|
3503
|
+
"$(grep '^| ' <<<"$bout" | sed -n '3p' | awk -F'|' '{gsub(/ /,"",$2); print $2}')"
|
|
3504
|
+
check "budget: every OVER artifact is counted in the failure line" 1 \
|
|
3505
|
+
"$(grep -c '^2 artifact(s) OVER budget' <<<"$bout")"
|
|
3506
|
+
|
|
3507
|
+
# A behaviors file shares the domain model's 60-line page budget.
|
|
3508
|
+
mkdir -p "$bdir/devos/product/behaviors"
|
|
3509
|
+
seq 1 61 >"$bdir/devos/product/behaviors/thing.md"
|
|
3510
|
+
check "budget: a behaviors capability page shares the 60-line cap" 1 \
|
|
3511
|
+
"$(bash "$AB" "$bdir" 2>&1 | grep -c '^| OVER | devos/product/behaviors/thing.md | 61 | 60 |')"
|
|
3512
|
+
|
|
3513
|
+
check "budget: a missing repo directory fails loudly" 1 \
|
|
3514
|
+
"$(bash "$AB" "$tmp/no-such-dir" >/dev/null 2>&1; echo $?)"
|
|
3515
|
+
|
|
3516
|
+
# --- convention-usage.sh: which conventions capsules actually exercised -----
|
|
3517
|
+
CU="$SKILLS/status/scripts/convention-usage.sh"
|
|
3518
|
+
cdir="$tmp/convuse"
|
|
3519
|
+
cbdir="$tmp/convuse-bullets"
|
|
3520
|
+
|
|
3521
|
+
mkdir -p "$cdir"
|
|
3522
|
+
check "convuse: no conventions.md is not a failure" 0 \
|
|
3523
|
+
"$(bash "$CU" "$cdir" >/dev/null 2>&1; echo $?)"
|
|
3524
|
+
check "convuse: no conventions.md says so" 1 \
|
|
3525
|
+
"$(bash "$CU" "$cdir" 2>&1 | grep -c 'nothing onboarded')"
|
|
3526
|
+
|
|
3527
|
+
mkdir -p "$cdir/devos/standards" "$cdir/devos/specs/archive"
|
|
3528
|
+
{
|
|
3529
|
+
echo "# Project Conventions"
|
|
3530
|
+
echo ""
|
|
3531
|
+
echo "## Testing"
|
|
3532
|
+
echo ""
|
|
3533
|
+
echo "- Asserts live in \`plugins/devos/scripts/gates/test-gates.sh\`, no framework."
|
|
3534
|
+
echo "- Fixtures under \`evals/fixtures/\` are copied to a temp dir first."
|
|
3535
|
+
echo "- One logical change per commit."
|
|
3536
|
+
echo "- Never invoke \`legacy/old-runner.sh\`; it was replaced."
|
|
3537
|
+
echo "- Each spec writes a \`devos/specs/<name>/spec.md\` capsule."
|
|
3538
|
+
} >"$cdir/devos/standards/conventions.md"
|
|
3539
|
+
|
|
3540
|
+
# Below the floor the ratio is meaningless, so it must refuse rather than guess.
|
|
3541
|
+
for n in a b c; do
|
|
3542
|
+
mkdir -p "$cdir/devos/specs/archive/spec-$n"
|
|
3543
|
+
echo "ran test-gates.sh over evals/fixtures/thing" >"$cdir/devos/specs/archive/spec-$n/spec.md"
|
|
3544
|
+
done
|
|
3545
|
+
check "convuse: below 5 capsules reports INSUFFICIENT DATA" 1 \
|
|
3546
|
+
"$(bash "$CU" "$cdir" 2>&1 | grep -c 'INSUFFICIENT DATA: only 3 archived capsule')"
|
|
3547
|
+
check "convuse: INSUFFICIENT DATA still exits 0 (a report, not a gate)" 0 \
|
|
3548
|
+
"$(bash "$CU" "$cdir" >/dev/null 2>&1; echo $?)"
|
|
3549
|
+
|
|
3550
|
+
for n in d e; do
|
|
3551
|
+
mkdir -p "$cdir/devos/specs/archive/spec-$n"
|
|
3552
|
+
echo "ran test-gates.sh over evals/fixtures/thing" >"$cdir/devos/specs/archive/spec-$n/spec.md"
|
|
3553
|
+
done
|
|
3554
|
+
cout=$(bash "$CU" "$cdir" 2>&1)
|
|
3555
|
+
|
|
3556
|
+
check "convuse: at the floor it reports instead of refusing" 1 \
|
|
3557
|
+
"$(grep -c '^Source: 5 archived capsule' <<<"$cout")"
|
|
3558
|
+
check "convuse: an exact path match counts as CITED" 1 \
|
|
3559
|
+
"$(grep -c '^| CITED | 2 |' <<<"$cout")"
|
|
3560
|
+
check "convuse: a full-path convention is CITED when the capsule names just the file" 1 \
|
|
3561
|
+
"$(grep -c '| test-gates.sh |' <<<"$cout")"
|
|
3562
|
+
check "convuse: a basename-only match is flagged for a second look" 1 \
|
|
3563
|
+
"$(grep -c '^## CITED by filename only' <<<"$cout")"
|
|
3564
|
+
check "convuse: an unambiguous full-path match is not listed as needing one" 0 \
|
|
3565
|
+
"$(sed -n '/^## CITED by filename only/,/^$/p' <<<"$cout" | grep -c 'evals/fixtures')"
|
|
3566
|
+
check "convuse: an identifier no capsule mentions is UNCITED" 1 \
|
|
3567
|
+
"$(grep -c 'Never invoke legacy/old-runner.sh' <<<"$cout")"
|
|
3568
|
+
check "convuse: a prose-only entry is UNMEASURABLE, never UNCITED" 1 \
|
|
3569
|
+
"$(grep -c '^| UNMEASURABLE | 1 |' <<<"$cout")"
|
|
3570
|
+
check "convuse: a generic basename never produces a false CITED" 1 \
|
|
3571
|
+
"$(grep -c 'devos/specs/<name>/spec.md' <<<"$cout")"
|
|
3572
|
+
check "convuse: every entry lands in exactly one class" "5" \
|
|
3573
|
+
"$(awk -F'|' '/^\| (CITED|UNCITED|UNMEASURABLE) \|/ {s+=$3} END{print s}' <<<"$cout")"
|
|
3574
|
+
mkdir -p "$cdir/devos/specs/archive/spec-f"
|
|
3575
|
+
printf '%s\n' "ran test-gates.sh over evals/fixtures/thing" > "$cdir/devos/specs/archive/spec-f/spec.md"
|
|
3576
|
+
printf '%s' '- Never invoke `legacy/final-runner.sh`; it was replaced.' > "$cdir/devos/standards/conventions.md"
|
|
3577
|
+
cfinal=$(bash "$CU" "$cdir" 2>&1)
|
|
3578
|
+
check "convuse: a final unterminated bullet is counted" 1 \
|
|
3579
|
+
"$(grep -c '^| UNCITED | 1 |$' <<<"$cfinal")"
|
|
3580
|
+
check "convuse: uncited entries never change the exit code" 0 \
|
|
3581
|
+
"$(bash "$CU" "$cdir" >/dev/null 2>&1; echo $?)"
|
|
3582
|
+
check "convuse: a missing repo directory fails loudly" 1 \
|
|
3583
|
+
"$(bash "$CU" "$tmp/no-such-dir" >/dev/null 2>&1; echo $?)"
|
|
3584
|
+
|
|
3585
|
+
# Both markdown bullet markers make an entry. Only the dash form was
|
|
3586
|
+
# recognized, so a convention written `* foo` was dropped from every class and
|
|
3587
|
+
# from the total: a file with two entries reported one.
|
|
3588
|
+
mkdir -p "$cbdir/devos/specs/archive" "$cbdir/devos/standards"
|
|
3589
|
+
for n in one two three four five; do
|
|
3590
|
+
mkdir -p "$cbdir/devos/specs/archive/spec-$n"
|
|
3591
|
+
echo "uses src/index.js" >"$cbdir/devos/specs/archive/spec-$n/spec.md"
|
|
3592
|
+
done
|
|
3593
|
+
{
|
|
3594
|
+
echo "# Project Conventions"
|
|
3595
|
+
echo ""
|
|
3596
|
+
echo "## Testing"
|
|
3597
|
+
echo ""
|
|
3598
|
+
echo "- \`src/index.js\`: keep the entry point thin."
|
|
3599
|
+
echo "* \`lib/unused.ts\`: written with a star bullet."
|
|
3600
|
+
} >"$cbdir/devos/standards/conventions.md"
|
|
3601
|
+
cbout=$(bash "$CU" "$cbdir" 2>&1)
|
|
3602
|
+
check "convuse: a star-bulleted entry is counted" 1 \
|
|
3603
|
+
"$(grep -c '^| Total entries | 2 |' <<<"$cbout")"
|
|
3604
|
+
check "convuse: an uncited star-bulleted entry is reported, not dropped" 1 \
|
|
3605
|
+
"$(grep -c 'lib/unused.ts' <<<"$cbout")"
|
|
3606
|
+
check "convuse: both bullet forms land in exactly one class" "2" \
|
|
3607
|
+
"$(awk -F'|' '/^\| (CITED|UNCITED|UNMEASURABLE) \|/ {s+=$3} END{print s}' <<<"$cbout")"
|
|
3608
|
+
printf 'uses lib/unused.ts\n' >>"$cbdir/devos/specs/archive/spec-one/spec.md"
|
|
3609
|
+
cbcited=$(bash "$CU" "$cbdir" 2>&1)
|
|
3610
|
+
check "convuse: a cited star-bulleted entry is CITED" 1 \
|
|
3611
|
+
"$(grep -c '^| CITED | 2 |' <<<"$cbcited")"
|
|
3612
|
+
check "convuse: a cited star-bulleted entry is not left in UNCITED" 1 \
|
|
3613
|
+
"$(grep -c '^| UNCITED | 0 |' <<<"$cbcited")"
|
|
3614
|
+
# A star-bulleted prose-only entry is still unmeasurable, not cited by accident.
|
|
3615
|
+
{
|
|
3616
|
+
echo "# Project Conventions"
|
|
3617
|
+
echo ""
|
|
3618
|
+
echo "- one logical change per commit."
|
|
3619
|
+
echo "* \`legacy/old-runner.sh\` was replaced."
|
|
3620
|
+
} >"$cbdir/devos/standards/conventions.md"
|
|
3621
|
+
cbprose=$(bash "$CU" "$cbdir" 2>&1)
|
|
3622
|
+
check "convuse: an uncited star-bulleted entry is UNCITED by identifier" 1 \
|
|
3623
|
+
"$(grep -c '^| UNCITED | 1 |' <<<"$cbprose")"
|
|
3624
|
+
check "convuse: a prose-only star-bulleted entry is UNMEASURABLE" 1 \
|
|
3625
|
+
"$(grep -c '^| UNMEASURABLE | 1 |' <<<"$cbprose")"
|
|
3626
|
+
|
|
3627
|
+
# A spec directory name is a free-form slug, so it can contain a space. The
|
|
3628
|
+
# haystack was built with cat over an unquoted expansion, so such a path split
|
|
3629
|
+
# into two arguments that both failed to open, stderr was discarded, and the
|
|
3630
|
+
# capsule vanished from the search. Every entry only that capsule mentioned was
|
|
3631
|
+
# then listed UNCITED, which is the list a reader is told to consider deleting.
|
|
3632
|
+
csdir="$tmp/convuse-spaced"
|
|
3633
|
+
mkdir -p "$csdir/devos/standards" "$csdir/devos/specs/archive/alpha feature"
|
|
3634
|
+
printf 'uses lib/spaced.ts\n' > "$csdir/devos/specs/archive/alpha feature/spec.md"
|
|
3635
|
+
for n in beta gamma delta epsilon; do
|
|
3636
|
+
mkdir -p "$csdir/devos/specs/archive/spec-$n"
|
|
3637
|
+
echo "ran test-gates.sh over evals/fixtures/thing" >"$csdir/devos/specs/archive/spec-$n/spec.md"
|
|
3638
|
+
done
|
|
3639
|
+
{
|
|
3640
|
+
echo "# Project Conventions"
|
|
3641
|
+
echo ""
|
|
3642
|
+
echo "- \`lib/spaced.ts\`: keep the archive loader in sync."
|
|
3643
|
+
echo "- \`lib/absent.ts\`: delete once it is dead."
|
|
3644
|
+
} >"$csdir/devos/standards/conventions.md"
|
|
3645
|
+
csout=$(bash "$CU" "$csdir" 2>&1)
|
|
3646
|
+
# Control: the identifier no capsule mentions must still be UNCITED, so a
|
|
3647
|
+
# script that classified nothing at all cannot satisfy the next check.
|
|
3648
|
+
check "convuse: an identifier no capsule mentions is still UNCITED" 1 \
|
|
3649
|
+
"$(grep -c 'lib/absent.ts' <<<"$csout")"
|
|
3650
|
+
check "convuse: a capsule under a spaced directory is searched" 1 \
|
|
3651
|
+
"$(grep -c '^| CITED | 1 |' <<<"$csout")"
|
|
3652
|
+
check "convuse: a spaced-capsule entry is not put on the delete list" 0 \
|
|
3653
|
+
"$(grep -c 'lib/spaced.ts' <<<"$csout")"
|
|
3654
|
+
check "convuse: a spaced spec directory still counts toward the capsule floor" 1 \
|
|
3655
|
+
"$(grep -c '^Source: 5 archived capsule' <<<"$csout")"
|
|
3656
|
+
|
|
3657
|
+
# --- Sizing calibration ------------------------------------------------------
|
|
3658
|
+
# The router's size letter lands in the started event's detail; the capsule's
|
|
3659
|
+
# task groups are reality. M with many groups is under-sized, L with one is
|
|
3660
|
+
# over-sized, and a started detail without an S/M/L (resume notes) is skipped.
|
|
3661
|
+
SC="$SKILLS/status/scripts/sizing-calibration.sh"
|
|
3662
|
+
sz="$tmp/sizing/devos/specs"
|
|
3663
|
+
mkdir -p "$sz/a" "$sz/b" "$sz/archive/c" "$sz/archive/d" "$sz/archive/e"
|
|
3664
|
+
printf '{"at":"t","event":"started","detail":"M"}\n' > "$sz/a/ops.jsonl"
|
|
3665
|
+
for i in 1 2 3 4 5 6; do printf '### Task Group %d: g\n' "$i"; done > "$sz/a/spec.md"
|
|
3666
|
+
printf '{"at":"t","event":"started","detail":"L"}\n' > "$sz/b/ops.jsonl"
|
|
3667
|
+
printf '### Task Group 1: only\n' > "$sz/b/spec.md"
|
|
3668
|
+
printf '{"at":"t","event":"started","detail":"resuming at Phase C"}\n' > "$sz/archive/c/ops.jsonl"
|
|
3669
|
+
printf '### Task Group 1: g\n' > "$sz/archive/c/spec.md"
|
|
3670
|
+
# A capital in a path is not a sizing decision. d names a file and nothing
|
|
3671
|
+
# else, so the router never sized it and there is nothing to calibrate; e
|
|
3672
|
+
# names the same file and then carries the real size call, which must still
|
|
3673
|
+
# be read.
|
|
3674
|
+
printf '{"at":"t","event":"started","detail":"resuming: rewrite src/L.ts parser"}\n' > "$sz/archive/d/ops.jsonl"
|
|
3675
|
+
printf '### Task Group 1: g\n' > "$sz/archive/d/spec.md"
|
|
3676
|
+
printf '{"at":"t","event":"started","detail":"resuming: rewrite src/L.ts parser, routed L"}\n' > "$sz/archive/e/ops.jsonl"
|
|
3677
|
+
printf '### Task Group 1: g\n' > "$sz/archive/e/spec.md"
|
|
3678
|
+
out=$(bash "$SC" "$tmp/sizing" 2>&1)
|
|
3679
|
+
grep -q "UNDERSIZED a: sized M, capsule has 6 task groups" <<<"$out" \
|
|
3680
|
+
&& check "sizing: M with 6 groups reported under-sized" 0 $? \
|
|
3681
|
+
|| { echo "FAIL sizing: under-sized M not flagged"; echo "$out"; fail=1; }
|
|
3682
|
+
grep -q "OVERSIZED b: sized L, capsule has 1 task group" <<<"$out" \
|
|
3683
|
+
&& check "sizing: L with 1 group reported over-sized" 0 $? \
|
|
3684
|
+
|| { echo "FAIL sizing: over-sized L not flagged"; fail=1; }
|
|
3685
|
+
grep -q "UNDERSIZED c\|OVERSIZED c" <<<"$out" \
|
|
3686
|
+
&& { echo "FAIL sizing: resume detail without size letter was scored"; fail=1; } || true
|
|
3687
|
+
grep -q "UNDERSIZED d\|OVERSIZED d" <<<"$out" \
|
|
3688
|
+
&& { echo "FAIL sizing: a detail naming a file was read as a size call"; echo "$out"; fail=1; } || true
|
|
3689
|
+
grep -q "OVERSIZED e: sized L, capsule has 1 task group" <<<"$out" \
|
|
3690
|
+
&& check "sizing: a size call after a file name is still read" 0 $? \
|
|
3691
|
+
|| { echo "FAIL sizing: size call after a file name was lost"; echo "$out"; fail=1; }
|
|
3692
|
+
grep -q "calibrated: 0 ok, 1 under-sized M, 2 over-sized L" <<<"$out" \
|
|
3693
|
+
&& check "sizing: summary counts mismatches and skips resume details" 0 $? \
|
|
3694
|
+
|| { echo "FAIL sizing: wrong summary counts"; echo "$out"; fail=1; }
|
|
3695
|
+
out=$(bash "$SC" "$tmp/no-such-project" 2>&1); rc=$?
|
|
3696
|
+
[[ $rc -eq 0 && -n "$out" ]] || { echo "FAIL sizing: missing specs dir must advise, got rc=$rc"; fail=1; }
|
|
3697
|
+
|
|
3698
|
+
# --- Scenario <-> criteria consistency -------------------------------------
|
|
3699
|
+
# Every scenario must have a criteria file and vice versa. A scenario without
|
|
3700
|
+
# criteria can never be scored; criteria without a scenario is dead config.
|
|
3701
|
+
missing=0
|
|
3702
|
+
for s in "$EVALS"/scenarios/*.md; do
|
|
3703
|
+
name=$(basename "$s" .md)
|
|
3704
|
+
[[ -f "$EVALS/criteria/${name}.json" ]] || { echo "FAIL scenario $name: no criteria.json" >&2; missing=1; }
|
|
3705
|
+
done
|
|
3706
|
+
for c in "$EVALS"/criteria/*.json; do
|
|
3707
|
+
name=$(basename "$c" .json)
|
|
3708
|
+
[[ "$name" == "_selftest-needs" || "$name" == "_selftest-flip" || "$name" == "_selftest-hold" ]] && continue
|
|
3709
|
+
[[ -f "$EVALS/scenarios/${name}.md" ]] || { echo "FAIL criteria $name: no scenario.md" >&2; missing=1; }
|
|
3710
|
+
done
|
|
3711
|
+
check "every scenario has criteria and every criteria has a scenario" 0 "$missing"
|
|
3712
|
+
|
|
3713
|
+
# --- Concurrent template-cache invalidation ---------------------------------
|
|
3714
|
+
# no-op-gate and oracle-gate start together under `just verify`. If both see
|
|
3715
|
+
# a stale fingerprint before either takes the lock, the second process must
|
|
3716
|
+
# re-check after acquisition. Otherwise it deletes templates the first process
|
|
3717
|
+
# is already building. Hold the lock, advance the fingerprint, then call ensure.
|
|
3718
|
+
(
|
|
3719
|
+
export EVAL_DIR="$EVALS"
|
|
3720
|
+
TEMPLATES="$tmp/template-cache"
|
|
3721
|
+
source "$EVALS/build-templates.sh" || exit 1
|
|
3722
|
+
mkdir -p "$TEMPLATES"
|
|
3723
|
+
printf 'stale\n' >"$TEMPLATES/.fingerprint"
|
|
3724
|
+
mkdir "$TEMPLATES.lock"
|
|
3725
|
+
printf 'keep\n' >"$TEMPLATES/sentinel"
|
|
3726
|
+
want=$(templates_fingerprint)
|
|
3727
|
+
(
|
|
3728
|
+
sleep 0.2
|
|
3729
|
+
printf '%s\n' "$want" >"$TEMPLATES/.fingerprint"
|
|
3730
|
+
rmdir "$TEMPLATES.lock"
|
|
3731
|
+
) &
|
|
3732
|
+
ensure_templates_fresh
|
|
3733
|
+
wait
|
|
3734
|
+
test -f "$TEMPLATES/sentinel"
|
|
3735
|
+
)
|
|
3736
|
+
check "concurrent cache invalidation preserves templates built by the lock holder" 0 "$?"
|
|
3737
|
+
|
|
3738
|
+
# --- Every scenario has a template build case --------------------------------
|
|
3739
|
+
# A scenario with no build_template case in run.sh silently runs against an
|
|
3740
|
+
# empty repo, producing garbage scores. EVAL_TEMPLATES_ONLY=1 reports EMPTY,
|
|
3741
|
+
# but `just evals` doesn't set it. Catch the gap in `just verify` by dry-running
|
|
3742
|
+
# the same template build and flagging any EMPTY result.
|
|
3743
|
+
tmpl_out=$(EVAL_TEMPLATES_ONLY=1 bash "$EVALS/run.sh" 2>&1)
|
|
3744
|
+
empty_count=$(grep -c 'EMPTY' <<<"$tmpl_out" || true)
|
|
3745
|
+
check "no scenario has an empty template build" 0 "$empty_count"
|
|
3746
|
+
|
|
3747
|
+
# --- Criteria must discriminate (at least one non-transcript criterion fails) ---
|
|
3748
|
+
# A criterion that passes against the pre-fix template is dead weight: it adds
|
|
3749
|
+
# to the total but can never catch a regression. For each scenario, at least
|
|
3750
|
+
# one filesystem-checkable criterion MUST fail against the untouched template.
|
|
3751
|
+
# Pure-analysis scenarios (all criteria need a transcript) are validated by the
|
|
3752
|
+
# transcript-dependency check instead — they need at least one non-transcript
|
|
3753
|
+
# pre-condition criterion to be scored at all.
|
|
3754
|
+
disc_fail=0
|
|
3755
|
+
for s in "$EVALS"/scenarios/*.md; do
|
|
3756
|
+
name=$(basename "$s" .md)
|
|
3757
|
+
[[ "$name" == "_selftest-needs" || "$name" == "_selftest-flip" || "$name" == "_selftest-hold" ]] && continue
|
|
3758
|
+
crit="$EVALS/criteria/${name}.json"
|
|
3759
|
+
[[ -f "$crit" ]] || continue
|
|
3760
|
+
tmpl="$EVALS/templates/$name"
|
|
3761
|
+
if [[ ! -d "$tmpl" ]]; then
|
|
3762
|
+
( cd "$EVALS" && EVAL_TEMPLATES_ONLY=1 bash run.sh >/dev/null 2>&1 )
|
|
3763
|
+
fi
|
|
3764
|
+
[[ -d "$tmpl" ]] || continue
|
|
3765
|
+
# Count filesystem vs transcript criteria.
|
|
3766
|
+
total=$(python3 -c "import json;print(len(json.load(open('$crit'))))")
|
|
3767
|
+
total_fs=$(python3 -c "import json;print(sum(1 for x in json.load(open('$crit')) if not x.get('needs')))")
|
|
3768
|
+
total_tx=$((total - total_fs))
|
|
3769
|
+
# Pure-analysis (all need transcript): validated by transcript-dependency check.
|
|
3770
|
+
[[ "$total_fs" -eq 0 ]] && continue
|
|
3771
|
+
# If there are transcript criteria AND filesystem criteria are pre-conditions
|
|
3772
|
+
# (pass against template), that's OK — the transcript criteria do the real testing.
|
|
3773
|
+
# Only flag if there are NO transcript criteria AND all filesystem criteria pass.
|
|
3774
|
+
if [[ "$total_tx" -gt 0 ]]; then
|
|
3775
|
+
# Has transcript criteria: filesystem criteria are pre-conditions. Skip discrimination check.
|
|
3776
|
+
continue
|
|
3777
|
+
fi
|
|
3778
|
+
# Run each filesystem criterion against the template and count failures.
|
|
3779
|
+
# Each check is designed to run with cwd=PROJ, so cd there first.
|
|
3780
|
+
# The Python script prints the count of failing criteria to stdout.
|
|
3781
|
+
fails=$(python3 - "$crit" "$tmpl" "$EVAL_DIR" <<'PYEOF'
|
|
3782
|
+
import json, os, subprocess, sys
|
|
3783
|
+
crit, proj, eval_dir = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
3784
|
+
with open(crit) as f:
|
|
3785
|
+
criteria = json.load(f)
|
|
3786
|
+
fs = [c for c in criteria if not c.get('needs')]
|
|
3787
|
+
env = {**os.environ, 'PROJ': proj, 'EVAL_DIR': eval_dir, 'TRANSCRIPT': proj + '/transcript.jsonl'}
|
|
3788
|
+
fails = 0
|
|
3789
|
+
for c in fs:
|
|
3790
|
+
try:
|
|
3791
|
+
cmd = 'cd "$PROJ" && ' + c['check'] if not c['check'].strip().startswith('cd ') and 'cd "$PROJ"' not in c['check'] else c['check']
|
|
3792
|
+
r = subprocess.run(['bash', '-c', cmd], capture_output=True, text=True, timeout=10, env=env)
|
|
3793
|
+
if r.returncode != 0:
|
|
3794
|
+
fails += 1
|
|
3795
|
+
except Exception:
|
|
3796
|
+
fails += 1
|
|
3797
|
+
print(fails)
|
|
3798
|
+
PYEOF
|
|
3799
|
+
)
|
|
3800
|
+
# fails is the count of failing criteria (captured from stdout).
|
|
3801
|
+
if [[ "$fails" -eq 0 ]]; then
|
|
3802
|
+
echo "FAIL $name: all $total_fs filesystem criteria pass against pre-fix template (none discriminate)" >&2
|
|
3803
|
+
disc_fail=1
|
|
3804
|
+
fi
|
|
3805
|
+
done
|
|
3806
|
+
check "every scenario has at least one discriminating filesystem criterion" 0 "$disc_fail"
|
|
3807
|
+
|
|
3808
|
+
# --- No scenario is entirely transcript-dependent --------------------------------
|
|
3809
|
+
# If every criterion in a scenario needs a transcript, the scenario can never be
|
|
3810
|
+
# scored without one — it's a manual-read item in disguise, not an eval. Flag it.
|
|
3811
|
+
all_transcript=0
|
|
3812
|
+
for c in "$EVALS"/criteria/*.json; do
|
|
3813
|
+
name=$(basename "$c" .json)
|
|
3814
|
+
[[ "$name" == "_selftest-needs" || "$name" == "_selftest-flip" || "$name" == "_selftest-hold" ]] && continue
|
|
3815
|
+
total=$(python3 -c "import json;print(len(json.load(open('$c'))))")
|
|
3816
|
+
needs_tx=$(python3 -c "import json;print(sum(1 for x in json.load(open('$c')) if x.get('needs')))")
|
|
3817
|
+
if [[ "$total" -gt 0 && "$needs_tx" -eq "$total" ]]; then
|
|
3818
|
+
echo "FAIL $name: all $total criteria need a transcript (scores 0 without one)" >&2
|
|
3819
|
+
all_transcript=1
|
|
3820
|
+
fi
|
|
3821
|
+
done
|
|
3822
|
+
check "no scenario is entirely transcript-dependent" 0 "$all_transcript"
|
|
3823
|
+
|
|
3824
|
+
# --- gate-check.mjs ---------------------------------------------------------
|
|
3825
|
+
GC="$(dirname "$DOD")/gate-check.mjs"
|
|
3826
|
+
g="$tmp/gates.md"
|
|
3827
|
+
cat > "$g" <<'EOF'
|
|
3828
|
+
# Gates: demo
|
|
3829
|
+
|
|
3830
|
+
- [ ] G1: passes via exit code
|
|
3831
|
+
CHECK: echo ok
|
|
3832
|
+
EVIDENCE: pending
|
|
3833
|
+
|
|
3834
|
+
- [ ] G2: passes via EXPECT regex
|
|
3835
|
+
CHECK: echo "count: 42"
|
|
3836
|
+
EXPECT: /count: 42/
|
|
3837
|
+
EVIDENCE: pending
|
|
3838
|
+
|
|
3839
|
+
- [ ] G3: failing check stays unchecked
|
|
3840
|
+
CHECK: false
|
|
3841
|
+
EVIDENCE: pending
|
|
3842
|
+
|
|
3843
|
+
- [x] G4: checked box with pending evidence counts UNMET
|
|
3844
|
+
EVIDENCE: pending
|
|
3845
|
+
|
|
3846
|
+
- [ ] G5: honestly abandoned
|
|
3847
|
+
EVIDENCE: pending
|
|
3848
|
+
ABANDON: G5 needs prod access this machine lacks
|
|
3849
|
+
EOF
|
|
3850
|
+
node "$GC" "$g" >/dev/null; check "gate-check: unmet gates exit 1" 1 $?
|
|
3851
|
+
grep -q '^- \[x\] G1' "$g"; check "gate-check: passing gate flipped" 0 $?
|
|
3852
|
+
grep -q '^- \[ \] G3' "$g"; check "gate-check: failing gate stays unchecked" 0 $?
|
|
3853
|
+
grep -qE '^ EVIDENCE: ok' "$g"; check "gate-check: evidence recorded from output" 0 $?
|
|
3854
|
+
grep -qE '^ EVIDENCE: count: 42' "$g"; check "gate-check: regex EXPECT recorded evidence" 0 $?
|
|
3855
|
+
node "$GC" --status "$g" >/dev/null; check "gate-check: --status reports remaining unmet as exit 1" 1 $?
|
|
3856
|
+
|
|
3857
|
+
# A gate file may omit the EVIDENCE placeholder. The pass path used to flip the
|
|
3858
|
+
# box and report ALL MET while writing nothing, so the next --status run on the
|
|
3859
|
+
# same bytes read the checked box as an unproven claim and exited 1: one run's
|
|
3860
|
+
# result contradicted the next on an unchanged file.
|
|
3861
|
+
gnoev="$tmp/gates-no-evidence.md"
|
|
3862
|
+
printf -- '- [ ] G1: no evidence line\n CHECK: echo ok\n EXPECT: ok\n- [ ] G2: second gate\n CHECK: echo world\n EXPECT: world\n' > "$gnoev"
|
|
3863
|
+
node "$GC" "$gnoev" >/dev/null; check "gate-check: gates without an EVIDENCE line can pass" 0 $?
|
|
3864
|
+
grep -qE '^ EVIDENCE: ok$' "$gnoev"; check "gate-check: a missing EVIDENCE line is written, not skipped" 0 $?
|
|
3865
|
+
grep -qE '^ EVIDENCE: world$' "$gnoev"; check "gate-check: each gate keeps its own evidence after a line insert" 0 $?
|
|
3866
|
+
[[ "$(grep -cE '^ EVIDENCE: ' "$gnoev")" == 2 ]]; check "gate-check: exactly one evidence line per gate" 0 $?
|
|
3867
|
+
node "$GC" --status "$gnoev" >/dev/null; check "gate-check: the recorded evidence satisfies a later --status run" 0 $?
|
|
3868
|
+
|
|
3869
|
+
gok="$tmp/gates-ok.md"
|
|
3870
|
+
printf -- '- [x] G1: done\n EVIDENCE: 3 rows rendered\n' > "$gok"
|
|
3871
|
+
node "$GC" "$gok" >/dev/null; check "gate-check: all met exits 0" 0 $?
|
|
3872
|
+
|
|
3873
|
+
gdupe="$tmp/gates-duplicate.md"
|
|
3874
|
+
cat > "$gdupe" <<'EOF'
|
|
3875
|
+
- [ ] G1: first outcome
|
|
3876
|
+
CHECK: false
|
|
3877
|
+
EVIDENCE: pending
|
|
3878
|
+
- [ ] G1: second outcome
|
|
3879
|
+
CHECK: false
|
|
3880
|
+
EVIDENCE: pending
|
|
3881
|
+
ABANDON: G1 only one outcome was abandoned
|
|
3882
|
+
EOF
|
|
3883
|
+
out=$(node "$GC" --status "$gdupe" 2>&1); rc=$?
|
|
3884
|
+
check "gate-check: duplicate identifiers are a parse error" 2 $rc
|
|
3885
|
+
printf '%s\n' "$out" | grep -Fq 'duplicate gate identifier G1'; check "gate-check: duplicate diagnostic names the identifier" 0 $?
|
|
3886
|
+
|
|
3887
|
+
# The gates.md format is enforced by gate-check.mjs alone. check-dod-core.sh
|
|
3888
|
+
# deliberately does not parse it, so there is no DoD
|
|
3889
|
+
# blocking case to test here until a real spec uses gates and earns enforcement.
|
|
3890
|
+
|
|
3891
|
+
# --- evals/run.sh isolation and timeout -------------------------------------
|
|
3892
|
+
# A machine-local pi extension can fail before an eval starts. Both DevOS and
|
|
3893
|
+
# baseline runs must disable those extensions. A child that ignores TERM must
|
|
3894
|
+
# also not leave the eval harness waiting forever.
|
|
3895
|
+
run_sh=$(cat "$EVALS/run.sh")
|
|
3896
|
+
if grep -q 'local ext_args=(-ne)' <<<"$run_sh" && grep -q 'pi -p "$prompt" ${PROVIDER:+--provider "$PROVIDER"}' <<<"$run_sh"; then isolation_ok=0; else isolation_ok=1; fi
|
|
3897
|
+
check "eval pi invocations disable machine-local extensions" 0 "$isolation_ok"
|
|
3898
|
+
|
|
3899
|
+
timeout_fn=$(awk '/^run_with_timeout\(\)/,/^}/' "$EVALS/run.sh")
|
|
3900
|
+
timeout_rc=0
|
|
3901
|
+
timeout_start=$(date +%s)
|
|
3902
|
+
(eval "$timeout_fn"; run_with_timeout 1 bash -c 'trap "" TERM; sleep 5') >/dev/null 2>&1 || timeout_rc=$?
|
|
3903
|
+
timeout_elapsed=$(( $(date +%s) - timeout_start ))
|
|
3904
|
+
if [[ "$timeout_rc" -ne 0 && "$timeout_elapsed" -le 3 ]]; then timeout_ok=0; else timeout_ok=1; fi
|
|
3905
|
+
check "run timeout kills a TERM-ignoring child" 0 "$timeout_ok"
|
|
3906
|
+
(eval "$timeout_fn"; run_with_timeout 5 bash -c 'exit 0') >/dev/null 2>&1
|
|
3907
|
+
check "run timeout preserves a successful child" 0 "$?"
|
|
3908
|
+
|
|
3909
|
+
# --- build/scripts/ledger.sh (H route) --------------------------------------
|
|
3910
|
+
# The H route's two evidence-backed decisions live in the script, so they get a
|
|
3911
|
+
# real test instead of a prose promise: `next` must flag a reissued task, and
|
|
3912
|
+
# `notes` must replace and cap instead of growing. Every action runs against a
|
|
3913
|
+
# throwaway run dir, the same way the route runs it.
|
|
3914
|
+
LEDGER="$REPO_ROOT/plugins/devos/skills/build/scripts/ledger.sh"
|
|
3915
|
+
|
|
3916
|
+
lr="$tmp/hard-run"
|
|
3917
|
+
bash "$LEDGER" init --dir "$lr" --task "make the widget pass" --check "true" >/dev/null
|
|
3918
|
+
check "ledger: init creates the run" 0 $?
|
|
3919
|
+
grep -q 'make the widget pass' "$lr/task.md"; check "ledger: init writes task.md" 0 $?
|
|
3920
|
+
|
|
3921
|
+
lpa="$tmp/ledger-path-artifact"
|
|
3922
|
+
out=$(bash "$LEDGER" init --dir "$lpa" --task "make a path artifact" --artifact ../escaped.txt 2>&1); rc=$?
|
|
3923
|
+
check "ledger: rejects a path artifact" 2 "$rc"
|
|
3924
|
+
grep -Fq 'artifact must be a file name' <<<"$out"; check "ledger: names the artifact path contract" 0 $?
|
|
3925
|
+
[[ ! -e "$tmp/escaped.txt" ]]; check "ledger: does not create an escaped artifact" 0 $?
|
|
3926
|
+
|
|
3927
|
+
out=$(bash "$LEDGER" read --dir "$lr")
|
|
3928
|
+
printf '%s' "$out" | grep -q 'round: 0/10'; check "ledger: read shows the round budget" 0 $?
|
|
3929
|
+
printf '%s' "$out" | grep -q 'artifact: solution.py (empty)'; check "ledger: read reports an empty artifact" 0 $?
|
|
3930
|
+
|
|
3931
|
+
# Plan is capped, not trusted whole.
|
|
3932
|
+
out=$(head -c 5000 /dev/zero | tr '\0' 'p' | bash "$LEDGER" plan --dir "$lr")
|
|
3933
|
+
[ "$(wc -c < "$lr/plan.md" | tr -d ' ')" -le 4000 ]; check "ledger: plan capped at 4000" 0 $?
|
|
3934
|
+
# Capping without saying so reads as a plan that landed whole, and the worker
|
|
3935
|
+
# is handed the cut half.
|
|
3936
|
+
printf '%s' "$out" | grep -q 'TRUNCATED'; check "ledger: an over-cap plan says it was cut" 0 $?
|
|
3937
|
+
printf 'short plan\n' | bash "$LEDGER" plan --dir "$lr" | grep -q 'TRUNCATED'
|
|
3938
|
+
[[ $? -ne 0 ]]; check "ledger: a plan that fits is not reported as cut" 0 $?
|
|
3939
|
+
|
|
3940
|
+
printf -- '- [todo] first\n- [todo] second\n' | bash "$LEDGER" tasks --dir "$lr" >/dev/null
|
|
3941
|
+
[ "$(grep -c '^- \[' "$lr/tasks.md" || true)" = "2" ]; check "ledger: tasks keeps only list lines" 0 $?
|
|
3942
|
+
|
|
3943
|
+
# The guard: the same task twice is the no-progress signal the paper relies on.
|
|
3944
|
+
bash "$LEDGER" next --dir "$lr" --task "patch the parser" >/dev/null
|
|
3945
|
+
out=$(bash "$LEDGER" next --dir "$lr" --task "Patch the parser")
|
|
3946
|
+
printf '%s' "$out" | grep -q 'GUARD:'; check "ledger: reissued task raises the guard" 0 $?
|
|
3947
|
+
printf '%s' "$out" | grep -q 'round 2/10'; check "ledger: next increments the round" 0 $?
|
|
3948
|
+
|
|
3949
|
+
# The round budget is a guard, not display text. Exhaustion must stop before
|
|
3950
|
+
# the task or round mutates, or the manager can run forever past its cap.
|
|
3951
|
+
lbr="$tmp/hard-run-budget"
|
|
3952
|
+
bash "$LEDGER" init --dir "$lbr" --task "initial task" --check "true" --max-iters 1 >/dev/null
|
|
3953
|
+
bash "$LEDGER" next --dir "$lbr" --task "allowed task" >/dev/null
|
|
3954
|
+
out=$(bash "$LEDGER" next --dir "$lbr" --task "over-budget task" 2>&1); rc=$?
|
|
3955
|
+
check "ledger: exhausted round budget refuses another task" 1 "$rc"
|
|
3956
|
+
printf '%s' "$out" | grep -q 'STOP: iteration budget exhausted (1/1)'; check "ledger: exhausted budget names the limit" 0 $?
|
|
3957
|
+
[[ "$(awk -F= '$1=="round" {print $2}' "$lbr/meta")" == "1" ]]; check "ledger: refused task does not increment the round" 0 $?
|
|
3958
|
+
[[ "$(awk -F= '$1=="last_task" {sub(/^[^=]*=/, ""); print}' "$lbr/meta")" == "allowed task" ]]; check "ledger: refused task does not replace the last task" 0 $?
|
|
3959
|
+
|
|
3960
|
+
# Budget validation must happen before init creates any run state. A valid
|
|
3961
|
+
# positive integer remains a usable control for the same command.
|
|
3962
|
+
for invalid_budget in nope 0 -1; do
|
|
3963
|
+
invalid_dir="$tmp/invalid-budget-$invalid_budget"
|
|
3964
|
+
out=$(bash "$LEDGER" init --dir "$invalid_dir" --task "invalid budget" --check true --max-iters "$invalid_budget" 2>&1); rc=$?
|
|
3965
|
+
check "ledger: rejects invalid budget $invalid_budget" 2 "$rc"
|
|
3966
|
+
grep -Fq 'max-iters must be a positive integer' <<<"$out"; check "ledger: names invalid budget $invalid_budget" 0 $?
|
|
3967
|
+
[[ ! -e "$invalid_dir" ]]; check "ledger: does not create state for invalid budget $invalid_budget" 0 $?
|
|
3968
|
+
done
|
|
3969
|
+
valid_budget_dir="$tmp/valid-budget"
|
|
3970
|
+
bash "$LEDGER" init --dir "$valid_budget_dir" --task "valid budget" --check true --max-iters 2 >/dev/null
|
|
3971
|
+
check "ledger: accepts a positive budget" 0 $?
|
|
3972
|
+
grep -q '^max_iters=2$' "$valid_budget_dir/meta"; check "ledger: stores the validated positive budget" 0 $?
|
|
3973
|
+
|
|
3974
|
+
# Notes replace by default and cap; --append adds without unbounded growth.
|
|
3975
|
+
printf 'first finding\n' | bash "$LEDGER" notes --dir "$lr" >/dev/null
|
|
3976
|
+
printf 'second finding\n' | bash "$LEDGER" notes --dir "$lr" >/dev/null
|
|
3977
|
+
[ "$(cat "$lr/notes.md")" = "second finding" ]; check "ledger: notes replace, old content gone" 0 $?
|
|
3978
|
+
head -c 9000 /dev/zero | tr '\0' 'n' | bash "$LEDGER" notes --dir "$lr" --append >/dev/null
|
|
3979
|
+
[ "$(wc -c < "$lr/notes.md" | tr -d ' ')" -le 8000 ]; check "ledger: notes capped at 8000" 0 $?
|
|
3980
|
+
out=$(head -c 9000 /dev/zero | tr '\0' 'n' | bash "$LEDGER" notes --dir "$lr")
|
|
3981
|
+
printf '%s' "$out" | grep -q 'TRUNCATED'; check "ledger: over-cap notes say they were cut" 0 $?
|
|
3982
|
+
|
|
3983
|
+
# Artifact write and read.
|
|
3984
|
+
printf 'print("ok")\n' | bash "$LEDGER" artifact --dir "$lr" >/dev/null
|
|
3985
|
+
bash "$LEDGER" artifact --dir "$lr" --read | grep -q 'print("ok")'; check "ledger: artifact round-trips" 0 $?
|
|
3986
|
+
|
|
3987
|
+
# Check is ground truth: exit code and recorded verdict.
|
|
3988
|
+
bash "$LEDGER" check --dir "$lr" --cmd "grep -q ok solution.py" >/dev/null; check "ledger: check passes on the artifact" 0 $?
|
|
3989
|
+
grep -q '^verdict=pass' "$lr/meta"; check "ledger: a pass is recorded" 0 $?
|
|
3990
|
+
bash "$LEDGER" check --dir "$lr" --cmd "grep -q missing solution.py" >/dev/null 2>&1; check "ledger: a failing check exits non-zero" 1 $?
|
|
3991
|
+
grep -q '^verdict=fail' "$lr/meta"; check "ledger: a fail overrides the recorded pass" 0 $?
|
|
3992
|
+
|
|
3993
|
+
# Refuses to clobber an initialized run, and errors on an uninitialized dir.
|
|
3994
|
+
bash "$LEDGER" init --dir "$lr" --task x >/dev/null 2>&1; check "ledger: init will not clobber a run" 2 $?
|
|
3995
|
+
bash "$LEDGER" read --dir "$tmp/no-such-run" >/dev/null 2>&1; check "ledger: read on an uninitialized dir errors" 2 $?
|
|
3996
|
+
|
|
3997
|
+
# --- design-taste/scripts/slop-scan.py: what a skipped file is ----------------
|
|
3998
|
+
# The skipped tally says binary/generated. A file was skipped when grep found no
|
|
3999
|
+
# line of text in it, which is also true of an empty or blank file, so an empty
|
|
4000
|
+
# .css was described as binary and a project whose only frontend file is empty
|
|
4001
|
+
# was told it had no frontend files. The rules cannot fire on a file with no
|
|
4002
|
+
# content, but the file is still an authored frontend file and belongs in the
|
|
4003
|
+
# count.
|
|
4004
|
+
SLOP="$REPO_ROOT/plugins/devos/skills/design-taste/scripts/slop-scan.py"
|
|
4005
|
+
slopd="$tmp/slop-empty"
|
|
4006
|
+
mkdir -p "$slopd"
|
|
4007
|
+
: > "$slopd/app.css"
|
|
4008
|
+
out=$(python3 "$SLOP" "$slopd")
|
|
4009
|
+
! printf '%s' "$out" | grep -q 'no frontend files'; check "slop: an empty-only tree is not called frontend-free" 0 $?
|
|
4010
|
+
printf '%s' "$out" | grep -q 'across 1 files'; check "slop: an empty frontend file is still counted" 0 $?
|
|
4011
|
+
|
|
4012
|
+
printf 'body { color: #16110d; }\n' > "$slopd/real.css"
|
|
4013
|
+
printf 'GITCRYPT\000\314"\332\376' > "$slopd/locked.css"
|
|
4014
|
+
: > "$slopd/blank.css"
|
|
4015
|
+
out=$(python3 "$SLOP" "$slopd")
|
|
4016
|
+
printf '%s' "$out" | grep -q 'across 3 files (1 binary/generated skipped)'; check "slop: only the binary file is skipped" 0 $?
|
|
4017
|
+
|
|
4018
|
+
# A blank file, not just a zero-byte one, is the same case.
|
|
4019
|
+
printf '\n\n' > "$slopd/blank.css"
|
|
4020
|
+
out=$(python3 "$SLOP" "$slopd")
|
|
4021
|
+
printf '%s' "$out" | grep -q 'across 3 files (1 binary/generated skipped)'; check "slop: a whitespace-only file is not skipped" 0 $?
|
|
4022
|
+
|
|
4023
|
+
# The skip path itself still works, so keeping every file cannot pass the
|
|
4024
|
+
# checks above.
|
|
4025
|
+
slopb="$tmp/slop-binary"
|
|
4026
|
+
mkdir -p "$slopb"
|
|
4027
|
+
printf 'GITCRYPT\000\314"\332\376' > "$slopb/locked.css"
|
|
4028
|
+
out=$(python3 "$SLOP" "$slopb")
|
|
4029
|
+
printf '%s' "$out" | grep -q 'no frontend files'; check "slop: a binary-only tree has nothing to scan" 0 $?
|
|
4030
|
+
python3 "$SLOP" --selftest >/dev/null; check "slop: selftest passes" 0 $?
|
|
4031
|
+
|
|
4032
|
+
# --- init/scripts/reverse-engineer.sh: a non-repo is not an empty history ---
|
|
4033
|
+
# The git log ran with stderr sent to /dev/null, so a directory that is not a
|
|
4034
|
+
# repository produced no output, fell into the same empty branch as a repo
|
|
4035
|
+
# whose last 6 months hold nothing, and printed "no clusters found in the last
|
|
4036
|
+
# 6 months". The reference tells the operator a greenfield repo is the case to
|
|
4037
|
+
# skip, so a directory with no history to read was green-lit on a claim about
|
|
4038
|
+
# history that was never read.
|
|
4039
|
+
REVENG="$REPO_ROOT/plugins/devos/skills/init/scripts/reverse-engineer.sh"
|
|
4040
|
+
reveng_notgit="$tmp/reveng-not-git"
|
|
4041
|
+
mkdir -p "$reveng_notgit"
|
|
4042
|
+
out=$(cd "$reveng_notgit" && bash "$REVENG" 2>&1); rc=$?
|
|
4043
|
+
[[ $rc -ne 0 ]] || { echo "FAIL reveng: a non-repo must not exit 0"; fail=1; }
|
|
4044
|
+
[[ "$out" == *"not a git repository"* ]] || { echo "FAIL reveng: a non-repo must name the cause: $out"; fail=1; }
|
|
4045
|
+
[[ "$out" != *"no clusters found"* ]] || { echo "FAIL reveng: a non-repo must not claim an empty history: $out"; fail=1; }
|
|
4046
|
+
|
|
4047
|
+
# Control: a real repository with no commits is the genuine zero-cluster case
|
|
4048
|
+
# and keeps the documented line and exit 0. Without it, "always exit non-zero"
|
|
4049
|
+
# would pass the checks above.
|
|
4050
|
+
reveng_empty="$tmp/reveng-empty"
|
|
4051
|
+
mkdir -p "$reveng_empty"
|
|
4052
|
+
git -C "$reveng_empty" init -q -b main
|
|
4053
|
+
out=$(cd "$reveng_empty" && bash "$REVENG" 2>&1); rc=$?
|
|
4054
|
+
check "reveng: an empty repository still reports no clusters" 0 "$rc"
|
|
4055
|
+
[[ "$out" == *"no clusters found in the last 6 months"* ]] || { echo "FAIL reveng: an empty repo lost its line: $out"; fail=1; }
|
|
4056
|
+
|
|
4057
|
+
# A missing directory is still its own error, not the repository one.
|
|
4058
|
+
out=$(cd "$reveng_empty" && bash "$REVENG" "$tmp/reveng-absent" 2>&1); rc=$?
|
|
4059
|
+
[[ $rc -ne 0 ]] || { echo "FAIL reveng: a missing directory must exit non-zero"; fail=1; }
|
|
4060
|
+
[[ "$out" == *"no such directory"* ]] || { echo "FAIL reveng: a missing directory must name itself: $out"; fail=1; }
|
|
4061
|
+
|
|
4062
|
+
# Control: a real repo with one cluster commit still writes its stub, so the
|
|
4063
|
+
# repository check cannot pass by refusing all work.
|
|
4064
|
+
reveng_real="$tmp/reveng-real"
|
|
4065
|
+
mkdir -p "$reveng_real/plugins/demo/skills/thing"
|
|
4066
|
+
git -C "$reveng_real" init -q -b main
|
|
4067
|
+
git -C "$reveng_real" config user.email t@t && git -C "$reveng_real" config user.name t
|
|
4068
|
+
echo x > "$reveng_real/plugins/demo/skills/thing/run.sh"
|
|
4069
|
+
git -C "$reveng_real" add -A && git -C "$reveng_real" commit -qm 'feat: thing'
|
|
4070
|
+
out=$(cd "$reveng_real" && bash "$REVENG" 2>&1); rc=$?
|
|
4071
|
+
check "reveng: a repository with history writes stubs" 0 "$rc"
|
|
4072
|
+
ls "$reveng_real/devos/specs/archive/"*-plugins-demo-skills-thing.spec.md >/dev/null 2>&1; check "reveng: the cluster stub lands on disk" 0 $?
|
|
4073
|
+
|
|
4074
|
+
# --------------------------------------------------------------------------
|
|
4075
|
+
# The smallest spec folder an M change is told to write, end to end.
|
|
4076
|
+
#
|
|
4077
|
+
# The router sends an M change through the same spec folder and the same plan
|
|
4078
|
+
# capsule as an L feature, so what a small change costs is set by what the
|
|
4079
|
+
# templates demand. write-spec.md names a one-page shape for M and a 300-line
|
|
4080
|
+
# ceiling for the capsule, and the claim that the short one works is only true
|
|
4081
|
+
# if every gate accepts it. It does not stop at validate-spec: the capsule is
|
|
4082
|
+
# archived and a verdict claimed, because a spec that validates but cannot
|
|
4083
|
+
# close the pipeline is not a shape the framework supports.
|
|
4084
|
+
mspec=$(mktemp -d)
|
|
4085
|
+
mspec_name=2026-01-01-invoice-pdf
|
|
4086
|
+
mkdir -p "$mspec/devos/specs/$mspec_name/verifications"
|
|
4087
|
+
cat > "$mspec/devos/specs/$mspec_name/spec-functional.md" <<'EOF'
|
|
4088
|
+
# Export invoices
|
|
4089
|
+
|
|
4090
|
+
## 1. Goal
|
|
4091
|
+
|
|
4092
|
+
A user can download a PDF of their invoices without asking support.
|
|
4093
|
+
|
|
4094
|
+
## 3. Functionality
|
|
4095
|
+
|
|
4096
|
+
### 3.1 Export
|
|
4097
|
+
**Scenarios:**
|
|
4098
|
+
- Scenario SC-1: plain export / WHEN an account has 3 invoices / THEN a PDF with all 3 is downloaded
|
|
4099
|
+
- Scenario SC-2: empty account / WHEN an account has no invoices / THEN the export is refused with one line naming why
|
|
4100
|
+
|
|
4101
|
+
## 5. Out of Scope
|
|
4102
|
+
|
|
4103
|
+
- Scheduled or recurring exports
|
|
4104
|
+
- Choosing a paper size or locale
|
|
4105
|
+
|
|
4106
|
+
## Appendix
|
|
4107
|
+
|
|
4108
|
+
**Explicit contract anchors:** `GET /invoices/:id/pdf`
|
|
4109
|
+
EOF
|
|
4110
|
+
cat > "$mspec/devos/specs/$mspec_name/spec.md" <<'EOF'
|
|
4111
|
+
---
|
|
4112
|
+
devos-contract: 1
|
|
4113
|
+
---
|
|
4114
|
+
# Export invoices
|
|
4115
|
+
|
|
4116
|
+
## Goal
|
|
4117
|
+
|
|
4118
|
+
A user downloads a PDF of their invoices.
|
|
4119
|
+
|
|
4120
|
+
## Local Craft
|
|
4121
|
+
|
|
4122
|
+
None
|
|
4123
|
+
|
|
4124
|
+
## Design Convergence
|
|
4125
|
+
- **Direct MVP**: the existing invoice list route renders a print view
|
|
4126
|
+
- **Failed Items**: None
|
|
4127
|
+
- **Adopted Additions**: None
|
|
4128
|
+
- **Rejected Additions**: None
|
|
4129
|
+
|
|
4130
|
+
## Boundaries
|
|
4131
|
+
- **Always** (in scope): the export route and its test
|
|
4132
|
+
- **Ask First** (none)
|
|
4133
|
+
- **Never** (out of scope): scheduled exports
|
|
4134
|
+
|
|
4135
|
+
## Code Map
|
|
4136
|
+
|
|
4137
|
+
- `src/invoices/pdf.ts`: the renderer
|
|
4138
|
+
|
|
4139
|
+
## Implementation Plan
|
|
4140
|
+
|
|
4141
|
+
### Overview
|
|
4142
|
+
Total: 1 task group.
|
|
4143
|
+
|
|
4144
|
+
### Task Group 1: Export route
|
|
4145
|
+
**Files:** `src/invoices/pdf.ts`
|
|
4146
|
+
- Render and test
|
|
4147
|
+
**Scenarios covered:** SC-1, SC-2
|
|
4148
|
+
**Acceptance Criteria:** None
|
|
4149
|
+
EOF
|
|
4150
|
+
printf 'Completeness: CLEAN\nQuality: PASS\nNo pre-existing failures.\n\nSC-1 pass: export test, 3 invoices.\nSC-2 pass: export test, empty account refused.\n' \
|
|
4151
|
+
> "$mspec/devos/specs/$mspec_name/verifications/verification.md"
|
|
4152
|
+
mkdir -p "$mspec/src/invoices"
|
|
4153
|
+
printf 'export const render = (invoices) => invoices.join(",");\n' \
|
|
4154
|
+
> "$mspec/src/invoices/pdf.ts"
|
|
4155
|
+
out=$(bash "$VSPEC" "$mspec/devos/specs/$mspec_name" 2>&1); rc=$?
|
|
4156
|
+
check "m-spec: the one-page functional spec validates" 0 "$rc"
|
|
4157
|
+
# The negative control, because a guard that has only ever passed is not a
|
|
4158
|
+
# guard. Deleting the one delivered source file has to fail the DoD gate, or
|
|
4159
|
+
# this block is asserting nothing.
|
|
4160
|
+
git -C "$mspec" init -q -b "$mspec_name"
|
|
4161
|
+
mv "$mspec/devos/specs/$mspec_name" "$mspec/devos/specs/archive/$mspec_name" 2>/dev/null \
|
|
4162
|
+
|| { mkdir -p "$mspec/devos/specs/archive" && mv "$mspec/devos/specs/$mspec_name" "$mspec/devos/specs/archive/$mspec_name"; }
|
|
4163
|
+
printf 'Implemented the export route.\nVerdict: DONE: devos/specs/archive/%s\n' "$mspec_name" \
|
|
4164
|
+
> "$mspec/transcript.txt"
|
|
4165
|
+
out=$(cd "$mspec" && bash "$DOD" transcript.txt 2>&1); rc=$?
|
|
4166
|
+
check "m-spec: the one-page folder closes the pipeline" 0 "$rc"
|
|
4167
|
+
rm "$mspec/src/invoices/pdf.ts"
|
|
4168
|
+
out=$(cd "$mspec" && bash "$DOD" transcript.txt 2>&1); rc=$?
|
|
4169
|
+
check "m-spec: the same folder fails once the delivered file is gone" 2 "$rc"
|
|
4170
|
+
|
|
4171
|
+
# The quality row the close path writes. verify-close.md 4.3 specifies six
|
|
4172
|
+
# columns, after dropping the three that no row in this repo has ever filled.
|
|
4173
|
+
# The contract that matters is not the column count but that the parser counts
|
|
4174
|
+
# the row: an instruction and a status script that disagree is how a metric
|
|
4175
|
+
# silently starts reporting on nothing. A legacy nine-column row has to keep
|
|
4176
|
+
# parsing too, because the archive is full of them.
|
|
4177
|
+
qrow=$(mktemp -d)
|
|
4178
|
+
mkdir -p "$qrow/devos/specs/archive"
|
|
4179
|
+
# Five rows minimum: quality-correlation.sh refuses to report below that, and a
|
|
4180
|
+
# refusal would mask the thing this check is about.
|
|
4181
|
+
for n in a b c d; do
|
|
4182
|
+
printf '%s\tL\tyes\t0\tnone\tyes\n' "$n" > "$qrow/devos/specs/archive/$n.quality.md"
|
|
4183
|
+
done
|
|
4184
|
+
printf 'legacy\tL\tyes\t2\twarning\tyes\t3\t1/4\tdebt\n' > "$qrow/devos/specs/archive/legacy.quality.md"
|
|
4185
|
+
out=$(bash "$QC" "$qrow" 2>&1); rc=$?
|
|
4186
|
+
check "quality-row: a six-column corpus does not error" 0 "$rc"
|
|
4187
|
+
# The real contract: both row shapes are counted, so the metric does not
|
|
4188
|
+
# silently start reporting on nothing once old rows age out.
|
|
4189
|
+
if grep -q 'Quality files: 5' <<<"$out" && grep -q 'Specs analyzed: 5' <<<"$out"; then
|
|
4190
|
+
pass=$((pass + 1))
|
|
4191
|
+
else
|
|
4192
|
+
echo "FAIL quality-row: six-column and legacy rows not both counted" >&2
|
|
4193
|
+
printf '%s\n' "$out" >&2
|
|
4194
|
+
fail=$((fail + 1))
|
|
4195
|
+
fi
|
|
4196
|
+
|
|
4197
|
+
# --------------------------------------------------------------------------
|
|
4198
|
+
# knowledge.py: the absorbed-source manifest. Its contract is that it needs no
|
|
4199
|
+
# network and no model, so drift is a diff between two files on disk, and that
|
|
4200
|
+
# a re-read never loses the date a collection was last distilled.
|
|
4201
|
+
KNOW="$REPO_ROOT/plugins/devos/skills/absorb/scripts/knowledge.py"
|
|
4202
|
+
kdir="$tmp/knowledge/devos/product/knowledge"
|
|
4203
|
+
know() { python3 -I "$KNOW" --dir "$kdir" "$@" 2>&1; }
|
|
4204
|
+
cat >"$tmp/sitemap.xml" <<'XML'
|
|
4205
|
+
<urlset><url><loc>https://help.example.com/fr/articles/111-statuses</loc><lastmod>2026-01-02T00:00:00Z</lastmod></url>
|
|
4206
|
+
<url><loc>https://help.example.com/fr/articles/222-quotes</loc><lastmod>2026-01-03T00:00:00Z</lastmod></url></urlset>
|
|
4207
|
+
XML
|
|
4208
|
+
cat >"$tmp/collection.md" <<'MD'
|
|
4209
|
+
- [Invoice statuses](https://help.example.com/fr/articles/111-statuses)
|
|
4210
|
+
- [Quotes](https://help.example.com/fr/articles/222-quotes)
|
|
4211
|
+
MD
|
|
4212
|
+
know init >/dev/null
|
|
4213
|
+
grep -q 'cache/' "$kdir/.gitignore" 2>/dev/null
|
|
4214
|
+
check "knowledge: init ignores the fetch cache" 0 $?
|
|
4215
|
+
out=$(know import --collection sales --listing "$tmp/collection.md" --lastmod "$tmp/sitemap.xml")
|
|
4216
|
+
check "knowledge: import counts the rows" "import sales: 2 new, 0 updated, 2 rows total" "$out"
|
|
4217
|
+
grep -q $'111\thttps://help.example.com/fr/articles/111-statuses\tInvoice statuses\tsales\tin\t2026-01-02' "$kdir/sources.tsv"
|
|
4218
|
+
check "knowledge: a row carries url, title, collection, scope and source date" 0 $?
|
|
4219
|
+
out=$(know check --lastmod "$tmp/sitemap.xml" | tail -1)
|
|
4220
|
+
check "knowledge: unread rows are reported before any read date" "check: 0 changed, 0 new, 0 gone, 2 unread" "$out"
|
|
4221
|
+
know read --collection sales --date 2026-02-01 >/dev/null
|
|
4222
|
+
out=$(know check --lastmod "$tmp/sitemap.xml" | tail -1)
|
|
4223
|
+
check "knowledge: no drift once read after the source date" "check: no drift, 2 in-scope articles current" "$out"
|
|
4224
|
+
cat >"$tmp/sitemap.xml" <<'XML'
|
|
4225
|
+
<urlset><url><loc>https://help.example.com/fr/articles/111-statuses</loc><lastmod>2026-03-09T00:00:00Z</lastmod></url>
|
|
4226
|
+
<url><loc>https://help.example.com/fr/articles/333-dunning</loc><lastmod>2026-03-01T00:00:00Z</lastmod></url></urlset>
|
|
4227
|
+
XML
|
|
4228
|
+
out=$(know check --lastmod "$tmp/sitemap.xml")
|
|
4229
|
+
grep -q '^changed sales/111 read 2026-02-01, source 2026-03-09' <<<"$out"; check "knowledge: a moved source is changed" 0 $?
|
|
4230
|
+
grep -q '^new 333' <<<"$out"; check "knowledge: an unlisted article is new" 0 $?
|
|
4231
|
+
grep -q '^gone sales/222' <<<"$out"; check "knowledge: a withdrawn article is gone" 0 $?
|
|
4232
|
+
know scope --collection purchases --reason "owned by the purchases service" >/dev/null
|
|
4233
|
+
grep -q $'collection:purchases\t\tcollection purchases\tpurchases\tout\t\t\t\towned by the purchases service' "$kdir/sources.tsv"
|
|
4234
|
+
check "knowledge: an out-of-scope collection keeps its reason" 0 $?
|
|
4235
|
+
out=$(know check --lastmod "$tmp/sitemap.xml" | tail -1)
|
|
4236
|
+
grep -qv 'purchases' <<<"$out"; check "knowledge: out-of-scope rows never drift" 0 $?
|
|
4237
|
+
before=$(cat "$kdir/sources.tsv")
|
|
4238
|
+
know check --lastmod "$tmp/sitemap.xml" >/dev/null
|
|
4239
|
+
check "knowledge: check writes nothing" "$before" "$(cat "$kdir/sources.tsv")"
|
|
4240
|
+
know import --collection sales --listing "$tmp/collection.md" --lastmod "$tmp/sitemap.xml" >/dev/null
|
|
4241
|
+
grep -q $'111\t.*\t2026-02-01' "$kdir/sources.tsv"; check "knowledge: re-import keeps the read date" 0 $?
|
|
4242
|
+
# A sitemap fetched through a markdown converter keeps the dates and loses the tags.
|
|
4243
|
+
cat >"$tmp/sitemap.md" <<'MD'
|
|
4244
|
+
[https://help.example.com/fr/articles/111-statuses](https://help.example.com/fr/articles/111-statuses)
|
|
4245
|
+
2026-04-04T00:00:00Z
|
|
4246
|
+
MD
|
|
4247
|
+
out=$(know check --lastmod "$tmp/sitemap.md")
|
|
4248
|
+
grep -q '^changed sales/111 read 2026-02-01, source 2026-04-04' <<<"$out"
|
|
4249
|
+
check "knowledge: a markdown-rendered sitemap still carries dates" 0 $?
|
|
4250
|
+
|
|
4251
|
+
# --------------------------------------------------------------------------
|
|
4252
|
+
# frame survey: where the project already touches a vocabulary
|
|
4253
|
+
SV="$SKILLS/frame/scripts/survey.sh"
|
|
4254
|
+
sv="$tmp/survey"
|
|
4255
|
+
mkdir -p "$sv/src/api/handlers" "$sv/src/jobs"
|
|
4256
|
+
printf 'void a payment\n' >"$sv/src/api/handlers/void.js"
|
|
4257
|
+
printf 'void a payment in a job\n' >"$sv/src/jobs/void.job.js"
|
|
4258
|
+
out=$(cd "$sv" && bash "$SV" void)
|
|
4259
|
+
grep -q 'code: 2 files' <<<"$out"
|
|
4260
|
+
check "survey: counts the files that touch a term" 0 $?
|
|
4261
|
+
grep -q 'src/api/handlers' <<<"$out"
|
|
4262
|
+
check "survey: groups matches by area" 0 $?
|
|
4263
|
+
out=$(cd "$sv" && bash "$SV" nothing-here)
|
|
4264
|
+
grep -q 'code: none under src' <<<"$out"
|
|
4265
|
+
check "survey: a term with no code says so" 0 $?
|
|
4266
|
+
out=$(cd "$sv" && bash "$SV")
|
|
4267
|
+
check "survey: no term is a usage error" 2 $?
|
|
4268
|
+
|
|
4269
|
+
CR="$SKILLS/absorb/scripts/check-refs.sh"
|
|
4270
|
+
cr="$tmp/checkrefs"
|
|
4271
|
+
mkdir -p "$cr/docs" "$cr/src/app" "$cr/src/db"
|
|
4272
|
+
printf 'model X {\n id String\n}\n' >/dev/null
|
|
4273
|
+
printf 'enum InvoiceStatus {\n PAID\n}\n' >"$cr/src/db/schema.prisma"
|
|
4274
|
+
printf 'export const kinds = [\x27DELETE\x27] as const\n' >"$cr/src/app/kinds.ts"
|
|
4275
|
+
printf 'page names `src/app/kinds.ts` and `InvoiceStatus.PAID` and `kinds.DELETE`\n' >"$cr/docs/ok.md"
|
|
4276
|
+
out=$(bash "$CR" --docs "$cr/docs" --src "$cr/src" 2>&1)
|
|
4277
|
+
check "check-refs: a page that resolves passes" 0 $?
|
|
4278
|
+
grep -q 'missing refs: 0' <<<"$out"
|
|
4279
|
+
check "check-refs: reports zero missing" 0 $?
|
|
4280
|
+
printf 'page names `src/app/gone.ts` and `InvoiceStatus.TERMINATED`\n' >"$cr/docs/bad.md"
|
|
4281
|
+
out=$(bash "$CR" --docs "$cr/docs" --src "$cr/src" 2>&1) && rc=0 || rc=$?
|
|
4282
|
+
check "check-refs: a missing path fails" 1 $rc
|
|
4283
|
+
grep -q 'MISSING src/app/gone.ts' <<<"$out"
|
|
4284
|
+
check "check-refs: names the missing path" 0 $?
|
|
4285
|
+
grep -q 'InvoiceStatus.TERMINATED' <<<"$out"
|
|
4286
|
+
check "check-refs: names the unknown member" 0 $?
|
|
4287
|
+
rm "$cr/docs/bad.md"
|
|
4288
|
+
out=$(bash "$CR" --docs "$cr/docs" 2>&1) && rc=0 || rc=$?
|
|
4289
|
+
check "check-refs: no source root is a usage error" 2 $rc
|
|
4290
|
+
|
|
4291
|
+
# --------------------------------------------------------------------------
|
|
4292
|
+
# The suite runs entirely inside $tmp. If that directory disappears mid-run,
|
|
4293
|
+
# every later check fails against files that are no longer there and the summary
|
|
4294
|
+
# reports gate failures that never happened: a `just verify` run showed 4 such
|
|
4295
|
+
# failures while this script passed 378/378 standalone, including under
|
|
4296
|
+
# concurrent load. The race behind it is not reproducible on demand, so this
|
|
4297
|
+
# does not fix it. It stops the framework reporting an environment collapse as
|
|
4298
|
+
# a verdict on the user's change, the same reason a verifier reports UNVERIFIED
|
|
4299
|
+
# instead of upgrading a missing input into a PASS.
|
|
4300
|
+
if [[ ! -d "$tmp" ]]; then
|
|
4301
|
+
echo "ENV test-gates: workspace $tmp vanished mid-run, results unusable (not a gate failure)" >&2
|
|
4302
|
+
exit 2
|
|
4303
|
+
fi
|
|
4304
|
+
echo "gate tests: $pass passed, $fail failed"
|
|
4305
|
+
[[ $fail -eq 0 ]] && echo "PASS gate scripts behave as documented"
|
|
4306
|
+
exit $((fail > 0))
|