@windyroad/risk-scorer 0.17.4 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +13 -2
- package/bin/install.mjs +42 -8
- package/hooks/codex-agent-completion.mjs +82 -0
- package/hooks/hooks.json +5 -2
- package/hooks/risk-scorer-dispatch.sh +3 -0
- package/hooks/risk-scorer-scaffold-nudge.sh +15 -4
- package/package.json +5 -2
- package/scripts/codex-agents.mjs +208 -0
- package/scripts/sync-codex-skills.mjs +8 -3
- package/skills/assess-external-comms/SKILL.md +5 -4
- package/skills/assess-inbound-report/SKILL.md +5 -4
- package/skills/assess-release/SKILL.md +5 -4
- package/skills/assess-wip/SKILL.md +5 -4
- package/skills/bootstrap-catalog/SKILL.md +5 -4
- package/skills/create-risk/SKILL.md +5 -4
- package/skills/external-comms/SKILL.md +5 -4
- package/skills/pipeline/SKILL.md +5 -4
- package/skills/update-policy/SKILL.md +6 -5
- package/skills/wip/SKILL.md +5 -4
- package/agents/test/inbound-report-contract.bats +0 -225
- package/agents/test/risk-scorer-above-appetite-stop.bats +0 -80
- package/agents/test/risk-scorer-catalog-consumption.bats +0 -138
- package/agents/test/risk-scorer-commit-verdict.bats +0 -66
- package/agents/test/risk-scorer-monitoring-not-a-control.bats +0 -76
- package/agents/test/risk-scorer-reducing-bypass-criteria.bats +0 -62
- package/agents/test/risk-scorer-register-hint.bats +0 -111
- package/agents/test/risk-scorer-structured-remediations.bats +0 -123
- package/agents/test/risk-scorer-user-stated-preconditions.bats +0 -89
- package/hooks/test/ci-status-gate.bats +0 -234
- package/hooks/test/external-comms-gate.bats +0 -528
- package/hooks/test/external-comms-key-substance.bats +0 -74
- package/hooks/test/git-push-gate.bats +0 -82
- package/hooks/test/pipeline-state-changeset-partition.bats +0 -181
- package/hooks/test/pipeline-state-hash.bats +0 -195
- package/hooks/test/plan-risk-guidance-once-per-session.bats +0 -95
- package/hooks/test/reducing-marker-persistence.bats +0 -236
- package/hooks/test/risk-gate.bats +0 -330
- package/hooks/test/risk-policy-project-root.bats +0 -20
- package/hooks/test/risk-score-commit-gate-adr-031-bypass.bats +0 -98
- package/hooks/test/risk-score-commit-gate-cadence-staleness.bats +0 -105
- package/hooks/test/risk-score-mark-external-comms-prompt-parse.bats +0 -94
- package/hooks/test/risk-score-mark-register-queue.bats +0 -253
- package/hooks/test/risk-score-mark.bats +0 -112
- package/hooks/test/risk-scorer-hook-dispatch.bats +0 -77
- package/hooks/test/risk-scorer-no-stop-hook.bats +0 -19
- package/hooks/test/risk-scorer-scaffold-nudge.bats +0 -132
- package/hooks/test/slide-marker-on-subprocess-return.bats +0 -118
- package/hooks/test/wip-risk-mark.bats +0 -64
- package/scripts/test/drain-register-queue.bats +0 -359
- package/scripts/test/evaluate-graduation.bats +0 -533
- package/scripts/test/extract-risks-from-reports.bats +0 -366
- package/scripts/test/restage-commit.bats +0 -137
|
@@ -1,330 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bats
|
|
2
|
-
# Tests for .claude/hooks/lib/risk-gate.sh
|
|
3
|
-
|
|
4
|
-
setup() {
|
|
5
|
-
HOOKS_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
|
6
|
-
source "$HOOKS_DIR/lib/gate-helpers.sh"
|
|
7
|
-
source "$HOOKS_DIR/lib/risk-gate.sh"
|
|
8
|
-
|
|
9
|
-
TEST_SESSION="bats-test-$$-${BATS_TEST_NUMBER}"
|
|
10
|
-
RDIR=$(_risk_dir "$TEST_SESSION")
|
|
11
|
-
SCORE_FILE="${RDIR}/commit"
|
|
12
|
-
HASH_FILE="${RDIR}/state-hash"
|
|
13
|
-
|
|
14
|
-
export RISK_TTL=5
|
|
15
|
-
rm -f "$SCORE_FILE" "$HASH_FILE"
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
teardown() {
|
|
19
|
-
rm -rf "${TMPDIR:-/tmp}/claude-risk-${TEST_SESSION}"
|
|
20
|
-
[ -n "${POLICY_DIR:-}" ] && rm -rf "$POLICY_DIR"
|
|
21
|
-
unset RISK_APPETITE 2>/dev/null || true
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
# Write a RISK-POLICY.md whose § Risk Appetite carries $1 into a fresh temp
|
|
25
|
-
# dir and cd into it, so check_risk_gate reads the appetite from cwd. Pass an
|
|
26
|
-
# empty string to OMIT the file entirely (absent-policy default path).
|
|
27
|
-
_use_policy() {
|
|
28
|
-
local appetite_section="$1"
|
|
29
|
-
POLICY_DIR=$(mktemp -d)
|
|
30
|
-
if [ -n "$appetite_section" ]; then
|
|
31
|
-
printf '# Risk Policy\n\n## Risk Appetite\n\n%s\n\n## Impact Levels\n| L | x |\n' \
|
|
32
|
-
"$appetite_section" > "$POLICY_DIR/RISK-POLICY.md"
|
|
33
|
-
fi
|
|
34
|
-
cd "$POLICY_DIR"
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
# Helper: call check_risk_gate directly (not via run) so RISK_GATE_REASON is visible
|
|
38
|
-
assert_gate_denies() {
|
|
39
|
-
local session="$1" action="$2" expected_reason="$3"
|
|
40
|
-
RISK_GATE_REASON=""
|
|
41
|
-
if check_risk_gate "$session" "$action"; then
|
|
42
|
-
echo "Expected gate to deny but it allowed"
|
|
43
|
-
return 1
|
|
44
|
-
fi
|
|
45
|
-
if [[ "$RISK_GATE_REASON" != *"$expected_reason"* ]]; then
|
|
46
|
-
echo "Expected reason to contain '$expected_reason' but got: $RISK_GATE_REASON"
|
|
47
|
-
return 1
|
|
48
|
-
fi
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
assert_gate_allows() {
|
|
52
|
-
local session="$1" action="$2"
|
|
53
|
-
if ! check_risk_gate "$session" "$action"; then
|
|
54
|
-
echo "Expected gate to allow but it denied: $RISK_GATE_REASON"
|
|
55
|
-
return 1
|
|
56
|
-
fi
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
@test "missing score file denies" {
|
|
60
|
-
assert_gate_denies "$TEST_SESSION" "commit" "No commit risk score found"
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
@test "missing-score deny instructs synchronous scorer dispatch (P402)" {
|
|
64
|
-
# A background-launched scorer's PostToolUse:Agent mark hook does not fire, so
|
|
65
|
-
# no marker persists and the gate re-blocks. The deny must tell the agent to
|
|
66
|
-
# dispatch synchronously. Mirrors external-comms-gate-canonical.bats.
|
|
67
|
-
assert_gate_denies "$TEST_SESSION" "commit" "run_in_background: false"
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
@test "score file with PENDING denies (non-numeric)" {
|
|
71
|
-
printf 'PENDING' > "$SCORE_FILE"
|
|
72
|
-
assert_gate_denies "$TEST_SESSION" "commit" "invalid value"
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
@test "score 4 allows (below threshold)" {
|
|
76
|
-
printf '4' > "$SCORE_FILE"
|
|
77
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
@test "score 5 allows (at threshold per ADR-086)" {
|
|
81
|
-
printf '5' > "$SCORE_FILE"
|
|
82
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
@test "score 8 denies (above threshold)" {
|
|
86
|
-
printf '8' > "$SCORE_FILE"
|
|
87
|
-
assert_gate_denies "$TEST_SESSION" "commit" "8/25"
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
@test "score 1 allows (very low)" {
|
|
91
|
-
printf '1' > "$SCORE_FILE"
|
|
92
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
@test "expired score file denies" {
|
|
96
|
-
printf '3' > "$SCORE_FILE"
|
|
97
|
-
# Backdate mtime by 10 seconds (TTL is 5)
|
|
98
|
-
touch -t "$(date -v-10S +%Y%m%d%H%M.%S 2>/dev/null || date -d '10 seconds ago' +%Y%m%d%H%M.%S 2>/dev/null)" "$SCORE_FILE"
|
|
99
|
-
assert_gate_denies "$TEST_SESSION" "commit" "expired"
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
@test "fresh score file allows" {
|
|
103
|
-
printf '3' > "$SCORE_FILE"
|
|
104
|
-
touch "$SCORE_FILE"
|
|
105
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
@test "drift detection: hash mismatch denies" {
|
|
109
|
-
printf '3' > "$SCORE_FILE"
|
|
110
|
-
touch "$SCORE_FILE"
|
|
111
|
-
echo "oldhash123" > "$HASH_FILE"
|
|
112
|
-
assert_gate_denies "$TEST_SESSION" "commit" "drift"
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
@test "no hash file skips drift check (backwards compat)" {
|
|
116
|
-
printf '3' > "$SCORE_FILE"
|
|
117
|
-
touch "$SCORE_FILE"
|
|
118
|
-
rm -f "$HASH_FILE"
|
|
119
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
@test "risk_gate_deny outputs valid JSON" {
|
|
123
|
-
run risk_gate_deny "Test reason"
|
|
124
|
-
[ "$status" -eq 0 ]
|
|
125
|
-
echo "$output" | python3 -c "import sys, json; json.load(sys.stdin)" 2>/dev/null
|
|
126
|
-
[[ "$output" == *"permissionDecision"* ]]
|
|
127
|
-
[[ "$output" == *"deny"* ]]
|
|
128
|
-
[[ "$output" == *"Test reason"* ]]
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
# ---------------------------------------------------------------------------
|
|
132
|
-
# Three-band TTL policy (P090)
|
|
133
|
-
# Band A: age < TTL/2 → pass silently, no slide
|
|
134
|
-
# Band B: TTL/2 <= age < TTL → consult state-hash; if invariant, pass + slide
|
|
135
|
-
# the marker forward (touch score file) bounded by 2*TTL hard cap
|
|
136
|
-
# from the scorer-run birth time (<action>-born); if drifted, halt
|
|
137
|
-
# Band C: age >= TTL → halt with existing "expired" message
|
|
138
|
-
# ---------------------------------------------------------------------------
|
|
139
|
-
|
|
140
|
-
# Helper: backdate file mtime by N seconds (portable between macOS and Linux)
|
|
141
|
-
_backdate() {
|
|
142
|
-
local file="$1" seconds="$2"
|
|
143
|
-
local stamp
|
|
144
|
-
stamp=$(date -v-${seconds}S +%Y%m%d%H%M.%S 2>/dev/null \
|
|
145
|
-
|| date -d "${seconds} seconds ago" +%Y%m%d%H%M.%S 2>/dev/null)
|
|
146
|
-
touch -t "$stamp" "$file"
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
# Helper: write a matching state-hash for the current working tree
|
|
150
|
-
_write_matching_hash() {
|
|
151
|
-
local target="$1"
|
|
152
|
-
local hash
|
|
153
|
-
hash=$("$HOOKS_DIR/lib/pipeline-state.sh" --hash-inputs 2>/dev/null | _hashcmd | cut -d' ' -f1)
|
|
154
|
-
echo "$hash" > "$target"
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
@test "Band A (age < TTL/2): passes, does NOT slide the marker" {
|
|
158
|
-
printf '3' > "$SCORE_FILE"
|
|
159
|
-
touch "$SCORE_FILE"
|
|
160
|
-
rm -f "$HASH_FILE"
|
|
161
|
-
BEFORE_MTIME=$(_mtime "$SCORE_FILE")
|
|
162
|
-
sleep 1
|
|
163
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
164
|
-
AFTER_MTIME=$(_mtime "$SCORE_FILE")
|
|
165
|
-
[ "$BEFORE_MTIME" = "$AFTER_MTIME" ]
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
@test "Band B (TTL/2 <= age < TTL) with hash invariant: passes AND slides marker forward" {
|
|
169
|
-
printf '3' > "$SCORE_FILE"
|
|
170
|
-
_backdate "$SCORE_FILE" 3
|
|
171
|
-
_write_matching_hash "$HASH_FILE"
|
|
172
|
-
touch "${SCORE_FILE}-born"
|
|
173
|
-
BEFORE_MTIME=$(_mtime "$SCORE_FILE")
|
|
174
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
175
|
-
AFTER_MTIME=$(_mtime "$SCORE_FILE")
|
|
176
|
-
[ "$AFTER_MTIME" -gt "$BEFORE_MTIME" ]
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
@test "Band B with no hash file: passes but does NOT slide (no invariance proof)" {
|
|
180
|
-
printf '3' > "$SCORE_FILE"
|
|
181
|
-
_backdate "$SCORE_FILE" 2
|
|
182
|
-
rm -f "$HASH_FILE"
|
|
183
|
-
BEFORE_MTIME=$(_mtime "$SCORE_FILE")
|
|
184
|
-
sleep 1
|
|
185
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
186
|
-
AFTER_MTIME=$(_mtime "$SCORE_FILE")
|
|
187
|
-
[ "$BEFORE_MTIME" = "$AFTER_MTIME" ]
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
@test "Band B with hash mismatch: denies with drift (no slide)" {
|
|
191
|
-
printf '3' > "$SCORE_FILE"
|
|
192
|
-
_backdate "$SCORE_FILE" 3
|
|
193
|
-
echo "staleold" > "$HASH_FILE"
|
|
194
|
-
touch "${SCORE_FILE}-born"
|
|
195
|
-
BEFORE_MTIME=$(_mtime "$SCORE_FILE")
|
|
196
|
-
assert_gate_denies "$TEST_SESSION" "commit" "drift"
|
|
197
|
-
AFTER_MTIME=$(_mtime "$SCORE_FILE")
|
|
198
|
-
[ "$BEFORE_MTIME" = "$AFTER_MTIME" ]
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
@test "Band B with hard-cap exceeded (born-age >= 2*TTL): denies even if hash invariant" {
|
|
202
|
-
printf '3' > "$SCORE_FILE"
|
|
203
|
-
_backdate "$SCORE_FILE" 3
|
|
204
|
-
_write_matching_hash "$HASH_FILE"
|
|
205
|
-
touch "${SCORE_FILE}-born"
|
|
206
|
-
_backdate "${SCORE_FILE}-born" 12
|
|
207
|
-
assert_gate_denies "$TEST_SESSION" "commit" "expired"
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
@test "Band C (age >= TTL): denies regardless of hash invariance" {
|
|
211
|
-
printf '3' > "$SCORE_FILE"
|
|
212
|
-
_backdate "$SCORE_FILE" 10
|
|
213
|
-
_write_matching_hash "$HASH_FILE"
|
|
214
|
-
touch "${SCORE_FILE}-born"
|
|
215
|
-
assert_gate_denies "$TEST_SESSION" "commit" "expired"
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
@test "Band B denial exports RISK_GATE_CATEGORY=drift on hash mismatch" {
|
|
219
|
-
printf '3' > "$SCORE_FILE"
|
|
220
|
-
_backdate "$SCORE_FILE" 3
|
|
221
|
-
echo "staleold" > "$HASH_FILE"
|
|
222
|
-
RISK_GATE_CATEGORY=""
|
|
223
|
-
! check_risk_gate "$TEST_SESSION" "commit"
|
|
224
|
-
[ "$RISK_GATE_CATEGORY" = "drift" ]
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
@test "Band C denial exports RISK_GATE_CATEGORY=expired" {
|
|
228
|
-
printf '3' > "$SCORE_FILE"
|
|
229
|
-
_backdate "$SCORE_FILE" 10
|
|
230
|
-
RISK_GATE_CATEGORY=""
|
|
231
|
-
! check_risk_gate "$TEST_SESSION" "commit"
|
|
232
|
-
[ "$RISK_GATE_CATEGORY" = "expired" ]
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
@test "Threshold denial exports RISK_GATE_CATEGORY=threshold and RISK_GATE_SCORE" {
|
|
236
|
-
printf '7' > "$SCORE_FILE"
|
|
237
|
-
touch "$SCORE_FILE"
|
|
238
|
-
RISK_GATE_CATEGORY=""
|
|
239
|
-
RISK_GATE_SCORE=""
|
|
240
|
-
! check_risk_gate "$TEST_SESSION" "commit"
|
|
241
|
-
[ "$RISK_GATE_CATEGORY" = "threshold" ]
|
|
242
|
-
[ "$RISK_GATE_SCORE" = "7" ]
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
# ---------------------------------------------------------------------------
|
|
246
|
-
# P007 / ADR-065 — the block threshold is the project's RISK-POLICY.md risk
|
|
247
|
-
# appetite (block when score > N), not a hardcoded 5. Default N=5 per ADR-086
|
|
248
|
-
# (was 4 under superseded ADR-065) when the policy is absent or unparseable,
|
|
249
|
-
# which tracks the new Low ceiling so the default fallback admits residual=5
|
|
250
|
-
# (the Impact=5/Likelihood=1 floor for severe-but-rare risks). Precedence:
|
|
251
|
-
# RISK_APPETITE env > RISK-POLICY.md parse > default 5. (Behavioural fixtures
|
|
252
|
-
# per ADR-052.)
|
|
253
|
-
# ---------------------------------------------------------------------------
|
|
254
|
-
|
|
255
|
-
@test "appetite 9 (exceeds 9): score 7 within the 5-9 band PASSES" {
|
|
256
|
-
_use_policy 'Pipeline gates block when cumulative residual risk exceeds 9.'
|
|
257
|
-
printf '7' > "$SCORE_FILE"
|
|
258
|
-
rm -f "$HASH_FILE"
|
|
259
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
@test "appetite 9 (exceeds 9): score 10 above the band FAILS and deny renders the parsed appetite 9/25" {
|
|
263
|
-
_use_policy 'Pipeline gates block when cumulative residual risk exceeds 9.'
|
|
264
|
-
printf '10' > "$SCORE_FILE"
|
|
265
|
-
rm -f "$HASH_FILE"
|
|
266
|
-
assert_gate_denies "$TEST_SESSION" "commit" "appetite of 9/25"
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
@test "appetite via 'Threshold: 9' phrasing: score 9 PASSES, score 10 FAILS" {
|
|
270
|
-
_use_policy '**Threshold: 9 (Medium)**'
|
|
271
|
-
rm -f "$HASH_FILE"
|
|
272
|
-
printf '9' > "$SCORE_FILE"
|
|
273
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
274
|
-
printf '10' > "$SCORE_FILE"
|
|
275
|
-
assert_gate_denies "$TEST_SESSION" "commit" "10/25"
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
@test "appetite 4 (exceeds 4): score 4 PASSES, score 5 FAILS" {
|
|
279
|
-
_use_policy 'Pipeline gates block when cumulative residual risk exceeds 4.'
|
|
280
|
-
rm -f "$HASH_FILE"
|
|
281
|
-
printf '4' > "$SCORE_FILE"
|
|
282
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
283
|
-
printf '5' > "$SCORE_FILE"
|
|
284
|
-
assert_gate_denies "$TEST_SESSION" "commit" "appetite of 4/25"
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
@test "absent RISK-POLICY.md: defaults to appetite 5 per ADR-086 (5 PASSES, 6 FAILS)" {
|
|
288
|
-
_use_policy '' # no RISK-POLICY.md in the temp dir
|
|
289
|
-
rm -f "$HASH_FILE"
|
|
290
|
-
printf '5' > "$SCORE_FILE"
|
|
291
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
292
|
-
printf '6' > "$SCORE_FILE"
|
|
293
|
-
assert_gate_denies "$TEST_SESSION" "commit" "appetite of 5/25"
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
@test "unparseable RISK-POLICY.md (no appetite integer): defaults to appetite 5 per ADR-086" {
|
|
297
|
-
_use_policy 'We are conservative about risk but state no number here.'
|
|
298
|
-
rm -f "$HASH_FILE"
|
|
299
|
-
printf '5' > "$SCORE_FILE"
|
|
300
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
301
|
-
printf '6' > "$SCORE_FILE"
|
|
302
|
-
assert_gate_denies "$TEST_SESSION" "commit" "appetite of 5/25"
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
@test "fractional score 5.5 FAILS under default appetite 5 (5.5 > 5) per ADR-086" {
|
|
306
|
-
_use_policy ''
|
|
307
|
-
rm -f "$HASH_FILE"
|
|
308
|
-
printf '5.5' > "$SCORE_FILE"
|
|
309
|
-
assert_gate_denies "$TEST_SESSION" "commit" "5.5/25"
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
@test "appetite via 'Threshold: 5' phrasing (ADR-086 new Low ceiling): score 5 PASSES, score 6 FAILS" {
|
|
313
|
-
_use_policy '**Threshold: 5 (Low)**'
|
|
314
|
-
rm -f "$HASH_FILE"
|
|
315
|
-
printf '5' > "$SCORE_FILE"
|
|
316
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
317
|
-
printf '6' > "$SCORE_FILE"
|
|
318
|
-
assert_gate_denies "$TEST_SESSION" "commit" "appetite of 5/25"
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
@test "RISK_APPETITE env override takes precedence over RISK-POLICY.md parse" {
|
|
322
|
-
_use_policy 'Pipeline gates block when cumulative residual risk exceeds 4.'
|
|
323
|
-
rm -f "$HASH_FILE"
|
|
324
|
-
RISK_APPETITE=9
|
|
325
|
-
printf '7' > "$SCORE_FILE"
|
|
326
|
-
assert_gate_allows "$TEST_SESSION" "commit"
|
|
327
|
-
printf '10' > "$SCORE_FILE"
|
|
328
|
-
assert_gate_denies "$TEST_SESSION" "commit" "appetite of 9/25"
|
|
329
|
-
unset RISK_APPETITE
|
|
330
|
-
}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bats
|
|
2
|
-
|
|
3
|
-
# P004: risk-policy-enforce-edit.sh project-root check.
|
|
4
|
-
|
|
5
|
-
setup() {
|
|
6
|
-
SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
|
7
|
-
HOOK="$SCRIPT_DIR/risk-policy-enforce-edit.sh"
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
run_hook_with_file() {
|
|
11
|
-
local file_path="$1"
|
|
12
|
-
local json="{\"tool_input\":{\"file_path\":\"${file_path}\"},\"session_id\":\"test-$$\"}"
|
|
13
|
-
echo "$json" | bash "$HOOK"
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
@test "risk-policy project-root: RISK-POLICY.md outside project exits 0" {
|
|
17
|
-
run run_hook_with_file "/Users/other/project/RISK-POLICY.md"
|
|
18
|
-
[ "$status" -eq 0 ]
|
|
19
|
-
[[ "$output" != *"BLOCKED"* ]]
|
|
20
|
-
}
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bats
|
|
2
|
-
|
|
3
|
-
# P170 / RFC-002 / ADR-031 T11: risk-score-commit-gate.sh recognises
|
|
4
|
-
# the `RISK_BYPASS: adr-031-migration` self-attestation marker in the
|
|
5
|
-
# git commit command's message arguments. Migration commits emitted by
|
|
6
|
-
# `migrate_problems_to_per_state_layout` carry this marker in their
|
|
7
|
-
# body so adopter auto-migration commits skip the full risk-score
|
|
8
|
-
# overhead while preserving the audit trail (per ADR-031 § Open
|
|
9
|
-
# Execution-time Questions resolution Q3 lean (b)).
|
|
10
|
-
|
|
11
|
-
setup() {
|
|
12
|
-
HOOKS_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
|
13
|
-
GATE_SCRIPT="$HOOKS_DIR/risk-score-commit-gate.sh"
|
|
14
|
-
TEST_SESSION="bats-test-$$-${BATS_TEST_NUMBER}"
|
|
15
|
-
|
|
16
|
-
# Run inside a temp work tree so the gate's RISK-POLICY.md presence
|
|
17
|
-
# check has something to bind to. The migration test fixture (T10)
|
|
18
|
-
# creates a real adopter repo; here we just need a minimal stub.
|
|
19
|
-
TMP_REPO="$(mktemp -d)"
|
|
20
|
-
cd "$TMP_REPO"
|
|
21
|
-
cat > RISK-POLICY.md <<EOF
|
|
22
|
-
# Risk Policy
|
|
23
|
-
|
|
24
|
-
Last reviewed: $(date -u +%Y-%m-%d)
|
|
25
|
-
EOF
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
teardown() {
|
|
29
|
-
rm -rf "$TMP_REPO"
|
|
30
|
-
rm -rf "${TMPDIR:-/tmp}/claude-risk-${TEST_SESSION}"
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
# Helper: invoke the gate with a Bash tool_input json + given command.
|
|
34
|
-
# Returns 0 if gate allows (exits 0 with no JSON deny), non-zero on deny.
|
|
35
|
-
invoke_gate() {
|
|
36
|
-
local command="$1"
|
|
37
|
-
local input
|
|
38
|
-
input=$(python3 -c "
|
|
39
|
-
import json, sys
|
|
40
|
-
print(json.dumps({
|
|
41
|
-
'tool_name': 'Bash',
|
|
42
|
-
'tool_input': {'command': sys.argv[1]},
|
|
43
|
-
'session_id': sys.argv[2]
|
|
44
|
-
}))
|
|
45
|
-
" "$command" "$TEST_SESSION")
|
|
46
|
-
echo "$input" | bash "$GATE_SCRIPT"
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
@test "T11: commit with RISK_BYPASS: adr-031-migration marker bypasses the gate" {
|
|
50
|
-
local cmd='git commit -m "docs(problems): auto-migrate" -m "RISK_BYPASS: adr-031-migration"'
|
|
51
|
-
run invoke_gate "$cmd"
|
|
52
|
-
[ "$status" -eq 0 ]
|
|
53
|
-
# Gate should emit NO deny JSON when bypassed (silent allow).
|
|
54
|
-
[[ "$output" != *"permissionDecision"* ]] || [[ "$output" != *"deny"* ]]
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
@test "T11: commit message body containing RISK_BYPASS marker (multi-paragraph -m sequence) bypasses" {
|
|
58
|
-
local cmd='git commit -m "docs(problems): auto-migrate to per-state subdirectory layout (ADR-031)" -m "See: docs/decisions/031-problem-ticket-directory-layout.accepted.md" -m "RISK_BYPASS: adr-031-migration"'
|
|
59
|
-
run invoke_gate "$cmd"
|
|
60
|
-
[ "$status" -eq 0 ]
|
|
61
|
-
[[ "$output" != *"permissionDecision"* ]] || [[ "$output" != *"deny"* ]]
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
@test "T11: normal commit without RISK_BYPASS marker still gated (no score = deny)" {
|
|
65
|
-
local cmd='git commit -m "feat: normal commit"'
|
|
66
|
-
run invoke_gate "$cmd"
|
|
67
|
-
# Either exits with deny JSON output OR exits non-zero — both indicate the
|
|
68
|
-
# gate didn't silently allow. We assert deny JSON appears in the output.
|
|
69
|
-
[[ "$output" == *"deny"* ]]
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
@test "T11: heredoc-style git commit -m with embedded RISK_BYPASS marker bypasses" {
|
|
73
|
-
# Heredoc shape used by the migration routine + manual workflows;
|
|
74
|
-
# marker must be detected regardless of how the message reaches the
|
|
75
|
-
# command string.
|
|
76
|
-
local cmd=$'git commit -m "$(cat <<\'EOF\'\ndocs(problems): auto-migrate\n\nRISK_BYPASS: adr-031-migration\nEOF\n)"'
|
|
77
|
-
run invoke_gate "$cmd"
|
|
78
|
-
[ "$status" -eq 0 ]
|
|
79
|
-
[[ "$output" != *"permissionDecision"* ]] || [[ "$output" != *"deny"* ]]
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
@test "T11: marker is case-sensitive (adr-031-MIGRATION not recognised — security guard)" {
|
|
83
|
-
local cmd='git commit -m "RISK_BYPASS: adr-031-MIGRATION"'
|
|
84
|
-
run invoke_gate "$cmd"
|
|
85
|
-
[[ "$output" == *"deny"* ]]
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
@test "T11: unrelated RISK_BYPASS token (e.g. reducing) does NOT match adr-031-migration path" {
|
|
89
|
-
# `reducing` is the existing risk-score-mark.sh marker for risk-reducing
|
|
90
|
-
# scoring runs — different mechanism (filesystem marker via PostToolUse
|
|
91
|
-
# agent return), not commit-message detection. The commit-gate should
|
|
92
|
-
# NOT treat a `RISK_BYPASS: reducing` line in the COMMIT MESSAGE as a
|
|
93
|
-
# bypass — only `adr-031-migration` (and any future commit-message
|
|
94
|
-
# markers added explicitly).
|
|
95
|
-
local cmd='git commit -m "RISK_BYPASS: reducing"'
|
|
96
|
-
run invoke_gate "$cmd"
|
|
97
|
-
[[ "$output" == *"deny"* ]]
|
|
98
|
-
}
|
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bats
|
|
2
|
-
|
|
3
|
-
# P408 / ADR-091 / RFC-043 / STORY-037: the POLICY_STALE branch of
|
|
4
|
-
# risk-score-commit-gate.sh derives its staleness threshold from the
|
|
5
|
-
# policy's stated review cadence line (`> Reviewed <cadence> ...`)
|
|
6
|
-
# instead of a hardcoded 14 days. Vocabulary: weekly=7,
|
|
7
|
-
# fortnightly/biweekly=14, monthly=30, quarterly=90, annually/yearly=365;
|
|
8
|
-
# fallback 14 when the line is absent or the word unrecognised. The
|
|
9
|
-
# cadence regex is case-sensitive and must bind the capital-R
|
|
10
|
-
# `Reviewed <word>` line, never the lowercase `Last reviewed: <date>`
|
|
11
|
-
# line (ADR-091 machine-read contract).
|
|
12
|
-
|
|
13
|
-
setup() {
|
|
14
|
-
HOOKS_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
|
15
|
-
GATE_SCRIPT="$HOOKS_DIR/risk-score-commit-gate.sh"
|
|
16
|
-
TEST_SESSION="bats-test-$$-${BATS_TEST_NUMBER}"
|
|
17
|
-
TMP_REPO="$(mktemp -d)"
|
|
18
|
-
cd "$TMP_REPO"
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
teardown() {
|
|
22
|
-
rm -rf "$TMP_REPO"
|
|
23
|
-
rm -rf "${TMPDIR:-/tmp}/claude-risk-${TEST_SESSION}"
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
# Helper: date N days ago, ISO format.
|
|
27
|
-
days_ago() {
|
|
28
|
-
python3 -c "from datetime import date, timedelta; print(date.today() - timedelta(days=int('$1')))"
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
# Helper: write a RISK-POLICY.md reviewed N days ago with an optional
|
|
32
|
-
# cadence line supplied verbatim as the second argument.
|
|
33
|
-
write_policy() {
|
|
34
|
-
local reviewed_days_ago="$1"
|
|
35
|
-
local cadence_line="${2-}"
|
|
36
|
-
{
|
|
37
|
-
echo "# Risk Policy"
|
|
38
|
-
echo ""
|
|
39
|
-
echo "> Last reviewed: $(days_ago "$reviewed_days_ago")"
|
|
40
|
-
if [ -n "$cadence_line" ]; then
|
|
41
|
-
echo ""
|
|
42
|
-
echo "$cadence_line"
|
|
43
|
-
fi
|
|
44
|
-
} > RISK-POLICY.md
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
# Helper: invoke the gate with a git-commit Bash payload.
|
|
48
|
-
invoke_gate() {
|
|
49
|
-
local input
|
|
50
|
-
input=$(python3 -c "
|
|
51
|
-
import json, sys
|
|
52
|
-
print(json.dumps({
|
|
53
|
-
'tool_name': 'Bash',
|
|
54
|
-
'tool_input': {'command': 'git commit -m \"feat: change\"'},
|
|
55
|
-
'session_id': sys.argv[1]
|
|
56
|
-
}))
|
|
57
|
-
" "$TEST_SESSION")
|
|
58
|
-
echo "$input" | bash "$GATE_SCRIPT"
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
@test "P408: monthly cadence, reviewed 16 days ago — NOT stale (witnessed case; deny is the score gate, not staleness)" {
|
|
62
|
-
write_policy 16 "> Reviewed monthly and after any significant change to distribution channels."
|
|
63
|
-
run invoke_gate
|
|
64
|
-
# The gate still denies (no risk score for this session) but the reason
|
|
65
|
-
# must NOT be policy staleness.
|
|
66
|
-
[[ "$output" == *"deny"* ]]
|
|
67
|
-
[[ "$output" != *"stale"* ]]
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
@test "P408: no cadence line, reviewed 16 days ago — 14-day fallback still flags stale (behaviour preserved)" {
|
|
71
|
-
write_policy 16
|
|
72
|
-
run invoke_gate
|
|
73
|
-
[[ "$output" == *"stale"* ]]
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
@test "P408: unrecognised cadence word falls back to 14 days" {
|
|
77
|
-
write_policy 16 "> Reviewed sporadically whenever convenient."
|
|
78
|
-
run invoke_gate
|
|
79
|
-
[[ "$output" == *"stale"* ]]
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
@test "P408: regex binds the capital-R cadence line, never the lowercase 'Last reviewed:' date line" {
|
|
83
|
-
# Quarterly cadence, reviewed 20 days ago. If the regex wrongly bound
|
|
84
|
-
# the 'Last reviewed:' line the capture would fail and the 14-day
|
|
85
|
-
# fallback would flag stale at 20 days; correct capital-R binding
|
|
86
|
-
# derives 90 days and passes.
|
|
87
|
-
write_policy 20 "> Reviewed quarterly."
|
|
88
|
-
run invoke_gate
|
|
89
|
-
[[ "$output" == *"deny"* ]]
|
|
90
|
-
[[ "$output" != *"stale"* ]]
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
@test "P408: elapsed monthly cadence denies and the message names the derived threshold and cadence word" {
|
|
94
|
-
write_policy 45 "> Reviewed monthly."
|
|
95
|
-
run invoke_gate
|
|
96
|
-
[[ "$output" == *"stale"* ]]
|
|
97
|
-
[[ "$output" == *"30"* ]]
|
|
98
|
-
[[ "$output" == *"monthly"* ]]
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
@test "P408: weekly cadence tightens the threshold below the default (10 days is stale under weekly)" {
|
|
102
|
-
write_policy 10 "> Reviewed weekly."
|
|
103
|
-
run invoke_gate
|
|
104
|
-
[[ "$output" == *"stale"* ]]
|
|
105
|
-
}
|
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bats
|
|
2
|
-
# Behavioural tests for risk-score-mark.sh external-comms branch under
|
|
3
|
-
# P166 hook-side key derivation (ADR-028 amended 2026-05-16).
|
|
4
|
-
#
|
|
5
|
-
# Contract: the PostToolUse:Agent hook derives the marker key from
|
|
6
|
-
# tool_input.prompt's `SURFACE: <name>` + `<draft>...</draft>` structure
|
|
7
|
-
# instead of trusting an agent-emitted EXTERNAL_COMMS_RISK_KEY line.
|
|
8
|
-
# On PASS, writes external-comms-risk-reviewed-<KEY> at the derived key.
|
|
9
|
-
# Backward-compat: falls back to agent-emitted KEY when prompt has no
|
|
10
|
-
# structure (one release-cycle window).
|
|
11
|
-
|
|
12
|
-
setup() {
|
|
13
|
-
SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
|
14
|
-
HOOK="$SCRIPT_DIR/risk-score-mark.sh"
|
|
15
|
-
ORIG_DIR="$PWD"
|
|
16
|
-
TEST_DIR=$(mktemp -d)
|
|
17
|
-
cd "$TEST_DIR"
|
|
18
|
-
TMPDIR="$TEST_DIR/tmp"
|
|
19
|
-
export TMPDIR
|
|
20
|
-
mkdir -p "$TMPDIR"
|
|
21
|
-
SESSION_ID="test-rs-mark-extcomms-prompt-$$-${BATS_TEST_NUMBER}"
|
|
22
|
-
RDIR="$TMPDIR/claude-risk-${SESSION_ID}"
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
teardown() {
|
|
26
|
-
cd "$ORIG_DIR"
|
|
27
|
-
rm -rf "$TEST_DIR"
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
gate_key() {
|
|
31
|
-
local draft="$1" surface="$2"
|
|
32
|
-
printf '%s\n%s' "$draft" "$surface" | shasum -a 256 | cut -d' ' -f1
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
run_hook() {
|
|
36
|
-
local prompt="$1"
|
|
37
|
-
local agent_output="$2"
|
|
38
|
-
python3 -c "
|
|
39
|
-
import json, sys
|
|
40
|
-
print(json.dumps({
|
|
41
|
-
'tool_name': 'Agent',
|
|
42
|
-
'session_id': '${SESSION_ID}',
|
|
43
|
-
'tool_input': {'subagent_type': 'wr-risk-scorer:external-comms', 'prompt': sys.argv[1]},
|
|
44
|
-
'tool_response': {'content': [{'type': 'text', 'text': sys.argv[2]}]}
|
|
45
|
-
}))" "$prompt" "$agent_output" | bash "$HOOK"
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
@test "external-comms PASS with structured prompt: marker lands at hook-derived key" {
|
|
49
|
-
DRAFT="we observed a leaked secret pattern in the changeset"
|
|
50
|
-
SURFACE="changeset-author"
|
|
51
|
-
PROMPT=$'SURFACE: '"$SURFACE"$'\n<draft>\n'"$DRAFT"$'\n</draft>\nReview against RISK-POLICY.md.'
|
|
52
|
-
AGENT_OUTPUT=$'no Confidential Information class matched\nEXTERNAL_COMMS_RISK_VERDICT: PASS'
|
|
53
|
-
run_hook "$PROMPT" "$AGENT_OUTPUT"
|
|
54
|
-
KEY=$(gate_key "$DRAFT" "$SURFACE")
|
|
55
|
-
[ -f "$RDIR/external-comms-risk-reviewed-${KEY}" ]
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
@test "external-comms FAIL with structured prompt: no marker" {
|
|
59
|
-
DRAFT="client Acme Corp is hitting this"
|
|
60
|
-
SURFACE="gh-issue-create"
|
|
61
|
-
PROMPT=$'SURFACE: '"$SURFACE"$'\n<draft>\n'"$DRAFT"$'\n</draft>'
|
|
62
|
-
AGENT_OUTPUT=$'EXTERNAL_COMMS_RISK_VERDICT: FAIL\nEXTERNAL_COMMS_RISK_REASON: Client names class — "Acme Corp"'
|
|
63
|
-
run_hook "$PROMPT" "$AGENT_OUTPUT"
|
|
64
|
-
KEY=$(gate_key "$DRAFT" "$SURFACE")
|
|
65
|
-
[ ! -f "$RDIR/external-comms-risk-reviewed-${KEY}" ]
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
@test "external-comms PASS with structured prompt AND agent-emitted KEY: hook-derived key wins" {
|
|
69
|
-
DRAFT="hook-derived wins"
|
|
70
|
-
SURFACE="gh-pr-comment"
|
|
71
|
-
PROMPT=$'SURFACE: '"$SURFACE"$'\n<draft>\n'"$DRAFT"$'\n</draft>'
|
|
72
|
-
BOGUS_KEY="0000000000000000000000000000000000000000000000000000000000000000"
|
|
73
|
-
AGENT_OUTPUT=$'EXTERNAL_COMMS_RISK_VERDICT: PASS\nEXTERNAL_COMMS_RISK_KEY: '"$BOGUS_KEY"
|
|
74
|
-
run_hook "$PROMPT" "$AGENT_OUTPUT"
|
|
75
|
-
DERIVED_KEY=$(gate_key "$DRAFT" "$SURFACE")
|
|
76
|
-
[ -f "$RDIR/external-comms-risk-reviewed-${DERIVED_KEY}" ]
|
|
77
|
-
[ ! -f "$RDIR/external-comms-risk-reviewed-${BOGUS_KEY}" ]
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
@test "external-comms backward-compat: PASS with no structured prompt but agent KEY" {
|
|
81
|
-
LEGACY_KEY="fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"
|
|
82
|
-
PROMPT="legacy unstructured prompt"
|
|
83
|
-
AGENT_OUTPUT=$'EXTERNAL_COMMS_RISK_VERDICT: PASS\nEXTERNAL_COMMS_RISK_KEY: '"$LEGACY_KEY"
|
|
84
|
-
run_hook "$PROMPT" "$AGENT_OUTPUT"
|
|
85
|
-
[ -f "$RDIR/external-comms-risk-reviewed-${LEGACY_KEY}" ]
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
@test "external-comms no structured prompt and no agent KEY: no marker" {
|
|
89
|
-
PROMPT="legacy"
|
|
90
|
-
AGENT_OUTPUT=$'EXTERNAL_COMMS_RISK_VERDICT: PASS'
|
|
91
|
-
run_hook "$PROMPT" "$AGENT_OUTPUT"
|
|
92
|
-
ext_markers=$(find "$RDIR" -maxdepth 1 -name 'external-comms-risk-reviewed-*' 2>/dev/null | wc -l | tr -d ' ')
|
|
93
|
-
[ "$ext_markers" -eq 0 ]
|
|
94
|
-
}
|