loki-mode 8.36.0 → 8.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/dashboard/__init__.py +1 -1
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +6 -2
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/tests/detect-invariant-violations.sh +359 -0
- package/tests/detect-mock-problems.sh +473 -0
- package/tests/detect-semantic-test-problems.sh +332 -0
- package/tests/detect-test-mutations.sh +455 -0
|
@@ -0,0 +1,455 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Test Mutation Detector - Quality Gate #9
|
|
3
|
+
# Verifies that test assertions exercise real code paths
|
|
4
|
+
#
|
|
5
|
+
# Usage: ./tests/detect-test-mutations.sh [--strict] [--block-high] [--commit HASH]
|
|
6
|
+
# --strict: Exit 1 on ANY finding (HIGH/MEDIUM/LOW) -- for CI; over-blocks
|
|
7
|
+
# --block-high: Exit 2 when one or more HIGH-severity findings are present,
|
|
8
|
+
# 0 otherwise. Does NOT block on MEDIUM/LOW. This is the clean
|
|
9
|
+
# exit-code contract for the run.sh mutation gate wrapper, so it
|
|
10
|
+
# does not have to grep stdout. --strict takes precedence if both
|
|
11
|
+
# are passed.
|
|
12
|
+
# --commit HASH: Check specific commit for assertion value mutations
|
|
13
|
+
#
|
|
14
|
+
# Output contract: every HIGH-severity finding prints a line beginning with the
|
|
15
|
+
# literal token "[HIGH]" on stdout (ANSI-colored), so a wrapper may also grep
|
|
16
|
+
# '\[HIGH\]' as an alternative to --block-high.
|
|
17
|
+
#
|
|
18
|
+
# Detects:
|
|
19
|
+
# 1. Shell tests where functions are redefined to return canned output
|
|
20
|
+
# 2. Test files where all assertions check constant values
|
|
21
|
+
# 3. Test files with assertion-to-test ratio below threshold
|
|
22
|
+
# 4. Test harnesses that intercept console.error or suppress React act warnings
|
|
23
|
+
# 5. Optional UI or storage lookups whose required assertions can silently skip
|
|
24
|
+
# 6. Assertion value mutations: commits that change assertion expected values
|
|
25
|
+
# alongside implementation changes (sign of fitting tests to code)
|
|
26
|
+
|
|
27
|
+
set -uo pipefail
|
|
28
|
+
|
|
29
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
30
|
+
# Directory to scan. Defaults to the repo containing this script (so
|
|
31
|
+
# run-all-tests.sh keeps scanning loki-mode unchanged). A run.sh gate wrapper
|
|
32
|
+
# MUST set LOKI_SCAN_DIR to the target project; cwd is NOT used by find/git here,
|
|
33
|
+
# so `cd TARGET_DIR` alone does not redirect the scan. The Check-5 git history is
|
|
34
|
+
# also read from this directory.
|
|
35
|
+
PROJECT_DIR="${LOKI_SCAN_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}"
|
|
36
|
+
STRICT=""
|
|
37
|
+
BLOCK_HIGH=""
|
|
38
|
+
COMMIT_HASH=""
|
|
39
|
+
|
|
40
|
+
# Parse arguments
|
|
41
|
+
while [ $# -gt 0 ]; do
|
|
42
|
+
case "$1" in
|
|
43
|
+
--strict) STRICT="--strict"; shift ;;
|
|
44
|
+
--block-high) BLOCK_HIGH="--block-high"; shift ;;
|
|
45
|
+
--commit) COMMIT_HASH="$2"; shift 2 ;;
|
|
46
|
+
*) shift ;;
|
|
47
|
+
esac
|
|
48
|
+
done
|
|
49
|
+
|
|
50
|
+
RED='\033[0;31m'
|
|
51
|
+
YELLOW='\033[1;33m'
|
|
52
|
+
GREEN='\033[0;32m'
|
|
53
|
+
CYAN='\033[0;36m'
|
|
54
|
+
NC='\033[0m'
|
|
55
|
+
|
|
56
|
+
FINDINGS=0
|
|
57
|
+
HIGH_FINDINGS=0
|
|
58
|
+
|
|
59
|
+
echo "=========================================="
|
|
60
|
+
echo "Test Mutation Detector - Quality Gate #9"
|
|
61
|
+
echo "=========================================="
|
|
62
|
+
echo ""
|
|
63
|
+
|
|
64
|
+
report() {
|
|
65
|
+
local severity="$1"
|
|
66
|
+
local file="$2"
|
|
67
|
+
local message="$3"
|
|
68
|
+
|
|
69
|
+
case "$severity" in
|
|
70
|
+
HIGH) echo -e "${RED}[HIGH]${NC} $file - $message"; ((HIGH_FINDINGS++)) ;;
|
|
71
|
+
MEDIUM) echo -e "${YELLOW}[MEDIUM]${NC} $file - $message" ;;
|
|
72
|
+
LOW) echo -e "${CYAN}[LOW]${NC} $file - $message" ;;
|
|
73
|
+
esac
|
|
74
|
+
((FINDINGS++))
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
find_js_harness_files() {
|
|
78
|
+
# Filters the ONE shared tree walk (_ALL_MUT_FILES) instead of walking the
|
|
79
|
+
# tree again. The name and -path predicates below are the same set the
|
|
80
|
+
# original `find` expressed, re-expressed as an ERE over full paths; the
|
|
81
|
+
# trailing exclusion grep is unchanged.
|
|
82
|
+
printf '%s\n' "$_ALL_MUT_FILES" \
|
|
83
|
+
| grep -E '(\.(test|spec)\.(ts|tsx|js|jsx)|/(setupTests|test-setup|vitest\.setup|jest\.setup)\.(ts|tsx|js|jsx)|/(vitest|jest)\.config\.(ts|js|mjs|cjs)|/tests?/setup\.(ts|tsx|js|jsx))$' \
|
|
84
|
+
| grep -Ev '/(node_modules|dist|build|coverage|\.git|\.claude|\.loki)/'
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
# Run one grep over a whole file set and stream `path:lineno:text` back.
|
|
88
|
+
# THE STREAM IS THE ITERATION: grep -H already emits results grouped by file in
|
|
89
|
+
# list order, so a per-file loop plus a lookup is redundant work. One fork per
|
|
90
|
+
# pattern instead of one per file (2,779 greps measured before).
|
|
91
|
+
# -H is mandatory: on a single-file list grep omits the path and the caller
|
|
92
|
+
# would parse the line number as the filename.
|
|
93
|
+
# Deliberately duplicated from the mock detector rather than shared: a helper
|
|
94
|
+
# file under tests/ would itself be scanned by Checks 1, 3 and 4 and would
|
|
95
|
+
# change this detector's own finding counts.
|
|
96
|
+
scan_lines() {
|
|
97
|
+
local pattern="$1"; shift
|
|
98
|
+
[ "$#" -gt 0 ] || return 0
|
|
99
|
+
printf '%s\0' "$@" | xargs -0 grep -nHE -- "$pattern" 2>/dev/null || true
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
# Two count streams joined on path by awk. Emits `path:a:b` for every file, so a
|
|
103
|
+
# caller can apply the SAME threshold comparison the per-file bash did -- but
|
|
104
|
+
# with two greps total instead of two per file, and without the `echo | tr -d`
|
|
105
|
+
# laundering (382 forks) that existed only to clean up `grep -c` output.
|
|
106
|
+
count_pairs() {
|
|
107
|
+
local pat_a="$1" pat_b="$2"; shift 2
|
|
108
|
+
[ "$#" -gt 0 ] || return 0
|
|
109
|
+
{
|
|
110
|
+
printf '%s\0' "$@" | xargs -0 grep -cHE -- "$pat_a" 2>/dev/null | sed 's/^/A:/' || true
|
|
111
|
+
printf '%s\0' "$@" | xargs -0 grep -cHE -- "$pat_b" 2>/dev/null | sed 's/^/B:/' || true
|
|
112
|
+
} | awk -F: '
|
|
113
|
+
{
|
|
114
|
+
tag = $1; n = $NF
|
|
115
|
+
path = $2
|
|
116
|
+
for (i = 3; i < NF; i++) path = path ":" $i
|
|
117
|
+
if (tag == "A") a[path] = n; else b[path] = n
|
|
118
|
+
}
|
|
119
|
+
END { for (p in a) print p ":" (a[p]+0) ":" (b[p]+0) }
|
|
120
|
+
' | sort -t: -k1,1
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
# Shell test files, listed once for Checks 1 and 4 (each previously globbed and
|
|
124
|
+
# ran 3 greps per file over the same ~378 files).
|
|
125
|
+
SHELL_TESTS=()
|
|
126
|
+
while IFS= read -r _f; do
|
|
127
|
+
[ -n "$_f" ] && [ -f "$_f" ] && SHELL_TESTS+=("$_f")
|
|
128
|
+
done < <(printf '%s\n' "$PROJECT_DIR"/tests/test-*.sh)
|
|
129
|
+
|
|
130
|
+
# ONE tree walk feeding Checks 2, 3, 5 and 6. Each previously ran its own
|
|
131
|
+
# full-tree `find` (three walks, ~700 ms each under load). The walk below is a
|
|
132
|
+
# strict SUPERSET of all three; the per-check slices immediately after keep each
|
|
133
|
+
# check's own filter, so the distinct sets are preserved exactly:
|
|
134
|
+
# JS_DENSITY Check 2 -- *.test.ts, *.test.js, *.spec.js only
|
|
135
|
+
# PY_DENSITY Check 3 -- test_*.py, with its own exclusion list
|
|
136
|
+
# HARNESS_FILES Checks 5,6 -- the full harness/config/setup set
|
|
137
|
+
# The harness set has the widest name list and its own -path predicates, so it
|
|
138
|
+
# is matched by re-testing each candidate rather than by narrowing the walk.
|
|
139
|
+
_ALL_MUT_FILES=$(find "$PROJECT_DIR" -type f \( \
|
|
140
|
+
-name "*.test.ts" -o -name "*.test.tsx" -o -name "*.test.js" -o -name "*.test.jsx" \
|
|
141
|
+
-o -name "*.spec.ts" -o -name "*.spec.tsx" -o -name "*.spec.js" -o -name "*.spec.jsx" \
|
|
142
|
+
-o -name "test_*.py" \
|
|
143
|
+
-o -name "setupTests.*" -o -name "test-setup.*" \
|
|
144
|
+
-o -name "vitest.setup.*" -o -name "jest.setup.*" \
|
|
145
|
+
-o -name "vitest.config.*" -o -name "jest.config.*" \
|
|
146
|
+
-o -name "setup.ts" -o -name "setup.tsx" -o -name "setup.js" -o -name "setup.jsx" \
|
|
147
|
+
\) 2>/dev/null || true)
|
|
148
|
+
|
|
149
|
+
# Check 1: Shell tests with function redefinitions that mask real behavior
|
|
150
|
+
echo -e "${CYAN}Scanning shell tests for function masking...${NC}"
|
|
151
|
+
# Same pattern and same `> 3` threshold; one grep -cH for the whole set instead
|
|
152
|
+
# of one grep plus an `echo | tr` pair per file.
|
|
153
|
+
if [ "${#SHELL_TESTS[@]}" -gt 0 ]; then
|
|
154
|
+
while IFS=: read -r test_file mask_count; do
|
|
155
|
+
[ -n "$mask_count" ] || continue
|
|
156
|
+
if [ "$mask_count" -gt 3 ]; then
|
|
157
|
+
report "LOW" "${test_file#$PROJECT_DIR/}" "Redefines $mask_count source functions (acceptable for log suppression)"
|
|
158
|
+
fi
|
|
159
|
+
done < <(printf '%s\0' "${SHELL_TESTS[@]}" \
|
|
160
|
+
| xargs -0 grep -cHE -- '^\s*(log_info|log_warn|log_error|log_step|emit_event|emit_learning_signal)\(\)' 2>/dev/null \
|
|
161
|
+
| sort -t: -k1,1 || true)
|
|
162
|
+
fi
|
|
163
|
+
|
|
164
|
+
# Check 2: JS/TS test files with very low assertion density
|
|
165
|
+
echo -e "${CYAN}Scanning for low assertion density...${NC}"
|
|
166
|
+
JS_DENSITY=()
|
|
167
|
+
while IFS= read -r _f; do
|
|
168
|
+
[ -n "$_f" ] && [ -f "$_f" ] && JS_DENSITY+=("$_f")
|
|
169
|
+
done < <(printf '%s\n' "$_ALL_MUT_FILES" | grep -E '\.(test\.ts|test\.js|spec\.js)$' | grep -v node_modules | grep -v dist || true)
|
|
170
|
+
# Thresholds unchanged; count_pairs just supplies both counts from two greps
|
|
171
|
+
# total instead of two per file.
|
|
172
|
+
while IFS=: read -r test_file test_count assert_count; do
|
|
173
|
+
[ -n "$test_count" ] || continue
|
|
174
|
+
if [ "$test_count" -gt 5 ] && [ "$assert_count" -lt "$test_count" ]; then
|
|
175
|
+
report "MEDIUM" "${test_file#$PROJECT_DIR/}" "Low assertion density: $assert_count assertions in $test_count tests (some tests have no assertions)"
|
|
176
|
+
fi
|
|
177
|
+
done < <(count_pairs '(it\(|test\()' '(assert\.|expect\(|should\.)' ${JS_DENSITY[@]+"${JS_DENSITY[@]}"})
|
|
178
|
+
|
|
179
|
+
# Check 3: Python tests with no assertions
|
|
180
|
+
echo -e "${CYAN}Scanning Python tests for missing assertions...${NC}"
|
|
181
|
+
PY_DENSITY=()
|
|
182
|
+
while IFS= read -r _f; do
|
|
183
|
+
[ -n "$_f" ] && [ -f "$_f" ] && PY_DENSITY+=("$_f")
|
|
184
|
+
done < <(printf '%s\n' "$_ALL_MUT_FILES" | grep -E '/test_[^/]*\.py$' | grep -vE '/(node_modules|__pycache__|\.claude|\.loki)/' || true)
|
|
185
|
+
while IFS=: read -r test_file test_count assert_count; do
|
|
186
|
+
[ -n "$test_count" ] || continue
|
|
187
|
+
if [ "$test_count" -gt 3 ] && [ "$assert_count" -lt "$test_count" ]; then
|
|
188
|
+
report "MEDIUM" "${test_file#$PROJECT_DIR/}" "Low assertion density: $assert_count assertions in $test_count tests"
|
|
189
|
+
fi
|
|
190
|
+
done < <(count_pairs '^\s*def test_' '(assert |self\.assert|pytest\.raises|assertEqual|assertTrue|assertFalse|assertRaises|assertIn)' ${PY_DENSITY[@]+"${PY_DENSITY[@]}"})
|
|
191
|
+
|
|
192
|
+
# Check 4: Shell tests with no pass/fail tracking
|
|
193
|
+
echo -e "${CYAN}Scanning shell tests for assertion tracking...${NC}"
|
|
194
|
+
# NOTE: the originals used `grep -c` (BRE with \|), not -E. count_pairs uses
|
|
195
|
+
# -E, so the alternations are written in ERE here -- the SAME alternation, just
|
|
196
|
+
# spelled for the regex dialect in use. `((PASSED` is escaped as `\(\(PASSED`
|
|
197
|
+
# because parens are metacharacters in ERE.
|
|
198
|
+
if [ "${#SHELL_TESTS[@]}" -gt 0 ]; then
|
|
199
|
+
while IFS=: read -r test_file has_pass has_fail; do
|
|
200
|
+
[ -n "$has_pass" ] || continue
|
|
201
|
+
if [ "$has_pass" -eq 0 ] && [ "$has_fail" -eq 0 ]; then
|
|
202
|
+
report "MEDIUM" "${test_file#$PROJECT_DIR/}" "No pass/fail assertion tracking found"
|
|
203
|
+
fi
|
|
204
|
+
done < <(count_pairs 'log_pass|PASSED|\(\(PASSED' 'log_fail|FAILED|\(\(FAILED' "${SHELL_TESTS[@]}")
|
|
205
|
+
fi
|
|
206
|
+
|
|
207
|
+
# HARNESS_INTEGRITY_START
|
|
208
|
+
# Check 5: console and React warning suppression in test harnesses
|
|
209
|
+
echo -e "${CYAN}Scanning test harnesses for hidden console or React act failures...${NC}"
|
|
210
|
+
console_error_re="console[[:space:]]*(\.[[:space:]]*error|\[[[:space:]]*['\"]error['\"][[:space:]]*\])[[:space:]]*=|(spyOn|stub|method|replaceProperty)[[:space:]]*\([[:space:]]*([A-Za-z_\$][A-Za-z0-9_\$]*\.)?console[[:space:]]*,[[:space:]]*['\"]error['\"]|mocked[[:space:]]*\([[:space:]]*console[.]error|defineProperty[[:space:]]*\([[:space:]]*console[[:space:]]*,[[:space:]]*['\"]error['\"]"
|
|
211
|
+
console_warn_re="console[[:space:]]*(\.[[:space:]]*warn|\[[[:space:]]*['\"]warn['\"][[:space:]]*\])[[:space:]]*=|(spyOn|stub|method|replaceProperty)[[:space:]]*\([[:space:]]*([A-Za-z_\$][A-Za-z0-9_\$]*\.)?console[[:space:]]*,[[:space:]]*['\"]warn['\"]"
|
|
212
|
+
HARNESS_FILES=()
|
|
213
|
+
while IFS= read -r _f; do
|
|
214
|
+
[ -n "$_f" ] && [ -f "$_f" ] && HARNESS_FILES+=("$_f")
|
|
215
|
+
done < <(find_js_harness_files)
|
|
216
|
+
|
|
217
|
+
# Each of the four greps below ran PER FILE (4 x 172 = ~700 forks). Each now
|
|
218
|
+
# runs ONCE over the whole set, and `first_hit_table` keeps only the first
|
|
219
|
+
# matching line per path -- the exact effect of the per-file `| head -1`.
|
|
220
|
+
#
|
|
221
|
+
# The if/elif/elif CHAIN AND ITS `continue`s ARE PRESERVED VERBATIM below. That
|
|
222
|
+
# precedence is load-bearing: a file that trips console_error_re must NOT then
|
|
223
|
+
# be tested for the config or act patterns. Evaluating the three conditions
|
|
224
|
+
# independently would double-report and change the finding set.
|
|
225
|
+
first_hit_table() {
|
|
226
|
+
local pattern="$1" ci="$2"; shift 2
|
|
227
|
+
[ "$#" -gt 0 ] || { printf '\n'; return 0; }
|
|
228
|
+
local gflags="-nHE"
|
|
229
|
+
[ "$ci" = "ci" ] && gflags="-nHEi"
|
|
230
|
+
printf '\n'
|
|
231
|
+
printf '%s\0' "$@" | xargs -0 grep $gflags -- "$pattern" 2>/dev/null \
|
|
232
|
+
| awk -F: '!seen[$1]++ { print $1 "\t" $2 }' || true
|
|
233
|
+
}
|
|
234
|
+
_t_err=$(first_hit_table "$console_error_re" "" ${HARNESS_FILES[@]+"${HARNESS_FILES[@]}"})
|
|
235
|
+
_t_cfg=$(first_hit_table '(^|[,{[:space:]])silent[[:space:]]*:[[:space:]]*true|onConsoleLog[[:space:]]*[:(]' "" ${HARNESS_FILES[@]+"${HARNESS_FILES[@]}"})
|
|
236
|
+
_t_act=$(first_hit_table 'not wrapped in (an )?act|React act warning|IS_REACT_ACT_ENVIRONMENT[[:space:]]*=[[:space:]]*false' "ci" ${HARNESS_FILES[@]+"${HARNESS_FILES[@]}"})
|
|
237
|
+
_t_warn=$(first_hit_table "$console_warn_re|onConsoleLog[[:space:]]*[:(]" "" ${HARNESS_FILES[@]+"${HARNESS_FILES[@]}"})
|
|
238
|
+
|
|
239
|
+
# Pull one path's first-hit line number out of a table; empty when absent,
|
|
240
|
+
# which is what `$(grep ... | head -1)` produced for a non-matching file.
|
|
241
|
+
_lookup() {
|
|
242
|
+
local tbl="$1" key="$2" row
|
|
243
|
+
row="${tbl#*$'\n'"$key"$'\t'}"
|
|
244
|
+
[ "$row" != "$tbl" ] || { printf ''; return 0; }
|
|
245
|
+
printf '%s' "${row%%$'\n'*}"
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
for test_file in ${HARNESS_FILES[@]+"${HARNESS_FILES[@]}"}; do
|
|
249
|
+
rel_path="${test_file#$PROJECT_DIR/}"
|
|
250
|
+
hit=$(_lookup "$_t_err" "$test_file")
|
|
251
|
+
if [ -n "$hit" ]; then
|
|
252
|
+
report "HIGH" "$rel_path:${hit}" "Intercepts or mocks console.error, which can hide React errors and act warnings"
|
|
253
|
+
continue
|
|
254
|
+
fi
|
|
255
|
+
|
|
256
|
+
config_hit=$(_lookup "$_t_cfg" "$test_file")
|
|
257
|
+
if [ -n "$config_hit" ] && echo "$rel_path" | grep -qE '(vitest|jest)\.config\.'; then
|
|
258
|
+
report "HIGH" "$rel_path:${config_hit}" "Test config suppresses or filters console output"
|
|
259
|
+
continue
|
|
260
|
+
fi
|
|
261
|
+
|
|
262
|
+
act_hit=$(_lookup "$_t_act" "$test_file")
|
|
263
|
+
warn_hit=$(_lookup "$_t_warn" "$test_file")
|
|
264
|
+
if [ -n "$act_hit" ] && [ -n "$warn_hit" ]; then
|
|
265
|
+
report "HIGH" "$rel_path:${warn_hit}" "Filters React act warnings from console output"
|
|
266
|
+
fi
|
|
267
|
+
done
|
|
268
|
+
|
|
269
|
+
# Check 6: optional lookup guarded assertions that can execute zero assertions
|
|
270
|
+
echo -e "${CYAN}Scanning for vacuous conditional UI assertions...${NC}"
|
|
271
|
+
# Check 6 keeps its per-file grep and its awk window body verbatim. Its awk
|
|
272
|
+
# fires ~37 times on this repo and the discovery grep ~172 -- both noise next to
|
|
273
|
+
# the ~2,700 forks removed from Checks 1-5, and leaving it untouched means there
|
|
274
|
+
# is less behavior to re-prove. Only the file list is now the shared one.
|
|
275
|
+
# The discovery grep is batched into ONE stream of `path:lineno:text` (it ran
|
|
276
|
+
# once per harness file, 172 forks). The awk window body below is untouched --
|
|
277
|
+
# it fires ~37 times, which is noise, and leaving it alone means less behavior
|
|
278
|
+
# to re-prove. Streaming also removes the outer per-file loop: the path now
|
|
279
|
+
# arrives on each row.
|
|
280
|
+
_C6_STREAM=""
|
|
281
|
+
if [ "${#HARNESS_FILES[@]}" -gt 0 ]; then
|
|
282
|
+
_C6_STREAM=$(printf '%s\0' "${HARNESS_FILES[@]}" | xargs -0 grep -nHE -- '(const|let|var)[[:space:]]+[A-Za-z_$][A-Za-z0-9_$]*[^=]*=.*((local|session)Storage[.]getItem|query(By[A-Za-z]+)?[[:space:]]*\(|querySelector[[:space:]]*\(|getElementById[[:space:]]*\(|boundingBox[[:space:]]*\()' 2>/dev/null || true)
|
|
283
|
+
fi
|
|
284
|
+
while IFS=: read -r test_file assign_line source_line; do
|
|
285
|
+
[ -n "$assign_line" ] || continue
|
|
286
|
+
[ -f "$test_file" ] || continue
|
|
287
|
+
rel_path="${test_file#$PROJECT_DIR/}"
|
|
288
|
+
if true; then
|
|
289
|
+
[ -n "$assign_line" ] || continue
|
|
290
|
+
guarded_var=$(echo "$source_line" | sed -E 's/.*(const|let|var)[[:space:]]+([A-Za-z_$][A-Za-z0-9_$]*).*/\2/')
|
|
291
|
+
guard_line=$(awk -v s="$assign_line" -v e="$((assign_line + 40))" -v v="$guarded_var" '
|
|
292
|
+
NR <= s || NR > e { next }
|
|
293
|
+
{
|
|
294
|
+
compact=$0
|
|
295
|
+
gsub(/[[:space:]]/, "", compact)
|
|
296
|
+
if (index(compact, "if(" v ")") || index(compact, "if(" v "!==null)") ||
|
|
297
|
+
index(compact, "if(" v "!=null)") || index(compact, v "&&expect(")) {
|
|
298
|
+
print NR
|
|
299
|
+
exit
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
' "$test_file")
|
|
303
|
+
[ -n "$guard_line" ] || continue
|
|
304
|
+
|
|
305
|
+
presence_check=$(awk -v s="$assign_line" -v e="$guard_line" -v v="$guarded_var" '
|
|
306
|
+
NR > s && NR < e && index($0, v) &&
|
|
307
|
+
$0 ~ /(not[.]toBeNull|toBeTruthy|toBeDefined|assert[.](ok|notEqual))/ { print NR; exit }
|
|
308
|
+
' "$test_file")
|
|
309
|
+
[ -z "$presence_check" ] || continue
|
|
310
|
+
|
|
311
|
+
assertion_line=$(awk -v s="$guard_line" -v e="$((guard_line + 10))" '
|
|
312
|
+
NR >= s && NR <= e && $0 ~ /(expect|assert)[[:space:]]*\(/ { print NR; exit }
|
|
313
|
+
' "$test_file")
|
|
314
|
+
if [ -n "$assertion_line" ]; then
|
|
315
|
+
report "HIGH" "$rel_path:$guard_line" "Required assertion is conditional on optional lookup '$guarded_var' and can silently skip"
|
|
316
|
+
fi
|
|
317
|
+
fi
|
|
318
|
+
done <<< "$_C6_STREAM"
|
|
319
|
+
# HARNESS_INTEGRITY_END
|
|
320
|
+
|
|
321
|
+
# Check 7: Assertion value mutations in git commits
|
|
322
|
+
# Detects when a commit changes BOTH implementation code AND assertion expected values
|
|
323
|
+
# This is a sign of "fitting the test to the code" -- changing what the test expects
|
|
324
|
+
# to match what the code produces, rather than fixing the code
|
|
325
|
+
echo -e "${CYAN}Scanning for assertion value mutations in commits...${NC}"
|
|
326
|
+
|
|
327
|
+
# Use provided commit or check the last 5 commits
|
|
328
|
+
if [ -n "$COMMIT_HASH" ]; then
|
|
329
|
+
COMMITS_TO_CHECK="$COMMIT_HASH"
|
|
330
|
+
else
|
|
331
|
+
COMMITS_TO_CHECK=$(cd "$PROJECT_DIR" && git log --oneline -5 --format='%H' 2>/dev/null || true)
|
|
332
|
+
fi
|
|
333
|
+
|
|
334
|
+
if [ -n "$COMMITS_TO_CHECK" ]; then
|
|
335
|
+
for commit in $COMMITS_TO_CHECK; do
|
|
336
|
+
# Get files changed in this commit
|
|
337
|
+
changed_files=$(cd "$PROJECT_DIR" && git diff-tree --no-commit-id --name-only -r "$commit" 2>/dev/null || true)
|
|
338
|
+
[ -z "$changed_files" ] && continue
|
|
339
|
+
|
|
340
|
+
# Classify files: test files vs implementation files
|
|
341
|
+
has_impl=false
|
|
342
|
+
has_test=false
|
|
343
|
+
test_files_changed=""
|
|
344
|
+
impl_files_changed=""
|
|
345
|
+
|
|
346
|
+
while IFS= read -r file; do
|
|
347
|
+
# Classification is pure glob matching on a filename, so it is done
|
|
348
|
+
# with `case` instead of `echo | grep -qE` -- that pipeline cost TWO
|
|
349
|
+
# forks per changed file (137 subprocesses across 5 commits) to
|
|
350
|
+
# answer a question the shell answers natively. The alternations
|
|
351
|
+
# below are the same ones the two regexes expressed, enumerated:
|
|
352
|
+
# test: *.test.*/*.spec.* in ts|js|tsx|jsx, a leading test/ or
|
|
353
|
+
# tests/ path, test_*.py, and the vitest|jest|playwright|
|
|
354
|
+
# cypress config / vitest|jest setup files (repo root or
|
|
355
|
+
# any subdirectory).
|
|
356
|
+
# impl: any remaining .ts .js .tsx .jsx .py .sh
|
|
357
|
+
_is_test=false
|
|
358
|
+
case "$file" in
|
|
359
|
+
*.test.ts|*.test.js|*.test.tsx|*.test.jsx|\
|
|
360
|
+
*.spec.ts|*.spec.js|*.spec.tsx|*.spec.jsx|\
|
|
361
|
+
tests/*|test/*|\
|
|
362
|
+
test_*.py|*/test_*.py|*test_*.py|\
|
|
363
|
+
vitest.config.ts|vitest.config.js|vitest.config.mjs|vitest.config.cjs|\
|
|
364
|
+
jest.config.ts|jest.config.js|jest.config.mjs|jest.config.cjs|\
|
|
365
|
+
playwright.config.ts|playwright.config.js|playwright.config.mjs|playwright.config.cjs|\
|
|
366
|
+
cypress.config.ts|cypress.config.js|cypress.config.mjs|cypress.config.cjs|\
|
|
367
|
+
*/vitest.config.ts|*/vitest.config.js|*/vitest.config.mjs|*/vitest.config.cjs|\
|
|
368
|
+
*/jest.config.ts|*/jest.config.js|*/jest.config.mjs|*/jest.config.cjs|\
|
|
369
|
+
*/playwright.config.ts|*/playwright.config.js|*/playwright.config.mjs|*/playwright.config.cjs|\
|
|
370
|
+
*/cypress.config.ts|*/cypress.config.js|*/cypress.config.mjs|*/cypress.config.cjs|\
|
|
371
|
+
vitest.setup.ts|vitest.setup.js|vitest.setup.tsx|vitest.setup.jsx|\
|
|
372
|
+
jest.setup.ts|jest.setup.js|jest.setup.tsx|jest.setup.jsx|\
|
|
373
|
+
*/vitest.setup.ts|*/vitest.setup.js|*/vitest.setup.tsx|*/vitest.setup.jsx|\
|
|
374
|
+
*/jest.setup.ts|*/jest.setup.js|*/jest.setup.tsx|*/jest.setup.jsx)
|
|
375
|
+
_is_test=true ;;
|
|
376
|
+
esac
|
|
377
|
+
if [ "$_is_test" = true ]; then
|
|
378
|
+
has_test=true
|
|
379
|
+
test_files_changed="$test_files_changed $file"
|
|
380
|
+
elif case "$file" in *.ts|*.js|*.tsx|*.jsx|*.py|*.sh) true ;; *) false ;; esac; then
|
|
381
|
+
# Implementation source file. The broken `grep -q ... | grep -vq`
|
|
382
|
+
# pipe that previously gated this branch always evaluated false
|
|
383
|
+
# (grep -q emits no stdout, so the piped grep saw empty input and
|
|
384
|
+
# exited 1), which left has_impl permanently false and made the
|
|
385
|
+
# entire HIGH commit-mutation path dead. The .md/.json/.yml
|
|
386
|
+
# extensions cannot match the .ts/.js/... pattern above, so the
|
|
387
|
+
# exclusion grep was redundant and has been removed.
|
|
388
|
+
has_impl=true
|
|
389
|
+
impl_files_changed="$impl_files_changed $file"
|
|
390
|
+
fi
|
|
391
|
+
done <<< "$changed_files"
|
|
392
|
+
|
|
393
|
+
# Only flag if BOTH test and implementation files changed in same commit.
|
|
394
|
+
# New test files are not mutations. They have no prior assertions to
|
|
395
|
+
# weaken, and blocking them punishes greenfield projects for adding real
|
|
396
|
+
# coverage alongside their first implementation.
|
|
397
|
+
if [ "$has_impl" = true ] && [ "$has_test" = true ]; then
|
|
398
|
+
modified_test_files=""
|
|
399
|
+
for test_file in $test_files_changed; do
|
|
400
|
+
if (cd "$PROJECT_DIR" \
|
|
401
|
+
&& git cat-file -e "${commit}^:${test_file}" 2>/dev/null \
|
|
402
|
+
&& git cat-file -e "${commit}:${test_file}" 2>/dev/null); then
|
|
403
|
+
modified_test_files="$modified_test_files $test_file"
|
|
404
|
+
fi
|
|
405
|
+
done
|
|
406
|
+
[ -z "$modified_test_files" ] && continue
|
|
407
|
+
|
|
408
|
+
# A real expectation mutation has both a removed assertion and an
|
|
409
|
+
# added replacement assertion. Counting additions alone confused
|
|
410
|
+
# expanded coverage with test fitting. Require at least three paired
|
|
411
|
+
# replacements to preserve the existing high-confidence threshold.
|
|
412
|
+
test_diff=$(cd "$PROJECT_DIR" && git diff "$commit^" "$commit" -- $modified_test_files 2>/dev/null || true)
|
|
413
|
+
removed_assertions=$(echo "$test_diff" | grep -E '^-[^-].*(\.toBe\(|\.toEqual\(|\.toStrictEqual\(|strictEqual\(|deepEqual\(|assertEqual\(|assert.*==)' 2>/dev/null | wc -l | tr -d '[:space:]')
|
|
414
|
+
added_assertions=$(echo "$test_diff" | grep -E '^\+[^+].*(\.toBe\(|\.toEqual\(|\.toStrictEqual\(|strictEqual\(|deepEqual\(|assertEqual\(|assert.*==)' 2>/dev/null | wc -l | tr -d '[:space:]')
|
|
415
|
+
removed_assertions="${removed_assertions:-0}"
|
|
416
|
+
added_assertions="${added_assertions:-0}"
|
|
417
|
+
changed_assertions="$removed_assertions"
|
|
418
|
+
if [ "$added_assertions" -lt "$changed_assertions" ]; then
|
|
419
|
+
changed_assertions="$added_assertions"
|
|
420
|
+
fi
|
|
421
|
+
|
|
422
|
+
if [ "$changed_assertions" -gt 2 ]; then
|
|
423
|
+
short_hash=$(echo "$commit" | cut -c1-8)
|
|
424
|
+
report "HIGH" "commit:$short_hash" "Replaced $changed_assertions assertion values alongside implementation code -- possible test fitting"
|
|
425
|
+
fi
|
|
426
|
+
fi
|
|
427
|
+
done
|
|
428
|
+
fi
|
|
429
|
+
|
|
430
|
+
# Summary
|
|
431
|
+
echo ""
|
|
432
|
+
echo "=========================================="
|
|
433
|
+
echo "Results: $FINDINGS finding(s)"
|
|
434
|
+
echo "=========================================="
|
|
435
|
+
|
|
436
|
+
echo " HIGH: $HIGH_FINDINGS"
|
|
437
|
+
|
|
438
|
+
# --strict takes precedence: block on ANY finding (legacy CI behavior, unchanged).
|
|
439
|
+
if [ "$STRICT" = "--strict" ] && [ $FINDINGS -gt 0 ]; then
|
|
440
|
+
echo -e "${RED}GATE FAILED: $FINDINGS finding(s)${NC}"
|
|
441
|
+
exit 1
|
|
442
|
+
fi
|
|
443
|
+
|
|
444
|
+
# --block-high: exit 2 only when HIGH-severity findings are present. MEDIUM/LOW
|
|
445
|
+
# do not block (they are routed to the findings injector by the run.sh wrapper).
|
|
446
|
+
if [ "$BLOCK_HIGH" = "--block-high" ] && [ $HIGH_FINDINGS -gt 0 ]; then
|
|
447
|
+
echo -e "${RED}GATE FAILED: $HIGH_FINDINGS HIGH-severity finding(s)${NC}"
|
|
448
|
+
exit 2
|
|
449
|
+
fi
|
|
450
|
+
|
|
451
|
+
if [ $FINDINGS -eq 0 ]; then
|
|
452
|
+
echo -e "${GREEN}All tests pass mutation detection gate.${NC}"
|
|
453
|
+
fi
|
|
454
|
+
|
|
455
|
+
exit 0
|