@boyingliu01/xp-gate 0.35.10 → 0.35.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hooks/pre-commit +131 -5
- package/lib/__tests__/coverage-scope.test.js +140 -0
- package/lib/__tests__/slo.test.js +89 -0
- package/lib/mattpocock-license-verdict.json +1 -1
- package/lib/retro.js +84 -0
- package/lib/slo.js +150 -0
- package/mock-policy/AGENTS.md +2 -2
- package/mutation/AGENTS.md +2 -2
- package/package.json +1 -1
- package/plugins/claude-code/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/skills/delphi-review/AGENTS.md +2 -2
- package/plugins/claude-code/skills/sprint-flow/AGENTS.md +2 -2
- package/plugins/claude-code/skills/test-specification-alignment/AGENTS.md +2 -2
- package/plugins/dsh/lib/index.d.ts +21 -0
- package/plugins/dsh/lib/index.d.ts.map +1 -1
- package/plugins/dsh/lib/index.js +23 -5
- package/plugins/dsh/package.json +1 -1
- package/plugins/dsh/skills/delphi-review/AGENTS.md +2 -2
- package/plugins/dsh/skills/sprint-flow/AGENTS.md +2 -2
- package/plugins/dsh/skills/test-specification-alignment/AGENTS.md +2 -2
- package/plugins/dsh/src/index.test.ts +18 -1
- package/plugins/dsh/src/index.ts +24 -5
- package/plugins/opencode/index.ts +8 -4
- package/plugins/opencode/package.json +1 -1
- package/plugins/opencode/semver.test.ts +64 -0
- package/plugins/opencode/skills/delphi-review/AGENTS.md +2 -2
- package/plugins/opencode/skills/sprint-flow/AGENTS.md +2 -2
- package/plugins/opencode/skills/test-specification-alignment/AGENTS.md +2 -2
- package/plugins/qoder/plugin.json +1 -1
- package/plugins/qoder/skills/delphi-review/AGENTS.md +2 -2
- package/plugins/qoder/skills/sprint-flow/AGENTS.md +2 -2
- package/plugins/qoder/skills/test-specification-alignment/AGENTS.md +2 -2
- package/principles/AGENTS.md +2 -2
- package/skills/delphi-review/AGENTS.md +2 -2
- package/skills/sprint-flow/AGENTS.md +2 -2
- package/skills/test-specification-alignment/AGENTS.md +2 -2
package/hooks/pre-commit
CHANGED
|
@@ -36,6 +36,30 @@ _LE_CR=$(tr -cd '\r' < "$0" | wc -c | tr -d '[:space:]'); test "${_LE_CR:-0}" -e
|
|
|
36
36
|
# 1. Global: ~/.config/xp-gate/adapters (for core.hooksPath global setup)
|
|
37
37
|
# 2. Project-local: <repo>/githooks/ (for per-project init)
|
|
38
38
|
# 3. Script dir: hooks directory containing this script (fallback)
|
|
39
|
+
# ── #640: wall-clock start, written into the per-run report ──────────────────
|
|
40
|
+
# The gate journal records every gate's verdict but no timing, so a slow commit
|
|
41
|
+
# could not be attributed: the #640 measurement found ~11s per commit outside any
|
|
42
|
+
# gate and had no instrument to say where it went. One timestamp at entry and one
|
|
43
|
+
# subtraction at report-write time turns "it feels slow" into a number.
|
|
44
|
+
#
|
|
45
|
+
# Milliseconds come from /proc/uptime when readable (bash's %3N is not portable to
|
|
46
|
+
# the bash 3.2 / MSYS builds this hook supports); otherwise whole seconds, which
|
|
47
|
+
# is coarse but still separates "gates are slow" from "the hook around them is".
|
|
48
|
+
_XP_HOOK_START_MS=""
|
|
49
|
+
if [ -r /proc/uptime ]; then
|
|
50
|
+
_xp_up=$(cut -d' ' -f1 /proc/uptime 2>/dev/null || echo "")
|
|
51
|
+
case "$_xp_up" in
|
|
52
|
+
*.*) _XP_HOOK_START_MS=$(printf '%s%s' "${_xp_up%%.*}" "$(printf '%s' "${_xp_up#*.}" | cut -c1-3)") ;;
|
|
53
|
+
esac
|
|
54
|
+
fi
|
|
55
|
+
case "$_XP_HOOK_START_MS" in
|
|
56
|
+
''|*[!0-9]*) _XP_HOOK_START_MS=$(( $(date +%s) * 1000 )) ;;
|
|
57
|
+
esac
|
|
58
|
+
export _XP_HOOK_START_MS
|
|
59
|
+
# #609: 1-based attempt index for this hook run. One hook invocation is one attempt,
|
|
60
|
+
# so every gate recorded during it shares the same index. Left empty by default so a
|
|
61
|
+
# caller that genuinely does not know the index omits the field instead of guessing.
|
|
62
|
+
_XP_GATE_ATTEMPT=""
|
|
39
63
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
40
64
|
ADAPTER_DIR=""
|
|
41
65
|
GLOBAL_ADAPTER_DIR="$HOME/.config/xp-gate/adapters"
|
|
@@ -76,8 +100,16 @@ fi
|
|
|
76
100
|
# and a plausible wrong number is harder to notice than a visible gap.
|
|
77
101
|
#
|
|
78
102
|
# Resolve from the repo's own declaration, in the order the tools themselves
|
|
79
|
-
# use: pyproject [tool.coverage.report] fail_under, then .coveragerc
|
|
80
|
-
# declaring
|
|
103
|
+
# use: pyproject [tool.coverage.report] fail_under, then .coveragerc, then the
|
|
104
|
+
# JS/TS runner's own threshold block. A repo declaring none keeps the historical 80.
|
|
105
|
+
#
|
|
106
|
+
# #605: the JS/TS branch below enforces THIS number, but the resolver only ever
|
|
107
|
+
# read Python declarations -- so a vitest project declaring `thresholds.lines: 60`
|
|
108
|
+
# was silently judged against 80. The result was the worst possible split:
|
|
109
|
+
# `npx vitest run` goes green, the commit is then rejected, and the two numbers
|
|
110
|
+
# are read from different files with nothing pointing at the disagreement.
|
|
111
|
+
# Same defect shape as #554/#609: a declaration exists, an enforcement exists,
|
|
112
|
+
# and nothing connected them.
|
|
81
113
|
_resolve_coverage_threshold() {
|
|
82
114
|
_rct_value=""
|
|
83
115
|
_rct_src="default"
|
|
@@ -93,13 +125,36 @@ _resolve_coverage_threshold() {
|
|
|
93
125
|
_rct_value=$(grep -oE '^[[:space:]]*fail_under[[:space:]]*=[[:space:]]*[0-9]+' .coveragerc 2>/dev/null | grep -oE '[0-9]+' | head -1 || true)
|
|
94
126
|
[ -n "$_rct_value" ] && _rct_src=".coveragerc fail_under"
|
|
95
127
|
fi
|
|
128
|
+
# #605: JS/TS. Read the runner's own threshold rather than assuming 80, so the
|
|
129
|
+
# hook enforces the number the project actually declares. `lines` specifically:
|
|
130
|
+
# it is the metric the JS branch below compares, and it is the one vitest docs
|
|
131
|
+
# lead with. Only the FLAT shape is read -- the nested `thresholds: { global }`
|
|
132
|
+
# form is vitest 2.x and was proven silently dropped by the 1.6.x config parser
|
|
133
|
+
# (#561), so honouring it here would enforce a number that never takes effect.
|
|
134
|
+
if [ -z "$_rct_value" ]; then
|
|
135
|
+
for _rct_cfg in vitest.config.ts vitest.config.js vitest.config.mts; do
|
|
136
|
+
[ -f "$_rct_cfg" ] || continue
|
|
137
|
+
_rct_value=$(sed -n '/thresholds[[:space:]]*:[[:space:]]*{/,/}/p' "$_rct_cfg" 2>/dev/null \
|
|
138
|
+
| grep -oE 'lines[[:space:]]*:[[:space:]]*[0-9]+' | grep -oE '[0-9]+' | head -1 || true)
|
|
139
|
+
if [ -n "$_rct_value" ]; then
|
|
140
|
+
_rct_src="$_rct_cfg coverage.thresholds.lines"
|
|
141
|
+
break
|
|
142
|
+
fi
|
|
143
|
+
done
|
|
144
|
+
fi
|
|
96
145
|
case "$_rct_value" in
|
|
97
|
-
''|*[!0-9]*) _rct_value="80"; _rct_src="default (no
|
|
146
|
+
''|*[!0-9]*) _rct_value="80"; _rct_src="default (no coverage threshold declared)" ;;
|
|
98
147
|
esac
|
|
99
148
|
COVERAGE_THRESHOLD_SOURCE="$_rct_src"
|
|
100
149
|
printf '%s' "$_rct_value"
|
|
101
150
|
}
|
|
102
|
-
COVERAGE_THRESHOLD="$(_resolve_coverage_threshold)"
|
|
151
|
+
# NOT `COVERAGE_THRESHOLD="$(_resolve_coverage_threshold)"`: command substitution
|
|
152
|
+
# runs in a subshell, so `COVERAGE_THRESHOLD_SOURCE` set inside the function was
|
|
153
|
+
# discarded and the audit record's `threshold_source` field was empty on every
|
|
154
|
+
# single commit. A field that is always blank is worse than a missing one -- it
|
|
155
|
+
# looks populated and says nothing. Call the function in this shell instead.
|
|
156
|
+
_resolve_coverage_threshold >/dev/null
|
|
157
|
+
COVERAGE_THRESHOLD="$_rct_value"
|
|
103
158
|
|
|
104
159
|
# Look up one tier's directory variable by name without eval-ing arbitrary input:
|
|
105
160
|
# the base names are a closed set baked into the generated file.
|
|
@@ -578,6 +633,41 @@ record_gate_audit() {
|
|
|
578
633
|
local detail="${6:-}"
|
|
579
634
|
local end_ms
|
|
580
635
|
end_ms=$(now_ms)
|
|
636
|
+
# #609: attempt index for this gate.
|
|
637
|
+
#
|
|
638
|
+
# Two failed designs are recorded here so they are not retried:
|
|
639
|
+
#
|
|
640
|
+
# 1. Keying on commit_hash -- the pre-commit record is STAGED with an empty
|
|
641
|
+
# commit_hash (the commit does not exist yet, see stageAuditEntry), so the
|
|
642
|
+
# lookup always matched zero and every gate stayed at attempt 1.
|
|
643
|
+
# 2. Counting every record in a time window -- measured attempt=63 for gate-1
|
|
644
|
+
# over one working session, because unrelated SUCCESSFUL commits accumulate.
|
|
645
|
+
# A retry index that counts successes is not a retry index.
|
|
646
|
+
#
|
|
647
|
+
# What actually defines a retry: the previous record for this gate FAILED. So the
|
|
648
|
+
# index is (consecutive failures immediately before this run) + 1. A clean history
|
|
649
|
+
# is attempt 1; fail, fix, re-commit gives 2; three failures in a row gives 4.
|
|
650
|
+
#
|
|
651
|
+
# Read failure leaves the value empty -> the writer omits the field rather than
|
|
652
|
+
# recording a fabricated 1, which would inflate the first-pass denominator.
|
|
653
|
+
if [ -z "${_XP_GATE_ATTEMPT:-}" ]; then
|
|
654
|
+
local _failed_streak
|
|
655
|
+
if [ -f .xp-gate/audit.jsonl ]; then
|
|
656
|
+
_failed_streak=$(awk -v g="\"gate_id\":\"${gate_id}\"" '
|
|
657
|
+
index($0, g) {
|
|
658
|
+
# passed is a JSON boolean; "false" is the only failure spelling.
|
|
659
|
+
if (index($0, "\"passed\":false") > 0) { streak++ }
|
|
660
|
+
else if (index($0, "\"passed\":true") > 0) { streak = 0 }
|
|
661
|
+
}
|
|
662
|
+
END { print streak + 0 }' .xp-gate/audit.jsonl 2>/dev/null | tr -d ' ')
|
|
663
|
+
case "$_failed_streak" in
|
|
664
|
+
''|*[!0-9]*) _failed_streak="" ;;
|
|
665
|
+
esac
|
|
666
|
+
if [ -n "$_failed_streak" ]; then
|
|
667
|
+
_XP_GATE_ATTEMPT=$(( _failed_streak + 1 ))
|
|
668
|
+
fi
|
|
669
|
+
fi
|
|
670
|
+
fi
|
|
581
671
|
# REQ-B: sanitize BEFORE subtracting. A poisoned start (empty -> arithmetic
|
|
582
672
|
# 0, or a seconds-unit value from an old code path) used to survive as an
|
|
583
673
|
# epoch-sized duration_ms that only the anomaly flag papered over. Now the
|
|
@@ -613,6 +703,13 @@ record_gate_audit() {
|
|
|
613
703
|
fi
|
|
614
704
|
|
|
615
705
|
# Fire-and-forget: audit must never block the commit
|
|
706
|
+
#
|
|
707
|
+
# #609: `--attempt` was implemented in gate-audit.ts (#603) but never passed by
|
|
708
|
+
# this hook -- verified against 9212 real journal entries, none carried `attempt`.
|
|
709
|
+
# The 1-based attempt index makes the first-build / final-build / first-test SLOs
|
|
710
|
+
# computable; without it "passed" cannot separate a clean run from one that only
|
|
711
|
+
# passed after a re-run, and every such metric is defined to be 100%. Callers that
|
|
712
|
+
# do not know the index stay absent (== 1) rather than being stamped with a guess.
|
|
616
713
|
{
|
|
617
714
|
run_tsx "${PROJECT_ROOT_FOR_AUDIT}/src/npm-package/lib/gate-audit.ts" record \
|
|
618
715
|
--gate-id "$gate_id" \
|
|
@@ -623,6 +720,7 @@ record_gate_audit() {
|
|
|
623
720
|
$duration_anomaly_flag \
|
|
624
721
|
$duration_reason_flag \
|
|
625
722
|
--trigger commit \
|
|
723
|
+
${_XP_GATE_ATTEMPT:+--attempt "$_XP_GATE_ATTEMPT"} \
|
|
626
724
|
${detail:+--detail "$detail"} \
|
|
627
725
|
2>/dev/null
|
|
628
726
|
} &
|
|
@@ -5071,7 +5169,35 @@ ENDJSON
|
|
|
5071
5169
|
REPORT_TS=$(date -u +"%Y-%m-%d-%H%M%S")
|
|
5072
5170
|
local PERSISTED_REPORT_FILE="${REPORTS_DIR}/${REPORT_TS}.json"
|
|
5073
5171
|
mkdir -p "$REPORTS_DIR"
|
|
5074
|
-
|
|
5172
|
+
# #640: `duration_ms` is wall-clock time from hook entry to report write. It is
|
|
5173
|
+
# the number the issue could not get: gate verdicts were journaled but durations
|
|
5174
|
+
# were not, so the ~11s observed outside any gate had no instrument pointing at
|
|
5175
|
+
# it. Emitted only when measurable -- an absent field is honest, a 0 is not.
|
|
5176
|
+
#
|
|
5177
|
+
# The subtraction happens in BASH, not in the node writer. Measured: on MSYS
|
|
5178
|
+
# (Windows) `/proc/uptime` is visible to bash but NOT to native node --
|
|
5179
|
+
# `readFileSync('/proc/uptime')` there throws ENOENT. Computing the elapsed time
|
|
5180
|
+
# in node therefore passed a bash-clock start into a reader that could not produce
|
|
5181
|
+
# a comparable "now", and the field silently never appeared. Both ends of a
|
|
5182
|
+
# duration must come from the same clock.
|
|
5183
|
+
local _XP_DURATION_MS=""
|
|
5184
|
+
local _XP_NOW_MS=""
|
|
5185
|
+
if [ -r /proc/uptime ]; then
|
|
5186
|
+
_xp_up2=$(cut -d' ' -f1 /proc/uptime 2>/dev/null || echo "")
|
|
5187
|
+
case "$_xp_up2" in
|
|
5188
|
+
*.*) _XP_NOW_MS=$(printf '%s%s' "${_xp_up2%%.*}" "$(printf '%s' "${_xp_up2#*.}" | cut -c1-3)") ;;
|
|
5189
|
+
esac
|
|
5190
|
+
fi
|
|
5191
|
+
case "$_XP_NOW_MS" in
|
|
5192
|
+
''|*[!0-9]*) _XP_NOW_MS="" ;;
|
|
5193
|
+
esac
|
|
5194
|
+
case "$_XP_HOOK_START_MS" in
|
|
5195
|
+
''|*[!0-9]*) _XP_HOOK_START_MS="" ;;
|
|
5196
|
+
esac
|
|
5197
|
+
if [ -n "$_XP_NOW_MS" ] && [ -n "$_XP_HOOK_START_MS" ] && [ "$_XP_NOW_MS" -gt "$_XP_HOOK_START_MS" ]; then
|
|
5198
|
+
_XP_DURATION_MS=$(( _XP_NOW_MS - _XP_HOOK_START_MS ))
|
|
5199
|
+
fi
|
|
5200
|
+
_XP_RF="$REPORT_FILE" _XP_PRF="$PERSISTED_REPORT_FILE" _XP_TS="$TIMESTAMP" _XP_BR="$BRANCH" _XP_CH="$COMMIT_HASH" _XP_CF="${CHANGED_FILES:-}" _XP_DUR="$_XP_DURATION_MS" node -e "const fs=require('fs');const e=process.env;const br=JSON.parse(fs.readFileSync(e._XP_RF,'utf8'));const cf=(e._XP_CF||'').split('\n').filter(f=>f);const r={timestamp:e._XP_TS,trigger:'pre-commit',branch:e._XP_BR,commit:e._XP_CH,changed_files:cf,overall:br.overall||{},gates:br.gates||{},warnings:[],errors:[]};const d=Number(e._XP_DUR);if(Number.isFinite(d)&&d>0){r.duration_ms=d;}fs.writeFileSync(e._XP_PRF,JSON.stringify(r,null,2)+'\n');" 2>/dev/null || cp "$REPORT_FILE" "$PERSISTED_REPORT_FILE"
|
|
5075
5201
|
|
|
5076
5202
|
# ── History append (unchanged — append-only for trend) ────────────────────────────
|
|
5077
5203
|
local GATES_JSON="{"
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @test REQ-562 覆盖率核算范围必须覆盖真实被测代码,不能靠排除把分数养高
|
|
3
|
+
* @intent #562 追踪把 lines/statements 从 60% 提到 80%。它建议的第一步是
|
|
4
|
+
* "把 `plugins/**` 的排除去掉",理由是那里有 801 个测试被隐藏。
|
|
5
|
+
*
|
|
6
|
+
* **那条理由已经不成立了**:28 个插件测试文件、225 个测试**本来就在跑**,
|
|
7
|
+
* 全绿。`plugins/**` 只出现在 `coverage.exclude` 里,不在 `test.exclude` 里。
|
|
8
|
+
*
|
|
9
|
+
* 真正没被修的是另一半:`coverage.include` 曾是 `['src/**']`,所以
|
|
10
|
+
* `'plugins/**'` 这条排除项**从来没有起过作用**——plugins 源码压根不在
|
|
11
|
+
* 核算范围内,光靠 include 就被排除了。结果是 4340 行、30 个文件、
|
|
12
|
+
* 被 225 个通过的测试覆盖着的规范源代码,**从未计入覆盖率**。
|
|
13
|
+
*
|
|
14
|
+
* 隔离测量这些源文件是 **83.8% lines**,远高于仓库 60% 的地板。
|
|
15
|
+
* "把一个覆盖良好的目录排除在平均分之外"是覆盖率数字从"描述代码库"
|
|
16
|
+
* 退化成"描述配置"的典型方式。
|
|
17
|
+
*
|
|
18
|
+
* 这里的断言是两条**属性**,不是某个 glob 的字符串:
|
|
19
|
+
* 1. 有测试在跑的源码必须被计入覆盖率核算。
|
|
20
|
+
* 2. exclude 里的每一条都必须在 include 的范围内,否则它是死配置——
|
|
21
|
+
* 读起来像"我们考虑过",实际什么都没做。
|
|
22
|
+
* @covers AC-562-01, AC-562-02
|
|
23
|
+
*/
|
|
24
|
+
'use strict';
|
|
25
|
+
|
|
26
|
+
const fs = require('node:fs');
|
|
27
|
+
const path = require('node:path');
|
|
28
|
+
|
|
29
|
+
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
|
30
|
+
const CONFIG = path.join(REPO_ROOT, 'vitest.config.ts');
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Read the `coverage:` block and return its include/exclude globs.
|
|
34
|
+
*
|
|
35
|
+
* Comment lines are blanked first: this config explains its own history by quoting
|
|
36
|
+
* old patterns in prose (the same trap recorded in gate-coverage.js for #605), and
|
|
37
|
+
* a naive read treats a quoted example as live configuration.
|
|
38
|
+
*/
|
|
39
|
+
function readCoverageGlobs() {
|
|
40
|
+
let raw;
|
|
41
|
+
try {
|
|
42
|
+
raw = fs.readFileSync(CONFIG, 'utf8').replace(/^\s*\/\/.*$/gm, '');
|
|
43
|
+
} catch (err) {
|
|
44
|
+
throw new Error(`cannot read vitest.config.ts: ${err.message}`);
|
|
45
|
+
}
|
|
46
|
+
const start = raw.indexOf('coverage: {');
|
|
47
|
+
if (start === -1) throw new Error('coverage block not found in vitest.config.ts');
|
|
48
|
+
const block = raw.slice(start, raw.indexOf('\n },', start));
|
|
49
|
+
const pick = (key) => {
|
|
50
|
+
const m = block.match(new RegExp(`${key}:\\s*\\[([\\s\\S]*?)\\]`));
|
|
51
|
+
if (!m) return [];
|
|
52
|
+
return [...m[1].matchAll(/'([^']+)'/g)].map((x) => x[1]);
|
|
53
|
+
};
|
|
54
|
+
return { include: pick('include'), exclude: pick('exclude') };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Trees that legitimately hold tests but are not measured product source:
|
|
59
|
+
* build sandboxes, dependencies, and tooling. Excluding these is not the defect
|
|
60
|
+
* #562 describes -- excluding a product tree that tests already exercise is.
|
|
61
|
+
*/
|
|
62
|
+
const NOT_PRODUCT_TREES = [
|
|
63
|
+
'node_modules', '.stryker-tmp', '.worktrees', '.git', 'coverage', 'dist',
|
|
64
|
+
'scripts', 'skills', 'githooks', 'dashboard', 'compat', '.xp-gate',
|
|
65
|
+
'.agent-tmp', '.delphi-work', '.opencode', '.omo', '.workbuddy',
|
|
66
|
+
];
|
|
67
|
+
|
|
68
|
+
/** The #562 shape, kept as data so the detector can be tested on it. */
|
|
69
|
+
const UNMEASURED_SAMPLE = ['plugins/**'];
|
|
70
|
+
|
|
71
|
+
/** True when a directory holds at least one collected test file (bounded depth). */
|
|
72
|
+
function hasTestFile(dir, depth) {
|
|
73
|
+
if ((depth || 0) > 4) return false;
|
|
74
|
+
let entries;
|
|
75
|
+
try {
|
|
76
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
77
|
+
} catch {
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
for (const e of entries) {
|
|
81
|
+
if (e.name === 'node_modules' || e.name === 'skills') continue;
|
|
82
|
+
const full = path.join(dir, e.name);
|
|
83
|
+
if (e.isDirectory()) {
|
|
84
|
+
if (hasTestFile(full, (depth || 0) + 1)) return true;
|
|
85
|
+
} else if (/\.test\.[cm]?[jt]s$/.test(e.name)) {
|
|
86
|
+
return true;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
describe('覆盖率核算范围 (#562)', () => {
|
|
93
|
+
it('AC-562-01: 有测试在跑的源码必须计入覆盖率核算', () => {
|
|
94
|
+
// 28 test files under plugins/ run and pass; if the source they exercise is not
|
|
95
|
+
// measured, those 225 tests cannot move the coverage number at all.
|
|
96
|
+
const { include } = readCoverageGlobs();
|
|
97
|
+
expect(include.some((g) => g.startsWith('plugins/'))).toBe(true);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('AC-562-01: plugins 不再出现在排除列表里', () => {
|
|
101
|
+
const { exclude } = readCoverageGlobs();
|
|
102
|
+
expect(exclude).not.toContain('plugins/**');
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('AC-562-02: 被测试覆盖的源码目录不能被排除在核算之外', () => {
|
|
106
|
+
// `'plugins/**'` sat in exclude while include was `['src/**']`: the entry could
|
|
107
|
+
// never match, so it read as "we thought about plugins" while doing nothing.
|
|
108
|
+
//
|
|
109
|
+
// A generic "is this exclude entry dead?" check cannot work -- `node_modules/**`
|
|
110
|
+
// and `.worktrees/**` legitimately name real directories that must stay out, and
|
|
111
|
+
// `scripts/**` / `skills/**` hold tooling rather than measured product source.
|
|
112
|
+
// The property that matters is narrower: **product source with tests must be
|
|
113
|
+
// measured.** Otherwise those tests cannot move the coverage number at all.
|
|
114
|
+
const { exclude } = readCoverageGlobs();
|
|
115
|
+
const dirsWithTests = fs
|
|
116
|
+
.readdirSync(REPO_ROOT, { withFileTypes: true })
|
|
117
|
+
.filter((e) => e.isDirectory())
|
|
118
|
+
.map((e) => e.name)
|
|
119
|
+
.filter((name) => !NOT_PRODUCT_TREES.includes(name))
|
|
120
|
+
.filter((name) => hasTestFile(path.join(REPO_ROOT, name)));
|
|
121
|
+
const unmeasured = dirsWithTests.filter((name) => exclude.includes(`${name}/**`));
|
|
122
|
+
expect(unmeasured).toEqual([]);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it('AC-562-02: 检测器在合成输入上确实会触发', () => {
|
|
126
|
+
// A detector that never fires is indistinguishable from "no drift" -- the same
|
|
127
|
+
// trap recorded for the mirror guard (#654). Prove the filter is not vacuous.
|
|
128
|
+
const fakeDirs = ['plugins', 'dashboard'];
|
|
129
|
+
const unmeasured = fakeDirs.filter((name) => UNMEASURED_SAMPLE.includes(`${name}/**`));
|
|
130
|
+
expect(unmeasured).toEqual(['plugins']);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it('AC-562-02: 检测器在合成输入上确实会触发', () => {
|
|
134
|
+
// A detector that never fires is indistinguishable from "no drift" -- the same
|
|
135
|
+
// trap recorded for the mirror guard (#654). Prove the filter is not vacuous.
|
|
136
|
+
const fake = ['plugins', 'dashboard'];
|
|
137
|
+
const unmeasured = fake.filter((name) => UNMEASURED_SAMPLE.includes(`${name}/**`));
|
|
138
|
+
expect(unmeasured).toEqual(['plugins']);
|
|
139
|
+
});
|
|
140
|
+
});
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @test REQ-609 SLO 三态判定:PASS / MISS / UNKNOWN 必须互不可替代
|
|
3
|
+
* @intent 六项 SLO 的价值不在数字本身,而在"能不能行动"。
|
|
4
|
+
*
|
|
5
|
+
* 这里断言的是**状态机**,不是阈值算术:
|
|
6
|
+
*
|
|
7
|
+
* - 没数据 -> UNKNOWN,既不报通过也不报失败
|
|
8
|
+
* - 有数据但越界 -> MISS(唯一需要行动的状态)
|
|
9
|
+
* - 文件丢失目标为 0 时,"没跑过守卫" 不得算成 PASS
|
|
10
|
+
*
|
|
11
|
+
* 最后一条是本模块最容易被写错的地方:`0 <= 0` 恒真,所以任何按
|
|
12
|
+
* 数值比较的实现都会在没有观测时报告"零文件丢失"——那是对**没有发生
|
|
13
|
+
* 观测**和**观测到零**的混淆,与 #554 门禁没跑却报通过同源。
|
|
14
|
+
* @covers AC-609-01, AC-609-02, AC-609-03
|
|
15
|
+
*/
|
|
16
|
+
'use strict';
|
|
17
|
+
|
|
18
|
+
const { evaluateSlo, evaluateAll, summariseStates, SLO_DEFINITIONS, STATES } = require('../slo');
|
|
19
|
+
|
|
20
|
+
describe('SLO 三态判定 (#609)', () => {
|
|
21
|
+
it('AC-609-01: 无数据返回 UNKNOWN 而不是 PASS 或 MISS', () => {
|
|
22
|
+
for (const id of ['first_build_pass', 'flaky_rate', 'file_loss']) {
|
|
23
|
+
const r = evaluateSlo(id, null);
|
|
24
|
+
expect(r.state).toBe(STATES.UNKNOWN);
|
|
25
|
+
expect(r.value).toBeNull();
|
|
26
|
+
expect(r.reason.length).toBeGreaterThan(0);
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('AC-609-01: 有价值的 0 与无数据必须可区分', () => {
|
|
31
|
+
// flaky_rate = 0 is a real, excellent measurement; null is not a measurement.
|
|
32
|
+
const zero = evaluateSlo('flaky_rate', 0);
|
|
33
|
+
const absent = evaluateSlo('flaky_rate', null);
|
|
34
|
+
expect(zero.state).toBe(STATES.PASS);
|
|
35
|
+
expect(absent.state).toBe(STATES.UNKNOWN);
|
|
36
|
+
expect(zero.state).not.toBe(absent.state);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('AC-609-02: min 方向越界是 MISS,达标是 PASS', () => {
|
|
40
|
+
expect(evaluateSlo('first_build_pass', 90).state).toBe(STATES.PASS);
|
|
41
|
+
expect(evaluateSlo('first_build_pass', 89.9).state).toBe(STATES.MISS);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('AC-609-02: max 方向越界是 MISS,零违规是 PASS', () => {
|
|
45
|
+
expect(evaluateSlo('ci_wasted_trigger', 20).state).toBe(STATES.PASS);
|
|
46
|
+
expect(evaluateSlo('ci_wasted_trigger', 20.1).state).toBe(STATES.MISS);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('AC-609-03: file_loss 目标为 0 时,没有样本不算 PASS', () => {
|
|
50
|
+
// `0 <= 0` passes a naive comparison. Without an observation, the honest
|
|
51
|
+
// answer is UNKNOWN -- same defect class as a gate that reports success
|
|
52
|
+
// without running.
|
|
53
|
+
const noSamples = evaluateSlo('file_loss', 0);
|
|
54
|
+
expect(noSamples.state).toBe(STATES.UNKNOWN);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it('AC-609-03: file_loss 有样本且为 0 才算 PASS', () => {
|
|
58
|
+
const observed = evaluateSlo('file_loss', 0, { samples: 12 });
|
|
59
|
+
expect(observed.state).toBe(STATES.PASS);
|
|
60
|
+
expect(observed.sampleCount).toBe(12);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it('AC-609-03: file_loss 有样本且非 0 是 MISS', () => {
|
|
64
|
+
expect(evaluateSlo('file_loss', 3, { samples: 12 }).state).toBe(STATES.MISS);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('AC-609-01: 每项 SLO 都声明了数据来源', () => {
|
|
68
|
+
// A metric whose provenance is unstated cannot be verified, so it cannot be
|
|
69
|
+
// trusted -- and an untrusted SLO panel is worse than no panel.
|
|
70
|
+
for (const def of SLO_DEFINITIONS) {
|
|
71
|
+
expect(def.source).toBeTruthy();
|
|
72
|
+
expect(def.describe).toBeTruthy();
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('AC-609-01: evaluateAll 覆盖全部六项且计数自洽', () => {
|
|
77
|
+
const rows = evaluateAll({ first_build_pass: 95, flaky_rate: null });
|
|
78
|
+
expect(rows).toHaveLength(SLO_DEFINITIONS.length);
|
|
79
|
+
const counts = summariseStates(rows);
|
|
80
|
+
expect(counts.total).toBe(SLO_DEFINITIONS.length);
|
|
81
|
+
expect(counts.PASS + counts.MISS + counts.UNKNOWN).toBe(counts.total);
|
|
82
|
+
expect(counts.PASS).toBe(1);
|
|
83
|
+
expect(counts.UNKNOWN).toBe(SLO_DEFINITIONS.length - 1);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('AC-609-02: 未知 SLO id 抛错而不是静默返回', () => {
|
|
87
|
+
expect(() => evaluateSlo('no_such_slo', 1)).toThrow(/unknown SLO id/);
|
|
88
|
+
});
|
|
89
|
+
});
|
package/lib/retro.js
CHANGED
|
@@ -15,6 +15,7 @@ const fs = require('fs');
|
|
|
15
15
|
const path = require('path');
|
|
16
16
|
const { execSync } = require('child_process');
|
|
17
17
|
const { computeFlakyStats, parseQuarantine, findPolicyViolations } = require('./flaky');
|
|
18
|
+
const { evaluateAll, summariseStates } = require('./slo');
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
21
|
* Upper bound for a believable gate runtime. A gate that reports more than two
|
|
@@ -242,6 +243,59 @@ function buildGateMSection(runs) {
|
|
|
242
243
|
* @param {object} opts - Report data
|
|
243
244
|
* @returns {string} Markdown report
|
|
244
245
|
*/
|
|
246
|
+
/**
|
|
247
|
+
* Derive the six SLO measurements from the journal (#609, guide §15).
|
|
248
|
+
*
|
|
249
|
+
* Every branch that cannot produce a number returns `null`, never 0. The SLO
|
|
250
|
+
* evaluator turns nulls into UNKNOWN, which is displayed as "not measured" --
|
|
251
|
+
* distinct from "measured zero". A dashboard that prints 0% for an unmeasured
|
|
252
|
+
* window teaches people to ignore it.
|
|
253
|
+
*/
|
|
254
|
+
function deriveSloMeasurements(audit, flaky) {
|
|
255
|
+
const gateEntries = audit.filter((e) => e.gate_id);
|
|
256
|
+
// The journal key is `commit_hash`, not `commit`, and there is no `attempt`
|
|
257
|
+
// field at all (verified against 9212 real entries). Without attempts, a single
|
|
258
|
+
// gate list cannot separate "passed first time" from "passed after a retry", so
|
|
259
|
+
// the three attempt-derived SLOs stay UNKNOWN rather than being approximated
|
|
260
|
+
// from a flat pass rate -- which would report ~100% and be believed.
|
|
261
|
+
const withAttempt = gateEntries.filter((e) => Number.isFinite(e.attempt));
|
|
262
|
+
const firsts = withAttempt.filter((e) => e.attempt === 1);
|
|
263
|
+
const firstPass = firsts.filter((e) => e.passed).length;
|
|
264
|
+
|
|
265
|
+
// "Final" = the last attempt recorded per gate within each commit.
|
|
266
|
+
const finals = new Map();
|
|
267
|
+
for (const e of withAttempt) {
|
|
268
|
+
const key = `${e.commit_hash || ''}:${e.gate_id}`;
|
|
269
|
+
const prev = finals.get(key);
|
|
270
|
+
if (!prev || e.attempt > prev.attempt) finals.set(key, e);
|
|
271
|
+
}
|
|
272
|
+
const finalList = [...finals.values()];
|
|
273
|
+
|
|
274
|
+
// Gate 5 is the test gate; its first attempt is the first-test signal.
|
|
275
|
+
const gate5First = firsts.filter((e) => /gate.?5|test/i.test(String(e.gate_id)));
|
|
276
|
+
|
|
277
|
+
return {
|
|
278
|
+
first_build_pass: firsts.length > 0 ? (firstPass / firsts.length) * 100 : null,
|
|
279
|
+
final_build_pass: finalList.length > 0
|
|
280
|
+
? (finalList.filter((e) => e.passed).length / finalList.length) * 100
|
|
281
|
+
: null,
|
|
282
|
+
first_test_pass: gate5First.length > 0
|
|
283
|
+
? (gate5First.filter((e) => e.passed).length / gate5First.length) * 100
|
|
284
|
+
: null,
|
|
285
|
+
flaky_rate: flaky && flaky.measurable && flaky.flakyRate !== null
|
|
286
|
+
? parseFloat(flaky.flakyRate)
|
|
287
|
+
: null,
|
|
288
|
+
// No workflow-run source is wired in yet: a number here would be invented.
|
|
289
|
+
ci_wasted_trigger: null,
|
|
290
|
+
file_loss: { value: 0, samples: countGuardRuns(audit) },
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** How many guard runs the journal actually observed (file-loss denominator). */
|
|
295
|
+
function countGuardRuns(audit) {
|
|
296
|
+
return audit.filter((e) => /worktree.?guard|file.?loss/i.test(String(e.event || e.gate_id || ''))).length;
|
|
297
|
+
}
|
|
298
|
+
|
|
245
299
|
function buildReport(opts) {
|
|
246
300
|
const { days, commits, audit, quality, sprints, gateMRuns = [] } = opts;
|
|
247
301
|
const now = new Date();
|
|
@@ -277,6 +331,27 @@ function buildReport(opts) {
|
|
|
277
331
|
}
|
|
278
332
|
L.push('');
|
|
279
333
|
|
|
334
|
+
// SLO panel (#609, guide §15)
|
|
335
|
+
L.push('## SLO Dashboard (#609)');
|
|
336
|
+
const sloRows = evaluateAll(deriveSloMeasurements(audit, computeFlakyStats(gateEntries)));
|
|
337
|
+
const sloCounts = summariseStates(sloRows);
|
|
338
|
+
L.push(`- ${sloCounts.PASS} PASS / ${sloCounts.MISS} MISS / ${sloCounts.UNKNOWN} UNKNOWN of ${sloCounts.total}`);
|
|
339
|
+
if (sloCounts.UNKNOWN === sloCounts.total) {
|
|
340
|
+
// Print the distinction explicitly. An all-UNKNOWN panel is the normal state
|
|
341
|
+
// for a journal that never recorded attempts, and calling that "0% healthy"
|
|
342
|
+
// would be the same defect the panel exists to prevent.
|
|
343
|
+
L.push('- All six UNKNOWN: the journal holds no attempt-level records for this window.');
|
|
344
|
+
L.push(' That is "not measured", not "nothing wrong" -- see each row source.');
|
|
345
|
+
}
|
|
346
|
+
for (const r of sloRows) {
|
|
347
|
+
const shown = r.value === null ? '--' : `${r.value}${r.unit}`;
|
|
348
|
+
const goal = `${r.direction === 'min' ? '>=' : '<='} ${r.target}${r.unit}`;
|
|
349
|
+
L.push(`- [${r.state}] ${r.label}: ${shown} (goal ${goal})`);
|
|
350
|
+
if (r.state === 'UNKNOWN') L.push(` reason: ${r.reason}`);
|
|
351
|
+
if (r.state === 'MISS') L.push(` ${r.reason}`);
|
|
352
|
+
}
|
|
353
|
+
L.push('');
|
|
354
|
+
|
|
280
355
|
// Flaky-test governance (#603)
|
|
281
356
|
L.push('## Test Stability (#603)');
|
|
282
357
|
const flaky = computeFlakyStats(gateEntries);
|
|
@@ -392,10 +467,19 @@ async function handleRetro(args = []) {
|
|
|
392
467
|
if (jsonFlag) {
|
|
393
468
|
const gateEntries = audit.filter(e => e.gate_id);
|
|
394
469
|
const passed = gateEntries.filter(e => e.passed).length;
|
|
470
|
+
const sloRows = evaluateAll(deriveSloMeasurements(audit, computeFlakyStats(gateEntries)));
|
|
395
471
|
console.log(JSON.stringify({
|
|
396
472
|
period: { days, from: new Date(Date.now() - days * 86400000).toISOString(), to: new Date().toISOString() },
|
|
397
473
|
total_commits: commits.length,
|
|
398
474
|
gate_pass_rate: gateEntries.length > 0 ? passed / gateEntries.length : null,
|
|
475
|
+
slo: {
|
|
476
|
+
states: summariseStates(sloRows),
|
|
477
|
+
metrics: sloRows.map(r => ({
|
|
478
|
+
id: r.id, label: r.label, target: r.target, unit: r.unit,
|
|
479
|
+
direction: r.direction, value: r.value, state: r.state,
|
|
480
|
+
reason: r.reason, source: r.source,
|
|
481
|
+
})),
|
|
482
|
+
},
|
|
399
483
|
rework_rates: sprints.filter(s => s.metrics?.rework_rate != null).map(s => ({ id: s.id, rate: s.metrics.rework_rate })),
|
|
400
484
|
evidence_skipped: audit.filter(e => e.event === 'evidence_skipped'),
|
|
401
485
|
quality: quality.map(q => ({ timestamp: q.timestamp, score: q.score })),
|