@boyingliu01/xp-gate 0.35.2 → 0.35.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/gate-4.sh +33 -4
- package/lib/__tests__/flaky.test.js +128 -0
- package/lib/doctor.js +56 -0
- package/lib/flaky.js +208 -0
- package/lib/gate-audit.ts +15 -0
- package/lib/mattpocock-license-verdict.json +1 -1
- package/lib/retro.js +26 -0
- package/mock-policy/AGENTS.md +2 -2
- package/mutation/AGENTS.md +2 -2
- package/package.json +1 -1
- package/plugins/claude-code/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/skills/delphi-review/AGENTS.md +2 -2
- package/plugins/claude-code/skills/sprint-flow/AGENTS.md +2 -2
- package/plugins/claude-code/skills/test-specification-alignment/AGENTS.md +2 -2
- package/plugins/dsh/lib/command.d.ts +1 -1
- package/plugins/dsh/lib/command.d.ts.map +1 -1
- package/plugins/dsh/lib/command.js +9 -3
- package/plugins/dsh/lib/dialect.d.ts +22 -2
- package/plugins/dsh/lib/dialect.d.ts.map +1 -1
- package/plugins/dsh/lib/dialect.js +37 -2
- package/plugins/dsh/lib/gate-runner.d.ts.map +1 -1
- package/plugins/dsh/lib/gate-runner.js +22 -0
- package/plugins/dsh/lib/index.d.ts.map +1 -1
- package/plugins/dsh/lib/index.js +7 -4
- package/plugins/dsh/package.json +1 -1
- package/plugins/dsh/skills/delphi-review/AGENTS.md +2 -2
- package/plugins/dsh/skills/sprint-flow/AGENTS.md +2 -2
- package/plugins/dsh/skills/test-specification-alignment/AGENTS.md +2 -2
- package/plugins/dsh/src/command.ts +9 -3
- package/plugins/dsh/src/dialect.ts +45 -2
- package/plugins/dsh/src/exit-code.test.ts +83 -0
- package/plugins/dsh/src/gate-runner.ts +23 -0
- package/plugins/dsh/src/index.ts +7 -4
- package/plugins/opencode/package.json +1 -1
- package/plugins/opencode/skills/delphi-review/AGENTS.md +2 -2
- package/plugins/opencode/skills/sprint-flow/AGENTS.md +2 -2
- package/plugins/opencode/skills/test-specification-alignment/AGENTS.md +2 -2
- package/plugins/qoder/plugin.json +1 -1
- package/plugins/qoder/skills/delphi-review/AGENTS.md +2 -2
- package/plugins/qoder/skills/sprint-flow/AGENTS.md +2 -2
- package/plugins/qoder/skills/test-specification-alignment/AGENTS.md +2 -2
- package/principles/AGENTS.md +2 -2
- package/skills/delphi-review/AGENTS.md +2 -2
- package/skills/sprint-flow/AGENTS.md +2 -2
- package/skills/test-specification-alignment/AGENTS.md +2 -2
package/adapters/gate-4.sh
CHANGED
|
@@ -165,13 +165,42 @@ else
|
|
|
165
165
|
rm -f "$PRINCIPLES_STDERR"
|
|
166
166
|
|
|
167
167
|
if [ "$PRINCIPLES_EXIT" -ge 2 ]; then
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
#
|
|
168
|
+
# Fail-closed (#661), same rule as Gate 8 (#499): a gate that cannot run
|
|
169
|
+
# must not report success.
|
|
170
|
+
#
|
|
171
|
+
# This branch used to settle on SKIP, which the caller reads as
|
|
172
|
+
# non-blocking -- so a checker that crashed on startup
|
|
173
|
+
# (`Cannot find module 'C:\c\Users\...'`, the POSIX-path-to-native-node
|
|
174
|
+
# bug) still printed "PASSED - Principles checker (no errors found)".
|
|
175
|
+
# The path conversion fixed that one cause; any other way of breaking
|
|
176
|
+
# the launch (missing tsx, corrupt module, OOM) lands here too, and all
|
|
177
|
+
# of them would have been absorbed into a green light.
|
|
178
|
+
#
|
|
179
|
+
# SKIP stays reserved for "language not applicable / tool not
|
|
180
|
+
# installed", which is not what this is: the checker exists and failed.
|
|
181
|
+
echo ""
|
|
182
|
+
echo "❌ BLOCKED - Principles checker failed to execute (exit $PRINCIPLES_EXIT)"
|
|
183
|
+
echo "This is a tool fault, NOT a clean result. A gate that cannot run must not pass."
|
|
184
|
+
echo " Fix the checker, or set PRINCIPLES_SKIP_TOOL_FAULT=1 to downgrade to a warning."
|
|
185
|
+
echo " (downgrade is audit-logged and never allowed on main/master)"
|
|
186
|
+
GATE_4_STATUS="FAIL"
|
|
172
187
|
# shellcheck disable=SC2034
|
|
173
188
|
GATE_4_REASON="checker_execution_error"
|
|
174
189
|
rm -f "$PRINCIPLES_JSON"
|
|
190
|
+
if [ "${PRINCIPLES_SKIP_TOOL_FAULT:-0}" = "1" ]; then
|
|
191
|
+
echo "⚠️ ESCAPE VALVE: PRINCIPLES_SKIP_TOOL_FAULT=1 — downgrading to SKIP"
|
|
192
|
+
# Same asymmetry as Gate 5a: a downgrade is tolerable on a feature
|
|
193
|
+
# branch, never on main/master, where it would green-light the trunk.
|
|
194
|
+
_g4_branch="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)"
|
|
195
|
+
if [ "$_g4_branch" = "main" ] || [ "$_g4_branch" = "master" ]; then
|
|
196
|
+
echo "❌ ESCAPE VALVE BLOCKED: PRINCIPLES_SKIP_TOOL_FAULT not allowed on $_g4_branch"
|
|
197
|
+
GATE_4_STATUS="FAIL"
|
|
198
|
+
exit 1
|
|
199
|
+
fi
|
|
200
|
+
GATE_4_STATUS="SKIP"
|
|
201
|
+
else
|
|
202
|
+
exit 1
|
|
203
|
+
fi
|
|
175
204
|
else
|
|
176
205
|
# Check severity levels. The reporter emits JSON.stringify(out, null, 2),
|
|
177
206
|
# i.e. `"severity": "warning"` WITH a space; tolerate any whitespace so a
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @test REQ-603 Flaky 测试治理:首过率度量、重试上限、隔离台账
|
|
3
|
+
* @intent 指南 §11「Flaky 测试治理」整章空白:无重试上限、无首过率/重跑率度量、
|
|
4
|
+
* 无隔离台账与到期机制。`xp-gate retro` 只统计 rework rate 和 gate pass rate,
|
|
5
|
+
* 回答不了「我们的测试稳不稳」。
|
|
6
|
+
*
|
|
7
|
+
* 这个缺口有一个具体的代价:**不稳定的测试会被当成产品缺陷去修**。
|
|
8
|
+
* 一个失败-run 如果只是抖动,开发者花在"修"它上面的时间全是浪费;
|
|
9
|
+
* 而如果它是真的,把它当抖动重跑就是漏网。两条路都错在同一个地方——
|
|
10
|
+
* 我们没有任何数据区分这两者。
|
|
11
|
+
*
|
|
12
|
+
* 三个设计决定:
|
|
13
|
+
*
|
|
14
|
+
* 1. **重跑成功不计入首过率。** 这是整个度量的立身之本。如果 `passed: true`
|
|
15
|
+
* 不管是不是重跑得来的都算通过,首过率永远是 100%,度量就没有意义。
|
|
16
|
+
* 记录必须显式携带 `attempt`,否则"重跑后通过"和"一次通过"在数据上
|
|
17
|
+
* 无法区分——而这两者正是要区分的全部。
|
|
18
|
+
*
|
|
19
|
+
* 2. **重试必须有上限,且默认极小(1 次)。** 无上限重试等于把门禁变成
|
|
20
|
+
* "多试几次总会有一次过",那是退让而不是治理。上限的作用是让抖动
|
|
21
|
+
* 暴露出来,而不是让它消失。
|
|
22
|
+
*
|
|
23
|
+
* 3. **隔离台账必须有到期。** 没有到期的隔离就是永久删除(只是看起来还在)。
|
|
24
|
+
* 这条和 #608 的 escape valve 同一条教训:没有失效机制的安全阀,
|
|
25
|
+
* 一年后就成了常态。
|
|
26
|
+
* @covers AC-603-01, AC-603-02, AC-603-03, AC-603-04
|
|
27
|
+
*/
|
|
28
|
+
'use strict';
|
|
29
|
+
|
|
30
|
+
const fs = require('node:fs');
|
|
31
|
+
const os = require('node:os');
|
|
32
|
+
const path = require('node:path');
|
|
33
|
+
const { computeFlakyStats, parseQuarantine, findExpiredQuarantine, MAX_RETRIES } = require('../flaky');
|
|
34
|
+
|
|
35
|
+
function rec(overrides) {
|
|
36
|
+
return {
|
|
37
|
+
commit_hash: 'deadbeef',
|
|
38
|
+
duration_ms: 1000,
|
|
39
|
+
gate_id: '5',
|
|
40
|
+
gate_name: 'Tests + Coverage',
|
|
41
|
+
issues_found: 0,
|
|
42
|
+
passed: true,
|
|
43
|
+
repo_path: '/repo',
|
|
44
|
+
timestamp: '2026-10-30T00:00:00Z',
|
|
45
|
+
trigger: 'pre-commit',
|
|
46
|
+
...overrides,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
describe('flaky 测试治理 (#603)', () => {
|
|
51
|
+
it('AC-603-01: 重跑后通过不计入首过率', () => {
|
|
52
|
+
const stats = computeFlakyStats([
|
|
53
|
+
rec({ passed: true, attempt: 1 }),
|
|
54
|
+
rec({ passed: false, attempt: 1 }),
|
|
55
|
+
rec({ passed: true, attempt: 2, previously_failed: true }),
|
|
56
|
+
rec({ passed: true, attempt: 1 }),
|
|
57
|
+
]);
|
|
58
|
+
// 4 个 gate 运行,其中 3 个是首次尝试(1 过 1 失 1 过 —— 第 4 条也是首次)。
|
|
59
|
+
// 首过率 = 首过数 / 首次尝试数。重跑成功那一条既不算首过,也不算首次尝试。
|
|
60
|
+
expect(stats.firstAttempts).toBe(3);
|
|
61
|
+
expect(stats.firstPasses).toBe(2);
|
|
62
|
+
expect(stats.firstPassRate).toBeCloseTo(66.7, 1);
|
|
63
|
+
// 重跑成功被单独计量——这正是需要被看见的那一类。
|
|
64
|
+
// 分母是全部运行(3 次首次 + 1 次重跑 = 4),所以 1/4 = 25%。
|
|
65
|
+
expect(stats.flakyRecoveries).toBe(1);
|
|
66
|
+
expect(stats.flakyRate).toBeCloseTo(25, 1);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('AC-603-02: 没有重跑记录时 flakyRate 为 null 而非 0(不可测量≠零抖动)', () => {
|
|
70
|
+
// 一个从没记录过重跑的 journal 说明不了"测试很稳",只能说明"没度量"。
|
|
71
|
+
// 报 0 会让读者以为已经量过而且是好的——和 #651 的 NOT MEASURABLE 同一条教训。
|
|
72
|
+
const stats = computeFlakyStats([rec({ passed: true, attempt: 1 })]);
|
|
73
|
+
expect(stats.flakyRate).toBeNull();
|
|
74
|
+
expect(stats.measurable).toBe(false);
|
|
75
|
+
expect(stats.reason).toBeTruthy();
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('AC-603-03: 重试上限有硬上限,配置不能无限放大', () => {
|
|
79
|
+
// 无上限重试等于把门禁变成"多试几次总会有一次过"。
|
|
80
|
+
expect(MAX_RETRIES).toBeGreaterThan(0);
|
|
81
|
+
expect(MAX_RETRIES).toBeLessThanOrEqual(5);
|
|
82
|
+
const { resolveMaxRetries } = require('../flaky');
|
|
83
|
+
expect(resolveMaxRetries(undefined)).toBe(1);
|
|
84
|
+
expect(resolveMaxRetries('0')).toBe(0);
|
|
85
|
+
expect(resolveMaxRetries('99')).toBe(MAX_RETRIES);
|
|
86
|
+
expect(resolveMaxRetries('-3')).toBe(0);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('AC-603-04: 隔离台账解析出到期项,逾期必须被查出且核心链路不得长期隔离', () => {
|
|
90
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flaky-'));
|
|
91
|
+
const file = path.join(dir, '.flaky-quarantine.yaml');
|
|
92
|
+
fs.writeFileSync(file, [
|
|
93
|
+
'quarantined:',
|
|
94
|
+
' - test: "src/a.test.ts > flaky one"',
|
|
95
|
+
' owner: alice',
|
|
96
|
+
' issue: 601',
|
|
97
|
+
' expiry: 2026-01-01',
|
|
98
|
+
' reason: "timing-dependent assertion"',
|
|
99
|
+
' - test: "src/b.test.ts > ok one"',
|
|
100
|
+
' owner: bob',
|
|
101
|
+
' issue: 602',
|
|
102
|
+
' expiry: 2099-01-01',
|
|
103
|
+
' reason: "upstream bug"',
|
|
104
|
+
' - test: "src/c.test.ts > critical"',
|
|
105
|
+
' owner: carol',
|
|
106
|
+
' issue: 603',
|
|
107
|
+
' expiry: 2099-01-01',
|
|
108
|
+
' reason: "known"',
|
|
109
|
+
' critical_path: true',
|
|
110
|
+
'',
|
|
111
|
+
].join('\n'));
|
|
112
|
+
const entries = parseQuarantine(file);
|
|
113
|
+
expect(entries).toHaveLength(3);
|
|
114
|
+
expect(entries[0].owner).toBe('alice');
|
|
115
|
+
|
|
116
|
+
const expired = findExpiredQuarantine(entries, new Date('2026-06-01T00:00:00Z'));
|
|
117
|
+
expect(expired.map((e) => e.owner)).toEqual(['alice']);
|
|
118
|
+
|
|
119
|
+
const { findPolicyViolations } = require('../flaky');
|
|
120
|
+
// A critical-path test may not sit in quarantine past its expiry either --
|
|
121
|
+
// that is the whole point of marking it critical.
|
|
122
|
+
const violations = findPolicyViolations(entries, new Date('2026-06-01T00:00:00Z'));
|
|
123
|
+
const owners = violations.map((v) => v.entry.owner);
|
|
124
|
+
expect(owners).toContain('alice');
|
|
125
|
+
expect(owners).toContain('carol');
|
|
126
|
+
expect(owners).not.toContain('bob');
|
|
127
|
+
});
|
|
128
|
+
});
|
package/lib/doctor.js
CHANGED
|
@@ -26,6 +26,7 @@ const { GATE_CLI_TOOLS, checkCliTool, getToolInstallCmd, checkBash } = require('
|
|
|
26
26
|
// generated shell projection of this same module.
|
|
27
27
|
const { RESOLUTION_ORDER } = require('./hook-module-resolution.js');
|
|
28
28
|
const { inspectHookFile } = require('./hook-health.js');
|
|
29
|
+
const { parseQuarantine, findPolicyViolations } = require('./flaky.js');
|
|
29
30
|
|
|
30
31
|
/**
|
|
31
32
|
* Files above this size get a `bash -n` probe in addition to the CRLF scan.
|
|
@@ -830,6 +831,48 @@ function diagnoseAuditLedger(ctx = {}) {
|
|
|
830
831
|
return { checks, issues };
|
|
831
832
|
}
|
|
832
833
|
|
|
834
|
+
/**
|
|
835
|
+
* Check the flaky-test quarantine ledger for expired entries (#603).
|
|
836
|
+
*
|
|
837
|
+
* This lives in doctor rather than in the gate path on purpose: an expired
|
|
838
|
+
* quarantine is not a code defect, it is a decision that was never revisited.
|
|
839
|
+
* Nothing fails, everything keeps running, and a test meant to be fixed within
|
|
840
|
+
* two weeks quietly stays disabled for a year. That is exactly the class of drift
|
|
841
|
+
* doctor exists to surface, and exactly the class no gate can block.
|
|
842
|
+
*/
|
|
843
|
+
function diagnoseFlakyQuarantine(ctx = {}) {
|
|
844
|
+
const checks = [];
|
|
845
|
+
let issues = 0;
|
|
846
|
+
const root = ctx.repoRoot || process.cwd();
|
|
847
|
+
let entries = [];
|
|
848
|
+
try {
|
|
849
|
+
entries = parseQuarantine(path.join(root, '.flaky-quarantine.yaml'));
|
|
850
|
+
} catch {
|
|
851
|
+
return { checks, issues };
|
|
852
|
+
}
|
|
853
|
+
// No ledger is PASS, not SKIP: a project that quarantines nothing is genuinely
|
|
854
|
+
// in good shape, and SKIP would imply the check had not been performed.
|
|
855
|
+
if (entries.length === 0) {
|
|
856
|
+
checks.push({ name: 'Flaky quarantine', status: 'PASS', detail: 'no quarantined tests' });
|
|
857
|
+
return { checks, issues };
|
|
858
|
+
}
|
|
859
|
+
const violations = findPolicyViolations(entries, new Date());
|
|
860
|
+
if (violations.length === 0) {
|
|
861
|
+
checks.push({
|
|
862
|
+
name: 'Flaky quarantine',
|
|
863
|
+
status: 'PASS',
|
|
864
|
+
detail: `${entries.length} quarantined test(s), all within expiry`,
|
|
865
|
+
});
|
|
866
|
+
return { checks, issues };
|
|
867
|
+
}
|
|
868
|
+
issues += violations.length;
|
|
869
|
+
const detail = violations
|
|
870
|
+
.map((v) => `${v.rule}: ${v.entry.test} (owner ${v.entry.owner || 'unassigned'}, expiry ${v.entry.expiry || 'none'})`)
|
|
871
|
+
.join('; ');
|
|
872
|
+
checks.push({ name: 'Flaky quarantine', status: 'WARN', detail });
|
|
873
|
+
return { checks, issues };
|
|
874
|
+
}
|
|
875
|
+
|
|
833
876
|
function diagnoseModuleDrift(ctx = {}) {
|
|
834
877
|
const checks = [];
|
|
835
878
|
let issues = 0;
|
|
@@ -3020,6 +3063,19 @@ async function doctor(args) {
|
|
|
3020
3063
|
issues += hookDrift.issues + moduleDrift.issues;
|
|
3021
3064
|
}
|
|
3022
3065
|
|
|
3066
|
+
// Flaky-test quarantine expiry (#603). A WARN, not a FAIL: an overdue ledger
|
|
3067
|
+
// breaks nothing at runtime, it only means an intentional gap outlived its
|
|
3068
|
+
// intent -- doctor's business, not a gate's.
|
|
3069
|
+
try {
|
|
3070
|
+
const flaky = diagnoseFlakyQuarantine({ repoRoot: process.cwd() });
|
|
3071
|
+
checks.push(...flaky.checks);
|
|
3072
|
+
issues += flaky.issues;
|
|
3073
|
+
} catch (err) {
|
|
3074
|
+
// Surface the cause: a bare "could not be read" hid the real error and made
|
|
3075
|
+
// this look like a missing file rather than a bug in the check itself.
|
|
3076
|
+
checks.push({ name: 'Flaky quarantine', status: 'SKIP', detail: `ledger check failed: ${err && err.message ? err.message : String(err)}` });
|
|
3077
|
+
}
|
|
3078
|
+
|
|
3023
3079
|
let langIssues = 0;
|
|
3024
3080
|
// Upgrade + plugin + language-tool status, kept as one callable because a
|
|
3025
3081
|
// --format json run still has to execute all of it (#488).
|
package/lib/flaky.js
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flaky-test governance (#603).
|
|
3
|
+
*
|
|
4
|
+
* The gap this closes: a flaky test and a genuine failure look identical in the
|
|
5
|
+
* gate journal, so one of two wrong things happens — the flake gets treated as a
|
|
6
|
+
* product bug and someone spends a day "fixing" nothing, or the real failure gets
|
|
7
|
+
* re-run until it passes and slips through. Both are consequences of having no
|
|
8
|
+
* data that separates the two.
|
|
9
|
+
*
|
|
10
|
+
* Three rules, each earned from a specific failure mode:
|
|
11
|
+
*
|
|
12
|
+
* 1. A re-run that passes does NOT count as a first pass. If `passed: true` counted
|
|
13
|
+
* the same regardless of how it was obtained, first-pass rate would be
|
|
14
|
+
* permanently 100% and the metric would measure nothing.
|
|
15
|
+
* 2. Retries are capped, and the default is tiny. Unlimited retries turn a gate
|
|
16
|
+
* into "eventually something passes", which is a concession rather than
|
|
17
|
+
* governance.
|
|
18
|
+
* 3. Quarantine entries expire. An entry with no expiry is a permanent deletion
|
|
19
|
+
* that merely still looks present.
|
|
20
|
+
*
|
|
21
|
+
* No runtime dependencies: this ships inside the zero-install package.
|
|
22
|
+
*/
|
|
23
|
+
'use strict';
|
|
24
|
+
|
|
25
|
+
const fs = require('node:fs');
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Hard ceiling on retries. Config may lower this but never raise it: the cap
|
|
29
|
+
* exists so flakiness surfaces rather than being retried away.
|
|
30
|
+
*/
|
|
31
|
+
const MAX_RETRIES = 3;
|
|
32
|
+
|
|
33
|
+
/** Only schedule YAML is needed, so a tiny parser beats a dependency here. */
|
|
34
|
+
const FIELD_RE = /^\s*-\s+test:\s*(.+?)\s*$/
|
|
35
|
+
const KV_RE = /^\s+([A-Za-z_][\w]*):\s*(.*?)\s*$/
|
|
36
|
+
|
|
37
|
+
function stripQuotes(value) {
|
|
38
|
+
if (value.length >= 2) {
|
|
39
|
+
const first = value[0]
|
|
40
|
+
const last = value[value.length - 1]
|
|
41
|
+
if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
|
|
42
|
+
return value.slice(1, -1)
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return value
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Parse `.flaky-quarantine.yaml`.
|
|
50
|
+
*
|
|
51
|
+
* Returns `[]` for a missing file: no ledger is not an error, it just means
|
|
52
|
+
* nothing is quarantined. A malformed entry is skipped rather than throwing —
|
|
53
|
+
* a broken ledger must not take down `doctor`.
|
|
54
|
+
*
|
|
55
|
+
* @param {string} filePath
|
|
56
|
+
* @returns {Array<object>}
|
|
57
|
+
*/
|
|
58
|
+
function parseQuarantine(filePath) {
|
|
59
|
+
let text
|
|
60
|
+
try {
|
|
61
|
+
text = fs.readFileSync(filePath, 'utf8')
|
|
62
|
+
} catch {
|
|
63
|
+
return []
|
|
64
|
+
}
|
|
65
|
+
const entries = []
|
|
66
|
+
let current = null
|
|
67
|
+
for (const rawLine of text.split(/\r?\n/)) {
|
|
68
|
+
const line = rawLine.replace(/\s+$/, '')
|
|
69
|
+
if (line === '' || line.startsWith('#')) continue
|
|
70
|
+
const start = FIELD_RE.exec(line)
|
|
71
|
+
if (start) {
|
|
72
|
+
if (current) entries.push(current)
|
|
73
|
+
current = { test: stripQuotes(start[1]) }
|
|
74
|
+
continue
|
|
75
|
+
}
|
|
76
|
+
if (!current) continue
|
|
77
|
+
const kv = KV_RE.exec(line)
|
|
78
|
+
if (!kv) continue
|
|
79
|
+
const key = kv[1]
|
|
80
|
+
const value = stripQuotes(kv[2])
|
|
81
|
+
if (key === 'critical_path') {
|
|
82
|
+
current.critical_path = value === 'true'
|
|
83
|
+
} else {
|
|
84
|
+
current[key] = value
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (current) entries.push(current)
|
|
88
|
+
return entries
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Parse an `YYYY-MM-DD` expiry into a UTC midnight Date, or null. */
|
|
92
|
+
function parseExpiry(value) {
|
|
93
|
+
if (typeof value !== 'string') return null
|
|
94
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value.trim())
|
|
95
|
+
if (!m) return null
|
|
96
|
+
const date = new Date(`${m[1]}-${m[2]}-${m[3]}T00:00:00Z`)
|
|
97
|
+
return Number.isNaN(date.getTime()) ? null : date
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Entries whose expiry has passed. An unparseable expiry counts as expired. */
|
|
101
|
+
function findExpiredQuarantine(entries, now) {
|
|
102
|
+
const at = now || new Date()
|
|
103
|
+
return entries.filter((entry) => {
|
|
104
|
+
const expiry = parseExpiry(entry.expiry)
|
|
105
|
+
// No expiry at all is treated as expired: rule 3 says quarantine without an
|
|
106
|
+
// end date is a permanent deletion, so it must be reported, not tolerated.
|
|
107
|
+
if (expiry === null) return true
|
|
108
|
+
return expiry.getTime() < at.getTime()
|
|
109
|
+
})
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Policy violations across the ledger.
|
|
114
|
+
*
|
|
115
|
+
* Two distinct rules are reported separately because they need different
|
|
116
|
+
* responses: an expired entry is routine bookkeeping, whereas a critical-path
|
|
117
|
+
* entry is a decision someone has to make — a critical path that has been flaky
|
|
118
|
+
* for months means the team has stopped trusting its own safety net.
|
|
119
|
+
*/
|
|
120
|
+
function findPolicyViolations(entries, now) {
|
|
121
|
+
const at = now || new Date()
|
|
122
|
+
const violations = []
|
|
123
|
+
for (const entry of findExpiredQuarantine(entries, at)) {
|
|
124
|
+
violations.push({ rule: 'expired', entry })
|
|
125
|
+
}
|
|
126
|
+
for (const entry of entries) {
|
|
127
|
+
if (entry.critical_path && !violations.some((v) => v.entry === entry)) {
|
|
128
|
+
violations.push({ rule: 'critical-path-quarantined', entry })
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return violations
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Resolve the retry budget from configuration.
|
|
136
|
+
*
|
|
137
|
+
* Clamped, not validated-with-an-error: a bad value must not make the gate
|
|
138
|
+
* unusable, and the direction of clamping always favours surfacing flakiness.
|
|
139
|
+
*/
|
|
140
|
+
function resolveMaxRetries(value) {
|
|
141
|
+
if (value === undefined || value === null || value === '') return 1
|
|
142
|
+
const parsed = Number.parseInt(String(value), 10)
|
|
143
|
+
if (Number.isNaN(parsed)) return 1
|
|
144
|
+
if (parsed < 0) return 0
|
|
145
|
+
return Math.min(parsed, MAX_RETRIES)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Round to one decimal place without trailing `.0`. */
|
|
149
|
+
function pct(part, whole) {
|
|
150
|
+
if (whole === 0) return null
|
|
151
|
+
return Math.round((part / whole) * 1000) / 10
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Compute first-pass and flaky statistics from gate-journal entries.
|
|
156
|
+
*
|
|
157
|
+
* Records carry `attempt` (1-based). A record with `attempt > 1` is a re-run and
|
|
158
|
+
* is excluded from the first-attempt denominator — that exclusion IS the metric.
|
|
159
|
+
*/
|
|
160
|
+
function computeFlakyStats(entries) {
|
|
161
|
+
const rows = Array.isArray(entries) ? entries : []
|
|
162
|
+
let firstAttempts = 0
|
|
163
|
+
let firstPasses = 0
|
|
164
|
+
let flakyRecoveries = 0
|
|
165
|
+
let reruns = 0
|
|
166
|
+
let quarantined = 0
|
|
167
|
+
|
|
168
|
+
for (const row of rows) {
|
|
169
|
+
if (!row || typeof row !== 'object') continue
|
|
170
|
+
if (row.quarantined === true) quarantined += 1
|
|
171
|
+
const attempt = Number.isFinite(row.attempt) ? row.attempt : 1
|
|
172
|
+
if (attempt > 1) {
|
|
173
|
+
reruns += 1
|
|
174
|
+
if (row.passed === true) flakyRecoveries += 1
|
|
175
|
+
continue
|
|
176
|
+
}
|
|
177
|
+
firstAttempts += 1
|
|
178
|
+
if (row.passed === true) firstPasses += 1
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const measurable = reruns > 0
|
|
182
|
+
return {
|
|
183
|
+
totalRuns: rows.length,
|
|
184
|
+
firstAttempts,
|
|
185
|
+
firstPasses,
|
|
186
|
+
firstPassRate: pct(firstPasses, firstAttempts),
|
|
187
|
+
reruns,
|
|
188
|
+
flakyRecoveries,
|
|
189
|
+
quarantined,
|
|
190
|
+
// Without any re-run there is no evidence either way. Reporting 0 here would
|
|
191
|
+
// claim "measured, and stable" from data that only says "not measured".
|
|
192
|
+
flakyRate: measurable ? pct(flakyRecoveries, firstAttempts + reruns) : null,
|
|
193
|
+
measurable,
|
|
194
|
+
reason: measurable
|
|
195
|
+
? null
|
|
196
|
+
: 'no re-run records in the journal — flakiness is unmeasured, not zero',
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
module.exports = {
|
|
201
|
+
MAX_RETRIES,
|
|
202
|
+
parseQuarantine,
|
|
203
|
+
parseExpiry,
|
|
204
|
+
findExpiredQuarantine,
|
|
205
|
+
findPolicyViolations,
|
|
206
|
+
resolveMaxRetries,
|
|
207
|
+
computeFlakyStats,
|
|
208
|
+
}
|
package/lib/gate-audit.ts
CHANGED
|
@@ -47,6 +47,17 @@ export interface GateAuditEntry {
|
|
|
47
47
|
* one that drives fixes. Only meaningful when passed=false.
|
|
48
48
|
*/
|
|
49
49
|
failure_reason?: string;
|
|
50
|
+
/**
|
|
51
|
+
* Which attempt this record represents, 1-based (#603).
|
|
52
|
+
*
|
|
53
|
+
* Absent means 1 — older records predate re-run tracking. This field is what
|
|
54
|
+
* makes first-pass rate computable: without it, "passed" cannot distinguish a
|
|
55
|
+
* clean run from one that only passed after a re-run, and a metric that cannot
|
|
56
|
+
* tell those apart is always 100%.
|
|
57
|
+
*/
|
|
58
|
+
attempt?: number;
|
|
59
|
+
/** True when this row describes a test that is quarantined as flaky (#603). */
|
|
60
|
+
quarantined?: boolean;
|
|
50
61
|
}
|
|
51
62
|
|
|
52
63
|
// ── Public API ───────────────────────────────────────────────────────────────
|
|
@@ -423,6 +434,10 @@ function runCli(): void {
|
|
|
423
434
|
...(opts['detail'] ? { detail: opts['detail'] } : {}),
|
|
424
435
|
// #651: only set when provided, so successful records stay byte-identical.
|
|
425
436
|
...(opts['failure-reason'] ? { failure_reason: String(opts['failure-reason']).slice(0, 300) } : {}),
|
|
437
|
+
// #603: only set when provided, so single-attempt records stay byte-identical
|
|
438
|
+
// to what earlier versions wrote.
|
|
439
|
+
...(opts['attempt'] ? { attempt: Number.parseInt(String(opts['attempt']), 10) || 1 } : {}),
|
|
440
|
+
...(opts['quarantined'] === 'true' ? { quarantined: true } : {}),
|
|
426
441
|
};
|
|
427
442
|
// Pre-commit (trigger=commit) is staging for a commit that does not exist
|
|
428
443
|
// yet: park it in the pending sidecar so post-commit can stamp the real
|
package/lib/retro.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
const fs = require('fs');
|
|
15
15
|
const path = require('path');
|
|
16
16
|
const { execSync } = require('child_process');
|
|
17
|
+
const { computeFlakyStats, parseQuarantine, findPolicyViolations } = require('./flaky');
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Upper bound for a believable gate runtime. A gate that reports more than two
|
|
@@ -276,6 +277,31 @@ function buildReport(opts) {
|
|
|
276
277
|
}
|
|
277
278
|
L.push('');
|
|
278
279
|
|
|
280
|
+
// Flaky-test governance (#603)
|
|
281
|
+
L.push('## Test Stability (#603)');
|
|
282
|
+
const flaky = computeFlakyStats(gateEntries);
|
|
283
|
+
if (!flaky.measurable) {
|
|
284
|
+
// Say "not measured" rather than "0%". A journal with no re-runs cannot
|
|
285
|
+
// distinguish a stable suite from an unmeasured one, and printing 0% would
|
|
286
|
+
// claim the former.
|
|
287
|
+
L.push(`⚠️ NOT MEASURABLE — ${flaky.reason}`);
|
|
288
|
+
} else {
|
|
289
|
+
L.push(`- First-pass rate: ${flaky.firstPassRate}% (${flaky.firstPasses}/${flaky.firstAttempts} first attempts)`);
|
|
290
|
+
L.push(`- Flaky recoveries: ${flaky.flakyRecoveries} of ${flaky.totalRuns} run(s) passed only on re-run (${flaky.flakyRate}%)`);
|
|
291
|
+
// The pass rate above counts re-run successes, so state the gap explicitly.
|
|
292
|
+
L.push(`- Note: the gate pass rate above counts re-run successes; first-pass rate is the honest number.`);
|
|
293
|
+
}
|
|
294
|
+
const quarantineEntries = parseQuarantine(path.join(process.cwd(), '.flaky-quarantine.yaml'));
|
|
295
|
+
if (quarantineEntries.length > 0) {
|
|
296
|
+
const violations = findPolicyViolations(quarantineEntries, now);
|
|
297
|
+
L.push(`- Quarantined tests: ${quarantineEntries.length} (${violations.length} policy violation(s))`);
|
|
298
|
+
for (const v of violations) {
|
|
299
|
+
// An expired entry is a permanent deletion wearing a quarantine badge.
|
|
300
|
+
L.push(` - ${v.rule}: ${v.entry.test} (owner ${v.entry.owner || 'unassigned'}, expiry ${v.entry.expiry || 'none'})`);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
L.push('');
|
|
304
|
+
|
|
279
305
|
// Rework Rate Trend (#369)
|
|
280
306
|
L.push('## Rework Rate Trend (#369)');
|
|
281
307
|
const withRate = sprints.filter(s => s.metrics && s.metrics.rework_rate != null);
|
package/mock-policy/AGENTS.md
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# SRC/MOCK-POLICY KNOWLEDGE BASE
|
|
2
2
|
|
|
3
3
|
**Generated:** 2026-10-30
|
|
4
|
-
**Commit:**
|
|
4
|
+
**Commit:** aefe7f40
|
|
5
5
|
**Branch:** main
|
|
6
|
-
**Version:** 0.35.
|
|
6
|
+
**Version:** 0.35.4.0
|
|
7
7
|
|
|
8
8
|
## OVERVIEW
|
|
9
9
|
Mock layering policy enforcement — Gate M3 of pre-push hook. Ensures integration tests use real implementations for internal dependencies, mock external dependencies, and annotate pending mocks with removal plans. Combines project scope scanning, mock decision engine, and per-file validation into a single pipeline.
|
package/mutation/AGENTS.md
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# SRC/MUTATION KNOWLEDGE BASE
|
|
2
2
|
|
|
3
3
|
**Generated:** 2026-10-30
|
|
4
|
-
**Commit:**
|
|
4
|
+
**Commit:** aefe7f40
|
|
5
5
|
**Branch:** main
|
|
6
|
-
**Version:** 0.35.
|
|
6
|
+
**Version:** 0.35.4.0
|
|
7
7
|
|
|
8
8
|
## OVERVIEW
|
|
9
9
|
**Gate M** (incremental mutation testing) + **Gate M2** helpers (test-layer detection used by `src/mock-policy/`). Pre-push quality gate. TypeScript-only; uses Stryker.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "xp-gate",
|
|
3
|
-
"version": "0.35.
|
|
3
|
+
"version": "0.35.4",
|
|
4
4
|
"displayName": "XP-Gate",
|
|
5
5
|
"description": "Extreme Programming quality gates + AI workflow skills for Claude Code. Includes 12 pre-commit gates (Gate 0-11), 8 pre-push gates, Sprint Flow (6 phases), and Delphi multi-expert review (>=90% consensus).",
|
|
6
6
|
"author": {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# SKILLS/DELPHI-REVIEW KNOWLEDGE BASE
|
|
2
2
|
|
|
3
3
|
**Generated:** 2026-10-30
|
|
4
|
-
**Commit:**
|
|
4
|
+
**Commit:** aefe7f40
|
|
5
5
|
**Branch:** main
|
|
6
|
-
**Version:** 0.35.
|
|
6
|
+
**Version:** 0.35.4.0
|
|
7
7
|
|
|
8
8
|
## OVERVIEW
|
|
9
9
|
Delphi Consensus Review — multi-round anonymous expert review (≥90% threshold, exactly 3 experts with distinct executable model IDs). Supports design, requirements, and code-walkthrough modes.
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# SKILLS/SPRINT-FLOW KNOWLEDGE BASE
|
|
2
2
|
|
|
3
3
|
**Generated:** 2026-10-30
|
|
4
|
-
**Commit:**
|
|
4
|
+
**Commit:** aefe7f40
|
|
5
5
|
**Branch:** main
|
|
6
|
-
**Version:** 0.35.
|
|
6
|
+
**Version:** 0.35.4.0
|
|
7
7
|
|
|
8
8
|
## OVERVIEW
|
|
9
9
|
**6-phase** development pipeline (v2.0 compact redesign, Issue #290): PREP → DESIGN → BUILD → VERIFY → SHIP → CLOSE. Phase 3/6 BUILD default build mode is **ralph-loop** (REQ-level iteration, 40-67% token savings vs parallel). HARD-GATE between DESIGN (2/6) and BUILD (3/6): design must pass Delphi review (≥90% consensus) before any coding.
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# SKILLS/TEST-SPECIFICATION-ALIGNMENT KNOWLEDGE BASE
|
|
2
2
|
|
|
3
3
|
**Generated:** 2026-10-30
|
|
4
|
-
**Commit:**
|
|
4
|
+
**Commit:** aefe7f40
|
|
5
5
|
**Branch:** main
|
|
6
|
-
**Version:** 0.35.
|
|
6
|
+
**Version:** 0.35.4.0
|
|
7
7
|
|
|
8
8
|
## OVERVIEW
|
|
9
9
|
Test-Specification Alignment Engine — two-stage validation ensuring tests accurately reflect requirements and design specs.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type ShellDialect } from "./dialect.js";
|
|
2
|
-
export { type ShellDialect, detectDialect, shellQuote, shq } from "./dialect.js";
|
|
2
|
+
export { type ShellDialect, detectDialect, detectDialectForShell, shellQuote, shq } from "./dialect.js";
|
|
3
3
|
/**
|
|
4
4
|
* Allowlist of xp-gate gate aliases that are safe to invoke standalone
|
|
5
5
|
* (non-preCommitOnly gates + common aliases). Anything outside this set is
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../src/command.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,EAAc,MAAM,cAAc,CAAA;AAI5D,OAAO,EAAE,KAAK,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../src/command.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,EAAc,MAAM,cAAc,CAAA;AAI5D,OAAO,EAAE,KAAK,YAAY,EAAE,aAAa,EAAE,qBAAqB,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAEvG;;;;GAIG;AACH,eAAO,MAAM,cAAc,uGASjB,CAAA;AAEV,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAA;AAEpD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,OAAO,GAAG,YAAY,GAAG,MAAM,CAAA;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,YAAY,CAAA;CACvB;AA+BD;;;;;;;;;;;;GAYG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,MAAM,CAmBjE;AAED,qFAAqF;AACrF,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAElE"}
|
|
@@ -2,7 +2,7 @@ import { isAbsolute, resolve } from "node:path";
|
|
|
2
2
|
import { shellQuote } from "./dialect.js";
|
|
3
3
|
// Re-exported so existing importers of this module keep working; the dialect
|
|
4
4
|
// primitives themselves live in ./dialect.ts.
|
|
5
|
-
export { detectDialect, shellQuote, shq } from "./dialect.js";
|
|
5
|
+
export { detectDialect, detectDialectForShell, shellQuote, shq } from "./dialect.js";
|
|
6
6
|
/**
|
|
7
7
|
* Allowlist of xp-gate gate aliases that are safe to invoke standalone
|
|
8
8
|
* (non-preCommitOnly gates + common aliases). Anything outside this set is
|
|
@@ -73,12 +73,18 @@ export function buildCommand(options) {
|
|
|
73
73
|
// Merge stderr into stdout so a failure that only writes to stderr still
|
|
74
74
|
// reaches the renderer, then echo the status so the exit code survives the
|
|
75
75
|
// pipeline.
|
|
76
|
-
|
|
76
|
+
// Capture the status BEFORE printing it, then `exit` with it. Without the
|
|
77
|
+
// explicit `exit`, the compound command's status is printf's -- always 0 --
|
|
78
|
+
// so classifyGateRun() would read a failed gate as PASS (#669).
|
|
79
|
+
return `${inner} 2>&1; __rc=$?; printf '\\n[XP-Gate] exit:%s\\n' "$__rc"; exit $__rc`;
|
|
77
80
|
}
|
|
78
81
|
// PowerShell: merge streams and report $LASTEXITCODE explicitly. The
|
|
79
82
|
// .cmd/.ps1 shims npm writes under %APPDATA%\npm are visible here but not to a
|
|
80
83
|
// Git-Bash PATH lookup, which is why Windows uses this dialect.
|
|
81
|
-
|
|
84
|
+
//
|
|
85
|
+
// Same defect as the POSIX branch (#669): Write-Output resets the status, so
|
|
86
|
+
// the failing gate's code must be captured and re-exited explicitly.
|
|
87
|
+
return `${inner} 2>&1; $__rc=$LASTEXITCODE; Write-Output "[XP-Gate] exit:$__rc"; exit $__rc`;
|
|
82
88
|
}
|
|
83
89
|
/** Resolve a model-supplied path against the session cwd; absolutes pass through. */
|
|
84
90
|
export function resolveTarget(rawPath, cwd) {
|