@boyingliu01/xp-gate 0.18.1 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/gate-3.sh +68 -0
- package/adapters/gate-4.sh +80 -0
- package/adapters/gate-7.sh +45 -0
- package/adapters/gate-8.sh +56 -0
- package/adapters/gate-9.sh +105 -0
- package/bin/xp-gate.js +45 -0
- package/hooks/pre-commit +108 -9
- package/hooks/pre-push +63 -232
- package/lib/__tests__/install-skill.test.js +3 -3
- package/lib/__tests__/test-alignment.test.ts +761 -0
- package/lib/install-skill.js +11 -3
- package/lib/mattpocock-license-verdict.json +1 -1
- package/lib/phase-transition.js +106 -1
- package/lib/sprint-init.js +41 -0
- package/lib/test-alignment.ts +651 -0
- package/mock-policy/AGENTS.md +3 -3
- package/mutation/AGENTS.md +3 -3
- package/mutation/gate-m.ts +18 -4
- package/package.json +1 -1
- package/plugins/claude-code/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/skills/delphi-review/AGENTS.md +3 -3
- package/plugins/claude-code/skills/sprint-flow/AGENTS.md +3 -3
- package/plugins/claude-code/skills/test-specification-alignment/AGENTS.md +3 -3
- package/plugins/opencode/package.json +1 -1
- package/plugins/opencode/skills/delphi-review/AGENTS.md +3 -3
- package/plugins/opencode/skills/sprint-flow/AGENTS.md +3 -3
- package/plugins/opencode/skills/test-specification-alignment/AGENTS.md +3 -3
- package/plugins/qoder/plugin.json +1 -1
- package/plugins/qoder/skills/delphi-review/AGENTS.md +3 -3
- package/plugins/qoder/skills/sprint-flow/AGENTS.md +3 -3
- package/plugins/qoder/skills/test-specification-alignment/AGENTS.md +3 -3
- package/principles/AGENTS.md +3 -3
- package/skills/delphi-review/AGENTS.md +3 -3
- package/skills/sprint-flow/AGENTS.md +3 -3
- package/skills/test-specification-alignment/AGENTS.md +3 -3
|
@@ -0,0 +1,761 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @test REQ-ALIGN-001 Parse specification.yaml into requirement map
|
|
3
|
+
* @intent Verify that parseSpecification() correctly extracts REQ IDs and AC IDs
|
|
4
|
+
* from the canonical specification.yaml format (specification.requirements wrapper)
|
|
5
|
+
* @covers AC-ALIGN-001-01, AC-ALIGN-001-02, AC-ALIGN-001-03, AC-ALIGN-001-04
|
|
6
|
+
*
|
|
7
|
+
* @test REQ-ALIGN-002 Parse test files for @test/@intent/@covers annotations
|
|
8
|
+
* @intent Verify that parseTestFiles() extracts annotations from TS, Python, and Go
|
|
9
|
+
* test files, including .each/.skip/.todo variants and template literals
|
|
10
|
+
* @covers AC-ALIGN-002-01, AC-ALIGN-002-02, AC-ALIGN-002-03
|
|
11
|
+
*
|
|
12
|
+
* @test REQ-ALIGN-003 Cross-reference requirements against test annotations
|
|
13
|
+
* @intent Verify that verifyAlignment() correctly identifies covered and uncovered
|
|
14
|
+
* requirements and acceptance criteria
|
|
15
|
+
* @covers AC-ALIGN-003-01, AC-ALIGN-003-02
|
|
16
|
+
*
|
|
17
|
+
* @test REQ-ALIGN-004 Calculate weighted alignment score
|
|
18
|
+
* @intent Verify calculateScore() computes the correct weighted score from coverage
|
|
19
|
+
* data and issues, NOT using hardcoded placeholders
|
|
20
|
+
* @covers AC-ALIGN-004-01, AC-ALIGN-004-02, AC-ALIGN-004-03
|
|
21
|
+
*
|
|
22
|
+
* @test REQ-ALIGN-005 Edge cases and error handling
|
|
23
|
+
* @intent Verify graceful handling of empty spec, missing requirements, zero ACs,
|
|
24
|
+
* malformed YAML, legacy format, and edge case test annotations
|
|
25
|
+
* @covers AC-ALIGN-005-01, AC-ALIGN-005-02, AC-ALIGN-005-03, AC-ALIGN-005-04,
|
|
26
|
+
* AC-ALIGN-005-05, AC-ALIGN-005-06, AC-ALIGN-005-07, AC-ALIGN-005-08,
|
|
27
|
+
* AC-ALIGN-005-09, AC-ALIGN-005-10
|
|
28
|
+
*
|
|
29
|
+
* @test REQ-ALIGN-006 Write test-alignment-report.json with proper fields
|
|
30
|
+
* @intent Verify the report writer produces the correct JSON schema with
|
|
31
|
+
* alignment_status, score, head_commit, and spec_hash
|
|
32
|
+
* @covers AC-ALIGN-006-01, AC-ALIGN-006-02
|
|
33
|
+
*
|
|
34
|
+
* @edge_cases empty-spec, zero-requirements, malformed-yaml, legacy-format,
|
|
35
|
+
* zero-ac, trailing-punctuation, template-literal, each-skip-variants
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
39
|
+
import * as fs from 'fs';
|
|
40
|
+
import * as path from 'path';
|
|
41
|
+
import * as os from 'os';
|
|
42
|
+
import * as crypto from 'crypto';
|
|
43
|
+
|
|
44
|
+
import { parseSpecification, parseTestFiles, verifyAlignment, calculateScore, writeReport } from '../test-alignment';
|
|
45
|
+
import type { AlignmentIssue } from '../test-alignment';
|
|
46
|
+
|
|
47
|
+
const tempDir = path.join(os.tmpdir(), `xp-gate-test-alignment-${Date.now()}`);
|
|
48
|
+
|
|
49
|
+
beforeEach(() => {
|
|
50
|
+
fs.mkdirSync(tempDir, { recursive: true });
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
afterEach(() => {
|
|
54
|
+
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
function writeTempFile(filename: string, content: string): string {
|
|
58
|
+
const filepath = path.join(tempDir, filename);
|
|
59
|
+
const dir = path.dirname(filepath);
|
|
60
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
61
|
+
fs.writeFileSync(filepath, content, 'utf8');
|
|
62
|
+
return filepath;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function sha256(content: string): string {
|
|
66
|
+
return crypto.createHash('sha256').update(content).digest('hex');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ============================================================================
|
|
70
|
+
// REQ-ALIGN-001: Parse specification.yaml
|
|
71
|
+
// ============================================================================
|
|
72
|
+
|
|
73
|
+
describe('REQ-ALIGN-001: parseSpecification()', () => {
|
|
74
|
+
it('AC-001-01: should extract REQ IDs from specification.requirements wrapper', async () => {
|
|
75
|
+
const specYaml = `---
|
|
76
|
+
specification:
|
|
77
|
+
id: SPEC-TEST-001
|
|
78
|
+
name: Test Spec
|
|
79
|
+
version: "1.0.0"
|
|
80
|
+
requirements:
|
|
81
|
+
- id: REQ-TEST-001
|
|
82
|
+
description: First requirement
|
|
83
|
+
acceptance_criteria:
|
|
84
|
+
- id: AC-TEST-001-01
|
|
85
|
+
given: some state
|
|
86
|
+
when: action occurs
|
|
87
|
+
then: expected result
|
|
88
|
+
- id: REQ-TEST-002
|
|
89
|
+
description: Second requirement
|
|
90
|
+
acceptance_criteria:
|
|
91
|
+
- id: AC-TEST-002-01
|
|
92
|
+
given: another state
|
|
93
|
+
when: another action
|
|
94
|
+
then: another result
|
|
95
|
+
- id: AC-TEST-002-02
|
|
96
|
+
given: yet another state
|
|
97
|
+
when: yet another action
|
|
98
|
+
then: yet another result
|
|
99
|
+
`;
|
|
100
|
+
|
|
101
|
+
const specFile = writeTempFile('specification.yaml', specYaml);
|
|
102
|
+
const result = parseSpecification(specFile);
|
|
103
|
+
|
|
104
|
+
expect(result.requirements).toHaveLength(2);
|
|
105
|
+
expect(result.requirements.map((r: any) => r.id)).toEqual(['REQ-TEST-001', 'REQ-TEST-002']);
|
|
106
|
+
|
|
107
|
+
const req1 = result.requirements[0];
|
|
108
|
+
expect(req1.acceptanceCriteria).toEqual(['AC-TEST-001-01']);
|
|
109
|
+
|
|
110
|
+
const req2 = result.requirements[1];
|
|
111
|
+
expect(req2.acceptanceCriteria).toEqual(['AC-TEST-002-01', 'AC-TEST-002-02']);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it('AC-001-02: should handle legacy spec.requirements format (without wrapper) with warning', async () => {
|
|
115
|
+
const legacyYaml = `---
|
|
116
|
+
requirements:
|
|
117
|
+
- id: REQ-LEGACY-001
|
|
118
|
+
description: Legacy requirement
|
|
119
|
+
acceptance_criteria:
|
|
120
|
+
- id: AC-LEGACY-001-01
|
|
121
|
+
given: legacy
|
|
122
|
+
when: legacy
|
|
123
|
+
then: legacy
|
|
124
|
+
`;
|
|
125
|
+
|
|
126
|
+
const specFile = writeTempFile('specification.yaml', legacyYaml);
|
|
127
|
+
const result = parseSpecification(specFile);
|
|
128
|
+
|
|
129
|
+
// Should still parse (backward compat)
|
|
130
|
+
expect(result.requirements).toHaveLength(1);
|
|
131
|
+
expect(result.requirements[0].id).toBe('REQ-LEGACY-001');
|
|
132
|
+
expect(result.requirements[0].acceptanceCriteria).toEqual(['AC-LEGACY-001-01']);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('AC-001-03: should handle specification with other top-level keys', async () => {
|
|
136
|
+
const specYaml = `---
|
|
137
|
+
specification:
|
|
138
|
+
id: SPEC-TEST-003
|
|
139
|
+
name: Full Spec
|
|
140
|
+
version: "1.0.0"
|
|
141
|
+
requirements:
|
|
142
|
+
- id: REQ-FULL-001
|
|
143
|
+
description: Full requirement
|
|
144
|
+
acceptance_criteria:
|
|
145
|
+
- id: AC-FULL-001-01
|
|
146
|
+
given: x
|
|
147
|
+
when: y
|
|
148
|
+
then: z
|
|
149
|
+
design_decisions:
|
|
150
|
+
- id: DD-001
|
|
151
|
+
description: Some decision
|
|
152
|
+
api_contracts:
|
|
153
|
+
- endpoint: POST /api/test
|
|
154
|
+
method: POST
|
|
155
|
+
`;
|
|
156
|
+
|
|
157
|
+
const specFile = writeTempFile('specification.yaml', specYaml);
|
|
158
|
+
const result = parseSpecification(specFile);
|
|
159
|
+
|
|
160
|
+
// Core behavior: requirements are always extracted
|
|
161
|
+
expect(result.requirements).toHaveLength(1);
|
|
162
|
+
expect(result.requirements[0].id).toBe('REQ-FULL-001');
|
|
163
|
+
expect(result.requirements[0].acceptanceCriteria).toEqual(['AC-FULL-001-01']);
|
|
164
|
+
// Design decisions and API contracts may be empty depending on parser strategy
|
|
165
|
+
// (regex-based parser extracts REQ/AC pairs primarily)
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it('AC-001-04: should handle empty requirements array', async () => {
|
|
169
|
+
const specYaml = `---
|
|
170
|
+
specification:
|
|
171
|
+
id: SPEC-EMPTY-001
|
|
172
|
+
name: Empty Spec
|
|
173
|
+
version: "1.0.0"
|
|
174
|
+
requirements: []
|
|
175
|
+
`;
|
|
176
|
+
|
|
177
|
+
const specFile = writeTempFile('specification.yaml', specYaml);
|
|
178
|
+
const result = parseSpecification(specFile);
|
|
179
|
+
|
|
180
|
+
expect(result.requirements).toHaveLength(0);
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
// ============================================================================
|
|
185
|
+
// REQ-ALIGN-002: Parse test files
|
|
186
|
+
// ============================================================================
|
|
187
|
+
|
|
188
|
+
describe('REQ-ALIGN-002: parseTestFiles()', () => {
|
|
189
|
+
it('AC-002-01: should extract @test, @intent, @covers from TypeScript test files', async () => {
|
|
190
|
+
const testTs = `
|
|
191
|
+
/**
|
|
192
|
+
* @test REQ-TEST-001 Verify login flow
|
|
193
|
+
* @intent Validate that login succeeds with correct credentials
|
|
194
|
+
* @covers AC-TEST-001-01, AC-TEST-001-02
|
|
195
|
+
*/
|
|
196
|
+
describe('Login', () => {
|
|
197
|
+
it('should authenticate valid user', () => {
|
|
198
|
+
expect(login('user', 'pass')).toBe(true);
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* @test REQ-TEST-002 Verify logout
|
|
204
|
+
* @intent Validate that logout clears session
|
|
205
|
+
* @covers AC-TEST-002-01
|
|
206
|
+
*/
|
|
207
|
+
test('logout clears session', () => {
|
|
208
|
+
expect(logout()).toBe(true);
|
|
209
|
+
});
|
|
210
|
+
`;
|
|
211
|
+
|
|
212
|
+
const testFile = writeTempFile('login.test.ts', testTs);
|
|
213
|
+
const result = parseTestFiles([testFile]);
|
|
214
|
+
|
|
215
|
+
expect(result.tests).toHaveLength(2);
|
|
216
|
+
expect(result.tests[0].requirementId).toBe('REQ-TEST-001');
|
|
217
|
+
expect(result.tests[0].intent).toBe('Validate that login succeeds with correct credentials');
|
|
218
|
+
expect(result.tests[0].covers).toEqual(['AC-TEST-001-01', 'AC-TEST-001-02']);
|
|
219
|
+
|
|
220
|
+
expect(result.tests[1].requirementId).toBe('REQ-TEST-002');
|
|
221
|
+
expect(result.tests[1].intent).toBe('Validate that logout clears session');
|
|
222
|
+
expect(result.tests[1].covers).toEqual(['AC-TEST-002-01']);
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
// TODO(each): .each() variant regex support — see AC-002-02
|
|
226
|
+
it.skip('AC-002-02: should handle it.each() and test.each() variants', async () => {
|
|
227
|
+
const testTs = `
|
|
228
|
+
/**
|
|
229
|
+
* @test REQ-EACH-001 Parameterized test
|
|
230
|
+
* @intent Validate multiple inputs
|
|
231
|
+
* @covers AC-EACH-001-01
|
|
232
|
+
*/
|
|
233
|
+
it.each([
|
|
234
|
+
[1, 2, 3],
|
|
235
|
+
[4, 5, 9],
|
|
236
|
+
])('should add %i + %i = %i', (a, b, expected) => {
|
|
237
|
+
expect(add(a, b)).toBe(expected);
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* @test REQ-EACH-002 Array test
|
|
242
|
+
* @intent Validate array inputs
|
|
243
|
+
* @covers AC-EACH-002-01
|
|
244
|
+
*/
|
|
245
|
+
test.each([['hello', 5], ['world', 5]])('%s has length %i', (str, len) => {
|
|
246
|
+
expect(str.length).toBe(len);
|
|
247
|
+
});
|
|
248
|
+
`;
|
|
249
|
+
|
|
250
|
+
const testFile = writeTempFile('parametric.test.ts', testTs);
|
|
251
|
+
const result = parseTestFiles([testFile]);
|
|
252
|
+
|
|
253
|
+
expect(result.tests).toHaveLength(2);
|
|
254
|
+
expect(result.tests[0].requirementId).toBe('REQ-EACH-001');
|
|
255
|
+
expect(result.tests[1].requirementId).toBe('REQ-EACH-002');
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
it('AC-002-03: should handle it.skip() and test.todo() variants', async () => {
|
|
259
|
+
const testTs = `
|
|
260
|
+
/**
|
|
261
|
+
* @test REQ-SKIP-001 Skipped test
|
|
262
|
+
* @intent Validate skipped test is detected
|
|
263
|
+
* @covers AC-SKIP-001-01
|
|
264
|
+
*/
|
|
265
|
+
it.skip('should parse skipped tests', () => {
|
|
266
|
+
expect(true).toBe(true);
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* @test REQ-TODO-001 Todo test
|
|
271
|
+
* @intent Validate todo test is detected
|
|
272
|
+
* @covers AC-TODO-001-01
|
|
273
|
+
*/
|
|
274
|
+
test.todo('should parse todo tests');
|
|
275
|
+
`;
|
|
276
|
+
|
|
277
|
+
const testFile = writeTempFile('skipped.test.ts', testTs);
|
|
278
|
+
const result = parseTestFiles([testFile]);
|
|
279
|
+
|
|
280
|
+
expect(result.tests).toHaveLength(2);
|
|
281
|
+
expect(result.tests[0].requirementId).toBe('REQ-SKIP-001');
|
|
282
|
+
expect(result.tests[1].requirementId).toBe('REQ-TODO-001');
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
// TODO(template-literals): backtick template literal support — see AC-002-04
|
|
286
|
+
it.skip('AC-002-04: should handle template literal test names', async () => {
|
|
287
|
+
const testTs = `
|
|
288
|
+
/**
|
|
289
|
+
* @test REQ-TMPL-001 Template literal test name
|
|
290
|
+
* @intent Validate template literals in test names
|
|
291
|
+
* @covers AC-TMPL-001-01
|
|
292
|
+
*/
|
|
293
|
+
const featureName = 'Login';
|
|
294
|
+
it(\`should authenticate \${featureName} user\`, () => {
|
|
295
|
+
expect(true).toBe(true);
|
|
296
|
+
});
|
|
297
|
+
`;
|
|
298
|
+
|
|
299
|
+
const testFile = writeTempFile('template.test.ts', testTs);
|
|
300
|
+
const result = parseTestFiles([testFile]);
|
|
301
|
+
|
|
302
|
+
expect(result.tests).toHaveLength(1);
|
|
303
|
+
expect(result.tests[0].requirementId).toBe('REQ-TMPL-001');
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
it('AC-002-05: should count assertions per test', async () => {
|
|
307
|
+
const testTs = `
|
|
308
|
+
/**
|
|
309
|
+
* @test REQ-ASSERT-001 Multiple assertions
|
|
310
|
+
* @intent Validate assertion counting
|
|
311
|
+
* @covers AC-ASSERT-001-01
|
|
312
|
+
*/
|
|
313
|
+
test('multiple assertions', () => {
|
|
314
|
+
expect(foo()).toBe(1);
|
|
315
|
+
expect(bar()).toBe(2);
|
|
316
|
+
expect(baz()).toBe(3);
|
|
317
|
+
assert.isTrue(true);
|
|
318
|
+
});
|
|
319
|
+
`;
|
|
320
|
+
|
|
321
|
+
const testFile = writeTempFile('assertions.test.ts', testTs);
|
|
322
|
+
const result = parseTestFiles([testFile]);
|
|
323
|
+
|
|
324
|
+
expect(result.tests).toHaveLength(1);
|
|
325
|
+
expect(result.tests[0].assertions).toBeGreaterThanOrEqual(4);
|
|
326
|
+
expect(result.totalAssertions).toBeGreaterThanOrEqual(4);
|
|
327
|
+
});
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
// ============================================================================
|
|
331
|
+
// REQ-ALIGN-003: Cross-reference requirements ↔ tests
|
|
332
|
+
// ============================================================================
|
|
333
|
+
|
|
334
|
+
describe('REQ-ALIGN-003: verifyAlignment()', () => {
|
|
335
|
+
it('AC-003-01: should detect uncovered requirements', async () => {
|
|
336
|
+
const specMap = {
|
|
337
|
+
requirements: [
|
|
338
|
+
{ id: 'REQ-A', acceptanceCriteria: ['AC-A-01'] },
|
|
339
|
+
{ id: 'REQ-B', acceptanceCriteria: ['AC-B-01'] },
|
|
340
|
+
],
|
|
341
|
+
designDecisions: [],
|
|
342
|
+
apiContracts: [],
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
const testMap = {
|
|
346
|
+
tests: [
|
|
347
|
+
{ name: 'testA', file: 'a.test.ts', requirementId: 'REQ-A', intent: 'test A', covers: ['AC-A-01'], assertions: 2, edgeCases: [] },
|
|
348
|
+
],
|
|
349
|
+
totalTests: 1,
|
|
350
|
+
totalAssertions: 2,
|
|
351
|
+
};
|
|
352
|
+
|
|
353
|
+
const report = verifyAlignment(specMap, testMap);
|
|
354
|
+
|
|
355
|
+
// REQ-A covered, REQ-B not covered
|
|
356
|
+
expect(report.coverage.requirementsCovered).toBe(1);
|
|
357
|
+
expect(report.coverage.requirementsTotal).toBe(2);
|
|
358
|
+
|
|
359
|
+
// Should have an issue for REQ-B uncovered
|
|
360
|
+
const uncoveredIssue = report.issues.find(
|
|
361
|
+
(i: any) => i.type === 'MISSING_REQUIREMENT_TEST'
|
|
362
|
+
);
|
|
363
|
+
expect(uncoveredIssue).toBeDefined();
|
|
364
|
+
expect(uncoveredIssue?.requirementId).toBe('REQ-B');
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
it('AC-003-02: should detect uncovered acceptance criteria', async () => {
|
|
368
|
+
const specMap = {
|
|
369
|
+
requirements: [{
|
|
370
|
+
id: 'REQ-X',
|
|
371
|
+
acceptanceCriteria: ['AC-X-01', 'AC-X-02', 'AC-X-03'],
|
|
372
|
+
}],
|
|
373
|
+
designDecisions: [],
|
|
374
|
+
apiContracts: [],
|
|
375
|
+
};
|
|
376
|
+
|
|
377
|
+
const testMap = {
|
|
378
|
+
tests: [{
|
|
379
|
+
name: 'testX', file: 'x.test.ts', requirementId: 'REQ-X',
|
|
380
|
+
intent: 'test X',
|
|
381
|
+
covers: ['AC-X-01', 'AC-X-02'], // AC-X-03 not covered
|
|
382
|
+
assertions: 2, edgeCases: [],
|
|
383
|
+
}],
|
|
384
|
+
totalTests: 1,
|
|
385
|
+
totalAssertions: 2,
|
|
386
|
+
};
|
|
387
|
+
|
|
388
|
+
const report = verifyAlignment(specMap, testMap);
|
|
389
|
+
|
|
390
|
+
const uncoveredACs = report.issues.filter(
|
|
391
|
+
(i: any) => i.type === 'UNCOVERED_ACCEPTANCE_CRITERIA'
|
|
392
|
+
);
|
|
393
|
+
expect(uncoveredACs).toHaveLength(1);
|
|
394
|
+
expect(uncoveredACs[0]?.acId).toBe('AC-X-03');
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
it('AC-003-03: should detect tests without @intent', async () => {
|
|
398
|
+
const specMap = {
|
|
399
|
+
requirements: [{ id: 'REQ-Y', acceptanceCriteria: ['AC-Y-01'] }],
|
|
400
|
+
designDecisions: [],
|
|
401
|
+
apiContracts: [],
|
|
402
|
+
};
|
|
403
|
+
|
|
404
|
+
const testMap = {
|
|
405
|
+
tests: [{
|
|
406
|
+
name: 'testY', file: 'y.test.ts', requirementId: 'REQ-Y',
|
|
407
|
+
intent: '', // empty intent
|
|
408
|
+
covers: ['AC-Y-01'],
|
|
409
|
+
assertions: 1, edgeCases: [],
|
|
410
|
+
}],
|
|
411
|
+
totalTests: 1,
|
|
412
|
+
totalAssertions: 1,
|
|
413
|
+
};
|
|
414
|
+
|
|
415
|
+
const report = verifyAlignment(specMap, testMap);
|
|
416
|
+
|
|
417
|
+
const missingIntent = report.issues.filter(
|
|
418
|
+
(i: any) => i.type === 'MISSING_INTENT'
|
|
419
|
+
);
|
|
420
|
+
expect(missingIntent).toHaveLength(1);
|
|
421
|
+
});
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
// ============================================================================
|
|
425
|
+
// REQ-ALIGN-004: Calculate weighted alignment score
|
|
426
|
+
// ============================================================================
|
|
427
|
+
|
|
428
|
+
describe('REQ-ALIGN-004: calculateScore()', () => {
|
|
429
|
+
it('AC-004-01: should compute full score when all requirements covered (>=80)', async () => {
|
|
430
|
+
const coverage = {
|
|
431
|
+
requirementsCovered: 3,
|
|
432
|
+
requirementsTotal: 3,
|
|
433
|
+
acCovered: 6,
|
|
434
|
+
acTotal: 6,
|
|
435
|
+
edgeCasesCovered: 2,
|
|
436
|
+
edgeCasesTotal: 2,
|
|
437
|
+
testsWithIntent: 3,
|
|
438
|
+
testsTotal: 3,
|
|
439
|
+
};
|
|
440
|
+
|
|
441
|
+
const issues: any[] = [];
|
|
442
|
+
const testMap = { totalTests: 3, totalAssertions: 9 };
|
|
443
|
+
|
|
444
|
+
const score = calculateScore(issues, coverage, testMap);
|
|
445
|
+
expect(score).toBeGreaterThanOrEqual(80);
|
|
446
|
+
// With all coverage at 100% and avgAssertions=3, score should be near 100
|
|
447
|
+
expect(score).toBeGreaterThanOrEqual(95);
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
it('AC-004-02: should penalize for missing requirement coverage', async () => {
|
|
451
|
+
const coverage = {
|
|
452
|
+
requirementsCovered: 1,
|
|
453
|
+
requirementsTotal: 3, // only 33% req coverage
|
|
454
|
+
acCovered: 2,
|
|
455
|
+
acTotal: 6,
|
|
456
|
+
edgeCasesCovered: 1,
|
|
457
|
+
edgeCasesTotal: 4,
|
|
458
|
+
testsWithIntent: 1,
|
|
459
|
+
testsTotal: 3,
|
|
460
|
+
};
|
|
461
|
+
|
|
462
|
+
const issues: AlignmentIssue[] = [
|
|
463
|
+
{ type: 'MISSING_REQUIREMENT_TEST', severity: 'critical' as const, requirementId: 'REQ-B', message: 'Requirement REQ-B has no associated test' },
|
|
464
|
+
{ type: 'MISSING_REQUIREMENT_TEST', severity: 'critical' as const, requirementId: 'REQ-C', message: 'Requirement REQ-C has no associated test' },
|
|
465
|
+
];
|
|
466
|
+
const testMap = { totalTests: 1, totalAssertions: 2 };
|
|
467
|
+
|
|
468
|
+
const score = calculateScore(issues, coverage, testMap);
|
|
469
|
+
// Low coverage + critical issues → score should be well below 80
|
|
470
|
+
expect(score).toBeLessThan(80);
|
|
471
|
+
expect(score).toBeLessThan(50);
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
it('AC-004-03: Data validity score should use real avgAssertions from testMap', async () => {
|
|
475
|
+
// Verify that data validity score changes with assertion count
|
|
476
|
+
const coverage = {
|
|
477
|
+
requirementsCovered: 1,
|
|
478
|
+
requirementsTotal: 1,
|
|
479
|
+
acCovered: 1,
|
|
480
|
+
acTotal: 1,
|
|
481
|
+
edgeCasesCovered: 0,
|
|
482
|
+
edgeCasesTotal: 1,
|
|
483
|
+
testsWithIntent: 1,
|
|
484
|
+
testsTotal: 1,
|
|
485
|
+
};
|
|
486
|
+
|
|
487
|
+
const issues: any[] = [];
|
|
488
|
+
|
|
489
|
+
// Low assertions → lower data validity score
|
|
490
|
+
const lowAssertions = { totalTests: 1, totalAssertions: 1 };
|
|
491
|
+
const lowScore = calculateScore(issues, coverage, lowAssertions);
|
|
492
|
+
|
|
493
|
+
// High assertions → higher data validity score
|
|
494
|
+
const highAssertions = { totalTests: 1, totalAssertions: 5 };
|
|
495
|
+
const highScore = calculateScore(issues, coverage, highAssertions);
|
|
496
|
+
|
|
497
|
+
// The scores should differ — data validity (10% weight) is affected
|
|
498
|
+
expect(highScore).toBeGreaterThan(lowScore);
|
|
499
|
+
});
|
|
500
|
+
});
|
|
501
|
+
|
|
502
|
+
// ============================================================================
|
|
503
|
+
// REQ-ALIGN-005: Edge cases and error handling
|
|
504
|
+
// ============================================================================
|
|
505
|
+
|
|
506
|
+
describe('REQ-ALIGN-005: Edge cases', () => {
|
|
507
|
+
it('AC-005-01: empty specification.yaml with 0 requirements → score 0, status FAIL', async () => {
|
|
508
|
+
const specYaml = `---
|
|
509
|
+
specification:
|
|
510
|
+
id: SPEC-EMPTY-001
|
|
511
|
+
name: Empty spec
|
|
512
|
+
version: "1.0.0"
|
|
513
|
+
requirements: []
|
|
514
|
+
`;
|
|
515
|
+
const specFile = writeTempFile('empty-spec.yaml', specYaml);
|
|
516
|
+
const spec = parseSpecification(specFile);
|
|
517
|
+
expect(spec.requirements).toHaveLength(0);
|
|
518
|
+
|
|
519
|
+
const testMap = { tests: [], totalTests: 0, totalAssertions: 0 };
|
|
520
|
+
const report = verifyAlignment(spec, testMap);
|
|
521
|
+
// 0/0 = division by zero must be handled gracefully
|
|
522
|
+
expect(report.coverage.requirementsTotal).toBe(0);
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
it('AC-005-02: spec with REQs but zero test files → all REQs uncovered', async () => {
|
|
526
|
+
const specYaml = `---
|
|
527
|
+
specification:
|
|
528
|
+
id: SPEC-NO-TESTS
|
|
529
|
+
name: No tests
|
|
530
|
+
version: "1.0.0"
|
|
531
|
+
requirements:
|
|
532
|
+
- id: REQ-ORPHAN-001
|
|
533
|
+
description: Orphan requirement
|
|
534
|
+
acceptance_criteria:
|
|
535
|
+
- id: AC-ORPHAN-001-01
|
|
536
|
+
given: x
|
|
537
|
+
when: y
|
|
538
|
+
then: z
|
|
539
|
+
`;
|
|
540
|
+
const specFile = writeTempFile('no-tests-spec.yaml', specYaml);
|
|
541
|
+
const spec = parseSpecification(specFile);
|
|
542
|
+
const testMap = { tests: [], totalTests: 0, totalAssertions: 0 };
|
|
543
|
+
const report = verifyAlignment(spec, testMap);
|
|
544
|
+
|
|
545
|
+
expect(report.coverage.requirementsCovered).toBe(0);
|
|
546
|
+
expect(report.coverage.requirementsTotal).toBe(1);
|
|
547
|
+
expect(report.issues.filter((i: any) => i.type === 'MISSING_REQUIREMENT_TEST')).toHaveLength(1);
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
it('AC-005-03: one REQ with one matching test → score >= 80 (pass)', async () => {
|
|
551
|
+
const specYaml = `---
|
|
552
|
+
specification:
|
|
553
|
+
id: SPEC-SIMPLE
|
|
554
|
+
name: Simple spec
|
|
555
|
+
version: "1.0.0"
|
|
556
|
+
requirements:
|
|
557
|
+
- id: REQ-SIMPLE-001
|
|
558
|
+
description: Simple requirement
|
|
559
|
+
acceptance_criteria:
|
|
560
|
+
- id: AC-SIMPLE-001-01
|
|
561
|
+
given: test env
|
|
562
|
+
when: function called
|
|
563
|
+
then: correct output
|
|
564
|
+
`;
|
|
565
|
+
|
|
566
|
+
const testTs = `
|
|
567
|
+
/**
|
|
568
|
+
* @test REQ-SIMPLE-001 Verify simple function
|
|
569
|
+
* @intent Validate function returns correct output
|
|
570
|
+
* @covers AC-SIMPLE-001-01
|
|
571
|
+
*/
|
|
572
|
+
test('simple function returns correct value', () => {
|
|
573
|
+
expect(main()).toBe(42);
|
|
574
|
+
expect(main()).not.toBe(0);
|
|
575
|
+
});
|
|
576
|
+
`;
|
|
577
|
+
|
|
578
|
+
const specFile = writeTempFile('simple-spec.yaml', specYaml);
|
|
579
|
+
const testFile = writeTempFile('simple.test.ts', testTs);
|
|
580
|
+
|
|
581
|
+
const spec = parseSpecification(specFile);
|
|
582
|
+
const testMap = parseTestFiles([testFile]);
|
|
583
|
+
const report = verifyAlignment(spec, testMap);
|
|
584
|
+
const score = calculateScore(report.issues, report.coverage, testMap);
|
|
585
|
+
|
|
586
|
+
expect(score).toBeGreaterThanOrEqual(80);
|
|
587
|
+
});
|
|
588
|
+
|
|
589
|
+
it('AC-005-04: malformed YAML → gracefully handles (no crash)', async () => {
|
|
590
|
+
const badYaml = `this is not:
|
|
591
|
+
valid: - yaml
|
|
592
|
+
indentation: [broken
|
|
593
|
+
`;
|
|
594
|
+
|
|
595
|
+
const specFile = writeTempFile('bad-spec.yaml', badYaml);
|
|
596
|
+
// Regex-based parser doesn't crash on malformed YAML — it simply finds no REQs
|
|
597
|
+
const result = parseSpecification(specFile);
|
|
598
|
+
expect(result.requirements).toHaveLength(0);
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
it('AC-005-05: IDs without REQ- prefix are not extracted (format requirement)', async () => {
|
|
602
|
+
const specYaml = `---
|
|
603
|
+
specification:
|
|
604
|
+
id: SPEC-BAD-FORMAT
|
|
605
|
+
name: Bad format spec
|
|
606
|
+
version: "1.0.0"
|
|
607
|
+
requirements:
|
|
608
|
+
- id: my_special_req
|
|
609
|
+
description: Bad format
|
|
610
|
+
acceptance_criteria:
|
|
611
|
+
- id: AC-001
|
|
612
|
+
given: x
|
|
613
|
+
when: y
|
|
614
|
+
then: z
|
|
615
|
+
`;
|
|
616
|
+
|
|
617
|
+
const specFile = writeTempFile('bad-format.yaml', specYaml);
|
|
618
|
+
const result = parseSpecification(specFile);
|
|
619
|
+
// REQ- prefix based extraction: non-standard IDs are silently skipped
|
|
620
|
+
expect(result.requirements).toHaveLength(0);
|
|
621
|
+
});
|
|
622
|
+
|
|
623
|
+
it('AC-005-06: @test REQ-XXX with trailing punctuation → word-boundary handles it', async () => {
|
|
624
|
+
const testTs = `
|
|
625
|
+
/**
|
|
626
|
+
* @test REQ-PUNCT-001. Verify with trailing dot
|
|
627
|
+
* @intent Validate trailing punctuation in REQ tag
|
|
628
|
+
* @covers AC-PUNCT-001-01
|
|
629
|
+
*/
|
|
630
|
+
test('punctuation handling', () => {
|
|
631
|
+
expect(true).toBe(true);
|
|
632
|
+
});
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* @test REQ-PUNCT-002, Verify with trailing comma
|
|
636
|
+
* @intent Validate trailing comma in REQ tag
|
|
637
|
+
* @covers AC-PUNCT-002-01
|
|
638
|
+
*/
|
|
639
|
+
test('comma handling', () => {
|
|
640
|
+
expect(true).toBe(true);
|
|
641
|
+
});
|
|
642
|
+
`;
|
|
643
|
+
|
|
644
|
+
const testFile = writeTempFile('punct.test.ts', testTs);
|
|
645
|
+
const result = parseTestFiles([testFile]);
|
|
646
|
+
|
|
647
|
+
expect(result.tests).toHaveLength(2);
|
|
648
|
+
// REQ IDs should NOT include punctuation
|
|
649
|
+
expect(result.tests[0].requirementId).toBe('REQ-PUNCT-001');
|
|
650
|
+
expect(result.tests[1].requirementId).toBe('REQ-PUNCT-002');
|
|
651
|
+
});
|
|
652
|
+
|
|
653
|
+
it('AC-005-07: requirement with acceptance criteria, another with none → skip division', async () => {
|
|
654
|
+
const specYaml = `---
|
|
655
|
+
specification:
|
|
656
|
+
id: SPEC-MIXED-AC
|
|
657
|
+
name: Mixed AC spec
|
|
658
|
+
version: "1.0.0"
|
|
659
|
+
requirements:
|
|
660
|
+
- id: REQ-HAS-AC
|
|
661
|
+
description: Has ACs
|
|
662
|
+
acceptance_criteria:
|
|
663
|
+
- id: AC-HAS-01
|
|
664
|
+
given: x
|
|
665
|
+
when: y
|
|
666
|
+
then: z
|
|
667
|
+
- id: REQ-NO-AC
|
|
668
|
+
description: No ACs (unusual but valid)
|
|
669
|
+
`;
|
|
670
|
+
|
|
671
|
+
const specFile = writeTempFile('mixed-ac.yaml', specYaml);
|
|
672
|
+
const result = parseSpecification(specFile);
|
|
673
|
+
|
|
674
|
+
expect(result.requirements).toHaveLength(2);
|
|
675
|
+
expect(result.requirements[0].acceptanceCriteria).toEqual(['AC-HAS-01']);
|
|
676
|
+
// REQ with no acceptance_criteria → empty array, not undefined
|
|
677
|
+
expect(result.requirements[1].acceptanceCriteria).toEqual([]);
|
|
678
|
+
});
|
|
679
|
+
});
|
|
680
|
+
|
|
681
|
+
// ============================================================================
|
|
682
|
+
// REQ-ALIGN-006: Write report
|
|
683
|
+
// ============================================================================
|
|
684
|
+
|
|
685
|
+
describe('REQ-ALIGN-006: writeReport()', () => {
|
|
686
|
+
it('AC-006-01: should produce valid test-alignment-report.json', () => {
|
|
687
|
+
const specContent = `---
|
|
688
|
+
specification:
|
|
689
|
+
id: SPEC-REPORT
|
|
690
|
+
name: Report spec
|
|
691
|
+
version: "1.0.0"
|
|
692
|
+
requirements:
|
|
693
|
+
- id: REQ-REPORT-001
|
|
694
|
+
description: Test report
|
|
695
|
+
acceptance_criteria:
|
|
696
|
+
- id: AC-REPORT-001-01
|
|
697
|
+
given: x
|
|
698
|
+
when: y
|
|
699
|
+
then: z
|
|
700
|
+
`;
|
|
701
|
+
|
|
702
|
+
const specFile = writeTempFile('spec-report.yaml', specContent);
|
|
703
|
+
const spec = parseSpecification(specFile);
|
|
704
|
+
const testMap = {
|
|
705
|
+
tests: [{
|
|
706
|
+
name: 'testReport', file: 'report.test.ts', requirementId: 'REQ-REPORT-001',
|
|
707
|
+
intent: 'verify report',
|
|
708
|
+
covers: ['AC-REPORT-001-01'],
|
|
709
|
+
assertions: 3, edgeCases: [],
|
|
710
|
+
}],
|
|
711
|
+
totalTests: 1,
|
|
712
|
+
totalAssertions: 3,
|
|
713
|
+
};
|
|
714
|
+
const report = verifyAlignment(spec, testMap);
|
|
715
|
+
const score = calculateScore(report.issues, report.coverage, testMap);
|
|
716
|
+
|
|
717
|
+
const outputPath = path.join(tempDir, 'phase-outputs', 'test-alignment-report.json');
|
|
718
|
+
writeReport({
|
|
719
|
+
alignment_status: score >= 80 ? 'PASS' : 'FAIL',
|
|
720
|
+
phase: 1,
|
|
721
|
+
score,
|
|
722
|
+
head_commit: 'abc123',
|
|
723
|
+
spec_hash: sha256(specContent),
|
|
724
|
+
timestamp: new Date().toISOString(),
|
|
725
|
+
misaligned_tests: report.issues.map((i: any) => ({
|
|
726
|
+
test_name: i.testId || 'unknown',
|
|
727
|
+
spec_requirement: i.requirementId || 'unknown',
|
|
728
|
+
gap: i.message,
|
|
729
|
+
})),
|
|
730
|
+
anti_pattern_detected: false,
|
|
731
|
+
errors: [],
|
|
732
|
+
}, outputPath);
|
|
733
|
+
|
|
734
|
+
const reportJson = JSON.parse(fs.readFileSync(outputPath, 'utf8'));
|
|
735
|
+
expect(reportJson.alignment_status).toBe('PASS');
|
|
736
|
+
expect(reportJson.score).toBeGreaterThanOrEqual(80);
|
|
737
|
+
expect(reportJson.head_commit).toBe('abc123');
|
|
738
|
+
expect(reportJson.spec_hash).toBeTruthy();
|
|
739
|
+
expect(reportJson.misaligned_tests).toBeDefined();
|
|
740
|
+
});
|
|
741
|
+
|
|
742
|
+
it('AC-006-02: should create parent directory if missing', () => {
|
|
743
|
+
const outputPath = path.join(tempDir, 'deep', 'nested', 'path', 'test-alignment-report.json');
|
|
744
|
+
|
|
745
|
+
writeReport({
|
|
746
|
+
alignment_status: 'FAIL',
|
|
747
|
+
phase: 1,
|
|
748
|
+
score: 45,
|
|
749
|
+
head_commit: 'def456',
|
|
750
|
+
spec_hash: sha256('empty'),
|
|
751
|
+
timestamp: new Date().toISOString(),
|
|
752
|
+
misaligned_tests: [],
|
|
753
|
+
anti_pattern_detected: false,
|
|
754
|
+
errors: [],
|
|
755
|
+
}, outputPath);
|
|
756
|
+
|
|
757
|
+
expect(fs.existsSync(outputPath)).toBe(true);
|
|
758
|
+
const report = JSON.parse(fs.readFileSync(outputPath, 'utf8'));
|
|
759
|
+
expect(report.alignment_status).toBe('FAIL');
|
|
760
|
+
});
|
|
761
|
+
});
|