@sun-asterisk/sungen 3.2.15 → 3.2.16-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +139 -1
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/dashboard/templates/index.html +1 -1
- package/dist/exporters/feature-parser.js +1 -1
- package/dist/exporters/feature-parser.js.map +1 -1
- package/dist/exporters/matrix/build.d.ts +43 -0
- package/dist/exporters/matrix/build.d.ts.map +1 -0
- package/dist/exporters/matrix/build.js +403 -0
- package/dist/exporters/matrix/build.js.map +1 -0
- package/dist/exporters/matrix/export.d.ts +49 -0
- package/dist/exporters/matrix/export.d.ts.map +1 -0
- package/dist/exporters/matrix/export.js +143 -0
- package/dist/exporters/matrix/export.js.map +1 -0
- package/dist/exporters/matrix/fingerprint.d.ts +20 -0
- package/dist/exporters/matrix/fingerprint.d.ts.map +1 -0
- package/dist/exporters/matrix/fingerprint.js +83 -0
- package/dist/exporters/matrix/fingerprint.js.map +1 -0
- package/dist/exporters/matrix/gates.d.ts +21 -0
- package/dist/exporters/matrix/gates.d.ts.map +1 -0
- package/dist/exporters/matrix/gates.js +295 -0
- package/dist/exporters/matrix/gates.js.map +1 -0
- package/dist/exporters/matrix/map-loader.d.ts +22 -0
- package/dist/exporters/matrix/map-loader.d.ts.map +1 -0
- package/dist/exporters/matrix/map-loader.js +191 -0
- package/dist/exporters/matrix/map-loader.js.map +1 -0
- package/dist/exporters/matrix/render-csv.d.ts +9 -0
- package/dist/exporters/matrix/render-csv.d.ts.map +1 -0
- package/dist/exporters/matrix/render-csv.js +89 -0
- package/dist/exporters/matrix/render-csv.js.map +1 -0
- package/dist/exporters/matrix/render-xlsx.d.ts +25 -0
- package/dist/exporters/matrix/render-xlsx.d.ts.map +1 -0
- package/dist/exporters/matrix/render-xlsx.js +362 -0
- package/dist/exporters/matrix/render-xlsx.js.map +1 -0
- package/dist/exporters/matrix/types.d.ts +175 -0
- package/dist/exporters/matrix/types.d.ts.map +1 -0
- package/dist/exporters/matrix/types.js +15 -0
- package/dist/exporters/matrix/types.js.map +1 -0
- package/dist/exporters/matrix/wording.d.ts +45 -0
- package/dist/exporters/matrix/wording.d.ts.map +1 -0
- package/dist/exporters/matrix/wording.js +150 -0
- package/dist/exporters/matrix/wording.js.map +1 -0
- package/dist/exporters/scenario-merger.js +1 -1
- package/dist/exporters/scenario-merger.js.map +1 -1
- package/dist/exporters/spec-parser.js +1 -1
- package/dist/exporters/spec-parser.js.map +1 -1
- package/dist/exporters/xlsx-report-builder.d.ts +1 -1
- package/dist/exporters/xlsx-report-builder.d.ts.map +1 -1
- package/dist/exporters/xlsx-report-builder.js +3 -3
- package/dist/exporters/xlsx-report-builder.js.map +1 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +6 -1
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/capability.d.ts +25 -0
- package/dist/harness/capability.d.ts.map +1 -1
- package/dist/harness/capability.js +44 -2
- package/dist/harness/capability.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +1 -1
- package/dist/orchestrator/templates/ai-src/commands/delivery.md +144 -120
- package/dist/orchestrator/templates/ai-src/config/claude.md +1 -1
- package/dist/orchestrator/templates/ai-src/config/copilot.md +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +59 -6
- package/dist/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +1 -0
- package/package.json +4 -4
- package/src/cli/commands/delivery.ts +169 -2
- package/src/dashboard/templates/index.html +1 -1
- package/src/exporters/feature-parser.ts +1 -1
- package/src/exporters/matrix/build.ts +432 -0
- package/src/exporters/matrix/export.ts +134 -0
- package/src/exporters/matrix/fingerprint.ts +49 -0
- package/src/exporters/matrix/gates.ts +325 -0
- package/src/exporters/matrix/map-loader.ts +161 -0
- package/src/exporters/matrix/render-csv.ts +91 -0
- package/src/exporters/matrix/render-xlsx.ts +356 -0
- package/src/exporters/matrix/types.ts +202 -0
- package/src/exporters/matrix/wording.ts +157 -0
- package/src/exporters/scenario-merger.ts +1 -1
- package/src/exporters/spec-parser.ts +1 -1
- package/src/exporters/xlsx-report-builder.ts +3 -2
- package/src/harness/audit.ts +7 -2
- package/src/harness/capability.ts +53 -3
- package/src/orchestrator/templates/ai-src/commands/create-test.md +1 -1
- package/src/orchestrator/templates/ai-src/commands/delivery.md +144 -120
- package/src/orchestrator/templates/ai-src/config/claude.md +1 -1
- package/src/orchestrator/templates/ai-src/config/copilot.md +1 -1
- package/src/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +59 -6
- package/src/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +1 -1
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +1 -0
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic quality gates for the Test Case & Coverage Matrix
|
|
3
|
+
* (docs/spec/delivery-coverage-matrix-spec.md §6).
|
|
4
|
+
*
|
|
5
|
+
* Severity behavior:
|
|
6
|
+
* error → blocks any render
|
|
7
|
+
* review → blocks the official render; `--preview` renders with a DRAFT watermark
|
|
8
|
+
* warning → prints + renders
|
|
9
|
+
*
|
|
10
|
+
* The gates recompute every signature part from the .feature — the map is never
|
|
11
|
+
* trusted for derivable facts, only for the semantic fields (target/intent/oracle).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { splitVpAndName, extractTestcaseType } from '../feature-parser';
|
|
15
|
+
import { CoverageVariant, MatrixFinding, MAX_VARIANTS_PER_ITEM } from './types';
|
|
16
|
+
import { expandMapRef, BuildInputs } from './build';
|
|
17
|
+
|
|
18
|
+
export interface GateContext {
|
|
19
|
+
inputs: BuildInputs;
|
|
20
|
+
variants: CoverageVariant[];
|
|
21
|
+
variantsByVp: Map<string, CoverageVariant[]>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function runGates(ctx: GateContext): MatrixFinding[] {
|
|
25
|
+
const findings: MatrixFinding[] = [];
|
|
26
|
+
gateASource(ctx, findings);
|
|
27
|
+
gateBMapping(ctx, findings);
|
|
28
|
+
gateCAggregation(ctx, findings);
|
|
29
|
+
gateDExecutability(ctx, findings);
|
|
30
|
+
gateEDrift(ctx, findings);
|
|
31
|
+
gateGReviewState(ctx, findings);
|
|
32
|
+
gateWWording(ctx, findings);
|
|
33
|
+
return findings;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// --- Gate A — source quality ------------------------------------------------
|
|
37
|
+
|
|
38
|
+
function gateASource(ctx: GateContext, findings: MatrixFinding[]): void {
|
|
39
|
+
const { feature, merged } = ctx.inputs;
|
|
40
|
+
|
|
41
|
+
if (feature.backgroundWhenSteps.length > 0 || feature.backgroundThenSteps.length > 0) {
|
|
42
|
+
findings.push({
|
|
43
|
+
gate: 'A', severity: 'error',
|
|
44
|
+
message: 'Background contains When/Then steps — setup only; move the action/oracle into scenarios',
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const seen = new Map<string, number>();
|
|
49
|
+
for (const m of merged) {
|
|
50
|
+
const { vpId } = splitVpAndName(m.feature.name);
|
|
51
|
+
if (!vpId) {
|
|
52
|
+
findings.push({
|
|
53
|
+
gate: 'A', severity: 'error', ref: m.feature.name,
|
|
54
|
+
message: `scenario "${m.feature.name}" has no VP-id prefix — matrix variants must be addressable`,
|
|
55
|
+
});
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
seen.set(vpId, (seen.get(vpId) ?? 0) + 1);
|
|
59
|
+
|
|
60
|
+
const isManual = extractTestcaseType(m.feature.tags) === 'Manual';
|
|
61
|
+
if (m.resolvedExpected.length === 0) {
|
|
62
|
+
findings.push({
|
|
63
|
+
gate: 'A', severity: 'error', ref: vpId,
|
|
64
|
+
message: isManual
|
|
65
|
+
? `manual scenario ${vpId} has no observable oracle (no Then steps and no Observable/Oracle comment lines)`
|
|
66
|
+
: `scenario ${vpId} has no Then step — a variant without an observable oracle cannot be delivered`,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
for (const [vpId, n] of seen) {
|
|
71
|
+
if (n > 1) findings.push({ gate: 'A', severity: 'error', ref: vpId, message: `duplicate VP-id ${vpId} (${n} scenarios)` });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// --- Gate B — mapping completeness -------------------------------------------
|
|
76
|
+
|
|
77
|
+
function gateBMapping(ctx: GateContext, findings: MatrixFinding[]): void {
|
|
78
|
+
const { map } = ctx.inputs;
|
|
79
|
+
const { variants, variantsByVp } = ctx;
|
|
80
|
+
|
|
81
|
+
// Where each derived variant landed: 0 → unmapped, >1 → double-mapped.
|
|
82
|
+
const owners = new Map<string, string[]>();
|
|
83
|
+
for (const g of map.groups) {
|
|
84
|
+
for (const ref of g.variants) {
|
|
85
|
+
const expanded = expandMapRef(ref, variantsByVp);
|
|
86
|
+
if (expanded.length === 0) {
|
|
87
|
+
findings.push({
|
|
88
|
+
gate: 'B', severity: 'error', ref: g.id,
|
|
89
|
+
message: `group ${g.id} references ${ref} which matches no scenario/@cases row in the .feature`,
|
|
90
|
+
});
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
for (const v of expanded) {
|
|
94
|
+
const list = owners.get(v.ref) ?? [];
|
|
95
|
+
list.push(g.id);
|
|
96
|
+
owners.set(v.ref, list);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
for (const [ref, groups] of owners) {
|
|
102
|
+
if (groups.length > 1) {
|
|
103
|
+
findings.push({
|
|
104
|
+
gate: 'B', severity: 'error', ref,
|
|
105
|
+
message: `variant ${ref} belongs to ${groups.length} groups (${groups.join(', ')}) — exactly one required`,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
for (const v of variants) {
|
|
111
|
+
const grouped = owners.has(v.ref);
|
|
112
|
+
const disposed = v.vpId in map.dispositions;
|
|
113
|
+
if (!grouped && !disposed) {
|
|
114
|
+
findings.push({
|
|
115
|
+
gate: 'B', severity: 'error', ref: v.ref,
|
|
116
|
+
message: `variant ${v.ref} is in no group and has no disposition — 100% source disposition is required`,
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
if (grouped && disposed) {
|
|
120
|
+
findings.push({
|
|
121
|
+
gate: 'B', severity: 'error', ref: v.ref,
|
|
122
|
+
message: `variant ${v.ref} is grouped AND disposed (${map.dispositions[v.vpId].as}) — pick one`,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
for (const vpId of Object.keys(map.dispositions)) {
|
|
128
|
+
if (!variantsByVp.has(vpId)) {
|
|
129
|
+
findings.push({
|
|
130
|
+
gate: 'B', severity: 'warning', ref: vpId,
|
|
131
|
+
message: `disposition for ${vpId} matches no scenario (stale entry?)`,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// --- Gate C — aggregation safety ---------------------------------------------
|
|
138
|
+
|
|
139
|
+
function gateCAggregation(ctx: GateContext, findings: MatrixFinding[]): void {
|
|
140
|
+
const { map } = ctx.inputs;
|
|
141
|
+
const { variantsByVp } = ctx;
|
|
142
|
+
|
|
143
|
+
for (const g of map.groups) {
|
|
144
|
+
const vs = g.variants.flatMap((ref) => expandMapRef(ref, variantsByVp));
|
|
145
|
+
if (vs.length <= 1) continue;
|
|
146
|
+
|
|
147
|
+
// Approval IS the QA confirmation of the heuristic parts (oracle family,
|
|
148
|
+
// precondition equivalence, sequence safety) — once approved AND unchanged
|
|
149
|
+
// (fingerprints intact), only the hard signature checks keep firing. A
|
|
150
|
+
// drifted variant re-opens the heuristics (Gate E flips the item to
|
|
151
|
+
// proposed at the same time).
|
|
152
|
+
const stale = vs.some((v) => {
|
|
153
|
+
const stamped = map.fingerprints[v.ref] ?? map.fingerprints[v.vpId];
|
|
154
|
+
return !stamped || stamped !== v.fingerprint;
|
|
155
|
+
});
|
|
156
|
+
const heuristicsConfirmed = g.review === 'approved' && !stale;
|
|
157
|
+
|
|
158
|
+
// Hard signature parts — recomputed, never trusted from the map. ERROR on mismatch.
|
|
159
|
+
const hard: Array<[string, (v: CoverageVariant) => string]> = [
|
|
160
|
+
['execution mode', (v) => v.mode],
|
|
161
|
+
['test layer', (v) => [...v.layers].sort().join('+')],
|
|
162
|
+
['priority', (v) => v.priority],
|
|
163
|
+
];
|
|
164
|
+
for (const [label, keyOf] of hard) {
|
|
165
|
+
const distinct = new Set(vs.map(keyOf));
|
|
166
|
+
if (distinct.size > 1) {
|
|
167
|
+
findings.push({
|
|
168
|
+
gate: 'C', severity: 'error', ref: g.id,
|
|
169
|
+
message: `group ${g.id} mixes ${label}s (${[...distinct].join(' vs ')}) — mandatory split`,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Heuristic signature parts — REVIEW_REQUIRED (equivalence was inferred, not declared).
|
|
175
|
+
const preProfiles = new Set(vs.map((v) => v.preconditionProfile));
|
|
176
|
+
if (!heuristicsConfirmed && preProfiles.size > 1) {
|
|
177
|
+
findings.push({
|
|
178
|
+
gate: 'C', severity: 'review', ref: g.id,
|
|
179
|
+
message: `group ${g.id} variants have different precondition profiles — confirm the grouping is safe or split`,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
const oracleShapes = new Set(vs.map((v) => v.oracleShape.join(' ; ')));
|
|
183
|
+
if (!heuristicsConfirmed && oracleShapes.size > 1) {
|
|
184
|
+
findings.push({
|
|
185
|
+
gate: 'C', severity: 'review', ref: g.id,
|
|
186
|
+
message: `group ${g.id} variants assert different oracle shapes (${oracleShapes.size} shapes) — confirm they are one oracle family`,
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
if (!heuristicsConfirmed && vs.some((v) => v.procedureProfile === 'sequence')) {
|
|
190
|
+
findings.push({
|
|
191
|
+
gate: 'C', severity: 'review', ref: g.id,
|
|
192
|
+
message: `group ${g.id} contains a sequence-sensitive scenario — ordered flows stay solo items unless QA confirms`,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (vs.length > MAX_VARIANTS_PER_ITEM) {
|
|
197
|
+
findings.push({
|
|
198
|
+
gate: 'C', severity: 'warning', ref: g.id,
|
|
199
|
+
message: `group ${g.id} has ${vs.length} variants (> ${MAX_VARIANTS_PER_ITEM}) — consider splitting for reviewability`,
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// --- Gate D — manual executability -------------------------------------------
|
|
206
|
+
|
|
207
|
+
function gateDExecutability(ctx: GateContext, findings: MatrixFinding[]): void {
|
|
208
|
+
const { map, testData, feature } = ctx.inputs;
|
|
209
|
+
const { variantsByVp } = ctx;
|
|
210
|
+
|
|
211
|
+
for (const g of map.groups) {
|
|
212
|
+
const vs = g.variants.flatMap((ref) => expandMapRef(ref, variantsByVp));
|
|
213
|
+
for (const v of vs) {
|
|
214
|
+
const hasPrecondition = v.precondition.length > 0 || feature.backgroundGivenSteps.length > 0;
|
|
215
|
+
if (!hasPrecondition) {
|
|
216
|
+
findings.push({
|
|
217
|
+
gate: 'D', severity: 'error', ref: v.ref,
|
|
218
|
+
message: `${v.ref}: no precondition (no Background, auth tag, or Given) — a tester cannot establish the start state`,
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
if (v.trigger.length === 0) {
|
|
222
|
+
findings.push({
|
|
223
|
+
gate: 'D', severity: 'error', ref: v.ref,
|
|
224
|
+
message: `${v.ref}: no action/trigger (no When steps${v.mode === 'manual' ? ' and no Setup/Action comment lines' : ''})`,
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
// Oracle emptiness is Gate A (source) — not repeated here.
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// Every {{var}} must resolve — data the tester cannot obtain is not executable.
|
|
232
|
+
for (const [vpId, vs] of variantsByVp) {
|
|
233
|
+
const scenario = ctx.inputs.merged.find((m) => splitVpAndName(m.feature.name).vpId === vpId);
|
|
234
|
+
if (!scenario) continue;
|
|
235
|
+
for (const varName of scenario.feature.referencedVars) {
|
|
236
|
+
const inRows = vs.some((v) => v.caseLabel !== undefined); // dataset rows carry their own columns
|
|
237
|
+
if (!inRows && (!testData || !(varName in testData))) {
|
|
238
|
+
findings.push({
|
|
239
|
+
gate: 'D', severity: 'error', ref: vpId,
|
|
240
|
+
message: `${vpId}: {{${varName}}} does not resolve in test-data — the variant is not reproducible`,
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// No template token may survive into a RENDERED cell (review B-04) — this also
|
|
247
|
+
// catches cross-referencing test-data values the one-level resolver couldn't close.
|
|
248
|
+
for (const [, vs] of variantsByVp) {
|
|
249
|
+
for (const v of vs) {
|
|
250
|
+
const rendered = [...v.precondition, ...v.trigger, ...v.oracle, ...v.verification, ...v.data];
|
|
251
|
+
const token = rendered.map((t) => t.match(/\{\{[^}]*\}\}/)).find(Boolean);
|
|
252
|
+
if (token) {
|
|
253
|
+
findings.push({
|
|
254
|
+
gate: 'D', severity: 'error', ref: v.ref,
|
|
255
|
+
message: `${v.ref}: unresolved template token ${token[0]} remains in the rendered output — the export is not deterministic for a tester`,
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// --- Gate E — drift (stale approval) ------------------------------------------
|
|
263
|
+
|
|
264
|
+
function gateEDrift(ctx: GateContext, findings: MatrixFinding[]): void {
|
|
265
|
+
const { map } = ctx.inputs;
|
|
266
|
+
const { variantsByVp } = ctx;
|
|
267
|
+
|
|
268
|
+
for (const g of map.groups) {
|
|
269
|
+
if (g.review !== 'approved') continue; // proposed groups are already under review
|
|
270
|
+
for (const ref of g.variants) {
|
|
271
|
+
const vs = expandMapRef(ref, variantsByVp);
|
|
272
|
+
for (const v of vs) {
|
|
273
|
+
const stamped = map.fingerprints[v.ref] ?? map.fingerprints[v.vpId];
|
|
274
|
+
if (!stamped) {
|
|
275
|
+
findings.push({
|
|
276
|
+
gate: 'E', severity: 'review', ref: g.id,
|
|
277
|
+
message: `group ${g.id}: ${v.ref} has no stamped fingerprint — run \`sungen delivery --approve\` after review`,
|
|
278
|
+
});
|
|
279
|
+
} else if (stamped !== v.fingerprint) {
|
|
280
|
+
findings.push({
|
|
281
|
+
gate: 'E', severity: 'review', ref: g.id,
|
|
282
|
+
message: `group ${g.id}: ${v.ref} changed since approval (fingerprint drift) — re-review the grouping`,
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// --- Gate W — controlled-language lint on the map's semantic fields --------------
|
|
291
|
+
|
|
292
|
+
/** Patterns that make customer-facing wording read machine-generated (review §10). */
|
|
293
|
+
const WORDING_SMELLS: Array<[RegExp, string]> = [
|
|
294
|
+
[/\{\{[^}]*\}\}/, 'an unresolved {{token}}'],
|
|
295
|
+
[/\b(?:Setup|Observable|Oracle):/, 'a generator label (Setup:/Observable:/Oracle:)'],
|
|
296
|
+
[/\bUser (?:fill|click|press|see|wait)\b/i, 'DSL phrasing ("User fill/click/see…") — write plain product language'],
|
|
297
|
+
[/\[[^\]]+\]/, 'a [selector-style] reference — use the visible UI label'],
|
|
298
|
+
];
|
|
299
|
+
|
|
300
|
+
function gateWWording(ctx: GateContext, findings: MatrixFinding[]): void {
|
|
301
|
+
for (const g of ctx.inputs.map.groups) {
|
|
302
|
+
for (const [field, text] of [['intent', g.intent], ['oracle', g.oracle]] as const) {
|
|
303
|
+
for (const [re, what] of WORDING_SMELLS) {
|
|
304
|
+
if (re.test(text)) {
|
|
305
|
+
findings.push({
|
|
306
|
+
gate: 'W', severity: 'warning', ref: g.id,
|
|
307
|
+
message: `group ${g.id} ${field} contains ${what}`,
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// --- Gate G — review state -----------------------------------------------------
|
|
316
|
+
|
|
317
|
+
function gateGReviewState(ctx: GateContext, findings: MatrixFinding[]): void {
|
|
318
|
+
const proposed = ctx.inputs.map.groups.filter((g) => g.review !== 'approved');
|
|
319
|
+
if (proposed.length > 0) {
|
|
320
|
+
findings.push({
|
|
321
|
+
gate: 'G', severity: 'review',
|
|
322
|
+
message: `${proposed.length} group(s) still \`proposed\` (${proposed.slice(0, 5).map((g) => g.id).join(', ')}${proposed.length > 5 ? ', …' : ''}) — approve them or render with --preview`,
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Delivery Map loader — `qa/<kind>/<unit>/delivery/<unit>.map.yaml`.
|
|
3
|
+
*
|
|
4
|
+
* Structural validation only (shape, required fields, enum values, duplicate ids).
|
|
5
|
+
* Semantic validation against the .feature is the gates' job (gates.ts) — the loader
|
|
6
|
+
* never reads the feature.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import * as fs from 'fs';
|
|
10
|
+
import * as path from 'path';
|
|
11
|
+
import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
|
|
12
|
+
import { DeliveryMap, MapCategory, MapDisposition, MapGroup, RequirementOverride, RequirementStatus, ReviewState } from './types';
|
|
13
|
+
|
|
14
|
+
const CATEGORIES: MapCategory[] = ['normal', 'abnormal', 'security', 'nfr'];
|
|
15
|
+
const REVIEW_STATES: ReviewState[] = ['proposed', 'approved'];
|
|
16
|
+
const DISPOSITIONS = ['excluded', 'blocked', 'covered_elsewhere', 'accepted_risk'];
|
|
17
|
+
const REQUIREMENT_STATUSES: RequirementStatus[] =
|
|
18
|
+
['covered', 'partially_covered', 'covered_elsewhere', 'planned', 'gap', 'not_applicable'];
|
|
19
|
+
|
|
20
|
+
export interface MapLoadResult {
|
|
21
|
+
map: DeliveryMap | null;
|
|
22
|
+
/** Structural problems — any entry blocks the matrix path (fix the file first). */
|
|
23
|
+
errors: string[];
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Path of the map file for a unit directory (qa/<kind>/<screen>) + feature basename. */
|
|
27
|
+
export function mapFilePath(unitDir: string, featureBaseName: string): string {
|
|
28
|
+
return path.join(unitDir, 'delivery', `${featureBaseName}.map.yaml`);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function loadDeliveryMap(file: string): MapLoadResult {
|
|
32
|
+
if (!fs.existsSync(file)) return { map: null, errors: [] };
|
|
33
|
+
const errors: string[] = [];
|
|
34
|
+
let raw: unknown;
|
|
35
|
+
try {
|
|
36
|
+
raw = parseYaml(fs.readFileSync(file, 'utf-8'));
|
|
37
|
+
} catch (e) {
|
|
38
|
+
return { map: null, errors: [`map YAML does not parse: ${e instanceof Error ? e.message : e}`] };
|
|
39
|
+
}
|
|
40
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
41
|
+
return { map: null, errors: ['map root must be a YAML mapping'] };
|
|
42
|
+
}
|
|
43
|
+
const doc = raw as Record<string, unknown>;
|
|
44
|
+
|
|
45
|
+
const groups: MapGroup[] = [];
|
|
46
|
+
const seenIds = new Set<string>();
|
|
47
|
+
const rawGroups = doc.groups;
|
|
48
|
+
if (!Array.isArray(rawGroups) || rawGroups.length === 0) {
|
|
49
|
+
errors.push('map has no `groups` list');
|
|
50
|
+
} else {
|
|
51
|
+
rawGroups.forEach((g, i) => {
|
|
52
|
+
if (!g || typeof g !== 'object') { errors.push(`groups[${i}] is not a mapping`); return; }
|
|
53
|
+
const grp = g as Record<string, unknown>;
|
|
54
|
+
const where = `groups[${i}]${typeof grp.id === 'string' ? ` (${grp.id})` : ''}`;
|
|
55
|
+
for (const field of ['id', 'target', 'intent', 'oracle'] as const) {
|
|
56
|
+
if (typeof grp[field] !== 'string' || !(grp[field] as string).trim()) {
|
|
57
|
+
errors.push(`${where}: missing/empty \`${field}\``);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (!CATEGORIES.includes(grp.category as MapCategory)) {
|
|
61
|
+
errors.push(`${where}: \`category\` must be one of ${CATEGORIES.join(' | ')}`);
|
|
62
|
+
}
|
|
63
|
+
const review = (grp.review ?? 'proposed') as ReviewState;
|
|
64
|
+
if (!REVIEW_STATES.includes(review)) {
|
|
65
|
+
errors.push(`${where}: \`review\` must be proposed | approved`);
|
|
66
|
+
}
|
|
67
|
+
const variants = grp.variants;
|
|
68
|
+
if (!Array.isArray(variants) || variants.length === 0 || variants.some((v) => typeof v !== 'string' || !v.trim())) {
|
|
69
|
+
errors.push(`${where}: \`variants\` must be a non-empty list of VP refs`);
|
|
70
|
+
}
|
|
71
|
+
const id = String(grp.id ?? '');
|
|
72
|
+
if (id) {
|
|
73
|
+
if (seenIds.has(id)) errors.push(`duplicate group id \`${id}\``);
|
|
74
|
+
seenIds.add(id);
|
|
75
|
+
}
|
|
76
|
+
groups.push({
|
|
77
|
+
id,
|
|
78
|
+
target: String(grp.target ?? ''),
|
|
79
|
+
intent: String(grp.intent ?? ''),
|
|
80
|
+
oracle: String(grp.oracle ?? ''),
|
|
81
|
+
category: grp.category as MapCategory,
|
|
82
|
+
review,
|
|
83
|
+
variants: Array.isArray(variants) ? variants.map(String) : [],
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const dispositions: Record<string, MapDisposition> = {};
|
|
89
|
+
if (doc.dispositions !== undefined) {
|
|
90
|
+
if (!doc.dispositions || typeof doc.dispositions !== 'object' || Array.isArray(doc.dispositions)) {
|
|
91
|
+
errors.push('`dispositions` must be a mapping of VP-id → { as, reason }');
|
|
92
|
+
} else {
|
|
93
|
+
for (const [vpId, d] of Object.entries(doc.dispositions as Record<string, unknown>)) {
|
|
94
|
+
const dd = (d && typeof d === 'object' ? d : {}) as Record<string, unknown>;
|
|
95
|
+
if (!DISPOSITIONS.includes(String(dd.as))) {
|
|
96
|
+
errors.push(`dispositions.${vpId}: \`as\` must be one of ${DISPOSITIONS.join(' | ')}`);
|
|
97
|
+
}
|
|
98
|
+
dispositions[vpId] = { as: dd.as as MapDisposition['as'], reason: dd.reason ? String(dd.reason) : undefined };
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const requirements: Record<string, RequirementOverride> = {};
|
|
104
|
+
if (doc.requirements !== undefined) {
|
|
105
|
+
if (!doc.requirements || typeof doc.requirements !== 'object' || Array.isArray(doc.requirements)) {
|
|
106
|
+
errors.push('`requirements` must be a mapping of requirement-id → { status, note }');
|
|
107
|
+
} else {
|
|
108
|
+
for (const [reqId, r] of Object.entries(doc.requirements as Record<string, unknown>)) {
|
|
109
|
+
const rr = (r && typeof r === 'object' ? r : {}) as Record<string, unknown>;
|
|
110
|
+
if (!REQUIREMENT_STATUSES.includes(String(rr.status) as RequirementStatus)) {
|
|
111
|
+
errors.push(`requirements.${reqId}: \`status\` must be one of ${REQUIREMENT_STATUSES.join(' | ')}`);
|
|
112
|
+
}
|
|
113
|
+
requirements[reqId] = { status: rr.status as RequirementStatus, note: rr.note ? String(rr.note) : undefined };
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const fingerprints: Record<string, string> = {};
|
|
119
|
+
if (doc.fingerprints && typeof doc.fingerprints === 'object' && !Array.isArray(doc.fingerprints)) {
|
|
120
|
+
for (const [k, v] of Object.entries(doc.fingerprints as Record<string, unknown>)) {
|
|
121
|
+
fingerprints[k] = String(v);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const map: DeliveryMap = {
|
|
126
|
+
version: Number(doc.version ?? 1),
|
|
127
|
+
unit: String(doc.unit ?? ''),
|
|
128
|
+
formNo: doc.form_no !== undefined ? String(doc.form_no) : undefined,
|
|
129
|
+
groups,
|
|
130
|
+
dispositions,
|
|
131
|
+
requirements,
|
|
132
|
+
fingerprints,
|
|
133
|
+
};
|
|
134
|
+
return { map: errors.length > 0 ? null : map, errors };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Write the map back (used by `--approve` to flip review states + stamp fingerprints).
|
|
139
|
+
* Field order is kept stable so approve-diffs stay minimal and reviewable.
|
|
140
|
+
*/
|
|
141
|
+
export function writeDeliveryMap(file: string, map: DeliveryMap): void {
|
|
142
|
+
const doc: Record<string, unknown> = {
|
|
143
|
+
version: map.version,
|
|
144
|
+
unit: map.unit,
|
|
145
|
+
...(map.formNo !== undefined ? { form_no: map.formNo } : {}),
|
|
146
|
+
groups: map.groups.map((g) => ({
|
|
147
|
+
id: g.id,
|
|
148
|
+
target: g.target,
|
|
149
|
+
intent: g.intent,
|
|
150
|
+
oracle: g.oracle,
|
|
151
|
+
category: g.category,
|
|
152
|
+
review: g.review,
|
|
153
|
+
variants: g.variants,
|
|
154
|
+
})),
|
|
155
|
+
...(Object.keys(map.dispositions).length > 0 ? { dispositions: map.dispositions } : {}),
|
|
156
|
+
...(Object.keys(map.requirements).length > 0 ? { requirements: map.requirements } : {}),
|
|
157
|
+
...(Object.keys(map.fingerprints).length > 0 ? { fingerprints: map.fingerprints } : {}),
|
|
158
|
+
};
|
|
159
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
160
|
+
fs.writeFileSync(file, stringifyYaml(doc, { lineWidth: 0 }), 'utf-8');
|
|
161
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flat CSV rendering of the matrix — same model as the workbook, one row per
|
|
3
|
+
* delivery item (`Level: item`) followed by its variant rows (`Level: variant`;
|
|
4
|
+
* every item has them — the variant row carries the source scenario id, data,
|
|
5
|
+
* and result), so downstream pipelines can re-derive both views from one file.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { statusToTestResult } from '../playwright-report-parser';
|
|
9
|
+
import { itemResultLabel } from './render-xlsx';
|
|
10
|
+
import { MatrixModel } from './types';
|
|
11
|
+
|
|
12
|
+
const HEADERS = [
|
|
13
|
+
'Level', 'ID', 'Target', 'Test Intent / Condition', 'Category', 'Priority',
|
|
14
|
+
'Precondition', 'Coverage / Test Data', 'Action / Trigger', 'Expected Result',
|
|
15
|
+
'Mode', 'Trace', 'Result', 'Executed Date', 'Note',
|
|
16
|
+
];
|
|
17
|
+
|
|
18
|
+
function esc(value: string): string {
|
|
19
|
+
if (/[",\n]/.test(value)) return `"${value.replace(/"/g, '""')}"`;
|
|
20
|
+
return value;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function isoDate(startTime: string | undefined): string {
|
|
24
|
+
if (!startTime) return '';
|
|
25
|
+
const d = new Date(startTime);
|
|
26
|
+
if (isNaN(d.getTime())) return '';
|
|
27
|
+
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function renderMatrixCsv(model: MatrixModel): string {
|
|
31
|
+
const BOM = '\ufeff'; // Excel compatibility (Vietnamese/Japanese content)
|
|
32
|
+
const lines: string[] = [];
|
|
33
|
+
lines.push(`# Test Case & Coverage Matrix — ${model.unit}`);
|
|
34
|
+
lines.push(`# delivery items: ${model.manifest.itemCount} · coverage variants: ${model.manifest.variantCount} · state: ${model.manifest.approvalState}`);
|
|
35
|
+
lines.push(HEADERS.map(esc).join(','));
|
|
36
|
+
|
|
37
|
+
for (const item of model.items) {
|
|
38
|
+
const triggerTexts = new Set(item.variants.map((v) => v.trigger.join('\n')));
|
|
39
|
+
const commonTrigger = triggerTexts.size === 1 ? item.variants[0].trigger.join('\n') : '(differs per variant — see variant rows)';
|
|
40
|
+
lines.push([
|
|
41
|
+
'item',
|
|
42
|
+
item.id,
|
|
43
|
+
item.target,
|
|
44
|
+
item.intent,
|
|
45
|
+
item.category,
|
|
46
|
+
item.priority,
|
|
47
|
+
item.precondition.join('\n'),
|
|
48
|
+
`${item.variants.length} variant(s): ${item.variants.map((v) => v.condition).join(' · ')}`,
|
|
49
|
+
commonTrigger,
|
|
50
|
+
item.oracle,
|
|
51
|
+
item.mode === 'manual' ? 'Manual' : 'Auto',
|
|
52
|
+
item.traces.join(' '),
|
|
53
|
+
itemResultLabel(item),
|
|
54
|
+
'',
|
|
55
|
+
item.review === 'proposed' ? 'REVIEW REQUIRED' : '',
|
|
56
|
+
].map(esc).join(','));
|
|
57
|
+
|
|
58
|
+
for (const v of item.variants) {
|
|
59
|
+
const expected = v.verification.length > 0
|
|
60
|
+
? `${v.oracle.join('\n')}\nVerification method: ${v.verification.join(' ')}`
|
|
61
|
+
: v.oracle.join('\n');
|
|
62
|
+
lines.push([
|
|
63
|
+
'variant',
|
|
64
|
+
v.ref,
|
|
65
|
+
'',
|
|
66
|
+
v.condition,
|
|
67
|
+
'', '',
|
|
68
|
+
'',
|
|
69
|
+
v.data.join('\n'),
|
|
70
|
+
v.trigger.join('\n'),
|
|
71
|
+
expected,
|
|
72
|
+
v.mode === 'manual' ? 'Manual' : 'Auto',
|
|
73
|
+
v.traces.join(' '),
|
|
74
|
+
v.result ? statusToTestResult(v.result.status) : 'Pending',
|
|
75
|
+
isoDate(v.result?.startTime),
|
|
76
|
+
v.result?.error ? String(v.result.error).slice(0, 200) : '',
|
|
77
|
+
].map(esc).join(','));
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Requirement coverage appendix — keeps the CSV self-contained for pipelines.
|
|
82
|
+
if (model.requirements.length > 0) {
|
|
83
|
+
lines.push('');
|
|
84
|
+
lines.push('# Requirement coverage');
|
|
85
|
+
lines.push(['Requirement', 'Status', 'Delivery items', 'Variants', 'Note'].join(','));
|
|
86
|
+
for (const r of model.requirements) {
|
|
87
|
+
lines.push([r.id, r.status, r.items.join(' '), String(r.variantCount), r.note].map(esc).join(','));
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return BOM + lines.join('\n') + '\n';
|
|
91
|
+
}
|