@sun-asterisk/sungen 3.2.23 → 3.2.24-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +40 -8
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/dashboard/templates/index.html +1 -1
- package/dist/exporters/csv-exporter.d.ts.map +1 -1
- package/dist/exporters/csv-exporter.js +3 -1
- package/dist/exporters/csv-exporter.js.map +1 -1
- package/dist/exporters/feature-parser.d.ts +7 -0
- package/dist/exporters/feature-parser.d.ts.map +1 -1
- package/dist/exporters/feature-parser.js +27 -0
- package/dist/exporters/feature-parser.js.map +1 -1
- package/dist/exporters/xlsx-report-builder.d.ts.map +1 -1
- package/dist/exporters/xlsx-report-builder.js +3 -1
- package/dist/exporters/xlsx-report-builder.js.map +1 -1
- package/dist/harness/audit.d.ts +22 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +122 -13
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-contract.d.ts +104 -0
- package/dist/harness/flow-contract.d.ts.map +1 -1
- package/dist/harness/flow-contract.js +199 -12
- package/dist/harness/flow-contract.js.map +1 -1
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +22 -3
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/quality-gates.d.ts +2 -1
- package/dist/harness/quality-gates.d.ts.map +1 -1
- package/dist/harness/quality-gates.js +47 -5
- package/dist/harness/quality-gates.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +48 -5
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/spec-coverage.d.ts +20 -0
- package/dist/harness/spec-coverage.d.ts.map +1 -1
- package/dist/harness/spec-coverage.js +35 -0
- package/dist/harness/spec-coverage.js.map +1 -1
- package/dist/harness/viewpoint-ledger.d.ts +4 -0
- package/dist/harness/viewpoint-ledger.d.ts.map +1 -1
- package/dist/harness/viewpoint-ledger.js +59 -1
- package/dist/harness/viewpoint-ledger.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +28 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +98 -5
- package/dist/orchestrator/templates/ai-src/skills/sungen-viewpoint/group-e-identity.md +1 -1
- package/dist/orchestrator/test-data-guide.d.ts.map +1 -1
- package/dist/orchestrator/test-data-guide.js +8 -0
- package/dist/orchestrator/test-data-guide.js.map +1 -1
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +37 -8
- package/src/dashboard/templates/index.html +1 -1
- package/src/exporters/csv-exporter.ts +3 -1
- package/src/exporters/feature-parser.ts +22 -0
- package/src/exporters/xlsx-report-builder.ts +3 -1
- package/src/harness/audit.ts +131 -17
- package/src/harness/flow-contract.ts +258 -12
- package/src/harness/parse.ts +22 -3
- package/src/harness/quality-gates.ts +48 -5
- package/src/harness/sensors.ts +47 -4
- package/src/harness/spec-coverage.ts +38 -0
- package/src/harness/viewpoint-ledger.ts +58 -1
- package/src/orchestrator/templates/ai-src/commands/create-test.md +28 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +98 -5
- package/src/orchestrator/templates/ai-src/skills/sungen-viewpoint/group-e-identity.md +1 -1
- package/src/orchestrator/test-data-guide.ts +8 -0
|
@@ -27,6 +27,21 @@ import { parse as parseYaml } from 'yaml';
|
|
|
27
27
|
import { ScenarioInfo } from './parse';
|
|
28
28
|
import { readTextFile } from './read-text';
|
|
29
29
|
|
|
30
|
+
/** One declared flow of the use case: a branch point, its own steps, its own outcome. */
|
|
31
|
+
export interface FlowDecl {
|
|
32
|
+
/** The flow id as the use-case document numbers it — `BF`, `AF01`, `EF03`. */
|
|
33
|
+
id: string;
|
|
34
|
+
/** Where it leaves the basic flow (`BF step 2`). Omitted for the basic flow itself. */
|
|
35
|
+
branchFrom?: string;
|
|
36
|
+
/** Its OWN outcome/postcondition — what is different about where this flow ends. */
|
|
37
|
+
outcome?: string;
|
|
38
|
+
status: 'covered' | 'deferred' | 'pending-clarification' | 'out-of-scope';
|
|
39
|
+
/** Why, for anything other than `covered` — a deferral with no reason is a silent gap. */
|
|
40
|
+
reason?: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const FLOW_STATUSES = new Set(['covered', 'deferred', 'pending-clarification', 'out-of-scope']);
|
|
44
|
+
|
|
30
45
|
export interface FlowContract {
|
|
31
46
|
goal: string;
|
|
32
47
|
actor?: string;
|
|
@@ -34,9 +49,27 @@ export interface FlowContract {
|
|
|
34
49
|
precondition?: string;
|
|
35
50
|
outcome: { screen: string; assertion?: string };
|
|
36
51
|
value?: string;
|
|
52
|
+
/** Postcondition when the goal IS reached — the business value, not "API 200". */
|
|
53
|
+
successGuarantee?: string;
|
|
54
|
+
/** Postcondition that holds in EVERY flow, INCLUDING failure — the question
|
|
55
|
+
* "if this fails, what must still be true?" that Exception Flows assert against.
|
|
56
|
+
* Without it an EF has no anchor and its assertions are guesswork (#592). */
|
|
57
|
+
minimalGuarantee?: string;
|
|
37
58
|
/** Journey phases this flow declares. Default [HP, ER, EH]; UI is allowed but
|
|
38
59
|
* never demanded (presentation is the balance axis's business, not coverage's). */
|
|
39
60
|
phases: string[];
|
|
61
|
+
/**
|
|
62
|
+
* The use case's declared flow inventory — the answer to "how many flows does this use
|
|
63
|
+
* case HAVE?". `phases:` alone cannot answer it: a phase is present as soon as ONE
|
|
64
|
+
* scenario carries it, so a fifteen-flow use case with three scenarios reported full
|
|
65
|
+
* phase coverage. With an inventory, coverage is measured per DECLARED FLOW and a flow
|
|
66
|
+
* nobody wrote is a named gap instead of a silent absence (#595).
|
|
67
|
+
*
|
|
68
|
+
* `status` is what keeps a gap honest: every flow ends up `covered`, `deferred`,
|
|
69
|
+
* `pending-clarification` (an open question for the BA — the behaviour is not agreed yet,
|
|
70
|
+
* so no scenario can be right) or `out-of-scope`. Absent → `covered` is expected.
|
|
71
|
+
*/
|
|
72
|
+
flows?: FlowDecl[];
|
|
40
73
|
/** The mutated collection (cart, order, application …) — enables regression dims. */
|
|
41
74
|
stateful?: string;
|
|
42
75
|
budgets?: Record<string, number>;
|
|
@@ -72,6 +105,9 @@ export interface FlowQualityResult {
|
|
|
72
105
|
|
|
73
106
|
const DEFAULT_PHASES = ['HP', 'ER', 'EH'];
|
|
74
107
|
|
|
108
|
+
/** The phase token of a flow id segment, branch number removed: `EF01` → `EF`, `HP` → `HP`. */
|
|
109
|
+
const phaseToken = (seg: string): string => seg.replace(/\d+$/, '').toUpperCase();
|
|
110
|
+
|
|
75
111
|
export function flowContractPath(unitDir: string): string {
|
|
76
112
|
return path.join(unitDir, 'requirements', 'flow-contract.yaml');
|
|
77
113
|
}
|
|
@@ -105,7 +141,26 @@ export function loadFlowContract(unitDir: string): { contract: FlowContract | nu
|
|
|
105
141
|
precondition: raw.precondition !== undefined ? String(raw.precondition) : undefined,
|
|
106
142
|
outcome: { screen: String(outcome!.screen).toLowerCase(), assertion: outcome!.assertion !== undefined ? String(outcome!.assertion) : undefined },
|
|
107
143
|
value: raw.value !== undefined ? String(raw.value) : undefined,
|
|
144
|
+
successGuarantee: raw.successGuarantee !== undefined ? String(raw.successGuarantee) : (raw.success_guarantee !== undefined ? String(raw.success_guarantee) : undefined),
|
|
145
|
+
minimalGuarantee: raw.minimalGuarantee !== undefined ? String(raw.minimalGuarantee) : (raw.minimal_guarantee !== undefined ? String(raw.minimal_guarantee) : undefined),
|
|
108
146
|
phases,
|
|
147
|
+
flows: Array.isArray(raw.flows)
|
|
148
|
+
? (raw.flows as Array<Record<string, unknown>>)
|
|
149
|
+
.filter((f) => f && typeof f === 'object' && f.id)
|
|
150
|
+
.map((f) => {
|
|
151
|
+
const status = String(f.status ?? 'covered').toLowerCase();
|
|
152
|
+
if (!FLOW_STATUSES.has(status)) {
|
|
153
|
+
errors.push(`flows[${String(f.id)}].status "${status}" is not one of covered|deferred|pending-clarification|out-of-scope`);
|
|
154
|
+
}
|
|
155
|
+
return {
|
|
156
|
+
id: String(f.id).toUpperCase(),
|
|
157
|
+
branchFrom: f.branchFrom !== undefined ? String(f.branchFrom) : (f.branch_from !== undefined ? String(f.branch_from) : undefined),
|
|
158
|
+
outcome: f.outcome !== undefined ? String(f.outcome) : undefined,
|
|
159
|
+
status: (FLOW_STATUSES.has(status) ? status : 'covered') as FlowDecl['status'],
|
|
160
|
+
reason: f.reason !== undefined ? String(f.reason) : undefined,
|
|
161
|
+
};
|
|
162
|
+
})
|
|
163
|
+
: undefined,
|
|
109
164
|
stateful: raw.stateful !== undefined ? String(raw.stateful).toLowerCase() : undefined,
|
|
110
165
|
budgets: (raw.budgets && typeof raw.budgets === 'object') ? raw.budgets as Record<string, number> : undefined,
|
|
111
166
|
external: Array.isArray(raw.external)
|
|
@@ -119,7 +174,9 @@ export function loadFlowContract(unitDir: string): { contract: FlowContract | nu
|
|
|
119
174
|
: undefined,
|
|
120
175
|
golden: raw.golden === true,
|
|
121
176
|
},
|
|
122
|
-
errors
|
|
177
|
+
// Shape errors found while reading the inventory (a bad `status:`) are REPORTED with the
|
|
178
|
+
// contract, not swallowed — the contract is still usable, the typo is not silently ignored.
|
|
179
|
+
errors,
|
|
123
180
|
};
|
|
124
181
|
}
|
|
125
182
|
|
|
@@ -134,12 +191,32 @@ function touchesOutcome(s: ScenarioInfo, outcomeScreen: string): boolean {
|
|
|
134
191
|
return namespacesInOrder(s).includes(outcomeScreen);
|
|
135
192
|
}
|
|
136
193
|
|
|
194
|
+
/**
|
|
195
|
+
* Does the scenario ARRIVE at the outcome, as opposed to merely naming it?
|
|
196
|
+
*
|
|
197
|
+
* A guard scenario asserts the outcome screen is *absent* (`[Complete:Title] header is hidden`)
|
|
198
|
+
* — the strongest thing it can say — so "the outcome namespace appears somewhere in the steps"
|
|
199
|
+
* counts it as reaching a screen it exists to prove unreachable.
|
|
200
|
+
*/
|
|
201
|
+
function reachesOutcome(s: ScenarioInfo, outcomeScreen: string): boolean {
|
|
202
|
+
// Per STEP, from the structured list — `stepsText` is one space-joined blob, so any negation
|
|
203
|
+
// anywhere in the scenario would suppress every positive assertion in it.
|
|
204
|
+
const steps = (s.steps ?? []).map((st) => st.text.toLowerCase());
|
|
205
|
+
if (steps.length === 0) return touchesOutcome(s, outcomeScreen);
|
|
206
|
+
return steps.some((t) => t.includes(`[${outcomeScreen}`)
|
|
207
|
+
&& !/\b(is hidden|is not visible|does not exist|is absent|no longer)\b/.test(t));
|
|
208
|
+
}
|
|
209
|
+
|
|
137
210
|
/** Phase of a scenario: its declared phase token (FL-HP-001 / VP-FLOW-ER-02 / MS-EH-005)
|
|
138
211
|
* when present, else vocabulary detection. */
|
|
139
212
|
export function phaseOf(s: ScenarioInfo, declared: string[]): string | null {
|
|
140
213
|
const id = (s.vpId ?? '').toUpperCase();
|
|
214
|
+
// A DECLARED phase wins, matched on the id's segments with the branch number removed —
|
|
215
|
+
// so the use-case vocabulary (VP-EF01-01, VP-AF02-01: one id per Exception/Alternate
|
|
216
|
+
// Flow) resolves to its phase exactly like the flat HP/ER/EH form does.
|
|
217
|
+
const segs = id.split('-').map(phaseToken);
|
|
141
218
|
for (const ph of declared) {
|
|
142
|
-
if (
|
|
219
|
+
if (segs.includes(phaseToken(ph))) return ph;
|
|
143
220
|
}
|
|
144
221
|
const hay = s.haystack;
|
|
145
222
|
if (declared.includes('EH') && /\b(direct access|without (a |the )?(submit|login)|browser back|refresh|expired|tamper|unauthoriz|redirect(ed)? (back )?to|guard)\b/.test(hay)) return 'EH';
|
|
@@ -148,6 +225,101 @@ export function phaseOf(s: ScenarioInfo, declared: string[]): string | null {
|
|
|
148
225
|
return null;
|
|
149
226
|
}
|
|
150
227
|
|
|
228
|
+
/** The flow id a scenario claims: the `AF02`/`EF11`/`BF` segment of its viewpoint id. */
|
|
229
|
+
export function flowIdOf(s: ScenarioInfo, declaredPhases: string[]): string | null {
|
|
230
|
+
const tokens = new Set(declaredPhases.map(phaseToken));
|
|
231
|
+
for (const seg of (s.vpId ?? '').toUpperCase().split('-')) {
|
|
232
|
+
if (tokens.has(phaseToken(seg))) return seg;
|
|
233
|
+
}
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export interface InventoryResult {
|
|
238
|
+
/** Declared → the scenarios claiming it. A declared flow with none is a NAMED gap. */
|
|
239
|
+
covered: Array<{ id: string; scenarios: string[] }>;
|
|
240
|
+
/** Declared `covered` but nothing written — the silent-absence case, now named. */
|
|
241
|
+
uncovered: FlowDecl[];
|
|
242
|
+
/** Declared with a non-covered status, carried into the report so it stays visible. */
|
|
243
|
+
accounted: FlowDecl[];
|
|
244
|
+
/** A scenario's flow id that the inventory never declares — an id invented for one
|
|
245
|
+
* assertion (an "EF" that is really a success postcondition, an "AF" that is really a
|
|
246
|
+
* content check on the basic path). */
|
|
247
|
+
undeclared: Array<{ id: string; scenario: string }>;
|
|
248
|
+
/** Covered declared flows / declared flows that OUGHT to be covered. */
|
|
249
|
+
ratio: number;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Coverage per DECLARED FLOW, which is not the same question as coverage per phase.
|
|
254
|
+
*
|
|
255
|
+
* A phase is "covered" the moment one scenario carries it, so a use case decomposed into
|
|
256
|
+
* fifteen flows reported `BF=✓ AF=✓ EF=✓ → 100%` on three scenarios. Reviewers reading the
|
|
257
|
+
* suite counted the flows instead and got a very different number (#595). When the contract
|
|
258
|
+
* declares its inventory, this measures the thing the reviewer measures — and every gap is
|
|
259
|
+
* named, which is what makes "no silent missing flow" checkable rather than aspirational.
|
|
260
|
+
*/
|
|
261
|
+
export function flowInventory(contract: FlowContract, scenarios: ScenarioInfo[]): InventoryResult | null {
|
|
262
|
+
if (!contract.flows || contract.flows.length === 0) return null;
|
|
263
|
+
const byId = new Map<string, string[]>();
|
|
264
|
+
const undeclared: Array<{ id: string; scenario: string }> = [];
|
|
265
|
+
const declaredIds = new Set(contract.flows.map((f) => f.id));
|
|
266
|
+
for (const s of scenarios) {
|
|
267
|
+
const id = flowIdOf(s, contract.phases);
|
|
268
|
+
if (!id) continue;
|
|
269
|
+
if (!declaredIds.has(id)) { undeclared.push({ id, scenario: s.name }); continue; }
|
|
270
|
+
byId.set(id, [...(byId.get(id) ?? []), s.name]);
|
|
271
|
+
}
|
|
272
|
+
const expected = contract.flows.filter((f) => f.status === 'covered');
|
|
273
|
+
const covered = expected.filter((f) => (byId.get(f.id) ?? []).length > 0)
|
|
274
|
+
.map((f) => ({ id: f.id, scenarios: byId.get(f.id)! }));
|
|
275
|
+
return {
|
|
276
|
+
covered,
|
|
277
|
+
uncovered: expected.filter((f) => (byId.get(f.id) ?? []).length === 0),
|
|
278
|
+
accounted: contract.flows.filter((f) => f.status !== 'covered'),
|
|
279
|
+
undeclared,
|
|
280
|
+
ratio: expected.length ? covered.length / expected.length : 1,
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* A phase id used for something that is not that kind of flow.
|
|
286
|
+
*
|
|
287
|
+
* Two shapes, both seen on a real suite that the phase check scored 100%:
|
|
288
|
+
* - an `EF` scenario that REACHES the contract outcome. An exception flow is blocked by
|
|
289
|
+
* definition; one that completes the journey is a success-path postcondition wearing an
|
|
290
|
+
* exception's id ("the buffer does not survive a finalized registration").
|
|
291
|
+
* - an `AF` scenario with no branch: it walks the basic path and asserts extra content on
|
|
292
|
+
* it. An alternate flow needs a point where the actor does something else; without one it
|
|
293
|
+
* is an assertion belonging to the basic flow ("the shared component renders the new-email
|
|
294
|
+
* copy").
|
|
295
|
+
* Both inflate the flow count while adding no branch coverage, which is exactly what makes a
|
|
296
|
+
* suite look complete to the harness and thin to a reviewer.
|
|
297
|
+
*/
|
|
298
|
+
export function misfiledPhases(
|
|
299
|
+
contract: FlowContract, scenarios: ScenarioInfo[], basicPhase: string,
|
|
300
|
+
): Array<{ scenario: string; id: string; why: string }> {
|
|
301
|
+
const out: Array<{ scenario: string; id: string; why: string }> = [];
|
|
302
|
+
const outcome = contract.outcome.screen;
|
|
303
|
+
for (const s of scenarios) {
|
|
304
|
+
const id = flowIdOf(s, contract.phases);
|
|
305
|
+
if (!id || phaseToken(id) === phaseToken(basicPhase)) continue;
|
|
306
|
+
const ph = phaseToken(id);
|
|
307
|
+
if (ph === 'EF' && reachesOutcome(s, outcome)) {
|
|
308
|
+
// An error-then-recover flow legitimately ends at the outcome — it is the recovery that
|
|
309
|
+
// is being proven, and a guard flow is blocked rather than "failed". Only a scenario with
|
|
310
|
+
// no failure and no block at all is misfiled.
|
|
311
|
+
const failed = /\b(error|invalid|duplicate|reject|fail|denied|blocked|expired|refuse|unauthenticat|unauthoriz|redirect|guard|without (?:a |an |the )?\w+|not skippable|forces? \w+ to restart)\w*/i.test(s.haystack);
|
|
312
|
+
if (!failed) {
|
|
313
|
+
out.push({ scenario: s.name, id, why: `reaches the outcome screen [${outcome}] and nothing in it fails — this is a success-path postcondition, not an exception flow` });
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
if (ph === 'AF' && !/\b(back|cancel|return|instead|abandon|second tab|another tab|skip|leave|exit|retry|edit)\w*/i.test(s.haystack)) {
|
|
317
|
+
out.push({ scenario: s.name, id, why: 'no branch point — it walks the basic path and asserts extra content on it, so the assertion belongs to the basic flow' });
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
return out;
|
|
321
|
+
}
|
|
322
|
+
|
|
151
323
|
/**
|
|
152
324
|
* Verify the suite against the contract. Deterministic; a flow without a contract
|
|
153
325
|
* returns hasContract:false and neutral values (the audit reports the checklist).
|
|
@@ -171,10 +343,14 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
171
343
|
// --- Scope creep: a scenario that never touches the outcome and is not a ----
|
|
172
344
|
// guard/error phase is evidence of a SECOND business goal in this flow.
|
|
173
345
|
const declaredPhases = contract.phases;
|
|
346
|
+
const basic = declaredPhases.filter((p) => p !== 'UI')[0];
|
|
174
347
|
const offGoalScenarios = scenarios.filter((s) => {
|
|
175
348
|
if (touchesOutcome(s, outcomeScreen)) return false;
|
|
176
349
|
const ph = phaseOf(s, declaredPhases);
|
|
177
|
-
|
|
350
|
+
// Any NON-basic declared phase (guards, error recovery, alternate branches) legitimately
|
|
351
|
+
// stops before the outcome — that is what a branch IS. Only an unclassified scenario that
|
|
352
|
+
// never reaches the outcome is evidence of a second business goal.
|
|
353
|
+
return ph === null || ph === basic;
|
|
178
354
|
});
|
|
179
355
|
const offGoalRatio = scenarios.length ? offGoalScenarios.length / scenarios.length : 0;
|
|
180
356
|
const offGoalCategories = Array.from(new Set(
|
|
@@ -182,10 +358,13 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
182
358
|
|
|
183
359
|
// --- Phase coverage: the flow's coverage axis (UI never demanded) -----------
|
|
184
360
|
const demanded = declaredPhases.filter((p) => p !== 'UI');
|
|
361
|
+
// The FIRST declared phase is the Basic Flow (the primer allows exactly one) — it is the
|
|
362
|
+
// phase that must reach the declared outcome, whether the project spells it HP or BF.
|
|
363
|
+
const basicPhase = demanded[0];
|
|
185
364
|
const phases = demanded.map((phase) => {
|
|
186
365
|
const inPhase = scenarios.filter((s) => phaseOf(s, declaredPhases) === phase);
|
|
187
|
-
//
|
|
188
|
-
const relevant = phase ===
|
|
366
|
+
// The basic flow must additionally prove the outcome — a data assertion elsewhere is not the goal.
|
|
367
|
+
const relevant = phase === basicPhase ? inPhase.filter((s) => touchesOutcome(s, outcomeScreen)) : inPhase;
|
|
189
368
|
return {
|
|
190
369
|
phase,
|
|
191
370
|
covered: relevant.length > 0,
|
|
@@ -243,6 +422,47 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
243
422
|
};
|
|
244
423
|
}
|
|
245
424
|
|
|
425
|
+
/**
|
|
426
|
+
* A continuity claim states that data SURVIVES a step boundary (buffered, persists,
|
|
427
|
+
* round-trips, restored). Every such mechanism has two sides, and a suite that proves
|
|
428
|
+
* only the surviving side has not tested the mechanism — it has tested the happy half.
|
|
429
|
+
*
|
|
430
|
+
* Field report (#592): the viewpoint declared "basic-info values are buffered in
|
|
431
|
+
* sessionStorage … a page reload or session-storage clear between those steps LOSES the
|
|
432
|
+
* buffered data". The suite proved back-navigation RESTORES the values and never proved a
|
|
433
|
+
* reload loses them — one mechanism, one side. The QA framework caught it by construction
|
|
434
|
+
* (AF-02 "giữ được" beside EF-11 "mất"); this sensor catches it deterministically.
|
|
435
|
+
*
|
|
436
|
+
* Advisory. Reported per claim, naming which side is missing.
|
|
437
|
+
*/
|
|
438
|
+
export interface ContinuityGap { claim: string; missing: 'loss' | 'persistence' }
|
|
439
|
+
|
|
440
|
+
const CONTINUITY_CLAIM = /\b(buffer(?:s|ed|ing)?|persist(?:s|ed|ence)?|round[- ]?trips?|restor(?:e|es|ed|ation)|carr(?:y|ied|ies)|retain(?:s|ed)?|session[- ]?storage|local[- ]?storage)\b/i;
|
|
441
|
+
const SIDE_PERSIST = /\b(restor(?:e|es|ed)|persist(?:s|ed)?|unchanged|same value|round[- ]?trips?|re-?hydrat\w*|still (?:shows|holds)|carr(?:y|ied|ies)|retain(?:s|ed)?|prefilled|pre-?filled)\b/i;
|
|
442
|
+
const SIDE_LOSS = /\b(lose|loses|lost|cleared?|clears|discard\w*|reload\w*|refresh\w*|expire\w*|empty|blank|not restored|no longer|wiped?|reset)\b/i;
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* Continuity claims in the viewpoint file whose feature proves only ONE side.
|
|
446
|
+
* `claims` are the viewpoint's atomic item texts; `scenarioTexts` are per-scenario
|
|
447
|
+
* name+steps blobs (a claim's two sides may live in different scenarios).
|
|
448
|
+
*/
|
|
449
|
+
export function continuityGaps(claims: string[], scenarioTexts: string[]): ContinuityGap[] {
|
|
450
|
+
const out: ContinuityGap[] = [];
|
|
451
|
+
const anySide = (re: RegExp) => scenarioTexts.some((t) => re.test(t));
|
|
452
|
+
for (const claim of claims) {
|
|
453
|
+
if (!CONTINUITY_CLAIM.test(claim)) continue;
|
|
454
|
+
// Only judge a claim that itself names BOTH sides, or names the loss side: a claim that
|
|
455
|
+
// only ever promises persistence has no second side to demand.
|
|
456
|
+
const claimNamesLoss = SIDE_LOSS.test(claim);
|
|
457
|
+
if (!claimNamesLoss) continue;
|
|
458
|
+
const provesPersist = anySide(SIDE_PERSIST);
|
|
459
|
+
const provesLoss = anySide(SIDE_LOSS);
|
|
460
|
+
if (provesPersist && !provesLoss) out.push({ claim: claim.slice(0, 110), missing: 'loss' });
|
|
461
|
+
else if (provesLoss && !provesPersist) out.push({ claim: claim.slice(0, 110), missing: 'persistence' });
|
|
462
|
+
}
|
|
463
|
+
return out;
|
|
464
|
+
}
|
|
465
|
+
|
|
246
466
|
/**
|
|
247
467
|
* Generalized stateful regression depth: the contract names the mutated collection,
|
|
248
468
|
* so the three dims (count-proof · teardown · multi-source) stop being cart-only.
|
|
@@ -251,11 +471,37 @@ export function statefulDepthFor(collection: string, scenarios: ScenarioInfo[]):
|
|
|
251
471
|
const hay = scenarios.map((s) => s.haystack);
|
|
252
472
|
const any = (re: RegExp) => hay.some((h) => re.test(h));
|
|
253
473
|
const noun = collection.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
474
|
+
|
|
475
|
+
// Not every stateful flow is a COLLECTION. A cart accumulates rows, so counting them and
|
|
476
|
+
// feeding them from several sources are real regression dimensions; a registration state
|
|
477
|
+
// machine has neither, and demanding them produced advice no author could act on ("add to
|
|
478
|
+
// the cart from EVERY source" on a signup journey — #592). So each dimension is demanded
|
|
479
|
+
// only where the suite shows the vocabulary that makes it meaningful, and the ratio is
|
|
480
|
+
// renormalised over the applicable ones — the same "absent evidence scores nothing" rule
|
|
481
|
+
// the score axes already follow.
|
|
482
|
+
const accumulates = any(new RegExp(`\\b(rows?|items?|lines?|entries|quantity|qty|count)\\b`)) ||
|
|
483
|
+
any(new RegExp(`\\b${noun}\\s+(?:rows?|items?|lines?|count)\\b`));
|
|
484
|
+
const hasSources = any(/\b(recommended|related|you may also|another source|second (?:list|source)|from (?:the )?(?:list|rail|grid))\b/);
|
|
485
|
+
|
|
486
|
+
const countProof = any(/\b(quantity|qty|row count|count|number of|two (?:rows|lines|items)|\d+ (?:rows|lines|items))\b/);
|
|
487
|
+
// The inverse operation. A collection's is remove/clear; a state machine's is lose, expire,
|
|
488
|
+
// reset, restart, invalidate — and the outcome side below already accepted `lost`/`gone`
|
|
489
|
+
// while the verb side had no `lose`, so a suite proving exactly that was told it had not
|
|
490
|
+
// (#592: "add a REMOVE scenario" on a registration journey that already proves the buffer
|
|
491
|
+
// is lost and the link expires).
|
|
492
|
+
const teardown = any(/\b(remove|delete|clear|cancel|withdraw|abandon|lapse|expire|lose|lost|reset|restart|discard|invalidate|revert)(?:s|d|es|ed|ing|n)?\b/) &&
|
|
493
|
+
any(new RegExp(`\\b(empty|emptied|no items|zero|removed|cleared|cancelled|withdrawn|lost|gone|0 items|does not exist|no longer|awaiting|restart)\\b|empty[- ]${noun}`));
|
|
494
|
+
const multiSource = hasSources &&
|
|
495
|
+
any(new RegExp(`\\b(add|submit|create|place).{0,40}${noun}|${noun}.{0,40}\\b(add|submit|create|place)`));
|
|
496
|
+
|
|
497
|
+
const dims: Array<[string, boolean, boolean]> = [
|
|
498
|
+
['count-proof', countProof, accumulates],
|
|
499
|
+
['teardown', teardown, true],
|
|
500
|
+
['multi-source', multiSource, hasSources || accumulates],
|
|
501
|
+
];
|
|
502
|
+
const applicable = dims.filter(([, , app]) => app);
|
|
503
|
+
const missing = applicable.filter(([, v]) => !v).map(([k]) => k);
|
|
504
|
+
const ratio = applicable.length ? (applicable.length - missing.length) / applicable.length : 1;
|
|
505
|
+
return { countProof, teardown, multiSource, missing, ratio };
|
|
261
506
|
}
|
|
507
|
+
|
package/src/harness/parse.ts
CHANGED
|
@@ -65,7 +65,10 @@ export function idPrefix(id: string): string {
|
|
|
65
65
|
* QA to re-tag scenarios that were already correct.
|
|
66
66
|
*/
|
|
67
67
|
function isViewpointId(s: string): boolean {
|
|
68
|
-
|
|
68
|
+
// `FL-BF` / `FL-EF` are category ids exactly like `VP-LOGIC` — a flow declaring its
|
|
69
|
+
// phases under the FL- family had NO parsed viewpoints at all, which silently zeroed
|
|
70
|
+
// taxonomy and excluded traceability (#592).
|
|
71
|
+
return isIdLike(s) || /^(?:VP|FL)-[A-Z][A-Z0-9]*$/i.test(s);
|
|
69
72
|
}
|
|
70
73
|
|
|
71
74
|
export function parseViewpointOverview(filePath: string): ViewpointEntry[] {
|
|
@@ -141,6 +144,19 @@ export function parseViewpointOverview(filePath: string): ViewpointEntry[] {
|
|
|
141
144
|
|
|
142
145
|
const PRIORITY_TAGS: Record<string, Priority> = { '@high': 'high', '@normal': 'normal', '@low': 'low' };
|
|
143
146
|
|
|
147
|
+
/**
|
|
148
|
+
* Background steps EXECUTE for every scenario, so a sensor that reads only a scenario's own
|
|
149
|
+
* steps mis-measures the suite: a flow whose start state lives in `Background: Given User is
|
|
150
|
+
* on [Registration] page` looked like no scenario ever opened there, and downstream-scope
|
|
151
|
+
* reported the entry screen as merely navigated-to (#592). Prepending them here fixes every
|
|
152
|
+
* sensor at once — the same steps the runtime prepends.
|
|
153
|
+
*/
|
|
154
|
+
function withBackground(sc: ParsedScenario, background?: ParsedScenario): ParsedScenario {
|
|
155
|
+
const bg = background?.steps ?? [];
|
|
156
|
+
if (bg.length === 0) return sc;
|
|
157
|
+
return { ...sc, steps: [...bg, ...((sc.steps as ParsedStep[]) || [])] } as ParsedScenario;
|
|
158
|
+
}
|
|
159
|
+
|
|
144
160
|
function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
145
161
|
const tags = sc.tags || [];
|
|
146
162
|
const deferredToFlow = tags.includes('@deferred:flow');
|
|
@@ -171,7 +187,10 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
171
187
|
// not just single-word ones. A single-word category (VP-CART-001) still works. (H1)
|
|
172
188
|
// Flows use journey-phase ids (FL-HP-001 / FL-ER-002) — the VP- anchor rejected them, so every
|
|
173
189
|
// flow scenario had NO category and the whole suite bucketed `other` (taxonomy=0%, #569).
|
|
174
|
-
|
|
190
|
+
// Segments may carry digits: the use-case framework numbers its flows (VP-AF02-01,
|
|
191
|
+
// VP-EF11-01 — one id per Alternate/Exception Flow), and `[A-Z]+` also silently truncated
|
|
192
|
+
// VP-I18N-001 to no category at all (the `18` broke the class mid-segment).
|
|
193
|
+
const codeMatch = sc.name.match(/\b(?:VP|FL)-([A-Z][A-Z0-9]*(?:-[A-Z0-9]+)*)-\d+/i);
|
|
175
194
|
const vpCode = codeMatch ? codeMatch[0].toUpperCase() : undefined;
|
|
176
195
|
const category = codeMatch ? codeMatch[1].toUpperCase() : undefined;
|
|
177
196
|
// Project-scheme ID: the leading token of the title (VP0-001 / MS-HP-001 / VP-LIST-001).
|
|
@@ -245,7 +264,7 @@ export function loadScenarios(featurePath: string): ScenarioInfo[] {
|
|
|
245
264
|
const feature = parser.parseFeatureFile(featurePath);
|
|
246
265
|
return (feature.scenarios || [])
|
|
247
266
|
.filter((s) => !s.stepsName && !s.hookType) // skip @steps/@hook blocks
|
|
248
|
-
.map(classifyScenario);
|
|
267
|
+
.map((s) => classifyScenario(withBackground(s, feature.background)));
|
|
249
268
|
}
|
|
250
269
|
|
|
251
270
|
/**
|
|
@@ -39,9 +39,20 @@ function downstreamRoutes(specText: string): string[] {
|
|
|
39
39
|
* share state, so cart count/quantity asserts race. Mitigations: @cleanup:cart, @isolate, a fresh
|
|
40
40
|
* browser context, or a "Given … empty cart" background. Returns true when the risk is unmitigated.
|
|
41
41
|
*/
|
|
42
|
-
export function isolationRisk(featureText: string, stateful: boolean): boolean {
|
|
42
|
+
export function isolationRisk(featureText: string, stateful: boolean, collection = 'cart'): boolean {
|
|
43
43
|
if (!stateful || !/@parallel\b/i.test(featureText)) return false;
|
|
44
|
-
|
|
44
|
+
// What discharges the risk is isolation of THE MUTATED COLLECTION — so the cleanup tag has
|
|
45
|
+
// to name it (`@cleanup:cart` for a cart; `@cleanup:registration` for a registration flow).
|
|
46
|
+
// A cleanup of something else (`@cleanup:overlay`) leaves the shared state shared.
|
|
47
|
+
// Collection-independent mechanisms count for any flow: an explicit @isolate, a fresh
|
|
48
|
+
// browser context, unique data per run, or a Background restoring an empty start state.
|
|
49
|
+
// The collection is contract-declared now, so this no longer assumes a cart (#592: a
|
|
50
|
+
// registration flow was told it "mutates the cart"); `cart` stays the default for suites
|
|
51
|
+
// whose statefulness was inferred rather than declared.
|
|
52
|
+
const slug = collection.replace(/[^a-z0-9]+/gi, '[- ]?');
|
|
53
|
+
const named = new RegExp(`@cleanup:${slug}\\b|empty ${slug}\\b`, 'i');
|
|
54
|
+
const generic = /@isolate\b|empty state\b|fresh (?:browser )?context|new context|own fresh|{{\$timestamp}}/i;
|
|
55
|
+
return !named.test(featureText) && !generic.test(featureText);
|
|
45
56
|
}
|
|
46
57
|
|
|
47
58
|
/**
|
|
@@ -64,18 +75,50 @@ export function serialCascadeRisk(featureText: string, scenarioCount: number): n
|
|
|
64
75
|
return scenarioCount;
|
|
65
76
|
}
|
|
66
77
|
|
|
78
|
+
/**
|
|
79
|
+
* Does a scenario OPEN ON the screen this route names? The route gives a URL slug
|
|
80
|
+
* (`basic-info`, `register`); the feature names screens by human label (`[Basic Info]`,
|
|
81
|
+
* `[Registration]`). A raw substring test fails on BOTH shapes of that mismatch — and
|
|
82
|
+
* did, in the field: `"registration".includes("register")` is FALSE ("registration" is
|
|
83
|
+
* regist+ration), and `"basic info".includes("basic-info")` is false on the separator.
|
|
84
|
+
* Every `Given User is on [Registration] page` was therefore invisible, so the sensor
|
|
85
|
+
* reported "only a page-nav assertion" for routes the suite genuinely operates on.
|
|
86
|
+
*
|
|
87
|
+
* Compare normalized (alphanumeric-only, lowercase): containment either way, or a shared
|
|
88
|
+
* prefix long enough to be the same word family (≥5 chars AND ≥60% of the shorter form) —
|
|
89
|
+
* which pairs register/registration and basic-info/"Basic Info", while keeping
|
|
90
|
+
* login/logout apart (shared prefix "log" is 3).
|
|
91
|
+
*/
|
|
92
|
+
const norm = (s: string): string => s.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
93
|
+
|
|
94
|
+
export function sameScreenName(slug: string, label: string): boolean {
|
|
95
|
+
const a = norm(slug), b = norm(label);
|
|
96
|
+
if (!a || !b) return false;
|
|
97
|
+
if (a.includes(b) || b.includes(a)) return true;
|
|
98
|
+
let i = 0;
|
|
99
|
+
while (i < a.length && i < b.length && a[i] === b[i]) i++;
|
|
100
|
+
return i >= 5 && i / Math.min(a.length, b.length) >= 0.6;
|
|
101
|
+
}
|
|
102
|
+
|
|
67
103
|
export function downstreamScope(specText: string, scenarios: ScenarioInfo[]): DownstreamResult {
|
|
68
104
|
const routes = downstreamRoutes(specText);
|
|
69
105
|
const underCovered: { route: string; slug: string }[] = [];
|
|
70
106
|
for (const route of routes) {
|
|
71
107
|
const slug = (route.split('/').filter(Boolean).pop() || route).toLowerCase();
|
|
72
|
-
|
|
108
|
+
// Referenced when the route/slug appears literally, OR when any `[Ref]` in the suite
|
|
109
|
+
// names the same screen under its human label (register ↔ [Registration]).
|
|
110
|
+
const labelsOf = (s: ScenarioInfo): string[] =>
|
|
111
|
+
[...s.haystack.matchAll(/\[([^\]]+)\]/g)].map((m) => m[1].split(':')[0]);
|
|
112
|
+
const refs = scenarios.filter((s) =>
|
|
113
|
+
s.haystack.includes(slug) || s.haystack.includes(route.toLowerCase()) ||
|
|
114
|
+
labelsOf(s).some((l) => sameScreenName(slug, l)));
|
|
73
115
|
if (!refs.length) continue; // not referenced at all — out of this screen's scope entirely
|
|
74
116
|
// Substantively covered only if some scenario OPERATES on the downstream — i.e. it
|
|
75
117
|
// starts there (`is on [<downstream>]`) — not merely navigates to it as a terminal
|
|
76
118
|
// `see [<downstream>] page` assertion. The latter just proves the transition.
|
|
77
|
-
const
|
|
78
|
-
|
|
119
|
+
const contentCovered = refs.some((s) =>
|
|
120
|
+
[...s.haystack.matchAll(/\bis on \[([^\]]+)\]/g)]
|
|
121
|
+
.some((m) => sameScreenName(slug, m[1].split(':')[0])));
|
|
79
122
|
if (!contentCovered) underCovered.push({ route, slug });
|
|
80
123
|
}
|
|
81
124
|
return { downstreamRoutes: routes, underCovered };
|
package/src/harness/sensors.ts
CHANGED
|
@@ -40,6 +40,11 @@ const PHASE_BUCKETS: Record<string, string> = {
|
|
|
40
40
|
HP: 'business-core', // happy path = the business goal itself
|
|
41
41
|
ER: 'validation-security', // error recovery (validation must not trap the journey)
|
|
42
42
|
EH: 'validation-security', // guards & leakage (direct access, back, refresh)
|
|
43
|
+
// Use-case vocabulary (ISTQB/UC primer): one Basic Flow, N Alternate, N Exception.
|
|
44
|
+
// Alternate is a designed-for branch (not an error) → behaviour; Exception is blocked → guard side.
|
|
45
|
+
BF: 'business-core',
|
|
46
|
+
AF: 'behavior',
|
|
47
|
+
EF: 'validation-security',
|
|
43
48
|
};
|
|
44
49
|
|
|
45
50
|
/** Classify a VP category into a balance bucket by keyword containment + precedence (H1). */
|
|
@@ -47,7 +52,9 @@ export function bucketForCategory(category: string | undefined): string {
|
|
|
47
52
|
const cat = (category || '').toUpperCase();
|
|
48
53
|
if (!cat) return 'other';
|
|
49
54
|
for (const seg of cat.split('-')) {
|
|
50
|
-
|
|
55
|
+
// A flow id numbers its branch (AF02, EF11) — the phase is the letter part.
|
|
56
|
+
const phase = seg.replace(/\d+$/, '');
|
|
57
|
+
if (PHASE_BUCKETS[phase]) return PHASE_BUCKETS[phase];
|
|
51
58
|
}
|
|
52
59
|
for (const [bucket, kws] of BUCKET_ORDER) {
|
|
53
60
|
if (kws.some((k) => cat.includes(k))) return bucket;
|
|
@@ -498,8 +505,17 @@ export interface TraceResult {
|
|
|
498
505
|
note: string;
|
|
499
506
|
}
|
|
500
507
|
|
|
508
|
+
/**
|
|
509
|
+
* A viewpoint id without its family prefix. The use-case framework numbers flows and the
|
|
510
|
+
* two prefixes in circulation mean the same thing: a viewpoint declared `VP-BF` is the same
|
|
511
|
+
* family as a scenario coded `FL-BF-001` (and vice versa). Matching on the raw string made
|
|
512
|
+
* a correctly-traced flow suite read as 15% traceable (#592).
|
|
513
|
+
*/
|
|
514
|
+
const unprefixed = (id: string): string => id.replace(/^(?:VP|FL)-/i, '').toUpperCase();
|
|
515
|
+
|
|
501
516
|
export function traceability(scenarios: ScenarioInfo[], viewpoints: ViewpointEntry[]): TraceResult {
|
|
502
517
|
const overviewIds = new Set(viewpoints.map((v) => v.id.toUpperCase()));
|
|
518
|
+
const overviewBare = new Set(viewpoints.map((v) => unprefixed(v.id)));
|
|
503
519
|
// A scenario carries an ID if it has a project-scheme leading ID (vpId) or a VP-CAT code.
|
|
504
520
|
const withCode = scenarios.filter((s) => s.vpId || s.vpCode);
|
|
505
521
|
// Maps to overview if the scenario's ID, its sequence-stripped prefix, or its VP-CAT code
|
|
@@ -507,6 +523,12 @@ export function traceability(scenarios: ScenarioInfo[], viewpoints: ViewpointEnt
|
|
|
507
523
|
const mapped = withCode.filter((s) => {
|
|
508
524
|
const id = (s.vpId || s.vpCode || '').toUpperCase();
|
|
509
525
|
if (overviewIds.has(id) || overviewIds.has(idPrefix(id))) return true;
|
|
526
|
+
// Prefix-agnostic within the VP-/FL- family, and phase-family aware: FL-EF01-001 traces
|
|
527
|
+
// to a declared VP-EF (the branch number is part of the flow id, not of the family).
|
|
528
|
+
const bare = unprefixed(id), barePrefix = unprefixed(idPrefix(id));
|
|
529
|
+
if (overviewBare.has(bare) || overviewBare.has(barePrefix)) return true;
|
|
530
|
+
if ([...overviewBare].some((ob) => bare.startsWith(ob) || barePrefix.startsWith(ob) ||
|
|
531
|
+
bare.replace(/\d+$/, '').startsWith(ob.replace(/\d+$/, '')) && ob.length >= 2)) return true;
|
|
510
532
|
return [...overviewIds].some((oid) => id.startsWith(oid) || oid.startsWith(idPrefix(id)) || (!!s.category && oid.includes(s.category)));
|
|
511
533
|
});
|
|
512
534
|
return {
|
|
@@ -621,7 +643,11 @@ const CLAIM_RULES: ClaimRule[] = [
|
|
|
621
643
|
// "double-click does not create two orders" — not a per-feature keyword.
|
|
622
644
|
claim: 'no-side-effect/no-duplicate',
|
|
623
645
|
title: /(?=.*\b(submit|sen[dt]|resend|resubmit|re-?fire|re-?issue|re-?post|repost|create|charge|order|payment|\bpay\b|email|request|\botp\b|insert|register|book|duplicate|double[- ]?submit|again|twice)\b)(?=.*(\bno\b|\bnot\b|n['’]t\b|\bnever\b|\bwithout\b|\bcannot\b|prevent|block|avoid|reject|disabl|\bdeny\b|denies|\bkhông\b|\bchưa\b))/i,
|
|
624
|
-
|
|
646
|
+
// `is disabled` counts: when the spec's own mechanism against a repeat is "the control
|
|
647
|
+
// is disabled immediately" (FR-014-style), asserting the disabled state IS the contrast —
|
|
648
|
+
// the second activation cannot occur. Without it the canonical double-submit proof shape
|
|
649
|
+
// was unrecognised (#592).
|
|
650
|
+
proof: /\bcount\b|ok_count|status_counts|row with \{\{|table with|tohavecount|is hidden|are hidden|is disabled|are disabled|not complete|no longer/,
|
|
625
651
|
need: 'a record/request-count proof (count stays at one, e.g. `User see [Table] row with {{count}}`, an API `{{name.ok_count}}` invariant, or a `@query` DB count) or @manual with a request-count oracle',
|
|
626
652
|
hint: 'a "does-not-happen / does-not-repeat" claim about a state-changing action is NOT proven by a terminal `see [...] page` — that page is identical whether or not the action (re-)fired. Prove the side-effect count is unchanged, or mark @manual with a setup→action→assert-no-duplicate oracle.',
|
|
627
653
|
severity: 'fail',
|
|
@@ -728,9 +754,23 @@ const CATEGORY_SIGNALS: { cat: string; re: RegExp }[] = [
|
|
|
728
754
|
{ cat: 'LIST', re: /\b(product (list|grid)|featured product|product card|each (featured )?card|every (featured )?card)\b/ },
|
|
729
755
|
{ cat: 'NAV', re: /\b(navigates?|returns the user|redirect\w*|logo returns|menu navigates)\b/ },
|
|
730
756
|
{ cat: 'VAL', re: /\b(valid email|invalid\b|without (a |an )?@|without a domain|empty (email|input|field)|validation|required field)\b/ },
|
|
731
|
-
|
|
757
|
+
// An access GUARD is security even when its only observable is a redirect: "direct access
|
|
758
|
+
// … without a valid context redirects to X" used to match NAV (`redirect\w*`) and nothing in
|
|
759
|
+
// SEC, so the linter told QA to re-tag genuine auth-gate tests as navigation — diluting the
|
|
760
|
+
// visibility of the security suite (QA field report, 3.2.23). The guard vocabulary below is
|
|
761
|
+
// what makes the INTENT matchable, and GUARD_INTENT then wins over a mere observable.
|
|
762
|
+
{ cat: 'SEC', re: /\b(xss|sql|injection|payload|without authentication|unauthenticated|without an? (?:authenticated|valid|active) \w+|direct(?:ly)? access\w*|deep[- ]link\w*|auth[- ]?guard|not skippable|without having (?:completed|submitted|logged)|remains? unauthenticated|\binert\b|tamper\w*|malformed|url parameter|query param\w*|falls? back|gracefull?y|without a crash|not[- ]found|\b404\b|special character|abuse)\b/ },
|
|
732
763
|
];
|
|
733
764
|
|
|
765
|
+
/**
|
|
766
|
+
* Categories whose signal describes the test's INTENT/risk rather than the observable action.
|
|
767
|
+
* When one of these matches, a competing observable-only category (NAV: "redirects", "navigates")
|
|
768
|
+
* must not out-vote it — the redirect is HOW the guard shows itself, not what the test is about.
|
|
769
|
+
*/
|
|
770
|
+
const INTENT_OVER_OBSERVABLE: Record<string, Set<string>> = {
|
|
771
|
+
SEC: new Set(['NAV']),
|
|
772
|
+
};
|
|
773
|
+
|
|
734
774
|
export function taxonomyLint(scenarios: ScenarioInfo[]): TaxonomyResult {
|
|
735
775
|
const mislabeled: TaxonomyFinding[] = [];
|
|
736
776
|
let checked = 0;
|
|
@@ -741,7 +781,10 @@ export function taxonomyLint(scenarios: ScenarioInfo[]): TaxonomyResult {
|
|
|
741
781
|
const title = s.name.toLowerCase();
|
|
742
782
|
const matched = [...new Set(CATEGORY_SIGNALS.filter((c) => c.re.test(title)).map((c) => c.cat))];
|
|
743
783
|
if (matched.includes(cat)) continue; // title supports its own category → fine
|
|
744
|
-
|
|
784
|
+
// The declared category may be an INTENT category whose signal this title spells
|
|
785
|
+
// differently; never "correct" it toward a category that only names the observable.
|
|
786
|
+
const suppressed = INTENT_OVER_OBSERVABLE[cat];
|
|
787
|
+
const others = matched.filter((c) => c !== cat && !(suppressed && suppressed.has(c)));
|
|
745
788
|
if (others.length === 1) { // unambiguous mismatch
|
|
746
789
|
const sig = CATEGORY_SIGNALS.find((c) => c.cat === others[0])!;
|
|
747
790
|
const m = title.match(sig.re);
|
|
@@ -202,3 +202,41 @@ export function specCoverage(specPath: string, scenarios: ScenarioInfo[], featur
|
|
|
202
202
|
|
|
203
203
|
return { hasSpec: true, frTotal: frs.length, frCovered, uncoveredMust, inferredOnly, triggerGaps, verdict };
|
|
204
204
|
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* A flow's requirement list that is a HAND RESTATEMENT of the screen specs it traverses.
|
|
208
|
+
*
|
|
209
|
+
* A flow spec typically cites its requirements as belonging elsewhere — "ST_AUTH_002 FR-001",
|
|
210
|
+
* "restated here in flow terms". That restatement is lossy by construction, and nothing checked
|
|
211
|
+
* it: on a real run the flow spec restated two of the screens' FRs, `specFR` read **2/2 = 100%**,
|
|
212
|
+
* and the guard clause the flow most needed (a double-submit rule in one of those screen specs)
|
|
213
|
+
* had never entered the system at all. The axis was certifying complete coverage of a
|
|
214
|
+
* hand-truncated universe — a worse failure than a missing scenario, because the number said the
|
|
215
|
+
* opposite.
|
|
216
|
+
*
|
|
217
|
+
* So: name the source documents the flow says its requirements come from, and say plainly which
|
|
218
|
+
* of them the project does not hold. Whether the restatement is complete is then a question
|
|
219
|
+
* someone can answer, instead of one nothing was asking.
|
|
220
|
+
*/
|
|
221
|
+
export function restatedRequirementSources(specText: string, availableUnits: string[]): {
|
|
222
|
+
restated: boolean; sources: string[]; missing: string[];
|
|
223
|
+
} {
|
|
224
|
+
// "these ... originate in the SCREEN specs", "restated here", "per ST_AUTH_002 FR-001".
|
|
225
|
+
const restated = /\brestate[sd]?\b|\boriginate[sd]? in\b|\bderived from the (?:screen|per-screen) spec/i.test(specText);
|
|
226
|
+
// External document ids carrying their own requirement number: `ST_AUTH_002 FR-001`,
|
|
227
|
+
// `SCR-1-SYS-0001.FR-3`. Two+ segments and an uppercase head, so a bare `FR-001` (the flow's
|
|
228
|
+
// own) never matches.
|
|
229
|
+
// A citation may name SEVERAL documents at once — "ST_AUTH_002/ST_AUTH_004 FR-001" — so match
|
|
230
|
+
// the whole slash/comma-joined run and split it. Capturing only the token adjacent to the
|
|
231
|
+
// requirement number silently dropped every sibling.
|
|
232
|
+
const DOC = '[A-Z][A-Z0-9]*(?:[_-][A-Z0-9]+)+';
|
|
233
|
+
const cite = new RegExp(`\\b((?:${DOC})(?:\\s*[/,]\\s*(?:${DOC}))*)[\\s.]+(?:FR|NFR|BR)-\\d+`, 'g');
|
|
234
|
+
const sources = [...new Set(
|
|
235
|
+
[...specText.matchAll(cite)].flatMap((m) => m[1].split(/\s*[/,]\s*/)).map((x) => x.trim()).filter(Boolean),
|
|
236
|
+
)];
|
|
237
|
+
if (!restated && sources.length === 0) return { restated: false, sources: [], missing: [] };
|
|
238
|
+
const norm = (x: string): string => x.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
239
|
+
const have = availableUnits.map(norm);
|
|
240
|
+
const missing = sources.filter((src) => !have.some((u) => u.includes(norm(src)) || norm(src).includes(u)));
|
|
241
|
+
return { restated, sources, missing };
|
|
242
|
+
}
|