@sun-asterisk/sungen 3.2.25 → 3.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +23 -4
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/exporters/matrix/build.d.ts.map +1 -1
- package/dist/exporters/matrix/build.js +4 -1
- package/dist/exporters/matrix/build.js.map +1 -1
- package/dist/exporters/matrix/map-loader.d.ts.map +1 -1
- package/dist/exporters/matrix/map-loader.js +5 -0
- package/dist/exporters/matrix/map-loader.js.map +1 -1
- package/dist/exporters/matrix/types.d.ts +11 -0
- package/dist/exporters/matrix/types.d.ts.map +1 -1
- package/dist/exporters/matrix/types.js.map +1 -1
- package/dist/harness/audit.d.ts +7 -0
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +74 -6
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-contract.d.ts +18 -1
- package/dist/harness/flow-contract.d.ts.map +1 -1
- package/dist/harness/flow-contract.js +72 -9
- package/dist/harness/flow-contract.js.map +1 -1
- package/dist/harness/quality-gates.d.ts +12 -1
- package/dist/harness/quality-gates.d.ts.map +1 -1
- package/dist/harness/quality-gates.js +62 -7
- package/dist/harness/quality-gates.js.map +1 -1
- package/dist/harness/spec-branches.d.ts +88 -0
- package/dist/harness/spec-branches.d.ts.map +1 -0
- package/dist/harness/spec-branches.js +280 -0
- package/dist/harness/spec-branches.js.map +1 -0
- package/dist/harness/spec-coverage.d.ts +1 -1
- package/dist/harness/spec-coverage.js +4 -4
- package/dist/harness/spec-coverage.js.map +1 -1
- package/dist/harness/viewpoint-baseline.d.ts +9 -0
- package/dist/harness/viewpoint-baseline.d.ts.map +1 -1
- package/dist/harness/viewpoint-baseline.js +33 -3
- package/dist/harness/viewpoint-baseline.js.map +1 -1
- package/dist/harness/viewpoint-ledger.d.ts.map +1 -1
- package/dist/harness/viewpoint-ledger.js +63 -5
- package/dist/harness/viewpoint-ledger.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/add-flow.md +16 -0
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +9 -0
- package/dist/orchestrator/templates/ai-src/commands/delivery.md +9 -2
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +31 -1
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +22 -3
- package/src/exporters/matrix/build.ts +4 -1
- package/src/exporters/matrix/map-loader.ts +5 -0
- package/src/exporters/matrix/types.ts +11 -0
- package/src/harness/audit.ts +78 -8
- package/src/harness/flow-contract.ts +87 -9
- package/src/harness/quality-gates.ts +64 -6
- package/src/harness/spec-branches.ts +346 -0
- package/src/harness/spec-coverage.ts +4 -4
- package/src/harness/viewpoint-baseline.ts +41 -6
- package/src/harness/viewpoint-ledger.ts +56 -4
- package/src/orchestrator/templates/ai-src/commands/add-flow.md +16 -0
- package/src/orchestrator/templates/ai-src/commands/create-test.md +9 -0
- package/src/orchestrator/templates/ai-src/commands/delivery.md +9 -2
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +31 -1
package/src/harness/audit.ts
CHANGED
|
@@ -27,6 +27,8 @@ import { specCoverage, SpecCoverageResult, parseSpecClauses, restatedRequirement
|
|
|
27
27
|
import { downstreamScope, manualOracle, readText, DownstreamResult, ManualOracleResult,
|
|
28
28
|
negativeSideEffect, sourceBacked, crossArtifactOwnership, isolationRisk, serialCascadeRisk } from './quality-gates';
|
|
29
29
|
import { viewpointLedger, parseViewpointItems, browserGestureSubstitutions, viewpointIntegrity, LedgerResult } from './viewpoint-ledger';
|
|
30
|
+
import { specBranches, unenumeratedBranches, sameOracleClusters, permissionPairGaps, awaitingSeedData, unconsideredRisks, MULTI_SCREEN_RISKS } from './spec-branches';
|
|
31
|
+
const MULTI_SCREEN_RISK_COUNT = MULTI_SCREEN_RISKS.length;
|
|
30
32
|
import { capabilityRegistry } from '../capabilities/registry';
|
|
31
33
|
import { discoverAndRegisterCapabilities } from '../capabilities/discover';
|
|
32
34
|
import { contextRouter } from '../capabilities/context-router';
|
|
@@ -65,6 +67,12 @@ export interface AuditReport {
|
|
|
65
67
|
score: {
|
|
66
68
|
overall: number; // 0..10, business-weighted over the APPLICABLE axes
|
|
67
69
|
notApplicable?: string[]; // axes with no evidence — excluded, not scored as 1.0
|
|
70
|
+
/**
|
|
71
|
+
* Why each excluded axis is excluded. "No evidence" and "evidence WITHHELD pending review"
|
|
72
|
+
* are different states with different remedies, and reporting both as "no evidence" told the
|
|
73
|
+
* author of a 93%-covered ledger to go write the viewpoint they had just written (#657).
|
|
74
|
+
*/
|
|
75
|
+
naReasons?: Record<string, string>;
|
|
68
76
|
cappedAt?: number; // set when a weak critical axis holds the score down
|
|
69
77
|
coverage: number; // 0..1
|
|
70
78
|
businessDepth: number; // 0..1
|
|
@@ -76,7 +84,7 @@ export interface AuditReport {
|
|
|
76
84
|
* `specFR` and `atomicLedger` entirely — so a flow's real coverage axis was never printed
|
|
77
85
|
* while `balance`, which carried no weight, was.
|
|
78
86
|
*/
|
|
79
|
-
axes: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean; byDesign?: boolean }>;
|
|
87
|
+
axes: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean; byDesign?: boolean; withheld?: boolean }>;
|
|
80
88
|
formula: string;
|
|
81
89
|
};
|
|
82
90
|
gateStatus: 'PASS' | 'FAIL';
|
|
@@ -196,7 +204,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
196
204
|
const isUiFlowUnit = /^flows\//.test(catalogScreenName);
|
|
197
205
|
const flowQ: FlowQualityResult = isUiFlowUnit
|
|
198
206
|
? flowQuality(screenDir, scenarios)
|
|
199
|
-
: { hasContract: false, errors: [], outcomeProven: false, outcomeManualOnly: false, offGoal: [], offGoalRatio: 0, offGoalCategories: [], phases: [], phaseRatio: 1, handoffs: { total: 0, asserted: 0, ratio: 1 }, sit: [] };
|
|
207
|
+
: { hasContract: false, errors: [], outcomeProven: false, outcomeManualOnly: false, offGoal: [], offGoalRatio: 0, offGoalCategories: [], phases: [], phaseRatio: 1, danglingPhaseRefs: [], handoffs: { total: 0, asserted: 0, ratio: 1 }, sit: [] };
|
|
200
208
|
const gateCatalog = (flowQ.hasContract && !declaredPageType(viewpointText, catalog))
|
|
201
209
|
? { ...catalog, page_types: {} } as Catalog
|
|
202
210
|
: catalog;
|
|
@@ -230,7 +238,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
230
238
|
// #1 taxonomy-match: when the project defines a viewpoint taxonomy, scenarios must use it.
|
|
231
239
|
const taxonomyMismatch = viewpoints.length > 0 && trace.withVpCode > 0 && trace.mappedRatio < 0.6;
|
|
232
240
|
// #2 downstream-scope + #4 manual-oracle
|
|
233
|
-
const downstream = downstreamScope(readText(specPath), scenarios);
|
|
241
|
+
const downstream = downstreamScope(readText(specPath), scenarios, featureText);
|
|
234
242
|
const manualOracleResult = manualOracle(featureText);
|
|
235
243
|
const autoManual = automatableManual(scenarios); // TQ-2 — @manual that is really automatable
|
|
236
244
|
const ledger = viewpointLedger(viewpointPath, scenarios, featureText);
|
|
@@ -331,7 +339,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
331
339
|
// this KIND of unit". Only the first is a reason to cap the score: a contract flow can never
|
|
332
340
|
// supply balance evidence, so capping for it held every flow at 8.9 forever and printed
|
|
333
341
|
// "weakest: flowCoverage 100% — fix it to lift the number", which is not fixable advice (#595).
|
|
334
|
-
const axisDefs: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean; byDesign?: boolean }> = [
|
|
342
|
+
const axisDefs: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean; byDesign?: boolean; withheld?: boolean }> = [
|
|
335
343
|
flowScored
|
|
336
344
|
// A declared flow claimed only by @manual scenarios is ACCOUNTED FOR but nothing runs, so
|
|
337
345
|
// it cannot count the same as an automated one — `covered` was collapsing three very
|
|
@@ -340,10 +348,10 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
340
348
|
? { key: 'flowCoverage', value: inventory ? Math.min((inventory.ratio + inventory.automatedRatio) / 2, flowQ.phaseRatio) : flowQ.phaseRatio, weight: 0.22, applicable: true, critical: true }
|
|
341
349
|
: { key: 'coverage', value: coverage, weight: 0.22, applicable: !!gate.pageType && gate.themesTotal > 0, critical: true },
|
|
342
350
|
{ key: 'specFR', value: specRatio, weight: 0.15, applicable: spec.hasSpec && spec.frTotal > 0, critical: true },
|
|
343
|
-
{ key: 'atomicLedger', value: ledger.ratio, weight: 0.13, applicable: ledger.hasViewpoint && ledger.total > 0 && !viewpointMoved, critical: true },
|
|
351
|
+
{ key: 'atomicLedger', value: ledger.ratio, weight: 0.13, applicable: ledger.hasViewpoint && ledger.total > 0 && !viewpointMoved, critical: true, withheld: viewpointMoved && ledger.hasViewpoint && ledger.total > 0 },
|
|
344
352
|
{ key: 'businessDepth', value: businessDepth, weight: 0.20, applicable: true, critical: true },
|
|
345
353
|
{ key: 'claimProof', value: claim.ratio, weight: 0.15, applicable: claim.withClaims > 0, critical: true },
|
|
346
|
-
{ key: 'traceability', value: traceScore, weight: 0.09, applicable: viewpoints.length > 0 && !viewpointMoved, critical: false },
|
|
354
|
+
{ key: 'traceability', value: traceScore, weight: 0.09, applicable: viewpoints.length > 0 && !viewpointMoved, critical: false, withheld: viewpointMoved && viewpoints.length > 0 },
|
|
347
355
|
// A use-case decomposition is EF-heavy by construction (1 basic flow, N alternates, N
|
|
348
356
|
// exceptions) — that is the shape of a well-decomposed use case, not neglected business
|
|
349
357
|
// core. `flowCoverage` already measures whether the journey's phases are covered, so
|
|
@@ -394,10 +402,24 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
394
402
|
const overall = seniorBandedOverall(rawOverall, { flowStateful: flowDepth.stateful, flowRatio: flowDepth.ratio, oracleWeak: oracle.weak.length, isolationRisk: isoRisk });
|
|
395
403
|
|
|
396
404
|
const findings: string[] = [];
|
|
405
|
+
// A baseline that could not be compared was re-recorded rather than reported as an edit.
|
|
406
|
+
// Migrating in silence would be the same sin one level down, so it is stated.
|
|
407
|
+
if (viewpointBaseline.reparsed) {
|
|
408
|
+
findings.push(`VIEWPOINT-BASELINE-REPARSED: the recorded baseline for this unit was written by an earlier version whose viewpoint parser had a different granularity, so its ids cannot be compared with today's. \`test-viewpoint.md\` itself is unchanged — the file is now fingerprinted by its own bytes, which no parser upgrade can move, and the baseline has been re-recorded against it. \`atomicLedger\` and \`traceability\` count normally. If you DID edit the viewpoint since ${viewpointBaseline.recordedAt?.slice(0, 10) ?? 'it was accepted'}, review that diff yourself — this one migration could not tell an edit from the parser change.`);
|
|
409
|
+
}
|
|
397
410
|
if (capped) {
|
|
398
411
|
const why = weakestCritical && weakestCritical.value < 0.7
|
|
399
412
|
? `"${weakestCritical.key}" is ${(weakestCritical.value * 100).toFixed(0)}% — a weak axis is a hole, not something the other axes can average away`
|
|
400
|
-
:
|
|
413
|
+
: (() => {
|
|
414
|
+
const held = axisDefs.filter((a) => a.withheld).map((a) => a.key);
|
|
415
|
+
const absent = missingEvidence.filter((k) => !held.includes(k.replace(/ .*$/, '')));
|
|
416
|
+
const parts = [];
|
|
417
|
+
if (absent.length) parts.push(`supplies no evidence for [${absent.join(', ')}]`);
|
|
418
|
+
// An axis WAS measured and is being held — saying "no evidence" here sends the author
|
|
419
|
+
// to write a declaration that already exists, when the actual next move is to review a diff.
|
|
420
|
+
if (held.length) parts.push(`has [${held.join(', ')}] measured but WITHHELD until the changed viewpoint baseline is reviewed and accepted`);
|
|
421
|
+
return `this unit ${parts.join(', and ')}, and a top mark has to rest on complete evidence`;
|
|
422
|
+
})();
|
|
401
423
|
findings.push(`SCORE-CAPPED: overall held at ${cap.toFixed(1)} because ${why}.`);
|
|
402
424
|
}
|
|
403
425
|
if (viewpointBaseline.status === 'changed') {
|
|
@@ -622,6 +644,9 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
622
644
|
if (taxonomyMismatch) {
|
|
623
645
|
findings.push(`VP-TAXONOMY-MISMATCH: only ${(trace.mappedRatio * 100).toFixed(0)}% of scenarios use the viewpoint IDs declared in test-viewpoint.md — scenarios invented a generic VP-<CAT> scheme. Re-tag to the project's viewpoint IDs so the coverage matrix is accurate.`);
|
|
624
646
|
}
|
|
647
|
+
for (const a of downstream.absent.slice(0, 5)) {
|
|
648
|
+
findings.push(`NAV-TARGET-ABSENT: the spec declares "${a.route}" as a navigation target and NO scenario mentions it — not even a page assertion. Testing the render conditions of the region that links there does not test the link: a click-through case is what proves the destination. Add one, or record why this flow does not own it.`);
|
|
649
|
+
}
|
|
625
650
|
for (const d of downstream.underCovered) {
|
|
626
651
|
findings.push(`DOWNSTREAM-SCOPE-MISSING: "${d.route}" is a navigation target but is covered only by a page-nav assertion — cover its content/guards, or scaffold it (\`sungen add --screen ${d.slug}\`).`);
|
|
627
652
|
}
|
|
@@ -670,6 +695,46 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
670
695
|
if (depth.deferredBusinessCritical > 0 && depth.deferredBusinessCritical >= depth.businessCriticalTotal) {
|
|
671
696
|
findings.push(`DEPTH-DEFERRED: businessDepth ${businessDepth.toFixed(2)} is computed over only ${depth.businessCriticalTotal} on-screen scenario(s); ${depth.deferredBusinessCritical} business-critical scenario(s) are deferred to @manual (excluded from the ratio). Automate them in a flow and verify with \`sungen flow-check\` — this ratio is NOT "all business depth covered".`);
|
|
672
697
|
}
|
|
698
|
+
// #630 — the branches the spec ENUMERATES must each have a scenario. Narrative guidance for
|
|
699
|
+
// this already existed and was missed three times in three flows, so the check is an enforced
|
|
700
|
+
// enumeration rather than another sentence.
|
|
701
|
+
{
|
|
702
|
+
const branches = specBranches(readText(specPath) ?? '');
|
|
703
|
+
for (const b of unenumeratedBranches(branches, featureText).slice(0, 6)) {
|
|
704
|
+
findings.push(`SPEC-BRANCH-UNCOVERED: the spec enumerates "${b.label}" (${b.source}) and no scenario cites it. A mutually-exclusive branch is only covered when its OWN arm is verified — covering a sibling arm proves nothing about this one. Write it, or record the exclusion in the feature so the omission is a decision.`);
|
|
705
|
+
}
|
|
706
|
+
for (const c of sameOracleClusters(scenarios).slice(0, 4)) {
|
|
707
|
+
findings.push(`BRANCH-OVERCLAIMED: ${c.scenarios.length} scenarios assert an identical oracle (${c.scenarios.slice(0, 3).join(' | ')}${c.scenarios.length > 3 ? ' …' : ''}) — "${c.oracle}". They verify one branch several times while the coverage sheet counts them as ${c.scenarios.length}. Give each the assertion that distinguishes ITS branch, or merge them.`);
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
// #651 F1-7 — the fixed risk catalog, reconciled. SPEC-BRANCH-UNCOVERED enforces the branches a
|
|
711
|
+
// spec DECLARES; this is the half a spec is often silent about, which a reviewer catches and a
|
|
712
|
+
// generator does not. Consideration counts from anywhere the author reasons — a declared flow's
|
|
713
|
+
// branch point or reason, a scenario, the viewpoint — so "thought about, does not apply" is a
|
|
714
|
+
// one-line answer rather than a scenario nobody needs.
|
|
715
|
+
if (flowQ.hasContract) {
|
|
716
|
+
const unconsidered = unconsideredRisks(flowQ.contract!, scenarios, viewpointText);
|
|
717
|
+
if (unconsidered.length > 0) {
|
|
718
|
+
const list = unconsidered.map((r) => `${r.label} (${r.why})`).join(' · ');
|
|
719
|
+
findings.push(`RISK-FAMILY-UNCONSIDERED: ${unconsidered.length} of ${MULTI_SCREEN_RISK_COUNT} multi-screen risk families are not mentioned anywhere in this flow's contract, viewpoint or scenarios — ${list}. Each is a question, not a demand: if it does not apply here, record that in the \`flows:\` inventory as \`out-of-scope\` with the reason and it stops being asked. A flow is "enough" when every family has an answer, and a silent family is the one nobody weighed.`);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
// #630 — a deny-side security case with no allow-side counterpart cannot tell a working guard
|
|
723
|
+
// from a feature that is broken for everyone.
|
|
724
|
+
for (const g of permissionPairGaps(scenarios).slice(0, 4)) {
|
|
725
|
+
findings.push(`PERMISSION-PAIR-MISSING: "${g.scenario}" proves the content is hidden when permission is denied, but no scenario proves it is SHOWN when permission is granted. If the code hid it unconditionally this would still pass — the pair is the test, the way min-1/min/max/max+1 is the test for a boundary. Add the allow-side case${g.subject.length ? ` for [${g.subject.join(', ')}]` : ''}.`);
|
|
726
|
+
}
|
|
727
|
+
// #630 — a scenario blocked on unseeded data is a DATA task, not unfinished test design.
|
|
728
|
+
{
|
|
729
|
+
const tdDir = path.join(screenDir, 'test-data');
|
|
730
|
+
let tdText = '';
|
|
731
|
+
try {
|
|
732
|
+
for (const f of fs.readdirSync(tdDir).filter((x) => x.endsWith('.yaml'))) tdText += `${readText(path.join(tdDir, f)) ?? ''}\n`;
|
|
733
|
+
} catch { /* no test-data yet */ }
|
|
734
|
+
for (const a of awaitingSeedData(tdText, scenarios).slice(0, 4)) {
|
|
735
|
+
findings.push(`AWAITING-SEED-DATA: \`${a.key}\` is still a placeholder and ${a.scenarios.length} scenario(s) reference it (${a.scenarios.slice(0, 2).join(' | ')}${a.scenarios.length > 2 ? ' …' : ''}). These are Pending for a DATA reason, not an unresolved design — seed the value (or move it to \`<unit>.<env>.yaml\`) and they become runnable as written.`);
|
|
736
|
+
}
|
|
737
|
+
}
|
|
673
738
|
// #622 — a claim that names a count, proved by fewer assertions than it names. The count sits
|
|
674
739
|
// in the viewpoint CLAIM, so both the title-level and the claim-level shape are checked.
|
|
675
740
|
for (const g of ledger.partial.slice(0, 5)) {
|
|
@@ -791,7 +856,12 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
791
856
|
// flow) that carried no weight at all.
|
|
792
857
|
axes: axisDefs.map((a) => ({ ...a, weight: a.applicable ? a.weight / weightSum : 0 })),
|
|
793
858
|
formula: `overall = (${scored.map((a) => `${(a.weight / weightSum).toFixed(2)}*${a.key}`).join(' + ')}) * 10`
|
|
794
|
-
+ (
|
|
859
|
+
+ (() => {
|
|
860
|
+
const held = new Set(axisDefs.filter((a) => a.withheld).map((a) => a.key));
|
|
861
|
+
const absent = missingEvidence.filter((k) => !held.has(k.replace(/ .*$/, '')));
|
|
862
|
+
return (absent.length ? ` — n/a (no evidence): ${absent.join(', ')}` : '')
|
|
863
|
+
+ (held.size ? ` — measured but held: ${[...held].join(', ')}` : '');
|
|
864
|
+
})()
|
|
795
865
|
+ (notApplicableByDesign.length ? ` — n/a for this unit kind: ${notApplicableByDesign.join(', ')}` : ''),
|
|
796
866
|
},
|
|
797
867
|
gateStatus,
|
|
@@ -40,6 +40,13 @@ export interface FlowDecl {
|
|
|
40
40
|
reason?: string;
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
/** One declared journey phase. `scenarios:` is the author naming its evidence directly. */
|
|
44
|
+
export interface FlowPhaseDecl {
|
|
45
|
+
id: string;
|
|
46
|
+
description?: string;
|
|
47
|
+
scenarios?: string[];
|
|
48
|
+
}
|
|
49
|
+
|
|
43
50
|
const FLOW_STATUSES = new Set(['covered', 'deferred', 'pending-clarification', 'out-of-scope']);
|
|
44
51
|
|
|
45
52
|
export interface FlowContract {
|
|
@@ -58,6 +65,12 @@ export interface FlowContract {
|
|
|
58
65
|
/** Journey phases this flow declares. Default [HP, ER, EH]; UI is allowed but
|
|
59
66
|
* never demanded (presentation is the balance axis's business, not coverage's). */
|
|
60
67
|
phases: string[];
|
|
68
|
+
/**
|
|
69
|
+
* The phases as DECLARED, when the contract uses the object form. A phase may name the
|
|
70
|
+
* scenarios that carry it — direct evidence, rather than the harness inferring coverage from
|
|
71
|
+
* id segments — and that is what a project which writes `phases:` this way is telling us.
|
|
72
|
+
*/
|
|
73
|
+
phaseDetails?: FlowPhaseDecl[];
|
|
61
74
|
/**
|
|
62
75
|
* The use case's declared flow inventory — the answer to "how many flows does this use
|
|
63
76
|
* case HAVE?". `phases:` alone cannot answer it: a phase is present as soon as ONE
|
|
@@ -94,6 +107,8 @@ export interface FlowQualityResult {
|
|
|
94
107
|
/** Off-goal categories, for the split suggestion ("VP-FILTER-* looks like its own flow"). */
|
|
95
108
|
offGoalCategories: string[];
|
|
96
109
|
phases: { phase: string; covered: boolean; automated: boolean }[];
|
|
110
|
+
/** Phase-declared scenario refs that match no scenario — a phase whose evidence does not exist. */
|
|
111
|
+
danglingPhaseRefs: Array<{ phase: string; ref: string }>;
|
|
97
112
|
/** Covered-and-automated phases / declared phases (UI excluded) — the flow coverage axis. */
|
|
98
113
|
phaseRatio: number;
|
|
99
114
|
/** Cross-namespace transitions followed by an assertion / all transitions. */
|
|
@@ -130,9 +145,34 @@ export function loadFlowContract(unitDir: string): { contract: FlowContract | nu
|
|
|
130
145
|
errors.push('missing `outcome.screen:` — the screen namespace that carries the final proof');
|
|
131
146
|
}
|
|
132
147
|
if (errors.length > 0) return { contract: null, errors };
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
148
|
+
// `phases:` comes in two shapes, and the loader used to assume the first. A real project
|
|
149
|
+
// declared the RICHER one — each phase an object with `name`, `description` and an explicit
|
|
150
|
+
// `scenarios:` list — and `String(x).toUpperCase()` turned every one of them into
|
|
151
|
+
// `[OBJECT OBJECT]`. Nothing matched, phase coverage read 0%, the score fell to 6.9, and the
|
|
152
|
+
// report said only that the phases were uncovered. Garbage in place of an error is the worst
|
|
153
|
+
// of both: the author sees a broken number with no cause (#657).
|
|
154
|
+
//
|
|
155
|
+
// The object form is the better declaration — it names the scenarios that carry the phase
|
|
156
|
+
// instead of leaving the harness to infer them from ids — so it is read, not merely tolerated.
|
|
157
|
+
const phaseDetails: FlowPhaseDecl[] = Array.isArray(raw.phases)
|
|
158
|
+
? (raw.phases as unknown[]).map((x, i) => {
|
|
159
|
+
if (x !== null && typeof x === 'object') {
|
|
160
|
+
const o = x as Record<string, unknown>;
|
|
161
|
+
const name = o.name ?? o.id ?? o.phase;
|
|
162
|
+
if (name === undefined) {
|
|
163
|
+
errors.push(`phases[${i}] is an object with no \`name:\` (or \`id:\`) — a phase needs a label`);
|
|
164
|
+
return { id: `PHASE${i + 1}` };
|
|
165
|
+
}
|
|
166
|
+
return {
|
|
167
|
+
id: String(name).toUpperCase(),
|
|
168
|
+
description: o.description !== undefined ? String(o.description) : undefined,
|
|
169
|
+
scenarios: Array.isArray(o.scenarios) ? (o.scenarios as unknown[]).map((v) => String(v)) : undefined,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
return { id: String(x).toUpperCase() };
|
|
173
|
+
})
|
|
174
|
+
: [];
|
|
175
|
+
const phases = phaseDetails.length > 0 ? phaseDetails.map((p) => p.id) : DEFAULT_PHASES;
|
|
136
176
|
return {
|
|
137
177
|
contract: {
|
|
138
178
|
goal: String(raw.goal),
|
|
@@ -144,6 +184,7 @@ export function loadFlowContract(unitDir: string): { contract: FlowContract | nu
|
|
|
144
184
|
successGuarantee: raw.successGuarantee !== undefined ? String(raw.successGuarantee) : (raw.success_guarantee !== undefined ? String(raw.success_guarantee) : undefined),
|
|
145
185
|
minimalGuarantee: raw.minimalGuarantee !== undefined ? String(raw.minimalGuarantee) : (raw.minimal_guarantee !== undefined ? String(raw.minimal_guarantee) : undefined),
|
|
146
186
|
phases,
|
|
187
|
+
...(phaseDetails.some((p) => p.scenarios || p.description) ? { phaseDetails } : {}),
|
|
147
188
|
flows: Array.isArray(raw.flows)
|
|
148
189
|
? (raw.flows as Array<Record<string, unknown>>)
|
|
149
190
|
.filter((f) => f && typeof f === 'object' && f.id)
|
|
@@ -209,8 +250,17 @@ function reachesOutcome(s: ScenarioInfo, outcomeScreen: string): boolean {
|
|
|
209
250
|
|
|
210
251
|
/** Phase of a scenario: its declared phase token (FL-HP-001 / VP-FLOW-ER-02 / MS-EH-005)
|
|
211
252
|
* when present, else vocabulary detection. */
|
|
212
|
-
export function phaseOf(s: ScenarioInfo, declared: string[]): string | null {
|
|
253
|
+
export function phaseOf(s: ScenarioInfo, declared: string[], details?: FlowPhaseDecl[]): string | null {
|
|
213
254
|
const id = (s.vpId ?? '').toUpperCase();
|
|
255
|
+
// When a phase NAMES its scenarios, that list is the mapping — the author has already done
|
|
256
|
+
// the assignment, and inferring it from id segments instead would ignore them. A phase called
|
|
257
|
+
// "Cross-screen data" shares no token with `VP-LOGIC-002`; only the declaration connects them.
|
|
258
|
+
for (const ph of details ?? []) {
|
|
259
|
+
if (ph.scenarios?.some((ref) => {
|
|
260
|
+
const r = ref.trim().toUpperCase();
|
|
261
|
+
return r === id || (r.length > 0 && s.name.toUpperCase().includes(r));
|
|
262
|
+
})) return ph.id;
|
|
263
|
+
}
|
|
214
264
|
// A DECLARED phase wins, matched on the id's segments with the branch number removed —
|
|
215
265
|
// so the use-case vocabulary (VP-EF01-01, VP-AF02-01: one id per Exception/Alternate
|
|
216
266
|
// Flow) resolves to its phase exactly like the flat HP/ER/EH form does.
|
|
@@ -350,7 +400,7 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
350
400
|
const neutral: FlowQualityResult = {
|
|
351
401
|
hasContract: false, errors, outcomeProven: false, outcomeManualOnly: false,
|
|
352
402
|
offGoal: [], offGoalRatio: 0, offGoalCategories: [],
|
|
353
|
-
phases: [], phaseRatio: 1, handoffs: { total: 0, asserted: 0, ratio: 1 }, sit: [],
|
|
403
|
+
phases: [], phaseRatio: 1, danglingPhaseRefs: [], handoffs: { total: 0, asserted: 0, ratio: 1 }, sit: [],
|
|
354
404
|
};
|
|
355
405
|
if (!contract) return neutral;
|
|
356
406
|
|
|
@@ -367,7 +417,7 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
367
417
|
const basic = declaredPhases.filter((p) => p !== 'UI')[0];
|
|
368
418
|
const offGoalScenarios = scenarios.filter((s) => {
|
|
369
419
|
if (touchesOutcome(s, outcomeScreen)) return false;
|
|
370
|
-
const ph = phaseOf(s, declaredPhases);
|
|
420
|
+
const ph = phaseOf(s, declaredPhases, contract.phaseDetails);
|
|
371
421
|
// Any NON-basic declared phase (guards, error recovery, alternate branches) legitimately
|
|
372
422
|
// stops before the outcome — that is what a branch IS. Only an unclassified scenario that
|
|
373
423
|
// never reaches the outcome is evidence of a second business goal.
|
|
@@ -383,7 +433,7 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
383
433
|
// phase that must reach the declared outcome, whether the project spells it HP or BF.
|
|
384
434
|
const basicPhase = demanded[0];
|
|
385
435
|
const phases = demanded.map((phase) => {
|
|
386
|
-
const inPhase = scenarios.filter((s) => phaseOf(s, declaredPhases) === phase);
|
|
436
|
+
const inPhase = scenarios.filter((s) => phaseOf(s, declaredPhases, contract.phaseDetails) === phase);
|
|
387
437
|
// The basic flow must additionally prove the outcome — a data assertion elsewhere is not the goal.
|
|
388
438
|
const relevant = phase === basicPhase ? inPhase.filter((s) => touchesOutcome(s, outcomeScreen)) : inPhase;
|
|
389
439
|
return {
|
|
@@ -396,6 +446,17 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
396
446
|
? phases.filter((p) => p.covered && p.automated).length / demanded.length
|
|
397
447
|
: 1;
|
|
398
448
|
|
|
449
|
+
// A phase that names a scenario which does not exist is a declaration pointing at nothing:
|
|
450
|
+
// the phase reads covered in the contract and is empty in the suite. Same class as a dangling
|
|
451
|
+
// traceability ref — the map, not the territory.
|
|
452
|
+
const known = new Set(scenarios.flatMap((s) => [s.vpId?.toUpperCase(), s.name.toUpperCase()].filter(Boolean) as string[]));
|
|
453
|
+
const danglingPhaseRefs = (contract.phaseDetails ?? []).flatMap((ph) => (ph.scenarios ?? [])
|
|
454
|
+
.filter((ref) => {
|
|
455
|
+
const r = ref.trim().toUpperCase();
|
|
456
|
+
return !known.has(r) && !scenarios.some((s) => s.name.toUpperCase().includes(r));
|
|
457
|
+
})
|
|
458
|
+
.map((ref) => ({ phase: ph.id, ref })));
|
|
459
|
+
|
|
399
460
|
// --- Handoff integrity: no blind tail after a cross-namespace transition. ---
|
|
400
461
|
// A transition counts as asserted when ANY assertion follows it — in the entered
|
|
401
462
|
// namespace or later. Demanding the assertion in the entered namespace itself
|
|
@@ -439,7 +500,7 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
439
500
|
outcomeProven, outcomeManualOnly,
|
|
440
501
|
offGoal: offGoalScenarios.map((s) => s.name.slice(0, 80)),
|
|
441
502
|
offGoalRatio, offGoalCategories,
|
|
442
|
-
phases, phaseRatio, handoffs, sit,
|
|
503
|
+
phases, phaseRatio, danglingPhaseRefs, handoffs, sit,
|
|
443
504
|
};
|
|
444
505
|
}
|
|
445
506
|
|
|
@@ -460,7 +521,24 @@ export interface ContinuityGap { claim: string; missing: 'loss' | 'persistence'
|
|
|
460
521
|
|
|
461
522
|
const CONTINUITY_CLAIM = /\b(buffer(?:s|ed|ing)?|persist(?:s|ed|ence)?|round[- ]?trips?|restor(?:e|es|ed|ation)|carr(?:y|ied|ies)|retain(?:s|ed)?|session[- ]?storage|local[- ]?storage)\b/i;
|
|
462
523
|
const SIDE_PERSIST = /\b(restor(?:e|es|ed)|persist(?:s|ed)?|unchanged|same value|round[- ]?trips?|re-?hydrat\w*|still (?:shows|holds)|carr(?:y|ied|ies)|retain(?:s|ed)?|prefilled|pre-?filled)\b/i;
|
|
463
|
-
|
|
524
|
+
// A LOSS CONSTRUCTION, not a word list. `reset` and `expire` are domain NOUNS — every claim in a
|
|
525
|
+
// password-*reset* flow contains "reset", and every token flow contains "expire", so a bare-word
|
|
526
|
+
// list marked all of them as promising that state is lost and then demanded the missing half.
|
|
527
|
+
// A field report hit exactly that: the two sides of its mechanism were both written (in two
|
|
528
|
+
// different scenarios, which this already handles) and CONTINUITY-ONE-SIDED fired anyway (#630).
|
|
529
|
+
// The state has to be the thing that goes: `is cleared`, `loses the values`, `no longer holds`.
|
|
530
|
+
const SIDE_LOSS = new RegExp([
|
|
531
|
+
'\\b(?:lose|loses|losing|lost)\\b',
|
|
532
|
+
'\\b(?:is|are|gets?|got|becomes?|will be|was|were)\\s+(?:lost|cleared|emptied|discarded|wiped|reset|blanked)\\b',
|
|
533
|
+
'\\b(?:clears|clearing|discards|discarding|wipes|wiping)\\b',
|
|
534
|
+
'\\b(?:does|do|did)\\s+not\\s+(?:survive|persist|remain|carry)\\b',
|
|
535
|
+
'\\bnot\\s+(?:restored|retained|preserved|kept)\\b',
|
|
536
|
+
'\\bno longer\\b',
|
|
537
|
+
'\\bcomes? up (?:empty|blank)\\b',
|
|
538
|
+
// The triggers that CAUSE the loss still count when the claim names them as such.
|
|
539
|
+
'\\b(?:on|after) (?:a )?(?:reload|refresh)\\b',
|
|
540
|
+
'\\breload(?:ing|s|ed)? (?:the )?(?:page|screen|form|tab)\\b',
|
|
541
|
+
].join('|'), 'i');
|
|
464
542
|
|
|
465
543
|
/**
|
|
466
544
|
* Continuity claims in the viewpoint file whose feature proves only ONE side.
|
|
@@ -14,24 +14,46 @@ import { readTextFile } from './read-text';
|
|
|
14
14
|
export interface DownstreamResult {
|
|
15
15
|
downstreamRoutes: string[]; // success/navigation targets ≠ own route
|
|
16
16
|
underCovered: { route: string; slug: string }[]; // referenced only by a bare page-nav
|
|
17
|
+
/**
|
|
18
|
+
* Declared navigation targets NO scenario mentions at all. These used to be skipped as "out of
|
|
19
|
+
* this screen's scope", which is right for a screen and wrong for a FLOW: a field report found a
|
|
20
|
+
* dashboard whose spec declared four click-through destinations and whose suite tested only the
|
|
21
|
+
* render conditions of the regions containing them — the navigation itself had no case, and
|
|
22
|
+
* nothing said so (#630).
|
|
23
|
+
*/
|
|
24
|
+
absent: { route: string; slug: string }[];
|
|
17
25
|
}
|
|
18
26
|
|
|
19
27
|
/** Routes the spec hands off to (Navigation Flow / success), other than the screen's own route. */
|
|
20
28
|
function downstreamRoutes(specText: string): string[] {
|
|
21
29
|
const ownRoute = (specText.match(/\*\*Route\*\*\s*:\s*`?(\/[^\s`]+)/) || [])[1] || '';
|
|
22
30
|
const routes = new Set<string>();
|
|
31
|
+
const negated = new Set<string>();
|
|
23
32
|
for (const line of specText.split('\n')) {
|
|
24
33
|
if (!/success|navigat|to \(|→/i.test(line)) continue;
|
|
25
34
|
// A real route's leading `/` sits at a path boundary (start, whitespace, backtick, quote, paren),
|
|
26
35
|
// NOT after a letter/digit. The lookbehind rejects prose slashes like "text/icon" or
|
|
27
36
|
// "category/brand" that aren't routes at all (H2 — they produced /icon, /button, /brand).
|
|
37
|
+
let negRun = false; // inside a "…, not `/a`/`/b`" list
|
|
38
|
+
let prevEnd = 0;
|
|
28
39
|
for (const m of line.matchAll(/(?<![A-Za-z0-9])(\/[a-z][a-z0-9/_-]+)`?/gi)) {
|
|
29
40
|
const r = m[1];
|
|
41
|
+
// "the destination is `/a/b` (singular, not `/a/bs` as this spec assumed)" declares ONE
|
|
42
|
+
// target and CORRECTS another. A spec that says where a route is NOT is doing the reader a
|
|
43
|
+
// service; extracting the disowned one and demanding a test for it punishes the correction.
|
|
44
|
+
// The negation carries across a LIST — "not `/x`/`/y`" disowns both, not just the first.
|
|
45
|
+
const before = line.slice(Math.max(0, m.index - 24), m.index);
|
|
46
|
+
const gap = line.slice(prevEnd, m.index);
|
|
47
|
+
prevEnd = m.index + m[0].length;
|
|
48
|
+
if (/\b(?:not|instead of|rather than|no longer|never|không)\s*[`'"(\[]*\s*$/i.test(before)) negRun = true;
|
|
49
|
+
else if (negRun && !/^[`'"\s)\]]*[/,]?\s*(?:or|hoặc)?\s*[`'"(\[]*$/i.test(gap)) negRun = false;
|
|
50
|
+
if (negRun) { negated.add(r); continue; }
|
|
30
51
|
if (r !== ownRoute && r.split('/').length > ownRoute.split('/').length - 0) routes.add(r);
|
|
31
52
|
}
|
|
32
53
|
}
|
|
54
|
+
for (const n of negated) if (!routes.has(n)) routes.delete(n);
|
|
33
55
|
// keep only routes that extend beyond the own route (a distinct downstream surface)
|
|
34
|
-
return [...routes].filter((r) => r !== ownRoute && (!ownRoute || r.startsWith(ownRoute + '/') || r.split('/').length >= 3));
|
|
56
|
+
return [...routes].filter((r) => !negated.has(r) && r !== ownRoute && (!ownRoute || r.startsWith(ownRoute + '/') || r.split('/').length >= 3));
|
|
35
57
|
}
|
|
36
58
|
|
|
37
59
|
/**
|
|
@@ -100,9 +122,27 @@ export function sameScreenName(slug: string, label: string): boolean {
|
|
|
100
122
|
return i >= 5 && i / Math.min(a.length, b.length) >= 0.6;
|
|
101
123
|
}
|
|
102
124
|
|
|
103
|
-
export function downstreamScope(
|
|
104
|
-
|
|
125
|
+
export function downstreamScope(
|
|
126
|
+
specText: string, scenarios: ScenarioInfo[], featureText = '',
|
|
127
|
+
): DownstreamResult {
|
|
128
|
+
const all = downstreamRoutes(specText);
|
|
129
|
+
// A one-segment route that other declared routes are BUILT ON is a prefix, not a destination:
|
|
130
|
+
// a spec noting that "every route is served under a `/xx/` locale prefix (e.g. `/xx/register`)"
|
|
131
|
+
// was read as declaring `/xx/` a navigation target, and no suite will ever "navigate to" it.
|
|
132
|
+
// Structural, so it catches any mount point — locale, tenant, api version — without a list.
|
|
133
|
+
const mounts = all.filter((r) => {
|
|
134
|
+
const segs = r.split('/').filter(Boolean);
|
|
135
|
+
return segs.length === 1 && all.some((o) => o !== r && o.startsWith(r.endsWith('/') ? r : `${r}/`));
|
|
136
|
+
});
|
|
137
|
+
// The same destination written twice — once bare, once under the mount the spec documents —
|
|
138
|
+
// is one navigation target, and reporting both doubles the work it asks for.
|
|
139
|
+
const routes = all.filter((r) => !mounts.includes(r)
|
|
140
|
+
&& !mounts.some((m) => {
|
|
141
|
+
const bare = r.slice(m.replace(/\/$/, '').length);
|
|
142
|
+
return bare.startsWith('/') && all.includes(bare);
|
|
143
|
+
}));
|
|
105
144
|
const underCovered: { route: string; slug: string }[] = [];
|
|
145
|
+
const absent: { route: string; slug: string }[] = [];
|
|
106
146
|
for (const route of routes) {
|
|
107
147
|
const slug = (route.split('/').filter(Boolean).pop() || route).toLowerCase();
|
|
108
148
|
// Referenced when the route/slug appears literally, OR when any `[Ref]` in the suite
|
|
@@ -112,16 +152,34 @@ export function downstreamScope(specText: string, scenarios: ScenarioInfo[]): Do
|
|
|
112
152
|
const refs = scenarios.filter((s) =>
|
|
113
153
|
s.haystack.includes(slug) || s.haystack.includes(route.toLowerCase()) ||
|
|
114
154
|
labelsOf(s).some((l) => sameScreenName(slug, l)));
|
|
115
|
-
if (!refs.length)
|
|
155
|
+
if (!refs.length) {
|
|
156
|
+
// A scenario's haystack is its name + steps, so a route documented in a @manual scenario's
|
|
157
|
+
// tester procedure — a COMMENT — looks like nothing at all. That route is accounted for: a
|
|
158
|
+
// person has been told to check it. Reporting "no scenario mentions it, add one" over a
|
|
159
|
+
// deliberate manual deferral is the false positive this whole round is about, so absence is
|
|
160
|
+
// measured against the feature TEXT, comments included (#651 follow-up).
|
|
161
|
+
if (!featureText.toLowerCase().includes(route.toLowerCase())) absent.push({ route, slug });
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
116
164
|
// Substantively covered only if some scenario OPERATES on the downstream — i.e. it
|
|
117
165
|
// starts there (`is on [<downstream>]`) — not merely navigates to it as a terminal
|
|
118
166
|
// `see [<downstream>] page` assertion. The latter just proves the transition.
|
|
167
|
+
// Two shapes prove the screen's CONTENT was checked, not just its route:
|
|
168
|
+
// - the suite OPENS ON it (`is on [Registration] page`), and
|
|
169
|
+
// - it asserts an element NAMESPACED to it (`see [Registration:Submit] button`) — the
|
|
170
|
+
// namespace names the screen, so the assertion is about that screen by construction.
|
|
171
|
+
// Only the first counted, so a scenario that navigated there and then asserted the screen's
|
|
172
|
+
// own elements was still reported "covered only by a page-nav assertion" (#630). The field
|
|
173
|
+
// report's hypothesis — that the parser reads one line after the nav and stops — is still not
|
|
174
|
+
// what happens: there is no line window, and both shapes are matched over the whole scenario.
|
|
119
175
|
const contentCovered = refs.some((s) =>
|
|
120
176
|
[...s.haystack.matchAll(/\bis on \[([^\]]+)\]/g)]
|
|
121
|
-
.some((m) => sameScreenName(slug, m[1].split(':')[0]))
|
|
177
|
+
.some((m) => sameScreenName(slug, m[1].split(':')[0]))
|
|
178
|
+
|| [...s.haystack.matchAll(/\bsee \[([^\]:]+):[^\]]+\]/g)]
|
|
179
|
+
.some((m) => sameScreenName(slug, m[1])));
|
|
122
180
|
if (!contentCovered) underCovered.push({ route, slug });
|
|
123
181
|
}
|
|
124
|
-
return { downstreamRoutes: routes, underCovered };
|
|
182
|
+
return { downstreamRoutes: routes, underCovered, absent };
|
|
125
183
|
}
|
|
126
184
|
|
|
127
185
|
// ---------- #4 Manual-oracle ----------
|