@sun-asterisk/sungen 3.2.24-beta.4 → 3.2.24-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +9 -1
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +61 -4
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/next-step.d.ts +10 -0
- package/dist/harness/next-step.d.ts.map +1 -1
- package/dist/harness/next-step.js +36 -3
- package/dist/harness/next-step.js.map +1 -1
- package/dist/harness/sensors.d.ts +31 -0
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +74 -0
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/viewpoint-ledger.d.ts +21 -0
- package/dist/harness/viewpoint-ledger.d.ts.map +1 -1
- package/dist/harness/viewpoint-ledger.js +57 -5
- package/dist/harness/viewpoint-ledger.js.map +1 -1
- package/dist/orchestrator/flow-manager.d.ts +11 -0
- package/dist/orchestrator/flow-manager.d.ts.map +1 -1
- package/dist/orchestrator/flow-manager.js +48 -2
- package/dist/orchestrator/flow-manager.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +40 -0
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +9 -1
- package/src/harness/audit.ts +60 -4
- package/src/harness/next-step.ts +43 -3
- package/src/harness/sensors.ts +68 -0
- package/src/harness/viewpoint-ledger.ts +68 -5
- package/src/orchestrator/flow-manager.ts +52 -2
- package/src/orchestrator/templates/ai-src/commands/create-test.md +40 -0
package/src/harness/audit.ts
CHANGED
|
@@ -15,6 +15,7 @@ import { featureFilesFor } from './unit-paths';
|
|
|
15
15
|
import {
|
|
16
16
|
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, depthThresholdFor, coverageBalance, duplicateClusters, traceability, claimProof, taxonomyLint,
|
|
17
17
|
automatableManual, flowCoveredThemes, flowRegressionDepth, oracleStrength, declaredPageType,
|
|
18
|
+
numericClaimGaps, priorityMismatches,
|
|
18
19
|
GateResult, DepthResult, BalanceResult, DuplicateResult, TraceResult, ClaimProofResult, TaxonomyResult, Catalog, AutomatableManualResult, FlowDepthResult, OracleStrengthResult,
|
|
19
20
|
} from './sensors';
|
|
20
21
|
import { loadFlowScenarios } from './flow-check';
|
|
@@ -303,7 +304,22 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
303
304
|
// that rewrites that file scores them 100% by construction — which is exactly
|
|
304
305
|
// what a create-test run did, silently dropping the performance viewpoint on
|
|
305
306
|
// the way. While the declaration is unconfirmed, neither axis is evidence.
|
|
306
|
-
const
|
|
307
|
+
const adoptedPostHoc = viewpointBaseline.status === 'new' && scenarios.length > 0;
|
|
308
|
+
// `changed` was the only untrusted state, but `new` alongside an EXISTING suite is the same
|
|
309
|
+
// problem seen earlier: the yardstick is being introduced against scenarios that already exist,
|
|
310
|
+
// so it can only measure what their author already thought of. One field run wrote the file
|
|
311
|
+
// after generating the suite and said so in its own header — and both axes still read 100%,
|
|
312
|
+
// because a first audit records the baseline silently (#622). Untrusted until a human accepts it.
|
|
313
|
+
const viewpointMoved = viewpointBaseline.status === 'changed' || adoptedPostHoc;
|
|
314
|
+
// #618 — is there a YARDSTICK at all? `atomicLedger` and `traceability` are both measured
|
|
315
|
+
// against test-viewpoint.md, and four more sensors read it (the ledger, the declaration-integrity
|
|
316
|
+
// checks, the browser-gesture check, the continuity check). When the file is absent every one of
|
|
317
|
+
// them goes quiet — and worse, the checks that DERIVE from it keep talking: scenarios read as
|
|
318
|
+
// unsourceable because half their source contract is missing, `trace.mappedRatio` is 0 because
|
|
319
|
+
// there is nothing to map to, and the calibration axis reports a manufactured 0% that then trips
|
|
320
|
+
// SCORE-INFLATED-BY-BREADTH with advice nobody can act on. One absent file produced nine
|
|
321
|
+
// misleading findings and not one naming the file. So: say it once, and stop concluding.
|
|
322
|
+
const hasYardstick = ledger.hasViewpoint && (viewpoints.length > 0 || ledger.total > 0);
|
|
307
323
|
// 4. A flow with a contract is measured on ITS coverage: journey phases (HP/ER/EH
|
|
308
324
|
// automated) — not a screen's page-type themes. Same weight, same criticality,
|
|
309
325
|
// so a flow can finally EARN the coverage axis instead of always losing it to
|
|
@@ -565,7 +581,16 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
565
581
|
} else if (balance.imbalanced && !flowScored) {
|
|
566
582
|
findings.push(`BALANCE: ${balance.note} Stop expanding secondary viewpoints until business-core gaps are filled.`);
|
|
567
583
|
}
|
|
568
|
-
if (
|
|
584
|
+
if (adoptedPostHoc) {
|
|
585
|
+
findings.push(`VIEWPOINT-ADOPTED-POST-HOC: this is the first audit of \`requirements/test-viewpoint.md\` and the suite ALREADY has ${scenarios.length} scenarios, so the yardstick is being introduced against tests that already exist. A declaration written after the suite can only measure what its author already thought of — and if it was derived FROM the suite, \`atomicLedger\` and \`traceability\` read 100% by construction while anything dropped along the way stops being missing from anything. Both axes are held as unverified until you review the claims (add what you care about, delete what you do not) and confirm: \`sungen audit --screen ${screenName} --accept-viewpoint\`.`);
|
|
586
|
+
}
|
|
587
|
+
if (!hasYardstick) {
|
|
588
|
+
const lost = [
|
|
589
|
+
'atomicLedger', 'traceability',
|
|
590
|
+
...(unsourced.length ? [`${unsourced.length} scenario(s) could not be traced to a source`] : []),
|
|
591
|
+
];
|
|
592
|
+
findings.push(`VIEWPOINT-MISSING: \`requirements/test-viewpoint.md\` ${ledger.hasViewpoint ? 'declares no viewpoints' : 'does not exist'}, so this unit has no yardstick. Two scored axes are excluded (atomicLedger, traceability) and four sensors cannot run: the atomic ledger, the declaration-integrity checks (reused ids, dangling refs, uniform priority), the browser-gesture check and the two-way continuity check. Nothing here is a pass — it is unmeasured [${lost.join(', ')}]. A flow contract does NOT substitute: the contract declares the use case, the viewpoint declares the CLAIMS the suite is scored against, and only the second is independent of what was generated. Author it (\`sungen add-flow\` scaffolds a template), then \`sungen audit --screen ${screenName} --accept-viewpoint\`.`);
|
|
593
|
+
} else if (trace.mappedRatio < 0.5) {
|
|
569
594
|
findings.push(`TRACE: ${trace.note}`);
|
|
570
595
|
}
|
|
571
596
|
// (UNIVERSAL viewpoint-gap finding now emitted by the `ui` gate sensor — see the gate block below.)
|
|
@@ -643,6 +668,31 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
643
668
|
if (depth.deferredBusinessCritical > 0 && depth.deferredBusinessCritical >= depth.businessCriticalTotal) {
|
|
644
669
|
findings.push(`DEPTH-DEFERRED: businessDepth ${businessDepth.toFixed(2)} is computed over only ${depth.businessCriticalTotal} on-screen scenario(s); ${depth.deferredBusinessCritical} business-critical scenario(s) are deferred to @manual (excluded from the ratio). Automate them in a flow and verify with \`sungen flow-check\` — this ratio is NOT "all business depth covered".`);
|
|
645
670
|
}
|
|
671
|
+
// #622 — a claim that names a count, proved by fewer assertions than it names. The count sits
|
|
672
|
+
// in the viewpoint CLAIM, so both the title-level and the claim-level shape are checked.
|
|
673
|
+
for (const g of ledger.partial.slice(0, 5)) {
|
|
674
|
+
findings.push(`CLAIM-PARTIAL: claim ${g.id ?? ''} names ${g.claimed} but its scenario asserts ${g.asserted} value(s) — "${g.text.slice(0, 70)}". Mapping a claim to a scenario id proves it is ADDRESSED, not that the oracle demonstrates it: assert all ${g.claimed}, use a \`see all …\` set assertion, or narrow the claim to what is actually proven.`);
|
|
675
|
+
}
|
|
676
|
+
for (const g of numericClaimGaps(scenarios).slice(0, 3)) {
|
|
677
|
+
findings.push(`CLAIM-PARTIAL: "${g.name}" claims ${g.claimed} ${g.noun} but asserts ${g.asserted} value(s) — assert all ${g.claimed}, or say in the title what the scenario actually proves.`);
|
|
678
|
+
}
|
|
679
|
+
// #622 — the priority table is where release selection comes from; a tag that contradicts it
|
|
680
|
+
// means nobody can tell which of the two was the decision.
|
|
681
|
+
{
|
|
682
|
+
const pm = priorityMismatches(hasYardstick ? (readText(viewpointPath) ?? '') : '', scenarios);
|
|
683
|
+
const byId = new Map<string, { declared: string; tagged: string; n: number }>();
|
|
684
|
+
for (const m of pm) {
|
|
685
|
+
const cur = byId.get(m.id) ?? { declared: m.declared, tagged: m.tagged, n: 0 };
|
|
686
|
+
cur.n++;
|
|
687
|
+
byId.set(m.id, cur);
|
|
688
|
+
}
|
|
689
|
+
for (const [id, v] of [...byId.entries()].slice(0, 5)) {
|
|
690
|
+
findings.push(`PRIORITY-CONTRADICTED: the viewpoint declares ${id} as "${v.declared}" but ${v.n} of its scenario(s) are tagged @${v.tagged}. A smoke run selects on the tag, so it picks up work the declaration deprioritised — reconcile them, and if the tag is right, fix the declaration rather than leaving both on record.`);
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
if (ledger.manualOnly.length > 0) {
|
|
694
|
+
findings.push(`VIEWPOINT-ITEM-MANUAL-ONLY: ${ledger.manualOnly.length} viewpoint claim(s) are carried only by @manual scenarios — ${ledger.manualOnly.slice(0, 4).map((m) => m.id).join(', ')}${ledger.manualOnly.length > 4 ? ', …' : ''}. The claim is ADDRESSED (a procedure exists) but nothing runs for it, so a regression would not notice it breaking. \`atomicLedger\` counts them, because the design work is real — the delivery must not report them as automated coverage.`);
|
|
695
|
+
}
|
|
646
696
|
if (ledger.hasViewpoint && ledger.missing.length) {
|
|
647
697
|
const sample = ledger.missing.slice(0, 6).map((m) => m.id || `"${m.text}"`).join(', ');
|
|
648
698
|
findings.push(`VIEWPOINT-ITEM-MISSING: ${ledger.missing.length}/${ledger.total} atomic viewpoint items have no covering scenario (${(ledger.ratio * 100).toFixed(0)}% covered) — e.g. ${sample}. Cover each item or mark it deferred/spec-gap.`);
|
|
@@ -665,7 +715,10 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
665
715
|
findings.push(`FLOW-EXHAUSTIVE-INPUTS: ${c.scenarios.length} same-shape scenarios differ only in data (${c.scenarios.slice(0, 3).join(' | ')}${c.scenarios.length > 3 ? ' …' : ''}) — a system-test flow uses REPRESENTATIVE inputs (one valid + one that triggers the exception branch); the full EP/boundary matrix belongs to the owning screen (or a @cases dataset there). Keep one representative per branch here.`);
|
|
666
716
|
}
|
|
667
717
|
}
|
|
668
|
-
|
|
718
|
+
// Reported per scenario only when BOTH halves of the source contract exist. With no viewpoint,
|
|
719
|
+
// "doesn't trace to any FR / viewpoint item" is true by construction for anything that does not
|
|
720
|
+
// happen to cite an FR — the count is folded into VIEWPOINT-MISSING instead of six accusations.
|
|
721
|
+
for (const u of (hasYardstick ? unsourced : []).slice(0, 6)) {
|
|
669
722
|
findings.push(`UNSOURCEABLE-SCENARIO: "${u}" doesn't trace to any FR / viewpoint item — link it to a source, or tag it @exploration (not part of the official suite).`);
|
|
670
723
|
}
|
|
671
724
|
|
|
@@ -697,7 +750,10 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
697
750
|
for (const a of scored) axes[a.key] = Math.round(a.value * 100) / 100;
|
|
698
751
|
// Measured but not weighted: they drive findings, not the headline.
|
|
699
752
|
axes.manualOracle = Math.round(manualCompleteness * 100) / 100;
|
|
700
|
-
|
|
753
|
+
// Only a real mismatch scores 0. Without a yardstick `trace.mappedRatio` is 0 because there is
|
|
754
|
+
// nothing to map to, and reporting that as the weakest axis sent authors to "raise taxonomy" —
|
|
755
|
+
// an axis that was never measured.
|
|
756
|
+
if (hasYardstick) axes.taxonomy = taxonomyMismatch ? 0 : Math.round(trace.mappedRatio * 100) / 100;
|
|
701
757
|
const notApplicable = axisDefs.filter((a) => !a.applicable).map((a) => a.key);
|
|
702
758
|
const weakestEntry = Object.entries(axes).sort((a, b) => a[1] - b[1])[0];
|
|
703
759
|
const weakest = { axis: weakestEntry[0], value: weakestEntry[1] };
|
package/src/harness/next-step.ts
CHANGED
|
@@ -18,6 +18,7 @@ import * as fs from 'fs';
|
|
|
18
18
|
import * as path from 'path';
|
|
19
19
|
import { reportSlug, featureFilesFor } from './unit-paths';
|
|
20
20
|
import { readTextFile } from './read-text';
|
|
21
|
+
import { checkAssetsDrift } from '../orchestrator/assets-drift';
|
|
21
22
|
|
|
22
23
|
export interface NextStep {
|
|
23
24
|
/** The command to run, as the operator would type it. */
|
|
@@ -47,6 +48,13 @@ export interface UnitState {
|
|
|
47
48
|
findings: string[];
|
|
48
49
|
viewpointBaselineStatus?: string;
|
|
49
50
|
};
|
|
51
|
+
/**
|
|
52
|
+
* The project's AI commands/skills were written by a different sungen build. This belongs in
|
|
53
|
+
* "what to do next" because it silently voids every fix that ships as an INSTRUCTION: a HARD
|
|
54
|
+
* RULE added to `create-test` reaches nobody until `sungen update` runs, and the existing
|
|
55
|
+
* warning is one grey stderr line at the top of every command (#620).
|
|
56
|
+
*/
|
|
57
|
+
assetsDrift?: { recordedVersion: string; runningVersion: string };
|
|
50
58
|
steps: NextStep[];
|
|
51
59
|
}
|
|
52
60
|
|
|
@@ -63,7 +71,7 @@ function readJson(p: string): Record<string, unknown> | null {
|
|
|
63
71
|
function countScenarios(unitDir: string, unit: string): number {
|
|
64
72
|
let n = 0;
|
|
65
73
|
for (const f of featureFilesFor(unitDir, unit)) {
|
|
66
|
-
const text = readTextFile(f)
|
|
74
|
+
const text = fs.existsSync(f) ? readTextFile(f) : '';
|
|
67
75
|
n += (text.match(/^\s*Scenario(?: Outline)?:/gm) ?? []).length;
|
|
68
76
|
}
|
|
69
77
|
return n;
|
|
@@ -84,9 +92,12 @@ export function readUnitState(
|
|
|
84
92
|
try {
|
|
85
93
|
selectorFiles = fs.readdirSync(selectorsDir).filter((f) => f.endsWith('.yaml'));
|
|
86
94
|
} catch { /* api units carry no selectors */ }
|
|
87
|
-
|
|
95
|
+
// `readTextFile` throws on a missing file, and every path here is legitimately absent at some
|
|
96
|
+
// point in a unit's life — the resolver exists to report exactly that, so it must not crash on it.
|
|
97
|
+
const read = (p: string): string => (fs.existsSync(p) ? readTextFile(p) : '');
|
|
98
|
+
const selectorText = selectorFiles.map((f) => read(path.join(selectorsDir, f))).join('\n');
|
|
88
99
|
const contractPath = path.join(req, 'flow-contract.yaml');
|
|
89
|
-
const contractText =
|
|
100
|
+
const contractText = read(contractPath);
|
|
90
101
|
|
|
91
102
|
const specDirs = [
|
|
92
103
|
path.join(projectRoot, 'specs', 'generated', 'flows', unit),
|
|
@@ -125,6 +136,12 @@ export function readUnitState(
|
|
|
125
136
|
selectorsArePlaceholder: selectorFiles.length > 0
|
|
126
137
|
&& (/TODO|PLACEHOLDER|REPLACE ME/i.test(selectorText) || selectorText.trim().length < 40),
|
|
127
138
|
compiled, hasResults, hasDeliverable,
|
|
139
|
+
assetsDrift: (() => {
|
|
140
|
+
const d = checkAssetsDrift(projectRoot);
|
|
141
|
+
return d && d.recordedVersion !== d.runningVersion
|
|
142
|
+
? { recordedVersion: d.recordedVersion, runningVersion: d.runningVersion }
|
|
143
|
+
: undefined;
|
|
144
|
+
})(),
|
|
128
145
|
audit: audit
|
|
129
146
|
? {
|
|
130
147
|
overall: typeof score.overall === 'number' ? score.overall : 0,
|
|
@@ -163,6 +180,29 @@ export function deriveSteps(s: UnitState): NextStep[] {
|
|
|
163
180
|
});
|
|
164
181
|
return out;
|
|
165
182
|
}
|
|
183
|
+
// Stale AI assets come FIRST, because they invalidate the advice below: a HARD RULE added to
|
|
184
|
+
// `create-test` reaches nobody until `sungen update` runs, so the run that follows will repeat
|
|
185
|
+
// whatever the old instructions said. Upgrading the package alone does not refresh them, and the
|
|
186
|
+
// existing warning is one grey stderr line among a command's whole output (#620).
|
|
187
|
+
if (s.assetsDrift) {
|
|
188
|
+
out.push({
|
|
189
|
+
kind: 'do',
|
|
190
|
+
command: 'sungen update',
|
|
191
|
+
because: `this project's AI commands/skills were written by sungen ${s.assetsDrift.recordedVersion} and you are running ${s.assetsDrift.runningVersion} — upgrading the package does NOT refresh them, so any rule added since ${s.assetsDrift.recordedVersion} is not in effect for the next run`,
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
// The yardstick. `atomicLedger` and `traceability` are measured against test-viewpoint.md and
|
|
195
|
+
// four sensors read it, so its absence is not cosmetic — and it can go missing from a unit that
|
|
196
|
+
// already HAS a spec (a run deleted it), which the "no source at all" branch above never sees.
|
|
197
|
+
if (!s.hasViewpoint) {
|
|
198
|
+
out.push({
|
|
199
|
+
kind: 'blocked',
|
|
200
|
+
command: s.kind === 'flow'
|
|
201
|
+
? `sungen add-flow --flow ${s.unit} --path <start-url> # repairs the missing scaffold; never overwrites`
|
|
202
|
+
: `write qa/screens/${s.unit}/requirements/test-viewpoint.md`,
|
|
203
|
+
because: 'requirements/test-viewpoint.md is missing — it is the yardstick `atomicLedger` and `traceability` are measured against, and four sensors read it, so the audit reports VIEWPOINT-MISSING and those axes are excluded. A flow contract does not substitute: it declares the use case, the viewpoint declares the claims',
|
|
204
|
+
});
|
|
205
|
+
}
|
|
166
206
|
if (s.kind === 'flow' && !s.hasContract) {
|
|
167
207
|
out.push({
|
|
168
208
|
kind: 'blocked',
|
package/src/harness/sensors.ts
CHANGED
|
@@ -843,3 +843,71 @@ export function claimProof(scenarios: ScenarioInfo[], focus = 'functional'): Cla
|
|
|
843
843
|
|
|
844
844
|
return { total: measured.length, withClaims, proven, unproven, ratio, focus, threshold, verdict };
|
|
845
845
|
}
|
|
846
|
+
|
|
847
|
+
// ---------- Numeric claims, and priority that disagrees with the declaration ----------
|
|
848
|
+
|
|
849
|
+
const NUMBER_WORDS: Record<string, number> = {
|
|
850
|
+
two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10,
|
|
851
|
+
};
|
|
852
|
+
|
|
853
|
+
/**
|
|
854
|
+
* A title that names a COUNT, proved by fewer assertions than it claims.
|
|
855
|
+
*
|
|
856
|
+
* "restores all **six** buffered values" asserted four fields; "all **seven** confirmation rows
|
|
857
|
+
* are read-only" checked two. Exact id mapping made both read as covered, and `claimProof` had no
|
|
858
|
+
* rule for it — a claim can be mapped to a scenario and still be only partly proven, which is the
|
|
859
|
+
* gap between "declared coverage" and "coverage an oracle demonstrates" (#622).
|
|
860
|
+
*
|
|
861
|
+
* Counting is the whole point here: the number is in the title, so the check is arithmetic rather
|
|
862
|
+
* than semantic. A `see all …` assertion is exempt — it proves a set in one step, which is the
|
|
863
|
+
* stronger shape, not a weaker one.
|
|
864
|
+
*/
|
|
865
|
+
export function numericClaimGaps(scenarios: ScenarioInfo[]): Array<{ name: string; claimed: number; asserted: number; noun: string }> {
|
|
866
|
+
const out: Array<{ name: string; claimed: number; asserted: number; noun: string }> = [];
|
|
867
|
+
for (const s of scenarios) {
|
|
868
|
+
if (s.manual) continue; // a procedure counts by hand
|
|
869
|
+
if (/\bsee all\b/.test(s.stepsText)) continue; // a set proved in one assertion
|
|
870
|
+
const m = s.name.toLowerCase().match(/\b(?:all\s+)?(two|three|four|five|six|seven|eight|nine|ten|[2-9]|10)\s+([a-z][a-z-]{2,})/);
|
|
871
|
+
if (!m) continue;
|
|
872
|
+
const claimed = NUMBER_WORDS[m[1]] ?? Number(m[1]);
|
|
873
|
+
if (!Number.isFinite(claimed) || claimed < 2) continue;
|
|
874
|
+
// Only value-bearing assertions count: `see [X] … with/contains {{v}}`. A visibility check
|
|
875
|
+
// does not demonstrate one of N values.
|
|
876
|
+
const asserted = (s.stepsText.match(/\bsee \[[^\]]+\][^|]*?\b(?:with|contains) \{\{/g) ?? []).length;
|
|
877
|
+
if (asserted > 0 && asserted < claimed) out.push({ name: s.name, claimed, asserted, noun: m[2] });
|
|
878
|
+
}
|
|
879
|
+
return out;
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
/**
|
|
883
|
+
* A scenario whose priority tag contradicts the priority its viewpoint declares.
|
|
884
|
+
*
|
|
885
|
+
* The Priority-Viewpoints table is where release selection is supposed to come from. When the
|
|
886
|
+
* table says a flow is `Low` and its scenario is tagged `@high`, a smoke run picks up work the QA
|
|
887
|
+
* deliberately deprioritised — and nobody can tell which of the two was the decision (#622).
|
|
888
|
+
*/
|
|
889
|
+
export function priorityMismatches(
|
|
890
|
+
viewpointText: string, scenarios: ScenarioInfo[],
|
|
891
|
+
): Array<{ id: string; declared: string; tagged: string; scenario: string }> {
|
|
892
|
+
const declared = new Map<string, string>();
|
|
893
|
+
for (const m of viewpointText.matchAll(/^\|\s*((?:VP|FL)[A-Z0-9._-]*)\s*\|\s*(critical|high|medium|normal|low|deferred)\s*\|/gim)) {
|
|
894
|
+
declared.set(m[1].toUpperCase(), m[2].toLowerCase());
|
|
895
|
+
}
|
|
896
|
+
if (declared.size === 0) return [];
|
|
897
|
+
// `medium` and `normal` are the same rank under different names in real documents.
|
|
898
|
+
const rank = (p: string): string => (p === 'medium' ? 'normal' : p);
|
|
899
|
+
const out: Array<{ id: string; declared: string; tagged: string; scenario: string }> = [];
|
|
900
|
+
for (const s of scenarios) {
|
|
901
|
+
const id = (s.vpId ?? '').toUpperCase();
|
|
902
|
+
if (!id) continue;
|
|
903
|
+
// The most specific declared row that prefixes this id.
|
|
904
|
+
const key = [...declared.keys()].filter((k) => id.startsWith(k)).sort((a, b) => b.length - a.length)[0];
|
|
905
|
+
if (!key) continue;
|
|
906
|
+
const want = rank(declared.get(key)!);
|
|
907
|
+
const got = rank(String(s.priority).toLowerCase());
|
|
908
|
+
// `critical` and `deferred` have no tag equivalent — nothing to contradict.
|
|
909
|
+
if (want === 'critical' || want === 'deferred') continue;
|
|
910
|
+
if (want !== got) out.push({ id: key, declared: declared.get(key)!, tagged: got, scenario: s.name });
|
|
911
|
+
}
|
|
912
|
+
return out;
|
|
913
|
+
}
|
|
@@ -20,6 +20,39 @@ export interface LedgerResult {
|
|
|
20
20
|
covered: number;
|
|
21
21
|
ratio: number;
|
|
22
22
|
missing: { id?: string; text: string }[];
|
|
23
|
+
/**
|
|
24
|
+
* Items whose id is carried ONLY by `@manual` scenarios. Accounted for — a documented
|
|
25
|
+
* procedure exists — but nothing runs, so a regression would not notice. Counted as covered
|
|
26
|
+
* (the claim IS addressed) and reported separately, the same split `FLOW-MANUAL-ONLY` makes
|
|
27
|
+
* for declared flows (#622).
|
|
28
|
+
*/
|
|
29
|
+
manualOnly: { id?: string; text: string }[];
|
|
30
|
+
/**
|
|
31
|
+
* Items whose CLAIM names a count that the scenario carrying its id does not fully assert —
|
|
32
|
+
* "restores all six buffered values" proved on four fields. Exact id mapping shows a claim is
|
|
33
|
+
* ADDRESSED; it says nothing about whether the oracle demonstrates all of it (#622).
|
|
34
|
+
*/
|
|
35
|
+
partial: { id?: string; text: string; claimed: number; asserted: number }[];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const NUMBER_WORDS: Record<string, number> = {
|
|
39
|
+
two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10,
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/** The count a claim names, when it names one: "all six values", "seven rows", "3 items". */
|
|
43
|
+
function claimedCount(text: string): number | null {
|
|
44
|
+
// A digit after `step`/`screen`/`phase` is an ORDINAL, not a quantity — "the buffer exists at
|
|
45
|
+
// step 4" claims nothing about four of anything. And the counted thing has to be a noun: a
|
|
46
|
+
// function word after the number means the number was not counting.
|
|
47
|
+
const FUNCTION_WORD = /^(and|the|is|are|was|were|of|or|in|on|at|to|for|with|that|which|but|not|has|have|its)$/;
|
|
48
|
+
for (const m of text.toLowerCase().matchAll(/(\w+\s+)?\b(?:all\s+)?(two|three|four|five|six|seven|eight|nine|ten|[2-9]|10)\s+([a-z][a-z-]{2,})/g)) {
|
|
49
|
+
const before = (m[1] ?? '').trim();
|
|
50
|
+
if (/^(step|steps|screen|screens|phase|phases|page|pages|tier|round|part|section|version|mail_0?)$/.test(before)) continue;
|
|
51
|
+
if (FUNCTION_WORD.test(m[3])) continue;
|
|
52
|
+
const n = NUMBER_WORDS[m[2]] ?? Number(m[2]);
|
|
53
|
+
if (Number.isFinite(n) && n >= 2) return n;
|
|
54
|
+
}
|
|
55
|
+
return null;
|
|
23
56
|
}
|
|
24
57
|
|
|
25
58
|
const ID_RE = /\b([A-Z]{1,5}\d{0,2}(?:[.\-][A-Za-z0-9]+)*-?\d{0,3})\b/; // VP0.Title, VP7-002, MS-HP-001, TV-01
|
|
@@ -74,16 +107,43 @@ export function parseViewpointItems(viewpointPath: string): { id?: string; text:
|
|
|
74
107
|
export function viewpointLedger(viewpointPath: string, scenarios: ScenarioInfo[], featureText: string): LedgerResult {
|
|
75
108
|
const items = parseViewpointItems(viewpointPath);
|
|
76
109
|
if (!fs.existsSync(viewpointPath) || items.length === 0) {
|
|
77
|
-
return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [] };
|
|
110
|
+
return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [], manualOnly: [], partial: [] };
|
|
78
111
|
}
|
|
79
112
|
const featLower = featureText.toLowerCase();
|
|
80
113
|
const missing: { id?: string; text: string }[] = [];
|
|
114
|
+
const manualOnly: { id?: string; text: string }[] = [];
|
|
115
|
+
const partial: { id?: string; text: string; claimed: number; asserted: number }[] = [];
|
|
81
116
|
let covered = 0;
|
|
82
117
|
|
|
83
118
|
for (const item of items) {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
119
|
+
// WHEN AN ITEM DECLARES AN ID, THAT ID IS THE CONTRACT. It used to fall through to word
|
|
120
|
+
// matching when the id was absent from the suite, so an item whose id nobody carried was
|
|
121
|
+
// still counted covered on two shared non-generic words with ANY scenario. Two claims written
|
|
122
|
+
// deliberately WITHOUT a scenario were both scored covered — one on
|
|
123
|
+
// "confirmation"/"buffered", the other on "name"/"blocked". `atomicLedger 100%` then meant
|
|
124
|
+
// "no claim is more than two words away from some scenario", which is not a coverage
|
|
125
|
+
// statement at all, and the ledger could never report the gap it exists to report (#622).
|
|
126
|
+
//
|
|
127
|
+
// Word matching survives only where there is nothing better: a prose claim with no id.
|
|
128
|
+
let isCovered: boolean;
|
|
129
|
+
if (item.id) {
|
|
130
|
+
isCovered = featLower.includes(item.id.toLowerCase());
|
|
131
|
+
if (isCovered) {
|
|
132
|
+
// An id carried only by @manual scenarios is ADDRESSED but not RUNNING.
|
|
133
|
+
const carriers = scenarios.filter((s) => (s.vpId ?? '').toLowerCase() === item.id!.toLowerCase()
|
|
134
|
+
|| s.haystack.includes(item.id!.toLowerCase()));
|
|
135
|
+
if (carriers.length > 0 && carriers.every((s) => s.manual)) manualOnly.push({ id: item.id, text: item.text });
|
|
136
|
+
// The count is in the CLAIM, not in the scenario title — so this is the only place both
|
|
137
|
+
// halves are in hand. `see all …` proves a set in one step and is exempt.
|
|
138
|
+
const n = claimedCount(item.text);
|
|
139
|
+
const auto = carriers.filter((s) => !s.manual);
|
|
140
|
+
if (n !== null && auto.length > 0 && !auto.some((s) => /\bsee all\b/.test(s.stepsText))) {
|
|
141
|
+
const asserted = Math.max(...auto.map((s) =>
|
|
142
|
+
(s.stepsText.match(/\bsee \[[^\]]+\][^|]*?\b(?:with|contains) \{\{/g) ?? []).length));
|
|
143
|
+
if (asserted > 0 && asserted < n) partial.push({ id: item.id, text: item.text, claimed: n, asserted });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
} else {
|
|
87
147
|
const words = [...new Set((item.text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w)))];
|
|
88
148
|
const need = Math.min(2, words.length);
|
|
89
149
|
isCovered = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= need);
|
|
@@ -92,7 +152,10 @@ export function viewpointLedger(viewpointPath: string, scenarios: ScenarioInfo[]
|
|
|
92
152
|
else missing.push({ id: item.id, text: item.text });
|
|
93
153
|
}
|
|
94
154
|
|
|
95
|
-
return {
|
|
155
|
+
return {
|
|
156
|
+
hasViewpoint: true, total: items.length, covered,
|
|
157
|
+
ratio: items.length ? covered / items.length : 1, missing, manualOnly, partial,
|
|
158
|
+
};
|
|
96
159
|
}
|
|
97
160
|
|
|
98
161
|
/**
|
|
@@ -59,8 +59,26 @@ export class FlowManager {
|
|
|
59
59
|
const testDataPath = path.join(testDataDir, `${flowName}.yaml`);
|
|
60
60
|
|
|
61
61
|
if (fs.existsSync(flowDir)) {
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
// Refusing outright left no way back when a REQUIREMENT file went missing: a run deleted
|
|
63
|
+
// `requirements/test-viewpoint.md`, `add-flow` said "already exists", and the only remaining
|
|
64
|
+
// route was writing the scaffold by hand. The viewpoint is the yardstick two scored axes are
|
|
65
|
+
// measured against, so losing it silently costs real coverage evidence (#620).
|
|
66
|
+
//
|
|
67
|
+
// Repair CREATES ONLY WHAT IS MISSING — it never overwrites, so a filled spec or viewpoint is
|
|
68
|
+
// safe. Anything already present is reported as skipped.
|
|
69
|
+
const restored = this.restoreMissingScaffolds(options, flowName, isMobile, {
|
|
70
|
+
requirementsDir, featuresDir, selectorsDir, testDataDir, requirementsUiDir,
|
|
71
|
+
featurePath, selectorPath, testDataPath,
|
|
72
|
+
});
|
|
73
|
+
if (restored.length === 0) {
|
|
74
|
+
console.error(`Error: Flow "${options.name}" already exists at ${flowDir} — and every scaffold file is present, so there is nothing to repair.`);
|
|
75
|
+
process.exit(1);
|
|
76
|
+
}
|
|
77
|
+
console.log(`Flow "${options.name}" already exists — restored ${restored.length} missing file(s):`);
|
|
78
|
+
for (const f of restored) console.log(` + ${path.relative(this.cwd, f)}`);
|
|
79
|
+
console.log('');
|
|
80
|
+
console.log('Nothing else was touched: repair only creates what is absent, never overwrites.');
|
|
81
|
+
return;
|
|
64
82
|
}
|
|
65
83
|
|
|
66
84
|
console.log(`Creating flow: ${options.name}\n`);
|
|
@@ -172,6 +190,38 @@ export class FlowManager {
|
|
|
172
190
|
.replace(/-+/g, '-');
|
|
173
191
|
}
|
|
174
192
|
|
|
193
|
+
/**
|
|
194
|
+
* Re-create only the scaffold files that are ABSENT from an existing flow.
|
|
195
|
+
*
|
|
196
|
+
* The requirement files are not decoration: `requirements/test-viewpoint.md` is the yardstick
|
|
197
|
+
* `atomicLedger` and `traceability` are measured against, and four sensors read it. When a run
|
|
198
|
+
* removed it, `add-flow` refused ("already exists") and there was no CLI route back — so the
|
|
199
|
+
* yardstick stayed gone and the audit's two viewpoint axes stayed excluded (#620).
|
|
200
|
+
*
|
|
201
|
+
* Never overwrites: a filled spec or viewpoint is an INPUT the QA owns.
|
|
202
|
+
*/
|
|
203
|
+
private restoreMissingScaffolds(
|
|
204
|
+
options: FlowOptions, flowName: string, isMobile: boolean,
|
|
205
|
+
paths: {
|
|
206
|
+
requirementsDir: string; featuresDir: string; selectorsDir: string; testDataDir: string;
|
|
207
|
+
requirementsUiDir: string; featurePath: string; selectorPath: string; testDataPath: string;
|
|
208
|
+
},
|
|
209
|
+
): string[] {
|
|
210
|
+
const restored: string[] = [];
|
|
211
|
+
for (const d of [paths.featuresDir, paths.selectorsDir, paths.testDataDir, paths.requirementsUiDir]) {
|
|
212
|
+
if (!fs.existsSync(d)) fs.mkdirSync(d, { recursive: true });
|
|
213
|
+
}
|
|
214
|
+
const ensure = (file: string, body: () => string): void => {
|
|
215
|
+
if (fs.existsSync(file)) return;
|
|
216
|
+
fs.writeFileSync(file, body(), 'utf-8');
|
|
217
|
+
restored.push(file);
|
|
218
|
+
};
|
|
219
|
+
ensure(path.join(paths.requirementsDir, 'spec.md'), () => this.generateSpecTemplate(options, flowName, isMobile));
|
|
220
|
+
ensure(path.join(paths.requirementsDir, 'test-viewpoint.md'), () => this.generateViewpointTemplate(options));
|
|
221
|
+
ensure(paths.featurePath, () => this.generateFeatureTemplate(options, flowName, isMobile));
|
|
222
|
+
return restored;
|
|
223
|
+
}
|
|
224
|
+
|
|
175
225
|
private generateSpecTemplate(options: FlowOptions, flowName: string, isMobile: boolean): string {
|
|
176
226
|
if (isMobile) {
|
|
177
227
|
return `# ${options.name} Flow Specification
|
|
@@ -64,6 +64,46 @@ restated from screen specs the project does not hold, say so: the audit reports
|
|
|
64
64
|
`SPEC-RESTATED-UNVERIFIED` because `specFR 100%` over a hand-copied list certifies the copy, not
|
|
65
65
|
the source.
|
|
66
66
|
|
|
67
|
+
**The contract does NOT replace `test-viewpoint.md` — author BOTH.** They answer different
|
|
68
|
+
questions and only one of them is a yardstick:
|
|
69
|
+
|
|
70
|
+
| | declares | scored against it |
|
|
71
|
+
|---|---|---|
|
|
72
|
+
| `flow-contract.yaml` | the USE CASE — actor · trigger · goal · precondition · both guarantees · the flow inventory | `flowCoverage` |
|
|
73
|
+
| `test-viewpoint.md` | the CLAIMS the suite must prove — cross-screen concerns, edge cases, design decisions, open questions, priorities | `atomicLedger` · `traceability` |
|
|
74
|
+
|
|
75
|
+
If `requirements/test-viewpoint.md` is missing, **write it before generating** (`add-flow` scaffolds
|
|
76
|
+
a template; a run that deletes or replaces it has removed the yardstick). Without it two scored
|
|
77
|
+
axes are excluded and four sensors cannot run at all — the atomic ledger, the
|
|
78
|
+
declaration-integrity checks, the browser-gesture check and the two-way continuity check — so the
|
|
79
|
+
audit reports `VIEWPOINT-MISSING` and the result is *unmeasured*, not passing. And the reason the
|
|
80
|
+
contract cannot stand in: the viewpoint is the one artifact that must stay **independent of what
|
|
81
|
+
you generated**. That is why a filled one is an INPUT you never rewrite, and why authoring it
|
|
82
|
+
FIRST — from the spec, before the scenarios exist — is the only order that keeps it honest.
|
|
83
|
+
|
|
84
|
+
**If you find yourself writing it AFTER the suite, stop and say so.** That is not a neutral
|
|
85
|
+
ordering choice: a yardstick derived from the scenarios reads 100% by construction, and anything
|
|
86
|
+
dropped along the way stops being missing from anything. `sungen audit` now reports
|
|
87
|
+
`VIEWPOINT-ADOPTED-POST-HOC` on the first sighting of a viewpoint alongside an existing suite and
|
|
88
|
+
holds `atomicLedger` + `traceability` as unverified until a human accepts it — so the honest move
|
|
89
|
+
is to write the claims from `spec.md` and the contract, mark plainly that they are a draft for QA
|
|
90
|
+
review, and let the QA revise before `--accept-viewpoint`.
|
|
91
|
+
|
|
92
|
+
**When you give a claim an id, a scenario must carry that id.** The id is the contract: a claim
|
|
93
|
+
whose id no scenario carries is reported MISSING, and word overlap will not rescue it. Several
|
|
94
|
+
scenarios may prove one claim, and a claim carried only by `@manual` scenarios is `ADDRESSED` but
|
|
95
|
+
not running (`VIEWPOINT-ITEM-MANUAL-ONLY`) — do not let the delivery read it as automated coverage.
|
|
96
|
+
|
|
97
|
+
**A claim that names a count must be proved that many times.** "restores all six buffered values"
|
|
98
|
+
asserted on four fields is `CLAIM-PARTIAL`: mapping a claim to a scenario shows it is addressed,
|
|
99
|
+
not that the oracle demonstrates it. Assert all of them, use one `see all …` set assertion, or
|
|
100
|
+
narrow the claim to what is actually proven.
|
|
101
|
+
|
|
102
|
+
**Priority comes from the declaration.** A scenario tagged `@high` under a viewpoint row that says
|
|
103
|
+
`Low` is `PRIORITY-CONTRADICTED` — a smoke run selects on the tag, so it picks up work the QA
|
|
104
|
+
deliberately deprioritised. Inherit the declared priority; if the tag is right, fix the
|
|
105
|
+
declaration instead of leaving both on record.
|
|
106
|
+
|
|
67
107
|
---
|
|
68
108
|
|
|
69
109
|
## Platform detection (do this FIRST)
|