@sun-asterisk/sungen 3.2.25 → 3.2.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +23 -4
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/next.js +2 -2
- package/dist/cli/commands/next.js.map +1 -1
- package/dist/exporters/matrix/build.d.ts.map +1 -1
- package/dist/exporters/matrix/build.js +4 -1
- package/dist/exporters/matrix/build.js.map +1 -1
- package/dist/exporters/matrix/map-loader.d.ts.map +1 -1
- package/dist/exporters/matrix/map-loader.js +5 -0
- package/dist/exporters/matrix/map-loader.js.map +1 -1
- package/dist/exporters/matrix/types.d.ts +11 -0
- package/dist/exporters/matrix/types.d.ts.map +1 -1
- package/dist/exporters/matrix/types.js.map +1 -1
- package/dist/generators/test-generator/utils/selector-resolver.d.ts.map +1 -1
- package/dist/generators/test-generator/utils/selector-resolver.js +2 -1
- package/dist/generators/test-generator/utils/selector-resolver.js.map +1 -1
- package/dist/harness/audit.d.ts +7 -0
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +77 -9
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-contract.d.ts +18 -15
- package/dist/harness/flow-contract.d.ts.map +1 -1
- package/dist/harness/flow-contract.js +116 -18
- package/dist/harness/flow-contract.js.map +1 -1
- package/dist/harness/next-step.d.ts +8 -2
- package/dist/harness/next-step.d.ts.map +1 -1
- package/dist/harness/next-step.js +2 -2
- package/dist/harness/next-step.js.map +1 -1
- package/dist/harness/quality-gates.d.ts +12 -1
- package/dist/harness/quality-gates.d.ts.map +1 -1
- package/dist/harness/quality-gates.js +62 -7
- package/dist/harness/quality-gates.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +9 -1
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/spec-branches.d.ts +88 -0
- package/dist/harness/spec-branches.d.ts.map +1 -0
- package/dist/harness/spec-branches.js +280 -0
- package/dist/harness/spec-branches.js.map +1 -0
- package/dist/harness/spec-coverage.d.ts +1 -1
- package/dist/harness/spec-coverage.js +4 -4
- package/dist/harness/spec-coverage.js.map +1 -1
- package/dist/harness/viewpoint-baseline.d.ts +9 -0
- package/dist/harness/viewpoint-baseline.d.ts.map +1 -1
- package/dist/harness/viewpoint-baseline.js +33 -3
- package/dist/harness/viewpoint-baseline.js.map +1 -1
- package/dist/harness/viewpoint-ledger.d.ts.map +1 -1
- package/dist/harness/viewpoint-ledger.js +63 -5
- package/dist/harness/viewpoint-ledger.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/add-flow.md +16 -0
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +10 -1
- package/dist/orchestrator/templates/ai-src/commands/delivery.md +9 -2
- package/dist/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +17 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +42 -2
- package/dist/utils/ref-key.d.ts +30 -0
- package/dist/utils/ref-key.d.ts.map +1 -0
- package/dist/utils/ref-key.js +48 -0
- package/dist/utils/ref-key.js.map +1 -0
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +22 -3
- package/src/cli/commands/next.ts +2 -2
- package/src/exporters/matrix/build.ts +4 -1
- package/src/exporters/matrix/map-loader.ts +5 -0
- package/src/exporters/matrix/types.ts +11 -0
- package/src/generators/test-generator/utils/selector-resolver.ts +2 -1
- package/src/harness/audit.ts +81 -11
- package/src/harness/flow-contract.ts +128 -17
- package/src/harness/next-step.ts +10 -4
- package/src/harness/quality-gates.ts +64 -6
- package/src/harness/sensors.ts +9 -1
- package/src/harness/spec-branches.ts +346 -0
- package/src/harness/spec-coverage.ts +4 -4
- package/src/harness/viewpoint-baseline.ts +41 -6
- package/src/harness/viewpoint-ledger.ts +56 -4
- package/src/orchestrator/templates/ai-src/commands/add-flow.md +16 -0
- package/src/orchestrator/templates/ai-src/commands/create-test.md +10 -1
- package/src/orchestrator/templates/ai-src/commands/delivery.md +9 -2
- package/src/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +17 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +42 -2
- package/src/utils/ref-key.ts +43 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* ONE vocabulary for what a `[Reference]` means.
|
|
4
|
+
*
|
|
5
|
+
* The compiler's documented key rule is "the lowercase of the `[Reference]` text, spaces
|
|
6
|
+
* preserved" (`SelectorResolver.generateKey`). The harness re-implemented that rule with its own
|
|
7
|
+
* regex — `/\[([a-z0-9_.-]+):/` — which has no space in it, so a multi-word screen such as
|
|
8
|
+
* `[Basic Info:Email]` compiled, ran and passed, and was NEVER seen by `touchesOutcome`,
|
|
9
|
+
* `reachesOutcome`, phase coverage, handoffs or the navigation-target check. A real project
|
|
10
|
+
* hyphenated every namespace with zero functional change and its audit went 6.9 → 8.9 (#660).
|
|
11
|
+
*
|
|
12
|
+
* Two rules, one place:
|
|
13
|
+
* - `normalizeRefLabel` IS the compiler's key rule. The compiler delegates here.
|
|
14
|
+
* - `screenKey` is the COMPARISON form the harness uses for a screen/namespace: separators are
|
|
15
|
+
* equivalent (`basic info` ≡ `basic-info` ≡ `basic_info`), because a contract author writes
|
|
16
|
+
* `outcome.screen: basic-info` as readily as the feature writes `[Basic Info:…]`, and the
|
|
17
|
+
* harness only ever asks "is this the same screen?" — never "which YAML key resolves?".
|
|
18
|
+
*/
|
|
19
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
|
+
exports.normalizeRefLabel = normalizeRefLabel;
|
|
21
|
+
exports.screenKey = screenKey;
|
|
22
|
+
exports.refNamespaces = refNamespaces;
|
|
23
|
+
/** The compiler's selector-key rule: NFC, lowercase, trimmed, inner whitespace collapsed. */
|
|
24
|
+
function normalizeRefLabel(label) {
|
|
25
|
+
return label.normalize('NFC').toLowerCase().trim().replace(/\s+/g, ' ');
|
|
26
|
+
}
|
|
27
|
+
/** Comparison form for a screen / namespace name — separator-insensitive. */
|
|
28
|
+
function screenKey(label) {
|
|
29
|
+
return normalizeRefLabel(label).replace(/[\s_-]+/g, ' ').trim();
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Every namespace a piece of step text references, in order, as `screenKey`s.
|
|
33
|
+
* `[Screen:Element]` → `screen`; a bare `[Screen]` page reference is included too when `bare`
|
|
34
|
+
* is set (a `see [Done] page` assertion names the screen without an element).
|
|
35
|
+
*/
|
|
36
|
+
function refNamespaces(text, opts = {}) {
|
|
37
|
+
const out = [];
|
|
38
|
+
for (const m of text.matchAll(/\[([^\]]+)\]/g)) {
|
|
39
|
+
const inner = m[1];
|
|
40
|
+
const colon = inner.indexOf(':');
|
|
41
|
+
if (colon !== -1)
|
|
42
|
+
out.push(screenKey(inner.slice(0, colon)));
|
|
43
|
+
else if (opts.bare)
|
|
44
|
+
out.push(screenKey(inner));
|
|
45
|
+
}
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=ref-key.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ref-key.js","sourceRoot":"","sources":["../../src/utils/ref-key.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;GAgBG;;AAGH,8CAEC;AAGD,8BAEC;AAOD,sCASC;AAxBD,6FAA6F;AAC7F,SAAgB,iBAAiB,CAAC,KAAa;IAC7C,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAC1E,CAAC;AAED,6EAA6E;AAC7E,SAAgB,SAAS,CAAC,KAAa;IACrC,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAClE,CAAC;AAED;;;;GAIG;AACH,SAAgB,aAAa,CAAC,IAAY,EAAE,OAA2B,EAAE;IACvE,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;aACxD,IAAI,IAAI,CAAC,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sun-asterisk/sungen",
|
|
3
|
-
"version": "3.2.
|
|
3
|
+
"version": "3.2.27",
|
|
4
4
|
"description": "Deterministic E2E Test Compiler - Gherkin + Selectors → Playwright tests",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
@@ -40,8 +40,8 @@
|
|
|
40
40
|
"@babel/types": "^7.28.5",
|
|
41
41
|
"@cucumber/gherkin": "^37.0.0",
|
|
42
42
|
"@cucumber/messages": "^31.0.0",
|
|
43
|
-
"@sungen/driver-data-factory": "3.2.
|
|
44
|
-
"@sungen/driver-ui": "3.2.
|
|
43
|
+
"@sungen/driver-data-factory": "3.2.27",
|
|
44
|
+
"@sungen/driver-ui": "3.2.27",
|
|
45
45
|
"chalk": "^5.6.2",
|
|
46
46
|
"commander": "^14.0.2",
|
|
47
47
|
"dotenv": "^17.2.3",
|
|
@@ -32,7 +32,14 @@ function render(r: AuditReport): void {
|
|
|
32
32
|
const label = a.key.padEnd(width);
|
|
33
33
|
// Two different n/a: this unit supplied nothing, vs this axis does not apply to this KIND
|
|
34
34
|
// of unit. Only the first is something an author can act on.
|
|
35
|
-
if (!a.applicable) {
|
|
35
|
+
if (!a.applicable) {
|
|
36
|
+
// Three different states, three different remedies — collapsing them into "no evidence"
|
|
37
|
+
// contradicted the section right above, which printed the very number being called absent.
|
|
38
|
+
const why = a.byDesign ? 'not applicable to a contract flow'
|
|
39
|
+
: a.withheld ? `held — measured ${(a.value * 100).toFixed(0)}%, not counted until the viewpoint baseline is accepted`
|
|
40
|
+
: 'no evidence';
|
|
41
|
+
L(` ${label} ${' '.repeat(20)} n/a — ${why}`); continue;
|
|
42
|
+
}
|
|
36
43
|
L(` ${label} ${bar(a.value)} ${(a.value * 100).toFixed(0)}%${verdictOf[a.key] ?? ''}`);
|
|
37
44
|
}
|
|
38
45
|
L(` (${r.score.formula})`);
|
|
@@ -46,6 +53,9 @@ function render(r: AuditReport): void {
|
|
|
46
53
|
L(` ①F Flow contract — goal: ${fq.contract!.goal}`);
|
|
47
54
|
L(` outcome [${fq.contract!.outcome.screen}]: ${fq.outcomeProven ? '✓ proven (automated data assertion)' : fq.outcomeManualOnly ? '⚠ manual-only' : '✗ UNPROVEN'}`);
|
|
48
55
|
L(` phases: ${fq.phases.map((p) => `${p.phase}=${p.covered ? (p.automated ? '✓' : 'manual') : '✗'}`).join(' ')} → phase coverage ${(fq.phaseRatio * 100).toFixed(0)}%`);
|
|
56
|
+
for (const d of fq.danglingPhaseRefs) {
|
|
57
|
+
L(` ⚠ PHASE-REF-DANGLING: phase "${d.phase}" names \`${d.ref}\` — no scenario carries it`);
|
|
58
|
+
}
|
|
49
59
|
// The inventory, when declared, is the coverage that counts — phase coverage is only
|
|
50
60
|
// "each of BF/AF/EF appears at least once" and printing it as `flowCoverage` next to a 89%
|
|
51
61
|
// axis read as a contradiction.
|
|
@@ -103,7 +113,12 @@ function render(r: AuditReport): void {
|
|
|
103
113
|
const ax = Object.entries(r.calibration.axes).map(([k, v]) => `${k}=${(v * 100).toFixed(0)}%`).join(' · ');
|
|
104
114
|
L(` ⑨ Calibration — ${ax}`);
|
|
105
115
|
const na = r.calibration.notApplicable ?? [];
|
|
106
|
-
if (na.length)
|
|
116
|
+
if (na.length) {
|
|
117
|
+
const held = new Set(r.score.axes.filter((x) => x.withheld).map((x) => x.key));
|
|
118
|
+
const absent = na.filter((k) => !held.has(k));
|
|
119
|
+
if (absent.length) L(` not scored (no evidence in this unit): ${absent.join(', ')}`);
|
|
120
|
+
if (held.size) L(` measured but held (viewpoint baseline changed — review, then --accept-viewpoint): ${[...held].join(', ')}`);
|
|
121
|
+
}
|
|
107
122
|
L(` weakest: ${r.calibration.weakest.axis} ${(r.calibration.weakest.value * 100).toFixed(0)}%${r.calibration.inflated ? ' ⚠ SCORE-INFLATED-BY-BREADTH' : ''}`);
|
|
108
123
|
if (r.calibration.cappedAt !== undefined) {
|
|
109
124
|
// Only blame the weakest axis when it IS weak. A cap for missing evidence printed
|
|
@@ -154,8 +169,12 @@ export function registerAuditCommand(program: Command): void {
|
|
|
154
169
|
const unitId = catalogIdFromScreenDir(dir);
|
|
155
170
|
const vb = report.viewpointBaseline;
|
|
156
171
|
const postHoc = vb.status === 'new' && report.scenarioCount > 0;
|
|
172
|
+
// A re-parsed baseline is a CACHE migration, not an acceptance: the comparison already
|
|
173
|
+
// resolved to `unchanged`, and rewriting the record only lets the NEXT real edit be
|
|
174
|
+
// detected. Left unwritten it would re-migrate — and re-announce — on every run.
|
|
175
|
+
if (vb.reparsed) acceptViewpointBaseline(process.cwd(), unitId, { hash: vb.hash, ids: vb.ids, fileHash: vb.fileHash });
|
|
157
176
|
if ((vb.status === 'new' && !postHoc) || (options.acceptViewpoint && (vb.status === 'changed' || postHoc))) {
|
|
158
|
-
acceptViewpointBaseline(process.cwd(), unitId, { hash: vb.hash, ids: vb.ids });
|
|
177
|
+
acceptViewpointBaseline(process.cwd(), unitId, { hash: vb.hash, ids: vb.ids, fileHash: vb.fileHash });
|
|
159
178
|
if (options.acceptViewpoint && !options.json) {
|
|
160
179
|
console.log(`\n ✓ test-viewpoint.md accepted as the baseline for ${name} (${vb.ids.length} viewpoint id(s)).`);
|
|
161
180
|
console.log(' Re-run the audit to score atomicLedger + traceability against it.\n');
|
package/src/cli/commands/next.ts
CHANGED
|
@@ -40,8 +40,8 @@ function render(s: UnitState): void {
|
|
|
40
40
|
if (s.audit) L(` audit: ${s.audit.overall}/10 [${s.audit.gateStatus}] · ${s.audit.findings.length} finding(s)`);
|
|
41
41
|
L('');
|
|
42
42
|
for (const st of s.steps) {
|
|
43
|
-
const mark = st.kind === 'blocked' ? '⛔' : st.kind === 'optional' ? '○' : '→';
|
|
44
|
-
L(` ${mark} ${st.command}`);
|
|
43
|
+
const mark = st.kind === 'blocked' ? '⛔' : st.kind === 'optional' ? '○' : st.kind === 'decide' ? '⚖' : '→';
|
|
44
|
+
L(` ${mark} ${st.command}${st.kind === 'decide' ? ' (a QA decision — ask, do not run)' : ''}`);
|
|
45
45
|
L(` ${st.because}`);
|
|
46
46
|
}
|
|
47
47
|
L('');
|
|
@@ -559,7 +559,10 @@ export function buildMatrix(inputs: BuildInputs): MatrixModel {
|
|
|
559
559
|
priority,
|
|
560
560
|
mode: modes.size > 1 ? 'mixed' : (first?.mode ?? 'auto'),
|
|
561
561
|
layers: Array.from(new Set(groupVariants.flatMap((v) => v.layers))),
|
|
562
|
-
|
|
562
|
+
// The map's own `trace:` codes join the scenarios' `@spec:` ids on the SAME column: a
|
|
563
|
+
// reviewer wants one place to see what an item traces to, whether the id came from the spec
|
|
564
|
+
// or from the analysis. Map codes lead, because they are the ones a person chose for it.
|
|
565
|
+
traces: Array.from(new Set([...(g.trace ?? []), ...groupVariants.flatMap((v) => v.traces)])),
|
|
563
566
|
dimensions: g.dimensions,
|
|
564
567
|
// Only the preconditions COMMON to every variant belong on the parent —
|
|
565
568
|
// copying the first variant's setup mis-states the start state of the
|
|
@@ -79,6 +79,10 @@ export function loadDeliveryMap(file: string): MapLoadResult {
|
|
|
79
79
|
intent: String(grp.intent ?? ''),
|
|
80
80
|
oracle: String(grp.oracle ?? ''),
|
|
81
81
|
...(grp.dimensions !== undefined ? { dimensions: String(grp.dimensions) } : {}),
|
|
82
|
+
// Free-form by design: these are the project's own analysis codes, not a fixed scheme.
|
|
83
|
+
...(Array.isArray(grp.trace) && grp.trace.length
|
|
84
|
+
? { trace: (grp.trace as unknown[]).map((t) => String(t).trim()).filter(Boolean) }
|
|
85
|
+
: {}),
|
|
82
86
|
category: grp.category as MapCategory,
|
|
83
87
|
review,
|
|
84
88
|
variants: Array.isArray(variants) ? variants.map(String) : [],
|
|
@@ -150,6 +154,7 @@ export function writeDeliveryMap(file: string, map: DeliveryMap): void {
|
|
|
150
154
|
intent: g.intent,
|
|
151
155
|
oracle: g.oracle,
|
|
152
156
|
...(g.dimensions !== undefined ? { dimensions: g.dimensions } : {}),
|
|
157
|
+
...(g.trace && g.trace.length ? { trace: g.trace } : {}),
|
|
153
158
|
category: g.category,
|
|
154
159
|
review: g.review,
|
|
155
160
|
variants: g.variants,
|
|
@@ -33,6 +33,17 @@ export interface MapGroup {
|
|
|
33
33
|
* (e.g. "required ×3 · format ×9 · full-width ×2"). Rendered on the parent row
|
|
34
34
|
* instead of concatenating every variant's title — the compact review view. */
|
|
35
35
|
dimensions?: string;
|
|
36
|
+
/**
|
|
37
|
+
* Domain-logic reference ids this item traces to — a decision table, an algorithm, an internal
|
|
38
|
+
* business rule (`DEC-003`, `ALG-002`, `DISC-002`, `BR-001`).
|
|
39
|
+
*
|
|
40
|
+
* `requirements:` already traces to the FR/TR/NFR ids in `requirements/spec.md`. The codes a QA
|
|
41
|
+
* assigns while ANALYSING — which mechanism a case actually exercises — had no field, so they
|
|
42
|
+
* lived as comments in the `.feature` and never reached the Coverage sheet handed to the
|
|
43
|
+
* reviewer: the Trace column came out blank for exactly the items whose traceability mattered
|
|
44
|
+
* most (#651). Optional, and free-form on purpose: these ids are the project's own vocabulary.
|
|
45
|
+
*/
|
|
46
|
+
trace?: string[];
|
|
36
47
|
/** Variant refs: `VP-ID` (all @cases rows when the scenario has a dataset) or `VP-ID#label`. */
|
|
37
48
|
variants: string[];
|
|
38
49
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as fs from 'fs';
|
|
2
|
+
import { normalizeRefLabel } from '../../../utils/ref-key';
|
|
2
3
|
import * as path from 'path';
|
|
3
4
|
import yaml from 'yaml';
|
|
4
5
|
import { readYaml, readYamlIfExists } from '../../../utils/yaml-io';
|
|
@@ -189,7 +190,7 @@ export class SelectorResolver {
|
|
|
189
190
|
* "書類一覧" → "書類一覧"
|
|
190
191
|
*/
|
|
191
192
|
static generateKey(label: string): string {
|
|
192
|
-
return label
|
|
193
|
+
return normalizeRefLabel(label); // the one definition — the harness reads the same rule (#660)
|
|
193
194
|
}
|
|
194
195
|
|
|
195
196
|
/**
|
package/src/harness/audit.ts
CHANGED
|
@@ -27,6 +27,8 @@ import { specCoverage, SpecCoverageResult, parseSpecClauses, restatedRequirement
|
|
|
27
27
|
import { downstreamScope, manualOracle, readText, DownstreamResult, ManualOracleResult,
|
|
28
28
|
negativeSideEffect, sourceBacked, crossArtifactOwnership, isolationRisk, serialCascadeRisk } from './quality-gates';
|
|
29
29
|
import { viewpointLedger, parseViewpointItems, browserGestureSubstitutions, viewpointIntegrity, LedgerResult } from './viewpoint-ledger';
|
|
30
|
+
import { specBranches, unenumeratedBranches, sameOracleClusters, permissionPairGaps, awaitingSeedData, unconsideredRisks, MULTI_SCREEN_RISKS } from './spec-branches';
|
|
31
|
+
const MULTI_SCREEN_RISK_COUNT = MULTI_SCREEN_RISKS.length;
|
|
30
32
|
import { capabilityRegistry } from '../capabilities/registry';
|
|
31
33
|
import { discoverAndRegisterCapabilities } from '../capabilities/discover';
|
|
32
34
|
import { contextRouter } from '../capabilities/context-router';
|
|
@@ -65,6 +67,12 @@ export interface AuditReport {
|
|
|
65
67
|
score: {
|
|
66
68
|
overall: number; // 0..10, business-weighted over the APPLICABLE axes
|
|
67
69
|
notApplicable?: string[]; // axes with no evidence — excluded, not scored as 1.0
|
|
70
|
+
/**
|
|
71
|
+
* Why each excluded axis is excluded. "No evidence" and "evidence WITHHELD pending review"
|
|
72
|
+
* are different states with different remedies, and reporting both as "no evidence" told the
|
|
73
|
+
* author of a 93%-covered ledger to go write the viewpoint they had just written (#657).
|
|
74
|
+
*/
|
|
75
|
+
naReasons?: Record<string, string>;
|
|
68
76
|
cappedAt?: number; // set when a weak critical axis holds the score down
|
|
69
77
|
coverage: number; // 0..1
|
|
70
78
|
businessDepth: number; // 0..1
|
|
@@ -76,7 +84,7 @@ export interface AuditReport {
|
|
|
76
84
|
* `specFR` and `atomicLedger` entirely — so a flow's real coverage axis was never printed
|
|
77
85
|
* while `balance`, which carried no weight, was.
|
|
78
86
|
*/
|
|
79
|
-
axes: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean; byDesign?: boolean }>;
|
|
87
|
+
axes: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean; byDesign?: boolean; withheld?: boolean }>;
|
|
80
88
|
formula: string;
|
|
81
89
|
};
|
|
82
90
|
gateStatus: 'PASS' | 'FAIL';
|
|
@@ -196,7 +204,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
196
204
|
const isUiFlowUnit = /^flows\//.test(catalogScreenName);
|
|
197
205
|
const flowQ: FlowQualityResult = isUiFlowUnit
|
|
198
206
|
? flowQuality(screenDir, scenarios)
|
|
199
|
-
: { hasContract: false, errors: [], outcomeProven: false, outcomeManualOnly: false, offGoal: [], offGoalRatio: 0, offGoalCategories: [], phases: [], phaseRatio: 1, handoffs: { total: 0, asserted: 0, ratio: 1 }, sit: [] };
|
|
207
|
+
: { hasContract: false, errors: [], outcomeProven: false, outcomeManualOnly: false, offGoal: [], offGoalRatio: 0, offGoalCategories: [], phases: [], phaseRatio: 1, danglingPhaseRefs: [], handoffs: { total: 0, asserted: 0, ratio: 1 }, sit: [] };
|
|
200
208
|
const gateCatalog = (flowQ.hasContract && !declaredPageType(viewpointText, catalog))
|
|
201
209
|
? { ...catalog, page_types: {} } as Catalog
|
|
202
210
|
: catalog;
|
|
@@ -230,7 +238,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
230
238
|
// #1 taxonomy-match: when the project defines a viewpoint taxonomy, scenarios must use it.
|
|
231
239
|
const taxonomyMismatch = viewpoints.length > 0 && trace.withVpCode > 0 && trace.mappedRatio < 0.6;
|
|
232
240
|
// #2 downstream-scope + #4 manual-oracle
|
|
233
|
-
const downstream = downstreamScope(readText(specPath), scenarios);
|
|
241
|
+
const downstream = downstreamScope(readText(specPath), scenarios, featureText);
|
|
234
242
|
const manualOracleResult = manualOracle(featureText);
|
|
235
243
|
const autoManual = automatableManual(scenarios); // TQ-2 — @manual that is really automatable
|
|
236
244
|
const ledger = viewpointLedger(viewpointPath, scenarios, featureText);
|
|
@@ -331,7 +339,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
331
339
|
// this KIND of unit". Only the first is a reason to cap the score: a contract flow can never
|
|
332
340
|
// supply balance evidence, so capping for it held every flow at 8.9 forever and printed
|
|
333
341
|
// "weakest: flowCoverage 100% — fix it to lift the number", which is not fixable advice (#595).
|
|
334
|
-
const axisDefs: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean; byDesign?: boolean }> = [
|
|
342
|
+
const axisDefs: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean; byDesign?: boolean; withheld?: boolean }> = [
|
|
335
343
|
flowScored
|
|
336
344
|
// A declared flow claimed only by @manual scenarios is ACCOUNTED FOR but nothing runs, so
|
|
337
345
|
// it cannot count the same as an automated one — `covered` was collapsing three very
|
|
@@ -340,10 +348,10 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
340
348
|
? { key: 'flowCoverage', value: inventory ? Math.min((inventory.ratio + inventory.automatedRatio) / 2, flowQ.phaseRatio) : flowQ.phaseRatio, weight: 0.22, applicable: true, critical: true }
|
|
341
349
|
: { key: 'coverage', value: coverage, weight: 0.22, applicable: !!gate.pageType && gate.themesTotal > 0, critical: true },
|
|
342
350
|
{ key: 'specFR', value: specRatio, weight: 0.15, applicable: spec.hasSpec && spec.frTotal > 0, critical: true },
|
|
343
|
-
{ key: 'atomicLedger', value: ledger.ratio, weight: 0.13, applicable: ledger.hasViewpoint && ledger.total > 0 && !viewpointMoved, critical: true },
|
|
351
|
+
{ key: 'atomicLedger', value: ledger.ratio, weight: 0.13, applicable: ledger.hasViewpoint && ledger.total > 0 && !viewpointMoved, critical: true, withheld: viewpointMoved && ledger.hasViewpoint && ledger.total > 0 },
|
|
344
352
|
{ key: 'businessDepth', value: businessDepth, weight: 0.20, applicable: true, critical: true },
|
|
345
353
|
{ key: 'claimProof', value: claim.ratio, weight: 0.15, applicable: claim.withClaims > 0, critical: true },
|
|
346
|
-
{ key: 'traceability', value: traceScore, weight: 0.09, applicable: viewpoints.length > 0 && !viewpointMoved, critical: false },
|
|
354
|
+
{ key: 'traceability', value: traceScore, weight: 0.09, applicable: viewpoints.length > 0 && !viewpointMoved, critical: false, withheld: viewpointMoved && viewpoints.length > 0 },
|
|
347
355
|
// A use-case decomposition is EF-heavy by construction (1 basic flow, N alternates, N
|
|
348
356
|
// exceptions) — that is the shape of a well-decomposed use case, not neglected business
|
|
349
357
|
// core. `flowCoverage` already measures whether the journey's phases are covered, so
|
|
@@ -394,10 +402,24 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
394
402
|
const overall = seniorBandedOverall(rawOverall, { flowStateful: flowDepth.stateful, flowRatio: flowDepth.ratio, oracleWeak: oracle.weak.length, isolationRisk: isoRisk });
|
|
395
403
|
|
|
396
404
|
const findings: string[] = [];
|
|
405
|
+
// A baseline that could not be compared was re-recorded rather than reported as an edit.
|
|
406
|
+
// Migrating in silence would be the same sin one level down, so it is stated.
|
|
407
|
+
if (viewpointBaseline.reparsed) {
|
|
408
|
+
findings.push(`VIEWPOINT-BASELINE-REPARSED: the recorded baseline for this unit was written by an earlier version whose viewpoint parser had a different granularity, so its ids cannot be compared with today's. \`test-viewpoint.md\` itself is unchanged — the file is now fingerprinted by its own bytes, which no parser upgrade can move, and the baseline has been re-recorded against it. \`atomicLedger\` and \`traceability\` count normally. If you DID edit the viewpoint since ${viewpointBaseline.recordedAt?.slice(0, 10) ?? 'it was accepted'}, review that diff yourself — this one migration could not tell an edit from the parser change.`);
|
|
409
|
+
}
|
|
397
410
|
if (capped) {
|
|
398
411
|
const why = weakestCritical && weakestCritical.value < 0.7
|
|
399
412
|
? `"${weakestCritical.key}" is ${(weakestCritical.value * 100).toFixed(0)}% — a weak axis is a hole, not something the other axes can average away`
|
|
400
|
-
:
|
|
413
|
+
: (() => {
|
|
414
|
+
const held = axisDefs.filter((a) => a.withheld).map((a) => a.key);
|
|
415
|
+
const absent = missingEvidence.filter((k) => !held.includes(k.replace(/ .*$/, '')));
|
|
416
|
+
const parts = [];
|
|
417
|
+
if (absent.length) parts.push(`supplies no evidence for [${absent.join(', ')}]`);
|
|
418
|
+
// An axis WAS measured and is being held — saying "no evidence" here sends the author
|
|
419
|
+
// to write a declaration that already exists, when the actual next move is to review a diff.
|
|
420
|
+
if (held.length) parts.push(`has [${held.join(', ')}] measured but WITHHELD until the changed viewpoint baseline is reviewed and accepted`);
|
|
421
|
+
return `this unit ${parts.join(', and ')}, and a top mark has to rest on complete evidence`;
|
|
422
|
+
})();
|
|
401
423
|
findings.push(`SCORE-CAPPED: overall held at ${cap.toFixed(1)} because ${why}.`);
|
|
402
424
|
}
|
|
403
425
|
if (viewpointBaseline.status === 'changed') {
|
|
@@ -405,7 +427,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
405
427
|
viewpointBaseline.removed?.length ? `removed [${viewpointBaseline.removed.join(', ')}]` : '',
|
|
406
428
|
viewpointBaseline.added?.length ? `added [${viewpointBaseline.added.join(', ')}]` : '',
|
|
407
429
|
].filter(Boolean).join(', ');
|
|
408
|
-
findings.push(`VIEWPOINT-BASELINE-CHANGED: test-viewpoint.md no longer declares what it did when this unit was last accepted — ${moved || 'the declared ids were reordered or replaced'}. atomicLedger + traceability measure the suite AGAINST this file, so they are excluded from the score until the change is confirmed: a generator that rewrites the declaration scores both 100% by construction, and a viewpoint dropped from the file stops being missing from anything. Review the diff (a removed id means that coverage is now unclaimed), then run \`sungen audit --screen ${screenName} --accept-viewpoint
|
|
430
|
+
findings.push(`VIEWPOINT-BASELINE-CHANGED: test-viewpoint.md no longer declares what it did when this unit was last accepted — ${moved || 'the declared ids were reordered or replaced'}. atomicLedger + traceability measure the suite AGAINST this file, so they are excluded from the score until the change is confirmed: a generator that rewrites the declaration scores both 100% by construction, and a viewpoint dropped from the file stops being missing from anything. Review the diff (a removed id means that coverage is now unclaimed), then run \`sungen audit --screen ${screenName} --accept-viewpoint\` — a QA decision: an agent presents it and does not run it, because a yardstick accepted by the process it measures reads 100% by construction.`);
|
|
409
431
|
}
|
|
410
432
|
// On a contract flow the page type is INTENTIONALLY not applied — the contract is the checklist.
|
|
411
433
|
if (gate.pageTypeSource === 'undetermined' && !flowScored) {
|
|
@@ -423,7 +445,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
423
445
|
: `FLOW-OUTCOME-UNPROVEN: no scenario asserts data on the outcome screen \`${flowQ.contract!.outcome.screen}\` — the flow never proves its own goal ("${flowQ.contract!.goal}"). Add the happy-path scenario that ends in the declared final assertion.`);
|
|
424
446
|
}
|
|
425
447
|
if (flowQ.offGoalRatio > 0.35) {
|
|
426
|
-
findings.push(`FLOW-SCOPE-CREEP: ${flowQ.offGoal.length}/${scenarios.length} scenarios never touch the outcome screen \`${flowQ.contract!.outcome.screen}\` and are not guards/error-recovery (categories: ${flowQ.offGoalCategories.join(', ')}) — evidence of a SECOND business goal in this flow. Split them into their own flow (one flow = one goal, one observable outcome).`);
|
|
448
|
+
findings.push(`FLOW-SCOPE-CREEP: ${flowQ.offGoal.length}/${scenarios.length} scenarios never touch the outcome screen \`${flowQ.contract!.outcome.screen}\` and are not guards/error-recovery (categories: ${flowQ.offGoalCategories.join(', ')}) — evidence of a SECOND business goal in this flow. Split them into their own flow (one flow = one goal, one observable outcome) — or, when the spec KEEPS a variant deliberately light (an entry variant that only proves it reaches the action bar), declare that flow in \`flows:\` with its own \`outcome:\` and it is judged against that, not against the use case's. Never widen a scenario to satisfy this finding: that trades a reviewer's "over-testing" for the gate's approval, and the gate is the one that is wrong.`);
|
|
427
449
|
}
|
|
428
450
|
for (const ph of flowQ.phases.filter((p) => !p.covered || !p.automated)) {
|
|
429
451
|
findings.push(`FLOW-PHASE-${ph.phase}-MISSING: journey phase ${ph.phase} (${ph.phase === 'HP' ? 'happy path proving the outcome' : ph.phase === 'ER' ? 'error recovery — validation must not trap the journey' : 'guards — direct access / back / refresh'}) is ${ph.covered ? 'covered only by @manual' : 'not covered'} → it does not count toward flowCoverage until automated.`);
|
|
@@ -584,7 +606,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
584
606
|
findings.push(`BALANCE: ${balance.note} Stop expanding secondary viewpoints until business-core gaps are filled.`);
|
|
585
607
|
}
|
|
586
608
|
if (adoptedPostHoc) {
|
|
587
|
-
findings.push(`VIEWPOINT-ADOPTED-POST-HOC: this is the first audit of \`requirements/test-viewpoint.md\` and the suite ALREADY has ${scenarios.length} scenarios, so the yardstick is being introduced against tests that already exist. A declaration written after the suite can only measure what its author already thought of — and if it was derived FROM the suite, \`atomicLedger\` and \`traceability\` read 100% by construction while anything dropped along the way stops being missing from anything. Both axes are held as unverified until you review the claims (add what you care about, delete what you do not) and confirm: \`sungen audit --screen ${screenName} --accept-viewpoint
|
|
609
|
+
findings.push(`VIEWPOINT-ADOPTED-POST-HOC: this is the first audit of \`requirements/test-viewpoint.md\` and the suite ALREADY has ${scenarios.length} scenarios, so the yardstick is being introduced against tests that already exist. A declaration written after the suite can only measure what its author already thought of — and if it was derived FROM the suite, \`atomicLedger\` and \`traceability\` read 100% by construction while anything dropped along the way stops being missing from anything. Both axes are held as unverified until you review the claims (add what you care about, delete what you do not) and confirm: \`sungen audit --screen ${screenName} --accept-viewpoint\` — a QA decision: an agent presents it and does not run it.`);
|
|
588
610
|
}
|
|
589
611
|
if (!hasYardstick) {
|
|
590
612
|
const lost = [
|
|
@@ -622,6 +644,9 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
622
644
|
if (taxonomyMismatch) {
|
|
623
645
|
findings.push(`VP-TAXONOMY-MISMATCH: only ${(trace.mappedRatio * 100).toFixed(0)}% of scenarios use the viewpoint IDs declared in test-viewpoint.md — scenarios invented a generic VP-<CAT> scheme. Re-tag to the project's viewpoint IDs so the coverage matrix is accurate.`);
|
|
624
646
|
}
|
|
647
|
+
for (const a of downstream.absent.slice(0, 5)) {
|
|
648
|
+
findings.push(`NAV-TARGET-ABSENT: the spec declares "${a.route}" as a navigation target and NO scenario mentions it — not even a page assertion. Testing the render conditions of the region that links there does not test the link: a click-through case is what proves the destination. Add one, or record why this flow does not own it.`);
|
|
649
|
+
}
|
|
625
650
|
for (const d of downstream.underCovered) {
|
|
626
651
|
findings.push(`DOWNSTREAM-SCOPE-MISSING: "${d.route}" is a navigation target but is covered only by a page-nav assertion — cover its content/guards, or scaffold it (\`sungen add --screen ${d.slug}\`).`);
|
|
627
652
|
}
|
|
@@ -670,6 +695,46 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
670
695
|
if (depth.deferredBusinessCritical > 0 && depth.deferredBusinessCritical >= depth.businessCriticalTotal) {
|
|
671
696
|
findings.push(`DEPTH-DEFERRED: businessDepth ${businessDepth.toFixed(2)} is computed over only ${depth.businessCriticalTotal} on-screen scenario(s); ${depth.deferredBusinessCritical} business-critical scenario(s) are deferred to @manual (excluded from the ratio). Automate them in a flow and verify with \`sungen flow-check\` — this ratio is NOT "all business depth covered".`);
|
|
672
697
|
}
|
|
698
|
+
// #630 — the branches the spec ENUMERATES must each have a scenario. Narrative guidance for
|
|
699
|
+
// this already existed and was missed three times in three flows, so the check is an enforced
|
|
700
|
+
// enumeration rather than another sentence.
|
|
701
|
+
{
|
|
702
|
+
const branches = specBranches(readText(specPath) ?? '');
|
|
703
|
+
for (const b of unenumeratedBranches(branches, featureText).slice(0, 6)) {
|
|
704
|
+
findings.push(`SPEC-BRANCH-UNCOVERED: the spec enumerates "${b.label}" (${b.source}) and no scenario cites it. A mutually-exclusive branch is only covered when its OWN arm is verified — covering a sibling arm proves nothing about this one. Write it, or record the exclusion in the feature so the omission is a decision.`);
|
|
705
|
+
}
|
|
706
|
+
for (const c of sameOracleClusters(scenarios).slice(0, 4)) {
|
|
707
|
+
findings.push(`BRANCH-OVERCLAIMED: ${c.scenarios.length} scenarios assert an identical oracle (${c.scenarios.slice(0, 3).join(' | ')}${c.scenarios.length > 3 ? ' …' : ''}) — "${c.oracle}". They verify one branch several times while the coverage sheet counts them as ${c.scenarios.length}. Give each the assertion that distinguishes ITS branch, or merge them.`);
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
// #651 F1-7 — the fixed risk catalog, reconciled. SPEC-BRANCH-UNCOVERED enforces the branches a
|
|
711
|
+
// spec DECLARES; this is the half a spec is often silent about, which a reviewer catches and a
|
|
712
|
+
// generator does not. Consideration counts from anywhere the author reasons — a declared flow's
|
|
713
|
+
// branch point or reason, a scenario, the viewpoint — so "thought about, does not apply" is a
|
|
714
|
+
// one-line answer rather than a scenario nobody needs.
|
|
715
|
+
if (flowQ.hasContract) {
|
|
716
|
+
const unconsidered = unconsideredRisks(flowQ.contract!, scenarios, viewpointText);
|
|
717
|
+
if (unconsidered.length > 0) {
|
|
718
|
+
const list = unconsidered.map((r) => `${r.label} (${r.why})`).join(' · ');
|
|
719
|
+
findings.push(`RISK-FAMILY-UNCONSIDERED: ${unconsidered.length} of ${MULTI_SCREEN_RISK_COUNT} multi-screen risk families are not mentioned anywhere in this flow's contract, viewpoint or scenarios — ${list}. Each is a question, not a demand: if it does not apply here, record that in the \`flows:\` inventory as \`out-of-scope\` with the reason and it stops being asked. A flow is "enough" when every family has an answer, and a silent family is the one nobody weighed.`);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
// #630 — a deny-side security case with no allow-side counterpart cannot tell a working guard
|
|
723
|
+
// from a feature that is broken for everyone.
|
|
724
|
+
for (const g of permissionPairGaps(scenarios).slice(0, 4)) {
|
|
725
|
+
findings.push(`PERMISSION-PAIR-MISSING: "${g.scenario}" proves the content is hidden when permission is denied, but no scenario proves it is SHOWN when permission is granted. If the code hid it unconditionally this would still pass — the pair is the test, the way min-1/min/max/max+1 is the test for a boundary. Add the allow-side case${g.subject.length ? ` for [${g.subject.join(', ')}]` : ''}.`);
|
|
726
|
+
}
|
|
727
|
+
// #630 — a scenario blocked on unseeded data is a DATA task, not unfinished test design.
|
|
728
|
+
{
|
|
729
|
+
const tdDir = path.join(screenDir, 'test-data');
|
|
730
|
+
let tdText = '';
|
|
731
|
+
try {
|
|
732
|
+
for (const f of fs.readdirSync(tdDir).filter((x) => x.endsWith('.yaml'))) tdText += `${readText(path.join(tdDir, f)) ?? ''}\n`;
|
|
733
|
+
} catch { /* no test-data yet */ }
|
|
734
|
+
for (const a of awaitingSeedData(tdText, scenarios).slice(0, 4)) {
|
|
735
|
+
findings.push(`AWAITING-SEED-DATA: \`${a.key}\` is still a placeholder and ${a.scenarios.length} scenario(s) reference it (${a.scenarios.slice(0, 2).join(' | ')}${a.scenarios.length > 2 ? ' …' : ''}). These are Pending for a DATA reason, not an unresolved design — seed the value (or move it to \`<unit>.<env>.yaml\`) and they become runnable as written.`);
|
|
736
|
+
}
|
|
737
|
+
}
|
|
673
738
|
// #622 — a claim that names a count, proved by fewer assertions than it names. The count sits
|
|
674
739
|
// in the viewpoint CLAIM, so both the title-level and the claim-level shape are checked.
|
|
675
740
|
for (const g of ledger.partial.slice(0, 5)) {
|
|
@@ -791,7 +856,12 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
791
856
|
// flow) that carried no weight at all.
|
|
792
857
|
axes: axisDefs.map((a) => ({ ...a, weight: a.applicable ? a.weight / weightSum : 0 })),
|
|
793
858
|
formula: `overall = (${scored.map((a) => `${(a.weight / weightSum).toFixed(2)}*${a.key}`).join(' + ')}) * 10`
|
|
794
|
-
+ (
|
|
859
|
+
+ (() => {
|
|
860
|
+
const held = new Set(axisDefs.filter((a) => a.withheld).map((a) => a.key));
|
|
861
|
+
const absent = missingEvidence.filter((k) => !held.has(k.replace(/ .*$/, '')));
|
|
862
|
+
return (absent.length ? ` — n/a (no evidence): ${absent.join(', ')}` : '')
|
|
863
|
+
+ (held.size ? ` — measured but held: ${[...held].join(', ')}` : '');
|
|
864
|
+
})()
|
|
795
865
|
+ (notApplicableByDesign.length ? ` — n/a for this unit kind: ${notApplicableByDesign.join(', ')}` : ''),
|
|
796
866
|
},
|
|
797
867
|
gateStatus,
|