@sun-asterisk/sungen 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +24 -0
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +30 -14
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/eval.d.ts +3 -0
- package/dist/cli/commands/eval.d.ts.map +1 -0
- package/dist/cli/commands/eval.js +37 -0
- package/dist/cli/commands/eval.js.map +1 -0
- package/dist/cli/commands/ingest.d.ts +3 -0
- package/dist/cli/commands/ingest.d.ts.map +1 -0
- package/dist/cli/commands/ingest.js +179 -0
- package/dist/cli/commands/ingest.js.map +1 -0
- package/dist/cli/index.js +4 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/dashboard/templates/index.html +108 -194
- package/dist/generators/test-generator/adapters/adapter-interface.d.ts +1 -0
- package/dist/generators/test-generator/adapters/adapter-interface.d.ts.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/playwright-adapter.d.ts +1 -0
- package/dist/generators/test-generator/adapters/playwright/playwright-adapter.d.ts.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/playwright-adapter.js.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/templates/imports.hbs +3 -0
- package/dist/generators/test-generator/code-generator.d.ts +4 -0
- package/dist/generators/test-generator/code-generator.d.ts.map +1 -1
- package/dist/generators/test-generator/code-generator.js +31 -2
- package/dist/generators/test-generator/code-generator.js.map +1 -1
- package/dist/generators/test-generator/patterns/database-patterns.d.ts +5 -0
- package/dist/generators/test-generator/patterns/database-patterns.d.ts.map +1 -0
- package/dist/generators/test-generator/patterns/database-patterns.js +94 -0
- package/dist/generators/test-generator/patterns/database-patterns.js.map +1 -0
- package/dist/generators/test-generator/patterns/index.d.ts +1 -0
- package/dist/generators/test-generator/patterns/index.d.ts.map +1 -1
- package/dist/generators/test-generator/patterns/index.js +6 -1
- package/dist/generators/test-generator/patterns/index.js.map +1 -1
- package/dist/generators/test-generator/template-engine.d.ts +1 -0
- package/dist/generators/test-generator/template-engine.d.ts.map +1 -1
- package/dist/generators/test-generator/template-engine.js +1 -1
- package/dist/generators/test-generator/template-engine.js.map +1 -1
- package/dist/harness/audit.d.ts +16 -0
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +69 -5
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/capability-plan.d.ts +6 -0
- package/dist/harness/capability-plan.d.ts.map +1 -1
- package/dist/harness/capability-plan.js +14 -1
- package/dist/harness/capability-plan.js.map +1 -1
- package/dist/harness/catalog/drivers.yaml +1 -1
- package/dist/harness/catalog/universal-viewpoints.yaml +1 -1
- package/dist/harness/eval/skill-lint.d.ts +16 -0
- package/dist/harness/eval/skill-lint.d.ts.map +1 -0
- package/dist/harness/eval/skill-lint.js +129 -0
- package/dist/harness/eval/skill-lint.js.map +1 -0
- package/dist/harness/flow-plan.js +1 -1
- package/dist/harness/parse.d.ts +6 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +18 -3
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/quality-gates.d.ts +29 -0
- package/dist/harness/quality-gates.d.ts.map +1 -0
- package/dist/harness/quality-gates.js +183 -0
- package/dist/harness/quality-gates.js.map +1 -0
- package/dist/harness/script-check.d.ts.map +1 -1
- package/dist/harness/script-check.js +4 -1
- package/dist/harness/script-check.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +85 -6
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/spec-coverage.d.ts +37 -0
- package/dist/harness/spec-coverage.d.ts.map +1 -0
- package/dist/harness/spec-coverage.js +159 -0
- package/dist/harness/spec-coverage.js.map +1 -0
- package/dist/harness/viewpoint-ledger.d.ts +23 -0
- package/dist/harness/viewpoint-ledger.d.ts.map +1 -0
- package/dist/harness/viewpoint-ledger.js +118 -0
- package/dist/harness/viewpoint-ledger.js.map +1 -0
- package/dist/ingest/baseline-audit.d.ts +38 -0
- package/dist/ingest/baseline-audit.d.ts.map +1 -0
- package/dist/ingest/baseline-audit.js +85 -0
- package/dist/ingest/baseline-audit.js.map +1 -0
- package/dist/ingest/gsheet-fetch.d.ts +9 -0
- package/dist/ingest/gsheet-fetch.d.ts.map +1 -0
- package/dist/ingest/gsheet-fetch.js +180 -0
- package/dist/ingest/gsheet-fetch.js.map +1 -0
- package/dist/ingest/index.d.ts +6 -0
- package/dist/ingest/index.d.ts.map +1 -0
- package/dist/ingest/index.js +22 -0
- package/dist/ingest/index.js.map +1 -0
- package/dist/ingest/legacy-parser.d.ts +39 -0
- package/dist/ingest/legacy-parser.d.ts.map +1 -0
- package/dist/ingest/legacy-parser.js +218 -0
- package/dist/ingest/legacy-parser.js.map +1 -0
- package/dist/ingest/reconcile.d.ts +30 -0
- package/dist/ingest/reconcile.d.ts.map +1 -0
- package/dist/ingest/reconcile.js +65 -0
- package/dist/ingest/reconcile.js.map +1 -0
- package/dist/ingest/to-gherkin.d.ts +33 -0
- package/dist/ingest/to-gherkin.d.ts.map +1 -0
- package/dist/ingest/to-gherkin.js +93 -0
- package/dist/ingest/to-gherkin.js.map +1 -0
- package/dist/orchestrator/ai-rules-updater.d.ts.map +1 -1
- package/dist/orchestrator/ai-rules-updater.js +2 -0
- package/dist/orchestrator/ai-rules-updater.js.map +1 -1
- package/dist/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +1 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-delivery.md +10 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +1 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-ingest-legacy.md +79 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +25 -1
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-delivery.md +10 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +1 -1
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-ingest-legacy.md +79 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +44 -7
- package/dist/orchestrator/templates/specs-db.d.ts +18 -0
- package/dist/orchestrator/templates/specs-db.d.ts.map +1 -0
- package/dist/orchestrator/templates/specs-db.js +171 -0
- package/dist/orchestrator/templates/specs-db.js.map +1 -0
- package/dist/orchestrator/templates/specs-db.ts +147 -0
- package/docs/orchestration-spec.md +3 -3
- package/package.json +4 -4
- package/src/cli/commands/audit.ts +19 -0
- package/src/cli/commands/delivery.ts +31 -15
- package/src/cli/commands/eval.ts +28 -0
- package/src/cli/commands/ingest.ts +141 -0
- package/src/cli/index.ts +4 -0
- package/src/dashboard/templates/index.html +108 -194
- package/src/generators/test-generator/adapters/adapter-interface.ts +1 -1
- package/src/generators/test-generator/adapters/playwright/playwright-adapter.ts +1 -1
- package/src/generators/test-generator/adapters/playwright/templates/imports.hbs +3 -0
- package/src/generators/test-generator/code-generator.ts +29 -2
- package/src/generators/test-generator/patterns/database-patterns.ts +95 -0
- package/src/generators/test-generator/patterns/index.ts +3 -0
- package/src/generators/test-generator/template-engine.ts +2 -2
- package/src/harness/audit.ts +82 -5
- package/src/harness/capability-plan.ts +12 -1
- package/src/harness/catalog/drivers.yaml +1 -1
- package/src/harness/catalog/universal-viewpoints.yaml +1 -1
- package/src/harness/eval/skill-lint.ts +87 -0
- package/src/harness/flow-plan.ts +1 -1
- package/src/harness/parse.ts +19 -3
- package/src/harness/quality-gates.ts +152 -0
- package/src/harness/script-check.ts +4 -1
- package/src/harness/sensors.ts +84 -7
- package/src/harness/spec-coverage.ts +139 -0
- package/src/harness/viewpoint-ledger.ts +80 -0
- package/src/ingest/baseline-audit.ts +100 -0
- package/src/ingest/gsheet-fetch.ts +152 -0
- package/src/ingest/index.ts +5 -0
- package/src/ingest/legacy-parser.ts +184 -0
- package/src/ingest/reconcile.ts +80 -0
- package/src/ingest/to-gherkin.ts +108 -0
- package/src/orchestrator/ai-rules-updater.ts +2 -0
- package/src/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +1 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-delivery.md +10 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +1 -1
- package/src/orchestrator/templates/ai-instructions/claude-skill-ingest-legacy.md +79 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +25 -1
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-delivery.md +10 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +1 -1
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-ingest-legacy.md +79 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +44 -7
- package/src/orchestrator/templates/specs-db.ts +147 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quality gates (batch): downstream-scope + manual-oracle + negative-side-effect +
|
|
3
|
+
* cross-artifact ownership + source-backed strictness.
|
|
4
|
+
* Generic — read the project's own spec.md / feature text / sibling flows; no project data.
|
|
5
|
+
*/
|
|
6
|
+
import * as fs from 'fs';
|
|
7
|
+
import * as path from 'path';
|
|
8
|
+
import { ScenarioInfo, loadScenarios, idPrefix } from './parse';
|
|
9
|
+
|
|
10
|
+
// ---------- #2 Downstream-scope ----------
|
|
11
|
+
|
|
12
|
+
export interface DownstreamResult {
|
|
13
|
+
downstreamRoutes: string[]; // success/navigation targets ≠ own route
|
|
14
|
+
underCovered: { route: string; slug: string }[]; // referenced only by a bare page-nav
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** Routes the spec hands off to (Navigation Flow / success), other than the screen's own route. */
|
|
18
|
+
function downstreamRoutes(specText: string): string[] {
|
|
19
|
+
const ownRoute = (specText.match(/\*\*Route\*\*\s*:\s*`?(\/[^\s`]+)/) || [])[1] || '';
|
|
20
|
+
const routes = new Set<string>();
|
|
21
|
+
for (const line of specText.split('\n')) {
|
|
22
|
+
if (!/success|navigat|to \(|→/i.test(line)) continue;
|
|
23
|
+
for (const m of line.matchAll(/`?(\/[a-z][a-z0-9/_-]+)`?/gi)) {
|
|
24
|
+
const r = m[1];
|
|
25
|
+
if (r !== ownRoute && r.split('/').length > ownRoute.split('/').length - 0) routes.add(r);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
// keep only routes that extend beyond the own route (a distinct downstream surface)
|
|
29
|
+
return [...routes].filter((r) => r !== ownRoute && (!ownRoute || r.startsWith(ownRoute + '/') || r.split('/').length >= 3));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function downstreamScope(specText: string, scenarios: ScenarioInfo[]): DownstreamResult {
|
|
33
|
+
const routes = downstreamRoutes(specText);
|
|
34
|
+
const underCovered: { route: string; slug: string }[] = [];
|
|
35
|
+
for (const route of routes) {
|
|
36
|
+
const slug = (route.split('/').filter(Boolean).pop() || route).toLowerCase();
|
|
37
|
+
const refs = scenarios.filter((s) => s.haystack.includes(slug) || s.haystack.includes(route.toLowerCase()));
|
|
38
|
+
if (!refs.length) continue; // not referenced at all — out of this screen's scope entirely
|
|
39
|
+
// Substantively covered only if some scenario OPERATES on the downstream — i.e. it
|
|
40
|
+
// starts there (`is on [<downstream>]`) — not merely navigates to it as a terminal
|
|
41
|
+
// `see [<downstream>] page` assertion. The latter just proves the transition.
|
|
42
|
+
const opensOn = new RegExp(`\\bis on \\[[^\\]]*${slug}`, 'i');
|
|
43
|
+
const contentCovered = refs.some((s) => opensOn.test(s.haystack));
|
|
44
|
+
if (!contentCovered) underCovered.push({ route, slug });
|
|
45
|
+
}
|
|
46
|
+
return { downstreamRoutes: routes, underCovered };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// ---------- #4 Manual-oracle ----------
|
|
50
|
+
|
|
51
|
+
export interface ManualOracleResult {
|
|
52
|
+
manualTotal: number;
|
|
53
|
+
insufficient: string[]; // @manual scenarios lacking setup/action/oracle
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function blocks(featureText: string): string[] {
|
|
57
|
+
return featureText.split(/\n\s*\n/).filter((b) => /\bScenario:/.test(b));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function manualOracle(featureText: string): ManualOracleResult {
|
|
61
|
+
const insufficient: string[] = [];
|
|
62
|
+
let manualTotal = 0;
|
|
63
|
+
for (const b of blocks(featureText)) {
|
|
64
|
+
if (!/@manual\b/.test(b)) continue;
|
|
65
|
+
manualTotal++;
|
|
66
|
+
const commentLines = b.split('\n').filter((l) => /^\s*#/.test(l));
|
|
67
|
+
const hasOracle = /tester verifies|oracle\s*:|requires|verify that|expected\s*:|steps?\s*:/i.test(b);
|
|
68
|
+
const hasNumberedSteps = /^\s*#?\s*\d+\.\s/m.test(b);
|
|
69
|
+
// sufficient = an oracle/steps marker, OR a substantive comment block (≥3 comment lines)
|
|
70
|
+
if (!(hasOracle || hasNumberedSteps || commentLines.length >= 3)) {
|
|
71
|
+
const name = (b.match(/Scenario:\s*(.+)/) || [])[1] || '(unnamed)';
|
|
72
|
+
insufficient.push(name.trim().slice(0, 80));
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return { manualTotal, insufficient };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ---------- #4 Negative side-effect ----------
|
|
79
|
+
|
|
80
|
+
const NEG_TITLE = /\b(does not|doesn't|no second|not dispatch|not sent|without submitting|no leak|single request|exactly one|count is 1|only one request|no duplicate|not create)\b/i;
|
|
81
|
+
|
|
82
|
+
/** Titles asserting an ABSENCE must prove it (count / negative / @manual+oracle), not just a happy outcome. */
|
|
83
|
+
export function negativeSideEffect(scenarios: ScenarioInfo[]): string[] {
|
|
84
|
+
const flagged: string[] = [];
|
|
85
|
+
for (const s of scenarios) {
|
|
86
|
+
if (s.manual) continue; // @manual is a legitimate deferral (oracle checked by #4 manual-oracle)
|
|
87
|
+
if (!NEG_TITLE.test(s.name)) continue;
|
|
88
|
+
const proven = /\bcount\b|tohavecount|table with|is hidden|are hidden|not complete|message is hidden/.test(s.stepsText);
|
|
89
|
+
if (!proven) flagged.push(s.name.slice(0, 80));
|
|
90
|
+
}
|
|
91
|
+
return flagged;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ---------- #7 Source-backed strictness ----------
|
|
95
|
+
|
|
96
|
+
/** A scenario should trace to a source: a viewpoint ID (its own scheme), an FR id, or a
|
|
97
|
+
* viewpoint item (keyword overlap). ID match is language-agnostic and primary. */
|
|
98
|
+
export function sourceBacked(scenarios: ScenarioInfo[], frIds: string[], viewpointItems: string[], viewpointIds: string[], featureText: string): string[] {
|
|
99
|
+
if (!frIds.length && !viewpointItems.length && !viewpointIds.length) return []; // no contract
|
|
100
|
+
const vpIds = viewpointIds.map((s) => s.toUpperCase());
|
|
101
|
+
const itemWords = viewpointItems.map((t) => new Set((t.toLowerCase().match(/[a-z][a-z-]{4,}/g) || [])));
|
|
102
|
+
// per-scenario blocks (INCLUDING comments) so an FR cited in a comment counts as a source
|
|
103
|
+
const blockOf = new Map<string, string>();
|
|
104
|
+
for (const b of featureText.split(/\n\s*\n/)) {
|
|
105
|
+
const m = b.match(/Scenario:\s*(.+)/);
|
|
106
|
+
if (m) blockOf.set(m[1].trim().toLowerCase(), b.toLowerCase());
|
|
107
|
+
}
|
|
108
|
+
const unsourced: string[] = [];
|
|
109
|
+
for (const s of scenarios) {
|
|
110
|
+
const id = (s.vpId || s.vpCode || '').toUpperCase();
|
|
111
|
+
const mapsId = !!id && vpIds.some((v) => id === v || id.startsWith(v) || v.startsWith(idPrefix(id)));
|
|
112
|
+
const block = blockOf.get(s.name.trim().toLowerCase()) || s.haystack;
|
|
113
|
+
const citesFr = frIds.some((fid) => block.includes(fid.toLowerCase()));
|
|
114
|
+
const sWords = new Set((s.haystack.match(/[a-z][a-z-]{4,}/g) || []));
|
|
115
|
+
const mapsItem = itemWords.some((iw) => { let hits = 0; for (const w of iw) if (sWords.has(w)) hits++; return hits >= 2; });
|
|
116
|
+
if (!mapsId && !citesFr && !mapsItem) unsourced.push(s.name.slice(0, 80));
|
|
117
|
+
}
|
|
118
|
+
return unsourced;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// ---------- #6 Cross-artifact ownership ----------
|
|
122
|
+
|
|
123
|
+
export interface OwnershipResult { duplicates: { scenario: string; flow: string }[] }
|
|
124
|
+
|
|
125
|
+
/** Scenarios whose step-skeleton also appears in a sibling flow feature → duplicate ownership. */
|
|
126
|
+
export function crossArtifactOwnership(screenDir: string, scenarios: ScenarioInfo[]): OwnershipResult {
|
|
127
|
+
const duplicates: { scenario: string; flow: string }[] = [];
|
|
128
|
+
// screenDir = <root>/qa/screens/<name>; flows live at <root>/qa/flows/*/features/*.feature
|
|
129
|
+
const flowsRoot = path.resolve(screenDir, '..', '..', 'flows');
|
|
130
|
+
if (!fs.existsSync(flowsRoot)) return { duplicates };
|
|
131
|
+
const bySkeleton = new Map<string, string>();
|
|
132
|
+
for (const flow of fs.readdirSync(flowsRoot)) {
|
|
133
|
+
const fdir = path.join(flowsRoot, flow, 'features');
|
|
134
|
+
if (!fs.existsSync(fdir)) continue;
|
|
135
|
+
for (const f of fs.readdirSync(fdir).filter((x) => x.endsWith('.feature'))) {
|
|
136
|
+
for (const fs2 of loadScenarios(path.join(fdir, f))) {
|
|
137
|
+
if (fs2.stepSkeleton && fs2.stepSkeleton.length > 20) bySkeleton.set(fs2.stepSkeleton, flow);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
if (!bySkeleton.size) return { duplicates };
|
|
142
|
+
for (const s of scenarios) {
|
|
143
|
+
const flow = s.stepSkeleton && s.stepSkeleton.length > 20 ? bySkeleton.get(s.stepSkeleton) : undefined;
|
|
144
|
+
if (flow) duplicates.push({ scenario: s.name.slice(0, 70), flow });
|
|
145
|
+
}
|
|
146
|
+
return { duplicates };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// convenience reader
|
|
150
|
+
export function readText(p: string): string {
|
|
151
|
+
return fs.existsSync(p) ? fs.readFileSync(p, 'utf-8') : '';
|
|
152
|
+
}
|
|
@@ -68,7 +68,10 @@ export function analyzeFaithfulness(specSrc: string, automatedTitles: Set<string
|
|
|
68
68
|
for (const blk of extractTestBlocks(specSrc)) {
|
|
69
69
|
if (!automatedTitles.has(blk.title)) continue; // only non-@manual scenarios
|
|
70
70
|
const body = blk.body;
|
|
71
|
-
|
|
71
|
+
// An assertion is a Playwright `expect(...)` OR a Data Driver DB assertion
|
|
72
|
+
// (`db.assertRow/assertNoRow/assertCount/...`) — a DB check is a real oracle, so a
|
|
73
|
+
// DB-only scenario (no UI expect) is NOT a bypass.
|
|
74
|
+
if (!body.some((l) => /expect\(|\bdb\.assert\w*\s*\(/.test(l))) assertionlessTests.push(blk.title);
|
|
72
75
|
// hollow step: a `// step` whose region (until the NEXT step-comment / block end)
|
|
73
76
|
// contains no executable code. The region — not just the next line — is checked,
|
|
74
77
|
// so block-style steps (`// Assert all … { … expect … }`) are correctly counted.
|
package/src/harness/sensors.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import * as fs from 'fs';
|
|
10
10
|
import * as path from 'path';
|
|
11
11
|
import { parse as parseYaml } from 'yaml';
|
|
12
|
-
import { ScenarioInfo, ViewpointEntry } from './parse';
|
|
12
|
+
import { ScenarioInfo, ViewpointEntry, idPrefix } from './parse';
|
|
13
13
|
|
|
14
14
|
// Business-critical category codes (project VP-<CAT> prefixes). Configurable later.
|
|
15
15
|
const BUSINESS_CRITICAL_CATS = ['LIST', 'CART', 'PRODUCT', 'FILTER', 'CHECKOUT', 'ORDER'];
|
|
@@ -263,17 +263,23 @@ export interface TraceResult {
|
|
|
263
263
|
|
|
264
264
|
export function traceability(scenarios: ScenarioInfo[], viewpoints: ViewpointEntry[]): TraceResult {
|
|
265
265
|
const overviewIds = new Set(viewpoints.map((v) => v.id.toUpperCase()));
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
266
|
+
// A scenario carries an ID if it has a project-scheme leading ID (vpId) or a VP-CAT code.
|
|
267
|
+
const withCode = scenarios.filter((s) => s.vpId || s.vpCode);
|
|
268
|
+
// Maps to overview if the scenario's ID, its sequence-stripped prefix, or its VP-CAT code
|
|
269
|
+
// matches a declared viewpoint ID (format-tolerant: VP0-001↔VP0, MS-HP-001↔MS-HP-001).
|
|
270
|
+
const mapped = withCode.filter((s) => {
|
|
271
|
+
const id = (s.vpId || s.vpCode || '').toUpperCase();
|
|
272
|
+
if (overviewIds.has(id) || overviewIds.has(idPrefix(id))) return true;
|
|
273
|
+
return [...overviewIds].some((oid) => id.startsWith(oid) || oid.startsWith(idPrefix(id)) || (!!s.category && oid.includes(s.category)));
|
|
274
|
+
});
|
|
269
275
|
return {
|
|
270
276
|
total: scenarios.length,
|
|
271
277
|
withVpCode: withCode.length,
|
|
272
278
|
mappedToOverview: mapped.length,
|
|
273
279
|
withVpCodeRatio: scenarios.length ? withCode.length / scenarios.length : 0,
|
|
274
280
|
mappedRatio: scenarios.length ? mapped.length / scenarios.length : 0,
|
|
275
|
-
note: mapped.length < withCode.length * 0.5
|
|
276
|
-
? '
|
|
281
|
+
note: withCode.length && mapped.length < withCode.length * 0.5
|
|
282
|
+
? 'Scenario IDs do not match the viewpoint-overview ids (weak traceability — re-tag to the project viewpoint IDs).'
|
|
277
283
|
: 'Traceable.',
|
|
278
284
|
};
|
|
279
285
|
}
|
|
@@ -367,14 +373,85 @@ const CLAIM_RULES: ClaimRule[] = [
|
|
|
367
373
|
hint: 'capture the before-state and assert the after-state differs, or assert the visible/hidden transition.',
|
|
368
374
|
severity: 'warn',
|
|
369
375
|
},
|
|
376
|
+
{
|
|
377
|
+
// GENERAL — mutation-absence. A title asserts that a STATE-CHANGING action does NOT
|
|
378
|
+
// happen / does not repeat (submit, send, create, charge, order, pay, email, request,
|
|
379
|
+
// OTP, register, book, a re-/double-/again repeat…) paired with a negation in EITHER
|
|
380
|
+
// language. A mutation's absence is NOT observable from a positive `see [X] page` —
|
|
381
|
+
// that page looks identical whether or not the mutation fired — so it MUST prove a
|
|
382
|
+
// count/contrast (record count unchanged) or defer to @manual. This is the general
|
|
383
|
+
// category behind "browser back does not re-submit", "does not re-charge the card",
|
|
384
|
+
// "double-click does not create two orders" — not a per-feature keyword.
|
|
385
|
+
claim: 'no-side-effect/no-duplicate',
|
|
386
|
+
title: /(?=.*\b(submit|sen[dt]|resend|resubmit|re-?fire|re-?issue|re-?post|repost|create|charge|order|payment|\bpay\b|email|request|\botp\b|insert|register|book|duplicate|double[- ]?submit|again|twice)\b)(?=.*(\bno\b|\bnot\b|n['’]t\b|\bnever\b|\bwithout\b|\bcannot\b|prevent|block|avoid|reject|disabl|\bdeny\b|denies|\bkhông\b|\bchưa\b))/i,
|
|
387
|
+
proof: /\bcount\b|row with \{\{|table with|tohavecount|is hidden|are hidden|not complete|no longer/,
|
|
388
|
+
need: 'a record/request-count proof (count stays at one, e.g. `User see [Table] row with {{count}}`) or @manual with a request-count oracle',
|
|
389
|
+
hint: 'a "does-not-happen / does-not-repeat" claim about a state-changing action is NOT proven by a terminal `see [...] page` — that page is identical whether or not the action (re-)fired. Prove the side-effect count is unchanged, or mark @manual with a setup→action→assert-no-duplicate oracle.',
|
|
390
|
+
severity: 'fail',
|
|
391
|
+
},
|
|
370
392
|
{
|
|
371
393
|
claim: 'hidden/rejected/not-complete',
|
|
372
|
-
title: /\b(hidden|closed|dismiss(es|ed)?|
|
|
394
|
+
title: /\b(hidden|closed|dismiss(es|ed)?|not complete|rejected|inert)\b/,
|
|
373
395
|
proof: /\bis hidden\b|\bare hidden\b|message is hidden|not complete|\bhidden\b/,
|
|
374
396
|
need: 'a negative / hidden assertion (`… is hidden`)',
|
|
375
397
|
hint: 'assert the absence/hidden state that the title claims, not just an unrelated visible element.',
|
|
376
398
|
severity: 'fail',
|
|
377
399
|
},
|
|
400
|
+
{
|
|
401
|
+
claim: 'cleared/emptied',
|
|
402
|
+
title: /\b(cleared|clears|emptied|empties|reset to empty|wiped)\b/,
|
|
403
|
+
proof: /\bis empty\b|with \{\{empty|with ['"]?['"]?\s*$|\bempty\b/,
|
|
404
|
+
need: 'an empty/cleared assertion after the action (e.g. `field with {{empty_value}}` / `is empty`)',
|
|
405
|
+
hint: 'prove the value is actually gone — return to the screen and assert the field is empty, not just that the action ran.',
|
|
406
|
+
severity: 'fail',
|
|
407
|
+
},
|
|
408
|
+
{
|
|
409
|
+
claim: 'restored/preserved',
|
|
410
|
+
title: /\b(restored|preserved|persists?|retained|remembered|kept)\b/,
|
|
411
|
+
proof: /\bremember\b|with \{\{|field with/,
|
|
412
|
+
need: 'the value re-asserted after the transition (capture or `field with {{v}}` after returning)',
|
|
413
|
+
hint: 'prove the value survives — assert the field still holds the typed value after the reload/return, not just that it was typed.',
|
|
414
|
+
severity: 'warn',
|
|
415
|
+
},
|
|
416
|
+
{
|
|
417
|
+
claim: 'independent/separate',
|
|
418
|
+
title: /\b(independent|separate|isolat(ed|es)|per[- ]tab|two tabs|each tab)\b/,
|
|
419
|
+
proof: /\bcontext\b|tab a|tab b|second (tab|context)/,
|
|
420
|
+
need: 'a multi-context proof (tab A vs tab B)',
|
|
421
|
+
hint: 'independence across tabs/contexts is rarely DSL-expressible — mark @manual with a clear setup/action/oracle.',
|
|
422
|
+
severity: 'warn',
|
|
423
|
+
},
|
|
424
|
+
{
|
|
425
|
+
claim: 'sanitized/inert',
|
|
426
|
+
title: /\b(sanitized|sanitised|escaped|inert|not executed|not rendered|stripped)\b/,
|
|
427
|
+
proof: /field with \{\{|payload|inert|toContainText|is hidden/,
|
|
428
|
+
need: 'the payload echoed as inert text (`field with {{payload}}`) + no execution',
|
|
429
|
+
hint: 'prove the payload round-trips as literal text and triggers nothing — assert the field value and the absence of any effect.',
|
|
430
|
+
severity: 'warn',
|
|
431
|
+
},
|
|
432
|
+
{
|
|
433
|
+
claim: 'announces/aria',
|
|
434
|
+
title: /\b(announce[sd]?|aria|screen[- ]reader|programmatically associated)\b/,
|
|
435
|
+
proof: /aria|role|@manual|describedby|is focused/,
|
|
436
|
+
need: 'an aria/role assertion (or @manual with a screen-reader oracle)',
|
|
437
|
+
hint: 'ARIA announcement is usually not DSL-expressible — assert aria attributes if possible, else @manual with an NVDA/VoiceOver oracle.',
|
|
438
|
+
severity: 'warn',
|
|
439
|
+
},
|
|
440
|
+
{
|
|
441
|
+
// GENERAL CATCH-ALL (last) — any negative/absence title not handled by a specific
|
|
442
|
+
// rule above. Language-aware negation, NO verb list: if the title says "no / not /
|
|
443
|
+
// never / without / không / prevents …" the steps must carry a NEGATIVE/contrast
|
|
444
|
+
// assertion (hidden, empty, error, count, no-longer, a remembered before/after) — not
|
|
445
|
+
// only a positive presence. WARN, because a positive proxy is sometimes a valid
|
|
446
|
+
// negative proof (e.g. "stayed on the login page"); the semantic reviewer is the
|
|
447
|
+
// authoritative recall layer for the residue this can't judge structurally.
|
|
448
|
+
claim: 'negative-claim/absence',
|
|
449
|
+
title: /(\bno\b|\bnot\b|n['’]t\b|\bnever\b|\bwithout\b|\bcannot\b|prevent|block|avoid|reject|disabl|\bdeny\b|denies|\bkhông\b|\bchưa\b)/i,
|
|
450
|
+
proof: /is hidden|are hidden|is empty|no longer|not complete|disabl|invalid|rejected|\berror\b|\bcount\b|row with \{\{|table with|\bremember\b|\bexactly\b|tohavecount/i,
|
|
451
|
+
need: 'a proof of the ABSENCE — a contrast/empty/hidden/error/count assertion, or @manual with an oracle',
|
|
452
|
+
hint: 'a negative claim ("no / not / không …") is not proven by a positive `see [X]` that looks the same whether or not the claim holds. Assert the contrast (state hidden/empty, error shown, count unchanged), or mark @manual.',
|
|
453
|
+
severity: 'warn',
|
|
454
|
+
},
|
|
378
455
|
];
|
|
379
456
|
|
|
380
457
|
// ---------- Viewpoint taxonomy-lint (harness-roadmap §0.5 Q3) ----------
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spec-clause coverage (harness G2) — the spec-layer faithfulness check.
|
|
3
|
+
*
|
|
4
|
+
* Parses the structured parts of a screen `spec.md` (Functional Requirements + the
|
|
5
|
+
* Validation Rules table) and verifies (a) every MUST FR is covered by a scenario and
|
|
6
|
+
* (b) for EACH validation constraint, every trigger the spec mandates is actually
|
|
7
|
+
* exercised. The trigger check is PER-CONSTRAINT — so "validate format on blur AND on
|
|
8
|
+
* submit" tested only on blur is caught even when other constraints do use submit.
|
|
9
|
+
* Generic: no project data.
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from 'fs';
|
|
12
|
+
import { ScenarioInfo } from './parse';
|
|
13
|
+
|
|
14
|
+
export type Modality = 'MUST' | 'SHOULD' | 'MAY';
|
|
15
|
+
|
|
16
|
+
export interface TriggerGap { constraint: string; code: string; required: string[]; found: string[]; missing: string[] }
|
|
17
|
+
|
|
18
|
+
export interface SpecCoverageResult {
|
|
19
|
+
hasSpec: boolean;
|
|
20
|
+
frTotal: number;
|
|
21
|
+
frCovered: number;
|
|
22
|
+
uncoveredMust: { id: string; text: string }[];
|
|
23
|
+
triggerGaps: TriggerGap[]; // per-constraint trigger matrix gaps
|
|
24
|
+
verdict: 'pass' | 'warn' | 'fail';
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
interface FrClause { id: string; text: string; modality: Modality }
|
|
28
|
+
interface ValRow { constraint: string; code: string; triggers: string[] }
|
|
29
|
+
|
|
30
|
+
// Parsing the spec's Trigger CELL: loose word match ("blur, submit").
|
|
31
|
+
const SPEC_TRIGGER: { trigger: string; re: RegExp }[] = [
|
|
32
|
+
{ trigger: 'blur', re: /\b(blur|on blur|focus[- ]?out)\b/i },
|
|
33
|
+
{ trigger: 'submit', re: /\b(submit|on submit)\b/i },
|
|
34
|
+
];
|
|
35
|
+
// Detecting the trigger ACTION in a scenario: must be an actual action, NOT the word
|
|
36
|
+
// "submit" in a "[Submit] button is disabled" assertion.
|
|
37
|
+
const ACTION_TRIGGER: { trigger: string; re: RegExp }[] = [
|
|
38
|
+
{ trigger: 'blur', re: /\b(press tab|blur|focus[- ]?out|loses? focus|tab away|tab out)\b/i },
|
|
39
|
+
{ trigger: 'submit', re: /(click \[submit\]|press enter|\benter key\b|送信する)/i },
|
|
40
|
+
];
|
|
41
|
+
|
|
42
|
+
function modalityOf(text: string): Modality {
|
|
43
|
+
if (/\bMUST\b/.test(text)) return 'MUST';
|
|
44
|
+
if (/\bSHOULD\b/.test(text)) return 'SHOULD';
|
|
45
|
+
return 'MAY';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function specTriggersIn(text: string): string[] {
|
|
49
|
+
return SPEC_TRIGGER.filter((t) => t.re.test(text)).map((t) => t.trigger);
|
|
50
|
+
}
|
|
51
|
+
function actionTriggersIn(text: string): string[] {
|
|
52
|
+
return ACTION_TRIGGER.filter((t) => t.re.test(text)).map((t) => t.trigger);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function parseSpecClauses(specPath: string): { frs: FrClause[]; valRows: ValRow[] } {
|
|
56
|
+
if (!fs.existsSync(specPath)) return { frs: [], valRows: [] };
|
|
57
|
+
const lines = fs.readFileSync(specPath, 'utf-8').split('\n');
|
|
58
|
+
|
|
59
|
+
const frs: FrClause[] = [];
|
|
60
|
+
for (const line of lines) {
|
|
61
|
+
const m = line.match(/\*\*FR-(\d+)\*\*\s*:\s*(.+)$/);
|
|
62
|
+
if (m) frs.push({ id: `FR-${m[1]}`, text: m[2].replace(/\*\*/g, '').trim(), modality: modalityOf(m[2]) });
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Validation Rules table: a row carries a Constraint, a Trigger cell, and (often) a code.
|
|
66
|
+
const valRows: ValRow[] = [];
|
|
67
|
+
let cTrigger = -1, cConstraint = -1, cCode = -1, inTable = false;
|
|
68
|
+
for (const raw of lines) {
|
|
69
|
+
const line = raw.trim();
|
|
70
|
+
if (line.startsWith('|') && /\btrigger\b/i.test(line) && cTrigger < 0) {
|
|
71
|
+
const cells = line.split('|').map((c) => c.trim());
|
|
72
|
+
cTrigger = cells.findIndex((c) => /^trigger$/i.test(c));
|
|
73
|
+
cConstraint = cells.findIndex((c) => /constraint/i.test(c));
|
|
74
|
+
cCode = cells.findIndex((c) => /code/i.test(c));
|
|
75
|
+
inTable = cTrigger >= 0;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
if (inTable) {
|
|
79
|
+
if (!line.startsWith('|')) { inTable = false; cTrigger = -1; continue; }
|
|
80
|
+
if (/^\|[\s|:-]+\|?$/.test(line)) continue;
|
|
81
|
+
const cells = line.split('|').map((c) => c.trim());
|
|
82
|
+
const triggers = [...new Set((cells[cTrigger] || '').split(/,|\band\b/i).flatMap((p) => specTriggersIn(p)))];
|
|
83
|
+
if (!triggers.length) continue; // server-only rows etc.
|
|
84
|
+
const constraint = (cConstraint >= 0 ? cells[cConstraint] : '').replace(/`/g, '') || 'validation';
|
|
85
|
+
const codeM = (cCode >= 0 ? cells[cCode] : line).match(/M\d+/);
|
|
86
|
+
valRows.push({ constraint, code: codeM ? codeM[0] : '', triggers });
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return { frs, valRows };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Split a feature file into per-scenario blocks (tags + comments + steps), keyed by blank lines. */
|
|
93
|
+
function scenarioBlocks(featureText: string): string[] {
|
|
94
|
+
return featureText.split(/\n\s*\n/).filter((b) => /\bScenario:/.test(b)).map((b) => b.toLowerCase());
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function specCoverage(specPath: string, scenarios: ScenarioInfo[], featureText: string): SpecCoverageResult {
|
|
98
|
+
const { frs, valRows } = parseSpecClauses(specPath);
|
|
99
|
+
if (!fs.existsSync(specPath) || (frs.length === 0 && valRows.length === 0)) {
|
|
100
|
+
return { hasSpec: fs.existsSync(specPath), frTotal: 0, frCovered: 0, uncoveredMust: [], triggerGaps: [], verdict: 'pass' };
|
|
101
|
+
}
|
|
102
|
+
const featLower = featureText.toLowerCase();
|
|
103
|
+
|
|
104
|
+
// FR coverage: explicit @spec:FR / literal FR-id citation, else keyword fallback.
|
|
105
|
+
const uncoveredMust: { id: string; text: string }[] = [];
|
|
106
|
+
let frCovered = 0;
|
|
107
|
+
for (const fr of frs) {
|
|
108
|
+
const idLower = fr.id.toLowerCase();
|
|
109
|
+
const cited = featLower.includes(idLower);
|
|
110
|
+
const words = [...new Set((fr.text.toLowerCase().match(/[a-z][a-z-]{4,}/g) || []))]
|
|
111
|
+
.filter((w) => !/must|should|system|screen|users?|value|input|field/.test(w));
|
|
112
|
+
const kwHit = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= Math.min(2, words.length));
|
|
113
|
+
if (cited || kwHit) frCovered++;
|
|
114
|
+
else if (fr.modality === 'MUST') uncoveredMust.push({ id: fr.id, text: fr.text.slice(0, 90) });
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Per-constraint trigger coverage — the matrix-collapse catch.
|
|
118
|
+
// generic words that don't identify a constraint (would over-match happy-path scenarios)
|
|
119
|
+
const GENERIC = new Set(['valid', 'local', 'part', 'domain', 'email', 'address', 'must', 'exist', 'store', 'field', 'input', 'value', 'formed', 'well', 'character', 'characters']);
|
|
120
|
+
const blocks = scenarioBlocks(featureText);
|
|
121
|
+
const triggerGaps: TriggerGap[] = [];
|
|
122
|
+
for (const row of valRows) {
|
|
123
|
+
const kw = (row.constraint.toLowerCase().match(/[a-z]{4,}/g) || []).filter((w) => !GENERIC.has(w));
|
|
124
|
+
// blocks that belong to this constraint: cite its msg code (primary), or mention a
|
|
125
|
+
// DISTINCTIVE constraint word (generic words filtered to avoid matching happy paths).
|
|
126
|
+
const own = blocks.filter((b) =>
|
|
127
|
+
(row.code && b.includes(row.code.toLowerCase())) || kw.some((w) => b.includes(w)),
|
|
128
|
+
);
|
|
129
|
+
if (!own.length) continue; // constraint has no scenarios at all — FR/viewpoint gate covers that
|
|
130
|
+
const found = [...new Set(own.flatMap((b) => actionTriggersIn(b)))];
|
|
131
|
+
const missing = row.triggers.filter((t) => !found.includes(t));
|
|
132
|
+
if (missing.length) triggerGaps.push({ constraint: row.constraint, code: row.code, required: row.triggers, found, missing });
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const verdict: SpecCoverageResult['verdict'] =
|
|
136
|
+
uncoveredMust.length > 0 || triggerGaps.length > 0 ? 'fail' : 'pass';
|
|
137
|
+
|
|
138
|
+
return { hasSpec: true, frTotal: frs.length, frCovered, uncoveredMust, triggerGaps, verdict };
|
|
139
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Viewpoint Atomic Coverage Ledger (harness #2).
|
|
3
|
+
*
|
|
4
|
+
* The project's `test-viewpoint.md` IS the coverage contract. This parses it into ATOMIC
|
|
5
|
+
* items (each bullet / table row / ID-prefixed line) and reports the status of EACH —
|
|
6
|
+
* covered / missing — instead of the coarse "viewpoint mentioned" signal. It is fully
|
|
7
|
+
* project-driven (works on any project's viewpoint file, any domain), which is why it
|
|
8
|
+
* scales where a hardcoded domain catalog does not. Advisory: it surfaces the per-item
|
|
9
|
+
* gaps that inflate a "looks-covered" score; it does not fail the gate.
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from 'fs';
|
|
12
|
+
import { ScenarioInfo } from './parse';
|
|
13
|
+
|
|
14
|
+
export interface LedgerItem { id?: string; text: string; covered: boolean }
|
|
15
|
+
|
|
16
|
+
export interface LedgerResult {
|
|
17
|
+
hasViewpoint: boolean;
|
|
18
|
+
total: number;
|
|
19
|
+
covered: number;
|
|
20
|
+
ratio: number;
|
|
21
|
+
missing: { id?: string; text: string }[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const ID_RE = /\b([A-Z]{1,5}\d{0,2}(?:[.\-][A-Za-z0-9]+)*-?\d{0,3})\b/; // VP0.Title, VP7-002, MS-HP-001, TV-01
|
|
25
|
+
const GENERIC = new Set(['display', 'shown', 'value', 'field', 'input', 'page', 'screen', 'button', 'link', 'text', 'check', 'verify', 'should', 'with', 'when', 'then', 'user', 'this', 'that', 'each', 'item', 'items']);
|
|
26
|
+
|
|
27
|
+
/** Extract atomic checklist items from a viewpoint file (format-tolerant). */
|
|
28
|
+
export function parseViewpointItems(viewpointPath: string): { id?: string; text: string }[] {
|
|
29
|
+
if (!fs.existsSync(viewpointPath)) return [];
|
|
30
|
+
const lines = fs.readFileSync(viewpointPath, 'utf-8').split('\n');
|
|
31
|
+
const items: { id?: string; text: string }[] = [];
|
|
32
|
+
let inFence = false;
|
|
33
|
+
for (const raw of lines) {
|
|
34
|
+
const line = raw.trim();
|
|
35
|
+
if (line.startsWith('```')) { inFence = !inFence; continue; }
|
|
36
|
+
if (inFence || !line) continue;
|
|
37
|
+
if (/^#{1,6}\s/.test(line)) continue; // markdown heading
|
|
38
|
+
let text = '';
|
|
39
|
+
const bullet = line.match(/^(?:[-*+]|\d+[.)])\s+(.*)$/);
|
|
40
|
+
if (bullet) text = bullet[1];
|
|
41
|
+
else if (line.startsWith('|')) { // table data row
|
|
42
|
+
if (/^\|[\s|:-]+\|?$/.test(line)) continue; // separator
|
|
43
|
+
const cells = line.split('|').map((c) => c.trim()).filter(Boolean);
|
|
44
|
+
if (/^(vp|id|viewpoint|priority|reason|no\.?|category|item|trigger|#|pattern|applicable|notes|field|constraint|code|description|status)$/i.test(cells[0] || '')) continue; // header
|
|
45
|
+
text = cells.join(' — ');
|
|
46
|
+
} else continue;
|
|
47
|
+
text = text.replace(/[*`]/g, '').trim();
|
|
48
|
+
if (!text) continue;
|
|
49
|
+
const idM = text.match(ID_RE);
|
|
50
|
+
const id = idM && /\d/.test(idM[1]) ? idM[1] : undefined; // require a digit so prose words aren't IDs
|
|
51
|
+
const words = (text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w));
|
|
52
|
+
if (!id && words.length < 2) continue; // not substantive enough to track
|
|
53
|
+
items.push({ id, text: text.slice(0, 100) });
|
|
54
|
+
}
|
|
55
|
+
return items;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function viewpointLedger(viewpointPath: string, scenarios: ScenarioInfo[], featureText: string): LedgerResult {
|
|
59
|
+
const items = parseViewpointItems(viewpointPath);
|
|
60
|
+
if (!fs.existsSync(viewpointPath) || items.length === 0) {
|
|
61
|
+
return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [] };
|
|
62
|
+
}
|
|
63
|
+
const featLower = featureText.toLowerCase();
|
|
64
|
+
const missing: { id?: string; text: string }[] = [];
|
|
65
|
+
let covered = 0;
|
|
66
|
+
|
|
67
|
+
for (const item of items) {
|
|
68
|
+
let isCovered = false;
|
|
69
|
+
if (item.id && featLower.includes(item.id.toLowerCase())) isCovered = true;
|
|
70
|
+
else {
|
|
71
|
+
const words = [...new Set((item.text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w)))];
|
|
72
|
+
const need = Math.min(2, words.length);
|
|
73
|
+
isCovered = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= need);
|
|
74
|
+
}
|
|
75
|
+
if (isCovered) covered++;
|
|
76
|
+
else missing.push({ id: item.id, text: item.text });
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return { hasViewpoint: true, total: items.length, covered, ratio: items.length ? covered / items.length : 1, missing };
|
|
80
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QA baseline audit over a legacy testcase inventory (P-A). Reuses the harness
|
|
3
|
+
* duplicate sensor + the capability-plan reason classifier so the legacy view and
|
|
4
|
+
* the Gherkin view agree. Advisory — evaluates the *existing* manual suite and the
|
|
5
|
+
* automation/driver opportunity; it does not gate anything.
|
|
6
|
+
*/
|
|
7
|
+
import { LegacyInventory, LegacyTestcase } from './legacy-parser';
|
|
8
|
+
import { duplicateClusters } from '../harness/sensors';
|
|
9
|
+
import { ScenarioInfo } from '../harness/parse';
|
|
10
|
+
import { classifyReason, MANUAL_REASONS } from '../harness/capability-plan';
|
|
11
|
+
|
|
12
|
+
export interface BaselineReport {
|
|
13
|
+
total: number;
|
|
14
|
+
sheets: { name: string; type: string; rows: number }[];
|
|
15
|
+
byCategory: Record<string, number>;
|
|
16
|
+
byPriority: Record<string, number>;
|
|
17
|
+
byResult: Record<string, number>;
|
|
18
|
+
depthRatio: number; // % of TC whose Expected asserts a concrete value
|
|
19
|
+
deepCount: number;
|
|
20
|
+
duplicateClusters: number;
|
|
21
|
+
exactDuplicates: number;
|
|
22
|
+
reasons: { // capability-plan style distribution
|
|
23
|
+
ui: number; crossScreen: number; capabilityManual: number; keepManual: number;
|
|
24
|
+
byCode: Record<string, number>;
|
|
25
|
+
driverCandidates: { driver: string; count: number }[];
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** An Expected that asserts a concrete value (not just "displayed/visible"). */
|
|
30
|
+
function isDeep(tc: LegacyTestcase): boolean {
|
|
31
|
+
const e = (tc.expected || '').trim().toLowerCase();
|
|
32
|
+
if (!e) return false;
|
|
33
|
+
const shallowOnly = /^(.{0,40})(表示される|hiển thị|displayed|shown|is visible|表示)$/.test(e);
|
|
34
|
+
const hasValue = /(=|equals?|bằng|giá trị|\bvalue\b|"|「|\bmessage\b|エラー|error|\bcount\b|số lượng|\bformat\b|định dạng)/.test(e);
|
|
35
|
+
return hasValue || (!shallowOnly && e.length > 30);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Synthesize a ScenarioInfo-lite so we can reuse harness sensors on the inventory. */
|
|
39
|
+
export function legacyToScenarioInfo(tc: LegacyTestcase): ScenarioInfo {
|
|
40
|
+
const skeleton = (tc.steps || '')
|
|
41
|
+
.replace(/\{\{[^}]*\}\}/g, '{}').replace(/"[^"]*"/g, '""')
|
|
42
|
+
.replace(/\d+/g, 'N').replace(/\s+/g, ' ').trim().toLowerCase();
|
|
43
|
+
const hay = `${tc.category} ${tc.steps} ${tc.expected}`.toLowerCase();
|
|
44
|
+
return {
|
|
45
|
+
name: `${tc.category}: ${tc.id}`,
|
|
46
|
+
category: (tc.category || '').toUpperCase().replace(/[^A-Z0-9]+/g, '-').slice(0, 16),
|
|
47
|
+
priority: tc.priority,
|
|
48
|
+
manual: true,
|
|
49
|
+
thenCount: tc.expected ? 1 : 0,
|
|
50
|
+
hasDataAssertion: isDeep(tc),
|
|
51
|
+
shallow: !isDeep(tc),
|
|
52
|
+
stepSkeleton: skeleton,
|
|
53
|
+
haystack: hay,
|
|
54
|
+
} as ScenarioInfo;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function baselineAudit(inv: LegacyInventory): BaselineReport {
|
|
58
|
+
const tcs = inv.testcases;
|
|
59
|
+
const inc = (m: Record<string, number>, k: string) => { m[k] = (m[k] || 0) + 1; };
|
|
60
|
+
|
|
61
|
+
const byCategory: Record<string, number> = {};
|
|
62
|
+
const byPriority: Record<string, number> = {};
|
|
63
|
+
const byResult: Record<string, number> = {};
|
|
64
|
+
let deepCount = 0;
|
|
65
|
+
const byCode: Record<string, number> = {};
|
|
66
|
+
let ui = 0, crossScreen = 0, capabilityManual = 0, keepManual = 0;
|
|
67
|
+
const driverHits: Record<string, number> = {};
|
|
68
|
+
|
|
69
|
+
for (const tc of tcs) {
|
|
70
|
+
inc(byCategory, tc.category || '(uncategorized)');
|
|
71
|
+
inc(byPriority, tc.priority);
|
|
72
|
+
inc(byResult, (tc.result || '(not-run)').trim() || '(not-run)');
|
|
73
|
+
if (isDeep(tc)) deepCount++;
|
|
74
|
+
|
|
75
|
+
const code = classifyReason(`${tc.precondition || ''} ${tc.steps} ${tc.expected} ${tc.testData || ''}`);
|
|
76
|
+
if (!code) { ui++; continue; }
|
|
77
|
+
inc(byCode, code);
|
|
78
|
+
if (code === 'XS') { crossScreen++; continue; }
|
|
79
|
+
const def = MANUAL_REASONS[code];
|
|
80
|
+
if (def?.cls === 'keep') keepManual++;
|
|
81
|
+
else { capabilityManual++; for (const d of def?.drivers || []) driverHits[d] = (driverHits[d] || 0) + 1; }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const dup = duplicateClusters(tcs.map(legacyToScenarioInfo));
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
total: tcs.length,
|
|
88
|
+
sheets: inv.sheets,
|
|
89
|
+
byCategory, byPriority, byResult,
|
|
90
|
+
depthRatio: tcs.length ? deepCount / tcs.length : 0,
|
|
91
|
+
deepCount,
|
|
92
|
+
duplicateClusters: dup.clusters.length,
|
|
93
|
+
exactDuplicates: dup.exactDuplicateCount,
|
|
94
|
+
reasons: {
|
|
95
|
+
ui, crossScreen, capabilityManual, keepManual, byCode,
|
|
96
|
+
driverCandidates: Object.entries(driverHits).map(([driver, count]) => ({ driver, count }))
|
|
97
|
+
.sort((a, b) => b.count - a.count),
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
}
|