@sun-asterisk/sungen 3.0.0-beta.74 → 3.0.0-beta.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +17 -3
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +30 -14
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/ingest.d.ts +3 -0
- package/dist/cli/commands/ingest.d.ts.map +1 -0
- package/dist/cli/commands/ingest.js +179 -0
- package/dist/cli/commands/ingest.js.map +1 -0
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/dashboard/templates/index.html +54 -54
- package/dist/generators/gherkin-parser/index.d.ts +2 -0
- package/dist/generators/gherkin-parser/index.d.ts.map +1 -1
- package/dist/generators/gherkin-parser/index.js +15 -0
- package/dist/generators/gherkin-parser/index.js.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/all-contain-element.hbs +8 -0
- package/dist/generators/test-generator/patterns/capture-patterns.d.ts +5 -0
- package/dist/generators/test-generator/patterns/capture-patterns.d.ts.map +1 -1
- package/dist/generators/test-generator/patterns/capture-patterns.js +33 -0
- package/dist/generators/test-generator/patterns/capture-patterns.js.map +1 -1
- package/dist/harness/audit.d.ts +5 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +13 -2
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/capability-plan.d.ts +6 -0
- package/dist/harness/capability-plan.d.ts.map +1 -1
- package/dist/harness/capability-plan.js +13 -0
- package/dist/harness/capability-plan.js.map +1 -1
- package/dist/harness/parse.d.ts +1 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +3 -0
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/provenance.d.ts +6 -0
- package/dist/harness/provenance.d.ts.map +1 -0
- package/dist/harness/provenance.js +65 -0
- package/dist/harness/provenance.js.map +1 -0
- package/dist/harness/sensors.d.ts +30 -0
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +122 -0
- package/dist/harness/sensors.js.map +1 -1
- package/dist/ingest/baseline-audit.d.ts +38 -0
- package/dist/ingest/baseline-audit.d.ts.map +1 -0
- package/dist/ingest/baseline-audit.js +85 -0
- package/dist/ingest/baseline-audit.js.map +1 -0
- package/dist/ingest/gsheet-fetch.d.ts +9 -0
- package/dist/ingest/gsheet-fetch.d.ts.map +1 -0
- package/dist/ingest/gsheet-fetch.js +180 -0
- package/dist/ingest/gsheet-fetch.js.map +1 -0
- package/dist/ingest/index.d.ts +6 -0
- package/dist/ingest/index.d.ts.map +1 -0
- package/dist/ingest/index.js +22 -0
- package/dist/ingest/index.js.map +1 -0
- package/dist/ingest/legacy-parser.d.ts +39 -0
- package/dist/ingest/legacy-parser.d.ts.map +1 -0
- package/dist/ingest/legacy-parser.js +218 -0
- package/dist/ingest/legacy-parser.js.map +1 -0
- package/dist/ingest/reconcile.d.ts +30 -0
- package/dist/ingest/reconcile.d.ts.map +1 -0
- package/dist/ingest/reconcile.js +65 -0
- package/dist/ingest/reconcile.js.map +1 -0
- package/dist/ingest/to-gherkin.d.ts +33 -0
- package/dist/ingest/to-gherkin.d.ts.map +1 -0
- package/dist/ingest/to-gherkin.js +93 -0
- package/dist/ingest/to-gherkin.js.map +1 -0
- package/dist/orchestrator/ai-rules-updater.d.ts.map +1 -1
- package/dist/orchestrator/ai-rules-updater.js +2 -0
- package/dist/orchestrator/ai-rules-updater.js.map +1 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-delivery.md +10 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-gherkin-syntax.md +12 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-ingest-legacy.md +79 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-delivery.md +10 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-gherkin-syntax.md +12 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-ingest-legacy.md +79 -0
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +13 -3
- package/src/cli/commands/delivery.ts +31 -15
- package/src/cli/commands/ingest.ts +141 -0
- package/src/cli/index.ts +2 -0
- package/src/dashboard/templates/index.html +54 -54
- package/src/generators/gherkin-parser/index.ts +17 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/all-contain-element.hbs +8 -0
- package/src/generators/test-generator/patterns/capture-patterns.ts +38 -0
- package/src/harness/audit.ts +18 -4
- package/src/harness/capability-plan.ts +11 -0
- package/src/harness/parse.ts +4 -0
- package/src/harness/provenance.ts +33 -0
- package/src/harness/sensors.ts +189 -0
- package/src/ingest/baseline-audit.ts +100 -0
- package/src/ingest/gsheet-fetch.ts +152 -0
- package/src/ingest/index.ts +5 -0
- package/src/ingest/legacy-parser.ts +184 -0
- package/src/ingest/reconcile.ts +80 -0
- package/src/ingest/to-gherkin.ts +108 -0
- package/src/orchestrator/ai-rules-updater.ts +2 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-delivery.md +10 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-gherkin-syntax.md +12 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-ingest-legacy.md +79 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-delivery.md +10 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-gherkin-syntax.md +12 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-ingest-legacy.md +79 -0
|
@@ -18,6 +18,8 @@ export interface ParsedStep {
|
|
|
18
18
|
keyword: string; // Given, When, Then, And, But
|
|
19
19
|
text: string; // Original step text
|
|
20
20
|
selectorRef?: string; // Extracted selector reference like [auth-login.email]
|
|
21
|
+
childRef?: string; // Q2: second [ref] after "contain" — child element for all-card structural assertion
|
|
22
|
+
childElementType?: string; // Element type after the child [ref], e.g. "button" in "contain [Add To Cart] button"
|
|
21
23
|
dataRef?: string; // Extracted data reference like <valid_user.email>
|
|
22
24
|
value?: string; // Static value if any
|
|
23
25
|
elementType?: string; // Element type after [Selector] e.g., "button", "text", "link", "field"
|
|
@@ -173,12 +175,25 @@ export class GherkinParser {
|
|
|
173
175
|
if (selectorRef) {
|
|
174
176
|
const elementTypeMatch = text.match(/\]\s+([\w-]+)/);
|
|
175
177
|
elementType = elementTypeMatch ? elementTypeMatch[1].toLowerCase() : undefined;
|
|
178
|
+
// In "see all [Card] contain [Child]" the word after the first `]` is the
|
|
179
|
+
// connector "contain", not an element type — don't mistake it for one (Q2).
|
|
180
|
+
if (elementType && /^(contain|contains|include|includes)$/.test(elementType)) elementType = undefined;
|
|
176
181
|
} else {
|
|
177
182
|
// Extract final word if it's a known type (page, button, field, etc)
|
|
178
183
|
const finalWordMatch = text.match(/\b(page|button|field|link|text|checkbox|radio|select|dropdown|modal|dialog|drawer|message|error|warning|success|alert|table|list|list-item|listitem|row|search|option|slider|toggle|tab|breadcrumb|overlay|step|section|card|item|icon|image|header|label|badge|tooltip|tag|menuitem|cell|column|uploader|date-picker|key)\b$/i);
|
|
179
184
|
elementType = finalWordMatch ? finalWordMatch[1].toLowerCase() : undefined;
|
|
180
185
|
}
|
|
181
186
|
|
|
187
|
+
// Q2: a second [ref] after "contain"/"include" = the child element every container
|
|
188
|
+
// must hold, e.g. "see all [Product Card] contain [Add To Cart] button".
|
|
189
|
+
let childRef: string | undefined;
|
|
190
|
+
let childElementType: string | undefined;
|
|
191
|
+
const childMatch = text.match(/\b(?:contain|contains|include|includes)\s+\[([^\]]+)\]\s*([\w-]+)?/i);
|
|
192
|
+
if (childMatch) {
|
|
193
|
+
childRef = childMatch[1];
|
|
194
|
+
childElementType = childMatch[2] ? childMatch[2].toLowerCase() : undefined;
|
|
195
|
+
}
|
|
196
|
+
|
|
182
197
|
// Extract positional index from "[Element] field 3", "[Element] button 2", etc.
|
|
183
198
|
let nth = 0;
|
|
184
199
|
if (selectorRef && selectorMatch) {
|
|
@@ -202,6 +217,8 @@ export class GherkinParser {
|
|
|
202
217
|
keyword: step.keyword.trim(),
|
|
203
218
|
text: step.text, // Preserve original text (with parent scoping) for pattern matching
|
|
204
219
|
selectorRef,
|
|
220
|
+
childRef,
|
|
221
|
+
childElementType,
|
|
205
222
|
dataRef,
|
|
206
223
|
value,
|
|
207
224
|
elementType,
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
{
|
|
2
|
+
const __cards_{{stepCounter}} = {{> locator}};
|
|
3
|
+
const __n_{{stepCounter}} = await __cards_{{stepCounter}}.count();
|
|
4
|
+
expect(__n_{{stepCounter}}, 'Expected at least one [{{selectorRef}}]').toBeGreaterThan(0);
|
|
5
|
+
for (let __i_{{stepCounter}} = 0; __i_{{stepCounter}} < __n_{{stepCounter}}; __i_{{stepCounter}}++) {
|
|
6
|
+
await expect({{#with child}}{{> locator}}{{/with}}, 'Every [{{selectorRef}}] must contain [{{childRef}}]').toBeVisible();
|
|
7
|
+
}
|
|
8
|
+
}
|
|
@@ -13,6 +13,11 @@ import { StepPattern, StepTemplateData } from './types';
|
|
|
13
13
|
* 2. List: `User see all [Product Name] contain {{selected_category}}`
|
|
14
14
|
* → asserts EVERY matching element's text contains the value
|
|
15
15
|
* (e.g. all products belong to the selected category/brand).
|
|
16
|
+
*
|
|
17
|
+
* 3. All-card: `User see all [Product Card] contain [Add To Cart] button`
|
|
18
|
+
* → asserts EVERY container element holds the child element
|
|
19
|
+
* (structural per-card proof: every card has name/price/action).
|
|
20
|
+
* Roadmap Q2 — the fix for "each card exposes X" claims.
|
|
16
21
|
*/
|
|
17
22
|
export const capturePatterns: StepPattern[] = [
|
|
18
23
|
{
|
|
@@ -56,4 +61,37 @@ export const capturePatterns: StepPattern[] = [
|
|
|
56
61
|
},
|
|
57
62
|
priority: 34,
|
|
58
63
|
},
|
|
64
|
+
{
|
|
65
|
+
// Q2 — all-card structural assertion: every container holds the child element.
|
|
66
|
+
// Distinguished from `all-contain-assertion` by a second [ref] (childRef) and the
|
|
67
|
+
// ABSENCE of a value/data (which would make it a text-contains assertion instead).
|
|
68
|
+
name: 'all-contain-element',
|
|
69
|
+
matcher: (step: ParsedStep) =>
|
|
70
|
+
/\b(see|sees)\b/i.test(step.text) &&
|
|
71
|
+
/\ball\b/i.test(step.text) &&
|
|
72
|
+
/(contain|contains|include|includes)/i.test(step.text) &&
|
|
73
|
+
!!step.selectorRef && !!step.childRef && !step.value && !step.dataRef,
|
|
74
|
+
resolver: (step, context): StepTemplateData => {
|
|
75
|
+
const container = context.selectorResolver.resolveSelector(
|
|
76
|
+
step.selectorRef!, undefined, step.elementType, step.nth,
|
|
77
|
+
);
|
|
78
|
+
const child = context.selectorResolver.resolveSelector(
|
|
79
|
+
step.childRef!, undefined, step.childElementType, 0,
|
|
80
|
+
);
|
|
81
|
+
const sc = context.stepCounter;
|
|
82
|
+
return {
|
|
83
|
+
templateName: 'all-contain-element',
|
|
84
|
+
data: {
|
|
85
|
+
...container,
|
|
86
|
+
selectorRef: step.selectorRef,
|
|
87
|
+
childRef: step.childRef,
|
|
88
|
+
stepCounter: sc,
|
|
89
|
+
// Render the child RELATIVE to each container row via locator.hbs's parentLocator branch.
|
|
90
|
+
child: { ...child, parentLocator: `__cards_${sc}.nth(__i_${sc})` },
|
|
91
|
+
},
|
|
92
|
+
comment: `Assert all ${step.selectorRef} contain [${step.childRef}]`,
|
|
93
|
+
};
|
|
94
|
+
},
|
|
95
|
+
priority: 36,
|
|
96
|
+
},
|
|
59
97
|
];
|
package/src/harness/audit.ts
CHANGED
|
@@ -9,16 +9,19 @@
|
|
|
9
9
|
import * as path from 'path';
|
|
10
10
|
import { loadScenarios, parseViewpointOverview, ScenarioInfo, ViewpointEntry } from './parse';
|
|
11
11
|
import {
|
|
12
|
-
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, coverageBalance, duplicateClusters, traceability,
|
|
13
|
-
GateResult, DepthResult, BalanceResult, DuplicateResult, TraceResult,
|
|
12
|
+
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, coverageBalance, duplicateClusters, traceability, claimProof, taxonomyLint,
|
|
13
|
+
GateResult, DepthResult, BalanceResult, DuplicateResult, TraceResult, ClaimProofResult, TaxonomyResult,
|
|
14
14
|
} from './sensors';
|
|
15
15
|
import { readIntent, projectRootFromScreenDir, IntentProfile } from './intent';
|
|
16
|
+
import { getProvenance, Provenance } from './provenance';
|
|
16
17
|
|
|
17
18
|
export interface AuditReport {
|
|
18
19
|
screen: string;
|
|
19
20
|
scenarioCount: number;
|
|
20
21
|
gate: GateResult;
|
|
21
22
|
depth: DepthResult;
|
|
23
|
+
claim: ClaimProofResult;
|
|
24
|
+
taxonomy: TaxonomyResult;
|
|
22
25
|
balance: BalanceResult;
|
|
23
26
|
duplicates: DuplicateResult;
|
|
24
27
|
trace: TraceResult;
|
|
@@ -33,6 +36,7 @@ export interface AuditReport {
|
|
|
33
36
|
gateStatus: 'PASS' | 'FAIL';
|
|
34
37
|
findings: string[]; // human-actionable, what the Repair loop would target
|
|
35
38
|
intent: IntentProfile; // P3 — the intent profile that drove the thresholds
|
|
39
|
+
provenance: Provenance; // D1 — sungen version + catalog hash (diagnose cross-user score gaps)
|
|
36
40
|
}
|
|
37
41
|
|
|
38
42
|
export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
@@ -47,6 +51,8 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
47
51
|
// P3 — intent profile from qa/context.md drives the depth threshold (focus).
|
|
48
52
|
const intent = readIntent(projectRootFromScreenDir(screenDir));
|
|
49
53
|
const depth = assertionDepth(scenarios, dataThemesFor(catalog, gate.pageType), intent.focus);
|
|
54
|
+
const claim = claimProof(scenarios, intent.focus);
|
|
55
|
+
const taxonomy = taxonomyLint(scenarios);
|
|
50
56
|
const balance = coverageBalance(scenarios);
|
|
51
57
|
const duplicates = duplicateClusters(scenarios);
|
|
52
58
|
const trace = traceability(scenarios, viewpoints);
|
|
@@ -78,6 +84,13 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
78
84
|
`add data assertions (\`... with {{value}}\`, \`see all ... contain {{v}}\`) or, if cross-screen, defer to a flow with @manual + reason.`,
|
|
79
85
|
);
|
|
80
86
|
}
|
|
87
|
+
for (const u of claim.unproven) {
|
|
88
|
+
const tag = u.severity === 'fail' ? 'CLAIM-UNPROVEN' : 'CLAIM-WEAK';
|
|
89
|
+
findings.push(`${tag}: "${u.name}" — title claims [${u.claim}] but steps lack ${u.need}. ${u.hint}`);
|
|
90
|
+
}
|
|
91
|
+
for (const m of taxonomy.mislabeled) {
|
|
92
|
+
findings.push(`VP-MISLABEL: "${m.name}" is coded VP-${m.current} but reads as ${m.suggested} (signal: "${m.signal}") → re-tag VP-${m.suggested}-NNN so the coverage matrix isn't skewed.`);
|
|
93
|
+
}
|
|
81
94
|
if (balance.imbalanced) {
|
|
82
95
|
findings.push(`BALANCE: ${balance.note} Stop expanding secondary viewpoints until business-core gaps are filled.`);
|
|
83
96
|
}
|
|
@@ -91,12 +104,12 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
91
104
|
// Gate now spans coverage (viewpoint themes) AND depth (data-correctness).
|
|
92
105
|
// A depth 'fail' (below the intent threshold) fails the gate; 'warn' does not.
|
|
93
106
|
const gateStatus: 'PASS' | 'FAIL' =
|
|
94
|
-
gate.gaps.length === 0 && depth.verdict !== 'fail' ? 'PASS' : 'FAIL';
|
|
107
|
+
gate.gaps.length === 0 && depth.verdict !== 'fail' && claim.verdict !== 'fail' ? 'PASS' : 'FAIL';
|
|
95
108
|
|
|
96
109
|
return {
|
|
97
110
|
screen: screenName,
|
|
98
111
|
scenarioCount: scenarios.length,
|
|
99
|
-
gate, depth, balance, duplicates, trace,
|
|
112
|
+
gate, depth, claim, taxonomy, balance, duplicates, trace,
|
|
100
113
|
score: {
|
|
101
114
|
overall: Math.round(overall * 10) / 10,
|
|
102
115
|
coverage: Math.round(coverage * 100) / 100,
|
|
@@ -108,5 +121,6 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
108
121
|
gateStatus,
|
|
109
122
|
findings,
|
|
110
123
|
intent,
|
|
124
|
+
provenance: getProvenance(),
|
|
111
125
|
};
|
|
112
126
|
}
|
|
@@ -43,6 +43,17 @@ const INFER: { code: string; re: RegExp }[] = [
|
|
|
43
43
|
{ code: 'M9', re: /\b(judgment|human|subjective|manual review)\b/i },
|
|
44
44
|
];
|
|
45
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Classify free text (e.g. a legacy testcase's precondition+steps+expected) into a
|
|
48
|
+
* manual-reason code, or '' when nothing matches (→ UI-automatable). Reuses the same
|
|
49
|
+
* INFER patterns as the scenario planner so legacy-ingest and the Gherkin planner agree.
|
|
50
|
+
*/
|
|
51
|
+
export function classifyReason(text: string): string {
|
|
52
|
+
const t = (text || '').toLowerCase();
|
|
53
|
+
for (const { code, re } of INFER) if (re.test(t)) return code;
|
|
54
|
+
return '';
|
|
55
|
+
}
|
|
56
|
+
|
|
46
57
|
interface ParsedScenario { name: string; tags: string[]; manual: boolean; reason: string }
|
|
47
58
|
|
|
48
59
|
/** Parse scenarios with their tags + the reason comment line above (for @manual). */
|
package/src/harness/parse.ts
CHANGED
|
@@ -28,6 +28,7 @@ export interface ScenarioInfo {
|
|
|
28
28
|
shallow: boolean; // asserts only visibility/navigation (no data)
|
|
29
29
|
stepSkeleton: string; // normalized steps for duplicate clustering
|
|
30
30
|
haystack: string; // lowercase name + steps text (for keyword coverage)
|
|
31
|
+
stepsText: string; // lowercase steps ONLY (name excluded) — for claim-proof
|
|
31
32
|
}
|
|
32
33
|
|
|
33
34
|
// ---------- test-viewpoint.md ----------
|
|
@@ -98,11 +99,13 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
98
99
|
let hasData = false;
|
|
99
100
|
const skeletonParts: string[] = [];
|
|
100
101
|
const textParts: string[] = [sc.name];
|
|
102
|
+
const stepTextParts: string[] = [];
|
|
101
103
|
|
|
102
104
|
for (const step of sc.steps as ParsedStep[]) {
|
|
103
105
|
const kw = step.keyword.trim();
|
|
104
106
|
if (kw === 'Given' || kw === 'When' || kw === 'Then') last = kw;
|
|
105
107
|
textParts.push(step.text);
|
|
108
|
+
stepTextParts.push(step.text);
|
|
106
109
|
// normalized skeleton: keep [refs] (distinct targets = distinct tests),
|
|
107
110
|
// but neutralize {{vars}} and quoted values so EP/data families collapse.
|
|
108
111
|
const skel = step.text
|
|
@@ -132,6 +135,7 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
132
135
|
shallow,
|
|
133
136
|
stepSkeleton: skeletonParts.join(' | '),
|
|
134
137
|
haystack: textParts.join(' ').toLowerCase(),
|
|
138
|
+
stepsText: stepTextParts.join(' ').toLowerCase(),
|
|
135
139
|
};
|
|
136
140
|
}
|
|
137
141
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provenance stamp for deterministic reports (score-determinism D1).
|
|
3
|
+
*
|
|
4
|
+
* The audit score is a pure function of the project files + the bundled catalog.
|
|
5
|
+
* Stamping the sungen version + a catalog hash into the report lets two users who
|
|
6
|
+
* get different scores see IMMEDIATELY whether the cause is a version/catalog
|
|
7
|
+
* mismatch (the main residual source of cross-user variance) rather than a real
|
|
8
|
+
* difference in their test design.
|
|
9
|
+
*/
|
|
10
|
+
import * as fs from 'fs';
|
|
11
|
+
import * as path from 'path';
|
|
12
|
+
import * as crypto from 'crypto';
|
|
13
|
+
|
|
14
|
+
export interface Provenance {
|
|
15
|
+
sungenVersion: string;
|
|
16
|
+
catalogHash: string; // sha256(universal-viewpoints.yaml), first 12 hex — the catalog that drives the score
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function getProvenance(): Provenance {
|
|
20
|
+
let sungenVersion = 'unknown';
|
|
21
|
+
try {
|
|
22
|
+
const pkg = JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', 'package.json'), 'utf-8'));
|
|
23
|
+
if (pkg && typeof pkg.version === 'string') sungenVersion = pkg.version;
|
|
24
|
+
} catch { /* version stays 'unknown' */ }
|
|
25
|
+
|
|
26
|
+
let catalogHash = 'unknown';
|
|
27
|
+
try {
|
|
28
|
+
const cat = fs.readFileSync(path.join(__dirname, 'catalog', 'universal-viewpoints.yaml'), 'utf-8');
|
|
29
|
+
catalogHash = crypto.createHash('sha256').update(cat).digest('hex').slice(0, 12);
|
|
30
|
+
} catch { /* hash stays 'unknown' */ }
|
|
31
|
+
|
|
32
|
+
return { sungenVersion, catalogHash };
|
|
33
|
+
}
|
package/src/harness/sensors.ts
CHANGED
|
@@ -277,3 +277,192 @@ export function traceability(scenarios: ScenarioInfo[], viewpoints: ViewpointEnt
|
|
|
277
277
|
: 'Traceable.',
|
|
278
278
|
};
|
|
279
279
|
}
|
|
280
|
+
|
|
281
|
+
// ---------- Claim-Proof (harness-roadmap §0.5 Q1) ----------
|
|
282
|
+
//
|
|
283
|
+
// A scenario's TITLE makes a claim; its STEPS must prove it. This is the
|
|
284
|
+
// design-layer sibling of script-check's anti-bypass (code↔Gherkin): here it is
|
|
285
|
+
// title↔steps. We only measure scenarios whose title carries a claim keyword, so
|
|
286
|
+
// plain visibility/navigation scenarios are never falsely pressured. @manual
|
|
287
|
+
// scenarios are skipped — their claim is deferred to a flow (XS/flow-check owns it).
|
|
288
|
+
|
|
289
|
+
export interface ClaimUnproven {
|
|
290
|
+
name: string;
|
|
291
|
+
claim: string; // the claim keyword matched in the title
|
|
292
|
+
need: string; // the assertion shape the steps must contain
|
|
293
|
+
hint: string; // how to fix
|
|
294
|
+
severity: 'fail' | 'warn';
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
export interface ClaimProofResult {
|
|
298
|
+
total: number; // non-manual scenarios
|
|
299
|
+
withClaims: number; // scenarios whose title carries a claim
|
|
300
|
+
proven: number;
|
|
301
|
+
unproven: ClaimUnproven[];
|
|
302
|
+
ratio: number; // proven / withClaims (1 if withClaims === 0)
|
|
303
|
+
focus: string;
|
|
304
|
+
threshold: number;
|
|
305
|
+
verdict: DepthVerdict;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
interface ClaimRule {
|
|
309
|
+
claim: string;
|
|
310
|
+
title: RegExp; // claim keyword(s) in the title
|
|
311
|
+
proof: RegExp; // required assertion shape in the steps
|
|
312
|
+
need: string;
|
|
313
|
+
hint: string;
|
|
314
|
+
severity: 'fail' | 'warn';
|
|
315
|
+
// Optional: even when `proof` matches, if `underproof` also matches the proof is
|
|
316
|
+
// PARTIAL (the right shape exists but on the wrong/another element) → a warn-level
|
|
317
|
+
// finding. e.g. an "each card exposes <action>" claim that has a see-all on price
|
|
318
|
+
// but asserts the actions with singular presence.
|
|
319
|
+
underproof?: RegExp;
|
|
320
|
+
underNeed?: string;
|
|
321
|
+
underHint?: string;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// Ordered by specificity; the first matching rule per scenario wins.
|
|
325
|
+
const CLAIM_RULES: ClaimRule[] = [
|
|
326
|
+
{
|
|
327
|
+
claim: 'all/every/each',
|
|
328
|
+
title: /\b(all|every|each)\b/,
|
|
329
|
+
proof: /\bsee all\b/,
|
|
330
|
+
need: 'a `User see all …` assertion (every element, not one)',
|
|
331
|
+
hint: 'assert across all matched elements — for per-card structure use the all-card DSL (roadmap Q2).',
|
|
332
|
+
severity: 'fail',
|
|
333
|
+
// singular action presence under an "each/every" claim = partial proof
|
|
334
|
+
underproof: /\bsee \[[^\]]+\] (link|button)\b/,
|
|
335
|
+
underNeed: 'an all-card assertion for the claimed action (not single presence)',
|
|
336
|
+
underHint: 'the actions are asserted once, not per-card — wrap them with the all-card DSL `User see all [Card] contain [Action]` (roadmap Q2).',
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
claim: 'only/belong-to',
|
|
340
|
+
title: /\b(only|belongs? to|exclusively)\b/,
|
|
341
|
+
proof: /\bsee all\b[^.]*\bcontain/,
|
|
342
|
+
need: 'all-match (`User see all … contains …`)',
|
|
343
|
+
hint: 'all displayed items must match the facet; "no items OUTSIDE" is UI-limited → verify via hybrid API/DB or mark @manual with reason.',
|
|
344
|
+
severity: 'warn',
|
|
345
|
+
},
|
|
346
|
+
{
|
|
347
|
+
claim: 'single/exactly-one',
|
|
348
|
+
title: /\b(a single|single|exactly one|just one|only one)\b/,
|
|
349
|
+
proof: /\bcount\b|\btable with\b|\bexactly\b|tohavecount/,
|
|
350
|
+
need: 'a count assertion (e.g. count is 1)',
|
|
351
|
+
hint: 'assert the quantity, not just presence.',
|
|
352
|
+
severity: 'fail',
|
|
353
|
+
},
|
|
354
|
+
{
|
|
355
|
+
claim: 'correct/matching/identity',
|
|
356
|
+
title: /\b(correct|matching|the same|same (product|name|price|value|item)|consistency|identity|lands? in|is the (product|item))\b/,
|
|
357
|
+
proof: /\bremember\b/,
|
|
358
|
+
need: 'capture + compare (`User remember … as {{v}}` then assert {{v}})',
|
|
359
|
+
hint: 'remember the value on the source, then assert it on the target (cross-screen → defer to a flow as @manual).',
|
|
360
|
+
severity: 'fail',
|
|
361
|
+
},
|
|
362
|
+
{
|
|
363
|
+
claim: 'changes/toggles',
|
|
364
|
+
title: /\b(changes?|toggles?|updates?|reflects?|reopens?)\b/,
|
|
365
|
+
proof: /\bremember\b|\bis hidden\b/,
|
|
366
|
+
need: 'a state delta (before ≠ after, or a visible→hidden contrast)',
|
|
367
|
+
hint: 'capture the before-state and assert the after-state differs, or assert the visible/hidden transition.',
|
|
368
|
+
severity: 'warn',
|
|
369
|
+
},
|
|
370
|
+
{
|
|
371
|
+
claim: 'hidden/rejected/not-complete',
|
|
372
|
+
title: /\b(hidden|closed|dismiss(es|ed)?|does not|doesn't|not complete|rejected|inert)\b/,
|
|
373
|
+
proof: /\bis hidden\b|\bare hidden\b|message is hidden|not complete|\bhidden\b/,
|
|
374
|
+
need: 'a negative / hidden assertion (`… is hidden`)',
|
|
375
|
+
hint: 'assert the absence/hidden state that the title claims, not just an unrelated visible element.',
|
|
376
|
+
severity: 'fail',
|
|
377
|
+
},
|
|
378
|
+
];
|
|
379
|
+
|
|
380
|
+
// ---------- Viewpoint taxonomy-lint (harness-roadmap §0.5 Q3) ----------
|
|
381
|
+
//
|
|
382
|
+
// A scenario's VP-code prefix (VP-CART-001 → CART) should match what the title is
|
|
383
|
+
// actually about, or the coverage matrix / per-viewpoint score is skewed. Conservative:
|
|
384
|
+
// flag ONLY when the title carries NO signal for its own category AND exactly one OTHER
|
|
385
|
+
// known category's signal — so a legitimately mixed title (e.g. a LIST card that also
|
|
386
|
+
// mentions an action) is never flagged. LOGIC/UI are flexible lenses → not linted.
|
|
387
|
+
|
|
388
|
+
export interface TaxonomyFinding {
|
|
389
|
+
name: string;
|
|
390
|
+
current: string; // category from the VP code
|
|
391
|
+
suggested: string; // category the title signals
|
|
392
|
+
signal: string; // the phrase that triggered the suggestion
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
export interface TaxonomyResult {
|
|
396
|
+
total: number;
|
|
397
|
+
checked: number; // scenarios with a known, lintable category
|
|
398
|
+
mislabeled: TaxonomyFinding[];
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
const TAXONOMY_KNOWN = new Set(['LIST', 'PRODUCT', 'CART', 'FILTER', 'NAV', 'VAL', 'SEC']);
|
|
402
|
+
|
|
403
|
+
const CATEGORY_SIGNALS: { cat: string; re: RegExp }[] = [
|
|
404
|
+
{ cat: 'FILTER', re: /\b(categor(y|ies)|sub-?categor\w*|expand\w*|collaps\w*|\bfacet\b|filter\w*|belongs? to)\b/ },
|
|
405
|
+
{ cat: 'CART', re: /\b(add(ed|ing)? to cart|added (dialog|modal|message|confirmation)|cart (page|table|line|item)|view cart|continue shopping|quantit)\b/ },
|
|
406
|
+
{ cat: 'PRODUCT', re: /\b(product detail|product-detail|view product|product page|same product|product name and price|product identity|detail product)\b/ },
|
|
407
|
+
{ cat: 'LIST', re: /\b(product (list|grid)|featured product|product card|each (featured )?card|every (featured )?card)\b/ },
|
|
408
|
+
{ cat: 'NAV', re: /\b(navigates?|returns the user|redirect\w*|logo returns|menu navigates)\b/ },
|
|
409
|
+
{ cat: 'VAL', re: /\b(valid email|invalid\b|without (a |an )?@|without a domain|empty (email|input|field)|validation|required field)\b/ },
|
|
410
|
+
{ cat: 'SEC', re: /\b(xss|sql|injection|payload|without authentication|unauthenticated|\binert\b|tamper\w*|malformed|url parameter|query param\w*|falls? back|gracefull?y|without a crash|not[- ]found|\b404\b|special character|abuse)\b/ },
|
|
411
|
+
];
|
|
412
|
+
|
|
413
|
+
export function taxonomyLint(scenarios: ScenarioInfo[]): TaxonomyResult {
|
|
414
|
+
const mislabeled: TaxonomyFinding[] = [];
|
|
415
|
+
let checked = 0;
|
|
416
|
+
for (const s of scenarios) {
|
|
417
|
+
const cat = s.category;
|
|
418
|
+
if (!cat || !TAXONOMY_KNOWN.has(cat)) continue;
|
|
419
|
+
checked++;
|
|
420
|
+
const title = s.name.toLowerCase();
|
|
421
|
+
const matched = [...new Set(CATEGORY_SIGNALS.filter((c) => c.re.test(title)).map((c) => c.cat))];
|
|
422
|
+
if (matched.includes(cat)) continue; // title supports its own category → fine
|
|
423
|
+
const others = matched.filter((c) => c !== cat);
|
|
424
|
+
if (others.length === 1) { // unambiguous mismatch
|
|
425
|
+
const sig = CATEGORY_SIGNALS.find((c) => c.cat === others[0])!;
|
|
426
|
+
const m = title.match(sig.re);
|
|
427
|
+
mislabeled.push({ name: s.name, current: cat, suggested: others[0], signal: m ? m[0] : others[0] });
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return { total: scenarios.length, checked, mislabeled };
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
export function claimProof(scenarios: ScenarioInfo[], focus = 'functional'): ClaimProofResult {
|
|
434
|
+
const measured = scenarios.filter((s) => !s.manual);
|
|
435
|
+
const unproven: ClaimUnproven[] = [];
|
|
436
|
+
let withClaims = 0;
|
|
437
|
+
|
|
438
|
+
for (const s of measured) {
|
|
439
|
+
const title = s.name.toLowerCase();
|
|
440
|
+
const rule = CLAIM_RULES.find((r) => r.title.test(title));
|
|
441
|
+
if (!rule) continue; // no claim → not measured
|
|
442
|
+
withClaims++;
|
|
443
|
+
if (!rule.proof.test(s.stepsText)) {
|
|
444
|
+
// proof shape entirely absent → fail at the rule's severity
|
|
445
|
+
unproven.push({ name: s.name, claim: rule.claim, need: rule.need, hint: rule.hint, severity: rule.severity });
|
|
446
|
+
} else if (rule.underproof && rule.underproof.test(s.stepsText)) {
|
|
447
|
+
// right shape exists but only as a partial proof → warn
|
|
448
|
+
unproven.push({
|
|
449
|
+
name: s.name, claim: rule.claim,
|
|
450
|
+
need: rule.underNeed ?? rule.need,
|
|
451
|
+
hint: rule.underHint ?? rule.hint,
|
|
452
|
+
severity: 'warn',
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
const proven = withClaims - unproven.length;
|
|
458
|
+
const ratio = withClaims ? proven / withClaims : 1;
|
|
459
|
+
const threshold = DEPTH_THRESHOLDS[focus] ?? DEPTH_THRESHOLDS.functional;
|
|
460
|
+
|
|
461
|
+
const hasFail = unproven.some((u) => u.severity === 'fail');
|
|
462
|
+
let verdict: DepthVerdict = 'pass';
|
|
463
|
+
if (ratio < threshold) {
|
|
464
|
+
verdict = WARN_ONLY_FOCUS.has(focus) || !hasFail ? 'warn' : 'fail';
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
return { total: measured.length, withClaims, proven, unproven, ratio, focus, threshold, verdict };
|
|
468
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QA baseline audit over a legacy testcase inventory (P-A). Reuses the harness
|
|
3
|
+
* duplicate sensor + the capability-plan reason classifier so the legacy view and
|
|
4
|
+
* the Gherkin view agree. Advisory — evaluates the *existing* manual suite and the
|
|
5
|
+
* automation/driver opportunity; it does not gate anything.
|
|
6
|
+
*/
|
|
7
|
+
import { LegacyInventory, LegacyTestcase } from './legacy-parser';
|
|
8
|
+
import { duplicateClusters } from '../harness/sensors';
|
|
9
|
+
import { ScenarioInfo } from '../harness/parse';
|
|
10
|
+
import { classifyReason, MANUAL_REASONS } from '../harness/capability-plan';
|
|
11
|
+
|
|
12
|
+
export interface BaselineReport {
|
|
13
|
+
total: number;
|
|
14
|
+
sheets: { name: string; type: string; rows: number }[];
|
|
15
|
+
byCategory: Record<string, number>;
|
|
16
|
+
byPriority: Record<string, number>;
|
|
17
|
+
byResult: Record<string, number>;
|
|
18
|
+
depthRatio: number; // % of TC whose Expected asserts a concrete value
|
|
19
|
+
deepCount: number;
|
|
20
|
+
duplicateClusters: number;
|
|
21
|
+
exactDuplicates: number;
|
|
22
|
+
reasons: { // capability-plan style distribution
|
|
23
|
+
ui: number; crossScreen: number; capabilityManual: number; keepManual: number;
|
|
24
|
+
byCode: Record<string, number>;
|
|
25
|
+
driverCandidates: { driver: string; count: number }[];
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** An Expected that asserts a concrete value (not just "displayed/visible"). */
|
|
30
|
+
function isDeep(tc: LegacyTestcase): boolean {
|
|
31
|
+
const e = (tc.expected || '').trim().toLowerCase();
|
|
32
|
+
if (!e) return false;
|
|
33
|
+
const shallowOnly = /^(.{0,40})(表示される|hiển thị|displayed|shown|is visible|表示)$/.test(e);
|
|
34
|
+
const hasValue = /(=|equals?|bằng|giá trị|\bvalue\b|"|「|\bmessage\b|エラー|error|\bcount\b|số lượng|\bformat\b|định dạng)/.test(e);
|
|
35
|
+
return hasValue || (!shallowOnly && e.length > 30);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Synthesize a ScenarioInfo-lite so we can reuse harness sensors on the inventory. */
|
|
39
|
+
export function legacyToScenarioInfo(tc: LegacyTestcase): ScenarioInfo {
|
|
40
|
+
const skeleton = (tc.steps || '')
|
|
41
|
+
.replace(/\{\{[^}]*\}\}/g, '{}').replace(/"[^"]*"/g, '""')
|
|
42
|
+
.replace(/\d+/g, 'N').replace(/\s+/g, ' ').trim().toLowerCase();
|
|
43
|
+
const hay = `${tc.category} ${tc.steps} ${tc.expected}`.toLowerCase();
|
|
44
|
+
return {
|
|
45
|
+
name: `${tc.category}: ${tc.id}`,
|
|
46
|
+
category: (tc.category || '').toUpperCase().replace(/[^A-Z0-9]+/g, '-').slice(0, 16),
|
|
47
|
+
priority: tc.priority,
|
|
48
|
+
manual: true,
|
|
49
|
+
thenCount: tc.expected ? 1 : 0,
|
|
50
|
+
hasDataAssertion: isDeep(tc),
|
|
51
|
+
shallow: !isDeep(tc),
|
|
52
|
+
stepSkeleton: skeleton,
|
|
53
|
+
haystack: hay,
|
|
54
|
+
} as ScenarioInfo;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function baselineAudit(inv: LegacyInventory): BaselineReport {
|
|
58
|
+
const tcs = inv.testcases;
|
|
59
|
+
const inc = (m: Record<string, number>, k: string) => { m[k] = (m[k] || 0) + 1; };
|
|
60
|
+
|
|
61
|
+
const byCategory: Record<string, number> = {};
|
|
62
|
+
const byPriority: Record<string, number> = {};
|
|
63
|
+
const byResult: Record<string, number> = {};
|
|
64
|
+
let deepCount = 0;
|
|
65
|
+
const byCode: Record<string, number> = {};
|
|
66
|
+
let ui = 0, crossScreen = 0, capabilityManual = 0, keepManual = 0;
|
|
67
|
+
const driverHits: Record<string, number> = {};
|
|
68
|
+
|
|
69
|
+
for (const tc of tcs) {
|
|
70
|
+
inc(byCategory, tc.category || '(uncategorized)');
|
|
71
|
+
inc(byPriority, tc.priority);
|
|
72
|
+
inc(byResult, (tc.result || '(not-run)').trim() || '(not-run)');
|
|
73
|
+
if (isDeep(tc)) deepCount++;
|
|
74
|
+
|
|
75
|
+
const code = classifyReason(`${tc.precondition || ''} ${tc.steps} ${tc.expected} ${tc.testData || ''}`);
|
|
76
|
+
if (!code) { ui++; continue; }
|
|
77
|
+
inc(byCode, code);
|
|
78
|
+
if (code === 'XS') { crossScreen++; continue; }
|
|
79
|
+
const def = MANUAL_REASONS[code];
|
|
80
|
+
if (def?.cls === 'keep') keepManual++;
|
|
81
|
+
else { capabilityManual++; for (const d of def?.drivers || []) driverHits[d] = (driverHits[d] || 0) + 1; }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const dup = duplicateClusters(tcs.map(legacyToScenarioInfo));
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
total: tcs.length,
|
|
88
|
+
sheets: inv.sheets,
|
|
89
|
+
byCategory, byPriority, byResult,
|
|
90
|
+
depthRatio: tcs.length ? deepCount / tcs.length : 0,
|
|
91
|
+
deepCount,
|
|
92
|
+
duplicateClusters: dup.clusters.length,
|
|
93
|
+
exactDuplicates: dup.exactDuplicateCount,
|
|
94
|
+
reasons: {
|
|
95
|
+
ui, crossScreen, capabilityManual, keepManual, byCode,
|
|
96
|
+
driverCandidates: Object.entries(driverHits).map(([driver, count]) => ({ driver, count }))
|
|
97
|
+
.sort((a, b) => b.count - a.count),
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
}
|