@sun-asterisk/sungen 3.2.21-beta.1 → 3.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +0 -8
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +0 -7
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/trace.d.ts.map +1 -1
- package/dist/cli/commands/trace.js +0 -9
- package/dist/cli/commands/trace.js.map +1 -1
- package/dist/exporters/matrix/export.d.ts.map +1 -1
- package/dist/exporters/matrix/export.js +0 -11
- package/dist/exporters/matrix/export.js.map +1 -1
- package/dist/exporters/matrix/render-xlsx.d.ts.map +1 -1
- package/dist/exporters/matrix/render-xlsx.js +0 -15
- package/dist/exporters/matrix/render-xlsx.js.map +1 -1
- package/dist/exporters/matrix/types.d.ts +0 -2
- package/dist/exporters/matrix/types.d.ts.map +1 -1
- package/dist/exporters/matrix/types.js.map +1 -1
- package/dist/exporters/playwright-report-parser.d.ts.map +1 -1
- package/dist/exporters/playwright-report-parser.js +0 -1
- package/dist/exporters/playwright-report-parser.js.map +1 -1
- package/dist/exporters/types.d.ts +0 -2
- package/dist/exporters/types.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.d.ts +1 -6
- package/dist/generators/test-generator/diagnostics.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.js +0 -5
- package/dist/generators/test-generator/diagnostics.js.map +1 -1
- package/dist/generators/test-generator/patterns/index.d.ts.map +1 -1
- package/dist/generators/test-generator/patterns/index.js +19 -27
- package/dist/generators/test-generator/patterns/index.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.d.ts.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +0 -22
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/harness/audit.d.ts +0 -2
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +9 -79
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-plan.d.ts +0 -3
- package/dist/harness/flow-plan.d.ts.map +1 -1
- package/dist/harness/flow-plan.js +2 -6
- package/dist/harness/flow-plan.js.map +1 -1
- package/dist/harness/parse.d.ts +0 -5
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +1 -29
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +1 -13
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/spec-coverage.d.ts.map +1 -1
- package/dist/harness/spec-coverage.js +5 -29
- package/dist/harness/spec-coverage.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/add-flow.md +3 -41
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +0 -10
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +16 -35
- package/dist/orchestrator/templates/qa-context.md +1 -14
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +0 -8
- package/src/cli/commands/delivery.ts +0 -6
- package/src/cli/commands/trace.ts +0 -9
- package/src/exporters/matrix/export.ts +0 -11
- package/src/exporters/matrix/render-xlsx.ts +0 -15
- package/src/exporters/matrix/types.ts +0 -2
- package/src/exporters/playwright-report-parser.ts +0 -2
- package/src/exporters/types.ts +0 -2
- package/src/generators/test-generator/diagnostics.ts +1 -6
- package/src/generators/test-generator/patterns/index.ts +24 -30
- package/src/generators/test-generator/step-mapper.ts +0 -22
- package/src/harness/audit.ts +10 -82
- package/src/harness/flow-plan.ts +3 -10
- package/src/harness/parse.ts +1 -31
- package/src/harness/sensors.ts +1 -13
- package/src/harness/spec-coverage.ts +4 -26
- package/src/orchestrator/templates/ai-src/commands/add-flow.md +3 -41
- package/src/orchestrator/templates/ai-src/commands/create-test.md +0 -10
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +16 -35
- package/src/orchestrator/templates/qa-context.md +1 -14
- package/dist/harness/flow-contract.d.ts +0 -71
- package/dist/harness/flow-contract.d.ts.map +0 -1
- package/dist/harness/flow-contract.js +0 -235
- package/dist/harness/flow-contract.js.map +0 -1
- package/dist/harness/perf.d.ts +0 -40
- package/dist/harness/perf.d.ts.map +0 -1
- package/dist/harness/perf.js +0 -136
- package/dist/harness/perf.js.map +0 -1
- package/src/harness/flow-contract.ts +0 -229
- package/src/harness/perf.ts +0 -112
|
@@ -86,52 +86,15 @@ qa/flows/${input:flow}/
|
|
|
86
86
|
└── ui/ # Screenshots, mockups
|
|
87
87
|
```
|
|
88
88
|
|
|
89
|
-
### 1a.
|
|
89
|
+
### 1a. Identify the screens in the flow
|
|
90
90
|
|
|
91
|
-
|
|
92
|
-
observable, valuable outcome. Before asking for screens, walk this checklist with the user —
|
|
93
|
-
if 1, 3 or 8 fails, propose SPLITTING into separate flows:
|
|
94
|
-
|
|
95
|
-
1. Exactly **one business goal**? (cart correctness + category filtering = two flows)
|
|
96
|
-
2. A clear **trigger** and precondition?
|
|
97
|
-
3. **One observable final outcome**? (a final assertion you can write in one sentence)
|
|
98
|
-
4. Is that outcome **valuable to the actor**? (an order placed, a password reset — not "a page rendered")
|
|
99
|
-
5. Is **every step necessary** for that outcome?
|
|
100
|
-
6. Are all steps at the **same business abstraction**?
|
|
101
|
-
7. Are optional/error branches **phases of this goal** (ER/EH), not new goals?
|
|
102
|
-
8. Does **no segment** form an independently valuable flow on its own?
|
|
103
|
-
9. Can you write **a single clear final assertion**?
|
|
104
|
-
10. Can you name it "**Verb + outcome**"? (`place-order`, `reset-password` — not `cart-and-filter`)
|
|
105
|
-
|
|
106
|
-
Then ask: "Which screens does this flow visit, in order? (e.g., login → dashboard → award-form → confirmation)"
|
|
91
|
+
Ask the user: "Which screens does this flow visit, in order? (e.g., login → dashboard → award-form → confirmation)"
|
|
107
92
|
|
|
108
93
|
Record the screen list — you will need it for:
|
|
109
94
|
- Filling `spec.md` (Step 3)
|
|
110
95
|
- Suggesting `[Screen:Element]` namespace prefixes
|
|
111
96
|
- Capturing visuals per screen (Step 2)
|
|
112
97
|
|
|
113
|
-
### 1b. Author the Flow Contract (`requirements/flow-contract.yaml`)
|
|
114
|
-
|
|
115
|
-
Write the answers down as the flow's contract — `sungen audit` scores the flow **against it**
|
|
116
|
-
(the `flowCoverage` axis: HP/ER/EH journey phases; `FLOW-OUTCOME-UNPROVEN` when no automated
|
|
117
|
-
scenario asserts data on the outcome screen; `FLOW-SCOPE-CREEP` when scenarios never touch it):
|
|
118
|
-
|
|
119
|
-
```yaml
|
|
120
|
-
goal: "Place an order for a product added from home" # Verb + outcome
|
|
121
|
-
actor: user
|
|
122
|
-
trigger: "Add a product to the cart from the home featured list"
|
|
123
|
-
precondition: "A registered account; an empty cart"
|
|
124
|
-
outcome:
|
|
125
|
-
screen: checkout # the [Screen:...] namespace carrying the final proof
|
|
126
|
-
assertion: "The confirmation shows the order number and the paid total"
|
|
127
|
-
value: "The customer has paid; the shop has a new order"
|
|
128
|
-
phases: [HP, ER, EH] # journey phases (default); add UI only if the flow owns UI states
|
|
129
|
-
stateful: cart # the mutated collection, if any — enables regression-depth dims
|
|
130
|
-
```
|
|
131
|
-
|
|
132
|
-
**A filled contract is an INPUT to generation — never an output.** Like `test-viewpoint.md`,
|
|
133
|
-
generation must not rewrite it to match what was generated; disagree → propose the diff and ask.
|
|
134
|
-
|
|
135
98
|
### 2. Capture visual source
|
|
136
99
|
|
|
137
100
|
**Mobile path** (`platform: mobile`):
|
|
@@ -224,8 +187,7 @@ If user picks `/sungen:create-test`, **you MUST use the Skill tool** to invoke i
|
|
|
224
187
|
- Test data namespaced by phase: `login.email`, `submission.nominee`
|
|
225
188
|
- `@flow` tag required at feature level
|
|
226
189
|
- `Background:` should only contain the starting navigation — the URL path (web) or the `--reach` nav recipe (mobile)
|
|
227
|
-
- Each scenario = one phase of the journey
|
|
228
|
-
- One flow = ONE business goal with ONE observable outcome (`requirements/flow-contract.yaml`) — a segment with its own value is its own flow
|
|
190
|
+
- Each scenario = one phase of the journey
|
|
229
191
|
{{#cap parallel-subagents}}
|
|
230
192
|
- Mobile flows are tagged `@platform:mobile` and run via `/sungen:run-test <flow>` (WebdriverIO, not Playwright)
|
|
231
193
|
{{/cap}}
|
|
@@ -6,16 +6,6 @@ order: 20
|
|
|
6
6
|
claude-tools: "Read, Grep, Bash, Glob, Write, AskUserQuestion, Skill, mcp__playwright__browser_navigate, mcp__playwright__browser_snapshot, mcp__playwright__browser_take_screenshot"
|
|
7
7
|
copilot-tools: "[vscode, execute, read, agent, edit, search, web, browser, todo, 'playwright/*']"
|
|
8
8
|
codex-trigger: "Run when the user asks to CREATE, generate, write, or author test cases / a .feature file for a screen or flow. Step 2 (after add-screen/add-flow, before run-test). Do NOT use for executing, running, or compiling existing tests."
|
|
9
|
-
---
|
|
10
|
-
## ⛔ HARD RULE — the run's LAST action is the next-step hand-back
|
|
11
|
-
|
|
12
|
-
A create-test run is NOT finished when the files are written or the audit prints. The final
|
|
13
|
-
action of EVERY run — success, partial, or aborted — is the next-step hand-back
|
|
14
|
-
({{#cap parallel-subagents}}an `AskUserQuestion` offering the next actions{{/cap}}{{^cap parallel-subagents}}a numbered list of next-action choices{{/cap}};
|
|
15
|
-
see "Finish — always hand the next step back" at the end of this file). Ending with a prose
|
|
16
|
-
summary and no choices is a broken run: the operator is left guessing. This holds no matter
|
|
17
|
-
how long the generation/repair loop ran.
|
|
18
|
-
|
|
19
9
|
---
|
|
20
10
|
{{#cap parallel-subagents}}
|
|
21
11
|
## ⛔ HARD RULE — No Figma MCP when PAT data exists
|
|
@@ -605,40 +605,22 @@ error:
|
|
|
605
605
|
|
|
606
606
|
> **Auto-detect**: if path is `qa/flows/<name>/` → use this section. Skip Steps 1–4 above.
|
|
607
607
|
|
|
608
|
-
**Read `requirements/flow-contract.yaml` FIRST — it is the flow's boundary and the yardstick
|
|
609
|
-
`sungen audit` scores the flow against** (`flowCoverage` axis = journey phases HP/ER/EH automated;
|
|
610
|
-
`FLOW-OUTCOME-UNPROVEN`; `FLOW-SCOPE-CREEP`). No contract yet → author it with the user via the
|
|
611
|
-
boundary checklist in `add-flow` (one business goal · clear trigger · ONE observable outcome
|
|
612
|
-
valuable to the actor · name = "Verb + outcome"), THEN generate. **A filled contract is an INPUT —
|
|
613
|
-
never rewrite it to match your output** (same rule as `test-viewpoint.md`).
|
|
614
|
-
|
|
615
608
|
| Aspect | Screen | Flow |
|
|
616
609
|
|---|---|---|
|
|
617
|
-
| Section focus | UI patterns per section | Journey phases
|
|
610
|
+
| Section focus | UI patterns per section | Journey phases across screens |
|
|
618
611
|
| Selector format | `[Element]` | `[Screen:Element]` (namespaced) |
|
|
619
612
|
| Test data keys | `{{variable}}` | `{{phase.variable}}` |
|
|
620
613
|
| Feature tag | `@auto` / `@smoke` etc. | `@flow` (required) |
|
|
621
|
-
|
|
|
614
|
+
| Viewpoints | VP-UI/VAL/LOGIC/SEC per section | VP-LOGIC (transitions), VP-SEC (auth persistence), VP-VAL (cross-screen data) |
|
|
622
615
|
|
|
623
|
-
**Scenarios to generate
|
|
616
|
+
**Scenarios to generate:**
|
|
624
617
|
|
|
625
|
-
|
|
|
626
|
-
|
|
627
|
-
|
|
|
628
|
-
|
|
|
629
|
-
|
|
|
630
|
-
| Cross-screen
|
|
631
|
-
| Stateful regression (when `stateful:` declared) | Count/quantity proof · teardown (remove → empty) · multi-source add. | missing dims cap `businessDepth` (`FLOW-DEPTH`) |
|
|
632
|
-
|
|
633
|
-
**Boundary discipline while generating:** every scenario must serve the contract's goal. A scenario
|
|
634
|
-
that never touches `outcome.screen` and is not a guard (`EH`) or error-recovery (`ER`) belongs in a
|
|
635
|
-
DIFFERENT flow — propose the split instead of writing it here (`FLOW-SCOPE-CREEP` will flag it).
|
|
636
|
-
Auth persistence across transitions is part of `EH` unless the project declares it its own phase.
|
|
637
|
-
|
|
638
|
-
**Manual in flows**: always `@manual:Mx` with the reason code — bare `@manual` is flagged
|
|
639
|
-
(`MANUAL-CODE-MISSING`) because the capability planner cannot route it. Typical flow deferrals:
|
|
640
|
-
inbox/mail oracle → `M5`, network-request count → `M3`, context expiry control → `M7`. A
|
|
641
|
-
cross-screen scenario inside the flow's own goal is NOT manual — automate it here.
|
|
618
|
+
| Category | What to test |
|
|
619
|
+
|---|---|
|
|
620
|
+
| Happy path | Complete flow end-to-end with valid data |
|
|
621
|
+
| Auth persistence | Auth state maintained across screen transitions |
|
|
622
|
+
| Error recovery | Invalid input mid-flow → fix → continue |
|
|
623
|
+
| Cross-screen data | Data entered on screen A visible on screen B |
|
|
642
624
|
|
|
643
625
|
```gherkin
|
|
644
626
|
@flow @auth:user
|
|
@@ -648,20 +630,19 @@ Feature: Award Submission Flow
|
|
|
648
630
|
Given User is on [Login] page
|
|
649
631
|
|
|
650
632
|
@high
|
|
651
|
-
Scenario:
|
|
633
|
+
Scenario: User logs in successfully
|
|
652
634
|
When User fill [Login:Email] field with {{login.email}}
|
|
653
635
|
And User fill [Login:Password] field with {{login.password}}
|
|
654
636
|
And User click [Login:Submit] button
|
|
655
637
|
Then User see [Dashboard] page
|
|
656
|
-
When User click [Dashboard:Awards] link
|
|
657
|
-
And User fill [Awards:Nominee] field with {{submission.nominee}}
|
|
658
|
-
And User click [Awards:Submit] button
|
|
659
|
-
Then User see [Awards:Success Message] text with {{success_message}}
|
|
660
638
|
|
|
661
639
|
@high
|
|
662
|
-
Scenario:
|
|
663
|
-
When User
|
|
664
|
-
Then User see [
|
|
640
|
+
Scenario: User submits nomination
|
|
641
|
+
When User click [Dashboard:Awards] link
|
|
642
|
+
Then User see [Awards] page
|
|
643
|
+
When User fill [Awards:Nominee] field with {{submission.nominee}}
|
|
644
|
+
And User click [Awards:Submit] button
|
|
645
|
+
Then User see {{success_message}} message
|
|
665
646
|
```
|
|
666
647
|
|
|
667
648
|
```yaml
|
|
@@ -45,21 +45,8 @@ Example:
|
|
|
45
45
|
|
|
46
46
|
## Testing Strategy
|
|
47
47
|
|
|
48
|
-
Machine-readable intent — `sungen audit` reads these keys (Intent Profile). Values here are
|
|
49
|
-
live even when the surrounding text changes; an invalid value silently falls back to the default.
|
|
50
|
-
|
|
51
|
-
focus: functional
|
|
52
|
-
<!-- focus: functional | e-commerce | security | smoke — drives the audit's depth threshold -->
|
|
53
|
-
|
|
54
|
-
risk_tier: normal
|
|
55
|
-
<!-- risk_tier: high | normal | low -->
|
|
56
|
-
|
|
57
|
-
To silence driver suggestions in audit findings, add a line: capability_suggestions with value off.
|
|
58
|
-
|
|
59
48
|
**Focus areas** — what to cover thoroughly:
|
|
60
|
-
<!--
|
|
61
|
-
Response-time budgets are NOT a focus value — declare them in qa/perf.yaml
|
|
62
|
-
(percentile + scenario_ms budgets; reported by `sungen delivery`). -->
|
|
49
|
+
<!-- List from: functional, security, ui, accessibility, performance -->
|
|
63
50
|
<!-- Example: functional, security -->
|
|
64
51
|
|
|
65
52
|
**Mandatory coverage:**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sun-asterisk/sungen",
|
|
3
|
-
"version": "3.2.21
|
|
3
|
+
"version": "3.2.21",
|
|
4
4
|
"description": "Deterministic E2E Test Compiler - Gherkin + Selectors → Playwright tests",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
@@ -39,8 +39,8 @@
|
|
|
39
39
|
"@babel/types": "^7.28.5",
|
|
40
40
|
"@cucumber/gherkin": "^37.0.0",
|
|
41
41
|
"@cucumber/messages": "^31.0.0",
|
|
42
|
-
"@sungen/driver-data-factory": "3.2.21
|
|
43
|
-
"@sungen/driver-ui": "3.2.21
|
|
42
|
+
"@sungen/driver-data-factory": "3.2.21",
|
|
43
|
+
"@sungen/driver-ui": "3.2.21",
|
|
44
44
|
"chalk": "^5.6.2",
|
|
45
45
|
"commander": "^14.0.2",
|
|
46
46
|
"dotenv": "^17.2.3",
|
|
@@ -40,14 +40,6 @@ function render(r: AuditReport): void {
|
|
|
40
40
|
if (r.gate.gaps.length) for (const g of r.gate.gaps) L(` ✗ ${g.status === 'shallow' ? 'SHALLOW' : 'MISSING'}: ${g.theme}`);
|
|
41
41
|
else L(' ✓ all critical themes covered (with data assertions)');
|
|
42
42
|
L('');
|
|
43
|
-
if (r.flowQuality?.hasContract) {
|
|
44
|
-
const fq = r.flowQuality;
|
|
45
|
-
L(` ①F Flow contract — goal: ${fq.contract!.goal}`);
|
|
46
|
-
L(` outcome [${fq.contract!.outcome.screen}]: ${fq.outcomeProven ? '✓ proven (automated data assertion)' : fq.outcomeManualOnly ? '⚠ manual-only' : '✗ UNPROVEN'}`);
|
|
47
|
-
L(` phases: ${fq.phases.map((p) => `${p.phase}=${p.covered ? (p.automated ? '✓' : 'manual') : '✗'}`).join(' ')} → flowCoverage ${(fq.phaseRatio * 100).toFixed(0)}%`);
|
|
48
|
-
L(` handoffs asserted: ${fq.handoffs.asserted}/${fq.handoffs.total}${fq.offGoal.length ? ` ⚠ off-goal scenarios: ${fq.offGoal.length} (${fq.offGoalCategories.join(', ')})` : ''}`);
|
|
49
|
-
L('');
|
|
50
|
-
}
|
|
51
43
|
L(` ② Assertion depth — ${r.depth.businessCriticalShallow}/${r.depth.businessCriticalTotal} business-critical scenarios are shallow (visibility/nav only)`);
|
|
52
44
|
for (const s of r.depth.shallowBusinessCritical.slice(0, 8)) L(` ⚠ ${s.category}: ${s.name}`);
|
|
53
45
|
if (r.depth.shallowBusinessCritical.length > 8) L(` … +${r.depth.shallowBusinessCritical.length - 8} more`);
|
|
@@ -690,12 +690,6 @@ async function exportMatrixTarget(
|
|
|
690
690
|
}
|
|
691
691
|
|
|
692
692
|
log(`${COLOR.bold}${paths.label}${COLOR.reset} — ${model.manifest.itemCount} delivery item(s), ${model.manifest.variantCount} coverage variant(s), state: ${model.manifest.approvalState}`);
|
|
693
|
-
if (model.perf) {
|
|
694
|
-
const p = model.perf;
|
|
695
|
-
const tone = p.pass ? COLOR.green : COLOR.yellow;
|
|
696
|
-
log(` ${tone}PERF ${p.pass ? 'PASS' : 'OVER BUDGET'}${COLOR.reset}: p${p.percentile} ${p.metric.replace('_ms', '')} ${p.measuredMs}ms ${p.pass ? '≤' : '>'} budget ${p.budgetMs}ms (${p.samples} run${p.samples === 1 ? '' : 's'})`);
|
|
697
|
-
for (const s of p.slowest) log(` ${COLOR.yellow}slowest:${COLOR.reset} ${s.ms}ms — ${s.title.slice(0, 90)}`);
|
|
698
|
-
}
|
|
699
693
|
printFindings(model.findings);
|
|
700
694
|
|
|
701
695
|
if (opts.check) {
|
|
@@ -55,15 +55,6 @@ export function registerTraceCommand(program: Command): void {
|
|
|
55
55
|
for (const line of r.mermaid.split('\n')) L(' ' + line);
|
|
56
56
|
L(' ```');
|
|
57
57
|
L('');
|
|
58
|
-
// ⑥ Deterministic next-step hand-back. The AI commands are INSTRUCTED to close
|
|
59
|
-
// every run by offering next actions, but a long generation run can end without
|
|
60
|
-
// it — leaving the operator guessing. Printing the choices here makes the
|
|
61
|
-
// hand-back survive the AI forgetting: trace is the converge step's last call.
|
|
62
|
-
L(' ⑥ NEXT STEPS');
|
|
63
|
-
L(` → /sungen:run-test ${name} (generate selectors, compile, run, auto-fix)`);
|
|
64
|
-
L(` → /sungen:create-test ${name} (extend coverage / next tier)`);
|
|
65
|
-
L(` → /sungen:delivery ${name} (export the Test Case & Coverage Matrix)`);
|
|
66
|
-
L('');
|
|
67
58
|
} catch (error) {
|
|
68
59
|
console.error('Error:', error instanceof Error ? error.message : error);
|
|
69
60
|
process.exit(1);
|
|
@@ -17,7 +17,6 @@ import { mergeFeatureAndSpec } from '../scenario-merger';
|
|
|
17
17
|
import { getPackageVersion } from '../package-info';
|
|
18
18
|
import { writeCsv } from '../csv-exporter';
|
|
19
19
|
import { writeXlsx } from '../xlsx-exporter';
|
|
20
|
-
import { loadPerfConfig, perfVerdict } from '../../harness/perf';
|
|
21
20
|
import { loadDeliveryMap, writeDeliveryMap } from './map-loader';
|
|
22
21
|
import { mapContentFingerprint } from './fingerprint';
|
|
23
22
|
import { buildMatrix, deriveVariants } from './build';
|
|
@@ -90,16 +89,6 @@ export function loadMatrixModel(paths: MatrixTargetPaths): MatrixLoadResult {
|
|
|
90
89
|
specText: readSpecText(paths),
|
|
91
90
|
apiCatalog: catalogFor(paths),
|
|
92
91
|
});
|
|
93
|
-
// Perf verdict (#569): budgets are config (qa/perf.yaml), samples are the run's
|
|
94
|
-
// per-test durations. Advisory — reported here because delivery is where runs
|
|
95
|
-
// are read; the audit's score stays a pure function of the design artifacts.
|
|
96
|
-
const perfConfig = loadPerfConfig(paths.cwd);
|
|
97
|
-
if (perfConfig && results) {
|
|
98
|
-
const samples = [...results.values()]
|
|
99
|
-
.filter((r) => (r.status === 'passed' || r.status === 'failed' || r.status === 'timedOut') && typeof r.duration === 'number')
|
|
100
|
-
.map((r) => ({ title: r.testTitle, ms: r.duration as number }));
|
|
101
|
-
model.perf = perfVerdict(perfConfig, paths.unit, samples);
|
|
102
|
-
}
|
|
103
92
|
return { model, map, mapErrors: [] };
|
|
104
93
|
}
|
|
105
94
|
|
|
@@ -315,21 +315,6 @@ function addCoverageSheet(wb: ExcelJS.Workbook, model: MatrixModel, sungenVersio
|
|
|
315
315
|
|
|
316
316
|
let rowIdx = 7;
|
|
317
317
|
|
|
318
|
-
// --- Performance budget (#569) — advisory: config (qa/perf.yaml) vs measured runs.
|
|
319
|
-
if (model.perf) {
|
|
320
|
-
const p = model.perf;
|
|
321
|
-
dataCell(ws.getRow(rowIdx).getCell(2), 'Performance budget', { bold: true });
|
|
322
|
-
rowIdx++;
|
|
323
|
-
const r = ws.getRow(rowIdx);
|
|
324
|
-
dataCell(r.getCell(2), `p${p.percentile} ${p.metric.replace('_ms', ' time')}`);
|
|
325
|
-
dataCell(r.getCell(3), p.pass ? 'PASS' : 'OVER BUDGET', { bold: true });
|
|
326
|
-
dataCell(r.getCell(4), `measured ${p.measuredMs}ms`);
|
|
327
|
-
dataCell(r.getCell(5), `budget ${p.budgetMs}ms`);
|
|
328
|
-
dataCell(r.getCell(6), `${p.samples} runs`);
|
|
329
|
-
if (!p.pass && p.slowest.length) dataCell(r.getCell(8), `slowest: ${p.slowest.map((x) => `${x.ms}ms ${x.title.slice(0, 40)}`).join(' · ')}`);
|
|
330
|
-
rowIdx += 2;
|
|
331
|
-
}
|
|
332
|
-
|
|
333
318
|
// --- Requirement coverage (review §6): every requirement id gets ONE explicit status.
|
|
334
319
|
if (model.requirements.length > 0) {
|
|
335
320
|
dataCell(ws.getRow(rowIdx).getCell(2), 'Requirement coverage — every id has an explicit status', { bold: true });
|
|
@@ -225,8 +225,6 @@ export interface MatrixModel {
|
|
|
225
225
|
requirements: RequirementCoverage[];
|
|
226
226
|
findings: MatrixFinding[];
|
|
227
227
|
manifest: MatrixManifest;
|
|
228
|
-
/** Perf verdict (#569) — present when qa/perf.yaml sets a budget AND the run produced durations. */
|
|
229
|
-
perf?: import('../../harness/perf').PerfVerdict | null;
|
|
230
228
|
}
|
|
231
229
|
|
|
232
230
|
/** Complexity warning threshold (rules draft Gate I; raised for compact grouping) — echoed in the manifest. */
|
|
@@ -54,7 +54,6 @@ interface PlaywrightSpec {
|
|
|
54
54
|
projectId?: string;
|
|
55
55
|
results?: Array<{
|
|
56
56
|
status?: string; // 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'
|
|
57
|
-
duration?: number; // ms — Playwright emits this on every result (#569 perf budgets)
|
|
58
57
|
startTime?: string;
|
|
59
58
|
error?: { message?: string; stack?: string };
|
|
60
59
|
attachments?: Array<{ name?: string; path?: string; contentType?: string }>;
|
|
@@ -121,7 +120,6 @@ export function loadPlaywrightReport(reportPath: string): Map<string, Playwright
|
|
|
121
120
|
result.set(stripped, {
|
|
122
121
|
testTitle: stripped,
|
|
123
122
|
status,
|
|
124
|
-
duration: typeof res?.duration === 'number' ? res.duration : undefined,
|
|
125
123
|
startTime: res?.startTime,
|
|
126
124
|
error: errorMsg,
|
|
127
125
|
tracePath: trace,
|
package/src/exporters/types.ts
CHANGED
|
@@ -121,8 +121,6 @@ export interface SpecFileData {
|
|
|
121
121
|
export interface PlaywrightResult {
|
|
122
122
|
testTitle: string;
|
|
123
123
|
status: 'passed' | 'failed' | 'skipped' | 'timedOut' | 'interrupted' | 'unknown';
|
|
124
|
-
/** Wall-clock run time in ms (from Playwright's results[].duration) — perf budgets (#569). */
|
|
125
|
-
duration?: number;
|
|
126
124
|
startTime?: string;
|
|
127
125
|
error?: string;
|
|
128
126
|
tracePath?: string;
|
|
@@ -35,17 +35,12 @@
|
|
|
35
35
|
* nothing for the query half to assert, so the step checks the PATH only. Not an
|
|
36
36
|
* error (the value may legitimately vary per environment), but the step asserts
|
|
37
37
|
* strictly less than it appears to
|
|
38
|
-
* SG-W015 — `see [X] card/item with {{n}}` where the value is a bare number: this compiles
|
|
39
|
-
* to a TEXT assertion (toHaveText('3')), NOT a count — an author reaching for
|
|
40
|
-
* "3 cards are shown" gets an assertion that can only pass if the card's text
|
|
41
|
-
* IS the digit. Count semantics exist on row/table ("see [T] table with {{n}}",
|
|
42
|
-
* "see [Ref] row with {{n}}") and lists ("should have count") only
|
|
43
38
|
*
|
|
44
39
|
* Diagnostics never block generation by themselves; `sungen generate --strict`
|
|
45
40
|
* turns any collected diagnostic into a non-zero exit.
|
|
46
41
|
*/
|
|
47
42
|
|
|
48
|
-
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007' | 'SG-W008' | 'SG-W009' | 'SG-W010' | 'SG-W011' | 'SG-W012' | 'SG-W013' | 'SG-W014'
|
|
43
|
+
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007' | 'SG-W008' | 'SG-W009' | 'SG-W010' | 'SG-W011' | 'SG-W012' | 'SG-W013' | 'SG-W014';
|
|
49
44
|
|
|
50
45
|
export interface Diagnostic {
|
|
51
46
|
code: DiagnosticCode;
|
|
@@ -73,40 +73,34 @@ export class PatternRegistry {
|
|
|
73
73
|
* Prefers resolver (returns template data) over generator (renders directly).
|
|
74
74
|
*/
|
|
75
75
|
generateStep(step: ParsedStep, context: PatternContext): MappedStep | null {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
const matchStep: ParsedStep = { ...step, text: maskRefsForMatching(step.text) };
|
|
81
|
-
for (const pattern of this.patterns) {
|
|
82
|
-
if (!this.matchesPattern(matchStep, pattern.matcher)) continue;
|
|
83
|
-
|
|
84
|
-
// Prefer resolver (framework-agnostic) over generator (legacy)
|
|
85
|
-
if (pattern.resolver) {
|
|
86
|
-
const resolved = pattern.resolver(step, context);
|
|
87
|
-
if (!resolved) continue; // resolver declined — fall through
|
|
88
|
-
|
|
89
|
-
// Auto-inject parent scoping if step has parentRef
|
|
90
|
-
// Skip for table-* patterns — they resolve the table name internally from step text
|
|
91
|
-
if (step.parentRef && step.parentType && !pattern.name.startsWith('table-')) {
|
|
92
|
-
resolved.data.parentLocator = PatternRegistry.resolveParentLocator(
|
|
93
|
-
step.parentRef, step.parentType, context
|
|
94
|
-
);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
const code = context.templateEngine.renderStep(resolved.templateName, resolved.data);
|
|
98
|
-
return {
|
|
99
|
-
code,
|
|
100
|
-
comment: resolved.comment,
|
|
101
|
-
};
|
|
102
|
-
}
|
|
76
|
+
const pattern = this.findPattern(step);
|
|
77
|
+
if (!pattern) {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
103
80
|
|
|
104
|
-
|
|
105
|
-
|
|
81
|
+
// Prefer resolver (framework-agnostic) over generator (legacy)
|
|
82
|
+
if (pattern.resolver) {
|
|
83
|
+
const resolved = pattern.resolver(step, context);
|
|
84
|
+
|
|
85
|
+
// Auto-inject parent scoping if step has parentRef
|
|
86
|
+
// Skip for table-* patterns — they resolve the table name internally from step text
|
|
87
|
+
if (step.parentRef && step.parentType && !pattern.name.startsWith('table-')) {
|
|
88
|
+
resolved.data.parentLocator = PatternRegistry.resolveParentLocator(
|
|
89
|
+
step.parentRef, step.parentType, context
|
|
90
|
+
);
|
|
106
91
|
}
|
|
107
92
|
|
|
108
|
-
|
|
93
|
+
const code = context.templateEngine.renderStep(resolved.templateName, resolved.data);
|
|
94
|
+
return {
|
|
95
|
+
code,
|
|
96
|
+
comment: resolved.comment,
|
|
97
|
+
};
|
|
109
98
|
}
|
|
99
|
+
|
|
100
|
+
if (pattern.generator) {
|
|
101
|
+
return pattern.generator(step, context);
|
|
102
|
+
}
|
|
103
|
+
|
|
110
104
|
return null;
|
|
111
105
|
}
|
|
112
106
|
|
|
@@ -175,28 +175,6 @@ export class StepMapper {
|
|
|
175
175
|
this.lastPrimaryKeyword = keyword;
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
-
// SG-W015 — `see [X] card with {{n}}` where n is a bare number compiles to a TEXT
|
|
179
|
-
// assertion (toHaveText('3')), not a count. An author reaching for "3 cards are shown"
|
|
180
|
-
// gets an assertion that can only pass if the card's TEXT is the digit — a wrong test
|
|
181
|
-
// with no error. Count semantics live on row/table ("see [T] table with {{n}}",
|
|
182
|
-
// "see [Ref] row with {{n}}") and lists ("should have count"); on the container types
|
|
183
|
-
// below, a bare-number text expectation is almost never what the author meant. A text
|
|
184
|
-
// element asserting a numeric readout ("[Row Quantity] text with 4") stays silent.
|
|
185
|
-
const COUNT_MISREAD_TYPES = new Set(['card', 'item', 'items', 'cards', 'option', 'options']);
|
|
186
|
-
if (this.diagnostics && step.dataRef && step.selectorRef &&
|
|
187
|
-
COUNT_MISREAD_TYPES.has((step.elementType || '').toLowerCase()) &&
|
|
188
|
-
/\bsees?\s+\[/.test(step.text) && /\bwith\b/.test(step.text)) {
|
|
189
|
-
// peekBaseValue reads the literal from test-data even in runtime-data mode,
|
|
190
|
-
// where resolveData returns an opaque marker (never numeric-looking).
|
|
191
|
-
let resolvedValue: string | undefined;
|
|
192
|
-
try { resolvedValue = this.dataResolver.peekBaseValue(step.dataRef, this.featureName); } catch { /* unresolved at compile time → cannot judge */ }
|
|
193
|
-
if (resolvedValue !== undefined && /^\d+$/.test(resolvedValue.trim())) {
|
|
194
|
-
this.diagnostics.add('SG-W015',
|
|
195
|
-
`\`see [${step.selectorRef}] ${step.elementType} with {{${step.dataRef}}}\` compiles to a TEXT assertion toHaveText('${resolvedValue.trim()}'), NOT a count of ${step.elementType}s. For a count use \`see [Ref] row with {{n}}\` / \`see [T] table with {{n}}\` (or a list "should have count"); if '${resolvedValue.trim()}' really is the ${step.elementType}'s text, assert it on a text/label element instead`,
|
|
196
|
-
{ feature: this.featureName, step: step.text });
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
|
|
200
178
|
// SG-W007 — a two-step-form browser-alert handler (`click [OK] alert`) written AFTER
|
|
201
179
|
// an action step registers its page.once('dialog') listener too late: a dialog
|
|
202
180
|
// triggered by the earlier action auto-dismisses with no error (issue #472).
|