@sun-asterisk/sungen 3.2.20 → 3.2.21-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +8 -0
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +9 -0
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/trace.d.ts.map +1 -1
- package/dist/cli/commands/trace.js +9 -0
- package/dist/cli/commands/trace.js.map +1 -1
- package/dist/exporters/api-testcase-formatter.d.ts +8 -0
- package/dist/exporters/api-testcase-formatter.d.ts.map +1 -1
- package/dist/exporters/api-testcase-formatter.js +44 -22
- package/dist/exporters/api-testcase-formatter.js.map +1 -1
- package/dist/exporters/matrix/build.d.ts +4 -2
- package/dist/exporters/matrix/build.d.ts.map +1 -1
- package/dist/exporters/matrix/build.js +25 -2
- package/dist/exporters/matrix/build.js.map +1 -1
- package/dist/exporters/matrix/export.d.ts +6 -0
- package/dist/exporters/matrix/export.d.ts.map +1 -1
- package/dist/exporters/matrix/export.js +20 -1
- package/dist/exporters/matrix/export.js.map +1 -1
- package/dist/exporters/matrix/gates.js +7 -1
- package/dist/exporters/matrix/gates.js.map +1 -1
- package/dist/exporters/matrix/render-xlsx.d.ts.map +1 -1
- package/dist/exporters/matrix/render-xlsx.js +15 -0
- package/dist/exporters/matrix/render-xlsx.js.map +1 -1
- package/dist/exporters/matrix/types.d.ts +7 -0
- package/dist/exporters/matrix/types.d.ts.map +1 -1
- package/dist/exporters/matrix/types.js.map +1 -1
- package/dist/exporters/playwright-report-parser.d.ts.map +1 -1
- package/dist/exporters/playwright-report-parser.js +1 -0
- package/dist/exporters/playwright-report-parser.js.map +1 -1
- package/dist/exporters/types.d.ts +2 -0
- package/dist/exporters/types.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.d.ts +6 -1
- package/dist/generators/test-generator/diagnostics.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.js +5 -0
- package/dist/generators/test-generator/diagnostics.js.map +1 -1
- package/dist/generators/test-generator/patterns/index.d.ts.map +1 -1
- package/dist/generators/test-generator/patterns/index.js +27 -19
- package/dist/generators/test-generator/patterns/index.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.d.ts.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +22 -0
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/harness/audit.d.ts +2 -0
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +79 -9
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-contract.d.ts +71 -0
- package/dist/harness/flow-contract.d.ts.map +1 -0
- package/dist/harness/flow-contract.js +235 -0
- package/dist/harness/flow-contract.js.map +1 -0
- package/dist/harness/flow-plan.d.ts +3 -0
- package/dist/harness/flow-plan.d.ts.map +1 -1
- package/dist/harness/flow-plan.js +6 -2
- package/dist/harness/flow-plan.js.map +1 -1
- package/dist/harness/parse.d.ts +5 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +29 -1
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/perf.d.ts +40 -0
- package/dist/harness/perf.d.ts.map +1 -0
- package/dist/harness/perf.js +136 -0
- package/dist/harness/perf.js.map +1 -0
- package/dist/harness/script-check.d.ts.map +1 -1
- package/dist/harness/script-check.js +17 -4
- package/dist/harness/script-check.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +13 -1
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/spec-coverage.d.ts.map +1 -1
- package/dist/harness/spec-coverage.js +29 -5
- package/dist/harness/spec-coverage.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/add-flow.md +41 -3
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +10 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +35 -16
- package/dist/orchestrator/templates/qa-context.md +14 -1
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +8 -0
- package/src/cli/commands/delivery.ts +8 -0
- package/src/cli/commands/trace.ts +9 -0
- package/src/exporters/api-testcase-formatter.ts +46 -20
- package/src/exporters/matrix/build.ts +30 -4
- package/src/exporters/matrix/export.ts +28 -1
- package/src/exporters/matrix/gates.ts +7 -1
- package/src/exporters/matrix/render-xlsx.ts +15 -0
- package/src/exporters/matrix/types.ts +7 -0
- package/src/exporters/playwright-report-parser.ts +2 -0
- package/src/exporters/types.ts +2 -0
- package/src/generators/test-generator/diagnostics.ts +6 -1
- package/src/generators/test-generator/patterns/index.ts +30 -24
- package/src/generators/test-generator/step-mapper.ts +22 -0
- package/src/harness/audit.ts +82 -10
- package/src/harness/flow-contract.ts +229 -0
- package/src/harness/flow-plan.ts +10 -3
- package/src/harness/parse.ts +31 -1
- package/src/harness/perf.ts +112 -0
- package/src/harness/script-check.ts +18 -4
- package/src/harness/sensors.ts +13 -1
- package/src/harness/spec-coverage.ts +26 -4
- package/src/orchestrator/templates/ai-src/commands/add-flow.md +41 -3
- package/src/orchestrator/templates/ai-src/commands/create-test.md +10 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +35 -16
- package/src/orchestrator/templates/qa-context.md +14 -1
|
@@ -86,15 +86,52 @@ qa/flows/${input:flow}/
|
|
|
86
86
|
└── ui/ # Screenshots, mockups
|
|
87
87
|
```
|
|
88
88
|
|
|
89
|
-
### 1a.
|
|
89
|
+
### 1a. Define the flow's BOUNDARY, then its screens
|
|
90
90
|
|
|
91
|
-
|
|
91
|
+
A flow is the **smallest complete business action chain**: one clear trigger ending in ONE
|
|
92
|
+
observable, valuable outcome. Before asking for screens, walk this checklist with the user —
|
|
93
|
+
if 1, 3 or 8 fails, propose SPLITTING into separate flows:
|
|
94
|
+
|
|
95
|
+
1. Exactly **one business goal**? (cart correctness + category filtering = two flows)
|
|
96
|
+
2. A clear **trigger** and precondition?
|
|
97
|
+
3. **One observable final outcome**? (a final assertion you can write in one sentence)
|
|
98
|
+
4. Is that outcome **valuable to the actor**? (an order placed, a password reset — not "a page rendered")
|
|
99
|
+
5. Is **every step necessary** for that outcome?
|
|
100
|
+
6. Are all steps at the **same business abstraction**?
|
|
101
|
+
7. Are optional/error branches **phases of this goal** (ER/EH), not new goals?
|
|
102
|
+
8. Does **no segment** form an independently valuable flow on its own?
|
|
103
|
+
9. Can you write **a single clear final assertion**?
|
|
104
|
+
10. Can you name it "**Verb + outcome**"? (`place-order`, `reset-password` — not `cart-and-filter`)
|
|
105
|
+
|
|
106
|
+
Then ask: "Which screens does this flow visit, in order? (e.g., login → dashboard → award-form → confirmation)"
|
|
92
107
|
|
|
93
108
|
Record the screen list — you will need it for:
|
|
94
109
|
- Filling `spec.md` (Step 3)
|
|
95
110
|
- Suggesting `[Screen:Element]` namespace prefixes
|
|
96
111
|
- Capturing visuals per screen (Step 2)
|
|
97
112
|
|
|
113
|
+
### 1b. Author the Flow Contract (`requirements/flow-contract.yaml`)
|
|
114
|
+
|
|
115
|
+
Write the answers down as the flow's contract — `sungen audit` scores the flow **against it**
|
|
116
|
+
(the `flowCoverage` axis: HP/ER/EH journey phases; `FLOW-OUTCOME-UNPROVEN` when no automated
|
|
117
|
+
scenario asserts data on the outcome screen; `FLOW-SCOPE-CREEP` when scenarios never touch it):
|
|
118
|
+
|
|
119
|
+
```yaml
|
|
120
|
+
goal: "Place an order for a product added from home" # Verb + outcome
|
|
121
|
+
actor: user
|
|
122
|
+
trigger: "Add a product to the cart from the home featured list"
|
|
123
|
+
precondition: "A registered account; an empty cart"
|
|
124
|
+
outcome:
|
|
125
|
+
screen: checkout # the [Screen:...] namespace carrying the final proof
|
|
126
|
+
assertion: "The confirmation shows the order number and the paid total"
|
|
127
|
+
value: "The customer has paid; the shop has a new order"
|
|
128
|
+
phases: [HP, ER, EH] # journey phases (default); add UI only if the flow owns UI states
|
|
129
|
+
stateful: cart # the mutated collection, if any — enables regression-depth dims
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
**A filled contract is an INPUT to generation — never an output.** Like `test-viewpoint.md`,
|
|
133
|
+
generation must not rewrite it to match what was generated; disagree → propose the diff and ask.
|
|
134
|
+
|
|
98
135
|
### 2. Capture visual source
|
|
99
136
|
|
|
100
137
|
**Mobile path** (`platform: mobile`):
|
|
@@ -187,7 +224,8 @@ If user picks `/sungen:create-test`, **you MUST use the Skill tool** to invoke i
|
|
|
187
224
|
- Test data namespaced by phase: `login.email`, `submission.nominee`
|
|
188
225
|
- `@flow` tag required at feature level
|
|
189
226
|
- `Background:` should only contain the starting navigation — the URL path (web) or the `--reach` nav recipe (mobile)
|
|
190
|
-
- Each scenario = one phase of the journey
|
|
227
|
+
- Each scenario = one phase of the journey; ids are `FL-<PHASE>-NNN` (`HP`/`ER`/`EH`, optional `UI`)
|
|
228
|
+
- One flow = ONE business goal with ONE observable outcome (`requirements/flow-contract.yaml`) — a segment with its own value is its own flow
|
|
191
229
|
{{#cap parallel-subagents}}
|
|
192
230
|
- Mobile flows are tagged `@platform:mobile` and run via `/sungen:run-test <flow>` (WebdriverIO, not Playwright)
|
|
193
231
|
{{/cap}}
|
|
@@ -6,6 +6,16 @@ order: 20
|
|
|
6
6
|
claude-tools: "Read, Grep, Bash, Glob, Write, AskUserQuestion, Skill, mcp__playwright__browser_navigate, mcp__playwright__browser_snapshot, mcp__playwright__browser_take_screenshot"
|
|
7
7
|
copilot-tools: "[vscode, execute, read, agent, edit, search, web, browser, todo, 'playwright/*']"
|
|
8
8
|
codex-trigger: "Run when the user asks to CREATE, generate, write, or author test cases / a .feature file for a screen or flow. Step 2 (after add-screen/add-flow, before run-test). Do NOT use for executing, running, or compiling existing tests."
|
|
9
|
+
---
|
|
10
|
+
## ⛔ HARD RULE — the run's LAST action is the next-step hand-back
|
|
11
|
+
|
|
12
|
+
A create-test run is NOT finished when the files are written or the audit prints. The final
|
|
13
|
+
action of EVERY run — success, partial, or aborted — is the next-step hand-back
|
|
14
|
+
({{#cap parallel-subagents}}an `AskUserQuestion` offering the next actions{{/cap}}{{^cap parallel-subagents}}a numbered list of next-action choices{{/cap}};
|
|
15
|
+
see "Finish — always hand the next step back" at the end of this file). Ending with a prose
|
|
16
|
+
summary and no choices is a broken run: the operator is left guessing. This holds no matter
|
|
17
|
+
how long the generation/repair loop ran.
|
|
18
|
+
|
|
9
19
|
---
|
|
10
20
|
{{#cap parallel-subagents}}
|
|
11
21
|
## ⛔ HARD RULE — No Figma MCP when PAT data exists
|
|
@@ -605,22 +605,40 @@ error:
|
|
|
605
605
|
|
|
606
606
|
> **Auto-detect**: if path is `qa/flows/<name>/` → use this section. Skip Steps 1–4 above.
|
|
607
607
|
|
|
608
|
+
**Read `requirements/flow-contract.yaml` FIRST — it is the flow's boundary and the yardstick
|
|
609
|
+
`sungen audit` scores the flow against** (`flowCoverage` axis = journey phases HP/ER/EH automated;
|
|
610
|
+
`FLOW-OUTCOME-UNPROVEN`; `FLOW-SCOPE-CREEP`). No contract yet → author it with the user via the
|
|
611
|
+
boundary checklist in `add-flow` (one business goal · clear trigger · ONE observable outcome
|
|
612
|
+
valuable to the actor · name = "Verb + outcome"), THEN generate. **A filled contract is an INPUT —
|
|
613
|
+
never rewrite it to match your output** (same rule as `test-viewpoint.md`).
|
|
614
|
+
|
|
608
615
|
| Aspect | Screen | Flow |
|
|
609
616
|
|---|---|---|
|
|
610
|
-
| Section focus | UI patterns per section | Journey phases
|
|
617
|
+
| Section focus | UI patterns per section | Journey phases toward ONE declared outcome |
|
|
611
618
|
| Selector format | `[Element]` | `[Screen:Element]` (namespaced) |
|
|
612
619
|
| Test data keys | `{{variable}}` | `{{phase.variable}}` |
|
|
613
620
|
| Feature tag | `@auto` / `@smoke` etc. | `@flow` (required) |
|
|
614
|
-
|
|
|
621
|
+
| Scenario ids | `VP-<CATEGORY>-NNN` | `FL-<PHASE>-NNN` — phases: `HP` (happy path), `ER` (error recovery), `EH` (guards), `UI` (journey UI states, optional) |
|
|
615
622
|
|
|
616
|
-
**Scenarios to generate:**
|
|
623
|
+
**Scenarios to generate — every phase demanded by the contract, automated:**
|
|
617
624
|
|
|
618
|
-
|
|
|
619
|
-
|
|
620
|
-
|
|
|
621
|
-
|
|
|
622
|
-
|
|
|
623
|
-
| Cross-screen
|
|
625
|
+
| Phase | What to test | Scoring |
|
|
626
|
+
|---|---|---|
|
|
627
|
+
| `FL-HP` happy path | The complete journey ending in the contract's `outcome.assertion` — an AUTOMATED **data** assertion on `outcome.screen` (an order number, a summed total — not just "page visible"). This scenario is WHY the flow exists: it is the regression proof after a lib/framework upgrade. | uncovered → `flowCoverage` drops + `FLOW-OUTCOME-UNPROVEN` |
|
|
628
|
+
| `FL-ER` error recovery | Invalid input mid-flow → error shown → fix → the journey still completes. Validation must not trap the journey. | uncovered → `flowCoverage` drops |
|
|
629
|
+
| `FL-EH` guards | Direct URL access without the precondition · browser back · refresh · expired context — each ends in a safe observable state. | uncovered → `flowCoverage` drops |
|
|
630
|
+
| Cross-screen handoff | After every screen transition, assert the CARRIED state on the new screen (the added product's name in the cart, the email echoed on the sent screen). | blind tails cap `businessDepth` (`FLOW-HANDOFF-SHALLOW`) |
|
|
631
|
+
| Stateful regression (when `stateful:` declared) | Count/quantity proof · teardown (remove → empty) · multi-source add. | missing dims cap `businessDepth` (`FLOW-DEPTH`) |
|
|
632
|
+
|
|
633
|
+
**Boundary discipline while generating:** every scenario must serve the contract's goal. A scenario
|
|
634
|
+
that never touches `outcome.screen` and is not a guard (`EH`) or error-recovery (`ER`) belongs in a
|
|
635
|
+
DIFFERENT flow — propose the split instead of writing it here (`FLOW-SCOPE-CREEP` will flag it).
|
|
636
|
+
Auth persistence across transitions is part of `EH` unless the project declares it its own phase.
|
|
637
|
+
|
|
638
|
+
**Manual in flows**: always `@manual:Mx` with the reason code — bare `@manual` is flagged
|
|
639
|
+
(`MANUAL-CODE-MISSING`) because the capability planner cannot route it. Typical flow deferrals:
|
|
640
|
+
inbox/mail oracle → `M5`, network-request count → `M3`, context expiry control → `M7`. A
|
|
641
|
+
cross-screen scenario inside the flow's own goal is NOT manual — automate it here.
|
|
624
642
|
|
|
625
643
|
```gherkin
|
|
626
644
|
@flow @auth:user
|
|
@@ -630,19 +648,20 @@ Feature: Award Submission Flow
|
|
|
630
648
|
Given User is on [Login] page
|
|
631
649
|
|
|
632
650
|
@high
|
|
633
|
-
Scenario:
|
|
651
|
+
Scenario: FL-HP-001 A signed-in user's nomination is submitted and confirmed
|
|
634
652
|
When User fill [Login:Email] field with {{login.email}}
|
|
635
653
|
And User fill [Login:Password] field with {{login.password}}
|
|
636
654
|
And User click [Login:Submit] button
|
|
637
655
|
Then User see [Dashboard] page
|
|
638
|
-
|
|
639
|
-
@high
|
|
640
|
-
Scenario: User submits nomination
|
|
641
656
|
When User click [Dashboard:Awards] link
|
|
642
|
-
|
|
643
|
-
When User fill [Awards:Nominee] field with {{submission.nominee}}
|
|
657
|
+
And User fill [Awards:Nominee] field with {{submission.nominee}}
|
|
644
658
|
And User click [Awards:Submit] button
|
|
645
|
-
Then User see {{success_message}}
|
|
659
|
+
Then User see [Awards:Success Message] text with {{success_message}}
|
|
660
|
+
|
|
661
|
+
@high
|
|
662
|
+
Scenario: FL-EH-001 Direct access to the award form without login redirects to login
|
|
663
|
+
When User go to [Awards] page
|
|
664
|
+
Then User see [Login] page
|
|
646
665
|
```
|
|
647
666
|
|
|
648
667
|
```yaml
|
|
@@ -45,8 +45,21 @@ Example:
|
|
|
45
45
|
|
|
46
46
|
## Testing Strategy
|
|
47
47
|
|
|
48
|
+
Machine-readable intent — `sungen audit` reads these keys (Intent Profile). Values here are
|
|
49
|
+
live even when the surrounding text changes; an invalid value silently falls back to the default.
|
|
50
|
+
|
|
51
|
+
focus: functional
|
|
52
|
+
<!-- focus: functional | e-commerce | security | smoke — drives the audit's depth threshold -->
|
|
53
|
+
|
|
54
|
+
risk_tier: normal
|
|
55
|
+
<!-- risk_tier: high | normal | low -->
|
|
56
|
+
|
|
57
|
+
To silence driver suggestions in audit findings, add a line: capability_suggestions with value off.
|
|
58
|
+
|
|
48
59
|
**Focus areas** — what to cover thoroughly:
|
|
49
|
-
<!--
|
|
60
|
+
<!-- Prose for humans; the parseable value is the `focus:` key above.
|
|
61
|
+
Response-time budgets are NOT a focus value — declare them in qa/perf.yaml
|
|
62
|
+
(percentile + scenario_ms budgets; reported by `sungen delivery`). -->
|
|
50
63
|
<!-- Example: functional, security -->
|
|
51
64
|
|
|
52
65
|
**Mandatory coverage:**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sun-asterisk/sungen",
|
|
3
|
-
"version": "3.2.
|
|
3
|
+
"version": "3.2.21-beta.1",
|
|
4
4
|
"description": "Deterministic E2E Test Compiler - Gherkin + Selectors → Playwright tests",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
@@ -39,8 +39,8 @@
|
|
|
39
39
|
"@babel/types": "^7.28.5",
|
|
40
40
|
"@cucumber/gherkin": "^37.0.0",
|
|
41
41
|
"@cucumber/messages": "^31.0.0",
|
|
42
|
-
"@sungen/driver-data-factory": "3.2.
|
|
43
|
-
"@sungen/driver-ui": "3.2.
|
|
42
|
+
"@sungen/driver-data-factory": "3.2.21-beta.1",
|
|
43
|
+
"@sungen/driver-ui": "3.2.21-beta.1",
|
|
44
44
|
"chalk": "^5.6.2",
|
|
45
45
|
"commander": "^14.0.2",
|
|
46
46
|
"dotenv": "^17.2.3",
|
|
@@ -40,6 +40,14 @@ function render(r: AuditReport): void {
|
|
|
40
40
|
if (r.gate.gaps.length) for (const g of r.gate.gaps) L(` ✗ ${g.status === 'shallow' ? 'SHALLOW' : 'MISSING'}: ${g.theme}`);
|
|
41
41
|
else L(' ✓ all critical themes covered (with data assertions)');
|
|
42
42
|
L('');
|
|
43
|
+
if (r.flowQuality?.hasContract) {
|
|
44
|
+
const fq = r.flowQuality;
|
|
45
|
+
L(` ①F Flow contract — goal: ${fq.contract!.goal}`);
|
|
46
|
+
L(` outcome [${fq.contract!.outcome.screen}]: ${fq.outcomeProven ? '✓ proven (automated data assertion)' : fq.outcomeManualOnly ? '⚠ manual-only' : '✗ UNPROVEN'}`);
|
|
47
|
+
L(` phases: ${fq.phases.map((p) => `${p.phase}=${p.covered ? (p.automated ? '✓' : 'manual') : '✗'}`).join(' ')} → flowCoverage ${(fq.phaseRatio * 100).toFixed(0)}%`);
|
|
48
|
+
L(` handoffs asserted: ${fq.handoffs.asserted}/${fq.handoffs.total}${fq.offGoal.length ? ` ⚠ off-goal scenarios: ${fq.offGoal.length} (${fq.offGoalCategories.join(', ')})` : ''}`);
|
|
49
|
+
L('');
|
|
50
|
+
}
|
|
43
51
|
L(` ② Assertion depth — ${r.depth.businessCriticalShallow}/${r.depth.businessCriticalTotal} business-critical scenarios are shallow (visibility/nav only)`);
|
|
44
52
|
for (const s of r.depth.shallowBusinessCritical.slice(0, 8)) L(` ⚠ ${s.category}: ${s.name}`);
|
|
45
53
|
if (r.depth.shallowBusinessCritical.length > 8) L(` … +${r.depth.shallowBusinessCritical.length - 8} more`);
|
|
@@ -635,6 +635,8 @@ function matrixPathsFor(cwd: string, target: DeliveryTarget): MatrixTargetPaths
|
|
|
635
635
|
return {
|
|
636
636
|
cwd,
|
|
637
637
|
unit: target.featureBaseName,
|
|
638
|
+
kind: target.kind,
|
|
639
|
+
area: target.screen,
|
|
638
640
|
label: target.isFlow ? `flow/${target.featureBaseName}` : target.featureBaseName,
|
|
639
641
|
featureFile: path.join(base, 'features', `${target.featureBaseName}.feature`),
|
|
640
642
|
testDataFile: resolveTestDataPathForTarget(cwd, target),
|
|
@@ -688,6 +690,12 @@ async function exportMatrixTarget(
|
|
|
688
690
|
}
|
|
689
691
|
|
|
690
692
|
log(`${COLOR.bold}${paths.label}${COLOR.reset} — ${model.manifest.itemCount} delivery item(s), ${model.manifest.variantCount} coverage variant(s), state: ${model.manifest.approvalState}`);
|
|
693
|
+
if (model.perf) {
|
|
694
|
+
const p = model.perf;
|
|
695
|
+
const tone = p.pass ? COLOR.green : COLOR.yellow;
|
|
696
|
+
log(` ${tone}PERF ${p.pass ? 'PASS' : 'OVER BUDGET'}${COLOR.reset}: p${p.percentile} ${p.metric.replace('_ms', '')} ${p.measuredMs}ms ${p.pass ? '≤' : '>'} budget ${p.budgetMs}ms (${p.samples} run${p.samples === 1 ? '' : 's'})`);
|
|
697
|
+
for (const s of p.slowest) log(` ${COLOR.yellow}slowest:${COLOR.reset} ${s.ms}ms — ${s.title.slice(0, 90)}`);
|
|
698
|
+
}
|
|
691
699
|
printFindings(model.findings);
|
|
692
700
|
|
|
693
701
|
if (opts.check) {
|
|
@@ -55,6 +55,15 @@ export function registerTraceCommand(program: Command): void {
|
|
|
55
55
|
for (const line of r.mermaid.split('\n')) L(' ' + line);
|
|
56
56
|
L(' ```');
|
|
57
57
|
L('');
|
|
58
|
+
// ⑥ Deterministic next-step hand-back. The AI commands are INSTRUCTED to close
|
|
59
|
+
// every run by offering next actions, but a long generation run can end without
|
|
60
|
+
// it — leaving the operator guessing. Printing the choices here makes the
|
|
61
|
+
// hand-back survive the AI forgetting: trace is the converge step's last call.
|
|
62
|
+
L(' ⑥ NEXT STEPS');
|
|
63
|
+
L(` → /sungen:run-test ${name} (generate selectors, compile, run, auto-fix)`);
|
|
64
|
+
L(` → /sungen:create-test ${name} (extend coverage / next tier)`);
|
|
65
|
+
L(` → /sungen:delivery ${name} (export the Test Case & Coverage Matrix)`);
|
|
66
|
+
L('');
|
|
58
67
|
} catch (error) {
|
|
59
68
|
console.error('Error:', error instanceof Error ? error.message : error);
|
|
60
69
|
process.exit(1);
|
|
@@ -74,26 +74,7 @@ export function formatApiRequest(
|
|
|
74
74
|
const calls = parseApiCalls(tags).filter((c) => catalog[c.name]);
|
|
75
75
|
if (calls.length === 0) return '';
|
|
76
76
|
|
|
77
|
-
const blocks = calls.map((call) =>
|
|
78
|
-
const entry = catalog[call.name];
|
|
79
|
-
const resolve = (raw: string): string => localize(bindParams(raw, call.args));
|
|
80
|
-
// Coerce method/path — loadApiCatalog is a raw passthrough, so a malformed entry
|
|
81
|
-
// (numeric method/path) must degrade gracefully, not crash the unit's export.
|
|
82
|
-
const method = String(entry.method ?? '').toUpperCase() || '—';
|
|
83
|
-
const api = entry.path ? resolve(String(entry.path)) : '—';
|
|
84
|
-
const header = formatHeaders(entry, call.args, localize);
|
|
85
|
-
const lines = [`Method: ${method}`, `API: ${api}`, `Header: ${header}`];
|
|
86
|
-
// A file upload carries its parts in `files:` / `bodyFile:`, NOT `body:` — render them so the
|
|
87
|
-
// delivery Steps reflect the actual request (without this, fields like `gift_image` vanished from
|
|
88
|
-
// the export even though the compiled .spec.ts sent them).
|
|
89
|
-
const filesStr = formatFiles(entry.files, call.args, localize);
|
|
90
|
-
const bodyFileStr = formatBodyFile(entry.bodyFile, call.args, localize);
|
|
91
|
-
// Body is scalar fields; a pure file upload may have none → still show `Body: —` for a stable shape.
|
|
92
|
-
lines.push(`Body: ${formatBody(entry.body, call.args, localize)}`);
|
|
93
|
-
if (filesStr) lines.push(`Files: ${filesStr}`);
|
|
94
|
-
if (bodyFileStr) lines.push(`Body file: ${bodyFileStr}`);
|
|
95
|
-
return lines.join('\n');
|
|
96
|
-
});
|
|
77
|
+
const blocks = calls.map((call) => callBlockLines(call, catalog[call.name], localize).join('\n'));
|
|
97
78
|
|
|
98
79
|
if (blocks.length === 1) return blocks[0];
|
|
99
80
|
// Multiple endpoints (flow) — number them so the call order is explicit.
|
|
@@ -102,6 +83,51 @@ export function formatApiRequest(
|
|
|
102
83
|
.join('\n');
|
|
103
84
|
}
|
|
104
85
|
|
|
86
|
+
/** Render one invoked endpoint as its Method / API / Header / Body block lines. */
|
|
87
|
+
function callBlockLines(call: ApiCall, entry: ApiCatalogEntry, localize: (text: string) => string): string[] {
|
|
88
|
+
const resolve = (raw: string): string => localize(bindParams(raw, call.args));
|
|
89
|
+
// Coerce method/path — loadApiCatalog is a raw passthrough, so a malformed entry
|
|
90
|
+
// (numeric method/path) must degrade gracefully, not crash the unit's export.
|
|
91
|
+
const method = String(entry.method ?? '').toUpperCase() || '—';
|
|
92
|
+
const api = entry.path ? resolve(String(entry.path)) : '—';
|
|
93
|
+
const header = formatHeaders(entry, call.args, localize);
|
|
94
|
+
const lines = [`Method: ${method}`, `API: ${api}`, `Header: ${header}`];
|
|
95
|
+
// A file upload carries its parts in `files:` / `bodyFile:`, NOT `body:` — render them so the
|
|
96
|
+
// delivery Steps reflect the actual request (without this, fields like `gift_image` vanished from
|
|
97
|
+
// the export even though the compiled .spec.ts sent them).
|
|
98
|
+
const filesStr = formatFiles(entry.files, call.args, localize);
|
|
99
|
+
const bodyFileStr = formatBodyFile(entry.bodyFile, call.args, localize);
|
|
100
|
+
// Body is scalar fields; a pure file upload may have none → still show `Body: —` for a stable shape.
|
|
101
|
+
lines.push(`Body: ${formatBody(entry.body, call.args, localize)}`);
|
|
102
|
+
if (filesStr) lines.push(`Files: ${filesStr}`);
|
|
103
|
+
if (bodyFileStr) lines.push(`Body file: ${bodyFileStr}`);
|
|
104
|
+
return lines;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Matrix-cell view of the ordered `@api:` chain: ONE string per call — the full
|
|
109
|
+
* request block for cataloged endpoints, a name-only line otherwise (a call must
|
|
110
|
+
* never silently vanish from the deliverable). Blocks are numbered when the chain
|
|
111
|
+
* has more than one call — same style as formatApiRequest — which also keeps a
|
|
112
|
+
* deliberately repeated call distinct through the matrix's precondition Set-dedup.
|
|
113
|
+
*/
|
|
114
|
+
export function formatApiCallLines(
|
|
115
|
+
calls: ApiCall[],
|
|
116
|
+
catalog: Record<string, ApiCatalogEntry>,
|
|
117
|
+
localize: (text: string) => string,
|
|
118
|
+
): string[] {
|
|
119
|
+
const blocks = calls.map((call) => {
|
|
120
|
+
const entry = catalog[call.name];
|
|
121
|
+
if (!entry) {
|
|
122
|
+
const args = Object.entries(call.args).map(([k, v]) => `${k}=${v}`).join(', ');
|
|
123
|
+
return [`Call API ${call.name}${args ? ` (${args})` : ''}.`];
|
|
124
|
+
}
|
|
125
|
+
return callBlockLines(call, entry, localize);
|
|
126
|
+
});
|
|
127
|
+
if (blocks.length === 1) return [blocks[0].join('\n')];
|
|
128
|
+
return blocks.map((b, i) => b.map((l, j) => (j === 0 ? `${i + 1}. ${l}` : ` ${l}`)).join('\n'));
|
|
129
|
+
}
|
|
130
|
+
|
|
105
131
|
/**
|
|
106
132
|
* Rewrite an API scenario's `Then` assertions into readable Status / Response lines.
|
|
107
133
|
* Raw assertions reference the runtime response object
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* is structural: there is no independent parent result input).
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import { FeatureMetadata, PlaywrightResult } from '../types';
|
|
8
|
+
import { ApiCatalogEntry, FeatureMetadata, PlaywrightResult } from '../types';
|
|
9
|
+
import { formatApiCallLines, parseApiCalls } from '../api-testcase-formatter';
|
|
9
10
|
import { bucketTexts } from './step-buckets';
|
|
10
11
|
import { MergedScenario } from '../scenario-merger';
|
|
11
12
|
import {
|
|
@@ -95,6 +96,8 @@ export interface BuildInputs {
|
|
|
95
96
|
transformerVersion: string;
|
|
96
97
|
/** requirements/spec.md content — source of the requirement-id inventory (FR/TR/NFR). */
|
|
97
98
|
specText?: string;
|
|
99
|
+
/** apis.yaml endpoint catalog (api units) — renders `@api:` calls as request blocks. */
|
|
100
|
+
apiCatalog?: Record<string, ApiCatalogEntry>;
|
|
98
101
|
}
|
|
99
102
|
|
|
100
103
|
/**
|
|
@@ -102,11 +105,12 @@ export interface BuildInputs {
|
|
|
102
105
|
* `@cases` dataset row). Exported separately so the gates + the map-proposal
|
|
103
106
|
* support tooling see the same universe the builder does.
|
|
104
107
|
*/
|
|
105
|
-
export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' | 'testData' | 'results'>): CoverageVariant[] {
|
|
108
|
+
export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' | 'testData' | 'results' | 'apiCatalog'>): CoverageVariant[] {
|
|
106
109
|
const { merged, results } = inputs;
|
|
107
110
|
// Test-data values may cross-reference other keys (email_padded: " {{valid_email}} ") —
|
|
108
111
|
// resolve one level so display cells never leak a template token (review B-04).
|
|
109
112
|
const testData = resolveCrossRefs(inputs.testData);
|
|
113
|
+
const apiCatalog = inputs.apiCatalog ?? {};
|
|
110
114
|
const variants: CoverageVariant[] = [];
|
|
111
115
|
|
|
112
116
|
for (const m of merged) {
|
|
@@ -141,11 +145,27 @@ export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' |
|
|
|
141
145
|
tags,
|
|
142
146
|
);
|
|
143
147
|
|
|
148
|
+
// `@api:<name>(args)` invocations — the runtime's precondition codegen (driver-api):
|
|
149
|
+
// each call sends its catalog request BEFORE the scenario's own steps and binds
|
|
150
|
+
// {{<name>.*}}. Cell placement follows what the tester actually does:
|
|
151
|
+
// - pure api scenario (no When steps): the calls ARE the procedure → trigger,
|
|
152
|
+
// rendered as Method/API/Header/Body blocks (the legacy Steps-cell language) —
|
|
153
|
+
// the page-load fallback would misdescribe an HTTP request;
|
|
154
|
+
// - hybrid (@api + own When steps): the calls are setup → precondition.
|
|
155
|
+
// Blocks keep raw `{{var}}` tokens here; renderCells' sub resolves them per data
|
|
156
|
+
// context (so @cases rows render their own arg values).
|
|
157
|
+
const apiCalls = parseApiCalls(tags);
|
|
158
|
+
const apiCallsAreTrigger = apiCalls.length > 0 && rawTrigger.length === 0;
|
|
159
|
+
const apiRequestLines = apiCalls.length > 0 ? formatApiCallLines(apiCalls, apiCatalog, (s) => s) : [];
|
|
160
|
+
// Shape entry per call: endpoint identity + arg KEYS (values are data, not shape).
|
|
161
|
+
const apiCallShapes = apiCalls.map((c) => `api:${c.name}(${Object.keys(c.args).join(',')})`);
|
|
162
|
+
|
|
144
163
|
const preconditionProfile = [
|
|
145
164
|
authRole ?? '-',
|
|
146
165
|
m.feature.extendsName ?? '-',
|
|
147
166
|
...bucketTexts(m.feature.orderedSteps, 'given').map(normalizeShape),
|
|
148
167
|
...(manual?.preconditions ?? []).map(normalizeShape),
|
|
168
|
+
...(apiCallsAreTrigger ? [] : apiCallShapes),
|
|
149
169
|
].join(' | ');
|
|
150
170
|
// Display precondition = auth state + Background Given (shared start state) +
|
|
151
171
|
// the scenario's own Given steps + manual Setup lines — deduplicated (a manual
|
|
@@ -155,6 +175,7 @@ export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' |
|
|
|
155
175
|
...bucketTexts(inputs.feature.backgroundOrderedSteps, 'given').map(renderPrecondition),
|
|
156
176
|
...bucketTexts(m.feature.orderedSteps, 'given').map(renderPrecondition),
|
|
157
177
|
...(manual?.preconditions ?? []).map((t) => renderSetupInstruction(t)),
|
|
178
|
+
...(apiCallsAreTrigger ? [] : apiRequestLines),
|
|
158
179
|
]));
|
|
159
180
|
|
|
160
181
|
const base = {
|
|
@@ -166,7 +187,8 @@ export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' |
|
|
|
166
187
|
layers: deriveLayers(tags),
|
|
167
188
|
traces: tags.filter((t) => t.startsWith('@spec:')).map((t) => t.slice('@spec:'.length)),
|
|
168
189
|
runtimeVars: [...runtimeVars],
|
|
169
|
-
|
|
190
|
+
apiCalls: apiCalls.length,
|
|
191
|
+
triggerShape: [...rawTrigger.map(normalizeShape), ...(apiCallsAreTrigger ? apiCallShapes : [])],
|
|
170
192
|
oracleShape: rawOracle.map(normalizeShape),
|
|
171
193
|
preconditionProfile,
|
|
172
194
|
precondition,
|
|
@@ -196,7 +218,11 @@ export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' |
|
|
|
196
218
|
});
|
|
197
219
|
return { trigger, oracle, precondition: preconditionOut, verification: verificationOut };
|
|
198
220
|
}
|
|
199
|
-
|
|
221
|
+
// A pure api scenario's procedure is its request blocks (not renderAction —
|
|
222
|
+
// Method/API/Header/Body lines are already reader-facing, not step DSL).
|
|
223
|
+
const trigger = apiCallsAreTrigger
|
|
224
|
+
? apiRequestLines.map(sub)
|
|
225
|
+
: rawTrigger.map((s) => renderAction(sub(s)));
|
|
200
226
|
const oracle = rawOracle.map((s) => renderExpected(sub(s)));
|
|
201
227
|
return {
|
|
202
228
|
// A pure-render check (Given + Then only) is checked on page load — say so
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import * as fs from 'fs';
|
|
11
|
+
import { loadApiCatalog } from '../api-catalog-loader';
|
|
11
12
|
import { parseFeatureMetadata } from '../feature-parser';
|
|
12
13
|
import { parseSpecFile } from '../spec-parser';
|
|
13
14
|
import { loadTestData } from '../test-data-resolver';
|
|
@@ -16,17 +17,25 @@ import { mergeFeatureAndSpec } from '../scenario-merger';
|
|
|
16
17
|
import { getPackageVersion } from '../package-info';
|
|
17
18
|
import { writeCsv } from '../csv-exporter';
|
|
18
19
|
import { writeXlsx } from '../xlsx-exporter';
|
|
20
|
+
import { loadPerfConfig, perfVerdict } from '../../harness/perf';
|
|
19
21
|
import { loadDeliveryMap, writeDeliveryMap } from './map-loader';
|
|
20
22
|
import { mapContentFingerprint } from './fingerprint';
|
|
21
23
|
import { buildMatrix, deriveVariants } from './build';
|
|
22
24
|
import { renderMatrixXlsx } from './render-xlsx';
|
|
23
25
|
import { renderMatrixCsv } from './render-csv';
|
|
24
26
|
import { DeliveryMap, MatrixFinding, MatrixModel } from './types';
|
|
27
|
+
import { ApiCatalogEntry } from '../types';
|
|
25
28
|
|
|
26
29
|
export interface MatrixTargetPaths {
|
|
27
30
|
cwd: string;
|
|
28
31
|
/** Feature basename — names the deliverable files. */
|
|
29
32
|
unit: string;
|
|
33
|
+
/** Unit kind — 'api' loads the endpoint catalog so cells render request blocks.
|
|
34
|
+
* Optional for back-compat: absent means non-api (no catalog). */
|
|
35
|
+
kind?: 'screen' | 'flow' | 'api';
|
|
36
|
+
/** Area/screen directory name (differs from `unit` for sub-features) — keys the
|
|
37
|
+
* unit-level apis.yaml path. Defaults to `unit`. */
|
|
38
|
+
area?: string;
|
|
30
39
|
/** Display label (flow/… prefix for flows). */
|
|
31
40
|
label: string;
|
|
32
41
|
featureFile: string;
|
|
@@ -44,6 +53,11 @@ function readSpecText(paths: MatrixTargetPaths): string {
|
|
|
44
53
|
: '';
|
|
45
54
|
}
|
|
46
55
|
|
|
56
|
+
/** apis.yaml endpoint catalog for an api unit ({} for screen/flow — loader no-ops). */
|
|
57
|
+
function catalogFor(paths: MatrixTargetPaths): Record<string, ApiCatalogEntry> {
|
|
58
|
+
return loadApiCatalog(paths.cwd, paths.kind ?? 'screen', paths.area ?? paths.unit);
|
|
59
|
+
}
|
|
60
|
+
|
|
47
61
|
export interface MatrixLoadResult {
|
|
48
62
|
model?: MatrixModel;
|
|
49
63
|
map?: DeliveryMap;
|
|
@@ -74,7 +88,18 @@ export function loadMatrixModel(paths: MatrixTargetPaths): MatrixLoadResult {
|
|
|
74
88
|
map,
|
|
75
89
|
transformerVersion: getPackageVersion(),
|
|
76
90
|
specText: readSpecText(paths),
|
|
91
|
+
apiCatalog: catalogFor(paths),
|
|
77
92
|
});
|
|
93
|
+
// Perf verdict (#569): budgets are config (qa/perf.yaml), samples are the run's
|
|
94
|
+
// per-test durations. Advisory — reported here because delivery is where runs
|
|
95
|
+
// are read; the audit's score stays a pure function of the design artifacts.
|
|
96
|
+
const perfConfig = loadPerfConfig(paths.cwd);
|
|
97
|
+
if (perfConfig && results) {
|
|
98
|
+
const samples = [...results.values()]
|
|
99
|
+
.filter((r) => (r.status === 'passed' || r.status === 'failed' || r.status === 'timedOut') && typeof r.duration === 'number')
|
|
100
|
+
.map((r) => ({ title: r.testTitle, ms: r.duration as number }));
|
|
101
|
+
model.perf = perfVerdict(perfConfig, paths.unit, samples);
|
|
102
|
+
}
|
|
78
103
|
return { model, map, mapErrors: [] };
|
|
79
104
|
}
|
|
80
105
|
|
|
@@ -95,15 +120,17 @@ export function approveMatrix(paths: MatrixTargetPaths, groupIds?: string[]): {
|
|
|
95
120
|
const merged = mergeFeatureAndSpec(feature, spec);
|
|
96
121
|
const testData = fs.existsSync(paths.testDataFile) ? loadTestData(paths.testDataFile, process.env.SUNGEN_ENV ?? null) : null;
|
|
97
122
|
|
|
123
|
+
const apiCatalog = catalogFor(paths);
|
|
98
124
|
const model = buildMatrix({
|
|
99
125
|
unit: paths.unit, feature, merged, testData, results: null, map,
|
|
100
126
|
transformerVersion: getPackageVersion(),
|
|
101
127
|
specText: readSpecText(paths),
|
|
128
|
+
apiCatalog,
|
|
102
129
|
});
|
|
103
130
|
const blocking = model.findings.filter((f) => f.severity === 'error');
|
|
104
131
|
if (blocking.length > 0) return { findings: blocking, approved: [] };
|
|
105
132
|
|
|
106
|
-
const variants = deriveVariants({ feature, merged, testData, results: null });
|
|
133
|
+
const variants = deriveVariants({ feature, merged, testData, results: null, apiCatalog });
|
|
107
134
|
const byRef = new Map(variants.map((v) => [v.ref, v] as const));
|
|
108
135
|
const byVp = new Map<string, typeof variants>();
|
|
109
136
|
for (const v of variants) {
|
|
@@ -248,7 +248,13 @@ function gateDExecutability(ctx: GateContext, findings: MatrixFinding[]): void {
|
|
|
248
248
|
for (const g of map.groups) {
|
|
249
249
|
const vs = g.variants.flatMap((ref) => expandMapRef(ref, variantsByVp));
|
|
250
250
|
for (const v of vs) {
|
|
251
|
-
|
|
251
|
+
// An api-first variant (`@api:` calls, no UI steps) legitimately has no UI start
|
|
252
|
+
// state — the request blocks in its trigger (Method/API/Header/Body) carry
|
|
253
|
+
// everything a tester needs. Demanding a Background/Given here blocked 100% of
|
|
254
|
+
// api-first units (field report 3.2.19).
|
|
255
|
+
const hasPrecondition = v.precondition.length > 0
|
|
256
|
+
|| bucketTexts(feature.backgroundOrderedSteps, 'given').length > 0
|
|
257
|
+
|| v.apiCalls > 0;
|
|
252
258
|
if (!hasPrecondition) {
|
|
253
259
|
findings.push({
|
|
254
260
|
gate: 'D', severity: 'error', ref: v.ref,
|
|
@@ -315,6 +315,21 @@ function addCoverageSheet(wb: ExcelJS.Workbook, model: MatrixModel, sungenVersio
|
|
|
315
315
|
|
|
316
316
|
let rowIdx = 7;
|
|
317
317
|
|
|
318
|
+
// --- Performance budget (#569) — advisory: config (qa/perf.yaml) vs measured runs.
|
|
319
|
+
if (model.perf) {
|
|
320
|
+
const p = model.perf;
|
|
321
|
+
dataCell(ws.getRow(rowIdx).getCell(2), 'Performance budget', { bold: true });
|
|
322
|
+
rowIdx++;
|
|
323
|
+
const r = ws.getRow(rowIdx);
|
|
324
|
+
dataCell(r.getCell(2), `p${p.percentile} ${p.metric.replace('_ms', ' time')}`);
|
|
325
|
+
dataCell(r.getCell(3), p.pass ? 'PASS' : 'OVER BUDGET', { bold: true });
|
|
326
|
+
dataCell(r.getCell(4), `measured ${p.measuredMs}ms`);
|
|
327
|
+
dataCell(r.getCell(5), `budget ${p.budgetMs}ms`);
|
|
328
|
+
dataCell(r.getCell(6), `${p.samples} runs`);
|
|
329
|
+
if (!p.pass && p.slowest.length) dataCell(r.getCell(8), `slowest: ${p.slowest.map((x) => `${x.ms}ms ${x.title.slice(0, 40)}`).join(' · ')}`);
|
|
330
|
+
rowIdx += 2;
|
|
331
|
+
}
|
|
332
|
+
|
|
318
333
|
// --- Requirement coverage (review §6): every requirement id gets ONE explicit status.
|
|
319
334
|
if (model.requirements.length > 0) {
|
|
320
335
|
dataCell(ws.getRow(rowIdx).getCell(2), 'Requirement coverage — every id has an explicit status', { bold: true });
|
|
@@ -98,6 +98,11 @@ export interface CoverageVariant {
|
|
|
98
98
|
/** Variables this scenario produces at RUN TIME (`remember … as {{v}}`, capability
|
|
99
99
|
* bindings) — they never exist in test-data, so Gate D must not demand them. */
|
|
100
100
|
runtimeVars: string[];
|
|
101
|
+
/** Count of `@api:<name>` invocations. An api-first variant's start state lives in
|
|
102
|
+
* its request blocks (Method/API/Header/Body), so Gate D's precondition demand is
|
|
103
|
+
* waived when this is > 0 — requiring a Background/Given blocked EVERY api-first
|
|
104
|
+
* unit (field report 3.2.19). */
|
|
105
|
+
apiCalls: number;
|
|
101
106
|
/** Condition summary: differentiating dimension shown on the sub-row. */
|
|
102
107
|
condition: string;
|
|
103
108
|
/** Resolved data pairs (test-data values / dataset-row columns), `key: value` lines. */
|
|
@@ -220,6 +225,8 @@ export interface MatrixModel {
|
|
|
220
225
|
requirements: RequirementCoverage[];
|
|
221
226
|
findings: MatrixFinding[];
|
|
222
227
|
manifest: MatrixManifest;
|
|
228
|
+
/** Perf verdict (#569) — present when qa/perf.yaml sets a budget AND the run produced durations. */
|
|
229
|
+
perf?: import('../../harness/perf').PerfVerdict | null;
|
|
223
230
|
}
|
|
224
231
|
|
|
225
232
|
/** Complexity warning threshold (rules draft Gate I; raised for compact grouping) — echoed in the manifest. */
|
|
@@ -54,6 +54,7 @@ interface PlaywrightSpec {
|
|
|
54
54
|
projectId?: string;
|
|
55
55
|
results?: Array<{
|
|
56
56
|
status?: string; // 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'
|
|
57
|
+
duration?: number; // ms — Playwright emits this on every result (#569 perf budgets)
|
|
57
58
|
startTime?: string;
|
|
58
59
|
error?: { message?: string; stack?: string };
|
|
59
60
|
attachments?: Array<{ name?: string; path?: string; contentType?: string }>;
|
|
@@ -120,6 +121,7 @@ export function loadPlaywrightReport(reportPath: string): Map<string, Playwright
|
|
|
120
121
|
result.set(stripped, {
|
|
121
122
|
testTitle: stripped,
|
|
122
123
|
status,
|
|
124
|
+
duration: typeof res?.duration === 'number' ? res.duration : undefined,
|
|
123
125
|
startTime: res?.startTime,
|
|
124
126
|
error: errorMsg,
|
|
125
127
|
tracePath: trace,
|
package/src/exporters/types.ts
CHANGED
|
@@ -121,6 +121,8 @@ export interface SpecFileData {
|
|
|
121
121
|
export interface PlaywrightResult {
|
|
122
122
|
testTitle: string;
|
|
123
123
|
status: 'passed' | 'failed' | 'skipped' | 'timedOut' | 'interrupted' | 'unknown';
|
|
124
|
+
/** Wall-clock run time in ms (from Playwright's results[].duration) — perf budgets (#569). */
|
|
125
|
+
duration?: number;
|
|
124
126
|
startTime?: string;
|
|
125
127
|
error?: string;
|
|
126
128
|
tracePath?: string;
|
|
@@ -35,12 +35,17 @@
|
|
|
35
35
|
* nothing for the query half to assert, so the step checks the PATH only. Not an
|
|
36
36
|
* error (the value may legitimately vary per environment), but the step asserts
|
|
37
37
|
* strictly less than it appears to
|
|
38
|
+
* SG-W015 — `see [X] card/item with {{n}}` where the value is a bare number: this compiles
|
|
39
|
+
* to a TEXT assertion (toHaveText('3')), NOT a count — an author reaching for
|
|
40
|
+
* "3 cards are shown" gets an assertion that can only pass if the card's text
|
|
41
|
+
* IS the digit. Count semantics exist on row/table ("see [T] table with {{n}}",
|
|
42
|
+
* "see [Ref] row with {{n}}") and lists ("should have count") only
|
|
38
43
|
*
|
|
39
44
|
* Diagnostics never block generation by themselves; `sungen generate --strict`
|
|
40
45
|
* turns any collected diagnostic into a non-zero exit.
|
|
41
46
|
*/
|
|
42
47
|
|
|
43
|
-
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007' | 'SG-W008' | 'SG-W009' | 'SG-W010' | 'SG-W011' | 'SG-W012' | 'SG-W013' | 'SG-W014';
|
|
48
|
+
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007' | 'SG-W008' | 'SG-W009' | 'SG-W010' | 'SG-W011' | 'SG-W012' | 'SG-W013' | 'SG-W014' | 'SG-W015';
|
|
44
49
|
|
|
45
50
|
export interface Diagnostic {
|
|
46
51
|
code: DiagnosticCode;
|