@sun-asterisk/sungen 3.2.15 → 3.2.16-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +139 -1
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/dashboard/templates/index.html +1 -1
- package/dist/exporters/feature-parser.js +1 -1
- package/dist/exporters/feature-parser.js.map +1 -1
- package/dist/exporters/matrix/build.d.ts +43 -0
- package/dist/exporters/matrix/build.d.ts.map +1 -0
- package/dist/exporters/matrix/build.js +403 -0
- package/dist/exporters/matrix/build.js.map +1 -0
- package/dist/exporters/matrix/export.d.ts +49 -0
- package/dist/exporters/matrix/export.d.ts.map +1 -0
- package/dist/exporters/matrix/export.js +143 -0
- package/dist/exporters/matrix/export.js.map +1 -0
- package/dist/exporters/matrix/fingerprint.d.ts +20 -0
- package/dist/exporters/matrix/fingerprint.d.ts.map +1 -0
- package/dist/exporters/matrix/fingerprint.js +83 -0
- package/dist/exporters/matrix/fingerprint.js.map +1 -0
- package/dist/exporters/matrix/gates.d.ts +21 -0
- package/dist/exporters/matrix/gates.d.ts.map +1 -0
- package/dist/exporters/matrix/gates.js +295 -0
- package/dist/exporters/matrix/gates.js.map +1 -0
- package/dist/exporters/matrix/map-loader.d.ts +22 -0
- package/dist/exporters/matrix/map-loader.d.ts.map +1 -0
- package/dist/exporters/matrix/map-loader.js +191 -0
- package/dist/exporters/matrix/map-loader.js.map +1 -0
- package/dist/exporters/matrix/render-csv.d.ts +9 -0
- package/dist/exporters/matrix/render-csv.d.ts.map +1 -0
- package/dist/exporters/matrix/render-csv.js +89 -0
- package/dist/exporters/matrix/render-csv.js.map +1 -0
- package/dist/exporters/matrix/render-xlsx.d.ts +25 -0
- package/dist/exporters/matrix/render-xlsx.d.ts.map +1 -0
- package/dist/exporters/matrix/render-xlsx.js +362 -0
- package/dist/exporters/matrix/render-xlsx.js.map +1 -0
- package/dist/exporters/matrix/types.d.ts +175 -0
- package/dist/exporters/matrix/types.d.ts.map +1 -0
- package/dist/exporters/matrix/types.js +15 -0
- package/dist/exporters/matrix/types.js.map +1 -0
- package/dist/exporters/matrix/wording.d.ts +45 -0
- package/dist/exporters/matrix/wording.d.ts.map +1 -0
- package/dist/exporters/matrix/wording.js +150 -0
- package/dist/exporters/matrix/wording.js.map +1 -0
- package/dist/exporters/scenario-merger.js +1 -1
- package/dist/exporters/scenario-merger.js.map +1 -1
- package/dist/exporters/spec-parser.js +1 -1
- package/dist/exporters/spec-parser.js.map +1 -1
- package/dist/exporters/xlsx-report-builder.d.ts +1 -1
- package/dist/exporters/xlsx-report-builder.d.ts.map +1 -1
- package/dist/exporters/xlsx-report-builder.js +3 -3
- package/dist/exporters/xlsx-report-builder.js.map +1 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +6 -1
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/capability.d.ts +25 -0
- package/dist/harness/capability.d.ts.map +1 -1
- package/dist/harness/capability.js +44 -2
- package/dist/harness/capability.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +1 -1
- package/dist/orchestrator/templates/ai-src/commands/delivery.md +144 -120
- package/dist/orchestrator/templates/ai-src/config/claude.md +1 -1
- package/dist/orchestrator/templates/ai-src/config/copilot.md +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +59 -6
- package/dist/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +1 -0
- package/package.json +4 -4
- package/src/cli/commands/delivery.ts +169 -2
- package/src/dashboard/templates/index.html +1 -1
- package/src/exporters/feature-parser.ts +1 -1
- package/src/exporters/matrix/build.ts +432 -0
- package/src/exporters/matrix/export.ts +134 -0
- package/src/exporters/matrix/fingerprint.ts +49 -0
- package/src/exporters/matrix/gates.ts +325 -0
- package/src/exporters/matrix/map-loader.ts +161 -0
- package/src/exporters/matrix/render-csv.ts +91 -0
- package/src/exporters/matrix/render-xlsx.ts +356 -0
- package/src/exporters/matrix/types.ts +202 -0
- package/src/exporters/matrix/wording.ts +157 -0
- package/src/exporters/scenario-merger.ts +1 -1
- package/src/exporters/spec-parser.ts +1 -1
- package/src/exporters/xlsx-report-builder.ts +3 -2
- package/src/harness/audit.ts +7 -2
- package/src/harness/capability.ts +53 -3
- package/src/orchestrator/templates/ai-src/commands/create-test.md +1 -1
- package/src/orchestrator/templates/ai-src/commands/delivery.md +144 -120
- package/src/orchestrator/templates/ai-src/config/claude.md +1 -1
- package/src/orchestrator/templates/ai-src/config/copilot.md +1 -1
- package/src/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +59 -6
- package/src/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +1 -1
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +1 -0
|
@@ -33,6 +33,8 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.verificationScope = verificationScope;
|
|
37
|
+
exports.verificationScopeFindings = verificationScopeFindings;
|
|
36
38
|
exports.capabilitiesPath = capabilitiesPath;
|
|
37
39
|
exports.readCapabilities = readCapabilities;
|
|
38
40
|
exports.writeCapabilities = writeCapabilities;
|
|
@@ -51,6 +53,42 @@ exports.driverMeta = driverMeta;
|
|
|
51
53
|
const fs = __importStar(require("fs"));
|
|
52
54
|
const path = __importStar(require("path"));
|
|
53
55
|
const yaml_1 = require("yaml");
|
|
56
|
+
/**
|
|
57
|
+
* Effective verification scope = the recorded "test type". Explicit `verification` wins; otherwise
|
|
58
|
+
* derived from the enabled drivers. `ui` is always in scope (every project observes the UI/E2E result).
|
|
59
|
+
*/
|
|
60
|
+
function verificationScope(profile) {
|
|
61
|
+
if (profile.verification?.length) {
|
|
62
|
+
const s = new Set(['ui']);
|
|
63
|
+
for (const v of profile.verification)
|
|
64
|
+
if (v === 'api' || v === 'db' || v === 'ui')
|
|
65
|
+
s.add(v);
|
|
66
|
+
return s;
|
|
67
|
+
}
|
|
68
|
+
const s = new Set(['ui']);
|
|
69
|
+
if (profile.enabled.includes('api'))
|
|
70
|
+
s.add('api');
|
|
71
|
+
if (profile.enabled.includes('db'))
|
|
72
|
+
s.add('db');
|
|
73
|
+
return s;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Flag API/DB verification that is OUT OF the project's recorded scope: a unit whose scenarios use
|
|
77
|
+
* `@api`/`@requires:api` (or `@query`/`@requires:db`) while the project is not API/DB-testing AND the
|
|
78
|
+
* viewpoint never asked for that layer. Deterministic; advisory only (a nudge, never a hard gate).
|
|
79
|
+
*/
|
|
80
|
+
function verificationScopeFindings(featureText, viewpointText, profile) {
|
|
81
|
+
const scope = verificationScope(profile);
|
|
82
|
+
const vp = (viewpointText || '').toLowerCase();
|
|
83
|
+
const out = [];
|
|
84
|
+
if (/@api\b|@requires:\s*api\b/.test(featureText) && !scope.has('api') && !/\bapi\b/.test(vp)) {
|
|
85
|
+
out.push({ layer: 'api', message: 'uses @api / @requires:api verification, but this project is not API-testing (capabilities `enabled` has no `api`) and the viewpoint does not request API verification. Keep the oracle UI-observable, OR record API in scope (capabilities `verification: [ui, api]` or `sungen capability add api`). Reserve API/DB verify for the ≤20% of effects with no UI observation.' });
|
|
86
|
+
}
|
|
87
|
+
if (/@query\b|@requires:\s*db\b/.test(featureText) && !scope.has('db') && !/\b(database|db)\b/.test(vp)) {
|
|
88
|
+
out.push({ layer: 'db', message: 'uses @query / @requires:db verification, but this project is not DB-testing (capabilities `enabled` has no `db`) and the viewpoint does not request DB verification. Keep the oracle UI-observable, OR record DB in scope (capabilities `verification: [ui, db]` or `sungen capability add db`). Reserve API/DB verify for the ≤20% of effects with no UI observation.' });
|
|
89
|
+
}
|
|
90
|
+
return out;
|
|
91
|
+
}
|
|
54
92
|
function capabilitiesPath(cwd) {
|
|
55
93
|
return path.join(cwd, 'qa', 'capabilities.yaml');
|
|
56
94
|
}
|
|
@@ -63,6 +101,7 @@ function readCapabilities(cwd) {
|
|
|
63
101
|
return {
|
|
64
102
|
platform: typeof y.platform === 'string' ? y.platform : undefined,
|
|
65
103
|
enabled: Array.isArray(y.enabled) ? y.enabled.map(String) : [],
|
|
104
|
+
verification: Array.isArray(y.verification) ? y.verification.map(String) : undefined,
|
|
66
105
|
source: 'capabilities.yaml',
|
|
67
106
|
};
|
|
68
107
|
}
|
|
@@ -75,8 +114,11 @@ function writeCapabilities(cwd, profile) {
|
|
|
75
114
|
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
76
115
|
const body = '# Sungen capability profile — which runtime/drivers this project uses.\n' +
|
|
77
116
|
'# platform: the runtime adapter (web → Playwright, mobile → Appium). No default.\n' +
|
|
78
|
-
'# enabled: drivers turned on (added via `sungen capability add <driver>`).\n
|
|
79
|
-
|
|
117
|
+
'# enabled: drivers turned on (added via `sungen capability add <driver>`).\n' +
|
|
118
|
+
'# verification: the TEST TYPE — which layers this project verifies (`ui` = E2E only; add `api`/`db`\n' +
|
|
119
|
+
'# only if the project actually verifies via API/Database). Absent → derived from `enabled`.\n' +
|
|
120
|
+
'# This keeps an E2E-only project from getting @api/@query verification it never asked for.\n\n' +
|
|
121
|
+
(0, yaml_1.stringify)({ platform: profile.platform, enabled: profile.enabled, ...(profile.verification ? { verification: profile.verification } : {}) });
|
|
80
122
|
fs.writeFileSync(p, body, 'utf-8');
|
|
81
123
|
return p;
|
|
82
124
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"capability.js","sourceRoot":"","sources":["../../src/harness/capability.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"capability.js","sourceRoot":"","sources":["../../src/harness/capability.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,8CAUC;AASD,8DAWC;AAaD,4CAEC;AAED,4CAcC;AAED,8CAaC;AAGD,8CAQC;AAED,gCAEC;AA7HD;;;;;;;;;GASG;AACH,uCAAyB;AACzB,2CAA6B;AAC7B,+BAAsE;AAkBtE;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,OAA0B;IAC1D,IAAI,OAAO,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QACjC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QACvC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,YAAY;YAAE,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;gBAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5F,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IACvC,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChD,OAAO,CAAC,CAAC;AACX,CAAC;AAID;;;;GAIG;AACH,SAAgB,yBAAyB,CAAC,WAAmB,EAAE,aAAqB,EAAE,OAA0B;IAC9G,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/C,MAAM,GAAG,GAAyB,EAAE,CAAC;IACrC,IAAI,2BAA2B,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAC9F,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,6WAA6W,EAAE,CAAC,CAAC;IACrZ,CAAC;IACD,IAAI,4BAA4B,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QACxG,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,wWAAwW,EAAE,CAAC,CAAC;IAC/Y,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAaD,SAAgB,gBAAgB,CAAC,GAAW;IAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC;AACnD,CAAC;AAED,SAAgB,gBAAgB,CAAC,GAAW;IAC1C,MAAM,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACrF,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAA,YAAS,EAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,OAAO;YACL,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YACjE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;YAC9D,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;YACpF,MAAM,EAAE,mBAAmB;SAC5B,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAChE,CAAC;AACH,CAAC;AAED,SAAgB,iBAAiB,CAAC,GAAW,EAAE,OAA0E;IACvH,MAAM,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAChC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,IAAI,GACR,0EAA0E;QAC1E,oFAAoF;QACpF,8EAA8E;QAC9E,uGAAuG;QACvG,iGAAiG;QACjG,kGAAkG;QAClG,IAAA,gBAAa,EAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACnJ,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACnC,OAAO,CAAC,CAAC;AACX,CAAC;AAED,IAAI,QAAQ,GAAsC,IAAI,CAAC;AACvD,SAAgB,iBAAiB;IAC/B,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;IAC1D,MAAM,CAAC,GAAG,IAAA,YAAS,EAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAwD,CAAC;IACxG,MAAM,GAAG,GAA+B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;QAAE,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC;IAC9E,QAAQ,GAAG,GAAG,CAAC;IACf,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAgB,UAAU,CAAC,EAAU;IACnC,OAAO,iBAAiB,EAAE,CAAC,EAAE,CAAC,CAAC;AACjC,CAAC"}
|
|
@@ -22,7 +22,7 @@ If `spec_figma.md` exists OR the user provides a Figma URL for the PAT flow:
|
|
|
22
22
|
|
|
23
23
|
## Platform detection (do this FIRST)
|
|
24
24
|
|
|
25
|
-
Read `qa/capabilities.yaml` and check the `platform` field.
|
|
25
|
+
Read `qa/capabilities.yaml` and check the `platform` field — **and the verification scope** (`verification:` if set, else derived from `enabled`: `ui` always; `api`/`db` only if that driver is on). This is the project's recorded **test type**: an **E2E/UI-only** project (no `api`/`db`) must NOT get `@api`/`@query`/`@requires:api|db` verification unless the test-viewpoint explicitly asks for it — keep oracles UI-observable, and cap any in-scope API/DB-in-E2E verification at the **≤20% band** (see `sungen-tc-generation` → "Respect the project's verification scope"). `sungen audit` flags `VERIFICATION-OUT-OF-SCOPE`.
|
|
26
26
|
|
|
27
27
|
- If `platform: mobile` → the visual-capture step (Step 3, "Auto-detect visual source") uses the **`sungen-capture-mobile`** skill (Appium MCP live capture) **instead of** the web `sungen-capture` (figma/live). Scenarios use mobile gesture vocabulary from `sungen-mobile-gestures` (tap, swipe, long-press — not click/hover) and `accessibility-id`-first selectors; mobile-specific steps (permission dialogs, clipboard, geolocation) are valid Gherkin, not errors. The `Background:` is a **navigation recipe** (see `add-screen --reach`), not a URL path. Everything else — viewpoints, `sungen-tc-generation`, the harness/audit loop — is platform-agnostic and unchanged.
|
|
28
28
|
- If `platform: web` or field absent → follow the **web path** (all steps below, unchanged) — Figma/live capture via `sungen-capture` as documented in Step 3.
|
|
@@ -1,146 +1,170 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: delivery
|
|
3
|
-
description: "Export Gherkin
|
|
4
|
-
argument-hint: "[
|
|
3
|
+
description: "Export the Test Case & Coverage Matrix (review/manual/customer deliverable) from Gherkin + Playwright results. --legacy exports the classic per-scenario CSV/XLSX."
|
|
4
|
+
argument-hint: "[name...] [--env <locale>] [--legacy] (omit names for all; --env for locale-specific export)"
|
|
5
5
|
order: 50
|
|
6
|
-
claude-tools: "Bash, Read, AskUserQuestion"
|
|
6
|
+
claude-tools: "Bash, Read, Write, AskUserQuestion"
|
|
7
7
|
copilot-tools: "[read, execute, edit, vscode/askQuestions]"
|
|
8
|
-
codex-trigger: "Run when the user asks to EXPORT or deliver test cases
|
|
8
|
+
codex-trigger: "Run when the user asks to EXPORT or deliver test cases for QA hand-off. Final step. Do NOT use for creating or running tests."
|
|
9
9
|
---
|
|
10
10
|
## Role
|
|
11
11
|
|
|
12
|
-
You are a **QA Test Delivery Engineer**.
|
|
12
|
+
You are a **QA Test Delivery Engineer**. The deterministic `sungen delivery` CLI does all parsing,
|
|
13
|
+
gating, and rendering. Your job is (1) invoking it, (2) **proposing the Delivery Map** when a unit
|
|
14
|
+
does not have one yet — the one semantic step the CLI cannot do — and (3) walking the user through
|
|
15
|
+
review → approve → official render.
|
|
16
|
+
|
|
17
|
+
## The two delivery modes
|
|
18
|
+
|
|
19
|
+
- **Matrix (default)** — the Test Case & Coverage Matrix: one row per *delivery item* (one target +
|
|
20
|
+
one test intent + one oracle family), with each Gherkin scenario / `@cases` row preserved as a
|
|
21
|
+
collapsible *coverage variant* sub-row. No numbered step dumps. This is the reviewable /
|
|
22
|
+
manually-executable / customer-facing standard.
|
|
23
|
+
- **Legacy (`--legacy`, alias `--full`)** — the classic per-scenario-steps workbook (BM-2-901-13).
|
|
24
|
+
Use it when the user asks for the old format or a project is mid-engagement on it.
|
|
13
25
|
|
|
14
26
|
## Parameters
|
|
15
27
|
|
|
16
28
|
Parse from `$ARGUMENTS`:
|
|
17
|
-
|
|
18
|
-
-
|
|
19
|
-
|
|
20
|
-
{{^cap parallel-subagents}}
|
|
21
|
-
- **names** — zero or more screen/flow names. Empty → CLI processes all targets in `qa/screens/` + `qa/flows/`.
|
|
22
|
-
{{/cap}}
|
|
23
|
-
- **`--env <locale>`** — optional. Sets `SUNGEN_ENV=<locale>` for the run so the CLI merges `<name>.<locale>.yaml` over the base test-data and writes `<name>-testcases.<locale>.csv` / `.xlsx`. Accept `--locale <locale>` as an alias.
|
|
24
|
-
|
|
25
|
-
If `--env` is passed but no value follows, ask the user which locale to use.
|
|
29
|
+
- **names** — zero or more screen/flow/api names. Empty → all targets.
|
|
30
|
+
- **`--env <locale>`** — sets `SUNGEN_ENV=<locale>` for the run (accept `--locale` as alias).
|
|
31
|
+
- **`--legacy` / `--full`** — pass through to the CLI and skip the map flow entirely.
|
|
26
32
|
|
|
27
33
|
## Steps
|
|
28
34
|
|
|
29
|
-
### 1.
|
|
30
|
-
|
|
31
|
-
{{#cap parallel-subagents}}
|
|
32
|
-
Run via Bash (single command, no extra parsing). Prefer the local `./bin/sungen.js` when it exists — the sungen monorepo ships local-only features the global npm package doesn't have yet (multi-sheet locale aggregation, `.<env>` filename suffix, locale-aware step rendering). Fall back to `npx sungen` in downstream projects.
|
|
33
|
-
{{/cap}}
|
|
34
|
-
{{^cap parallel-subagents}}
|
|
35
|
-
Run via Bash (single command, no extra parsing):
|
|
36
|
-
{{/cap}}
|
|
35
|
+
### 1. Run the CLI once
|
|
37
36
|
|
|
38
37
|
```bash
|
|
39
|
-
# No env — local-first dispatcher:
|
|
40
|
-
{{#cap parallel-subagents}}
|
|
41
|
-
[ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <screens> || npx sungen delivery <screens>
|
|
42
|
-
{{/cap}}
|
|
43
|
-
{{^cap parallel-subagents}}
|
|
44
38
|
[ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <names> || npx sungen delivery <names>
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
39
|
+
```
|
|
40
|
+
(prepend `SUNGEN_ENV=<locale>` when `--env` was given; append `--legacy` when requested — then skip
|
|
41
|
+
to step 5.)
|
|
42
|
+
|
|
43
|
+
Three outcomes per target:
|
|
44
|
+
- **Rendered** → done, go to step 5.
|
|
45
|
+
- **"no delivery map"** → go to step 2 (propose it).
|
|
46
|
+
- **Gate findings / "not approved"** → go to step 4 (review & approve).
|
|
47
|
+
|
|
48
|
+
### 2. Propose the Delivery Map (only when missing or the user asks to regroup)
|
|
49
|
+
|
|
50
|
+
Read the unit's `.feature` (and `requirements/spec.md` for target naming). Write
|
|
51
|
+
`qa/<screens|flows|api>/<unit>/delivery/<unit>.map.yaml`:
|
|
52
|
+
|
|
53
|
+
```yaml
|
|
54
|
+
version: 1
|
|
55
|
+
unit: <unit>
|
|
56
|
+
groups:
|
|
57
|
+
- id: DI-<TARGET>-<THEME> # stable kebab id, never renumbered
|
|
58
|
+
target: login.email # ONE target: field/component dot-path, flow phrase, or METHOD /path
|
|
59
|
+
intent: <one behavior/rule this item verifies>
|
|
60
|
+
oracle: <the shared observable Pass/Fail statement>
|
|
61
|
+
category: normal | abnormal | security | nfr
|
|
62
|
+
review: proposed # ALWAYS proposed — only QA approval flips it
|
|
63
|
+
variants: [VP-VAL-001-B, VP-VAL-001-S] # VP-ids; bare id on a @cases scenario = all its rows
|
|
64
|
+
dispositions: # scenarios intentionally NOT delivered as test cases
|
|
65
|
+
VP-DATA-000: { as: excluded, reason: data-setup checklist }
|
|
66
|
+
# as: excluded | blocked | covered_elsewhere | accepted_risk
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
**Grouping rules (the aggregation signature):**
|
|
70
|
+
- One group = **one target + one intent + one oracle family**. When unsure, keep items separate —
|
|
71
|
+
the gates and QA decide, never guess-merge.
|
|
72
|
+
- MAY share a group (become coverage dimensions): equivalence partitions, boundary values,
|
|
73
|
+
different data (`@cases` rows), a different trigger with the same oracle (blur vs submit),
|
|
74
|
+
locales.
|
|
75
|
+
- MUST split: different target, intent, oracle family, category, execution mode (`@manual` vs
|
|
76
|
+
auto), test layer (`@api`/`@query`), or priority tag; sequence-sensitive flows (re-Given/When
|
|
77
|
+
after a Then) stay solo. **Different risk classes never merge** — XSS and SQL injection are
|
|
78
|
+
separate items even though both are "injection on the same field" (different risk, action,
|
|
79
|
+
and oracle family); a component's visibility rule and its action/revalidation rule are two
|
|
80
|
+
intents, not one.
|
|
81
|
+
- Every scenario must land in exactly one group **or** one disposition (Gate B enforces 100%
|
|
82
|
+
disposition). Data-setup blocks (`@manual:data-setup`) → `excluded`; SPEC-GAP placeholders →
|
|
83
|
+
`blocked`.
|
|
84
|
+
|
|
85
|
+
**Wording rules for `intent`/`oracle` (customer-facing — Gate W lints these):**
|
|
86
|
+
- Plain product language, present simple, ~10–20 words, one behavior:
|
|
87
|
+
"A user can sign in with valid credentials and is redirected to the Jobs page."
|
|
88
|
+
- Oracle = the observable outcome as a definite assertion ("The Jobs page is displayed and the
|
|
89
|
+
Logout link is visible.") — no `should`, no tester actions.
|
|
90
|
+
- NEVER: `{{tokens}}`, `[Selector]` references, DSL phrasing (`User fill/click/see`), generator
|
|
91
|
+
labels (`Setup:`/`Observable:`/`Oracle:`), or vague verbs (`handles`, `surfaces`) when a precise
|
|
92
|
+
behavior exists. Use the visible UI label (the Login button, the Email field).
|
|
93
|
+
- **Preserve the source meaning exactly** — never strengthen, weaken, or reinterpret an oracle
|
|
94
|
+
(a security assertion especially: if the source says "the password appears ONLY in the HTTPS
|
|
95
|
+
POST body", do not write "no plaintext password on the network").
|
|
96
|
+
|
|
97
|
+
**Requirement coverage (`requirements:` section, optional):** `sungen delivery` scans
|
|
98
|
+
`requirements/spec.md` for FR-/TR-/NFR- ids; ids traced by `@spec:` tags are `covered`, the rest
|
|
99
|
+
are `gap` (Gate R warning). Record the reviewed status for genuine non-gaps:
|
|
100
|
+
|
|
101
|
+
```yaml
|
|
102
|
+
requirements:
|
|
103
|
+
TR-007: { status: planned, note: Performance needs Lighthouse-style tooling }
|
|
104
|
+
TR-004: { status: partially_covered, note: client-side covered by VP-SEC-003; hashing needs DB verify }
|
|
105
|
+
# status: covered | partially_covered | covered_elsewhere | planned | gap | not_applicable
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Then validate and fix any ERROR findings:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
[ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <unit> --check || npx sungen delivery <unit> --check
|
|
54
112
|
```
|
|
55
113
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
**Options:**
|
|
89
|
-
- **Fix missing sources** (Recommended) — Print the suggested commands from CLI output and stop. User will run those commands manually, then re-invoke `/sungen:delivery`.
|
|
90
|
-
{{#cap parallel-subagents}}
|
|
91
|
-
- **Continue with available screens** — Re-run as `npx sungen delivery <screens> --continue-on-missing` to skip screens with blocking issues.
|
|
92
|
-
{{/cap}}
|
|
93
|
-
{{^cap parallel-subagents}}
|
|
94
|
-
- **Continue with available targets** — Re-run as `npx sungen delivery <names> --continue-on-missing` to skip targets with blocking issues.
|
|
95
|
-
{{/cap}}
|
|
96
|
-
- **Cancel** — Exit.
|
|
97
|
-
|
|
98
|
-
### 3. Show summary + offer next steps (on success)
|
|
99
|
-
|
|
100
|
-
Forward the CLI's summary table to the user verbatim. Then use `AskUserQuestion`:
|
|
101
|
-
|
|
102
|
-
- **Open a specific CSV** — Help user inspect one of the exported files with Read tool.
|
|
103
|
-
{{#cap parallel-subagents}}
|
|
104
|
-
- **Run tests to refresh results** — Suggest `/sungen:run-test <screen>` to update `test-results/results.json`, then re-run delivery.
|
|
105
|
-
- **Export another screen** — User can run `/sungen:delivery <other-screen>`.
|
|
106
|
-
{{/cap}}
|
|
107
|
-
{{^cap parallel-subagents}}
|
|
108
|
-
- **Run tests to refresh results** — Suggest `/sungen-run-test <name>` to update test results, then re-run delivery.
|
|
109
|
-
- **Export another target** — User can run `/sungen-delivery <other-name>`.
|
|
110
|
-
{{/cap}}
|
|
111
|
-
- **Done** — Exit.
|
|
114
|
+
### 3. Present the grouping for review
|
|
115
|
+
|
|
116
|
+
Show the user a compact summary: item count vs variant count, the groups (id · target · intent ·
|
|
117
|
+
variant count), dispositions, and every remaining `review`-level finding (oracle-shape /
|
|
118
|
+
precondition heuristics — these are exactly what QA must confirm). Then `AskUserQuestion`:
|
|
119
|
+
|
|
120
|
+
- **Approve all groups** (Recommended when the findings look right)
|
|
121
|
+
- **Adjust the grouping** — user names the groups to change; edit the map, re-run `--check`, re-present.
|
|
122
|
+
- **Preview draft first** — `sungen delivery <unit> --preview` (DRAFT watermark), then re-ask.
|
|
123
|
+
- **Cancel**
|
|
124
|
+
|
|
125
|
+
### 4. Approve + official render
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
[ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <unit> --approve || npx sungen delivery <unit> --approve
|
|
129
|
+
[ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <unit> || npx sungen delivery <unit>
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`--approve` stamps scenario fingerprints into the map — if the `.feature` changes later, Gate E
|
|
133
|
+
flips the affected groups back to review (drift). Approved + unchanged groups render without
|
|
134
|
+
re-asking.
|
|
135
|
+
|
|
136
|
+
### 5. Show summary + offer next steps
|
|
137
|
+
|
|
138
|
+
Forward the CLI summary verbatim (Items = review rows; **Variants = execution units — progress is
|
|
139
|
+
counted in variants, never items**). Then `AskUserQuestion`:
|
|
140
|
+
|
|
141
|
+
- **Open the workbook** — inspect `qa/deliverables/<unit>-testcases.xlsx` (Testcases sheet:
|
|
142
|
+
collapse outline level 1 for the customer view; Coverage sheet: target × category grid + gaps).
|
|
143
|
+
- **Run tests to refresh results** — `/sungen:run-test <unit>`, then re-run delivery.
|
|
144
|
+
- **Export the legacy workbook too** — `sungen delivery <unit> --legacy`.
|
|
145
|
+
- **Done**
|
|
112
146
|
|
|
113
147
|
## Important notes
|
|
114
148
|
|
|
115
|
-
- **Do NOT parse
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
- **
|
|
149
|
+
- **Do NOT parse/duplicate what the CLI computes** — gates recompute mode/layer/priority/oracle
|
|
150
|
+
shapes from the `.feature`; the map only carries the semantic fields (target/intent/oracle/
|
|
151
|
+
category/grouping).
|
|
152
|
+
- **Never write `review: approved` yourself** — only `--approve` after the user agreed. Proposing
|
|
153
|
+
with `review: approved` defeats the QA review this format exists for.
|
|
154
|
+
- **Do NOT modify feature/test-data files during delivery** — read-only except the map file.
|
|
155
|
+
- The map is committed and stable: adding a group never renumbers others; deleting scenarios →
|
|
156
|
+
update the map (Gate B lists exactly what's unmapped/stale).
|
|
119
157
|
|
|
120
|
-
## CLI
|
|
158
|
+
## CLI reference
|
|
121
159
|
|
|
122
160
|
```
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
[--skip-preflight] Skip pre-flight checks (not recommended)
|
|
130
|
-
{{#cap parallel-subagents}}
|
|
131
|
-
[--continue-on-missing] Skip screens with blocking misses
|
|
132
|
-
{{/cap}}
|
|
133
|
-
{{^cap parallel-subagents}}
|
|
134
|
-
[--continue-on-missing] Skip targets with blocking misses
|
|
135
|
-
{{/cap}}
|
|
136
|
-
|
|
137
|
-
# Locale-aware export (env var, not a CLI flag):
|
|
138
|
-
{{#cap parallel-subagents}}
|
|
139
|
-
SUNGEN_ENV=<locale> sungen delivery [screens...]
|
|
140
|
-
{{/cap}}
|
|
141
|
-
{{^cap parallel-subagents}}
|
|
142
|
-
SUNGEN_ENV=<locale> sungen delivery [names...]
|
|
143
|
-
{{/cap}}
|
|
144
|
-
→ reads <name>.<locale>.yaml overlay, picks <name>-test-result.<locale>.json,
|
|
145
|
-
writes <name>-testcases.<locale>.csv / .xlsx
|
|
161
|
+
sungen delivery [names...] # matrix (default; needs the map)
|
|
162
|
+
--check # gates only — validate the map, write nothing
|
|
163
|
+
--approve [DI-a,DI-b] # flip proposed→approved (+ stamp fingerprints); all groups when bare
|
|
164
|
+
--preview # render despite review findings (DRAFT watermark)
|
|
165
|
+
--legacy | --full # classic per-scenario workbook (no map needed)
|
|
166
|
+
--skip-preflight | --continue-on-missing | --env <env> # as before
|
|
146
167
|
```
|
|
168
|
+
|
|
169
|
+
Outputs: `qa/deliverables/<unit>-testcases.xlsx` (Testcases + Coverage sheets) + `.csv`
|
|
170
|
+
(flat, `Level` column `item|variant`). Legacy mode writes the classic files instead.
|
|
@@ -30,7 +30,7 @@ You generate 3 files for sungen — a Gherkin compiler that produces Playwright
|
|
|
30
30
|
| `/sungen:create-data-test <name>` | Generate standardized test-data (valid/boundary/invalid + CHK trace) from the Data Factory catalog; no name = all units (asks to confirm) |
|
|
31
31
|
| `/sungen:review <name>` | Score syntax, coverage, viewpoint quality (auto-detects screen or flow) |
|
|
32
32
|
| `/sungen:run-test <name>` | Generate `selectors.yaml`, compile, run, auto-fix (auto-detects screen or flow) |
|
|
33
|
-
| `/sungen:delivery [name...]` | Export
|
|
33
|
+
| `/sungen:delivery [name...]` | Export the Test Case & Coverage Matrix (review/manual/customer deliverable; `--legacy` = classic CSV; all screens if no arg) |
|
|
34
34
|
| `/sungen:locale <name> <locale>` | Bootstrap i18n for a screen — audit selectors, detect locale switch, generate overlay (run before `/sungen:run-test --env <locale>`) |
|
|
35
35
|
|
|
36
36
|
**Screen path:** add-screen → create-test → review → run-test → delivery.
|
|
@@ -30,7 +30,7 @@ You generate 3 files for sungen — a Gherkin compiler that produces Playwright
|
|
|
30
30
|
| `/sungen-create-data-test <name>` | Generate standardized test-data (valid/boundary/invalid + CHK trace) from the Data Factory catalog; no name = all units (asks to confirm) |
|
|
31
31
|
| `/sungen-review <name>` | Score syntax, coverage, viewpoint quality (auto-detects screen or flow) |
|
|
32
32
|
| `/sungen-run-test <name>` | Generate `selectors.yaml`, compile, run, auto-fix (auto-detects screen or flow) |
|
|
33
|
-
| `/sungen-delivery [name...]` | Export
|
|
33
|
+
| `/sungen-delivery [name...]` | Export the Test Case & Coverage Matrix (review/manual/customer deliverable; `--legacy` = classic CSV; all screens if no arg) |
|
|
34
34
|
| `/sungen-locale <name> <locale>` | Bootstrap i18n for a screen — audit selectors, detect locale switch, generate overlay (run before `/sungen-run-test --env <locale>`) |
|
|
35
35
|
|
|
36
36
|
**Screen path:** add-screen → create-test → review → run-test → delivery.
|
|
@@ -1,21 +1,74 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: sungen-delivery
|
|
3
|
-
description: 'Export
|
|
3
|
+
description: 'Export the Test Case & Coverage Matrix (or the legacy per-scenario CSV) from Gherkin + Playwright results. Auto-loaded by delivery command.'
|
|
4
4
|
user-invocable: false
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
## Purpose
|
|
8
8
|
|
|
9
|
-
Export test cases from Sungen
|
|
9
|
+
Export test cases from Sungen units to the QA deliverable. Two modes:
|
|
10
10
|
|
|
11
|
-
**
|
|
11
|
+
- **Matrix (default)** — the **Test Case & Coverage Matrix**: one row per *delivery item* (one
|
|
12
|
+
target + one test intent + one oracle family); scenarios / `@cases` rows stay as collapsible
|
|
13
|
+
*coverage variant* sub-rows. Built for review, manual execution, customer hand-off, and
|
|
14
|
+
reporting — no numbered step dumps.
|
|
15
|
+
- **Legacy (`--legacy`/`--full`)** — the classic per-scenario BM-2-901-13 workbook (documented
|
|
16
|
+
in the second half of this skill; unchanged).
|
|
12
17
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
18
|
+
**This skill delegates all heavy work to the `sungen delivery` CLI.** The CLI is the single source
|
|
19
|
+
of truth for parsing + gating. The AI's only semantic contribution is **proposing the Delivery
|
|
20
|
+
Map** (grouping) — everything else is: invoke the CLI, show output verbatim, help on failures.
|
|
16
21
|
|
|
17
22
|
---
|
|
18
23
|
|
|
24
|
+
## Matrix mode — the Delivery Map
|
|
25
|
+
|
|
26
|
+
`qa/<screens|flows|api>/<unit>/delivery/<unit>.map.yaml` — committed, AI-proposed
|
|
27
|
+
(`review: proposed`), gate-validated, QA-approved (`sungen delivery --approve` stamps scenario
|
|
28
|
+
fingerprints). Schema + grouping rules live in the delivery command instructions; the normative
|
|
29
|
+
spec is `docs/spec/delivery-coverage-matrix-spec.md`.
|
|
30
|
+
|
|
31
|
+
**Gates** (CLI `--check`): A source (VP-ids unique, oracle present, Background setup-only) ·
|
|
32
|
+
B mapping (every scenario in exactly one group XOR one disposition) · C aggregation (mode/layer/
|
|
33
|
+
priority recomputed and equal within a group — heuristic oracle-shape/precondition mismatches are
|
|
34
|
+
review-level, silenced once approved and unchanged) · D executability (precondition · condition+
|
|
35
|
+
data · trigger · oracle all renderable; every `{{var}}` resolves; **no template token may survive
|
|
36
|
+
into a rendered cell** — test-data cross-references are resolved for display) · E drift
|
|
37
|
+
(fingerprint mismatch → back to review) · G review state (proposed groups block the official
|
|
38
|
+
render; `--preview` renders a DRAFT watermark) · R requirement coverage (spec FR/TR/NFR ids with
|
|
39
|
+
no trace and no `requirements:` status → warning) · W wording lint (map intent/oracle containing
|
|
40
|
+
tokens, `[Selector]` refs, DSL phrasing, or generator labels → warning).
|
|
41
|
+
|
|
42
|
+
**Wording normalization (deterministic, after semantic normalization):** DSL steps render as
|
|
43
|
+
controlled manual-test English without changing meaning — actions in the imperative (`User fill
|
|
44
|
+
[Email] field with X` → `Enter X in the Email field.`), expected results as observable assertions
|
|
45
|
+
(`User see [Jobs] page` → `The Jobs page is displayed.`), preconditions as states (`The user is
|
|
46
|
+
signed out.`). Manual `# Tester verifies:` labels become structured fields: `Setup:` →
|
|
47
|
+
Precondition, `Action:` → Action, `Observable:` → Expected Result, `Oracle:` → a separate
|
|
48
|
+
`Verification method:` line. Sequence-sensitive flows keep event order: actions numbered with
|
|
49
|
+
mid-flow assertions inline as `Verify: …`; only the final Then block is the Expected Result.
|
|
50
|
+
Empty test values render as `(empty)`.
|
|
51
|
+
|
|
52
|
+
**Workbook**: `Testcases` sheet — parent rows + outline-level-1 variant sub-rows for **every**
|
|
53
|
+
item (single-variant included: the sub-row carries the source VP-id, resolved data, and the
|
|
54
|
+
result/evidence entry). Collapse outline for the customer view, expand to execute. Variant Result
|
|
55
|
+
cells have a dropdown (Passed/Failed/Blocked/Pending/N/A) and the parent Result is a **live Excel
|
|
56
|
+
formula** over its children (failed→blocked→pending→partial→passed, e.g. `2/3 Passed · 1 Failed`)
|
|
57
|
+
— a parent can never contradict its variants, even after manual edits. ID + Target columns are
|
|
58
|
+
frozen; dates are ISO (`2026-08-04`). `Coverage` sheet — requirement coverage table (every FR/TR/
|
|
59
|
+
NFR id with an explicit status), target × category grid with explicit `—` gaps, dispositions,
|
|
60
|
+
manifest. CSV is flat with a `Level` column (`item`/`variant`) + a requirement-coverage appendix.
|
|
61
|
+
`delivery_item_count` ≠ progress — variants are the execution metric.
|
|
62
|
+
|
|
63
|
+
**Authoring guidance the matrix rewards** (create-test side): payload/provider matrices (SQLi
|
|
64
|
+
payload lists, OAuth provider sets) belong in `@cases` datasets so each case is an atomic,
|
|
65
|
+
independently-reportable variant; keep dataset `case:` labels short and stable (`CHK-EMAIL-I1`),
|
|
66
|
+
with descriptions in other columns — the label is part of the variant's identity.
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## Legacy mode (--legacy / --full)
|
|
71
|
+
|
|
19
72
|
## Architecture
|
|
20
73
|
|
|
21
74
|
```
|
|
@@ -66,7 +66,7 @@ Use these when repairing GATE/DEPTH findings for the hard viewpoints (cart/detai
|
|
|
66
66
|
2. **Stop when** `gateStatus == PASS` AND `findings` empty — or budget exhausted.
|
|
67
67
|
3. **Never fake a pass.** A shallow `see [Cart] page` does not satisfy `cart-correctness`. If a gap is genuinely cross-screen or needs capabilities the DSL lacks (e.g. capture an element value to compare elsewhere), **report it as a residual gap / flow item** instead of forcing a green gate.
|
|
68
68
|
4. **EP/data families are OK.** A `duplicates` cluster with `sameDataLikely=false` is an intentional equivalence-partition family (e.g. many invalid-email cases) — keep it; only collapse `sameDataLikely=true` exact duplicates.
|
|
69
|
-
5. **Advisory findings — surface, don't gate.** `MANUAL-REASON-MISMATCH` → fix the scenario's `@manual:Mx` code (so the planner recommends the right driver) during repair. `CAPABILITY-SUGGESTION` → **present it to the user as a next-step option** (e.g. "N @manual could be automated — `sungen capability add api db`?"), **recommend-only — never auto-install**. Neither fails the gate.
|
|
69
|
+
5. **Advisory findings — surface, don't gate.** `MANUAL-REASON-MISMATCH` → fix the scenario's `@manual:Mx` code (so the planner recommends the right driver) during repair. `CAPABILITY-SUGGESTION` → **present it to the user as a next-step option** (e.g. "N @manual could be automated — `sungen capability add api db`?"), **recommend-only — never auto-install**. `VERIFICATION-OUT-OF-SCOPE (api|db)` → the unit uses `@api`/`@query`/`@requires:api|db` verification the project's recorded **test type** never opted into (E2E/UI-only per `capabilities.yaml`, and the viewpoint doesn't ask for it). During repair: **rewrite the oracle to be UI-observable** (or downgrade the deep check to an `(optional deep check: …)` note); only keep the API/DB verification if the project genuinely tests that layer — then record it (`verification: [ui, api]` / `sungen capability add`). Keep API/DB-in-E2E to the ≤20% band. Neither of these fails the gate.
|
|
70
70
|
|
|
71
71
|
## Discovery / fallback tree (when input is limited)
|
|
72
72
|
|
|
@@ -328,6 +328,7 @@ Security: [S1 – admin only]
|
|
|
328
328
|
Gate (`SG-W008`, both directions): a `@manual` with Gherkin steps but no `Then` fires — comments (even labeled `Oracle:` lines) can NOT substitute for the missing `Then`. A `@manual` with NO steps ALSO fires when its comments lack ≥1 numbered action line + ≥1 labeled `Observable:`/`Oracle:`/`Expected:` line — prose paragraphs and rationale-only comments export EMPTY Steps/Expected cells.
|
|
329
329
|
In shape A the observable/oracle lines MUST carry their **label** (`Observable:` / `Oracle:` / `Expected:`) — the delivery parser extracts Expected results **by label**; unlabeled prose all lands in the Steps column and the Expected cell exports EMPTY. And the numbered action steps MUST precede them — without the actions the tester has an oracle but no procedure.
|
|
330
330
|
**Oracle layer must match the unit** — a screen-unit manual defaults to a **UI oracle**: phrase the expected as what the tester SEES (on-screen message, list/table state after reload). If the business effect IS UI-observable, do NOT cite API responses or DB state as the oracle — "server rejects the request" is verified by *the error message on screen + no new row after reload*, not by reading the API response. Only when the effect has NO UI observation (mail content, stored hash, request count, response-body leak) may the oracle use another layer — then: (a) put the **layer label inside the line**: `Oracle: UI — …` / `Oracle: API — …` / `Oracle: DB — …` so QA knows the tooling needed from the deliverable alone; (b) tag `@manual:M2` so the capability planner recommends the api/db driver; (c) prefer the automation-ready `@requires:api`/`@query` form when the steps are expressible. A deep engineering check may ride along as `(optional deep check: …)` appended to the UI oracle — it must never REPLACE a UI-observable oracle.
|
|
331
|
+
**Respect the project's VERIFICATION SCOPE (the recorded test type).** Read `qa/capabilities.yaml` — the effective scope is `verification:` if set, else derived from `enabled` (`ui` always; `api`/`db` only if that driver is on). If the project is **E2E/UI-only** (no `api`/`db`) **and the test-viewpoint does not ask for API/DB verification**, do **NOT** author `@api`/`@query`/`@requires:api|db` verification — keep the oracle **UI-observable**, or leave the deep check as an `(optional deep check: …)` note. Author API/DB verification only when the scope (or an explicit viewpoint) calls for it, and even then keep the API/DB-in-E2E combination to a **≤20% band** of scenarios (the effects with no UI observation) — never the default. `sungen audit` reports `VERIFICATION-OUT-OF-SCOPE` when a unit uses API/DB verify the project never opted into.
|
|
331
332
|
Use this comment shape:
|
|
332
333
|
```gherkin
|
|
333
334
|
@high @manual
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sun-asterisk/sungen",
|
|
3
|
-
"version": "3.2.
|
|
3
|
+
"version": "3.2.16-beta.2",
|
|
4
4
|
"description": "Deterministic E2E Test Compiler - Gherkin + Selectors → Playwright tests",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
"copy-templates": "mkdir -p dist/generators/test-generator/adapters/playwright/templates/steps && mkdir -p dist/generators/test-generator/templates && mkdir -p dist/orchestrator/templates && mkdir -p dist/dashboard/templates && cp -r src/generators/test-generator/adapters/playwright/templates/*.hbs dist/generators/test-generator/adapters/playwright/templates/ 2>/dev/null || true && cp -r src/generators/test-generator/adapters/playwright/templates/steps dist/generators/test-generator/adapters/playwright/templates/ && mkdir -p dist/generators/test-generator/adapters/appium/templates/steps && cp -r src/generators/test-generator/adapters/appium/templates/*.hbs dist/generators/test-generator/adapters/appium/templates/ 2>/dev/null || true && cp -r src/generators/test-generator/adapters/appium/templates/steps dist/generators/test-generator/adapters/appium/templates/ && cp src/generators/test-generator/templates/*.hbs dist/generators/test-generator/templates/ 2>/dev/null || true && cp -r src/orchestrator/templates/* dist/orchestrator/templates/ && cp src/dashboard/templates/index.html dist/dashboard/templates/index.html && mkdir -p dist/harness/catalog && cp src/harness/catalog/*.yaml dist/harness/catalog/",
|
|
13
13
|
"build:dashboard": "cd ../../dashboard && npm install --silent && npm run build && cd - && cp ../../dashboard/dist/index.html src/dashboard/templates/index.html",
|
|
14
14
|
"dev": "tsx src/cli/index.ts",
|
|
15
|
-
"test": "tsx tests/golden/run.ts && tsx tests/audit/run.ts && tsx tests/ingest/run.ts && tsx tests/eval/run.ts && tsx tests/exporter/run.ts && tsx tests/exporter/feature-parser-category.run.ts && tsx tests/exporter/api-detail-sheet.run.ts && tsx tests/exporter/overview-sheet.run.ts && tsx tests/exporter/delivery-divider-tcid.run.ts && tsx tests/exporter/note-expected-actual.run.ts && tsx tests/exporter/test-data-vars.run.ts && tsx tests/exporter/mobile-app-id.run.ts && tsx tests/exporter/api-testcase-cells.run.ts && tsx tests/exporter/delivery-cases-result.run.ts && tsx tests/exporter/precondition-auth-role.run.ts && tsx tests/exporter/report-builder-parity.run.ts && tsx tests/exporter/delivery-preflight-testdata.run.ts && tsx tests/exporter/delivery-preflight-selectors.run.ts && tsx tests/exporter/delivery-prerun-overview.run.ts && tsx tests/exporter/delivery-locale-env-discovery.run.ts && tsx tests/exporter/manual-testcase-testdata.run.ts && tsx tests/dashboard/api-flows-discovery.run.ts && tsx tests/dashboard/status-results-fallback.run.ts && tsx tests/codegen/verb-ref-masking.run.ts && tsx tests/codegen/table-aria-hidden-locators.run.ts && tsx tests/codegen/steps-base-not-compiled.run.ts && tsx tests/codegen/manual-oracle-gate.run.ts && tsx tests/codegen/locator-value-qualified-assertions.run.ts && tsx tests/codegen/table-match-data-column-mapping.run.ts && tsx tests/codegen/param-binding-native-type.run.ts && tsx tests/codegen/table-row-scope-redeclaration.run.ts && tsx tests/codegen/table-row-label-resolution.run.ts && tsx tests/api-runtime/base-path-url-join.run.ts && tsx tests/api-runtime/file-upload-multipart.run.ts && tsx tests/api-runtime/schema-assertion.run.ts && tsx tests/runtime/test-data-cross-ref.run.ts && tsx tests/runtime/test-data-array-projection.run.ts && tsx tests/runtime/env-secret-interpolation.run.ts && tsx tests/capabilities/run.ts && tsx tests/openapi/run.ts && tsx tests/api-field-coverage/run.ts && tsx tests/packaging/run.ts && tsx tests/generate-hint/run.ts && tsx tests/template-assertion/run.ts && tsx tests/journey/run.ts && tsx tests/harness/serial-cascade.run.ts && tsx tests/ai-skills/cross-assistant-orphan.run.ts && tsx tests/init/codex-mcp-config.run.ts && tsx src/orchestrator/ai-skills/golden-skills.test.ts && tsx tests/db-runtime/sql-placeholder-rewrite.run.ts && tsx tests/db-runtime/fallback-flow-codified.run.ts && tsx tests/db-runtime/cosmos-engine.run.ts && tsx tests/db-runtime/mongodb-engine.run.ts && tsx tests/db-runtime/dynamodb-engine.run.ts && tsx tests/db-runtime/mysql-integration.opt-in.run.ts && tsx tests/db-runtime/mongodb-integration.opt-in.run.ts && tsx tests/db-runtime/dynamodb-integration.opt-in.run.ts",
|
|
15
|
+
"test": "tsx tests/golden/run.ts && tsx tests/audit/run.ts && tsx tests/ingest/run.ts && tsx tests/eval/run.ts && tsx tests/exporter/run.ts && tsx tests/exporter/delivery-matrix.run.ts && tsx tests/exporter/feature-parser-category.run.ts && tsx tests/exporter/api-detail-sheet.run.ts && tsx tests/exporter/overview-sheet.run.ts && tsx tests/exporter/delivery-divider-tcid.run.ts && tsx tests/exporter/note-expected-actual.run.ts && tsx tests/exporter/test-data-vars.run.ts && tsx tests/exporter/mobile-app-id.run.ts && tsx tests/exporter/api-testcase-cells.run.ts && tsx tests/exporter/delivery-cases-result.run.ts && tsx tests/exporter/precondition-auth-role.run.ts && tsx tests/exporter/report-builder-parity.run.ts && tsx tests/exporter/delivery-preflight-testdata.run.ts && tsx tests/exporter/delivery-preflight-selectors.run.ts && tsx tests/exporter/delivery-prerun-overview.run.ts && tsx tests/exporter/delivery-locale-env-discovery.run.ts && tsx tests/exporter/manual-testcase-testdata.run.ts && tsx tests/dashboard/api-flows-discovery.run.ts && tsx tests/dashboard/status-results-fallback.run.ts && tsx tests/codegen/verb-ref-masking.run.ts && tsx tests/codegen/table-aria-hidden-locators.run.ts && tsx tests/codegen/steps-base-not-compiled.run.ts && tsx tests/codegen/manual-oracle-gate.run.ts && tsx tests/codegen/locator-value-qualified-assertions.run.ts && tsx tests/codegen/table-match-data-column-mapping.run.ts && tsx tests/codegen/param-binding-native-type.run.ts && tsx tests/codegen/table-row-scope-redeclaration.run.ts && tsx tests/codegen/table-row-label-resolution.run.ts && tsx tests/api-runtime/base-path-url-join.run.ts && tsx tests/api-runtime/file-upload-multipart.run.ts && tsx tests/api-runtime/schema-assertion.run.ts && tsx tests/runtime/test-data-cross-ref.run.ts && tsx tests/runtime/test-data-array-projection.run.ts && tsx tests/runtime/env-secret-interpolation.run.ts && tsx tests/capabilities/run.ts && tsx tests/openapi/run.ts && tsx tests/api-field-coverage/run.ts && tsx tests/packaging/run.ts && tsx tests/generate-hint/run.ts && tsx tests/template-assertion/run.ts && tsx tests/journey/run.ts && tsx tests/harness/serial-cascade.run.ts && tsx tests/ai-skills/cross-assistant-orphan.run.ts && tsx tests/init/codex-mcp-config.run.ts && tsx src/orchestrator/ai-skills/golden-skills.test.ts && tsx tests/db-runtime/sql-placeholder-rewrite.run.ts && tsx tests/db-runtime/fallback-flow-codified.run.ts && tsx tests/db-runtime/cosmos-engine.run.ts && tsx tests/db-runtime/mongodb-engine.run.ts && tsx tests/db-runtime/dynamodb-engine.run.ts && tsx tests/db-runtime/mysql-integration.opt-in.run.ts && tsx tests/db-runtime/mongodb-integration.opt-in.run.ts && tsx tests/db-runtime/dynamodb-integration.opt-in.run.ts",
|
|
16
16
|
"test:update": "tsx tests/golden/run.ts --update && tsx tests/audit/run.ts --update && tsx tests/ingest/run.ts --update",
|
|
17
17
|
"prepublishOnly": "npm run build:dashboard && npm run build"
|
|
18
18
|
},
|
|
@@ -39,8 +39,8 @@
|
|
|
39
39
|
"@babel/types": "^7.28.5",
|
|
40
40
|
"@cucumber/gherkin": "^37.0.0",
|
|
41
41
|
"@cucumber/messages": "^31.0.0",
|
|
42
|
-
"@sungen/driver-data-factory": "3.2.
|
|
43
|
-
"@sungen/driver-ui": "3.2.
|
|
42
|
+
"@sungen/driver-data-factory": "3.2.16-beta.2",
|
|
43
|
+
"@sungen/driver-ui": "3.2.16-beta.2",
|
|
44
44
|
"chalk": "^5.6.2",
|
|
45
45
|
"commander": "^14.0.2",
|
|
46
46
|
"dotenv": "^17.2.3",
|