@sun-asterisk/sungen 3.2.15 → 3.2.16-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/dist/cli/commands/delivery.d.ts.map +1 -1
  2. package/dist/cli/commands/delivery.js +139 -1
  3. package/dist/cli/commands/delivery.js.map +1 -1
  4. package/dist/dashboard/templates/index.html +1 -1
  5. package/dist/exporters/feature-parser.js +1 -1
  6. package/dist/exporters/feature-parser.js.map +1 -1
  7. package/dist/exporters/matrix/build.d.ts +43 -0
  8. package/dist/exporters/matrix/build.d.ts.map +1 -0
  9. package/dist/exporters/matrix/build.js +403 -0
  10. package/dist/exporters/matrix/build.js.map +1 -0
  11. package/dist/exporters/matrix/export.d.ts +49 -0
  12. package/dist/exporters/matrix/export.d.ts.map +1 -0
  13. package/dist/exporters/matrix/export.js +143 -0
  14. package/dist/exporters/matrix/export.js.map +1 -0
  15. package/dist/exporters/matrix/fingerprint.d.ts +20 -0
  16. package/dist/exporters/matrix/fingerprint.d.ts.map +1 -0
  17. package/dist/exporters/matrix/fingerprint.js +83 -0
  18. package/dist/exporters/matrix/fingerprint.js.map +1 -0
  19. package/dist/exporters/matrix/gates.d.ts +21 -0
  20. package/dist/exporters/matrix/gates.d.ts.map +1 -0
  21. package/dist/exporters/matrix/gates.js +295 -0
  22. package/dist/exporters/matrix/gates.js.map +1 -0
  23. package/dist/exporters/matrix/map-loader.d.ts +22 -0
  24. package/dist/exporters/matrix/map-loader.d.ts.map +1 -0
  25. package/dist/exporters/matrix/map-loader.js +191 -0
  26. package/dist/exporters/matrix/map-loader.js.map +1 -0
  27. package/dist/exporters/matrix/render-csv.d.ts +9 -0
  28. package/dist/exporters/matrix/render-csv.d.ts.map +1 -0
  29. package/dist/exporters/matrix/render-csv.js +89 -0
  30. package/dist/exporters/matrix/render-csv.js.map +1 -0
  31. package/dist/exporters/matrix/render-xlsx.d.ts +25 -0
  32. package/dist/exporters/matrix/render-xlsx.d.ts.map +1 -0
  33. package/dist/exporters/matrix/render-xlsx.js +362 -0
  34. package/dist/exporters/matrix/render-xlsx.js.map +1 -0
  35. package/dist/exporters/matrix/types.d.ts +175 -0
  36. package/dist/exporters/matrix/types.d.ts.map +1 -0
  37. package/dist/exporters/matrix/types.js +15 -0
  38. package/dist/exporters/matrix/types.js.map +1 -0
  39. package/dist/exporters/matrix/wording.d.ts +45 -0
  40. package/dist/exporters/matrix/wording.d.ts.map +1 -0
  41. package/dist/exporters/matrix/wording.js +150 -0
  42. package/dist/exporters/matrix/wording.js.map +1 -0
  43. package/dist/exporters/scenario-merger.js +1 -1
  44. package/dist/exporters/scenario-merger.js.map +1 -1
  45. package/dist/exporters/spec-parser.js +1 -1
  46. package/dist/exporters/spec-parser.js.map +1 -1
  47. package/dist/exporters/xlsx-report-builder.d.ts +1 -1
  48. package/dist/exporters/xlsx-report-builder.d.ts.map +1 -1
  49. package/dist/exporters/xlsx-report-builder.js +3 -3
  50. package/dist/exporters/xlsx-report-builder.js.map +1 -1
  51. package/dist/harness/audit.d.ts.map +1 -1
  52. package/dist/harness/audit.js +6 -1
  53. package/dist/harness/audit.js.map +1 -1
  54. package/dist/harness/capability.d.ts +25 -0
  55. package/dist/harness/capability.d.ts.map +1 -1
  56. package/dist/harness/capability.js +44 -2
  57. package/dist/harness/capability.js.map +1 -1
  58. package/dist/orchestrator/templates/ai-src/commands/create-test.md +1 -1
  59. package/dist/orchestrator/templates/ai-src/commands/delivery.md +144 -120
  60. package/dist/orchestrator/templates/ai-src/config/claude.md +1 -1
  61. package/dist/orchestrator/templates/ai-src/config/copilot.md +1 -1
  62. package/dist/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +59 -6
  63. package/dist/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +1 -1
  64. package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +1 -0
  65. package/package.json +4 -4
  66. package/src/cli/commands/delivery.ts +169 -2
  67. package/src/dashboard/templates/index.html +1 -1
  68. package/src/exporters/feature-parser.ts +1 -1
  69. package/src/exporters/matrix/build.ts +432 -0
  70. package/src/exporters/matrix/export.ts +134 -0
  71. package/src/exporters/matrix/fingerprint.ts +49 -0
  72. package/src/exporters/matrix/gates.ts +325 -0
  73. package/src/exporters/matrix/map-loader.ts +161 -0
  74. package/src/exporters/matrix/render-csv.ts +91 -0
  75. package/src/exporters/matrix/render-xlsx.ts +356 -0
  76. package/src/exporters/matrix/types.ts +202 -0
  77. package/src/exporters/matrix/wording.ts +157 -0
  78. package/src/exporters/scenario-merger.ts +1 -1
  79. package/src/exporters/spec-parser.ts +1 -1
  80. package/src/exporters/xlsx-report-builder.ts +3 -2
  81. package/src/harness/audit.ts +7 -2
  82. package/src/harness/capability.ts +53 -3
  83. package/src/orchestrator/templates/ai-src/commands/create-test.md +1 -1
  84. package/src/orchestrator/templates/ai-src/commands/delivery.md +144 -120
  85. package/src/orchestrator/templates/ai-src/config/claude.md +1 -1
  86. package/src/orchestrator/templates/ai-src/config/copilot.md +1 -1
  87. package/src/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +59 -6
  88. package/src/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +1 -1
  89. package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +1 -0
@@ -0,0 +1,157 @@
1
+ /**
2
+ * Wording normalization — deterministic rendering step AFTER semantic
3
+ * normalization (review feedback §10): turn sungen DSL steps into controlled
4
+ * manual-test English without changing the target, condition, trigger,
5
+ * precondition, oracle, or trace.
6
+ *
7
+ * - Actions render in the imperative: "User fill [Email] field with X"
8
+ * → "Enter X in the Email field."
9
+ * - Expected results render as observable assertions (never tester actions):
10
+ * "User see [Jobs] page" → "The Jobs page is displayed."
11
+ * - Manual `# Tester verifies:` labels (Setup:/Action:/Observable:/Oracle:)
12
+ * become structured fields instead of prose: Setup → precondition,
13
+ * Action → action, Observable → expected, Oracle → verification method.
14
+ */
15
+
16
+ // `[Email] field` → `Email field` (the visible label + its element type).
17
+ function deRef(text: string): string {
18
+ return text.replace(/\[([^\]]+)\]/g, '$1');
19
+ }
20
+
21
+ function sentence(text: string): string {
22
+ let s = text.trim().replace(/\s+/g, ' ');
23
+ if (!s) return s;
24
+ s = s.charAt(0).toUpperCase() + s.slice(1);
25
+ if (!/[.!?…]$/.test(s)) s += '.';
26
+ return s;
27
+ }
28
+
29
+ /**
30
+ * Render one action step in the imperative. Pattern table covers the common
31
+ * sungen step verbs; anything unmatched falls back to actor-stripped text —
32
+ * still readable, never a raw `User fill`.
33
+ */
34
+ export function renderAction(raw: string): string {
35
+ let s = raw.trim().replace(/^(User|The user)\s+/i, '');
36
+
37
+ const rules: Array<[RegExp, (m: RegExpMatchArray) => string]> = [
38
+ // fill [X] field with V
39
+ [/^fills? \[([^\]]+)\][a-z ]* with (.+)$/i, (m) => `Enter ${m[2]} in the ${m[1]} field`],
40
+ // clear [X] field
41
+ [/^clears? \[([^\]]+)\](.*)$/i, (m) => `Clear the ${m[1]}${m[2] || ' field'}`],
42
+ // click [X] <type>
43
+ [/^clicks? \[([^\]]+)\]\s*(\w+)?$/i, (m) => `Click the ${m[1]}${m[2] ? ` ${m[2]}` : ''}`],
44
+ // press <Key> at/in [X] field
45
+ [/^press(?:es)? (.+?) (?:at|in|inside) \[([^\]]+)\](?: field)?$/i, (m) => `Press ${m[1]} in the ${m[2]} field`],
46
+ [/^press(?:es)? (.+)$/i, (m) => `Press ${m[1]}`],
47
+ // select V in/from [X] dropdown
48
+ [/^selects? (.+?) (?:in|from) \[([^\]]+)\]\s*(\w+)?$/i, (m) => `Select ${m[1]} in the ${m[2]}${m[3] ? ` ${m[3]}` : ''}`],
49
+ // check/uncheck [X] checkbox
50
+ [/^(un)?checks? \[([^\]]+)\]\s*(\w+)?$/i, (m) => `${m[1] ? 'Uncheck' : 'Check'} the ${m[2]}${m[3] ? ` ${m[3]}` : ''}`],
51
+ // hover [X]
52
+ [/^hovers? (?:over )?\[([^\]]+)\]\s*(\w+)?$/i, (m) => `Hover over the ${m[1]}${m[2] ? ` ${m[2]}` : ''}`],
53
+ // upload V to [X]
54
+ [/^uploads? (.+?) (?:to|into) \[([^\]]+)\]\s*(\w+)?$/i, (m) => `Upload ${m[1]} to the ${m[2]}${m[3] ? ` ${m[3]}` : ''}`],
55
+ // is on [X] page (as an action = navigate)
56
+ [/^is on \[([^\]]+)\] page(.*)$/i, (m) => `Open the ${m[1]} page${m[2] ?? ''}`],
57
+ // wait for [X] <type> (is )?visible
58
+ [/^waits? for \[([^\]]+)\]\s*(\w+)?(?: is)?(?: visible)?$/i, (m) => `Wait until the ${m[1]}${m[2] ? ` ${m[2]}` : ''} is visible`],
59
+ // scroll to [X]
60
+ [/^scrolls? (?:to|into) \[([^\]]+)\]\s*(\w+)?$/i, (m) => `Scroll to the ${m[1]}${m[2] ? ` ${m[2]}` : ''}`],
61
+ ];
62
+
63
+ for (const [re, out] of rules) {
64
+ const m = s.match(re);
65
+ if (m) return sentence(deRef(out(m)));
66
+ }
67
+ return sentence(deRef(s));
68
+ }
69
+
70
+ /**
71
+ * Render one expected step as an observable assertion (no tester action, no
72
+ * `should`, no DSL `User see`).
73
+ */
74
+ export function renderExpected(raw: string): string {
75
+ let s = raw.trim().replace(/^(User|The user)\s+/i, '');
76
+
77
+ const rules: Array<[RegExp, (m: RegExpMatchArray) => string]> = [
78
+ // see [X] page
79
+ [/^sees? \[([^\]]+)\] page$/i, (m) => `The ${m[1]} page is displayed`],
80
+ // see [X] <type> with V
81
+ [/^sees? \[([^\]]+)\]\s*(\w+)? with (.+)$/i, (m) => `The ${m[1]}${m[2] ? ` ${m[2]}` : ''} shows ${m[3]}`],
82
+ // see [X] <type> contains V
83
+ [/^sees? \[([^\]]+)\]\s*(\w+)? contains (.+)$/i, (m) => `The ${m[1]}${m[2] ? ` ${m[2]}` : ''} contains ${m[3]}`],
84
+ // see [X] <type> has text V
85
+ [/^sees? \[([^\]]+)\]\s*(\w+)? has text (.+)$/i, (m) => `The ${m[1]}${m[2] ? ` ${m[2]}` : ''} shows ${m[3]}`],
86
+ // see [X] <type> is hidden / is disabled / is enabled / …
87
+ [/^sees? \[([^\]]+)\]\s*(\w+)? is (.+)$/i, (m) => `The ${m[1]}${m[2] ? ` ${m[2]}` : ''} is ${m[3]}`],
88
+ // not see [X] <type>
89
+ [/^(?:do(?:es)? )?not sees? \[([^\]]+)\]\s*(\w+)?$/i, (m) => `The ${m[1]}${m[2] ? ` ${m[2]}` : ''} is not displayed`],
90
+ // see [X] <type>
91
+ [/^sees? \[([^\]]+)\]\s*(\w+)?$/i, (m) => `The ${m[1]}${m[2] ? ` ${m[2]}` : ''} is visible`],
92
+ ];
93
+
94
+ for (const [re, out] of rules) {
95
+ const m = s.match(re);
96
+ if (m) return sentence(deRef(out(m)));
97
+ }
98
+ return sentence(deRef(s));
99
+ }
100
+
101
+ /** Precondition wording: a state, not an action ("The user is signed out."). */
102
+ export function renderPrecondition(raw: string): string {
103
+ const s = raw.trim().replace(/^(User|The user)\s+/i, '');
104
+ const m = s.match(/^is on \[([^\]]+)\] page(.*)$/i);
105
+ if (m) return sentence(`The user is on the ${m[1]} page${m[2] ?? ''}`);
106
+ return sentence(deRef(`The user ${s.charAt(0).toLowerCase()}${s.slice(1)}`));
107
+ }
108
+
109
+ // ---------------------------------------------------------------------------
110
+ // Manual `# Tester verifies:` comment classification (structured, label-free)
111
+ // ---------------------------------------------------------------------------
112
+
113
+ export interface ManualProcedure {
114
+ /** Setup/Precondition/Arrange lines — the state to establish first. */
115
+ preconditions: string[];
116
+ /** Action/unlabelled lines — the imperative procedure. */
117
+ actions: string[];
118
+ /** Observable/Expect/Result/Assert lines — the observable outcome. */
119
+ expected: string[];
120
+ /** Oracle/Verify lines — HOW to check (tools, panes, queries). */
121
+ verification: string[];
122
+ }
123
+
124
+ /**
125
+ * Split a manual scenario's numbered comment lines into the four structured
126
+ * fields. Labels are consumed (structured), never left inside the prose —
127
+ * review feedback §10.2(3). Continuation lines append to the previous item;
128
+ * pre-amble (rationale/header/dividers) is skipped.
129
+ */
130
+ export function classifyManualComments(comments: string[]): ManualProcedure {
131
+ const out: ManualProcedure = { preconditions: [], actions: [], expected: [], verification: [] };
132
+ let last: { list: string[]; idx: number } | null = null;
133
+
134
+ const bucketOf = (label: string): keyof ManualProcedure => {
135
+ if (/setup|precondition|arrange|given/i.test(label)) return 'preconditions';
136
+ if (/oracle|verify|verification|how to check/i.test(label)) return 'verification';
137
+ if (/observ|expect|result|then|assert/i.test(label)) return 'expected';
138
+ return 'actions';
139
+ };
140
+
141
+ for (const raw of comments) {
142
+ const line = raw.trim();
143
+ if (!line) continue;
144
+ if (/^[-=*_]{2,}/.test(line)) { last = null; continue; }
145
+ const m = line.match(/^\d+[.)]\s*(?:([A-Za-z][A-Za-z /]*?):\s*)?(.+)$/);
146
+ if (m) {
147
+ const label = (m[1] || '').trim();
148
+ const text = m[2].trim();
149
+ const list = out[bucketOf(label)];
150
+ list.push(text);
151
+ last = { list, idx: list.length - 1 };
152
+ } else if (last) {
153
+ last.list[last.idx] += ' ' + line;
154
+ }
155
+ }
156
+ return out;
157
+ }
@@ -162,7 +162,7 @@ function findMatchingSpecTest(scenario: ScenarioMetadata, tests: SpecTest[]): Sp
162
162
  if (exact) return exact;
163
163
 
164
164
  // Fallback: match by VP-ID prefix (ignore trailing text differences)
165
- const vpMatch = scenario.name.match(/^(VP-[A-Z][A-Z0-9]*-\d+[a-zA-Z]?)/);
165
+ const vpMatch = scenario.name.match(/^(VP-[A-Z][A-Z0-9]*-\d+(?:[a-zA-Z]|-[A-Z0-9]+)?)/);
166
166
  if (vpMatch) {
167
167
  const vpId = vpMatch[1];
168
168
  const byVp = tests.find((t) => t.vpId === vpId);
@@ -165,7 +165,7 @@ function classifyBlock(block: CodeBlock): 'precondition' | 'step' | 'expectation
165
165
  * Try to extract VP ID (e.g., VP-UI-001) from the start of a scenario title.
166
166
  */
167
167
  function extractVpId(title: string): string | undefined {
168
- const match = title.match(/^(VP-[A-Z][A-Z0-9]*-\d+[a-zA-Z]?)\b/);
168
+ const match = title.match(/^(VP-[A-Z][A-Z0-9]*-\d+(?:[a-zA-Z]|-[A-Z0-9]+)?)\b/);
169
169
  return match ? match[1] : undefined;
170
170
  }
171
171
 
@@ -143,6 +143,7 @@ export function renderReportHeaderBand(
143
143
  ws: ExcelJS.Worksheet,
144
144
  titleLabel: string,
145
145
  sungenVersion?: string,
146
+ formNo: string = 'BM-2-901-13',
146
147
  ): void {
147
148
  const issueDate = (() => {
148
149
  const d = new Date();
@@ -187,9 +188,9 @@ export function renderReportHeaderBand(
187
188
  d1.alignment = { horizontal: 'center', vertical: 'middle', wrapText: true };
188
189
  d1.border = allBordersBlack;
189
190
 
190
- // G1 — No: BM-2-901-13
191
+ // G1 — No: <form number> (the company ISO form id; configurable via the delivery map)
191
192
  const g1 = ws.getCell('G1');
192
- g1.value = 'No: BM-2-901-13';
193
+ g1.value = `No: ${formNo}`;
193
194
  g1.font = { size: 12, name: TIMES };
194
195
  g1.alignment = { vertical: 'middle' };
195
196
  g1.border = { top: thinBlack, left: thinBlack, right: thinBlack };
@@ -17,7 +17,7 @@ import {
17
17
  } from './sensors';
18
18
  import { loadFlowScenarios } from './flow-check';
19
19
  import { manualReasonMismatches, MANUAL_REASONS, buildPlan } from './capability-plan';
20
- import { readCapabilities } from './capability';
20
+ import { readCapabilities, verificationScopeFindings } from './capability';
21
21
  import { readIntent, projectRootFromScreenDir, IntentProfile } from './intent';
22
22
  import { getProvenance, Provenance } from './provenance';
23
23
  import { specCoverage, SpecCoverageResult, parseSpecClauses } from './spec-coverage';
@@ -129,7 +129,8 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
129
129
  discoverAndRegisterCapabilities();
130
130
  // The active platform (web | mobile | …) activates its own capability for scoring + sensor routing.
131
131
  // `web` has no capability of its own → scoringCap stays the default `ui` (byte-identical).
132
- const platform = readCapabilities(projectRootFromScreenDir(screenDir)).platform;
132
+ const capProfile = readCapabilities(projectRootFromScreenDir(screenDir));
133
+ const platform = capProfile.platform;
133
134
  const defaultCap = capabilityRegistry.defaultCapabilityId();
134
135
  const scoringCapId = scoringCapabilityFor(catalogScreenName, defaultCap, platform);
135
136
  const scoringCap = scoringCapId ? capabilityRegistry.get(scoringCapId) : undefined;
@@ -223,6 +224,10 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
223
224
  for (const c of flowCredits) {
224
225
  findings.push(`COVERED-VIA-FLOW: critical theme "${c.theme}" is not covered on this screen but is deeply covered by flow "${c.flow}" — cross-screen depth correctly owned by the flow, so it is credited (not a screen gap). Verify with \`sungen flow-check\`.`);
225
226
  }
227
+ // Verification-scope (advisory): API/DB verification the project's recorded test type never asked for.
228
+ for (const v of verificationScopeFindings(featureText, fs.existsSync(viewpointPath) ? fs.readFileSync(viewpointPath, 'utf-8') : '', capProfile)) {
229
+ findings.push(`VERIFICATION-OUT-OF-SCOPE (${v.layer}): this unit ${v.message}`);
230
+ }
226
231
  for (const g of gate.gaps) {
227
232
  if (g.status === 'shallow') {
228
233
  findings.push(`GATE: critical theme "${g.theme}" is covered only by SHALLOW scenarios (no data assertion) → deepen with \`... with {{value}}\` / \`table ... with {{value}}\` (count @manual cross-screen too).`);
@@ -15,9 +15,55 @@ import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
15
15
  export interface CapabilityProfile {
16
16
  platform?: string; // web | mobile | … (the runtime adapter)
17
17
  enabled: string[]; // driver ids enabled for this project
18
+ /**
19
+ * Which VERIFICATION layers this project's test scope uses — the recorded "test type"
20
+ * (`ui` = E2E-only; add `api` / `db` when the project also verifies via API/DB). Explicit here wins;
21
+ * when absent it is DERIVED from `enabled` (ui always; api if the api driver is on; db if db is on).
22
+ * Generation and the audit use it so an E2E-only project does not get @api/@query verification it
23
+ * never asked for.
24
+ */
25
+ verification?: string[];
18
26
  source: 'capabilities.yaml' | 'absent';
19
27
  }
20
28
 
29
+ export type VerifyLayer = 'ui' | 'api' | 'db';
30
+
31
+ /**
32
+ * Effective verification scope = the recorded "test type". Explicit `verification` wins; otherwise
33
+ * derived from the enabled drivers. `ui` is always in scope (every project observes the UI/E2E result).
34
+ */
35
+ export function verificationScope(profile: CapabilityProfile): Set<VerifyLayer> {
36
+ if (profile.verification?.length) {
37
+ const s = new Set<VerifyLayer>(['ui']);
38
+ for (const v of profile.verification) if (v === 'api' || v === 'db' || v === 'ui') s.add(v);
39
+ return s;
40
+ }
41
+ const s = new Set<VerifyLayer>(['ui']);
42
+ if (profile.enabled.includes('api')) s.add('api');
43
+ if (profile.enabled.includes('db')) s.add('db');
44
+ return s;
45
+ }
46
+
47
+ export interface VerifyScopeFinding { layer: 'api' | 'db'; message: string; }
48
+
49
+ /**
50
+ * Flag API/DB verification that is OUT OF the project's recorded scope: a unit whose scenarios use
51
+ * `@api`/`@requires:api` (or `@query`/`@requires:db`) while the project is not API/DB-testing AND the
52
+ * viewpoint never asked for that layer. Deterministic; advisory only (a nudge, never a hard gate).
53
+ */
54
+ export function verificationScopeFindings(featureText: string, viewpointText: string, profile: CapabilityProfile): VerifyScopeFinding[] {
55
+ const scope = verificationScope(profile);
56
+ const vp = (viewpointText || '').toLowerCase();
57
+ const out: VerifyScopeFinding[] = [];
58
+ if (/@api\b|@requires:\s*api\b/.test(featureText) && !scope.has('api') && !/\bapi\b/.test(vp)) {
59
+ out.push({ layer: 'api', message: 'uses @api / @requires:api verification, but this project is not API-testing (capabilities `enabled` has no `api`) and the viewpoint does not request API verification. Keep the oracle UI-observable, OR record API in scope (capabilities `verification: [ui, api]` or `sungen capability add api`). Reserve API/DB verify for the ≤20% of effects with no UI observation.' });
60
+ }
61
+ if (/@query\b|@requires:\s*db\b/.test(featureText) && !scope.has('db') && !/\b(database|db)\b/.test(vp)) {
62
+ out.push({ layer: 'db', message: 'uses @query / @requires:db verification, but this project is not DB-testing (capabilities `enabled` has no `db`) and the viewpoint does not request DB verification. Keep the oracle UI-observable, OR record DB in scope (capabilities `verification: [ui, db]` or `sungen capability add db`). Reserve API/DB verify for the ≤20% of effects with no UI observation.' });
63
+ }
64
+ return out;
65
+ }
66
+
21
67
  export interface DriverMeta {
22
68
  id: string;
23
69
  kind: 'platform' | 'capability';
@@ -41,6 +87,7 @@ export function readCapabilities(cwd: string): CapabilityProfile {
41
87
  return {
42
88
  platform: typeof y.platform === 'string' ? y.platform : undefined,
43
89
  enabled: Array.isArray(y.enabled) ? y.enabled.map(String) : [],
90
+ verification: Array.isArray(y.verification) ? y.verification.map(String) : undefined,
44
91
  source: 'capabilities.yaml',
45
92
  };
46
93
  } catch {
@@ -48,14 +95,17 @@ export function readCapabilities(cwd: string): CapabilityProfile {
48
95
  }
49
96
  }
50
97
 
51
- export function writeCapabilities(cwd: string, profile: { platform?: string; enabled: string[] }): string {
98
+ export function writeCapabilities(cwd: string, profile: { platform?: string; enabled: string[]; verification?: string[] }): string {
52
99
  const p = capabilitiesPath(cwd);
53
100
  fs.mkdirSync(path.dirname(p), { recursive: true });
54
101
  const body =
55
102
  '# Sungen capability profile — which runtime/drivers this project uses.\n' +
56
103
  '# platform: the runtime adapter (web → Playwright, mobile → Appium). No default.\n' +
57
- '# enabled: drivers turned on (added via `sungen capability add <driver>`).\n\n' +
58
- stringifyYaml({ platform: profile.platform, enabled: profile.enabled });
104
+ '# enabled: drivers turned on (added via `sungen capability add <driver>`).\n' +
105
+ '# verification: the TEST TYPE — which layers this project verifies (`ui` = E2E only; add `api`/`db`\n' +
106
+ '# only if the project actually verifies via API/Database). Absent → derived from `enabled`.\n' +
107
+ '# This keeps an E2E-only project from getting @api/@query verification it never asked for.\n\n' +
108
+ stringifyYaml({ platform: profile.platform, enabled: profile.enabled, ...(profile.verification ? { verification: profile.verification } : {}) });
59
109
  fs.writeFileSync(p, body, 'utf-8');
60
110
  return p;
61
111
  }
@@ -22,7 +22,7 @@ If `spec_figma.md` exists OR the user provides a Figma URL for the PAT flow:
22
22
 
23
23
  ## Platform detection (do this FIRST)
24
24
 
25
- Read `qa/capabilities.yaml` and check the `platform` field.
25
+ Read `qa/capabilities.yaml` and check the `platform` field — **and the verification scope** (`verification:` if set, else derived from `enabled`: `ui` always; `api`/`db` only if that driver is on). This is the project's recorded **test type**: an **E2E/UI-only** project (no `api`/`db`) must NOT get `@api`/`@query`/`@requires:api|db` verification unless the test-viewpoint explicitly asks for it — keep oracles UI-observable, and cap any in-scope API/DB-in-E2E verification at the **≤20% band** (see `sungen-tc-generation` → "Respect the project's verification scope"). `sungen audit` flags `VERIFICATION-OUT-OF-SCOPE`.
26
26
 
27
27
  - If `platform: mobile` → the visual-capture step (Step 3, "Auto-detect visual source") uses the **`sungen-capture-mobile`** skill (Appium MCP live capture) **instead of** the web `sungen-capture` (figma/live). Scenarios use mobile gesture vocabulary from `sungen-mobile-gestures` (tap, swipe, long-press — not click/hover) and `accessibility-id`-first selectors; mobile-specific steps (permission dialogs, clipboard, geolocation) are valid Gherkin, not errors. The `Background:` is a **navigation recipe** (see `add-screen --reach`), not a URL path. Everything else — viewpoints, `sungen-tc-generation`, the harness/audit loop — is platform-agnostic and unchanged.
28
28
  - If `platform: web` or field absent → follow the **web path** (all steps below, unchanged) — Figma/live capture via `sungen-capture` as documented in Step 3.
@@ -1,146 +1,170 @@
1
1
  ---
2
2
  name: delivery
3
- description: "Export Gherkin scenarios + Playwright results to CSV test case file for QA delivery."
4
- argument-hint: "[screen-name...] [--env <locale>] (omit screens for all; --env for locale-specific export)"
3
+ description: "Export the Test Case & Coverage Matrix (review/manual/customer deliverable) from Gherkin + Playwright results. --legacy exports the classic per-scenario CSV/XLSX."
4
+ argument-hint: "[name...] [--env <locale>] [--legacy] (omit names for all; --env for locale-specific export)"
5
5
  order: 50
6
- claude-tools: "Bash, Read, AskUserQuestion"
6
+ claude-tools: "Bash, Read, Write, AskUserQuestion"
7
7
  copilot-tools: "[read, execute, edit, vscode/askQuestions]"
8
- codex-trigger: "Run when the user asks to EXPORT or deliver test cases to CSV/XLSX for QA hand-off. Final step. Do NOT use for creating or running tests."
8
+ codex-trigger: "Run when the user asks to EXPORT or deliver test cases for QA hand-off. Final step. Do NOT use for creating or running tests."
9
9
  ---
10
10
  ## Role
11
11
 
12
- You are a **QA Test Delivery Engineer**. Your job is to invoke the deterministic `sungen delivery` CLI that performs all parsing and CSV export. Your role is minimal — just run the CLI and help the user if pre-flight checks fail.
12
+ You are a **QA Test Delivery Engineer**. The deterministic `sungen delivery` CLI does all parsing,
13
+ gating, and rendering. Your job is (1) invoking it, (2) **proposing the Delivery Map** when a unit
14
+ does not have one yet — the one semantic step the CLI cannot do — and (3) walking the user through
15
+ review → approve → official render.
16
+
17
+ ## The two delivery modes
18
+
19
+ - **Matrix (default)** — the Test Case & Coverage Matrix: one row per *delivery item* (one target +
20
+ one test intent + one oracle family), with each Gherkin scenario / `@cases` row preserved as a
21
+ collapsible *coverage variant* sub-row. No numbered step dumps. This is the reviewable /
22
+ manually-executable / customer-facing standard.
23
+ - **Legacy (`--legacy`, alias `--full`)** — the classic per-scenario-steps workbook (BM-2-901-13).
24
+ Use it when the user asks for the old format or a project is mid-engagement on it.
13
25
 
14
26
  ## Parameters
15
27
 
16
28
  Parse from `$ARGUMENTS`:
17
- {{#cap parallel-subagents}}
18
- - **screens**zero or more screen/flow names. Empty → CLI processes all targets in `qa/screens/` + `qa/flows/`.
19
- {{/cap}}
20
- {{^cap parallel-subagents}}
21
- - **names** — zero or more screen/flow names. Empty → CLI processes all targets in `qa/screens/` + `qa/flows/`.
22
- {{/cap}}
23
- - **`--env <locale>`** — optional. Sets `SUNGEN_ENV=<locale>` for the run so the CLI merges `<name>.<locale>.yaml` over the base test-data and writes `<name>-testcases.<locale>.csv` / `.xlsx`. Accept `--locale <locale>` as an alias.
24
-
25
- If `--env` is passed but no value follows, ask the user which locale to use.
29
+ - **names** — zero or more screen/flow/api names. Empty → all targets.
30
+ - **`--env <locale>`** sets `SUNGEN_ENV=<locale>` for the run (accept `--locale` as alias).
31
+ - **`--legacy` / `--full`** — pass through to the CLI and skip the map flow entirely.
26
32
 
27
33
  ## Steps
28
34
 
29
- ### 1. Invoke the CLI
30
-
31
- {{#cap parallel-subagents}}
32
- Run via Bash (single command, no extra parsing). Prefer the local `./bin/sungen.js` when it exists — the sungen monorepo ships local-only features the global npm package doesn't have yet (multi-sheet locale aggregation, `.<env>` filename suffix, locale-aware step rendering). Fall back to `npx sungen` in downstream projects.
33
- {{/cap}}
34
- {{^cap parallel-subagents}}
35
- Run via Bash (single command, no extra parsing):
36
- {{/cap}}
35
+ ### 1. Run the CLI once
37
36
 
38
37
  ```bash
39
- # No env — local-first dispatcher:
40
- {{#cap parallel-subagents}}
41
- [ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <screens> || npx sungen delivery <screens>
42
- {{/cap}}
43
- {{^cap parallel-subagents}}
44
38
  [ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <names> || npx sungen delivery <names>
45
- {{/cap}}
46
-
47
- # Locale-specific:
48
- {{#cap parallel-subagents}}
49
- [ -x ./bin/sungen.js ] && SUNGEN_ENV=<locale> ./bin/sungen.js delivery <screens> || SUNGEN_ENV=<locale> npx sungen delivery <screens>
50
- {{/cap}}
51
- {{^cap parallel-subagents}}
52
- [ -x ./bin/sungen.js ] && SUNGEN_ENV=<locale> ./bin/sungen.js delivery <names> || SUNGEN_ENV=<locale> npx sungen delivery <names>
53
- {{/cap}}
39
+ ```
40
+ (prepend `SUNGEN_ENV=<locale>` when `--env` was given; append `--legacy` when requested — then skip
41
+ to step 5.)
42
+
43
+ Three outcomes per target:
44
+ - **Rendered** → done, go to step 5.
45
+ - **"no delivery map"** → go to step 2 (propose it).
46
+ - **Gate findings / "not approved"** go to step 4 (review & approve).
47
+
48
+ ### 2. Propose the Delivery Map (only when missing or the user asks to regroup)
49
+
50
+ Read the unit's `.feature` (and `requirements/spec.md` for target naming). Write
51
+ `qa/<screens|flows|api>/<unit>/delivery/<unit>.map.yaml`:
52
+
53
+ ```yaml
54
+ version: 1
55
+ unit: <unit>
56
+ groups:
57
+ - id: DI-<TARGET>-<THEME> # stable kebab id, never renumbered
58
+ target: login.email # ONE target: field/component dot-path, flow phrase, or METHOD /path
59
+ intent: <one behavior/rule this item verifies>
60
+ oracle: <the shared observable Pass/Fail statement>
61
+ category: normal | abnormal | security | nfr
62
+ review: proposed # ALWAYS proposed — only QA approval flips it
63
+ variants: [VP-VAL-001-B, VP-VAL-001-S] # VP-ids; bare id on a @cases scenario = all its rows
64
+ dispositions: # scenarios intentionally NOT delivered as test cases
65
+ VP-DATA-000: { as: excluded, reason: data-setup checklist }
66
+ # as: excluded | blocked | covered_elsewhere | accepted_risk
67
+ ```
68
+
69
+ **Grouping rules (the aggregation signature):**
70
+ - One group = **one target + one intent + one oracle family**. When unsure, keep items separate —
71
+ the gates and QA decide, never guess-merge.
72
+ - MAY share a group (become coverage dimensions): equivalence partitions, boundary values,
73
+ different data (`@cases` rows), a different trigger with the same oracle (blur vs submit),
74
+ locales.
75
+ - MUST split: different target, intent, oracle family, category, execution mode (`@manual` vs
76
+ auto), test layer (`@api`/`@query`), or priority tag; sequence-sensitive flows (re-Given/When
77
+ after a Then) stay solo. **Different risk classes never merge** — XSS and SQL injection are
78
+ separate items even though both are "injection on the same field" (different risk, action,
79
+ and oracle family); a component's visibility rule and its action/revalidation rule are two
80
+ intents, not one.
81
+ - Every scenario must land in exactly one group **or** one disposition (Gate B enforces 100%
82
+ disposition). Data-setup blocks (`@manual:data-setup`) → `excluded`; SPEC-GAP placeholders →
83
+ `blocked`.
84
+
85
+ **Wording rules for `intent`/`oracle` (customer-facing — Gate W lints these):**
86
+ - Plain product language, present simple, ~10–20 words, one behavior:
87
+ "A user can sign in with valid credentials and is redirected to the Jobs page."
88
+ - Oracle = the observable outcome as a definite assertion ("The Jobs page is displayed and the
89
+ Logout link is visible.") — no `should`, no tester actions.
90
+ - NEVER: `{{tokens}}`, `[Selector]` references, DSL phrasing (`User fill/click/see`), generator
91
+ labels (`Setup:`/`Observable:`/`Oracle:`), or vague verbs (`handles`, `surfaces`) when a precise
92
+ behavior exists. Use the visible UI label (the Login button, the Email field).
93
+ - **Preserve the source meaning exactly** — never strengthen, weaken, or reinterpret an oracle
94
+ (a security assertion especially: if the source says "the password appears ONLY in the HTTPS
95
+ POST body", do not write "no plaintext password on the network").
96
+
97
+ **Requirement coverage (`requirements:` section, optional):** `sungen delivery` scans
98
+ `requirements/spec.md` for FR-/TR-/NFR- ids; ids traced by `@spec:` tags are `covered`, the rest
99
+ are `gap` (Gate R warning). Record the reviewed status for genuine non-gaps:
100
+
101
+ ```yaml
102
+ requirements:
103
+ TR-007: { status: planned, note: Performance needs Lighthouse-style tooling }
104
+ TR-004: { status: partially_covered, note: client-side covered by VP-SEC-003; hashing needs DB verify }
105
+ # status: covered | partially_covered | covered_elsewhere | planned | gap | not_applicable
106
+ ```
107
+
108
+ Then validate and fix any ERROR findings:
109
+
110
+ ```bash
111
+ [ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <unit> --check || npx sungen delivery <unit> --check
54
112
  ```
55
113
 
56
- {{#cap parallel-subagents}}
57
- - If no screen args → omit `<screens>` (CLI processes all targets).
58
- {{/cap}}
59
- {{^cap parallel-subagents}}
60
- - If no name args omit `<names>` (CLI processes all targets).
61
- {{/cap}}
62
- - If `--env <locale>` was provided → prepend `SUNGEN_ENV=<locale>` to the command. Do NOT pass `--env` to the CLI itself — it's not a CLI flag, only a slash-command convenience.
63
-
64
- The CLI handles:
65
- {{#cap parallel-subagents}}
66
- - Scope detection (all screens + flows vs specific)
67
- {{/cap}}
68
- {{^cap parallel-subagents}}
69
- - Scope detection (all screens + flows vs specific names)
70
- - Auto-detect: `qa/flows/<name>/` flow, `qa/screens/<name>/` screen
71
- {{/cap}}
72
- - Pre-flight source checks with colorful output
73
- - Parsing `.feature`, `.spec.ts`, `test-data.yaml` (+ `<name>.<env>.yaml` overlay when `SUNGEN_ENV` is set), and per-target `<name>-test-result[.<env>].json`
74
- - Generating CSV/XLSX at `qa/deliverables/<name>-testcases[.<env>].csv` / `.xlsx`
75
- - Printing summary table
76
-
77
- ### 2. Handle pre-flight failures (if CLI exits non-zero)
78
-
79
- {{#cap parallel-subagents}}
80
- If the CLI exits with blocking issues, it will have already printed a clear table showing exactly what's missing per screen.
81
- {{/cap}}
82
- {{^cap parallel-subagents}}
83
- If the CLI exits with blocking issues, it will have already printed a clear table showing exactly what's missing per target.
84
- {{/cap}}
85
-
86
- Use `AskUserQuestion` to offer next steps:
87
-
88
- **Options:**
89
- - **Fix missing sources** (Recommended) — Print the suggested commands from CLI output and stop. User will run those commands manually, then re-invoke `/sungen:delivery`.
90
- {{#cap parallel-subagents}}
91
- - **Continue with available screens** — Re-run as `npx sungen delivery <screens> --continue-on-missing` to skip screens with blocking issues.
92
- {{/cap}}
93
- {{^cap parallel-subagents}}
94
- - **Continue with available targets** — Re-run as `npx sungen delivery <names> --continue-on-missing` to skip targets with blocking issues.
95
- {{/cap}}
96
- - **Cancel** — Exit.
97
-
98
- ### 3. Show summary + offer next steps (on success)
99
-
100
- Forward the CLI's summary table to the user verbatim. Then use `AskUserQuestion`:
101
-
102
- - **Open a specific CSV** — Help user inspect one of the exported files with Read tool.
103
- {{#cap parallel-subagents}}
104
- - **Run tests to refresh results** — Suggest `/sungen:run-test <screen>` to update `test-results/results.json`, then re-run delivery.
105
- - **Export another screen** — User can run `/sungen:delivery <other-screen>`.
106
- {{/cap}}
107
- {{^cap parallel-subagents}}
108
- - **Run tests to refresh results** — Suggest `/sungen-run-test <name>` to update test results, then re-run delivery.
109
- - **Export another target** — User can run `/sungen-delivery <other-name>`.
110
- {{/cap}}
111
- - **Done** — Exit.
114
+ ### 3. Present the grouping for review
115
+
116
+ Show the user a compact summary: item count vs variant count, the groups (id · target · intent ·
117
+ variant count), dispositions, and every remaining `review`-level finding (oracle-shape /
118
+ precondition heuristics these are exactly what QA must confirm). Then `AskUserQuestion`:
119
+
120
+ - **Approve all groups** (Recommended when the findings look right)
121
+ - **Adjust the grouping** — user names the groups to change; edit the map, re-run `--check`, re-present.
122
+ - **Preview draft first** — `sungen delivery <unit> --preview` (DRAFT watermark), then re-ask.
123
+ - **Cancel**
124
+
125
+ ### 4. Approve + official render
126
+
127
+ ```bash
128
+ [ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <unit> --approve || npx sungen delivery <unit> --approve
129
+ [ -x ./bin/sungen.js ] && ./bin/sungen.js delivery <unit> || npx sungen delivery <unit>
130
+ ```
131
+
132
+ `--approve` stamps scenario fingerprints into the map — if the `.feature` changes later, Gate E
133
+ flips the affected groups back to review (drift). Approved + unchanged groups render without
134
+ re-asking.
135
+
136
+ ### 5. Show summary + offer next steps
137
+
138
+ Forward the CLI summary verbatim (Items = review rows; **Variants = execution units progress is
139
+ counted in variants, never items**). Then `AskUserQuestion`:
140
+
141
+ - **Open the workbook** inspect `qa/deliverables/<unit>-testcases.xlsx` (Testcases sheet:
142
+ collapse outline level 1 for the customer view; Coverage sheet: target × category grid + gaps).
143
+ - **Run tests to refresh results** — `/sungen:run-test <unit>`, then re-run delivery.
144
+ - **Export the legacy workbook too** — `sungen delivery <unit> --legacy`.
145
+ - **Done**
112
146
 
113
147
  ## Important notes
114
148
 
115
- - **Do NOT parse files yourself** — the CLI is the source of truth for parsing logic. Your job is orchestration + user interaction.
116
- - **Do NOT modify feature/spec.ts/test-data files** the delivery is read-only.
117
- - **The CLI already respects `@manual` tags, skips `@steps:` base scenarios, groups by Category 2, and generates UTF-8 BOM CSV for Excel compatibility with Vietnamese.**
118
- - **Pre-flight check is built into the CLI** — use `--skip-preflight` only in CI/automated pipelines where checks are done externally.
149
+ - **Do NOT parse/duplicate what the CLI computes** gates recompute mode/layer/priority/oracle
150
+ shapes from the `.feature`; the map only carries the semantic fields (target/intent/oracle/
151
+ category/grouping).
152
+ - **Never write `review: approved` yourself** — only `--approve` after the user agreed. Proposing
153
+ with `review: approved` defeats the QA review this format exists for.
154
+ - **Do NOT modify feature/test-data files during delivery** — read-only except the map file.
155
+ - The map is committed and stable: adding a group never renumbers others; deleting scenarios →
156
+ update the map (Gate B lists exactly what's unmapped/stale).
119
157
 
120
- ## CLI Reference
158
+ ## CLI reference
121
159
 
122
160
  ```
123
- {{#cap parallel-subagents}}
124
- sungen delivery [screens...]
125
- {{/cap}}
126
- {{^cap parallel-subagents}}
127
- sungen delivery [names...]
128
- {{/cap}}
129
- [--skip-preflight] Skip pre-flight checks (not recommended)
130
- {{#cap parallel-subagents}}
131
- [--continue-on-missing] Skip screens with blocking misses
132
- {{/cap}}
133
- {{^cap parallel-subagents}}
134
- [--continue-on-missing] Skip targets with blocking misses
135
- {{/cap}}
136
-
137
- # Locale-aware export (env var, not a CLI flag):
138
- {{#cap parallel-subagents}}
139
- SUNGEN_ENV=<locale> sungen delivery [screens...]
140
- {{/cap}}
141
- {{^cap parallel-subagents}}
142
- SUNGEN_ENV=<locale> sungen delivery [names...]
143
- {{/cap}}
144
- → reads <name>.<locale>.yaml overlay, picks <name>-test-result.<locale>.json,
145
- writes <name>-testcases.<locale>.csv / .xlsx
161
+ sungen delivery [names...] # matrix (default; needs the map)
162
+ --check # gates only — validate the map, write nothing
163
+ --approve [DI-a,DI-b] # flip proposed→approved (+ stamp fingerprints); all groups when bare
164
+ --preview # render despite review findings (DRAFT watermark)
165
+ --legacy | --full # classic per-scenario workbook (no map needed)
166
+ --skip-preflight | --continue-on-missing | --env <env> # as before
146
167
  ```
168
+
169
+ Outputs: `qa/deliverables/<unit>-testcases.xlsx` (Testcases + Coverage sheets) + `.csv`
170
+ (flat, `Level` column `item|variant`). Legacy mode writes the classic files instead.
@@ -30,7 +30,7 @@ You generate 3 files for sungen — a Gherkin compiler that produces Playwright
30
30
  | `/sungen:create-data-test <name>` | Generate standardized test-data (valid/boundary/invalid + CHK trace) from the Data Factory catalog; no name = all units (asks to confirm) |
31
31
  | `/sungen:review <name>` | Score syntax, coverage, viewpoint quality (auto-detects screen or flow) |
32
32
  | `/sungen:run-test <name>` | Generate `selectors.yaml`, compile, run, auto-fix (auto-detects screen or flow) |
33
- | `/sungen:delivery [name...]` | Export test cases CSV for QA delivery (all screens if no arg) |
33
+ | `/sungen:delivery [name...]` | Export the Test Case & Coverage Matrix (review/manual/customer deliverable; `--legacy` = classic CSV; all screens if no arg) |
34
34
  | `/sungen:locale <name> <locale>` | Bootstrap i18n for a screen — audit selectors, detect locale switch, generate overlay (run before `/sungen:run-test --env <locale>`) |
35
35
 
36
36
  **Screen path:** add-screen → create-test → review → run-test → delivery.
@@ -30,7 +30,7 @@ You generate 3 files for sungen — a Gherkin compiler that produces Playwright
30
30
  | `/sungen-create-data-test <name>` | Generate standardized test-data (valid/boundary/invalid + CHK trace) from the Data Factory catalog; no name = all units (asks to confirm) |
31
31
  | `/sungen-review <name>` | Score syntax, coverage, viewpoint quality (auto-detects screen or flow) |
32
32
  | `/sungen-run-test <name>` | Generate `selectors.yaml`, compile, run, auto-fix (auto-detects screen or flow) |
33
- | `/sungen-delivery [name...]` | Export test cases CSV for QA delivery (all screens if no arg) |
33
+ | `/sungen-delivery [name...]` | Export the Test Case & Coverage Matrix (review/manual/customer deliverable; `--legacy` = classic CSV; all screens if no arg) |
34
34
  | `/sungen-locale <name> <locale>` | Bootstrap i18n for a screen — audit selectors, detect locale switch, generate overlay (run before `/sungen-run-test --env <locale>`) |
35
35
 
36
36
  **Screen path:** add-screen → create-test → review → run-test → delivery.