@sun-asterisk/sungen 3.2.13 → 3.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +29 -50
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/dashboard/snapshot-builder.d.ts.map +1 -1
- package/dist/dashboard/snapshot-builder.js +3 -46
- package/dist/dashboard/snapshot-builder.js.map +1 -1
- package/dist/exporters/locale-variants.d.ts +38 -0
- package/dist/exporters/locale-variants.d.ts.map +1 -0
- package/dist/exporters/locale-variants.js +159 -0
- package/dist/exporters/locale-variants.js.map +1 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/state-with-filter-assertion.hbs +11 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/table-match-data.hbs +3 -11
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/table-row-exists.hbs +11 -1
- package/dist/generators/test-generator/code-generator.d.ts +10 -0
- package/dist/generators/test-generator/code-generator.d.ts.map +1 -1
- package/dist/generators/test-generator/code-generator.js +41 -4
- package/dist/generators/test-generator/code-generator.js.map +1 -1
- package/dist/generators/test-generator/diagnostics.d.ts +6 -1
- package/dist/generators/test-generator/diagnostics.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.js +5 -0
- package/dist/generators/test-generator/diagnostics.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.d.ts +6 -0
- package/dist/generators/test-generator/step-mapper.d.ts.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +41 -2
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/harness/annotation-overrides.d.ts +4 -1
- package/dist/harness/annotation-overrides.d.ts.map +1 -1
- package/dist/harness/annotation-overrides.js +5 -2
- package/dist/harness/annotation-overrides.js.map +1 -1
- package/dist/harness/quality-gates.d.ts.map +1 -1
- package/dist/harness/quality-gates.js +5 -0
- package/dist/harness/quality-gates.js.map +1 -1
- package/dist/orchestrator/flow-manager.d.ts.map +1 -1
- package/dist/orchestrator/flow-manager.js +4 -3
- package/dist/orchestrator/flow-manager.js.map +1 -1
- package/dist/orchestrator/screen-manager.d.ts.map +1 -1
- package/dist/orchestrator/screen-manager.js +4 -7
- package/dist/orchestrator/screen-manager.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +8 -2
- package/dist/orchestrator/templates/ai-src/skills/sungen-error-mapping/SKILL.md +6 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-gherkin-syntax/SKILL.md +8 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-locale/SKILL.md +8 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +9 -2
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-review/SKILL.md +1 -0
- package/dist/orchestrator/templates/specs-api.d.ts.map +1 -1
- package/dist/orchestrator/templates/specs-api.js +3 -1
- package/dist/orchestrator/templates/specs-api.js.map +1 -1
- package/dist/orchestrator/templates/specs-api.ts +3 -1
- package/dist/orchestrator/templates/specs-db.d.ts.map +1 -1
- package/dist/orchestrator/templates/specs-db.js +3 -1
- package/dist/orchestrator/templates/specs-db.js.map +1 -1
- package/dist/orchestrator/templates/specs-db.ts +3 -1
- package/dist/orchestrator/templates/specs-test-data.ts +77 -5
- package/dist/orchestrator/test-data-guide.d.ts +8 -0
- package/dist/orchestrator/test-data-guide.d.ts.map +1 -0
- package/dist/orchestrator/test-data-guide.js +193 -0
- package/dist/orchestrator/test-data-guide.js.map +1 -0
- package/package.json +4 -4
- package/src/cli/commands/delivery.ts +32 -70
- package/src/dashboard/snapshot-builder.ts +11 -61
- package/src/exporters/locale-variants.ts +157 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/state-with-filter-assertion.hbs +11 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/table-match-data.hbs +3 -11
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/table-row-exists.hbs +11 -1
- package/src/generators/test-generator/code-generator.ts +36 -1
- package/src/generators/test-generator/diagnostics.ts +6 -1
- package/src/generators/test-generator/step-mapper.ts +43 -2
- package/src/harness/annotation-overrides.ts +5 -2
- package/src/harness/quality-gates.ts +4 -0
- package/src/orchestrator/flow-manager.ts +4 -3
- package/src/orchestrator/screen-manager.ts +4 -7
- package/src/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +8 -2
- package/src/orchestrator/templates/ai-src/skills/sungen-error-mapping/SKILL.md +6 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-gherkin-syntax/SKILL.md +8 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-locale/SKILL.md +8 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +9 -2
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-review/SKILL.md +1 -0
- package/src/orchestrator/templates/specs-api.ts +3 -1
- package/src/orchestrator/templates/specs-db.ts +3 -1
- package/src/orchestrator/templates/specs-test-data.ts +77 -5
- package/src/orchestrator/test-data-guide.ts +158 -0
|
@@ -37,6 +37,10 @@ export class StepMapper {
|
|
|
37
37
|
/** Row scope: when active, column assertions are scoped to this row */
|
|
38
38
|
private inRowScope: boolean = false;
|
|
39
39
|
private rowScopeTable: string = '';
|
|
40
|
+
/** Count of row-scope-entering steps in the current scenario + whether the shared
|
|
41
|
+
* `tableRow` binding has been emitted (let-once + reassign when there are ≥2). */
|
|
42
|
+
private tableRowScopeSteps: number = 0;
|
|
43
|
+
private tableRowDeclared: boolean = false;
|
|
40
44
|
/** SG-W007: an action step already ran in this scenario (alert handlers registered after it are too late) */
|
|
41
45
|
private sawActionStep: boolean = false;
|
|
42
46
|
/** Compile-time warning channel (SG-W001/W002); optional — absent means silent legacy behavior */
|
|
@@ -109,6 +113,12 @@ export class StepMapper {
|
|
|
109
113
|
/**
|
|
110
114
|
* Set scenario context for path variable resolution
|
|
111
115
|
*/
|
|
116
|
+
/** A `see [Ref] row in [Table] table with {{v}}` step — declares `tableRow` and enters row scope. */
|
|
117
|
+
private isRowScopeEntryStep(step: ParsedStep): boolean {
|
|
118
|
+
return step.elementType === 'row' && /\bin\b.*\btable\b/i.test(step.text) &&
|
|
119
|
+
/\bwith\b/i.test(step.text) && !/\bis\s+hidden\b/i.test(step.text);
|
|
120
|
+
}
|
|
121
|
+
|
|
112
122
|
setScenarioContext(steps: ParsedStep[]): void {
|
|
113
123
|
this.currentScenarioSteps = steps;
|
|
114
124
|
// Reset dialog and frame scope at the start of each new scenario
|
|
@@ -116,6 +126,16 @@ export class StepMapper {
|
|
|
116
126
|
this.inFrameScope = false;
|
|
117
127
|
this.inRowScope = false;
|
|
118
128
|
this.rowScopeTable = '';
|
|
129
|
+
// Row-scope steps declare `const tableRow` at FUNCTION scope on purpose — the
|
|
130
|
+
// following `[Col] column with {{v}}` steps reference it, so the declaration
|
|
131
|
+
// can't be block-wrapped like the self-contained row ACTIONS are. But two or
|
|
132
|
+
// more row steps in one scenario then redeclare the same const → the whole
|
|
133
|
+
// spec fails to parse ("Identifier 'tableRow' has already been declared").
|
|
134
|
+
// Count them up front: with ≥2 the first becomes `let` and the rest plain
|
|
135
|
+
// reassignments (which also correctly REBINDS the row scope); a single
|
|
136
|
+
// occurrence keeps `const` so existing single-row specs stay byte-identical.
|
|
137
|
+
this.tableRowScopeSteps = steps.filter((s) => this.isRowScopeEntryStep(s)).length;
|
|
138
|
+
this.tableRowDeclared = false;
|
|
119
139
|
this.sawActionStep = false;
|
|
120
140
|
this.lastPrimaryKeyword = 'Given';
|
|
121
141
|
this.dataResolver.clearCaptured(); // captured vars are scenario-scoped
|
|
@@ -245,13 +265,34 @@ export class StepMapper {
|
|
|
245
265
|
|
|
246
266
|
if (mappedStep) {
|
|
247
267
|
// Enter row scope when a row-in-table step is matched
|
|
248
|
-
if (
|
|
249
|
-
/\bwith\b/i.test(step.text) && !/\bis\s+hidden\b/i.test(step.text)) {
|
|
268
|
+
if (this.isRowScopeEntryStep(step)) {
|
|
250
269
|
const brackets = step.text.match(/\[([^\]]+)\]/g) || [];
|
|
251
270
|
this.inRowScope = true;
|
|
252
271
|
this.rowScopeTable = brackets.length >= 2
|
|
253
272
|
? brackets[1].replace(/[\[\]]/g, '')
|
|
254
273
|
: '';
|
|
274
|
+
// ≥2 row steps in this scenario: turn the duplicate `const tableRow`
|
|
275
|
+
// declarations into one `let` + reassignments (see setScenarioContext).
|
|
276
|
+
if (this.tableRowScopeSteps > 1 && mappedStep.code && mappedStep.code.includes('const tableRow')) {
|
|
277
|
+
if (this.tableRowDeclared) {
|
|
278
|
+
mappedStep.code = mappedStep.code.replace('const tableRow', 'tableRow');
|
|
279
|
+
} else {
|
|
280
|
+
mappedStep.code = mappedStep.code.replace('const tableRow', 'let tableRow');
|
|
281
|
+
this.tableRowDeclared = true;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// SG-W010 — a POSITIONAL-looking row label ([First X Row], [Last …]) without a
|
|
285
|
+
// selectors.yaml entry is display-only: the step degrades to an anywhere-in-the-
|
|
286
|
+
// table existence check and an ordering rule silently never fails. A resolved
|
|
287
|
+
// entry compiles to a positioned toContainText — its absence is detectable here.
|
|
288
|
+
const rowLabel = (brackets[0] || '').replace(/[\[\]]/g, '');
|
|
289
|
+
if (this.diagnostics && rowLabel &&
|
|
290
|
+
/\b(first|last|top|bottom|nth|\d+(st|nd|rd|th)?)\b/i.test(rowLabel) &&
|
|
291
|
+
mappedStep.code && !mappedStep.code.includes('toContainText')) {
|
|
292
|
+
this.diagnostics.add('SG-W010',
|
|
293
|
+
`row label "[${rowLabel}]" looks positional but has no selectors.yaml entry — it is a display label only, so this step checks existence ANYWHERE in the table (an ordering rule would pass even when the row moved). Declare it to position the row, e.g. "${rowLabel.toLowerCase()}:\\n type: locator\\n value: 'tbody tr:first-child'"`,
|
|
294
|
+
{ feature: this.featureName, step: step.text });
|
|
295
|
+
}
|
|
255
296
|
}
|
|
256
297
|
|
|
257
298
|
if (this.verbose) {
|
|
@@ -2,10 +2,13 @@
|
|
|
2
2
|
* Shared annotation-override grammar for precondition annotations (`@query`/`@api`).
|
|
3
3
|
*
|
|
4
4
|
* Parses `name(a={{x}},b="lit",c=3)` overrides into a map of JS expressions, e.g.
|
|
5
|
-
* `{ a: "testData.
|
|
5
|
+
* `{ a: "testData.raw('x')", b: "\"lit\"", c: "3" }`. Used by the DB and API capability drivers'
|
|
6
6
|
* precondition codegen; lives in core so both drivers can share it. Gherkin tags carry no whitespace,
|
|
7
7
|
* so values are single tokens — flows thread a prior response via a whole-value ref, e.g.
|
|
8
8
|
* `@api:get_profile(token={{login.body.token}})`, with the auth scheme declared in the catalog header.
|
|
9
|
+
* A `{{ref}}` override binds into a typed sink (DB query param, JSON request body), so it resolves
|
|
10
|
+
* via `testData.raw()` (native type, still interpolates embedded refs) rather than `get()`
|
|
11
|
+
* (Gherkin-text stringify) — a numeric override otherwise reaches the DB/API as a JSON string.
|
|
9
12
|
*/
|
|
10
13
|
export function parseQueryOverrides(raw?: string): Record<string, string> {
|
|
11
14
|
const out: Record<string, string> = {};
|
|
@@ -18,7 +21,7 @@ export function parseQueryOverrides(raw?: string): Record<string, string> {
|
|
|
18
21
|
if (!key) continue;
|
|
19
22
|
const v = val.match(/^\{\{\s*([^}]+?)\s*\}\}$/);
|
|
20
23
|
const q = val.match(/^["'](.*)["']$/);
|
|
21
|
-
if (v) out[key] = `testData.
|
|
24
|
+
if (v) out[key] = `testData.raw(${JSON.stringify(v[1])})`;
|
|
22
25
|
else if (q) out[key] = JSON.stringify(q[1]);
|
|
23
26
|
else if (/^-?\d+(?:\.\d+)?$/.test(val)) out[key] = val;
|
|
24
27
|
else out[key] = JSON.stringify(val);
|
|
@@ -174,6 +174,10 @@ export interface OwnershipResult { duplicates: { scenario: string; flow: string
|
|
|
174
174
|
|
|
175
175
|
/** Scenarios whose step-skeleton also appears in a sibling flow feature → duplicate ownership. */
|
|
176
176
|
export function crossArtifactOwnership(screenDir: string, scenarios: ScenarioInfo[]): OwnershipResult {
|
|
177
|
+
// A behavior's ownership is judged from the screen side; flow/api units are the owners, so the
|
|
178
|
+
// gate does not apply to them. Without this guard, auditing qa/flows/<x>, qa/api/<x>, or
|
|
179
|
+
// qa/api/flows/<x> resolves flowsRoot to the unit's own parent and every scenario self-matches.
|
|
180
|
+
if (path.basename(path.dirname(screenDir)) !== 'screens') return { duplicates: [] };
|
|
177
181
|
const duplicates: { scenario: string; flow: string }[] = [];
|
|
178
182
|
// screenDir = <root>/qa/screens/<name>; flows live at <root>/qa/flows/*/features/*.feature
|
|
179
183
|
const flowsRoot = path.resolve(screenDir, '..', '..', 'flows');
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import * as fs from 'fs';
|
|
8
8
|
import * as path from 'path';
|
|
9
9
|
import { readCapabilities } from '../harness/capability';
|
|
10
|
+
import { testDataHeader, ensureTestDataGuideFiles } from './test-data-guide';
|
|
10
11
|
|
|
11
12
|
export interface FlowOptions {
|
|
12
13
|
name: string;
|
|
@@ -114,11 +115,11 @@ export class FlowManager {
|
|
|
114
115
|
fs.writeFileSync(selectorPath, selectorLines.join('\n'), 'utf-8');
|
|
115
116
|
|
|
116
117
|
fs.writeFileSync(testDataPath, [
|
|
117
|
-
|
|
118
|
-
`# Namespace by phase: login.email, submission.nominee`,
|
|
119
|
-
`# Reference in features using {{variable}} syntax`,
|
|
118
|
+
testDataHeader(`${options.name} Flow`).trimEnd(),
|
|
119
|
+
`# Namespace keys by phase: login.email, submission.nominee`,
|
|
120
120
|
``,
|
|
121
121
|
].join('\n'), 'utf-8');
|
|
122
|
+
ensureTestDataGuideFiles(process.cwd());
|
|
122
123
|
|
|
123
124
|
const specPath = path.join(requirementsDir, 'spec.md');
|
|
124
125
|
fs.writeFileSync(specPath, this.generateSpecTemplate(options, flowName, isMobile), 'utf-8');
|
|
@@ -7,6 +7,7 @@ import * as fs from 'fs';
|
|
|
7
7
|
import * as path from 'path';
|
|
8
8
|
import { chromium } from '@playwright/test';
|
|
9
9
|
import { readCapabilities } from '../harness/capability';
|
|
10
|
+
import { testDataHeader, ensureTestDataGuideFiles } from './test-data-guide';
|
|
10
11
|
|
|
11
12
|
export interface ScreenOptions {
|
|
12
13
|
name: string;
|
|
@@ -144,15 +145,11 @@ export class ScreenManager {
|
|
|
144
145
|
fs.writeFileSync(selectorPath, selectorLines.join('\n'), 'utf-8');
|
|
145
146
|
}
|
|
146
147
|
|
|
147
|
-
// Generate empty test-data YAML
|
|
148
|
+
// Generate empty test-data YAML (header = the 3-kinds authoring guide)
|
|
148
149
|
if (!fs.existsSync(testDataPath)) {
|
|
149
|
-
fs.writeFileSync(testDataPath,
|
|
150
|
-
`# ${options.name} Screen Test Data`,
|
|
151
|
-
`# Fill via AI or manually`,
|
|
152
|
-
`# Reference in features using {{variable}} syntax`,
|
|
153
|
-
``,
|
|
154
|
-
].join('\n'), 'utf-8');
|
|
150
|
+
fs.writeFileSync(testDataPath, testDataHeader(`${options.name} Screen`), 'utf-8');
|
|
155
151
|
}
|
|
152
|
+
ensureTestDataGuideFiles(process.cwd());
|
|
156
153
|
|
|
157
154
|
// Generate requirements spec.md (only on first screen creation)
|
|
158
155
|
const specPath = path.join(requirementsDir, 'spec.md');
|
|
@@ -65,7 +65,7 @@ The CLI reads the **per-target result file first** (co-located with `.spec.ts`),
|
|
|
65
65
|
| Category 3 | Feature name (first line of `.feature`) |
|
|
66
66
|
| Category 4 | Screen name |
|
|
67
67
|
| Pre-condition | Auth tag → "Logged in as X" / "Not authenticated" + Given steps (natural language) |
|
|
68
|
-
| Test Data | `{{vars}}` from scenario resolved via test-data.yaml → `key: value; key2: value2` |
|
|
68
|
+
| Test Data | `{{vars}}` from scenario resolved via test-data.yaml → `key: value; key2: value2`. SECRET refs are printed **verbatim** as `${QA_*}` — the exporter never resolves them, so credentials never enter CSV/XLSX/dashboard |
|
|
69
69
|
| Steps | `.spec.ts` code comments for interactions (numbered) |
|
|
70
70
|
| Expected results | `.spec.ts` `expect(...)` comments (numbered) |
|
|
71
71
|
| Priority | Tag: `@high`/`@normal`/`@low` (default: Normal) |
|
|
@@ -85,7 +85,13 @@ The `.xlsx` leads with an **`Overview`** roll-up sheet, then splits the test cas
|
|
|
85
85
|
- **`Auto`** — automatable test cases (`Auto` + `Not compiled`).
|
|
86
86
|
- **`Manual`** — `@manual` test cases (always present, header-only when there are none).
|
|
87
87
|
|
|
88
|
-
Multi-locale
|
|
88
|
+
Multi-locale: `Overview` + one **`<LOCALE> Auto`** sheet per locale + a single shared **`Manual`** sheet (manual TCs are locale-invariant). The **CSV stays one file with every row** — the `Testcase type` column distinguishes Auto vs Manual. The Overview lists every detail sheet, so each locale's counts appear on its own row.
|
|
89
|
+
|
|
90
|
+
**Which result files become locale sheets — locales are DECLARED, never guessed.** A result-file suffix (`<name>-test-result.<X>.json`) is a locale ONLY when `X` is listed in `qa/capabilities.yaml` `locales:` (written by `/sungen:locale`), or — when delivering with `--env <E>` / `SUNGEN_ENV=<E>` — when `X = <E>-<locale>`. Every delivery mixes results of ONE environment:
|
|
91
|
+
- `sungen delivery login` → base results + bare declared locales (`ja`); `dev`/`stg` suffixes are IGNORED with a hint (`→ sungen delivery login --env dev`)
|
|
92
|
+
- `sungen delivery login --env dev` → `dev` results as base + `dev-ja` sheets; everything else ignored
|
|
93
|
+
- `--env ja` (SUNGEN_ENV=<locale> convention) → that locale's run IS the base, labeled JA
|
|
94
|
+
The base sheet label comes from `base_locale:` (default **EN**). Undeclared env suffixes NEVER fabricate locale sheets — a project without `locales:` always gets one clean `Auto` sheet. Mobile per-OS suffixes (`android`, `ios.ja`, …) are exempt and keep the per-OS files; their Test Data column merges the after-OS tail overlay (`ios.ja` → `<name>.ja.yaml`), matching what the runtime merged.
|
|
89
95
|
|
|
90
96
|
---
|
|
91
97
|
|
|
@@ -93,6 +93,12 @@ needs any of these, it is a **finding for QA** — surface it in the run summary
|
|
|
93
93
|
| `SG-W006` | `qa/app.<env>.yaml` found — unsupported by design | Describe the app once in `qa/app.yaml`; env differences belong in test-data overlays |
|
|
94
94
|
| `SG-W007` | Browser-alert handler step written AFTER an action — listener registers too late, the dialog auto-dismisses silently | Use the compound form `click [X] button and accept [OK] alert` (preferred), or move the alert step before the trigger |
|
|
95
95
|
| `SG-W008` | `@manual` scenario incomplete in either direction: (a) HAS Gherkin steps but NO `Then` — a dangling `When` is invalid Gherkin, comments can NOT substitute; (b) documentation-style (NO steps) but its comments lack numbered action lines and/or a labeled oracle line — prose paragraphs / rationale-only comments export EMPTY Steps + Expected cells | Two valid shapes only: complete the Gherkin with a `Then`, or write the FULL procedure in the `# Tester verifies:` block as NUMBERED lines — action steps first (`1. <action>`), then labeled `Observable:`/`Oracle:`/`Expected:` lines (the number is what fills Steps, the label is what fills Expected results) |
|
|
96
|
+
| `SG-W009` | test-data references `${QA_<NAME>}` (a SECRET env ref) but the name is declared in neither `.env.qa` nor `.env.qa.example` — the test throws "environment variable not set" the moment the key is used. (Only the `${QA_*}` shape is checked; any other `${...}` is literal data and never flagged.) | Add `QA_<NAME>=<value>` to `.env.qa` (gitignored) and list the NAME in `.env.qa.example` (the committed checklist). See `qa/TEST-DATA-GUIDE.md` |
|
|
97
|
+
| `SG-W010` | A row step's `[Ref]` looks POSITIONAL (`First…`/`Last…`/`Top…`/`nth`) but has no selectors.yaml entry — the label is display-only, so the step checks existence ANYWHERE in the table; an ordering/sort-stability rule would pass even when the row moved | Declare the row in selectors.yaml (a `type: locator` value RELATIVE to the table, e.g. `first contact row:` → `value: 'tbody tr:first-child'`) — the step then asserts `toContainText` on that exact row. This is a selectors.yaml-only fix — allowed inside the run-test auto-fix loop |
|
|
98
|
+
|
|
99
|
+
### Runtime error → `Test data "<key>" references ${QA_*} but the environment variable is not set`
|
|
100
|
+
|
|
101
|
+
The scenario uses a SECRET test-data key whose `QA_*` variable is missing in this environment. Fix: copy `.env.qa.example` → `.env.qa` and fill the value (or export it in the shell/CI). Never "fix" it by writing the real secret into test-data.yaml.
|
|
96
102
|
|
|
97
103
|
### Assertion errors → apply the Source-of-truth gate above FIRST
|
|
98
104
|
|
|
@@ -127,6 +127,14 @@ User see [Table] table match data:
|
|
|
127
127
|
|
|
128
128
|
Row scope: `see [Ref] row in [Table] table with {{v}}` enters scope. Subsequent `see [Col] column with {{v}}` checks cell in that row. Use `table match data:` for multi-row verification.
|
|
129
129
|
|
|
130
|
+
**[Ref] of a row step is a display label UNLESS declared in selectors.yaml.** Without an entry the step checks the value exists in ANY row — a name like `[First Contact Row]` does NOT position the row, and an ordering rule would pass even when the row moved (SG-W010 warns on positional-looking undeclared labels). To assert a POSITION, declare the row (a `type: locator` value RELATIVE to the table):
|
|
131
|
+
```yaml
|
|
132
|
+
first contact row:
|
|
133
|
+
type: locator
|
|
134
|
+
value: 'tbody tr:first-child'
|
|
135
|
+
```
|
|
136
|
+
→ compiles to `expect(table.locator('tbody tr:first-child')).toContainText(v)` — the exact row must hold the value — and still enters row scope for `[Col] column` checks.
|
|
137
|
+
|
|
130
138
|
### Database verification (optional Data Driver)
|
|
131
139
|
|
|
132
140
|
Read-only DB-state checks. **Prefer named queries** — SQL lives in `qa/screens/<screen>/database/queries.yaml` (reviewed once, parameterized). Invoke with the `@query:<name>` annotation; it binds the result rows to `{{name}}`, then assert with `expect`:
|
|
@@ -34,6 +34,14 @@ Take a screen/flow whose `selectors/*.yaml` and `.feature` files were authored a
|
|
|
34
34
|
1. `selectors/<feature>.yaml` — hardcoded `name`/`value` replaced with `{{var}}`
|
|
35
35
|
2. `test-data/<feature>.yaml` — base locale, complete with all new keys
|
|
36
36
|
3. `test-data/<feature>.<locale>.yaml` — overlay with only the keys that change
|
|
37
|
+
4. **`qa/capabilities.yaml` — DECLARE the locale** (required since 3.2.15): append the code to `locales:` and set `base_locale:` (language of the base test-data; tool default is `en`). Delivery/dashboard treat ONLY declared codes as locales — undeclared result-file suffixes are assumed to be ENVIRONMENTS (dev/staging) and excluded from locale sheets:
|
|
38
|
+
```yaml
|
|
39
|
+
platform: web
|
|
40
|
+
enabled: [web]
|
|
41
|
+
base_locale: en # or vi/ja — whatever the base test-data is written in
|
|
42
|
+
locales: [ja] # append every bootstrapped locale here
|
|
43
|
+
```
|
|
44
|
+
Env × locale combos use the compound `SUNGEN_ENV=<env>-<locale>` (e.g. `staging-ja`, overlay `<feature>.staging-ja.yaml`) — the locale segment LAST; delivery recognizes it when the tail matches a declared locale.
|
|
37
45
|
4. (Optional) `selectors/<feature>.yaml` Pages block updated when locale uses URL prefix or query param
|
|
38
46
|
|
|
39
47
|
After this skill finishes, `sungen run-test <name> --env <locale>` Just Works.
|
|
@@ -29,6 +29,11 @@ user-invocable: false
|
|
|
29
29
|
- Hardcode exact error messages in test-data.yaml — never leave `{{error}}` vague
|
|
30
30
|
→ Assertions are string-equal; vague placeholders produce false-positive passes.
|
|
31
31
|
|
|
32
|
+
- **Data classification — decide per value which of 3 kinds it is** (the scaffolded test-data header + `qa/TEST-DATA-GUIDE.md` explain this to QA; PRESERVE that header when rewriting the file):
|
|
33
|
+
1. **INVENTED** (backend need not accept it): validation inputs, boundaries, XSS payloads, expected messages → hardcode in the yaml.
|
|
34
|
+
2. **ENV-BOUND** (must exist in the test environment; harmless if leaked): seeded ids, registered emails, valid tokens → keep in yaml with a `# ENV-BOUND: <what to seed>` marker; per-environment values go in `<unit>.<env>.yaml`.
|
|
35
|
+
3. **SECRET** (real login / harmful if leaked): NEVER write the real value — reference `${QA_<NAME>}` (uppercase, `QA_` prefix mandatory; only that shape resolves from `.env.qa` at runtime, any other `${...}` stays literal). Whenever you emit a new `${QA_*}` ref, ALSO append the name to `.env.qa.example`. Note: "password" is not automatically SECRET — a made-up password probing validation is INVENTED; only a real account's password is SECRET (and `@auth:role` via `sungen makeauth` usually removes the need entirely).
|
|
36
|
+
|
|
32
37
|
- `@parallel` required when mixing `@auth:X` + `@no-auth` in the same feature
|
|
33
38
|
→ Playwright shares browser context per worker; auth state leaks across scenarios without isolation.
|
|
34
39
|
|
|
@@ -321,7 +326,9 @@ Security: [S1 – admin only]
|
|
|
321
326
|
- **Shape A — documentation-style** (NO Gherkin steps at all): the whole procedure lives in comments — numbered **action steps first**, then labeled `Observable:`/`Oracle:` lines. Use whenever the expected is unconfirmed (SPEC-GAP) or any part of the procedure is outside the DSL (password manager, DevTools, browser zoom/viewport, multi-tab, DB seeding). Do NOT keep a stray Gherkin fragment (`Given … page` + one fill) — a `When` without `Then` is invalid Gherkin, and a partial fragment can INVERT the chronology (e.g. "save a credential first" rendering AFTER "open the page").
|
|
322
327
|
- **Shape B — complete Gherkin**: `Given/When/Then`, all steps present. Use when the expected IS known and only execution is manual.
|
|
323
328
|
Gate (`SG-W008`, both directions): a `@manual` with Gherkin steps but no `Then` fires — comments (even labeled `Oracle:` lines) can NOT substitute for the missing `Then`. A `@manual` with NO steps ALSO fires when its comments lack ≥1 numbered action line + ≥1 labeled `Observable:`/`Oracle:`/`Expected:` line — prose paragraphs and rationale-only comments export EMPTY Steps/Expected cells.
|
|
324
|
-
In shape A the observable/oracle lines MUST carry their **label** (`Observable:` / `Oracle:` / `Expected:`) — the delivery parser extracts Expected results **by label**; unlabeled prose all lands in the Steps column and the Expected cell exports EMPTY. And the numbered action steps MUST precede them — without the actions the tester has an oracle but no procedure.
|
|
329
|
+
In shape A the observable/oracle lines MUST carry their **label** (`Observable:` / `Oracle:` / `Expected:`) — the delivery parser extracts Expected results **by label**; unlabeled prose all lands in the Steps column and the Expected cell exports EMPTY. And the numbered action steps MUST precede them — without the actions the tester has an oracle but no procedure.
|
|
330
|
+
**Oracle layer must match the unit** — a screen-unit manual defaults to a **UI oracle**: phrase the expected as what the tester SEES (on-screen message, list/table state after reload). If the business effect IS UI-observable, do NOT cite API responses or DB state as the oracle — "server rejects the request" is verified by *the error message on screen + no new row after reload*, not by reading the API response. Only when the effect has NO UI observation (mail content, stored hash, request count, response-body leak) may the oracle use another layer — then: (a) put the **layer label inside the line**: `Oracle: UI — …` / `Oracle: API — …` / `Oracle: DB — …` so QA knows the tooling needed from the deliverable alone; (b) tag `@manual:M2` so the capability planner recommends the api/db driver; (c) prefer the automation-ready `@requires:api`/`@query` form when the steps are expressible. A deep engineering check may ride along as `(optional deep check: …)` appended to the UI oracle — it must never REPLACE a UI-observable oracle.
|
|
331
|
+
Use this comment shape:
|
|
325
332
|
```gherkin
|
|
326
333
|
@high @manual
|
|
327
334
|
Scenario: VP-… <claim>
|
|
@@ -330,7 +337,7 @@ Security: [S1 – admin only]
|
|
|
330
337
|
# 1. <setup> e.g. seed a registered email; throttle the network
|
|
331
338
|
# 2. <action> e.g. click [Submit] with the request in flight
|
|
332
339
|
# 3. Observable: <what> e.g. Observable: only ONE POST is dispatched
|
|
333
|
-
# 4. Oracle: <tool>
|
|
340
|
+
# 4. Oracle: <layer — tool> e.g. Oracle: API — DevTools Network panel · Oracle: UI — visual inspection · Oracle: DB — read the row via DB client
|
|
334
341
|
```
|
|
335
342
|
|
|
336
343
|
#### Tier 1 guard — minimum before writing scenarios
|
|
@@ -188,6 +188,7 @@ Build a mapping table: for each applicable group, does the feature have a matchi
|
|
|
188
188
|
| Check | Pts |
|
|
189
189
|
|---|---|
|
|
190
190
|
| `@manual` has a valid **technical** reason (not "don't know how to automate" — document as `# TODO: automate when...` comment instead) | 3 |
|
|
191
|
+
| **Manual oracle speaks the unit's layer** — in a screen unit the oracle is UI-observable (on-screen message, list state after reload), NOT "API response/DB state" when the effect is visible in the UI; a genuinely non-UI oracle carries its layer label (`Oracle: API — …` / `Oracle: DB — …`) + `@manual:M2` | 2 |
|
|
191
192
|
| Stable selectors — per `sungen-selector-keys`: keys use spaces, standard types, `locator` only as last resort | 3 |
|
|
192
193
|
| Each step names a concrete element by name/label/role (enough to implement without asking) | 2 |
|
|
193
194
|
| Idempotent + no fixed waits (`wait for [T] dialog` instead of `wait N seconds`) | 2 |
|
|
@@ -24,7 +24,9 @@ interface ApiDataSource {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
function loadEnvQa(): void {
|
|
27
|
-
|
|
27
|
+
// Precedence (first set wins, real env highest): shell/CI env > .env.qa.<SUNGEN_ENV>
|
|
28
|
+
// > .env.qa — the env-specific file OVERLAYS the base, mirroring test-data overlays.
|
|
29
|
+
for (const name of [`.env.qa.${process.env.SUNGEN_ENV || ''}`, '.env.qa']) {
|
|
28
30
|
const p = path.join(process.cwd(), name);
|
|
29
31
|
if (!name.endsWith('.') && fs.existsSync(p)) {
|
|
30
32
|
for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
|
|
@@ -110,7 +110,9 @@ async function openSshTunnel(ssh: SshConfig, dstHost: string, dstPort: number):
|
|
|
110
110
|
}
|
|
111
111
|
|
|
112
112
|
function loadEnvQa(): void {
|
|
113
|
-
|
|
113
|
+
// Precedence (first set wins, real env highest): shell/CI env > .env.qa.<SUNGEN_ENV>
|
|
114
|
+
// > .env.qa — the env-specific file OVERLAYS the base, mirroring test-data overlays.
|
|
115
|
+
for (const name of [`.env.qa.${process.env.SUNGEN_ENV || ''}`, '.env.qa']) {
|
|
114
116
|
const p = path.join(process.cwd(), name);
|
|
115
117
|
if (!name.endsWith('.') && fs.existsSync(p)) {
|
|
116
118
|
for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
|
|
@@ -39,6 +39,8 @@ export class TestDataLoader {
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
data = resolveDynamicVars(data);
|
|
42
|
+
loadEnvQa();
|
|
43
|
+
data = resolveEnvRefs(data);
|
|
42
44
|
|
|
43
45
|
return new TestDataLoader(data);
|
|
44
46
|
}
|
|
@@ -51,21 +53,34 @@ export class TestDataLoader {
|
|
|
51
53
|
// A projected array (e.g. `{{q.rows[*].name}}`) serializes as a JSON list so the full
|
|
52
54
|
// ordered set is preserved for the assert-compare — String([...]) would flatten it to a
|
|
53
55
|
// lossy comma-join that can't be distinguished from a scalar containing commas.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
+
const result = Array.isArray(value)
|
|
57
|
+
? this.interpolate(JSON.stringify(value))
|
|
58
|
+
: this.interpolate(String(value));
|
|
59
|
+
assertNoUnresolvedEnvRef(key, result);
|
|
60
|
+
return result;
|
|
56
61
|
}
|
|
57
62
|
|
|
58
63
|
/**
|
|
59
64
|
* Resolve a key to its RAW, uncoerced value (array/object/number kept as-is) — for binding
|
|
60
|
-
* DB query
|
|
61
|
-
* native type. get() stringifies for Gherkin text; raw()
|
|
62
|
-
*
|
|
65
|
+
* DB query / API request params, where type-strict sinks (MySQL numeric columns, MongoDB,
|
|
66
|
+
* a JSON request body) require the native type. get() stringifies for Gherkin text; raw()
|
|
67
|
+
* must not. A string value still has its embedded `{{ref}}` cross-references resolved (same
|
|
68
|
+
* as get()), so an override like `token={{login.body.token}}` or `id={{prefix}}-001` still
|
|
69
|
+
* interpolates — only the final scalar/array/object type is preserved uncoerced. Arrays and
|
|
70
|
+
* objects are returned as-is; refs embedded inside their elements are not interpolated (unlike
|
|
71
|
+
* get()'s JSON-text pass), since raw() callers bind structured values directly, not text.
|
|
72
|
+
* Throws on missing, mirroring get()'s guard.
|
|
63
73
|
*/
|
|
64
74
|
raw(key: string): any {
|
|
65
75
|
const value = this.resolve(key);
|
|
66
76
|
if (value === undefined || value === null) {
|
|
67
77
|
throw new Error(`Test data key not found: ${key}`);
|
|
68
78
|
}
|
|
79
|
+
if (typeof value === 'string') {
|
|
80
|
+
const result = this.interpolate(value);
|
|
81
|
+
assertNoUnresolvedEnvRef(key, result);
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
69
84
|
return value;
|
|
70
85
|
}
|
|
71
86
|
|
|
@@ -206,6 +221,63 @@ function deepMerge(base: Record<string, any>, override: Record<string, any>): Re
|
|
|
206
221
|
return result;
|
|
207
222
|
}
|
|
208
223
|
|
|
224
|
+
/**
|
|
225
|
+
* SECRET test data: only `${QA_*}` (uppercase, QA_ prefix) is an env reference — resolved
|
|
226
|
+
* from .env.qa / process.env so real credentials never live in the committed yaml. The
|
|
227
|
+
* prefix doubles as the resolution ALLOWLIST: any other `${...}` text is ordinary literal
|
|
228
|
+
* data (XSS payloads, template strings, passwords containing `${`) — never touched, and a
|
|
229
|
+
* yaml can never exfiltrate arbitrary environment variables like ${AWS_SECRET_ACCESS_KEY}.
|
|
230
|
+
*/
|
|
231
|
+
const QA_ENV_REF = /\$\{(QA_[A-Z0-9_]+)\}/g;
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Load .env.qa files into process.env. Precedence (first set wins, real env highest):
|
|
235
|
+
* shell/CI env > .env.qa.<SUNGEN_ENV> > .env.qa — the env-specific file OVERLAYS the base,
|
|
236
|
+
* mirroring how <feature>.<env>.yaml overlays <feature>.yaml.
|
|
237
|
+
*/
|
|
238
|
+
function loadEnvQa(): void {
|
|
239
|
+
const env = process.env.SUNGEN_ENV;
|
|
240
|
+
for (const name of [env ? `.env.qa.${env}` : '', '.env.qa']) {
|
|
241
|
+
if (!name) continue;
|
|
242
|
+
const p = path.join(process.cwd(), name);
|
|
243
|
+
if (!fs.existsSync(p)) continue;
|
|
244
|
+
for (const line of fs.readFileSync(p, 'utf-8').split('\n')) {
|
|
245
|
+
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
|
|
246
|
+
if (m && process.env[m[1]] === undefined) process.env[m[1]] = m[2].replace(/^["']|["']$/g, '');
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** Substitute `${QA_*}` refs in every string value. Unset vars stay literal — the guard in
|
|
252
|
+
* get()/raw() throws with the var name the moment such a key is actually used. */
|
|
253
|
+
function resolveEnvRefs(data: Record<string, any>): Record<string, any> {
|
|
254
|
+
function resolveValue(value: any): any {
|
|
255
|
+
if (typeof value === 'string') {
|
|
256
|
+
return value.replace(QA_ENV_REF, (match, name) =>
|
|
257
|
+
process.env[name] !== undefined ? process.env[name]! : match);
|
|
258
|
+
}
|
|
259
|
+
if (Array.isArray(value)) return value.map(resolveValue);
|
|
260
|
+
if (value && typeof value === 'object') {
|
|
261
|
+
const resolved: Record<string, any> = {};
|
|
262
|
+
for (const [k, v] of Object.entries(value)) resolved[k] = resolveValue(v);
|
|
263
|
+
return resolved;
|
|
264
|
+
}
|
|
265
|
+
return value;
|
|
266
|
+
}
|
|
267
|
+
return resolveValue(data);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Fail fast — and only for the test that actually uses the key — when a `${QA_*}` secret
|
|
271
|
+
* was referenced but never provided. */
|
|
272
|
+
function assertNoUnresolvedEnvRef(key: string, value: string): void {
|
|
273
|
+
const m = value.match(/\$\{(QA_[A-Z0-9_]+)\}/);
|
|
274
|
+
if (m) {
|
|
275
|
+
throw new Error(
|
|
276
|
+
`Test data "${key}" references \${${m[1]}} but the environment variable is not set — ` +
|
|
277
|
+
`add ${m[1]}=... to .env.qa (gitignored; see .env.qa.example), or export it in the shell/CI.`);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
209
281
|
function resolveDynamicVars(data: Record<string, any>): Record<string, any> {
|
|
210
282
|
const ts = String(Date.now());
|
|
211
283
|
const uid = crypto.randomUUID();
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test-data authoring guide — the QA-facing explanation of the three data classes
|
|
3
|
+
* (INVENTED / ENV-BOUND / SECRET `${QA_*}`) and the secret-handling convention.
|
|
4
|
+
*
|
|
5
|
+
* Two layers, both generated here so they can never drift:
|
|
6
|
+
* - a compact header embedded at the top of every scaffolded test-data yaml
|
|
7
|
+
* (create-test's skill instructs the AI to preserve it when rewriting the file);
|
|
8
|
+
* - the full guide at qa/TEST-DATA-GUIDE.md + the .env.qa.example starter, created
|
|
9
|
+
* once per project by add-screen/add-flow.
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from 'fs';
|
|
12
|
+
import * as path from 'path';
|
|
13
|
+
|
|
14
|
+
/** Compact per-file header — examples first, one line per class. */
|
|
15
|
+
export function testDataHeader(unitName: string): string {
|
|
16
|
+
return [
|
|
17
|
+
`# ${unitName} Test Data — reference in features using {{variable}} syntax.`,
|
|
18
|
+
`# ─── HOW TO FILL VALUES (full guide: qa/TEST-DATA-GUIDE.md) ────────────────`,
|
|
19
|
+
`# 3 kinds of values:`,
|
|
20
|
+
`# 1) INVENTED — only exists to probe the screen; make it up from the spec`,
|
|
21
|
+
`# wrong_email: "abc@" ← write the value right here`,
|
|
22
|
+
`# 2) ENV-BOUND — must EXIST in the test environment (seeded id/email),`,
|
|
23
|
+
`# but leaking it is harmless`,
|
|
24
|
+
`# active_user_id: "d4f8-001" # ENV-BOUND: seeded on staging`,
|
|
25
|
+
`# 3) SECRET — a real login / harmful if leaked (real password, API key)`,
|
|
26
|
+
`# admin_password: \${QA_ADMIN_PASSWORD} ← never write the real value;`,
|
|
27
|
+
`# put it in .env.qa (gitignored), variable name MUST start with QA_`,
|
|
28
|
+
`# ───────────────────────────────────────────────────────────────────────────`,
|
|
29
|
+
``,
|
|
30
|
+
].join('\n');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const GUIDE_MD = `# Test Data Guide
|
|
34
|
+
|
|
35
|
+
Every \`{{variable}}\` in a \`.feature\` file maps to a key in that unit's
|
|
36
|
+
\`test-data/*.yaml\`. When you create or edit a value, decide which of the
|
|
37
|
+
**three kinds** it is — each kind is handled differently.
|
|
38
|
+
|
|
39
|
+
## 1) INVENTED — made-up values that only probe the screen
|
|
40
|
+
|
|
41
|
+
**What it is:** the value exists only to trigger a reaction (an error message, a
|
|
42
|
+
boundary, a rendering) — the backend does not need to accept it as real data.
|
|
43
|
+
|
|
44
|
+
**Examples:** a wrong-format email \`"abc@"\`; a 6-char password to trigger the
|
|
45
|
+
"minimum 7 characters" error; a 256-char string to test a 255 limit; an XSS
|
|
46
|
+
payload \`<script>alert(1)</script>\`; the exact expected error message from the spec.
|
|
47
|
+
|
|
48
|
+
**What to do:** write the value directly in the yaml. Nothing else.
|
|
49
|
+
|
|
50
|
+
**Self-check:** *"If I swapped this for another value of the same kind, would the
|
|
51
|
+
test still mean the same thing?"* → yes = INVENTED.
|
|
52
|
+
|
|
53
|
+
## 2) ENV-BOUND — real records of the test environment (not secret)
|
|
54
|
+
|
|
55
|
+
**What it is:** the value must EXIST in the environment for the test to work — a
|
|
56
|
+
seeded record id, a registered email, a valid invite token — but leaking it
|
|
57
|
+
causes no harm.
|
|
58
|
+
|
|
59
|
+
**Examples:**
|
|
60
|
+
\`\`\`yaml
|
|
61
|
+
active_user_id: "d4f8-001" # ENV-BOUND: seeded user on staging
|
|
62
|
+
existing_email: "tester01@example.com" # ENV-BOUND: pre-registered account
|
|
63
|
+
\`\`\`
|
|
64
|
+
|
|
65
|
+
**What to do:** keep it in the yaml and mark the line with \`# ENV-BOUND: ...\` so
|
|
66
|
+
the next person knows it must be re-seeded/replaced per environment. When the
|
|
67
|
+
value differs per environment, override it in \`<unit>.<env>.yaml\`
|
|
68
|
+
(e.g. \`login.staging.yaml\`) and run with \`SUNGEN_ENV=staging\`.
|
|
69
|
+
|
|
70
|
+
**Self-check:** *"If someone deleted this record from the DB, would the test
|
|
71
|
+
fail?"* → yes = ENV-BOUND.
|
|
72
|
+
|
|
73
|
+
## 3) SECRET — real credentials; harmful if leaked
|
|
74
|
+
|
|
75
|
+
**What it is:** values that grant real access or cause harm when exposed — a
|
|
76
|
+
real account password, an API key, a system token.
|
|
77
|
+
|
|
78
|
+
\`\`\`yaml
|
|
79
|
+
# ❌ WRONG — the real password lives in git; anyone who clones the repo reads it:
|
|
80
|
+
admin_password: "Sun@2026"
|
|
81
|
+
|
|
82
|
+
# ✅ RIGHT — the yaml holds only the variable name; the real value stays in
|
|
83
|
+
# .env.qa, which is gitignored:
|
|
84
|
+
admin_password: \${QA_ADMIN_PASSWORD}
|
|
85
|
+
\`\`\`
|
|
86
|
+
|
|
87
|
+
and in \`.env.qa\` (NOT committed — copy \`.env.qa.example\` and fill it):
|
|
88
|
+
|
|
89
|
+
\`\`\`bash
|
|
90
|
+
QA_ADMIN_PASSWORD=Sun@2026
|
|
91
|
+
\`\`\`
|
|
92
|
+
|
|
93
|
+
**Rules that matter:**
|
|
94
|
+
- The variable name MUST start with \`QA_\` and be UPPERCASE — only \`\${QA_...}\`
|
|
95
|
+
is replaced at runtime. Any other \`\${...}\` text is treated as ordinary
|
|
96
|
+
characters (so XSS payloads / template strings containing \`\${\` are safe).
|
|
97
|
+
- Whenever you add a new \`\${QA_*}\` reference, also add the NAME to
|
|
98
|
+
\`.env.qa.example\` — that file is the committed checklist of variables each
|
|
99
|
+
environment must provide.
|
|
100
|
+
- Delivery reports (CSV/Excel) and the dashboard always print \`\${QA_ADMIN_PASSWORD}\`
|
|
101
|
+
literally — secrets never enter deliverables.
|
|
102
|
+
- Per environment: \`.env.qa.<env>\` overlays \`.env.qa\` (run with \`SUNGEN_ENV=<env>\`);
|
|
103
|
+
a variable exported in the shell/CI beats both files.
|
|
104
|
+
- If a referenced \`QA_*\` variable is missing at run time, the test that uses it
|
|
105
|
+
fails immediately with the variable name — check \`.env.qa\`.
|
|
106
|
+
|
|
107
|
+
**Self-check:** *"Could sharing this file outside the team cause harm?"* → yes = SECRET.
|
|
108
|
+
|
|
109
|
+
## Easy to confuse
|
|
110
|
+
|
|
111
|
+
- **"password" is not automatically SECRET.** \`password_below_min: "Aa1bcd"\` used
|
|
112
|
+
to trigger a validation error is INVENTED — make it up. Only a password that
|
|
113
|
+
actually logs in to a real account is SECRET.
|
|
114
|
+
- **Testing screens behind login usually needs NO password at all** — run
|
|
115
|
+
\`sungen makeauth\` once: it opens a browser, you log in by hand, and the saved
|
|
116
|
+
session (storageState) is reused by \`@auth:<role>\` scenarios.
|
|
117
|
+
|
|
118
|
+
## Quick reference
|
|
119
|
+
|
|
120
|
+
| Kind | Recognize by | Where the value lives |
|
|
121
|
+
|---|---|---|
|
|
122
|
+
| INVENTED | backend need not accept it | the yaml, as-is |
|
|
123
|
+
| ENV-BOUND | must exist in the environment; harmless if leaked | the yaml + \`# ENV-BOUND:\` marker (+ \`<unit>.<env>.yaml\` overlay) |
|
|
124
|
+
| SECRET | real access / harmful if leaked | \`\${QA_*}\` in the yaml → real value in \`.env.qa\` (gitignored), name listed in \`.env.qa.example\` |
|
|
125
|
+
`;
|
|
126
|
+
|
|
127
|
+
const ENV_QA_EXAMPLE = `# Secret test data referenced as \${QA_*} in qa/**/test-data/*.yaml.
|
|
128
|
+
# Copy this file to .env.qa (gitignored) and fill the real values per environment.
|
|
129
|
+
# Per-environment overrides: .env.qa.<env> + run with SUNGEN_ENV=<env>.
|
|
130
|
+
#
|
|
131
|
+
# QA_ADMIN_PASSWORD=
|
|
132
|
+
`;
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Idempotently create qa/TEST-DATA-GUIDE.md and .env.qa.example, and make sure
|
|
136
|
+
* .gitignore covers .env.qa (incl. .env.qa.<env> overlays, but NOT .env.qa.example).
|
|
137
|
+
*/
|
|
138
|
+
export function ensureTestDataGuideFiles(cwd: string): void {
|
|
139
|
+
const guidePath = path.join(cwd, 'qa', 'TEST-DATA-GUIDE.md');
|
|
140
|
+
if (fs.existsSync(path.join(cwd, 'qa')) && !fs.existsSync(guidePath)) {
|
|
141
|
+
fs.writeFileSync(guidePath, GUIDE_MD, 'utf-8');
|
|
142
|
+
}
|
|
143
|
+
const examplePath = path.join(cwd, '.env.qa.example');
|
|
144
|
+
if (!fs.existsSync(examplePath)) {
|
|
145
|
+
fs.writeFileSync(examplePath, ENV_QA_EXAMPLE, 'utf-8');
|
|
146
|
+
}
|
|
147
|
+
const gitignorePath = path.join(cwd, '.gitignore');
|
|
148
|
+
if (fs.existsSync(gitignorePath)) {
|
|
149
|
+
const lines = fs.readFileSync(gitignorePath, 'utf-8').split('\n').map((l) => l.trim());
|
|
150
|
+
// `.env.qa` alone does not cover `.env.qa.staging` — ignore the whole family
|
|
151
|
+
// except the committed example (negation must come after the glob).
|
|
152
|
+
if (!lines.includes('.env.qa') && !lines.includes('.env.qa*') && !lines.includes('.env*')) {
|
|
153
|
+
fs.appendFileSync(gitignorePath, `\n# sungen secret test data (see .env.qa.example)\n.env.qa*\n!.env.qa.example\n`);
|
|
154
|
+
}
|
|
155
|
+
} else {
|
|
156
|
+
fs.writeFileSync(gitignorePath, `# sungen secret test data (see .env.qa.example)\n.env.qa*\n!.env.qa.example\n`, 'utf-8');
|
|
157
|
+
}
|
|
158
|
+
}
|