@sun-asterisk/sungen 3.2.12-beta.9 → 3.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +17 -7
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/exporters/result-variants.d.ts +6 -4
- package/dist/exporters/result-variants.d.ts.map +1 -1
- package/dist/exporters/result-variants.js +7 -6
- package/dist/exporters/result-variants.js.map +1 -1
- package/dist/exporters/scenario-merger.d.ts +14 -0
- package/dist/exporters/scenario-merger.d.ts.map +1 -1
- package/dist/exporters/scenario-merger.js +1 -0
- package/dist/exporters/scenario-merger.js.map +1 -1
- package/dist/exporters/spec-parser.d.ts.map +1 -1
- package/dist/exporters/spec-parser.js +10 -2
- package/dist/exporters/spec-parser.js.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/templates/steps/actions/hover-element-with-text.hbs +1 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/visible-filtered-assertion.hbs +1 -0
- package/dist/generators/test-generator/code-generator.d.ts.map +1 -1
- package/dist/generators/test-generator/code-generator.js +51 -0
- package/dist/generators/test-generator/code-generator.js.map +1 -1
- package/dist/generators/test-generator/diagnostics.d.ts +5 -1
- package/dist/generators/test-generator/diagnostics.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.js +4 -0
- package/dist/generators/test-generator/diagnostics.js.map +1 -1
- package/dist/generators/test-generator/utils/runtime-data-transformer.d.ts.map +1 -1
- package/dist/generators/test-generator/utils/runtime-data-transformer.js +9 -3
- package/dist/generators/test-generator/utils/runtime-data-transformer.js.map +1 -1
- package/dist/harness/audit.js +1 -1
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/quality-gates.d.ts.map +1 -1
- package/dist/harness/quality-gates.js +23 -5
- package/dist/harness/quality-gates.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-data-factory/SKILL.md +36 -6
- package/dist/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +2 -2
- package/dist/orchestrator/templates/ai-src/skills/sungen-error-mapping/SKILL.md +2 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +7 -3
- package/package.json +4 -4
- package/src/cli/commands/delivery.ts +15 -7
- package/src/exporters/result-variants.ts +7 -6
- package/src/exporters/scenario-merger.ts +1 -1
- package/src/exporters/spec-parser.ts +9 -2
- package/src/generators/test-generator/adapters/playwright/templates/steps/actions/hover-element-with-text.hbs +1 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/visible-filtered-assertion.hbs +1 -0
- package/src/generators/test-generator/code-generator.ts +54 -0
- package/src/generators/test-generator/diagnostics.ts +5 -1
- package/src/generators/test-generator/utils/runtime-data-transformer.ts +10 -4
- package/src/harness/audit.ts +1 -1
- package/src/harness/quality-gates.ts +22 -5
- package/src/orchestrator/templates/ai-src/skills/sungen-data-factory/SKILL.md +36 -6
- package/src/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +2 -2
- package/src/orchestrator/templates/ai-src/skills/sungen-error-mapping/SKILL.md +2 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +7 -3
|
@@ -291,11 +291,11 @@ export function runPreflight(cwd: string, target: DeliveryTarget): PreflightChec
|
|
|
291
291
|
suggestions.push(`/sungen:create-test ${target.screen}`);
|
|
292
292
|
}
|
|
293
293
|
if (!selectorsOk) {
|
|
294
|
-
missing.push(`selectors.yaml missing entries: ${path.relative(cwd, selectorsFile)}`);
|
|
294
|
+
missing.push(`selectors.yaml missing entries (optional — filled by run-test): ${path.relative(cwd, selectorsFile)}`);
|
|
295
295
|
suggestions.push(`/sungen:run-test ${target.screen}`);
|
|
296
296
|
}
|
|
297
297
|
if (!specOk) {
|
|
298
|
-
missing.push(`compiled .spec.ts missing: ${path.relative(cwd, specFile)}`);
|
|
298
|
+
missing.push(`compiled .spec.ts missing (optional — needed to run): ${path.relative(cwd, specFile)}`);
|
|
299
299
|
suggestions.push(`sungen generate --${target.kind === 'flow' ? 'flow' : target.kind === 'api' ? 'api' : 'screen'} ${target.screen}`);
|
|
300
300
|
}
|
|
301
301
|
if (!resultsOk) {
|
|
@@ -371,7 +371,12 @@ function checkSelectorsHasEntries(selectorsFile: string, screen: string): boolea
|
|
|
371
371
|
}
|
|
372
372
|
|
|
373
373
|
export function hasBlockingMissing(p: PreflightCheck): boolean {
|
|
374
|
-
|
|
374
|
+
// Only the CONTENT sources block: feature + test-data are what every export row is
|
|
375
|
+
// built from. selectors / .spec.ts / test-result.json only affect execution status —
|
|
376
|
+
// when absent, delivery still exports a pre-run overview (parseSpecFile returns
|
|
377
|
+
// {tests: []}, results resolve to null) with every executable row PENDING, so QA can
|
|
378
|
+
// review the test-case list right after create-test, before the first run.
|
|
379
|
+
return !p.featureOk || !p.testDataOk;
|
|
375
380
|
}
|
|
376
381
|
|
|
377
382
|
// ----------------------------------------------------------------------------
|
|
@@ -664,12 +669,15 @@ function printPreflightTable(checks: PreflightCheck[]): void {
|
|
|
664
669
|
log('');
|
|
665
670
|
}
|
|
666
671
|
|
|
667
|
-
// Warn about optional misses (
|
|
668
|
-
|
|
672
|
+
// Warn about optional misses (selectors / .spec.ts / test-result.json). These no
|
|
673
|
+
// longer block: the export proceeds as a pre-run overview with Result = PENDING.
|
|
674
|
+
const warnings = checks.filter((c) => !hasBlockingMissing(c) && c.missing.length > 0);
|
|
669
675
|
if (warnings.length > 0) {
|
|
670
|
-
log(`${COLOR.yellow}
|
|
676
|
+
log(`${COLOR.yellow}Pre-run export (optional sources missing — executable rows will be PENDING):${COLOR.reset}`);
|
|
671
677
|
for (const w of warnings) {
|
|
672
|
-
log(` ${w.screen}
|
|
678
|
+
log(` ${COLOR.bold}${w.screen}${COLOR.reset}`);
|
|
679
|
+
for (const m of w.missing) log(` - ${m}`);
|
|
680
|
+
for (const s of w.suggestions) log(` ${COLOR.cyan}→ ${s}${COLOR.reset}`);
|
|
673
681
|
}
|
|
674
682
|
log('');
|
|
675
683
|
}
|
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { MergedScenario } from './scenario-merger';
|
|
7
7
|
import { PlaywrightResult } from './types';
|
|
8
|
-
import { extractTestcaseType } from './feature-parser';
|
|
9
8
|
|
|
10
9
|
export interface ResultVariant {
|
|
11
10
|
/** Appended to the scenario display name: `''` for a plain scenario, ` — <label>` per
|
|
@@ -50,10 +49,12 @@ function rowLabel(row: Record<string, unknown>, i: number): string {
|
|
|
50
49
|
* `<scenario> — <label>` — expand into one variant per EXECUTED row by matching that prefix (so an
|
|
51
50
|
* already-run row carries its real pass/fail), attaching the backing dataset row by label.
|
|
52
51
|
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
52
|
+
* When no row has EXECUTED yet there are no runtime titles to expand from — a MANUAL `@cases`
|
|
53
|
+
* never executes, and an auto `@cases` before its first run (the pre-run overview export) is in
|
|
54
|
+
* the same position. Both split from the DATASET instead: one variant per row (` — <label>`,
|
|
55
|
+
* no result → Pending/Manual), each carrying its row so the reader gets one report line + its
|
|
56
|
+
* own data per case. Executed rows always win (real pass/fail) — the dataset split only fills
|
|
57
|
+
* the not-yet-run gap.
|
|
57
58
|
*
|
|
58
59
|
* NOTE: this deliberately does NOT gate on `m.spec`. For @cases the compiled test title carries a
|
|
59
60
|
* `— ${__row.__label}` suffix, so `findMatchingSpecTest` (base-name match) often leaves `m.spec`
|
|
@@ -81,7 +82,7 @@ export function resolveResultVariants(
|
|
|
81
82
|
});
|
|
82
83
|
}
|
|
83
84
|
}
|
|
84
|
-
if (datasetRows
|
|
85
|
+
if (datasetRows) {
|
|
85
86
|
return datasetRows.map((row, i) => ({ nameSuffix: ` — ${rowLabel(row, i)}`, row }));
|
|
86
87
|
}
|
|
87
88
|
return [{ nameSuffix: '' }];
|
|
@@ -21,7 +21,7 @@ import { isStepsBaseScenario, isSampleScaffoldScenario, extractTestcaseType } fr
|
|
|
21
21
|
* numbered item is a continuation (e.g. a wrapped URL) and is appended to it. Pre-amble
|
|
22
22
|
* lines (the `MANUAL:` rationale, the `Tester verifies:` header, separators) are ignored.
|
|
23
23
|
*/
|
|
24
|
-
function parseManualComments(comments: string[]): { steps: OrderedStep[]; expected: OrderedStep[] } {
|
|
24
|
+
export function parseManualComments(comments: string[]): { steps: OrderedStep[]; expected: OrderedStep[] } {
|
|
25
25
|
const steps: OrderedStep[] = [];
|
|
26
26
|
const expected: OrderedStep[] = [];
|
|
27
27
|
let last: OrderedStep | null = null;
|
|
@@ -31,14 +31,21 @@ function extractTestBlock(content: string, startIdx: number): {
|
|
|
31
31
|
// single-line, so `.*?` anchored on `}, async` is safe.
|
|
32
32
|
// Accept both Playwright `test(...)` and mobile/Appium mocha `it(...)` (issue #391). The leading
|
|
33
33
|
// `\b` prevents `it(` from matching the tail of a larger word like `submit(` / `edit(`.
|
|
34
|
-
|
|
34
|
+
// A @cases title is emitted as CONCAT — `test('VP-… — ' + __row.__label, …)` (scenario.hbs) —
|
|
35
|
+
// so the closing quote is followed by `+ __row.__label` instead of the comma. Without the
|
|
36
|
+
// optional concat group the whole test() vanished from the parse and every @cases scenario
|
|
37
|
+
// exported as "Not compiled" whenever no test-result was present (the pre-run overview).
|
|
38
|
+
const testRegex = /\b(?:test|it)\s*\(\s*(['"`])((?:\\.|(?!\1).)+)\1\s*(?:\+\s*__row\.__label\s*)?,\s*(?:\{.*?\}\s*,\s*)?async\s*\([^)]*\)\s*=>\s*\{/g;
|
|
35
39
|
testRegex.lastIndex = startIdx;
|
|
36
40
|
const match = testRegex.exec(content);
|
|
37
41
|
if (!match) return null;
|
|
38
42
|
|
|
39
43
|
// [1] = outer quote char, [2] = title. Unescape `\'`/`\"`/backtick/backslash: the results.json
|
|
40
44
|
// title is the RUNTIME string (`row's`), so the escaped source form would never match it.
|
|
41
|
-
|
|
45
|
+
// Concat form: the captured literal ends with the ` — ` separator the template appends
|
|
46
|
+
// before __row.__label — strip it so the title equals the feature's scenario name.
|
|
47
|
+
let title = unescapeQuoted(match[2]);
|
|
48
|
+
if (match[0].includes('__row.__label')) title = title.replace(/\s*—\s*$/, '');
|
|
42
49
|
const bodyStart = match.index + match[0].length;
|
|
43
50
|
|
|
44
51
|
// Find matching closing brace (accounting for nested braces)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
await {{> locator-base}}.filter({ hasText: {{> has-text-value dataValue=dataValue exact=exact legacyTail=(and (eq strategy "text") (eq value ""))}} }){{> locator-nth}}.hover();
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
await expect({{> locator-base}}.filter({ hasText: {{> has-text-value dataValue=dataValue exact=exact legacyTail=(and (eq strategy "text") (eq value ""))}} }){{> locator-nth}}).toBeVisible();
|
|
@@ -8,6 +8,7 @@ import { DiagnosticsCollector } from './diagnostics';
|
|
|
8
8
|
import { capabilityRegistry } from '../../capabilities/registry';
|
|
9
9
|
import { discoverAndRegisterCapabilities } from '../../capabilities/discover';
|
|
10
10
|
import { readCapabilities } from '../../harness/capability';
|
|
11
|
+
import { parseManualComments } from '../../exporters/scenario-merger';
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
14
|
* Filter base scenario steps for @extend: only keep Given→When steps.
|
|
@@ -168,6 +169,20 @@ function getEffectiveAuthRole(
|
|
|
168
169
|
}
|
|
169
170
|
|
|
170
171
|
|
|
172
|
+
/**
|
|
173
|
+
* True when the scenario has at least one step in the Then bucket —
|
|
174
|
+
* And/But inherit the preceding primary keyword (Given/When/Then).
|
|
175
|
+
*/
|
|
176
|
+
function hasThenStep(scenario: ParsedScenario): boolean {
|
|
177
|
+
let bucket = '';
|
|
178
|
+
for (const step of scenario.steps) {
|
|
179
|
+
const kw = step.keyword.trim();
|
|
180
|
+
if (kw === 'Given' || kw === 'When' || kw === 'Then') bucket = kw;
|
|
181
|
+
if (bucket === 'Then') return true;
|
|
182
|
+
}
|
|
183
|
+
return false;
|
|
184
|
+
}
|
|
185
|
+
|
|
171
186
|
/**
|
|
172
187
|
* Check if tags contain @manual
|
|
173
188
|
* @manual at feature level → skip entire feature
|
|
@@ -538,6 +553,33 @@ export class CodeGenerator {
|
|
|
538
553
|
const renderedScenarios: Array<{ code: string; authRole?: string }> = [];
|
|
539
554
|
for (const scenario of feature.scenarios) {
|
|
540
555
|
if (isManual(scenario.tags)) {
|
|
556
|
+
// SG-W008 — two-direction rule for @manual scenarios:
|
|
557
|
+
// • HAS Gherkin steps → the Gherkin must be COMPLETE: Given/When/Then. A dangling
|
|
558
|
+
// When with no Then is invalid Gherkin, and comments can NOT substitute for the
|
|
559
|
+
// missing Then — either add the Then, or drop the steps and move the whole
|
|
560
|
+
// procedure into the `# Tester verifies:` block.
|
|
561
|
+
// • NO Gherkin steps → documentation-style manual: the comments must carry the
|
|
562
|
+
// FULL procedure the deliverable can extract — ≥1 numbered action line AND ≥1
|
|
563
|
+
// labeled `Observable:`/`Oracle:`/`Expected:` line (parseManualComments — the
|
|
564
|
+
// SAME parser delivery uses). Field finding (student-site, 87 scenarios): prose
|
|
565
|
+
// paragraphs and rationale-only comments exported EMPTY Steps + Expected cells.
|
|
566
|
+
const isStructured = scenario.steps.length > 0;
|
|
567
|
+
if (isStructured && !hasThenStep(scenario)) {
|
|
568
|
+
this.diagnostics.add('SG-W008',
|
|
569
|
+
`@manual scenario "${scenario.name}" has Gherkin steps but no Then — a When without Then is invalid Gherkin. Add the Then step, or remove the steps and write the full procedure (action steps, then "Observable:"/"Oracle:" lines) in its \`# Tester verifies:\` block`,
|
|
570
|
+
{ feature: feature.name, step: scenario.name });
|
|
571
|
+
} else if (!isStructured) {
|
|
572
|
+
const parsed = parseManualComments(scenario.comments ?? []);
|
|
573
|
+
if (!parsed.steps.length || !parsed.expected.length) {
|
|
574
|
+
const missing = !parsed.steps.length && !parsed.expected.length
|
|
575
|
+
? 'numbered action steps and a labeled oracle line'
|
|
576
|
+
: !parsed.steps.length ? 'numbered action steps before the Observable/Oracle lines'
|
|
577
|
+
: 'a labeled "Observable:"/"Oracle:"/"Expected:" line';
|
|
578
|
+
this.diagnostics.add('SG-W008',
|
|
579
|
+
`@manual scenario "${scenario.name}" is documentation-style but its \`# Tester verifies:\` comments lack ${missing} — prose the delivery parser cannot extract exports EMPTY Steps/Expected cells. Use numbered lines: "1. <action>" … "N. Observable: <what>" / "Oracle: <tool>"`,
|
|
580
|
+
{ feature: feature.name, step: scenario.name });
|
|
581
|
+
}
|
|
582
|
+
}
|
|
541
583
|
if (this.options.verbose) {
|
|
542
584
|
console.log(` ⊘ Skipped @manual scenario: ${scenario.name}`);
|
|
543
585
|
}
|
|
@@ -549,6 +591,18 @@ export class CodeGenerator {
|
|
|
549
591
|
continue;
|
|
550
592
|
}
|
|
551
593
|
|
|
594
|
+
// Skip @steps:<name> base scenarios — they are setup blocks that live ONLY inlined
|
|
595
|
+
// into @extend scenarios (stepsRegistry above). Compiling them as standalone tests
|
|
596
|
+
// duplicated coverage, skewed the Playwright count against the deliverable (the
|
|
597
|
+
// exporter rightly excludes them), and made script-check flag the extra test()
|
|
598
|
+
// as a hand-edit. Mirrors countSteps() and harness/parse loadScenarios().
|
|
599
|
+
if (scenario.stepsName) {
|
|
600
|
+
if (this.options.verbose) {
|
|
601
|
+
console.log(` ⊘ Skipped @steps base scenario: ${scenario.name}`);
|
|
602
|
+
}
|
|
603
|
+
continue;
|
|
604
|
+
}
|
|
605
|
+
|
|
552
606
|
// TQ-11 — @requires:<cap> with the cap NOT enabled → emit a clean skip stub (no driver
|
|
553
607
|
// imports, so the spec still loads), visible as skipped-with-reason. With the cap enabled it
|
|
554
608
|
// falls through and compiles as a real test.
|
|
@@ -11,12 +11,16 @@
|
|
|
11
11
|
* SG-W006 — qa/app.<env>.yaml found (per-environment profiles are unsupported by design)
|
|
12
12
|
* SG-W007 — browser-alert handler step written AFTER an action step: the listener
|
|
13
13
|
* registers too late, an earlier dialog auto-dismisses with no error
|
|
14
|
+
* SG-W008 — @manual scenario is incomplete in either direction: Gherkin steps without a
|
|
15
|
+
* Then (invalid Gherkin; comments cannot substitute), or a documentation-style
|
|
16
|
+
* manual (no steps) whose comments lack the extractable procedure — ≥1 numbered
|
|
17
|
+
* action line + ≥1 labeled Observable:/Oracle:/Expected: line
|
|
14
18
|
*
|
|
15
19
|
* Diagnostics never block generation by themselves; `sungen generate --strict`
|
|
16
20
|
* turns any collected diagnostic into a non-zero exit.
|
|
17
21
|
*/
|
|
18
22
|
|
|
19
|
-
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007';
|
|
23
|
+
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007' | 'SG-W008';
|
|
20
24
|
|
|
21
25
|
export interface Diagnostic {
|
|
22
26
|
code: DiagnosticCode;
|
|
@@ -5,11 +5,17 @@ const MARKER_PATTERN = /__SUNGEN_TD_([A-Za-z0-9_]+)__/;
|
|
|
5
5
|
* Three passes: comments, string literals, then regex literals.
|
|
6
6
|
*/
|
|
7
7
|
export function transformToRuntimeData(code: string, accessor: string = 'testData'): string {
|
|
8
|
-
// Pass 0: Comments — replace markers in // comments with decoded key name
|
|
9
|
-
// Prevents Pass 2 from misinterpreting // comment markers as regex delimiters
|
|
8
|
+
// Pass 0: Comments — replace markers in // comments with decoded key name.
|
|
9
|
+
// Prevents Pass 2 from misinterpreting // comment markers as regex delimiters.
|
|
10
|
+
// ANCHORED to line start (^\s*//) so a `//` INSIDE a string literal on a code line
|
|
11
|
+
// — e.g. a URL `https://careerforum.aki-inc.jp/` in a selector name — is NOT mistaken
|
|
12
|
+
// for a comment. Before this anchor, such a URL preceding a marker on the same line
|
|
13
|
+
// made Pass 0 strip the marker to its bare key, so Pass 1 never wrapped it in
|
|
14
|
+
// testData.get() (silent: `toHaveAttribute('href', 'official_site_url')`). Generated
|
|
15
|
+
// comments are always full-line, so anchoring loses nothing.
|
|
10
16
|
code = code.replace(
|
|
11
|
-
|
|
12
|
-
(_, before, enc, after) =>
|
|
17
|
+
/^(\s*)\/\/(.*)__SUNGEN_TD_([A-Za-z0-9_]+)__(.*)$/gm,
|
|
18
|
+
(_, indent, before, enc, after) => `${indent}//${before}${decodeKey(enc)}${after}`
|
|
13
19
|
);
|
|
14
20
|
|
|
15
21
|
// Pass 1: String literal context — handles both whole-string and embedded markers
|
package/src/harness/audit.ts
CHANGED
|
@@ -291,7 +291,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
291
291
|
findings.push(`DOWNSTREAM-SCOPE-MISSING: "${d.route}" is a navigation target but is covered only by a page-nav assertion — cover its content/guards, or scaffold it (\`sungen add --screen ${d.slug}\`).`);
|
|
292
292
|
}
|
|
293
293
|
for (const m of manualOracleResult.insufficient.slice(0, 8)) {
|
|
294
|
-
findings.push(`MANUAL-STEPS-INSUFFICIENT: "${m}" — a
|
|
294
|
+
findings.push(`MANUAL-STEPS-INSUFFICIENT: "${m}" — @manual is incomplete: either Gherkin steps without a Then (invalid Gherkin; comments can't substitute), or a documentation manual whose comments lack numbered action steps + a labeled Observable:/Oracle: line. Write \`# Tester verifies:\` as "1. <action>" … "N. Observable: <what>" / "Oracle: <tool>" — that is what the delivery parser extracts into Steps/Expected.`);
|
|
295
295
|
}
|
|
296
296
|
// TQ-2 — automatable @manual: deferred (usually cross-screen) but fully DSL-expressible.
|
|
297
297
|
for (const m of autoManual.scenarios.slice(0, 8)) {
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import * as fs from 'fs';
|
|
7
7
|
import * as path from 'path';
|
|
8
8
|
import { ScenarioInfo, loadScenarios, idPrefix } from './parse';
|
|
9
|
+
import { parseManualComments } from '../exporters/scenario-merger';
|
|
9
10
|
|
|
10
11
|
// ---------- #2 Downstream-scope ----------
|
|
11
12
|
|
|
@@ -96,11 +97,27 @@ export function manualOracle(featureText: string): ManualOracleResult {
|
|
|
96
97
|
for (const b of blocks(featureText)) {
|
|
97
98
|
if (!/@manual\b/.test(b)) continue;
|
|
98
99
|
manualTotal++;
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
//
|
|
103
|
-
|
|
100
|
+
// Two-direction rule (mirrors compile-time SG-W008):
|
|
101
|
+
// • HAS Gherkin steps → the Gherkin must be COMPLETE: Given/When/Then. A dangling
|
|
102
|
+
// When with no Then is invalid Gherkin, and comments can NOT substitute for the
|
|
103
|
+
// missing Then — the hybrid "steps + oracle-in-comments" shape is banned.
|
|
104
|
+
// • NO Gherkin steps → documentation-style manual: comments must carry the FULL
|
|
105
|
+
// procedure the deliverable can extract — ≥1 numbered action line AND ≥1 labeled
|
|
106
|
+
// Observable:/Oracle:/Expected: line (parseManualComments — the SAME parser the
|
|
107
|
+
// delivery exporter uses, so this gate can't drift from the exported cells).
|
|
108
|
+
// Prose paragraphs / rationale-only comments export EMPTY Steps + Expected.
|
|
109
|
+
const hasGherkinSteps = /^\s*(Given|When|Then)\s/m.test(b);
|
|
110
|
+
let bad: boolean;
|
|
111
|
+
if (hasGherkinSteps) {
|
|
112
|
+
bad = !/^\s*Then\s/m.test(b);
|
|
113
|
+
} else {
|
|
114
|
+
const commentLines = b.split('\n')
|
|
115
|
+
.filter((l) => /^\s*#/.test(l))
|
|
116
|
+
.map((l) => l.replace(/^\s*#\s?/, ''));
|
|
117
|
+
const parsed = parseManualComments(commentLines);
|
|
118
|
+
bad = !parsed.steps.length || !parsed.expected.length;
|
|
119
|
+
}
|
|
120
|
+
if (bad) {
|
|
104
121
|
const name = (b.match(/Scenario:\s*(.+)/) || [])[1] || '(unnamed)';
|
|
105
122
|
insufficient.push(name.trim().slice(0, 80));
|
|
106
123
|
}
|
|
@@ -177,12 +177,42 @@ create-test.
|
|
|
177
177
|
## Cross-artifact check — testcase ↔ data agree
|
|
178
178
|
|
|
179
179
|
`data validate` checks the data alone; **`sungen data crosscheck`** checks the testcase and its data
|
|
180
|
-
AGREE
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
180
|
+
AGREE **and are COMPLETE**: no dangling `{{var}}`; a cross-screen `shared` dataset is identical across
|
|
181
|
+
the journey; a declared state-precondition is surfaced; **and — even when you author `@cases` by hand
|
|
182
|
+
instead of merging `data gen` — the authored values COVER the BVA the field-map's constraints require
|
|
183
|
+
and the login experience checklist**:
|
|
184
|
+
- **`BVA_UNCOVERED`** — a bounded dimension is missing a boundary value. Classic miss: `email
|
|
185
|
+
minLength: 5` → you must have an email of length **4 (invalid), 5, 6**; a URL/code of length 5 does
|
|
186
|
+
NOT count (matched by type). Add the missing values (the exact BVA is in `data gen`'s output) or waive.
|
|
187
|
+
- **`EXPERIENCE_UNCOVERED`** — a login/auth unit is missing credential-state scenarios: **soft-deleted**,
|
|
188
|
+
locked, unverified, password-expired, **valid-with-surrounding-whitespace** (trim → accept). Add each
|
|
189
|
+
(a seeded account in that state) or waive per spec.
|
|
190
|
+
|
|
191
|
+
Treat these as MUST-ADDRESS in the create-test loop — they are exactly the gaps a QA reviewer catches.
|
|
192
|
+
Together `data validate` + `data crosscheck` are the **Data Gate + Cross-artifact + Coverage Gate** (the
|
|
193
|
+
`data-only` profile). Every `data gen` also stamps a `_fingerprint` (catalog+generator+context hash).
|
|
194
|
+
|
|
195
|
+
## Boundary completeness + trim
|
|
196
|
+
|
|
197
|
+
Boundary Value Analysis is **complete by construction**: for every bounded dimension in the field-map's
|
|
198
|
+
`constraints` — including ones the field-type's catalog didn't model (e.g. an **email `minLength`**) —
|
|
199
|
+
the generator auto-emits the full set **min-1 (invalid) · min · min+1 · max-1 · max · max+1 (invalid)**.
|
|
200
|
+
So a spec "email min 5" yields length 4 (invalid), 5, 6 — don't hand-add them. Text fields also carry a
|
|
201
|
+
**leading/trailing-whitespace** case (trim behaviour: the server must trim then accept, or reject per
|
|
202
|
+
spec — confirm). If `data validate`/the QA still finds a missing boundary, it means the constraint was
|
|
203
|
+
absent from the field-map — add it there.
|
|
204
|
+
|
|
205
|
+
## Login / authentication screens — credential-state cases (experience)
|
|
206
|
+
|
|
207
|
+
A login screen is not just field-format validation. The catalog carries an **experience checklist**
|
|
208
|
+
(`auth-account-states`) of credential/account STATES a login must cover — each a scenario, most needing
|
|
209
|
+
a domain-state precondition (a seeded account in that state; declare it with a state-recipe →
|
|
210
|
+
`@manual:data-setup` Background):
|
|
211
|
+
- **active** (happy path) · **wrong-password** · **non-existent** (same generic error — anti-enumeration)
|
|
212
|
+
- **soft-deleted** (must NOT authenticate — real-world bug precedent) · **locked/suspended** ·
|
|
213
|
+
**unverified-email** · **password-expired** · **valid-with-surrounding-whitespace** (trim → accept).
|
|
214
|
+
When the unit is a login/auth screen, generate these as scenarios (confirm the exact outcome/message
|
|
215
|
+
code with the spec). Do not assume the outcome — ask the QA for states the spec doesn't pin.
|
|
186
216
|
|
|
187
217
|
## Weaving into scenarios
|
|
188
218
|
|
|
@@ -48,8 +48,8 @@ Source modules: `src/exporters/*.ts`
|
|
|
48
48
|
| 4 | Compiled spec | `specs/generated/<name>/<name>.spec.ts` | `specs/generated/flows/<name>/<name>.spec.ts` | `sungen generate` |
|
|
49
49
|
| 5 | Test results | `specs/generated/<name>/<name>-test-result.json` or `test-results/results.json` | `specs/generated/flows/<name>/<name>-test-result.json` or global fallback | `run-test` |
|
|
50
50
|
|
|
51
|
-
**Sources 1-
|
|
52
|
-
**
|
|
51
|
+
**Sources 1-2 are blocking** (they are what every export row is built from) — CLI aborts if either is missing.
|
|
52
|
+
**Sources 3-5 are optional** — when selectors/spec/results are absent, delivery still exports a **pre-run overview**: content columns (Steps · Expected results · Test Data · Precondition) are complete from the feature + test-data, and every executable row shows **Pending**. This is the intended path for QA to review the test-case list right after `create-test`, before the first `run-test`. Once `.spec.ts`/results exist they are picked up automatically — no flag needed.
|
|
53
53
|
|
|
54
54
|
The CLI reads the **per-target result file first** (co-located with `.spec.ts`), then falls back to the global `test-results/results.json`. Per-target is preferred because the global file gets OVERWRITTEN each time Playwright runs, losing results from earlier targets.
|
|
55
55
|
|
|
@@ -76,6 +76,7 @@ needs any of these, it is a **finding for QA** — surface it in the run summary
|
|
|
76
76
|
| Error | Diagnosis | Fix |
|
|
77
77
|
|---|---|---|
|
|
78
78
|
| strict mode violation | Multiple elements match | Add `nth: 0`, `exact: true`, or more specific `name` |
|
|
79
|
+
| strict mode violation on `see`/`hover [X] item\|option\|cell with {{v}}` (dynamic list) | Older builds didn't filter the locator by the value | Since 3.2.13 `see`/`hover ... with {{v}}` on list-member roles filters by `hasText` like `click` (existence/hover the matching item) — keep the step data-driven, don't add per-item selectors. Update sungen if still unfiltered |
|
|
79
80
|
| Timeout / not found | Element doesn't exist or name wrong | Re-snapshot → copy exact accessible name. Check iframe/dialog scope |
|
|
80
81
|
| Element is not an input | Wrong element type targeted | Change `type` or `value` to match actual element |
|
|
81
82
|
| not a select | Custom dropdown, not native `<select>` | Set the widget variant: `variant: radix-select`/`antd-select` (built-in recipes), a custom recipe name from `qa/app.yaml`, or project-wide `widgets.select` in `qa/app.yaml` |
|
|
@@ -91,6 +92,7 @@ needs any of these, it is a **finding for QA** — surface it in the run summary
|
|
|
91
92
|
| `SG-W005` | `qa/app.yaml` unknown key / unknown preset / unparseable | Fix the key, or `extends: native\|radix\|antd` |
|
|
92
93
|
| `SG-W006` | `qa/app.<env>.yaml` found — unsupported by design | Describe the app once in `qa/app.yaml`; env differences belong in test-data overlays |
|
|
93
94
|
| `SG-W007` | Browser-alert handler step written AFTER an action — listener registers too late, the dialog auto-dismisses silently | Use the compound form `click [X] button and accept [OK] alert` (preferred), or move the alert step before the trigger |
|
|
95
|
+
| `SG-W008` | `@manual` scenario incomplete in either direction: (a) HAS Gherkin steps but NO `Then` — a dangling `When` is invalid Gherkin, comments can NOT substitute; (b) documentation-style (NO steps) but its comments lack numbered action lines and/or a labeled oracle line — prose paragraphs / rationale-only comments export EMPTY Steps + Expected cells | Two valid shapes only: complete the Gherkin with a `Then`, or write the FULL procedure in the `# Tester verifies:` block as NUMBERED lines — action steps first (`1. <action>`), then labeled `Observable:`/`Oracle:`/`Expected:` lines (the number is what fills Steps, the label is what fills Expected results) |
|
|
94
96
|
|
|
95
97
|
### Assertion errors → apply the Source-of-truth gate above FIRST
|
|
96
98
|
|
|
@@ -317,7 +317,11 @@ Security: [S1 – admin only]
|
|
|
317
317
|
3. **Claim-Proof** (`CLAIM-UNPROVEN`) — a title claiming `all`/`only`/`every`/`single`/`correct`/`same`/`changes`/`hidden`/`cleared`/`restored`/`independent`/`sanitized`/`announces` MUST have the matching assertion (`see all …`, count, `remember`+compare, `is hidden`, return-and-assert-empty, etc.). If the title promises it, the steps must prove it.
|
|
318
318
|
- **Negative / absence claims** (`does not` / `no` / `never` / `prevents` / `không` / `chưa` — any language; `no-side-effect/no-duplicate`, `negative-claim/absence`): the `Then` must **differ** between the claim holding and not holding. A terminal `see [X] page` that looks identical whether or not the bad thing happened proves nothing. For a side-effect that should NOT repeat (re-submit on back, re-charge, duplicate order, resend OTP), assert the **count is unchanged** (`User see [Records] table with {{one}}` / `row with {{count}}`); if it's not UI-observable, mark `@manual` with a request-count oracle (shape below). This is general — it covers any side-effect, not a fixed verb list.
|
|
319
319
|
4. **Downstream-scope** (`DOWNSTREAM-SCOPE-MISSING`) — when the spec's Navigation Flow / success target is **another screen** (e.g. a confirmation/sent page), don't stop at a terminal `see [X] page`. Either cover that screen's content/guards (if its viewpoint items are in scope — they often have their own `MS-*` IDs), or scaffold it (`sungen add --screen <name>`) and note the handoff. Do not silently drop the downstream surface.
|
|
320
|
-
5. **Manual-oracle** (`MANUAL-STEPS-INSUFFICIENT`) —
|
|
320
|
+
5. **Manual-oracle** (`MANUAL-STEPS-INSUFFICIENT` at audit, `SG-W008` at generate) — a `@manual` scenario has exactly TWO valid shapes; the hybrid (Gherkin steps with the oracle stashed in comments) is **banned**:
|
|
321
|
+
- **Shape A — documentation-style** (NO Gherkin steps at all): the whole procedure lives in comments — numbered **action steps first**, then labeled `Observable:`/`Oracle:` lines. Use whenever the expected is unconfirmed (SPEC-GAP) or any part of the procedure is outside the DSL (password manager, DevTools, browser zoom/viewport, multi-tab, DB seeding). Do NOT keep a stray Gherkin fragment (`Given … page` + one fill) — a `When` without `Then` is invalid Gherkin, and a partial fragment can INVERT the chronology (e.g. "save a credential first" rendering AFTER "open the page").
|
|
322
|
+
- **Shape B — complete Gherkin**: `Given/When/Then`, all steps present. Use when the expected IS known and only execution is manual.
|
|
323
|
+
Gate (`SG-W008`, both directions): a `@manual` with Gherkin steps but no `Then` fires — comments (even labeled `Oracle:` lines) can NOT substitute for the missing `Then`. A `@manual` with NO steps ALSO fires when its comments lack ≥1 numbered action line + ≥1 labeled `Observable:`/`Oracle:`/`Expected:` line — prose paragraphs and rationale-only comments export EMPTY Steps/Expected cells.
|
|
324
|
+
In shape A the observable/oracle lines MUST carry their **label** (`Observable:` / `Oracle:` / `Expected:`) — the delivery parser extracts Expected results **by label**; unlabeled prose all lands in the Steps column and the Expected cell exports EMPTY. And the numbered action steps MUST precede them — without the actions the tester has an oracle but no procedure. Use this comment shape:
|
|
321
325
|
```gherkin
|
|
322
326
|
@high @manual
|
|
323
327
|
Scenario: VP-… <claim>
|
|
@@ -325,8 +329,8 @@ Security: [S1 – admin only]
|
|
|
325
329
|
# Tester verifies:
|
|
326
330
|
# 1. <setup> e.g. seed a registered email; throttle the network
|
|
327
331
|
# 2. <action> e.g. click [Submit] with the request in flight
|
|
328
|
-
# 3. <
|
|
329
|
-
# 4. Oracle: <tool> e.g. DevTools Network panel / mail-catcher / NVDA
|
|
332
|
+
# 3. Observable: <what> e.g. Observable: only ONE POST is dispatched
|
|
333
|
+
# 4. Oracle: <tool> e.g. Oracle: DevTools Network panel / mail-catcher / NVDA
|
|
330
334
|
```
|
|
331
335
|
|
|
332
336
|
#### Tier 1 guard — minimum before writing scenarios
|