@sun-asterisk/sungen 3.2.12-beta.1 → 3.2.12-beta.3

package/src/exporters/csv-exporter.ts

@@ -25,7 +25,7 @@ import {
 import { ApiCatalogEntry, EnvironmentInfo, PlaywrightResult, ScreenSummary, TestCaseRow } from './types';
 import { collectSecretValues, formatApiExpected, formatApiRequest, hasApiCalls, redactSecrets } from './api-testcase-formatter';
 import { SelectorKeyMap, substituteSelectorKeysInStep } from './selector-key-resolver';
-import { resolveResultVariants } from './result-variants';
+import { formatVariantRow, resolveResultVariants } from './result-variants';
 
 export interface BuildCsvInput {
   screen: string;
@@ -70,7 +70,7 @@ export function buildTestCaseRows(input: BuildCsvInput): TestCaseRow[] {
     // Data-driven (@cases): expand to one CSV row per executed dataset input (Playwright titles
     // "<scenario> — <label>"); plain scenarios resolve their single result. Shared with the
     // dashboard/JSON exporter via resolveResultVariants so the two can't drift on @cases handling.
-    const variants = resolveResultVariants(m, input.results);
+    const variants = resolveResultVariants(m, input.results, input.testData);
 
    for (const variant of variants) {
     const displayName = `${m.feature.name}${variant.nameSuffix}`;
@@ -132,7 +132,11 @@ export function buildTestCaseRows(input: BuildCsvInput): TestCaseRow[] {
     // Multi-line bullet format keeps delivery CSV/XLSX in sync with the
     // dashboard's Test data section. Multi-line CSV cells are valid (quoted
     // automatically) and render natively as multi-line in XLSX.
-    const testData = formatTestData(m.feature.referencedVars, input.testData, Infinity, '\n');
+    let testData = formatTestData(m.feature.referencedVars, input.testData, Infinity, '\n');
+    // @cases: each report line shows ITS OWN dataset row (executed auto rows and manual
+    // split rows both carry variant.row) — one line per case, one dataset per line.
+    const variantRow = formatVariantRow(variant, '\n');
+    if (variantRow) testData = testData ? `${testData}\n${variantRow}` : variantRow;
 
     // Status for this (possibly per-input) row — resolved into `variant` above.
     const result: PlaywrightResult | undefined = variant.result;

package/src/exporters/feature-parser.ts

@@ -9,7 +9,7 @@ import { FeatureMetadata, OrderedStep, ScenarioMetadata } from './types';
 /**
  * Variables referenced in a scenario: find all {{var_name}} in step text.
  */
-function extractReferencedVars(scenario: ParsedScenario): string[] {
+function extractReferencedVars(scenario: ParsedScenario, includeComments = false): string[] {
   const vars = new Set<string>();
   for (const step of scenario.steps) {
     const matches = step.text.matchAll(/\{\{([a-zA-Z_][a-zA-Z0-9_.-]*)\}\}/g);
@@ -28,6 +28,19 @@ function extractReferencedVars(scenario: ParsedScenario): string[] {
       }
     }
   }
+  // Manual scenarios carry their real tester procedure in `# Tester verifies:` comment
+  // blocks — the {{var}} refs there are the data the tester needs, so include them
+  // (issue: manual test cases exported with an empty Test Data cell). Automated
+  // scenarios keep step-only scanning: their comments are notes (SPEC-GAP etc.) and
+  // would only add noise vars to the delivery report.
+  if (includeComments) {
+    for (const line of scenario.comments ?? []) {
+      const matches = line.matchAll(/\{\{([a-zA-Z_][a-zA-Z0-9_.-]*)\}\}/g);
+      for (const match of matches) {
+        vars.add(match[1]);
+      }
+    }
+  }
   return Array.from(vars);
 }
 
@@ -85,7 +98,7 @@ export function parseFeatureMetadata(featureFilePath: string): FeatureMetadata {
       tags: [...parsed.tags, ...sc.tags],
       stepsName: sc.stepsName,
       extendsName: sc.extendsName,
-      referencedVars: extractReferencedVars(sc),
+      referencedVars: extractReferencedVars(sc, extractTestcaseType([...parsed.tags, ...sc.tags]) === 'Manual'),
       rawGivenSteps: given,
       rawWhenSteps:  when,
       rawThenSteps:  then,

package/src/exporters/json-exporter.ts

@@ -23,7 +23,7 @@ import {
 } from './playwright-report-parser';
 import { ApiCatalogEntry, EnvironmentInfo, PlaywrightResult } from './types';
 import { SelectorKeyMap, substituteSelectorKeysInStep } from './selector-key-resolver';
-import { resolveResultVariants } from './result-variants';
+import { formatVariantRow, resolveResultVariants } from './result-variants';
 import {
   ScenarioSnapshot,
   ScreenSnapshot,
@@ -123,13 +123,17 @@ export function buildScreenSnapshot(input: BuildScreenSnapshotInput): ScreenSnap
     }
     // Dashboard modal uses whitespace-pre-wrap — pass '\n' to render one
     // key/value per line instead of the CSV-friendly "; " join.
-    const testDataStr = formatTestData(m.feature.referencedVars, input.testData, Infinity, '\n');
+    const baseTestDataStr = formatTestData(m.feature.referencedVars, input.testData, Infinity, '\n');
 
-    // @cases scenarios expand to one snapshot entry per executed dataset row (shared with the CSV
-    // exporter via resolveResultVariants), so their per-row pass/fail is counted — not collapsed
-    // into one Pending/N/A entry as before.
-    const variants = resolveResultVariants(m, input.results ?? null);
+    // @cases scenarios expand to one snapshot entry per executed dataset row — and manual
+    // @cases split per DATASET row (shared with the CSV exporter via resolveResultVariants),
+    // so their per-row pass/fail is counted — not collapsed into one Pending/N/A entry.
+    const variants = resolveResultVariants(m, input.results ?? null, input.testData);
     for (const variant of variants) {
+      // Per-line Test Data: shared vars + THIS variant's dataset row (parity with csv-exporter).
+      let testDataStr = baseTestDataStr;
+      const variantRow = formatVariantRow(variant, '\n');
+      if (variantRow) testDataStr = testDataStr ? `${testDataStr}\n${variantRow}` : variantRow;
       const displayName = `${m.feature.name}${variant.nameSuffix}`;
       const { vpId, category1 } = splitVpAndName(displayName);
       const tcId = generateTcId(input.screen, vpId, fallbackIndex);

package/src/exporters/result-variants.ts

@@ -5,12 +5,42 @@
  */
 import { MergedScenario } from './scenario-merger';
 import { PlaywrightResult } from './types';
+import { extractTestcaseType } from './feature-parser';
 
 export interface ResultVariant {
-  /** Appended to the scenario display name: `''` for a plain scenario, ` — <label>` per executed
+  /** Appended to the scenario display name: `''` for a plain scenario, ` — <label>` per
    *  @cases dataset row (so each row gets a distinct category1). */
   nameSuffix: string;
   result?: PlaywrightResult;
+  /** The @cases dataset row backing this variant — lets the exporters render the row's own
+   *  columns into the Test Data cell (`__`-prefixed bookkeeping keys excluded). Present on
+   *  executed auto rows (matched by label) and on manual split rows. */
+  row?: Record<string, unknown>;
+}
+
+/**
+ * Read the `@cases:<name>` dataset rows for a scenario from the (flattened) test-data map.
+ * `loadTestData()` flattens arrays to JSON strings — accept that shape and raw arrays.
+ * Scalar rows normalize to `{ value }`, mirroring the runtime TestDataLoader.cases().
+ */
+export function getCasesDatasetRows(
+  m: MergedScenario,
+  testData: Record<string, string> | null | undefined,
+): Array<Record<string, unknown>> | null {
+  const tag = m.feature.tags.find((t) => t.startsWith('@cases:'));
+  if (!tag || !testData) return null;
+  let list: unknown = testData[tag.slice('@cases:'.length)];
+  if (typeof list === 'string') {
+    try { list = JSON.parse(list); } catch { return null; }
+  }
+  if (!Array.isArray(list) || list.length === 0) return null;
+  return list.map((row) =>
+    row && typeof row === 'object' && !Array.isArray(row) ? (row as Record<string, unknown>) : { value: row });
+}
+
+/** Display label of a dataset row — mirrors the runtime TestDataLoader.cases() rule. */
+function rowLabel(row: Record<string, unknown>, i: number): string {
+  return String(row.case ?? row.name ?? row.label ?? `row ${i + 1}`);
 }
 
 /**
@@ -18,9 +48,12 @@ export interface ResultVariant {
  *
  * A `@cases:<dataset>` scenario compiles to one `test()` per dataset row, whose Playwright title is
  * `<scenario> — <label>` — expand into one variant per EXECUTED row by matching that prefix (so an
- * already-run row carries its real pass/fail). A plain scenario resolves its single result by the
- * spec test title. With no results — or a @cases scenario with no matching result yet — a lone
- * variant with no result is returned, so the caller marks it Pending / N/A.
+ * already-run row carries its real pass/fail), attaching the backing dataset row by label.
+ *
+ * A MANUAL `@cases` scenario never executes, so that expansion can never fire — split it from the
+ * DATASET instead: one variant per row (` — <label>`, no result → Pending), each carrying its row
+ * so the manual tester gets one report line + its own data per case to walk and record. An auto
+ * @cases scenario that has not run yet stays a single line (its expansion belongs to execution).
  *
  * NOTE: this deliberately does NOT gate on `m.spec`. For @cases the compiled test title carries a
  * `— ${__row.__label}` suffix, so `findMatchingSpecTest` (base-name match) often leaves `m.spec`
@@ -30,13 +63,26 @@ export interface ResultVariant {
 export function resolveResultVariants(
   m: MergedScenario,
   results: Map<string, PlaywrightResult> | null,
+  testData?: Record<string, string> | null,
 ): ResultVariant[] {
   const isCases = m.feature.tags.some((t) => t.startsWith('@cases:'));
-  if (isCases && results) {
-    const marker = `${m.feature.name} — `;
-    const rows = [...results.entries()].filter(([t]) => t.includes(marker));
-    if (rows.length) {
-      return rows.map(([t, r]) => ({ nameSuffix: ` — ${t.slice(t.indexOf(marker) + marker.length)}`, result: r }));
+  if (isCases) {
+    const datasetRows = getCasesDatasetRows(m, testData);
+    const byLabel = new Map<string, Record<string, unknown>>();
+    (datasetRows ?? []).forEach((row, i) => byLabel.set(rowLabel(row, i), row));
+
+    if (results) {
+      const marker = `${m.feature.name} — `;
+      const ran = [...results.entries()].filter(([t]) => t.includes(marker));
+      if (ran.length) {
+        return ran.map(([t, r]) => {
+          const label = t.slice(t.indexOf(marker) + marker.length);
+          return { nameSuffix: ` — ${label}`, result: r, row: byLabel.get(label) };
+        });
+      }
+    }
+    if (datasetRows && extractTestcaseType(m.feature.tags) === 'Manual') {
+      return datasetRows.map((row, i) => ({ nameSuffix: ` — ${rowLabel(row, i)}`, row }));
     }
     return [{ nameSuffix: '' }];
   }
@@ -45,3 +91,16 @@ export function resolveResultVariants(
   }
   return [{ nameSuffix: '' }];
 }
+
+/**
+ * Render a variant's dataset row for the Test Data cell: one `col: value` line per column,
+ * `__`-prefixed bookkeeping keys excluded. Returns '' when the variant carries no row.
+ */
+export function formatVariantRow(variant: ResultVariant, separator: string = '\n'): string {
+  if (!variant.row) return '';
+  const useBullet = separator.includes('\n');
+  return Object.entries(variant.row)
+    .filter(([k]) => !k.startsWith('__'))
+    .map(([k, v]) => `${useBullet ? '• ' : ''}${k}: ${v}`)
+    .join(separator);
+}

package/src/exporters/test-data-resolver.ts

@@ -133,6 +133,7 @@ function truncate(s: string, max: number): string {
   return s.substring(0, max - 1) + '…';
 }
 
+
 /**
  * Resolve `{{var}}` test-data placeholders in a step / expected-result line to their concrete values,
  * so the delivery report shows what QA actually verifies (e.g. `text contains {{app_version}}` →

package/src/orchestrator/templates/ai-src/commands/create-data-test.md

@@ -90,6 +90,10 @@ Run (local-first): `[ -x ./bin/sungen.js ] && ./bin/sungen.js data lint --screen
 
 Run (local-first): `[ -x ./bin/sungen.js ] && ./bin/sungen.js data gen --screen <name> || npx sungen data gen --screen <name>`. If `test-data/<name>.yaml` already exists it **merges** the standardized field data in — it **replaces only the field keys it owns and preserves every other key** (create-test's data, flow-namespaced keys, `@cases` datasets, overrides). It asks before merging (in ALL mode after they confirmed, add `--yes`; `--force` skips the prompt). The output is standardized `valid/boundary/invalid` values with `CHK-*` traceability.
 
+### 5.5 Validate semantic correctness
+
+Run (local-first): `[ -x ./bin/sungen.js ] && ./bin/sungen.js data validate --screen <name> || npx sungen data validate --screen <name>`. This is stronger than lint (coverage): it checks every generated value is **semantically** correct — each `valid`/`boundary` value actually satisfies the constraints, and each synthesized `invalid` value actually violates them (a value in the `valid` set that breaks `maxLength`, or a "below min" case that landed inside the range, is a `VALUE_NOT_VALID` error). Fix any `error`; `INVALID_LOOKS_VALID` warns are a prompt to confirm the value is invalid for a non-length reason (charset/format). Each generated value also carries an `expected: valid|invalid` field in the test-data for traceability.
+
 ### 6. Weave into scenarios (if create-test already ran)
 
 If the `.feature` exists, turn the generated invalid/boundary sets into data-driven `Scenario Outline` + `Examples` (`@cases`) referencing the values — following `sungen-gherkin-syntax`. Otherwise leave the standardized `test-data/<name>.yaml` for `/sungen:run-test` to consume.

package/src/orchestrator/templates/ai-src/skills/sungen-data-factory/SKILL.md

@@ -109,6 +109,14 @@ Then `sungen data gen` writes the standardized `test-data/<name>.yaml`. Do not h
 generated values — change the field-map or `.overwrite` and re-run (determinism: same input → same
 data).
 
+Then run **`sungen data validate`** — the semantic check beyond coverage. It verifies each generated
+value is *correct*, not just *present*: every `valid`/`boundary` value actually satisfies the
+constraints and every synthesized `invalid` value actually violates them. A boundary is expanded to
+**one case per point** (`min / min+1` → two cases; `< min` → min-1, below the minimum), each stamped
+`expected: valid|invalid` in the output. Any `VALUE_NOT_VALID` error means a value labeled valid breaks
+a constraint — fix the field-map/constraints and re-run. This is what stops the "structurally right but
+semantically wrong" data (e.g. a "below minimum length" case that is actually long enough to pass).
+
 ## Weaving into scenarios
 
 When a `.feature` exists, express invalid/boundary sets as data-driven `Scenario Outline` + `Examples`