@sungen/driver-data-factory 3.2.12-beta.1 → 3.2.12-beta.10

package/src/crosscheck.ts

@@ -0,0 +1,149 @@
+/**
+ * Data Factory — Cross-artifact Gate (M3-B).
+ *
+ * `data validate` checks the DATA in isolation (values are semantically correct). This checks that the
+ * TESTCASE and its test-data actually AGREE — the "genuinely new piece" from the harness-merge review:
+ *   - every `{{var}}` a scenario references resolves to a key in the unit's test-data (no dangling ref);
+ *   - a cross-screen SHARED dataset (M3-A `shared:`) is present and identical across its member screens
+ *     (the registration setup→confirmation→register lineage: confirmation must show what setup buffered);
+ *   - a unit that declares a manual state-precondition (a state-recipe) surfaces it in the feature
+ *     (a `@manual` tag or a Background) rather than silently dropping the required state.
+ *
+ * Deterministic; no runtime, no LLM. Invoked from the create-test / create-data-test AI loop (and by
+ * `sungen data crosscheck`), exit-1 on an error finding.
+ */
+import { readFileSync, existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { parse as parseYaml } from 'yaml';
+import { loadStateRecipes, buildStatePlan, hasDatasource } from './state';
+
+export interface CrossFinding {
+  unit: string;
+  scenario?: string;
+  severity: 'error' | 'warn' | 'info';
+  code: 'DANGLING_VAR' | 'SHARED_MISSING' | 'SHARED_MISMATCH' | 'PRECONDITION_MISSING';
+  message: string;
+}
+
+/** Resolve a unit's feature dir + test-data path (screen or flow). */
+function unitPaths(cwd: string, name: string): { featureDir: string; testData: string } | null {
+  for (const kind of ['screens', 'flows']) {
+    const base = join(cwd, 'qa', kind, name);
+    if (existsSync(base)) return { featureDir: join(base, 'features'), testData: join(base, 'test-data', `${name}.yaml`) };
+  }
+  return null;
+}
+
+function readFeatureText(featureDir: string): string {
+  if (!existsSync(featureDir)) return '';
+  return readdirSync(featureDir)
+    .filter((f) => f.endsWith('.feature'))
+    .map((f) => readFileSync(join(featureDir, f), 'utf8'))
+    .join('\n');
+}
+
+/** Extract `{{var}}` roots referenced in a feature, excluding dynamic `{{$...}}` runtime vars. */
+export function referencedVars(featureText: string): string[] {
+  const out = new Set<string>();
+  const re = /\{\{\s*([^}]+?)\s*\}\}/g;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(featureText))) {
+    const tok = m[1].trim();
+    if (tok.startsWith('$')) continue; // {{$timestamp}} etc. — runtime dynamic, not a test-data key
+    const root = tok.split(/[.|[]/)[0].trim(); // login.email → login ; name|upper → name
+    if (root) out.add(root);
+  }
+  return [...out];
+}
+
+function loadYaml(path: string): Record<string, any> {
+  if (!existsSync(path)) return {};
+  try { return (parseYaml(readFileSync(path, 'utf8')) as Record<string, any>) || {}; } catch { return {}; }
+}
+
+/**
+ * Column names supplied per-row by every `@cases:<dataset>` referenced in the feature (where the
+ * dataset is a top-level LIST in test-data). A `{{col}}` inside a data-driven scenario resolves to
+ * the current row, not a top-level key — so these are valid roots, not dangling. Mirrors the
+ * compiler's data-driven lint, which is @cases-aware; without this, any @cases scenario that reads
+ * a row column (`{{input}}`, `{{code}}`, …) would be a false-positive DANGLING_VAR.
+ */
+export function casesColumns(featureText: string, data: Record<string, any>): Set<string> {
+  const cols = new Set<string>();
+  const re = /@cases:([A-Za-z0-9_.-]+)/g;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(featureText))) {
+    const ds = data[m[1]];
+    if (!Array.isArray(ds)) continue;
+    for (const row of ds) {
+      if (row && typeof row === 'object' && !Array.isArray(row)) {
+        for (const k of Object.keys(row)) cols.add(k);
+      }
+    }
+  }
+  return cols;
+}
+
+/** Cross-check one unit's feature ↔ test-data. */
+export function crosscheckUnit(cwd: string, name: string): CrossFinding[] {
+  const out: CrossFinding[] = [];
+  const paths = unitPaths(cwd, name);
+  if (!paths) return out;
+  const feature = readFeatureText(paths.featureDir);
+  if (!feature) return out;
+  const data = loadYaml(paths.testData);
+  const topKeys = new Set(Object.keys(data).filter((k) => !k.startsWith('_')));
+  const caseCols = casesColumns(feature, data); // @cases row columns resolve per-row, not as top-level keys
+
+  // 1. Dangling vars — a {{var}} whose root key is absent from test-data (and not a @cases column).
+  for (const v of referencedVars(feature)) {
+    if (!topKeys.has(v) && !caseCols.has(v)) {
+      out.push({ unit: name, severity: 'error', code: 'DANGLING_VAR', message: `scenario references {{${v}}} but no "${v}" key exists in test-data/${name}.yaml` });
+    }
+  }
+
+  // 3. Manual state-precondition declared but not surfaced in the feature.
+  const ds = hasDatasource(cwd);
+  for (const r of loadStateRecipes(cwd)) {
+    const plan = buildStatePlan(r, { datasource: ds });
+    if (plan.manual && new RegExp(`@dataFactory:${r.name}\\b`).test(feature) === false) {
+      const surfaced = /@manual/.test(feature) || /Background:/.test(feature);
+      if (!surfaced) out.push({ unit: name, severity: 'warn', code: 'PRECONDITION_MISSING', message: `state-recipe "${r.name}" needs a manual precondition, but the feature has no Background / @manual tag — weave its precondition block (\`sungen data state --recipe ${r.name}\`)` });
+    }
+  }
+  return out;
+}
+
+/** Cross-check a cross-screen SHARED dataset: present + identical across all member screens. */
+export function crosscheckShared(cwd: string): CrossFinding[] {
+  const out: CrossFinding[] = [];
+  for (const r of loadStateRecipes(cwd)) {
+    if (!r.shared) continue;
+    const { entity, screens } = r.shared;
+    const datasets = screens.map((s) => {
+      const paths = unitPaths(cwd, s);
+      const data = paths ? loadYaml(paths.testData) : {};
+      return { screen: s, present: paths != null && entity in data, value: data[entity] };
+    });
+    const present = datasets.filter((d) => d.present);
+    for (const d of datasets) if (!d.present) out.push({ unit: d.screen, severity: 'warn', code: 'SHARED_MISSING', message: `shared dataset "${entity}" (recipe "${r.name}") is not present in ${d.screen}'s test-data` });
+    // Identity: every present copy must equal the first.
+    if (present.length >= 2) {
+      const ref = JSON.stringify(present[0].value);
+      for (const d of present.slice(1)) {
+        if (JSON.stringify(d.value) !== ref) {
+          out.push({ unit: d.screen, severity: 'error', code: 'SHARED_MISMATCH', message: `shared dataset "${entity}" differs between ${present[0].screen} and ${d.screen} — the same values must flow across the ${r.name} journey (e.g. confirmation must show what setup entered)` });
+        }
+      }
+    }
+  }
+  return out;
+}
+
+/** Cross-check the whole project (all units + shared datasets). */
+export function crosscheckAll(cwd: string, units: string[]): CrossFinding[] {
+  const out: CrossFinding[] = [];
+  for (const u of units) out.push(...crosscheckUnit(cwd, u));
+  out.push(...crosscheckShared(cwd));
+  return out;
+}

package/src/detect.ts

@@ -0,0 +1,122 @@
+/**
+ * Data Factory — auto-detection (Layer A of the auto-orchestration design).
+ *
+ * Makes Sungen NOTICE what test-data work a unit needs, without QA having to know to run a command:
+ *   - a unit that HAS input fields but has NO field-map → its data was never standardized;
+ *   - a spec that describes a STATEFUL PRECONDITION (a required entity/state) with no state-recipe and
+ *     no Background/@manual precondition → the stateful setup is undeclared.
+ * Both are surfaced as advisory findings at `sungen generate`/`sungen audit` (the DF advisory sensor).
+ * Deterministic heuristics over spec.md + the feature; no runtime, no LLM. Advisory only — a nudge +
+ * an auto-draft prompt, never a hard fail (the fuzzy precondition scan must not block).
+ */
+import { join } from 'path';
+import { existsSync, readFileSync, readdirSync } from 'fs';
+import { loadStateRecipes } from './state';
+
+export interface DetectFinding {
+  code: 'UNSTANDARDIZED' | 'PRECONDITION_UNDECLARED';
+  scenario: string;
+  message: string;
+  severity: 'warn' | 'info';
+}
+
+function readIfExists(path: string): string {
+  try { return existsSync(path) ? readFileSync(path, 'utf8') : ''; } catch { return ''; }
+}
+
+function readSpec(dir: string): string {
+  return readIfExists(join(dir, 'requirements', 'spec.md'));
+}
+
+function readFeature(dir: string): string {
+  const fdir = join(dir, 'features');
+  if (!existsSync(fdir)) return '';
+  try {
+    return readdirSync(fdir).filter((f) => f.endsWith('.feature')).map((f) => readIfExists(join(fdir, f))).join('\n');
+  } catch { return ''; }
+}
+
+/**
+ * Does the unit take user input? Strongest signal: a Gherkin `fill [X] field` / `type` step; plus a
+ * spec field table row typed as input/select/textarea. A pure navigation / list / capture screen has
+ * none → we must NOT nag it ("no input fields → nothing to generate").
+ */
+export function hasInputFields(specText: string, featureText: string): boolean {
+  if (/\b(fill|type|enter|input|select|choose)\b\s+\[[^\]]+\]\s*(field|input|dropdown|textbox)?/i.test(featureText)) return true;
+  if (/\|\s*(input|select|textarea|checkbox|radio|combobox)\b/i.test(specText)) return true;
+  if (/\binput\s*\(|\bselect\s*\(|\btextarea\b/i.test(specText)) return true;
+  return false;
+}
+
+/** A field-map exists for this unit. */
+export function hasFieldMap(cwd: string, name: string): boolean {
+  return existsSync(join(cwd, 'qa', 'data-factory', `${name}.fields.yaml`));
+}
+
+/** Precondition signal phrases — a spec sentence hinting the test needs a pre-existing entity/state. */
+const PRECONDITION_SIGNALS = [
+  'requires', 'required before', 'precondition', 'pre-condition', 'prerequisite',
+  'logged in', 'authenticated', 'signed in', 'must be logged',
+  'already exists', 'already registered', 'existing ', 'pre-existing', 'must exist',
+  'reset token', 'verification', 'provisional account', 'session-storage', 'session storage',
+  'buffer', 'seeded', 'must have', 'previously', 'valid token', 'auth token',
+];
+
+/** Extract spec sentences that look like a stateful precondition (deduped, capped). */
+export function detectPreconditionSignals(specText: string): string[] {
+  if (!specText) return [];
+  const hits: string[] = [];
+  const seen = new Set<string>();
+  // Split into sentence-ish units (lines / bullet items / sentences).
+  const units = specText.split(/\n|(?<=[.。])\s+/).map((s) => s.replace(/\s+/g, ' ').trim()).filter(Boolean);
+  for (const u of units) {
+    const low = u.toLowerCase();
+    // Skip markdown headings/table separators, and spec-template meta lines (US labels, rationale)
+    // that mention auth/priority only descriptively — not an actual precondition.
+    if (/^#{1,6}\s|^\|?\s*-{3,}/.test(u)) continue;
+    if (/^\*\*(why|independent test|acceptance|business context|target users|priority)\b/i.test(u)) continue;
+    if (/\bpriority\b/i.test(low) && /\bp[0-9]\b/i.test(low)) continue;
+    const sig = PRECONDITION_SIGNALS.find((s) => low.includes(s));
+    if (sig) {
+      const key = low.slice(0, 60);
+      if (!seen.has(key)) { seen.add(key); hits.push(u.length > 160 ? u.slice(0, 157) + '…' : u); }
+    }
+    if (hits.length >= 4) break;
+  }
+  return hits;
+}
+
+/** Is a stateful precondition already declared — a state-recipe with resources, or a Background/@manual? */
+export function hasStateDeclaration(cwd: string, featureText: string): boolean {
+  if (/^\s*Background:/m.test(featureText) || /@manual\b/.test(featureText)) return true;
+  return loadStateRecipes(cwd).some((r) => (r.resources && r.resources.length) || r.shared);
+}
+
+/** Run Layer-A detection for one unit dir. Returns advisory findings (possibly empty). */
+export function detectUnit(cwd: string, dir: string, name: string): DetectFinding[] {
+  const spec = readSpec(dir);
+  const feature = readFeature(dir);
+  const out: DetectFinding[] = [];
+
+  // 1. Inputs exist but never standardized.
+  if (!hasFieldMap(cwd, name) && hasInputFields(spec, feature)) {
+    out.push({
+      code: 'UNSTANDARDIZED',
+      scenario: name,
+      severity: 'warn',
+      message: `DATA-FACTORY-UNSTANDARDIZED: ${name} has input fields but no standardized test-data (no qa/data-factory/${name}.fields.yaml) — run \`/sungen:create-data-test ${name}\` (or it is authored inline by \`/sungen:create-test\`) so values are standardized valid/boundary/invalid, not hand-invented.`,
+    });
+  }
+
+  // 2. Stateful precondition described in the spec but not declared anywhere.
+  const signals = detectPreconditionSignals(spec);
+  if (signals.length && !hasStateDeclaration(cwd, feature)) {
+    out.push({
+      code: 'PRECONDITION_UNDECLARED',
+      scenario: name,
+      severity: 'info',
+      message: `DATA-FACTORY-PRECONDITION-UNDECLARED: ${name}'s spec describes a stateful precondition but the test declares none (no state-recipe, no Background/@manual). Declare it with a state-recipe + \`sungen data state\` so it renders a @manual:data-setup Background. Signals: ${signals.map((s) => `"${s}"`).join('; ')}`,
+    });
+  }
+  return out;
+}

package/src/fixture.ts

@@ -6,9 +6,11 @@
  * Export` checklist area. Two products:
  *   • the VALID fixture  — N rows of valid values (should import cleanly). Unique fields get a
  *     per-row suffix so rows don't collide.
- *   • the INVALID fixture — one row per field's first invalid case (all-valid except that one field),
- *     plus a trailing `_violation` column naming the field + reason + expected error — so a row-level
- *     import-rejection test knows which row should fail and why.
+ *   • the INVALID fixture — one row per field's first invalid case (all-valid except that one field).
+ *     It carries the SAME headers as the valid file (no extra column) so a schema-strict importer
+ *     fails on the DATA, not on an unexpected column. The "which row is bad and why" lives in a
+ *     SIDECAR (`<name>.invalid.expected.yaml`), keyed by 0-based data-row index — the expected result
+ *     never alters the payload under test.
  *
  * Deterministic: same field-map + catalog → same CSV. Columns are the flattened leaf paths (nested
  * groups → dotted headers, e.g. `address.city`). Reference the file from an `@api` upload (`files:`)
@@ -31,7 +33,11 @@ function leaves(expanded: ExpandedField[], prefix = ''): Leaf[] {
 }
 
 const validOf = (f: ExpandedField): string => f.points.find((p) => p.group === 'valid' && p.value != null)?.value ?? '';
-const firstInvalid = (f: ExpandedField) => f.points.find((p) => p.group === 'invalid' && p.value != null);
+// First concrete invalid, preferring a plain (non-adversarial) format/boundary invalid over a
+// security payload — a bulk-import row-rejection test wants a data violation, not an injection string.
+const firstInvalid = (f: ExpandedField) =>
+  f.points.find((p) => p.group === 'invalid' && p.category !== 'adversarial' && p.value != null) ??
+  f.points.find((p) => p.group === 'invalid' && p.value != null);
 
 /** RFC-4180-ish CSV cell quoting. */
 function cell(v: string): string {
@@ -41,9 +47,23 @@ function row(cells: string[]): string {
   return cells.map(cell).join(',');
 }
 
+/** One expected-rejection entry in the sidecar, keyed to a 0-based data row of the invalid CSV. */
+export interface ExpectedRow {
+  row: number;
+  field: string;
+  chk: string;
+  reason: string;
+  expected_error?: string;
+}
+export interface FixtureExpected {
+  _note: string;
+  rows: ExpectedRow[];
+}
+
 export interface FixtureResult {
   valid: string;               // CSV text (header + N valid rows)
-  invalid?: string;            // CSV text (header + _violation + one bad row per field)
+  invalid?: string;            // CSV text (SAME headers as valid + one bad row per field — no extra column)
+  expected?: FixtureExpected;  // sidecar: per-row expected rejection (0-based data-row index)
   headers: string[];
 }
 
@@ -62,18 +82,30 @@ export function generateFixture(map: FieldMap, catalog: Catalog, opts: { rows?:
   const valid = [row(headers), ...validRows].join('\n') + '\n';
 
   let invalid: string | undefined;
+  let expected: FixtureExpected | undefined;
   if (opts.includeInvalid) {
-    const invHeaders = [...headers, '_violation'];
     const invRows: string[] = [];
+    const expectedRows: ExpectedRow[] = [];
     for (const target of ls) {
       const bad = firstInvalid(target.field);
       if (!bad) continue; // no concrete invalid value to seed
+      // Same columns as the valid file — only the target cell is bad. No `_violation` column: a
+      // schema-strict importer must fail on the data, not reject the file for an unexpected column.
       const cells = ls.map((l) => (l === target ? (bad.value as string) : validOf(l.field).replace(/\{\{\$timestamp\}\}/g, '1')));
-      const why = `${target.path}: ${bad.reason || 'invalid'}${bad.expectedError ? ` (${bad.expectedError})` : ''}`;
-      invRows.push(row([...cells, why]));
+      expectedRows.push(trimExpected({ row: invRows.length, field: target.path, chk: bad.id, reason: bad.reason || 'invalid', expected_error: bad.expectedError }));
+      invRows.push(row(cells));
     }
-    invalid = [row(invHeaders), ...invRows].join('\n') + '\n';
+    invalid = [row(headers), ...invRows].join('\n') + '\n';
+    expected = {
+      _note: 'Each row of the .invalid.csv should be REJECTED. `row` is the 0-based data-row index (header excluded). The CSV carries no extra columns so a strict importer fails on the data, not the schema.',
+      rows: expectedRows,
+    };
   }
 
-  return { valid, invalid, headers };
+  return { valid, invalid, expected, headers };
+}
+
+function trimExpected(e: ExpectedRow): ExpectedRow {
+  if (e.expected_error == null) { const { expected_error, ...rest } = e; return rest as ExpectedRow; }
+  return e;
 }