@sungen/driver-data-factory 3.2.12-beta.3 → 3.2.12-beta.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sungen/driver-data-factory",
-  "version": "3.2.12-beta.3",
+  "version": "3.2.12-beta.4",
   "description": "Sungen Data Factory capability — standardized test-data generation from a field-type + security knowledge catalog (valid/boundary/invalid + traceability), the .overwrite merge layer, and (opt-in) @dataFactory runtime seeding. Design-time knowledge is runtime-free. Plugs into @sun-asterisk/sungen via the capability SPI.",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -14,7 +14,7 @@
   "author": "eqe team (engineer & quality) — Sun Asterisk",
   "license": "MIT",
   "dependencies": {
-    "@sun-asterisk/sungen": "3.2.12-beta.3",
+    "@sun-asterisk/sungen": "3.2.12-beta.4",
     "yaml": "^2.4.0"
   },
   "files": [

package/src/catalog-loader.ts

@@ -14,7 +14,7 @@ import { readFileSync, readdirSync, existsSync, statSync } from 'fs';
 import { join } from 'path';
 import { createHash } from 'crypto';
 import { parse as parseYaml } from 'yaml';
-import type { Catalog, FieldType, SecurityBank, Technique, DataPoint, OverwriteWarning } from './types';
+import type { Catalog, FieldType, SecurityBank, Technique, DataPoint, OverwriteWarning, CatalogProvenance, FieldTypeProvenance } from './types';
 
 /** Resolve the packaged `common/` dir (works from src/ in dev via tsx and from dist/ when built). */
 export function commonDir(): string {
@@ -52,27 +52,63 @@ export function loadShippedCatalog(root = commonDir()): Catalog {
   return { fieldTypes, security, techniques };
 }
 
-/** Merge a field-type's group by DataPoint.id — overlay entries replace same-id, new ids append. */
-function mergeGroup(base: DataPoint[] = [], overlay: DataPoint[] = []): DataPoint[] {
+/**
+ * Merge a field-type group by DataPoint.id, honoring three explicit operations:
+ *   - EXTEND: an overlay id not in `base` is appended (`added`).
+ *   - REPLACE: an overlay id already in `base` wins (`replaced`).
+ *   - DISABLE: an id in `disableSet` is dropped (`disabled`).
+ * Records provenance and stamps each point's `source` so a reader can trace where it came from.
+ */
+function mergeGroup(
+  base: DataPoint[] = [],
+  overlay: DataPoint[] = [],
+  disableSet: Set<string>,
+  prov: FieldTypeProvenance,
+): DataPoint[] {
   const byId = new Map<string, DataPoint>();
-  for (const dp of base) byId.set(dp.id, dp);
-  for (const dp of overlay) byId.set(dp.id, { ...byId.get(dp.id), ...dp });
+  for (const dp of base) byId.set(dp.id, { ...dp, source: 'shipped' });
+  for (const dp of overlay) {
+    if (byId.has(dp.id)) prov.replaced.push(dp.id);
+    else prov.added.push(dp.id);
+    byId.set(dp.id, { ...byId.get(dp.id), ...dp, source: 'overwrite' });
+  }
+  for (const id of disableSet) {
+    if (byId.delete(id)) prov.disabled.push(id);
+  }
   return [...byId.values()];
 }
 
-function mergeFieldType(base: FieldType, overlay: Partial<FieldType>): FieldType {
+/** An overlay field-type entry may carry a `disable:` list of ids alongside partial field-type data. */
+type FieldTypeOverlay = Partial<FieldType> & { disable?: string[] };
+
+function mergeFieldType(base: FieldType, overlay: FieldTypeOverlay, prov: FieldTypeProvenance): FieldType {
+  const disableSet = new Set(overlay.disable || []);
+  const constraintsOverridden = Object.keys(overlay.constraints || {}).filter(
+    (k) => (base.constraints || {})[k] !== (overlay.constraints || {})[k],
+  );
+  prov.constraintsOverridden = constraintsOverridden;
+  prov.securityChanged = overlay.uses_security != null;
+  const { disable: _disable, ...rest } = overlay;
   return {
     ...base,
-    ...overlay,
+    ...rest,
     constraints: { ...(base.constraints || {}), ...(overlay.constraints || {}) },
     uses_security: overlay.uses_security ?? base.uses_security,
-    valid: mergeGroup(base.valid, overlay.valid),
-    boundary: mergeGroup(base.boundary, overlay.boundary),
-    invalid: mergeGroup(base.invalid, overlay.invalid),
+    valid: mergeGroup(base.valid, overlay.valid, disableSet, prov),
+    boundary: mergeGroup(base.boundary, overlay.boundary, disableSet, prov),
+    invalid: mergeGroup(base.invalid, overlay.invalid, disableSet, prov),
     notes: [...(base.notes || []), ...(overlay.notes || [])],
   };
 }
 
+/** Merge a security bank by payload id (same rules: replace same-id, append new). */
+function mergeSecurity(base: SecurityBank, overlay: Partial<SecurityBank>): SecurityBank {
+  const byId = new Map<string, any>();
+  for (const p of base.payloads || []) byId.set(p.id, p);
+  for (const p of overlay.payloads || []) byId.set(p.id, { ...byId.get(p.id), ...p });
+  return { ...base, ...overlay, payloads: [...byId.values()] };
+}
+
 /**
  * Load the effective catalog = shipped common + project overwrite overlay.
  * `projectDir` is the QA repo root; overlay is read from `qa/data-factory/common.overwrite.yaml`.
@@ -80,22 +116,32 @@ function mergeFieldType(base: FieldType, overlay: Partial<FieldType>): FieldType
 export function loadEffectiveCatalog(projectDir: string, root = commonDir()): {
   catalog: Catalog;
   warnings: OverwriteWarning[];
+  provenance: CatalogProvenance;
 } {
   const shipped = loadShippedCatalog(root);
   const overlayPath = join(projectDir, 'qa', 'data-factory', 'common.overwrite.yaml');
   const warnings: OverwriteWarning[] = [];
-  if (!existsSync(overlayPath)) return { catalog: shipped, warnings };
+  const provenance: CatalogProvenance = { fieldTypes: {}, newFieldTypes: [], security: { added: [], replaced: [] }, techniques: { added: [], replaced: [] } };
+  if (!existsSync(overlayPath)) return { catalog: shipped, warnings, provenance };
 
   const overlay = parseYaml(readFileSync(overlayPath, 'utf8')) || {};
-  const overlayFieldTypes: Record<string, Partial<FieldType>> = overlay.fieldTypes || overlay['field-types'] || {};
+  const overlayFieldTypes: Record<string, FieldTypeOverlay> = overlay.fieldTypes || overlay['field-types'] || {};
   const merged: Catalog = {
     fieldTypes: { ...shipped.fieldTypes },
     security: { ...shipped.security },
     techniques: { ...shipped.techniques },
   };
+
   for (const [type, ov] of Object.entries(overlayFieldTypes)) {
     if (merged.fieldTypes[type]) {
-      merged.fieldTypes[type] = mergeFieldType(merged.fieldTypes[type], ov);
+      const prov: FieldTypeProvenance = { type, disabled: [], replaced: [], added: [], constraintsOverridden: [], securityChanged: false };
+      // Warn on a disable target that does not exist in the shipped field-type (likely a typo/stale id).
+      const known = new Set([...(merged.fieldTypes[type].valid || []), ...(merged.fieldTypes[type].boundary || []), ...(merged.fieldTypes[type].invalid || [])].map((d) => d.id));
+      for (const id of ov.disable || []) {
+        if (!known.has(id)) warnings.push({ fieldType: type, detail: `disable target "${id}" does not exist in the shipped "${type}" catalog`, suggestion: `remove the stale id from the "${type}".disable list in qa/data-factory/common.overwrite.yaml` });
+      }
+      merged.fieldTypes[type] = mergeFieldType(merged.fieldTypes[type], ov, prov);
+      provenance.fieldTypes[type] = prov;
       // Drift warning: overlay references a security bank the shipped common no longer has.
       for (const bank of ov.uses_security || []) {
         if (!merged.security[bank]) {
@@ -108,10 +154,45 @@ export function loadEffectiveCatalog(projectDir: string, root = commonDir()): {
       }
     } else {
       // Wholly new field-type from the user — accept as-is (project-specific type).
-      merged.fieldTypes[type] = ov as FieldType;
+      const { disable: _d, ...rest } = ov;
+      merged.fieldTypes[type] = rest as FieldType;
+      provenance.newFieldTypes.push(type);
     }
   }
-  return { catalog: merged, warnings };
+
+  // Security-bank overrides (`security:` in the overwrite) — replace same-id payloads, append new banks.
+  const overlaySecurity: Record<string, Partial<SecurityBank>> = overlay.security || {};
+  for (const [cls, ov] of Object.entries(overlaySecurity)) {
+    if (merged.security[cls]) { merged.security[cls] = mergeSecurity(merged.security[cls], ov); provenance.security.replaced.push(cls); }
+    else { merged.security[cls] = { class: cls, payloads: [], ...ov } as SecurityBank; provenance.security.added.push(cls); }
+  }
+
+  // Technique overrides (`techniques:` in the overwrite) — shallow replace/add by name.
+  const overlayTech: Record<string, Partial<Technique>> = overlay.techniques || {};
+  for (const [name, ov] of Object.entries(overlayTech)) {
+    if (merged.techniques[name]) { merged.techniques[name] = { ...merged.techniques[name], ...ov }; provenance.techniques.replaced.push(name); }
+    else { merged.techniques[name] = { technique: name, ...ov } as Technique; provenance.techniques.added.push(name); }
+  }
+
+  return { catalog: merged, warnings, provenance };
+}
+
+/** Human-readable one-liners describing what `.overwrite` changed (for lint/sensor "no silent overwrite"). */
+export function describeProvenance(p: CatalogProvenance): string[] {
+  const lines: string[] = [];
+  for (const ft of Object.values(p.fieldTypes)) {
+    const parts: string[] = [];
+    if (ft.disabled.length) parts.push(`disabled ${ft.disabled.join(', ')}`);
+    if (ft.replaced.length) parts.push(`replaced ${ft.replaced.join(', ')}`);
+    if (ft.added.length) parts.push(`added ${ft.added.join(', ')}`);
+    if (ft.constraintsOverridden.length) parts.push(`constraints[${ft.constraintsOverridden.join(', ')}]`);
+    if (ft.securityChanged) parts.push('uses_security replaced');
+    if (parts.length) lines.push(`${ft.type}: ${parts.join('; ')}`);
+  }
+  for (const t of p.newFieldTypes) lines.push(`${t}: new project field-type`);
+  if (p.security.replaced.length || p.security.added.length) lines.push(`security: ${[...p.security.replaced.map((s) => `replaced ${s}`), ...p.security.added.map((s) => `added ${s}`)].join('; ')}`);
+  if (p.techniques.replaced.length || p.techniques.added.length) lines.push(`techniques: ${[...p.techniques.replaced.map((s) => `replaced ${s}`), ...p.techniques.added.map((s) => `added ${s}`)].join('; ')}`);
+  return lines;
 }
 
 /** Stable hash of the shipped common — recorded in df-manifest.json to detect update drift. */

package/src/cli.ts

@@ -10,7 +10,7 @@ import { readFileSync, writeFileSync, existsSync, readdirSync, mkdirSync } from
 import { join, dirname } from 'path';
 import { createInterface } from 'readline';
 import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
-import { loadEffectiveCatalog } from './catalog-loader';
+import { loadEffectiveCatalog, describeProvenance } from './catalog-loader';
 import { expandFieldMap, renderTestData, type FieldMap } from './generator';
 import { lintFieldMap, summarize } from './lint';
 import { validateGenerated } from './validate';
@@ -51,8 +51,9 @@ function loadFieldMap(cwd: string, name: string): FieldMap | null {
 async function genOne(cwd: string, name: string, opts: { force?: boolean }): Promise<'written' | 'skipped' | 'missing'> {
   const map = loadFieldMap(cwd, name);
   if (!map) { console.error(`  ✗ ${name}: no field-map at qa/data-factory/${name}.fields.yaml (author it first, e.g. via /sungen:create-data-test)`); return 'missing'; }
-  const { catalog, warnings } = loadEffectiveCatalog(cwd);
+  const { catalog, warnings, provenance } = loadEffectiveCatalog(cwd);
   for (const w of warnings) console.warn(`  ⚠ overwrite: ${w.detail} → ${w.suggestion}`);
+  for (const line of describeProvenance(provenance)) console.log(`  ⟲ .overwrite → ${line}`);
 
   const expanded = expandFieldMap(map, catalog);
   const out = testDataPath(cwd, name);
@@ -174,6 +175,23 @@ export function registerDataFactoryCommand(program: any): void {
       if (totalErrors) process.exitCode = 1;
     });
 
+  data
+    .command('provenance')
+    .description('Show what qa/data-factory/common.overwrite.yaml changed vs the shipped company catalog (extend/replace/disable) — "no silent overwrite"')
+    .action(() => {
+      const cwd = process.cwd();
+      ensureDataFactoryReadme(cwd);
+      const { provenance, warnings } = loadEffectiveCatalog(cwd);
+      const lines = describeProvenance(provenance);
+      if (lines.length === 0) {
+        console.log('No project overrides — the effective catalog is the shipped company common as-is.');
+      } else {
+        console.log('Project overrides (qa/data-factory/common.overwrite.yaml):');
+        for (const l of lines) console.log(`  ⟲ ${l}`);
+      }
+      for (const w of warnings) console.warn(`  ⚠ ${w.detail} → ${w.suggestion}`);
+    });
+
   data
     .command('fixture')
     .description('Generate a CSV import fixture from the field-map (bulk-import / file-upload testing)')

package/src/generator.ts

@@ -71,6 +71,8 @@ export interface ExpandedPoint {
    * literal charset/format invalid (or a security payload) legitimately satisfies the length oracle.
    */
   synthesized?: boolean;
+  /** Provenance: `overwrite` when the project's `.overwrite` replaced/added this point (else shipped). */
+  source?: 'shipped' | 'overwrite';
 }
 
 export interface ExpandedField {
@@ -84,6 +86,27 @@ export interface ExpandedField {
   dependsOn?: { field: string; value?: string };
 }
 
+/**
+ * Coerce an out-of-range numeric VALID literal into range, so a project's `valid` representative is
+ * actually valid for its constraints (a generic catalog example like `123` under `max: 100` becomes
+ * `100`). Returns null when in range already, or the type isn't numeric (non-numeric mismatches are
+ * left for the validator to flag — the field-map should supply a fitting value).
+ */
+function coerceValidNumeric(type: string, value: string, c: Record<string, any>): string | null {
+  if (!['number-integer', 'number-float', 'currency'].includes(type)) return null;
+  const n = Number(value);
+  if (!Number.isFinite(n)) return null;
+  let out = n;
+  if (Number.isFinite(c.min) && out < c.min) out = c.min;
+  if (Number.isFinite(c.max) && out > c.max) out = c.max;
+  if (out === n) return null; // already in range
+  if (type === 'number-float' || (type === 'currency' && c.decimalPlaces > 0)) {
+    const dp = Number.isFinite(c.decimalPlaces) ? c.decimalPlaces : 2;
+    return out.toFixed(dp);
+  }
+  return String(Math.round(out));
+}
+
 /** A gen hint that references a numeric/length boundary the generator can expand structurally. */
 function isBoundaryHint(hint: string): boolean {
   return /\bmin\b|\bmax\b|minimum|maximum|length|byte|exact|decimal|precision|local|total|over|below|under|exceed|\+\s*1|-\s*1|<|>/.test(
@@ -138,19 +161,35 @@ export function expandField(field: FieldDef, catalog: Catalog): ExpandedField {
   const emit = (p: Omit<ExpandedPoint, 'expectedValidity'> & { expectedValidity: Validity }) => points.push(p);
 
   const pushLiteral = (dp: DataPoint, group: ExpandedPoint['group'], value: string | null, opts: { validity?: Validity; label?: string; todo?: string } = {}) => {
+    // A valid representative must be valid for the project: clamp an out-of-range numeric literal.
+    let coerceNote: string | undefined;
+    if (group === 'valid' && value != null && satisfiesConstraints(type, value, c) === false) {
+      const fixed = coerceValidNumeric(type, value, c);
+      if (fixed != null) { value = fixed; coerceNote = 'adjusted to fit constraints'; }
+    }
     const isUniqueValid = group === 'valid' && field.unique && value != null;
     const v = isUniqueValid ? makeUnique(value!, type) : value;
-    // Oracle: if the constraints can judge this value, they win; else fall back to declared/group validity.
-    const oracle = satisfiesConstraints(type, isUniqueValid ? value : v, c);
-    const validity: Validity = dp.validity ?? (oracle == null ? (opts.validity ?? groupValidity(group)) : oracle ? 'valid' : 'invalid');
+    // Expected validity reflects INTENT by group, so the validator (validateGenerated) can catch a
+    // mismatch against the oracle instead of the mismatch being silently absorbed here:
+    //   - valid group   → `valid` (a value that actually violates the constraints is a VALUE_NOT_VALID).
+    //   - invalid group → `invalid` (declared: charset/format/blocklist/behavior — the length oracle
+    //     cannot "prove" such a literal valid, so it must NOT flip it to valid).
+    //   - boundary group → the oracle decides (that group is precisely about range).
+    let validity: Validity;
+    if (dp.validity) validity = dp.validity;
+    else if (group === 'boundary') {
+      const oracle = satisfiesConstraints(type, isUniqueValid ? value : v, c);
+      validity = oracle == null ? (opts.validity ?? 'valid') : oracle ? 'valid' : 'invalid';
+    } else validity = opts.validity ?? groupValidity(group);
     emit({
       id: dp.id,
       group,
       value: v,
       todo: opts.todo,
       reason: dp.reason,
-      note: isUniqueValid ? `${dp.note ? dp.note + ' · ' : ''}fresh per run` : (opts.label ? `${dp.note ? dp.note + ' · ' : ''}${opts.label}` : dp.note),
+      note: isUniqueValid ? `${dp.note ? dp.note + ' · ' : ''}fresh per run` : coerceNote ? `${dp.note ? dp.note + ' · ' : ''}${coerceNote}` : (opts.label ? `${dp.note ? dp.note + ' · ' : ''}${opts.label}` : dp.note),
       expectedValidity: validity,
+      ...(dp.source === 'overwrite' ? { source: 'overwrite' as const } : {}),
       ...resolveError(dp, field.errorMap),
     });
   };
@@ -173,6 +212,7 @@ export function expandField(field: FieldDef, catalog: Catalog): ExpandedField {
           note: bp.label && !bp.todo ? `${dp.note ? dp.note + ' · ' : ''}${bp.label}` : dp.note,
           expectedValidity: dp.validity ?? bp.validity,
           synthesized: bp.value != null,
+          ...(dp.source === 'overwrite' ? { source: 'overwrite' as const } : {}),
           ...resolveError(dp, field.errorMap),
         });
       }
@@ -244,8 +284,8 @@ function renderOne(f: ExpandedField): Record<string, any> {
   return {
     ...dep,
     valid: valid[0]?.value ?? null,
-    boundary: boundary.map((p) => trim({ id: p.id, value: p.value, expected: p.expectedValidity, note: p.note, todo: p.todo })),
-    invalid: invalid.map((p) => trim({ id: p.id, value: p.value, expected: p.expectedValidity, expected_error: p.expectedError, open: p.open, reason: p.reason, todo: p.todo })),
+    boundary: boundary.map((p) => trim({ id: p.id, value: p.value, expected: p.expectedValidity, via: p.source === 'overwrite' ? '.overwrite' : undefined, note: p.note, todo: p.todo })),
+    invalid: invalid.map((p) => trim({ id: p.id, value: p.value, expected: p.expectedValidity, via: p.source === 'overwrite' ? '.overwrite' : undefined, expected_error: p.expectedError, open: p.open, reason: p.reason, todo: p.todo })),
   };
 }
 

package/src/index.ts

@@ -18,7 +18,7 @@ import { registerDataFactoryCommand } from './cli';
 import { dataFactoryCoverageSensor } from './sensor';
 import { dfPreconditionCodegen } from './precondition';
 
-export { loadShippedCatalog, loadEffectiveCatalog, shippedCommonHash, commonDir } from './catalog-loader';
+export { loadShippedCatalog, loadEffectiveCatalog, shippedCommonHash, commonDir, describeProvenance } from './catalog-loader';
 export { expandField, expandFieldMap, renderTestData } from './generator';
 export type { FieldDef, FieldMap, ExpandedField, ExpandedPoint } from './generator';
 export { lintFieldMap, summarize } from './lint';
@@ -32,7 +32,7 @@ export { generateFixture } from './fixture';
 export type { FixtureResult } from './fixture';
 export { dataFactoryCoverageSensor } from './sensor';
 export { dfPreconditionCodegen } from './precondition';
-export type { Catalog, FieldType, SecurityBank, Technique, DataPoint, OverwriteWarning } from './types';
+export type { Catalog, FieldType, SecurityBank, Technique, DataPoint, OverwriteWarning, CatalogProvenance, FieldTypeProvenance } from './types';
 
 /** Register the Data Factory capability. */
 export function register(registry: CapabilityRegistry): void {

package/src/types.ts

@@ -24,6 +24,12 @@ export interface DataPoint {
    * (e.g. a format-invalid value whose length happens to sit inside the range).
    */
   validity?: 'valid' | 'invalid';
+  /**
+   * Where this point ended up coming from after the `.overwrite` merge — `shipped` (company common) or
+   * `overwrite` (the project replaced/added it). Set by the loader; carried through to the generated
+   * value so a reader can answer "why is this value here?". Not authored by hand.
+   */
+  source?: 'shipped' | 'overwrite';
 }
 
 export interface FieldType {
@@ -66,3 +72,27 @@ export interface OverwriteWarning {
   detail: string;
   suggestion: string;
 }
+
+/** What the `.overwrite` layer changed for one field-type (provenance — "no silent overwrite"). */
+export interface FieldTypeProvenance {
+  type: string;
+  /** Ids removed via `disable:` (an id that was not in the shipped common is reported as a warning). */
+  disabled: string[];
+  /** Ids that existed in shipped AND the overlay — the overlay value won. */
+  replaced: string[];
+  /** Ids new from the overlay (a project-specific case). */
+  added: string[];
+  /** Constraint keys the overlay changed. */
+  constraintsOverridden: string[];
+  /** True when the overlay replaced `uses_security`. */
+  securityChanged: boolean;
+}
+
+/** Catalog-wide provenance of the shipped-common → project-overwrite merge. */
+export interface CatalogProvenance {
+  fieldTypes: Record<string, FieldTypeProvenance>;
+  /** Wholly new field-types added by the project (not in shipped common). */
+  newFieldTypes: string[];
+  security: { added: string[]; replaced: string[] };
+  techniques: { added: string[]; replaced: string[] };
+}