@sungen/driver-data-factory 3.2.10-beta.6 → 3.2.12-beta.10

package/src/applicability.ts

@@ -0,0 +1,80 @@
+/**
+ * Data Factory — applicability (M2b).
+ *
+ * A catalog case is not universally applicable: an "empty (required)" invalid means nothing for an
+ * OPTIONAL field, a negative-number example only applies when the field allows negatives, a katakana
+ * or locale-specific case only applies under that locale, and a security payload is an *adversarial*
+ * case that belongs in its own bucket rather than mixed into plain `invalid`. Both reviews flagged
+ * this: the driver must return `not-applicable` (with a reason) instead of silently emitting a case
+ * that doesn't fit — and it must let Core (or a CLI flag) select a reduced profile.
+ *
+ * Pure + deterministic. Field-derived predicates (`required`, `negativesAllowed`) are always enforced;
+ * environment predicates (`locale`, `channel`, `sink`) only filter when the context supplies them.
+ */
+import type { DataPoint, SelectionContext, Profile } from './types';
+import type { FieldDef } from './generator';
+
+export interface FieldFacts {
+  required?: boolean;
+  min?: number;
+}
+
+export interface Applicability {
+  applicable: boolean;
+  /** Why it was excluded — shown to the QA ("not-applicable, không âm thầm sinh data"). */
+  reason?: string;
+}
+
+/** Evaluate a point's `when` conditions against the field facts + selection context. */
+export function evaluateWhen(when: Record<string, unknown> | undefined, facts: FieldFacts, ctx: SelectionContext): Applicability {
+  if (!when) return { applicable: true };
+  // Field-derived: always enforced (a definite field fact can rule the case out).
+  if (when.required === true && facts.required === false) {
+    return { applicable: false, reason: 'requires a required field; this field is optional' };
+  }
+  if (when.negativesAllowed === true && Number.isFinite(facts.min) && (facts.min as number) >= 0) {
+    return { applicable: false, reason: `requires negatives (min < 0); this field has min=${facts.min}` };
+  }
+  // Environment-derived: only filter when the context supplies the axis (else permissive).
+  if (typeof when.locale === 'string' && ctx.locale && !ctx.locale.toLowerCase().startsWith(String(when.locale).toLowerCase())) {
+    return { applicable: false, reason: `locale ${when.locale} ≠ context ${ctx.locale}` };
+  }
+  if (typeof when.channel === 'string' && ctx.channel && ctx.channel !== when.channel) {
+    return { applicable: false, reason: `channel ${when.channel} ≠ context ${ctx.channel}` };
+  }
+  if (typeof when.sink === 'string' && ctx.sink && ctx.sink !== when.sink) {
+    return { applicable: false, reason: `sink ${when.sink} ≠ context ${ctx.sink}` };
+  }
+  return { applicable: true };
+}
+
+/** Field facts the field-derived predicates read (required + numeric min after constraint merge). */
+export function fieldFacts(field: FieldDef, mergedConstraints: Record<string, any>): FieldFacts {
+  const min = mergedConstraints.min;
+  return { required: field.required, min: typeof min === 'number' ? min : undefined };
+}
+
+/**
+ * Is a point kept under the chosen profile? (Applied AFTER `when` applicability.) Rule-based over
+ * group + category, so it needs no per-point tagging:
+ *   - regression (default): everything.
+ *   - functional: valid + boundary + non-adversarial invalid (drops security/adversarial).
+ *   - smoke: valid + boundary (the "does it work" set — no invalid, no adversarial).
+ *   - security-min: valid + adversarial (security sanity).
+ */
+export function inProfile(group: 'valid' | 'boundary' | 'invalid', category: string | undefined, profile: Profile | undefined): boolean {
+  const adversarial = category === 'adversarial';
+  switch (profile) {
+    case undefined:
+    case 'regression':
+      return true;
+    case 'functional':
+      return !adversarial;
+    case 'smoke':
+      return group === 'valid' || group === 'boundary';
+    case 'security-min':
+      return group === 'valid' || adversarial;
+    default:
+      return true;
+  }
+}

package/src/boundary.ts

@@ -0,0 +1,319 @@
+/**
+ * Data Factory — structured boundary expansion + the deterministic validity oracle.
+ *
+ * This replaces the old prose-parsing (`hint.includes('min')`) that produced semantically WRONG data:
+ *   - password `< min` (minLength 7) synthesized a 7-char string — which is VALID, not the invalid
+ *     "below minimum" case it claims to be;
+ *   - integer `min / min+1` collapsed to a single value, dropping one of the two boundary cases.
+ *
+ * A boundary/invalid hint is tokenized into one or more BoundaryPoints — one case per point
+ * (`min / min+1` → two, `min-1 / max+1` → two) — each carrying the value AND its expected validity,
+ * computed by the same oracle that later re-checks the value. So "the rule says valid" and "the value
+ * actually satisfies the constraints" can never silently disagree (that disagreement is a golden-test
+ * failure). Pure, deterministic; no runtime, no LLM.
+ */
+
+export type Validity = 'valid' | 'invalid';
+
+export interface BoundaryPoint {
+  /** Id suffix so one catalog point can expand to several: '' (first), 'b', 'c', … */
+  suffix: string;
+  value: string | null;
+  /** What the oracle expects this value to be — locked by golden tests against the real check. */
+  validity: Validity;
+  /** Human label for the point, e.g. 'min', 'min+1', 'max+1', 'length-1'. */
+  label: string;
+  todo?: string;
+}
+
+const STR_TYPES = new Set(['string', 'password', 'phone', 'postcode', 'katakana']);
+const NUM_TYPES = new Set(['number-integer', 'number-float', 'currency']);
+
+/** Deterministic filler: a string of exactly `n` chars from a charset seed. */
+export function fillLen(n: number, base = 'a'): string {
+  if (!Number.isFinite(n) || n <= 0) return '';
+  return base.repeat(Math.ceil(n / base.length)).slice(0, n);
+}
+
+/** Password seed that satisfies upper+lower+digit for any length ≥ 3. */
+function fillPassword(n: number): string {
+  if (n <= 0) return '';
+  // Prefix guarantees the character classes; pad with lowercase to reach n.
+  const prefix = 'Aa1';
+  if (n <= prefix.length) return prefix.slice(0, n);
+  return prefix + 'a'.repeat(n - prefix.length);
+}
+
+function seedFor(type: string): string {
+  return type === 'password' ? 'PW' : type === 'katakana' ? 'ア' : type === 'phone' || type === 'postcode' ? '0' : 'a';
+}
+
+function fillString(type: string, n: number): string {
+  return type === 'password' ? fillPassword(n) : fillLen(n, seedFor(type));
+}
+
+/** Precision step for a numeric type: 1 for integers/currency(0dp), 10^-dp for floats. */
+function step(type: string, c: Record<string, any>): number {
+  if (type === 'number-float' || (type === 'currency' && Number.isFinite(c.decimalPlaces) && c.decimalPlaces > 0)) {
+    const dp = Number.isFinite(c.decimalPlaces) ? c.decimalPlaces : 2;
+    return Number((1 / Math.pow(10, dp)).toFixed(dp));
+  }
+  return 1;
+}
+
+/** Round to the type's decimal places so we don't emit float noise like 999999.9899999999. */
+function fmtNum(type: string, c: Record<string, any>, n: number): string {
+  if (type === 'number-float' || (type === 'currency' && c.decimalPlaces > 0)) {
+    const dp = Number.isFinite(c.decimalPlaces) ? c.decimalPlaces : 2;
+    return n.toFixed(dp);
+  }
+  return String(Math.round(n));
+}
+
+/** Split a prose hint into per-point segments: "min / min+1" → ["min","min+1"]. */
+function segments(hint: string): string[] {
+  return hint
+    .split(/[/,]/)
+    .map((s) => s.trim())
+    .filter(Boolean);
+}
+
+/** Resolve a single segment to a signed offset off min/max: "< min" → {base:'min',off:-1}. */
+function readOffset(seg: string): { base: 'min' | 'max' | 'length' | null; off: number; label: string } {
+  const s = seg.toLowerCase();
+  const base: 'min' | 'max' | 'length' | null =
+    /\bmax\b|maximum|maxlength|maxlocal|maxtotal/.test(s) ? 'max'
+    : /\bmin\b|minimum|minlength/.test(s) ? 'min'
+    : /exact|length|byte/.test(s) ? 'length'
+    : null;
+  // Offset: explicit ±1 wins; then relational (< / over / exceeds → out-of-range).
+  let off = 0;
+  if (/\+\s*1|plus one|one extra/.test(s)) off = 1;
+  else if (/-\s*1|minus one/.test(s)) off = -1;
+  else if (/<|below|under|less than|shorter/.test(s)) off = -1;
+  else if (/>|over|above|exceed|greater than|longer|too long/.test(s)) off = 1;
+  const sign = off > 0 ? '+1' : off < 0 ? '-1' : '';
+  const label = (base || 'bound') + sign;
+  return { base, off, label };
+}
+
+/**
+ * Expand a length-bounded string hint into one BoundaryPoint per segment.
+ * min/max come from the constraints; a point OUTSIDE [min,max] is invalid, inside/on-edge is valid.
+ */
+function expandLength(type: string, hint: string, c: Record<string, any>): BoundaryPoint[] {
+  const min = c.minLength ?? c.length ?? c.minlength;
+  const max = c.maxLength ?? c.length ?? c.maxlength;
+  const segs = segments(hint);
+  const out: BoundaryPoint[] = [];
+  let idx = 0;
+  for (const seg of segs) {
+    const { base, off, label } = readOffset(seg);
+    const anchor = base === 'max' ? max : base === 'length' ? (c.length ?? max ?? min) : min;
+    if (!Number.isFinite(anchor)) {
+      out.push({ suffix: suffixOf(idx++), value: null, validity: base === null ? 'valid' : 'valid', label, todo: 'minLength/maxLength/length' });
+      continue;
+    }
+    const n = anchor + off;
+    const lo = Number.isFinite(min) ? min : 0;
+    const hi = Number.isFinite(max) ? max : Infinity;
+    const validity: Validity = n >= lo && n <= hi && n >= 1 ? 'valid' : 'invalid';
+    out.push({ suffix: suffixOf(idx++), value: fillString(type, Math.max(0, n)), validity, label });
+  }
+  return out.length ? out : [{ suffix: '', value: null, validity: 'valid', label: 'bound', todo: 'minLength/maxLength' }];
+}
+
+/** Expand a numeric value hint. Float/currency use a precision step, not a fixed ±1. */
+function expandNumeric(type: string, hint: string, c: Record<string, any>): BoundaryPoint[] {
+  const min = c.min, max = c.max;
+  const st = step(type, c);
+  const segs = segments(hint);
+  const out: BoundaryPoint[] = [];
+  let idx = 0;
+  for (const seg of segs) {
+    const s = seg.toLowerCase();
+    // Decimal-places boundary (float): "exact decimal places" valid; "one extra digit" invalid.
+    if (/decimal|precision|extra digit/.test(s)) {
+      const dp = Number.isFinite(c.decimalPlaces) ? c.decimalPlaces : 2;
+      if (/extra|one extra|\+1|too many/.test(s)) {
+        out.push({ suffix: suffixOf(idx++), value: '0.' + '1'.repeat(dp + 1), validity: 'invalid', label: `decimalPlaces+1` });
+      } else {
+        out.push({ suffix: suffixOf(idx++), value: '0.' + '1'.repeat(Math.max(1, dp)), validity: 'valid', label: `decimalPlaces` });
+      }
+      continue;
+    }
+    const { base, off, label } = readOffset(seg);
+    const anchor = base === 'max' ? max : min;
+    if (!Number.isFinite(anchor)) {
+      out.push({ suffix: suffixOf(idx++), value: null, validity: 'valid', label, todo: 'min/max' });
+      continue;
+    }
+    const n = anchor + off * st;
+    const lo = Number.isFinite(min) ? min : -Infinity;
+    const hi = Number.isFinite(max) ? max : Infinity;
+    const validity: Validity = n >= lo && n <= hi ? 'valid' : 'invalid';
+    out.push({ suffix: suffixOf(idx++), value: fmtNum(type, c, n), validity, label });
+  }
+  return out.length ? out : [{ suffix: '', value: null, validity: 'valid', label: 'bound', todo: 'min/max' }];
+}
+
+/** Build an email at a target local or total length while honoring BOTH maxLocal and maxTotal. */
+function synthEmail(c: Record<string, any>, which: 'maxLocal' | 'maxTotal', off: number): string | null {
+  const maxLocal = Number(c.maxLocal);
+  const maxTotal = Number(c.maxTotal);
+  if (which === 'maxLocal') {
+    if (!Number.isFinite(maxLocal)) return null;
+    const localLen = Math.max(1, maxLocal + off);
+    return fillLen(localLen) + '@example.com';
+  }
+  // maxTotal: keep the local part ≤ maxLocal, pad the DOMAIN to reach the total length.
+  if (!Number.isFinite(maxTotal)) return null;
+  const total = maxTotal + off;
+  const localLen = Number.isFinite(maxLocal) ? Math.min(maxLocal, 24) : 24;
+  const fixed = 1 /* @ */ + 4 /* .com */;
+  const subLen = total - localLen - fixed;
+  if (subLen < 1) return fillLen(Math.max(1, total - fixed)) + '@x.io'.slice(0, Math.max(4, fixed));
+  return fillLen(localLen) + '@' + fillLen(subLen) + '.com';
+}
+
+/** Expand an email length hint (single point per catalog entry today, but honors both constraints). */
+function expandEmail(hint: string, c: Record<string, any>): BoundaryPoint[] {
+  const segs = segments(hint);
+  const out: BoundaryPoint[] = [];
+  let idx = 0;
+  for (const seg of segs) {
+    const { off, label } = readOffset(seg);
+    const s = seg.toLowerCase();
+    const which: 'maxLocal' | 'maxTotal' = /total/.test(s) ? 'maxTotal' : 'maxLocal';
+    const value = synthEmail(c, which, off);
+    if (value == null) { out.push({ suffix: suffixOf(idx++), value: null, validity: 'valid', label, todo: which }); continue; }
+    const validity: Validity = satisfiesEmail(value, c) ? 'valid' : 'invalid';
+    out.push({ suffix: suffixOf(idx++), value, validity, label: `${which}${off > 0 ? '+1' : off < 0 ? '-1' : ''}` });
+  }
+  return out.length ? out : [{ suffix: '', value: null, validity: 'valid', label: 'bound', todo: 'maxLocal/maxTotal' }];
+}
+
+function suffixOf(i: number): string {
+  return i === 0 ? '' : String.fromCharCode(96 + i); // 0→'' , 1→'a', 2→'b', … (id becomes CHK-INT-B1, CHK-INT-B1a, …)
+}
+
+/**
+ * Public entry: expand a boundary/invalid hint into structured points.
+ * Returns one point per segment; each carries the value AND the oracle's expected validity.
+ */
+export function expandBoundaryHint(hint: string, type: string, c: Record<string, any>): BoundaryPoint[] {
+  if (type === 'email') return expandEmail(hint, c);
+  if (STR_TYPES.has(type)) return expandLength(type, hint, c);
+  if (NUM_TYPES.has(type)) return expandNumeric(type, hint, c);
+  return [{ suffix: '', value: null, validity: 'valid', label: 'bound', todo: 'value/gen not resolvable' }];
+}
+
+// ---- The validity oracle: does a value structurally satisfy the constraints? --------------------
+
+export function satisfiesEmail(value: string, c: Record<string, any>): boolean {
+  const at = value.indexOf('@');
+  if (at <= 0 || at === value.length - 1) return false;
+  const local = value.slice(0, at);
+  if (Number.isFinite(c.maxLocal) && local.length > c.maxLocal) return false;
+  if (Number.isFinite(c.maxTotal) && value.length > c.maxTotal) return false;
+  // Total-length MIN (a spec often sets an email min length, e.g. 5) — the oracle must enforce it so a
+  // below-min email is correctly INVALID (was ignored → the min boundary silently missing).
+  const minTotal = c.minLength ?? c.minTotal;
+  if (Number.isFinite(minTotal) && value.length < minTotal) return false;
+  return true;
+}
+
+/** Build an email of exactly `n` total chars (for the total-length MIN boundary — small n). */
+export function emailOfTotalLength(n: number): string | null {
+  if (!Number.isFinite(n) || n < 1) return null;
+  if (n < 3) return 'a'.repeat(n);              // too short to be a real address → malformed (below-min case)
+  const rest = n - 2;                            // chars after "a@"
+  if (rest < 3) return 'a@' + 'b'.repeat(rest);  // e.g. n=4 → "a@bb" (no dot → also malformed, below-min)
+  return 'a@' + 'b'.repeat(rest - 2) + '.c';     // e.g. n=5 → "a@b.c", n=6 → "a@bb.c"
+}
+
+function satisfiesLength(type: string, value: string, c: Record<string, any>): boolean {
+  const min = c.minLength ?? c.length;
+  const max = c.maxLength ?? c.length;
+  if (Number.isFinite(min) && value.length < min) return false;
+  if (Number.isFinite(max) && value.length > max) return false;
+  return true;
+}
+
+function satisfiesNumeric(type: string, value: string, c: Record<string, any>): boolean {
+  const n = Number(value);
+  if (!Number.isFinite(n)) return false;
+  if (Number.isFinite(c.min) && n < c.min) return false;
+  if (Number.isFinite(c.max) && n > c.max) return false;
+  if ((type === 'number-integer') && !Number.isInteger(n)) return false;
+  if ((type === 'number-float' || type === 'currency') && Number.isFinite(c.decimalPlaces)) {
+    const dec = (value.split('.')[1] || '').length;
+    if (dec > c.decimalPlaces) return false;
+  }
+  return true;
+}
+
+/**
+ * The oracle. Given a concrete value and the constraints, decide whether it structurally satisfies
+ * them (length/range/precision — NOT charset/format, which the catalog encodes as literal invalids).
+ * Returns null when the type has no numeric/length constraint to check against (nothing to assert).
+ */
+export function satisfiesConstraints(type: string, value: string | null, c: Record<string, any>): boolean | null {
+  if (value == null) return null;
+  if (type === 'email') return satisfiesEmail(value, c);
+  if (STR_TYPES.has(type)) {
+    if (!Number.isFinite(c.minLength ?? c.length) && !Number.isFinite(c.maxLength ?? c.length)) return null;
+    return satisfiesLength(type, value, c);
+  }
+  if (NUM_TYPES.has(type)) {
+    if (!Number.isFinite(c.min) && !Number.isFinite(c.max) && !Number.isFinite(c.decimalPlaces)) return null;
+    return satisfiesNumeric(type, value, c);
+  }
+  return null;
+}
+
+// ---- BVA completeness: every bounded dimension gets its full min-1/min/min+1/max-1/max/max+1 set ----
+
+export interface BvaExtra { label: string; value: string; validity: Validity; group: 'boundary' | 'invalid'; }
+
+/**
+ * Ensure the canonical Boundary-Value-Analysis set is present for EVERY bounded dimension declared in
+ * the constraints — even one the field-type's catalog `gen:` entries don't exercise. This closes the
+ * QA-reported gap: email declared `minLength: 5` but the catalog only modelled maxLocal/maxTotal, so
+ * the min side (length 4 = min-1 invalid, 5 = min, 6 = min+1) was silently missing. Emits only the
+ * MISSING points (dedup by exact value), so a type whose catalog already covers a dimension (string /
+ * password / number) yields nothing — no churn.
+ */
+export function ensureBvaComplete(type: string, c: Record<string, any>, existing: Array<string | null>): BvaExtra[] {
+  const have = new Set(existing.filter((v): v is string => v != null));
+  const out: BvaExtra[] = [];
+  const push = (label: string, value: string | null) => {
+    if (value == null || have.has(value)) return;
+    const ok = satisfiesConstraints(type, value, c);
+    const validity: Validity = ok === false ? 'invalid' : 'valid';
+    out.push({ label, value, validity, group: validity === 'invalid' ? 'invalid' : 'boundary' });
+    have.add(value);
+  };
+  if (type === 'email') {
+    const minTotal = c.minLength ?? c.minTotal;
+    if (Number.isFinite(minTotal)) {
+      push('minTotal-1', emailOfTotalLength((minTotal as number) - 1));
+      push('minTotal', emailOfTotalLength(minTotal as number));
+      push('minTotal+1', emailOfTotalLength((minTotal as number) + 1));
+    }
+    return out;
+  }
+  if (STR_TYPES.has(type)) {
+    const min = c.minLength ?? c.length, max = c.maxLength ?? c.length;
+    if (Number.isFinite(min)) { push('minLen-1', fillString(type, min - 1)); push('minLen', fillString(type, min)); push('minLen+1', fillString(type, min + 1)); }
+    if (Number.isFinite(max)) { push('maxLen-1', fillString(type, max - 1)); push('maxLen', fillString(type, max)); push('maxLen+1', fillString(type, max + 1)); }
+    return out;
+  }
+  if (NUM_TYPES.has(type)) {
+    const min = c.min, max = c.max, st = step(type, c);
+    if (Number.isFinite(min)) { push('min-1', fmtNum(type, c, min - st)); push('min', fmtNum(type, c, min)); push('min+1', fmtNum(type, c, min + st)); }
+    if (Number.isFinite(max)) { push('max-1', fmtNum(type, c, max - st)); push('max', fmtNum(type, c, max)); push('max+1', fmtNum(type, c, max + st)); }
+    return out;
+  }
+  return out;
+}

package/src/catalog-loader.ts

@@ -14,7 +14,7 @@ import { readFileSync, readdirSync, existsSync, statSync } from 'fs';
 import { join } from 'path';
 import { createHash } from 'crypto';
 import { parse as parseYaml } from 'yaml';
-import type { Catalog, FieldType, SecurityBank, Technique, DataPoint, OverwriteWarning } from './types';
+import type { Catalog, FieldType, SecurityBank, Technique, Experience, DataPoint, OverwriteWarning, CatalogProvenance, FieldTypeProvenance } from './types';
 
 /** Resolve the packaged `common/` dir (works from src/ in dev via tsx and from dist/ when built). */
 export function commonDir(): string {
@@ -49,30 +49,70 @@ export function loadShippedCatalog(root = commonDir()): Catalog {
   for (const { data } of readYamlFilesDeep(join(root, 'techniques'))) {
     if (data?.technique) techniques[data.technique] = data as Technique;
   }
-  return { fieldTypes, security, techniques };
+  const experience: Record<string, Experience> = {};
+  for (const { data } of readYamlFilesDeep(join(root, 'experience'))) {
+    if (data?.experience) experience[data.experience] = data as Experience;
+  }
+  return { fieldTypes, security, techniques, experience };
 }
 
-/** Merge a field-type's group by DataPoint.id — overlay entries replace same-id, new ids append. */
-function mergeGroup(base: DataPoint[] = [], overlay: DataPoint[] = []): DataPoint[] {
+/**
+ * Merge a field-type group by DataPoint.id, honoring three explicit operations:
+ *   - EXTEND: an overlay id not in `base` is appended (`added`).
+ *   - REPLACE: an overlay id already in `base` wins (`replaced`).
+ *   - DISABLE: an id in `disableSet` is dropped (`disabled`).
+ * Records provenance and stamps each point's `source` so a reader can trace where it came from.
+ */
+function mergeGroup(
+  base: DataPoint[] = [],
+  overlay: DataPoint[] = [],
+  disableSet: Set<string>,
+  prov: FieldTypeProvenance,
+): DataPoint[] {
   const byId = new Map<string, DataPoint>();
-  for (const dp of base) byId.set(dp.id, dp);
-  for (const dp of overlay) byId.set(dp.id, { ...byId.get(dp.id), ...dp });
+  for (const dp of base) byId.set(dp.id, { ...dp, source: 'shipped' });
+  for (const dp of overlay) {
+    if (byId.has(dp.id)) prov.replaced.push(dp.id);
+    else prov.added.push(dp.id);
+    byId.set(dp.id, { ...byId.get(dp.id), ...dp, source: 'overwrite' });
+  }
+  for (const id of disableSet) {
+    if (byId.delete(id)) prov.disabled.push(id);
+  }
   return [...byId.values()];
 }
 
-function mergeFieldType(base: FieldType, overlay: Partial<FieldType>): FieldType {
+/** An overlay field-type entry may carry a `disable:` list of ids alongside partial field-type data. */
+type FieldTypeOverlay = Partial<FieldType> & { disable?: string[] };
+
+function mergeFieldType(base: FieldType, overlay: FieldTypeOverlay, prov: FieldTypeProvenance): FieldType {
+  const disableSet = new Set(overlay.disable || []);
+  const constraintsOverridden = Object.keys(overlay.constraints || {}).filter(
+    (k) => (base.constraints || {})[k] !== (overlay.constraints || {})[k],
+  );
+  prov.constraintsOverridden = constraintsOverridden;
+  prov.securityChanged = overlay.uses_security != null;
+  const { disable: _disable, ...rest } = overlay;
   return {
     ...base,
-    ...overlay,
+    ...rest,
     constraints: { ...(base.constraints || {}), ...(overlay.constraints || {}) },
     uses_security: overlay.uses_security ?? base.uses_security,
-    valid: mergeGroup(base.valid, overlay.valid),
-    boundary: mergeGroup(base.boundary, overlay.boundary),
-    invalid: mergeGroup(base.invalid, overlay.invalid),
+    valid: mergeGroup(base.valid, overlay.valid, disableSet, prov),
+    boundary: mergeGroup(base.boundary, overlay.boundary, disableSet, prov),
+    invalid: mergeGroup(base.invalid, overlay.invalid, disableSet, prov),
     notes: [...(base.notes || []), ...(overlay.notes || [])],
   };
 }
 
+/** Merge a security bank by payload id (same rules: replace same-id, append new). */
+function mergeSecurity(base: SecurityBank, overlay: Partial<SecurityBank>): SecurityBank {
+  const byId = new Map<string, any>();
+  for (const p of base.payloads || []) byId.set(p.id, p);
+  for (const p of overlay.payloads || []) byId.set(p.id, { ...byId.get(p.id), ...p });
+  return { ...base, ...overlay, payloads: [...byId.values()] };
+}
+
 /**
  * Load the effective catalog = shipped common + project overwrite overlay.
  * `projectDir` is the QA repo root; overlay is read from `qa/data-factory/common.overwrite.yaml`.
@@ -80,38 +120,84 @@ function mergeFieldType(base: FieldType, overlay: Partial<FieldType>): FieldType
 export function loadEffectiveCatalog(projectDir: string, root = commonDir()): {
   catalog: Catalog;
   warnings: OverwriteWarning[];
+  provenance: CatalogProvenance;
 } {
   const shipped = loadShippedCatalog(root);
   const overlayPath = join(projectDir, 'qa', 'data-factory', 'common.overwrite.yaml');
   const warnings: OverwriteWarning[] = [];
-  if (!existsSync(overlayPath)) return { catalog: shipped, warnings };
+  const provenance: CatalogProvenance = { fieldTypes: {}, newFieldTypes: [], security: { added: [], replaced: [] }, techniques: { added: [], replaced: [] } };
+  if (!existsSync(overlayPath)) return { catalog: shipped, warnings, provenance };
 
   const overlay = parseYaml(readFileSync(overlayPath, 'utf8')) || {};
-  const overlayFieldTypes: Record<string, Partial<FieldType>> = overlay.fieldTypes || overlay['field-types'] || {};
+  const overlayFieldTypes: Record<string, FieldTypeOverlay> = overlay.fieldTypes || overlay['field-types'] || {};
   const merged: Catalog = {
     fieldTypes: { ...shipped.fieldTypes },
     security: { ...shipped.security },
     techniques: { ...shipped.techniques },
+    experience: { ...shipped.experience },
   };
+
   for (const [type, ov] of Object.entries(overlayFieldTypes)) {
     if (merged.fieldTypes[type]) {
-      merged.fieldTypes[type] = mergeFieldType(merged.fieldTypes[type], ov);
+      const prov: FieldTypeProvenance = { type, disabled: [], replaced: [], added: [], constraintsOverridden: [], securityChanged: false };
+      // Warn on a disable target that does not exist in the shipped field-type (likely a typo/stale id).
+      const known = new Set([...(merged.fieldTypes[type].valid || []), ...(merged.fieldTypes[type].boundary || []), ...(merged.fieldTypes[type].invalid || [])].map((d) => d.id));
+      for (const id of ov.disable || []) {
+        if (!known.has(id)) warnings.push({ fieldType: type, detail: `disable target "${id}" does not exist in the shipped "${type}" catalog`, suggestion: `remove the stale id from the "${type}".disable list in qa/data-factory/common.overwrite.yaml` });
+      }
+      merged.fieldTypes[type] = mergeFieldType(merged.fieldTypes[type], ov, prov);
+      provenance.fieldTypes[type] = prov;
       // Drift warning: overlay references a security bank the shipped common no longer has.
       for (const bank of ov.uses_security || []) {
         if (!merged.security[bank]) {
           warnings.push({
             fieldType: type,
-            detail: `overwrite tham chiếu security bank "${bank}" không còn trong common`,
-            suggestion: `cập nhật hoặc xoá mục "${type}" trong qa/data-factory/common.overwrite.yaml`,
+            detail: `overwrite references security bank "${bank}" that no longer exists in the common catalog`,
+            suggestion: `update or remove the "${type}" entry in qa/data-factory/common.overwrite.yaml`,
           });
         }
       }
     } else {
       // Wholly new field-type from the user — accept as-is (project-specific type).
-      merged.fieldTypes[type] = ov as FieldType;
+      const { disable: _d, ...rest } = ov;
+      merged.fieldTypes[type] = rest as FieldType;
+      provenance.newFieldTypes.push(type);
     }
   }
-  return { catalog: merged, warnings };
+
+  // Security-bank overrides (`security:` in the overwrite) — replace same-id payloads, append new banks.
+  const overlaySecurity: Record<string, Partial<SecurityBank>> = overlay.security || {};
+  for (const [cls, ov] of Object.entries(overlaySecurity)) {
+    if (merged.security[cls]) { merged.security[cls] = mergeSecurity(merged.security[cls], ov); provenance.security.replaced.push(cls); }
+    else { merged.security[cls] = { class: cls, payloads: [], ...ov } as SecurityBank; provenance.security.added.push(cls); }
+  }
+
+  // Technique overrides (`techniques:` in the overwrite) — shallow replace/add by name.
+  const overlayTech: Record<string, Partial<Technique>> = overlay.techniques || {};
+  for (const [name, ov] of Object.entries(overlayTech)) {
+    if (merged.techniques[name]) { merged.techniques[name] = { ...merged.techniques[name], ...ov }; provenance.techniques.replaced.push(name); }
+    else { merged.techniques[name] = { technique: name, ...ov } as Technique; provenance.techniques.added.push(name); }
+  }
+
+  return { catalog: merged, warnings, provenance };
+}
+
+/** Human-readable one-liners describing what `.overwrite` changed (for lint/sensor "no silent overwrite"). */
+export function describeProvenance(p: CatalogProvenance): string[] {
+  const lines: string[] = [];
+  for (const ft of Object.values(p.fieldTypes)) {
+    const parts: string[] = [];
+    if (ft.disabled.length) parts.push(`disabled ${ft.disabled.join(', ')}`);
+    if (ft.replaced.length) parts.push(`replaced ${ft.replaced.join(', ')}`);
+    if (ft.added.length) parts.push(`added ${ft.added.join(', ')}`);
+    if (ft.constraintsOverridden.length) parts.push(`constraints[${ft.constraintsOverridden.join(', ')}]`);
+    if (ft.securityChanged) parts.push('uses_security replaced');
+    if (parts.length) lines.push(`${ft.type}: ${parts.join('; ')}`);
+  }
+  for (const t of p.newFieldTypes) lines.push(`${t}: new project field-type`);
+  if (p.security.replaced.length || p.security.added.length) lines.push(`security: ${[...p.security.replaced.map((s) => `replaced ${s}`), ...p.security.added.map((s) => `added ${s}`)].join('; ')}`);
+  if (p.techniques.replaced.length || p.techniques.added.length) lines.push(`techniques: ${[...p.techniques.replaced.map((s) => `replaced ${s}`), ...p.techniques.added.map((s) => `added ${s}`)].join('; ')}`);
+  return lines;
 }
 
 /** Stable hash of the shipped common — recorded in df-manifest.json to detect update drift. */