goldenmatch 0.1.0 → 0.3.1

package/README.md

@@ -9,7 +9,7 @@ npm install goldenmatch
 [![npm](https://img.shields.io/npm/v/goldenmatch?color=d4a017)](https://www.npmjs.com/package/goldenmatch)
 [![Node](https://img.shields.io/node/v/goldenmatch?color=339933)](https://nodejs.org/)
 [![License: MIT](https://img.shields.io/badge/license-MIT-green)](https://github.com/benzsevern/goldenmatch/blob/main/LICENSE)
-[![Tests](https://img.shields.io/badge/tests-478%20passing-brightgreen)](https://github.com/benzsevern/goldenmatch/tree/main/packages/goldenmatch-js/tests)
+[![Tests](https://img.shields.io/badge/tests-590%20passing-brightgreen)](https://github.com/benzsevern/goldenmatch/tree/main/packages/goldenmatch-js/tests)
 
 ---
 
@@ -18,7 +18,7 @@ npm install goldenmatch
 - **Edge-safe core** — the matching engine runs in browsers, Workers, Vercel Edge Runtime, Deno
 - **Pure TypeScript** — no native dependencies required; peer deps unlock performance (hnswlib, ONNX, piscina)
 - **Feature parity with Python goldenmatch** — same scorers, same clustering, same YAML configs
-- **478 tests, strict TypeScript** — `noUncheckedIndexedAccess`, `exactOptionalPropertyTypes`
+- **590 tests, strict TypeScript** — `noUncheckedIndexedAccess`, `exactOptionalPropertyTypes`
 
 ## Quick Start
 
@@ -45,6 +45,58 @@ for (const record of result.goldenRecords) {
 }
 ```
 
+## Auto-Config Verification (v0.3)
+
+Auto-generated configs are now checked both before the pipeline runs and after
+scoring finishes, so you get actionable diagnostics instead of silent failures
+on edge-case data.
+
+### Preflight — six static checks
+
+When you call `autoConfigureRows(rows)`, the returned config ships with a
+`_preflightReport` summarising six config-time checks:
+
+1. **missing_column** — matchkey/blocking references a column not in the data
+2. **cardinality_high** — a column is near-unique (poor blocking signal)
+3. **cardinality_low** — a column has too few distinct values to discriminate
+4. **block_size** — a blocking key would produce oversized blocks
+5. **remote_asset** — a scorer requires a model download (gated offline)
+6. **weight_confidence** — a weighted matchkey's weights look unbalanced
+
+Many findings trigger **auto-repairs** (field dropped, scorer swapped,
+weight clamped). `hasErrors === true` on unrepairable errors raises
+`ConfigValidationError` with the full report attached.
+
+```ts
+import { autoConfigureRows, ConfigValidationError } from "goldenmatch";
+
+const cfg = autoConfigureRows(rows);
+for (const f of cfg._preflightReport!.findings) {
+  console.log(`[${f.severity}] ${f.check}/${f.subject}: ${f.message}`);
+}
+```
+
+Defaults are **offline-safe**: remote-asset scorers (cross-encoder, remote
+embeddings) are dropped unless you opt in with `allowRemoteAssets: true`.
+
+### Postflight — four runtime signals
+
+Inside `dedupe()` / `match()`, after scoring but before clustering, the
+pipeline computes four signals attached as `result.postflightReport`:
+
+1. **scoreHistogram** — 100-bin pair-score distribution
+2. **blockSizePercentiles** + **preliminaryClusterSizes** — p50/p95/p99/max
+3. **thresholdOverlapPct** — fraction of pairs near the current threshold
+4. **oversizedClusters** — components above size limit, with bottleneck pair
+
+If the score distribution is clearly bimodal, postflight proposes a
+threshold adjustment. In **strict mode** (`autoConfigureRows(rows, { strict: true })`
+or manual `_strictAutoconfig: true`) the signals are still emitted but the
+threshold is never touched — use this for reproducible CI pipelines.
+
+See `examples/verificationInspection.ts` and `examples/strictModeParity.ts`
+for runnable demos.
+
 ## Three entrypoints
 
 ```typescript

package/dist/cli.cjs

@@ -2492,6 +2492,261 @@ var init_golden = __esm({
     ];
   }
 });
+var init_domain = __esm({
+  "src/core/domain.ts"() {
+    new Set(
+      [
+        "apple",
+        "samsung",
+        "sony",
+        "lg",
+        "dell",
+        "hp",
+        "lenovo",
+        "asus",
+        "acer",
+        "microsoft",
+        "google",
+        "amazon",
+        "bose",
+        "canon",
+        "nikon",
+        "panasonic",
+        "philips",
+        "toshiba"
+      ].map((s) => s.toLowerCase())
+    );
+  }
+});
+
+// src/core/autoconfigVerify.ts
+function signalScoreHistogram(pairScores) {
+  const bins = [];
+  for (let i = 0; i <= 100; i++) bins.push(i / 100);
+  const counts = new Array(100).fill(0);
+  for (const p of pairScores) {
+    const idx = Math.min(99, Math.max(0, Math.floor(p.score * 100)));
+    counts[idx] += 1;
+  }
+  const smoothed = counts.map((_, i) => {
+    let sum = 0;
+    let n = 0;
+    for (let j = -2; j <= 2; j++) {
+      const k = i + j;
+      if (k >= 0 && k < counts.length) {
+        sum += counts[k];
+        n += 1;
+      }
+    }
+    return sum / Math.max(1, n);
+  });
+  const max = Math.max(...smoothed, 0);
+  const mean = smoothed.reduce((a, b) => a + b, 0) / Math.max(1, smoothed.length);
+  const minHeight = Math.max(max * 0.3, mean * 2);
+  const peaks = [];
+  for (let i = 1; i < smoothed.length - 1; i++) {
+    if (smoothed[i] >= minHeight && smoothed[i] > smoothed[i - 1] && smoothed[i] >= smoothed[i + 1]) {
+      peaks.push(i);
+    }
+  }
+  if (peaks.length < 2) {
+    return {
+      histogram: { bins, counts },
+      valleyLocation: null,
+      isBimodal: false
+    };
+  }
+  const first = peaks[0];
+  const last = peaks[peaks.length - 1];
+  if (last - first <= 10) {
+    return {
+      histogram: { bins, counts },
+      valleyLocation: null,
+      isBimodal: false
+    };
+  }
+  let valleyIdx = first;
+  let valleyVal = smoothed[first];
+  for (let i = first + 1; i < last; i++) {
+    if (smoothed[i] < valleyVal) {
+      valleyVal = smoothed[i];
+      valleyIdx = i;
+    }
+  }
+  const depthRatio = valleyVal / Math.min(smoothed[first], smoothed[last]);
+  if (depthRatio >= 0.5) {
+    return {
+      histogram: { bins, counts },
+      valleyLocation: null,
+      isBimodal: false
+    };
+  }
+  return {
+    histogram: { bins, counts },
+    valleyLocation: valleyIdx / 100,
+    isBimodal: true
+  };
+}
+function getFirstWeightedThreshold(config) {
+  for (const mk of config.matchkeys ?? []) {
+    if (mk.type === "weighted") return mk.threshold;
+  }
+  return null;
+}
+function signalBlockingRecall() {
+  return "deferred";
+}
+function signalBlockSizePercentiles(rows, config) {
+  const b = config.blocking;
+  if (b === void 0 || b.keys.length === 0 || rows.length === 0) {
+    return { p50: 0, p95: 0, p99: 0, max: 0 };
+  }
+  const sample = rows.slice(0, Math.min(rows.length, 1e4));
+  const sizes = [];
+  for (const key of b.keys) {
+    if (key.fields.length === 0) continue;
+    const counts = /* @__PURE__ */ new Map();
+    for (const row of sample) {
+      const parts = key.fields.map((f) => {
+        const v = row[f];
+        return v === null || v === void 0 ? "\0" : String(v);
+      });
+      const k = parts.join("
");
+      counts.set(k, (counts.get(k) ?? 0) + 1);
+    }
+    for (const n of counts.values()) sizes.push(n);
+  }
+  sizes.sort((a, b2) => a - b2);
+  const pct = (q) => sizes.length === 0 ? 0 : sizes[Math.min(sizes.length - 1, Math.floor(sizes.length * q))] ?? 0;
+  return {
+    p50: pct(0.5),
+    p95: pct(0.95),
+    p99: pct(0.99),
+    max: sizes.length === 0 ? 0 : sizes[sizes.length - 1]
+  };
+}
+function signalThresholdOverlap(pairScores, threshold) {
+  if (pairScores.length === 0) return 0;
+  const lo = threshold - 0.02;
+  const hi = threshold + 0.02;
+  let inBand = 0;
+  for (const p of pairScores) {
+    if (p.score >= lo && p.score <= hi) inBand += 1;
+  }
+  return inBand / pairScores.length;
+}
+function signalClusterSizes(pairScores, threshold) {
+  const above = pairScores.filter((p) => p.score >= threshold);
+  const parent = /* @__PURE__ */ new Map();
+  const find = (x) => {
+    let root = x;
+    while ((parent.get(root) ?? root) !== root) root = parent.get(root);
+    let y = x;
+    while ((parent.get(y) ?? y) !== root) {
+      const next = parent.get(y) ?? y;
+      parent.set(y, root);
+      y = next;
+    }
+    return root;
+  };
+  const union = (a, b) => {
+    const ra = find(a);
+    const rb = find(b);
+    if (ra !== rb) parent.set(ra, rb);
+  };
+  for (const p of above) {
+    if (!parent.has(p.idA)) parent.set(p.idA, p.idA);
+    if (!parent.has(p.idB)) parent.set(p.idB, p.idB);
+    union(p.idA, p.idB);
+  }
+  const sizeByRoot = /* @__PURE__ */ new Map();
+  const membersByRoot = /* @__PURE__ */ new Map();
+  for (const id of parent.keys()) {
+    const r = find(id);
+    sizeByRoot.set(r, (sizeByRoot.get(r) ?? 0) + 1);
+    if (!membersByRoot.has(r)) membersByRoot.set(r, /* @__PURE__ */ new Set());
+    membersByRoot.get(r).add(id);
+  }
+  const sizes = Array.from(sizeByRoot.values()).sort((a, b) => a - b);
+  const pct = (q) => sizes.length === 0 ? 0 : sizes[Math.min(sizes.length - 1, Math.floor(sizes.length * q))] ?? 0;
+  const percentiles = {
+    p50: pct(0.5),
+    p95: pct(0.95),
+    p99: pct(0.99),
+    max: sizes.length === 0 ? 0 : sizes[sizes.length - 1],
+    count: sizes.length
+  };
+  const oversized = [];
+  let clusterId = 0;
+  for (const [root, size] of sizeByRoot) {
+    if (size <= 100) continue;
+    const members = membersByRoot.get(root);
+    let bottleneckPair = [-1, -1];
+    let minScore = Infinity;
+    for (const p of above) {
+      if (members.has(p.idA) && members.has(p.idB) && p.score < minScore) {
+        minScore = p.score;
+        bottleneckPair = [
+          Math.min(p.idA, p.idB),
+          Math.max(p.idA, p.idB)
+        ];
+      }
+    }
+    oversized.push({ clusterId: clusterId++, size, bottleneckPair });
+  }
+  return { percentiles, oversized };
+}
+function postflight(rows, config, options) {
+  const currentThreshold = options.currentThreshold ?? getFirstWeightedThreshold(config) ?? 0.7;
+  const hist = signalScoreHistogram(options.pairScores);
+  const adjustments = [];
+  const advisories = [];
+  if (hist.isBimodal && hist.valleyLocation !== null) {
+    if (!config._strictAutoconfig && Math.abs(hist.valleyLocation - currentThreshold) > 0.05) {
+      adjustments.push({
+        field: "threshold",
+        fromValue: currentThreshold,
+        toValue: hist.valleyLocation,
+        reason: "histogram valley location differs from current threshold",
+        signal: "scoreHistogram"
+      });
+    }
+  } else {
+    advisories.push(
+      "score distribution is unimodal; threshold cannot be auto-set"
+    );
+  }
+  const clusterResult = signalClusterSizes(
+    options.pairScores,
+    currentThreshold
+  );
+  const overlapPct = signalThresholdOverlap(
+    options.pairScores,
+    currentThreshold
+  );
+  if (overlapPct > 0.2 && config.llmScorer?.enabled !== true) {
+    advisories.push(
+      `${(overlapPct * 100).toFixed(1)}% of pairs within threshold \xB10.02 \u2014 consider enabling LLM auto mode for calibration`
+    );
+  }
+  const blockSizePercentiles = signalBlockSizePercentiles(rows, config);
+  const signals = {
+    scoreHistogram: hist.histogram,
+    blockingRecall: signalBlockingRecall(),
+    blockSizePercentiles,
+    thresholdOverlapPct: overlapPct,
+    totalPairsScored: options.pairScores.length,
+    currentThreshold,
+    preliminaryClusterSizes: clusterResult.percentiles,
+    oversizedClusters: clusterResult.oversized
+  };
+  return { signals, adjustments, advisories };
+}
+var init_autoconfigVerify = __esm({
+  "src/core/autoconfigVerify.ts"() {
+    init_domain();
+  }
+});
 
 // src/core/pipeline.ts
 function buildSourceLookup(rows) {
@@ -2521,6 +2776,38 @@ function assignClusterIds(rows, clusters) {
     return cid !== void 0 ? { ...row, __cluster_id__: cid } : row;
   });
 }
+function isPreflightReport(v) {
+  return typeof v === "object" && v !== null && "findings" in v && Array.isArray(v.findings);
+}
+function applyPostflight(rows, config, pairScores) {
+  const pre = config._preflightReport;
+  if (!isPreflightReport(pre)) {
+    return { pairScores, report: void 0 };
+  }
+  const report = postflight(rows, config, {
+    pairScores: pairScores.map((p) => ({
+      idA: p.idA,
+      idB: p.idB,
+      score: p.score
+    }))
+  });
+  let filtered = pairScores;
+  if (config._strictAutoconfig !== true) {
+    for (const adj of report.adjustments) {
+      if (adj.field === "threshold") {
+        const newThreshold = adj.toValue;
+        const prev = filtered.length;
+        filtered = filtered.filter((p) => p.score >= newThreshold);
+        if (prev > 0 && filtered.length === 0) {
+          report.advisories.push(
+            `threshold adjustment to ${newThreshold.toFixed(3)} dropped all ${prev} pairs`
+          );
+        }
+      }
+    }
+  }
+  return { pairScores: filtered, report };
+}
 function runDedupePipeline(rows, config, options) {
   if (rows.length === 0) {
     return _emptyDedupeResult(config);
@@ -2570,8 +2857,13 @@ function runDedupePipeline(rows, config, options) {
       }
     }
   }
+  const { pairScores: finalPairs, report: postflightReport } = applyPostflight(
+    processed,
+    config,
+    allPairs
+  );
   const allIds = collectRowIds(processed);
-  const pairTuples = allPairs.map((p) => [
+  const pairTuples = finalPairs.map((p) => [
     p.idA,
     p.idB,
     p.score
@@ -2640,8 +2932,9 @@ function runDedupePipeline(rows, config, options) {
     dupes,
     unique,
     stats,
-    scoredPairs: allPairs,
-    config
+    scoredPairs: finalPairs,
+    config,
+    ...postflightReport !== void 0 ? { postflightReport } : {}
   };
 }
 function runMatchPipeline(targetRows, referenceRows, config) {
@@ -2695,7 +2988,8 @@ function runMatchPipeline(targetRows, referenceRows, config) {
       matchedCount: matched.length,
       unmatchedCount: unmatched.length,
       matchRate: targetRows.length > 0 ? matched.length / targetRows.length : 0
-    }
+    },
+    ...result.postflightReport !== void 0 ? { postflightReport: result.postflightReport } : {}
   };
 }
 function _emptyDedupeResult(config) {
@@ -2724,6 +3018,7 @@ var init_pipeline = __esm({
     init_scorer();
     init_cluster();
     init_golden();
+    init_autoconfigVerify();
   }
 });
 
@@ -3567,6 +3862,12 @@ var init_explain = __esm({
 });
 
 // src/core/profiler.ts
+function isYearValue(v) {
+  const normalized = v.replace(/\.0+$/, "");
+  const n = Number(normalized);
+  if (Number.isNaN(n) || !Number.isFinite(n) || !Number.isInteger(n)) return false;
+  return n >= 1900 && n <= 2100;
+}
 function toStringOrNull(value) {
   if (value === null || value === void 0) return null;
   if (typeof value === "string") {
@@ -3575,10 +3876,22 @@ function toStringOrNull(value) {
   }
   return String(value);
 }
-function guessType(values, columnName) {
-  if (values.length === 0) return "text";
-  const n = values.length;
+function guessTypeByName(columnName) {
   const lname = columnName.toLowerCase();
+  if (/email|e_mail|e-mail/i.test(lname)) return "email";
+  if (/phone|tel(?!e)|mobile|cell/i.test(lname)) return "phone";
+  if (/zip|postal|postcode/i.test(lname)) return "zip";
+  if (YEAR_NAME_RE.test(lname)) return "year";
+  if (/date|created|modified|updated|_at$|birth|dob/i.test(lname)) return "date";
+  if (/^(city|state|county|country|region|province)/i.test(lname)) return "geo";
+  if (/city_desc|state_cd|country_code|state_code/i.test(lname)) return "geo";
+  if (/^id$|_id$|uuid|guid/i.test(lname)) return "id";
+  if (/name|first|last|full_name|surname/i.test(lname)) return "name";
+  return null;
+}
+function guessTypeByData(values) {
+  if (values.length === 0) return null;
+  const n = values.length;
   const emailCount = values.reduce(
     (acc, v) => acc + (EMAIL_VALUE_RE.test(v) ? 1 : 0),
     0
@@ -3597,14 +3910,16 @@ function guessType(values, columnName) {
     0
   );
   if (zipCount / n > 0.6) return "zip";
+  let yearCount = 0;
+  for (const v of values) {
+    if (isYearValue(v)) yearCount++;
+  }
+  if (yearCount / n >= 0.95) return "year";
   let dateCount = 0;
   for (const v of values) {
     if (DATE_VALUE_RES.some((re) => re.test(v))) dateCount++;
   }
   if (dateCount / n > 0.6) return "date";
-  if (/^(city|state|county|country|region|province)/i.test(lname)) return "geo";
-  if (/city_desc|state_cd|country_code|state_code/i.test(lname)) return "geo";
-  if (/^id$|_id$|uuid|guid/i.test(lname)) return "id";
   const nameCount = values.reduce(
     (acc, v) => acc + (NAME_VALUE_RE.test(v) ? 1 : 0),
     0
@@ -3615,7 +3930,44 @@ function guessType(values, columnName) {
     if (/^-?\d+(\.\d+)?$/.test(v)) numericCount++;
   }
   if (numericCount / n > 0.8) return "numeric";
-  return "text";
+  let totalLen = 0;
+  let delimRows = 0;
+  let totalDelims = 0;
+  for (const v of values) {
+    totalLen += v.length;
+    const commas = (v.match(/,/g) ?? []).length;
+    const semis = (v.match(/;/g) ?? []).length;
+    const count = commas + semis;
+    if (count > 0) {
+      delimRows++;
+      totalDelims += count;
+    }
+  }
+  const avgLen = totalLen / n;
+  const delimFraction = delimRows / n;
+  const avgDelimsPerDelimRow = delimRows > 0 ? totalDelims / delimRows : 0;
+  if (avgLen > 30 && delimFraction >= 0.7 && avgDelimsPerDelimRow >= 2) {
+    return "multi_name";
+  }
+  return null;
+}
+function guessTypeAndConfidence(values, columnName) {
+  if (values.length === 0) return { type: "text", confidence: 0.3 };
+  const nameType = guessTypeByName(columnName);
+  const dataType = guessTypeByData(values);
+  if (nameType !== null && dataType !== null) {
+    if (nameType === dataType) {
+      return { type: nameType, confidence: 0.9 };
+    }
+    const nameAuthoritative = nameType === "date" || nameType === "year" || nameType === "geo" || nameType === "id" || nameType === "email" || nameType === "zip" || nameType === "name";
+    return {
+      type: nameAuthoritative ? nameType : dataType,
+      confidence: 0.7
+    };
+  }
+  if (nameType !== null) return { type: nameType, confidence: 0.7 };
+  if (dataType !== null) return { type: dataType, confidence: 0.7 };
+  return { type: "text", confidence: 0.3 };
 }
 function profileColumn(name, rawValues) {
   const totalCount = rawValues.length;
@@ -3643,7 +3995,10 @@ function profileColumn(name, rawValues) {
     if (sampleValues.length >= 5) break;
   }
   const sampleForType = nonNull.length > 500 ? nonNull.slice(0, 500) : nonNull;
-  const inferredType = guessType(sampleForType, name);
+  const { type: inferredType, confidence } = guessTypeAndConfidence(
+    sampleForType,
+    name
+  );
   return {
     name,
     nullRate,
@@ -3654,7 +4009,8 @@ function profileColumn(name, rawValues) {
     inferredType,
     avgLength,
     maxLength: maxLen,
-    sampleValues
+    sampleValues,
+    confidence
   };
 }
 function profileRows(rows) {
@@ -3682,7 +4038,7 @@ function profileRows(rows) {
     byName
   };
 }
-var EMAIL_VALUE_RE, PHONE_STRIP_RE, DATE_VALUE_RES, ZIP_VALUE_RE, NAME_VALUE_RE;
+var EMAIL_VALUE_RE, PHONE_STRIP_RE, DATE_VALUE_RES, ZIP_VALUE_RE, NAME_VALUE_RE, YEAR_NAME_RE;
 var init_profiler = __esm({
   "src/core/profiler.ts"() {
     EMAIL_VALUE_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
@@ -3694,6 +4050,7 @@ var init_profiler = __esm({
     ];
     ZIP_VALUE_RE = /^\d{5}(-?\d{4})?$/;
     NAME_VALUE_RE = /^[A-Za-z][A-Za-z \-']{0,28}[A-Za-z]$|^[A-Za-z]{2,3}$/;
+    YEAR_NAME_RE = /(^|_)(year|yr)(_|$)/i;
   }
 });