goldenmatch 0.1.0

package/src/core/graph-er.ts

@@ -0,0 +1,371 @@
+/**
+ * graph-er.ts — Multi-table entity resolution with evidence propagation.
+ * Edge-safe: no `node:` imports.
+ *
+ * Ports goldenmatch/core/graph_er.py. Each table is deduped independently
+ * first, then cluster assignments propagate across foreign-key edges:
+ * if row A.fk points into B's cluster, rows of A whose FK shares a cluster
+ * get a similarity boost before re-clustering.
+ */
+
+import type { ClusterInfo, PairKey, Row, ScoredPair } from "./types.js";
+import { pairKey } from "./cluster.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface TableSchema {
+  readonly name: string;
+  readonly rows: readonly Row[];
+  readonly idColumn: string;
+}
+
+export interface Relationship {
+  readonly tableA: string;
+  readonly tableB: string;
+  readonly fkColumn: string; // column in tableA referencing tableB
+}
+
+export interface GraphERResult {
+  readonly clustersByTable: ReadonlyMap<string, ReadonlyMap<number, ClusterInfo>>;
+  readonly converged: boolean;
+  readonly iterations: number;
+}
+
+/**
+ * A scorer for graph ER.
+ *
+ * **Contract:** Must return `ScoredPair.idA` / `.idB` as **0-based row indices
+ * into the input `rows` array**, NOT the `__row_id__` values (or any other
+ * external/stable row identifier) those rows may carry.
+ *
+ * Why: `runGraphER` seeds its Union-Find with `0..rows.length` and indexes
+ * foreign-key cluster lookups by row position. Returning external row IDs
+ * instead of 0-based indices causes the evidence-propagation boost to never
+ * apply (ids won't line up with the UF roots or the fk-index map) and can
+ * silently produce wrong clusters.
+ *
+ * If you have stable external row IDs, re-number your rows to 0-based
+ * positional indices before scoring, then map back afterward.
+ */
+export interface GraphERScorer {
+  (rows: readonly Row[]): readonly ScoredPair[];
+}
+
+export interface RunGraphEROptions {
+  readonly maxIterations?: number;
+  readonly convergenceThreshold?: number;
+  readonly similarityBoost?: number;
+  /** Per-table scorer: takes rows, returns scored pairs. Required. */
+  readonly scorerByTable: ReadonlyMap<string, GraphERScorer>;
+  /** Match threshold for building clusters. Default 0.85. */
+  readonly threshold?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Minimal Union-Find
+// ---------------------------------------------------------------------------
+
+class UnionFind {
+  private parent: number[] = [];
+  private size: number[] = [];
+
+  add(id: number): void {
+    while (this.parent.length <= id) {
+      this.parent.push(this.parent.length);
+      this.size.push(1);
+    }
+  }
+
+  find(id: number): number {
+    this.add(id);
+    let cur = id;
+    while (this.parent[cur] !== cur) {
+      const parent = this.parent[cur]!;
+      this.parent[cur] = this.parent[parent]!; // path compression
+      cur = this.parent[cur]!;
+    }
+    return cur;
+  }
+
+  union(a: number, b: number): void {
+    const rootA = this.find(a);
+    const rootB = this.find(b);
+    if (rootA === rootB) return;
+    if (this.size[rootA]! < this.size[rootB]!) {
+      this.parent[rootA] = rootB;
+      this.size[rootB]! += this.size[rootA]!;
+    } else {
+      this.parent[rootB] = rootA;
+      this.size[rootA]! += this.size[rootB]!;
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function toRowIndex(rows: readonly Row[], idColumn: string): Map<unknown, number> {
+  const map = new Map<unknown, number>();
+  for (let i = 0; i < rows.length; i++) {
+    const v = rows[i]![idColumn];
+    if (v !== null && v !== undefined) map.set(v, i);
+  }
+  return map;
+}
+
+function clustersFromPairs(
+  rowCount: number,
+  pairs: readonly ScoredPair[],
+  threshold: number,
+): Map<number, ClusterInfo> {
+  const uf = new UnionFind();
+  for (let i = 0; i < rowCount; i++) uf.add(i);
+
+  const scoreMap = new Map<PairKey, number>();
+  for (const p of pairs) {
+    if (p.score < threshold) continue;
+    uf.union(p.idA, p.idB);
+    scoreMap.set(pairKey(p.idA, p.idB), p.score);
+  }
+
+  const rootMembers = new Map<number, number[]>();
+  for (let i = 0; i < rowCount; i++) {
+    const root = uf.find(i);
+    const list = rootMembers.get(root);
+    if (list) list.push(i);
+    else rootMembers.set(root, [i]);
+  }
+
+  const clusters = new Map<number, ClusterInfo>();
+  let clusterId = 0;
+  for (const members of rootMembers.values()) {
+    const pairScores = new Map<PairKey, number>();
+    let minEdge = 1;
+    let edgeSum = 0;
+    let edgeCount = 0;
+    for (let i = 0; i < members.length; i++) {
+      for (let j = i + 1; j < members.length; j++) {
+        const a = members[i]!;
+        const b = members[j]!;
+        const k = pairKey(a, b);
+        const s = scoreMap.get(k);
+        if (s !== undefined) {
+          pairScores.set(k, s);
+          if (s < minEdge) minEdge = s;
+          edgeSum += s;
+          edgeCount++;
+        }
+      }
+    }
+    const avgEdge = edgeCount > 0 ? edgeSum / edgeCount : 0;
+    const connectivity = members.length <= 1 ? 0 : Math.min(1, edgeCount / (members.length - 1));
+    const confidence = members.length <= 1 ? 1 : 0.4 * minEdge + 0.3 * avgEdge + 0.3 * connectivity;
+
+    clusters.set(clusterId++, {
+      members,
+      size: members.length,
+      oversized: false,
+      pairScores,
+      confidence,
+      bottleneckPair: null,
+      clusterQuality: "strong",
+    });
+  }
+
+  return clusters;
+}
+
+function rowIdToCluster(clusters: ReadonlyMap<number, ClusterInfo>): Map<number, number> {
+  const map = new Map<number, number>();
+  for (const [cid, c] of clusters) {
+    for (const m of c.members) map.set(m, cid);
+  }
+  return map;
+}
+
+// ---------------------------------------------------------------------------
+// Core algorithm
+// ---------------------------------------------------------------------------
+
+/**
+ * Run multi-table entity resolution with iterative evidence propagation.
+ *
+ * For each table, the caller provides a scorer that produces pair scores
+ * from a row array. The algorithm:
+ *   1. Score & cluster each table independently.
+ *   2. For every relationship A->B: find pairs in A whose fk resolves to
+ *      the same cluster in B. Boost those pair scores by `similarityBoost`.
+ *   3. Re-cluster every table. Repeat until clusters stabilize or
+ *      `maxIterations` is reached.
+ *
+ * **Scorer contract (important):** scorers in `options.scorerByTable` must
+ * return `ScoredPair.idA` / `.idB` as **0-based row indices** into the
+ * `rows` array they were handed (NOT the stable `__row_id__` values those
+ * rows may carry). The evidence-propagation step keys foreign-key cluster
+ * lookups by row position; using external row IDs will silently make the
+ * boost no-op and can produce wrong clusters. See {@link GraphERScorer}.
+ */
+export function runGraphER(
+  tables: readonly TableSchema[],
+  relationships: readonly Relationship[],
+  options: RunGraphEROptions,
+): GraphERResult {
+  const maxIterations = options.maxIterations ?? 5;
+  const convergenceThreshold = options.convergenceThreshold ?? 0.01;
+  const similarityBoost = options.similarityBoost ?? 0.1;
+  const threshold = options.threshold ?? 0.85;
+
+  // Per-table state.
+  const tableByName = new Map<string, TableSchema>();
+  for (const t of tables) tableByName.set(t.name, t);
+
+  const idIndexByTable = new Map<string, Map<unknown, number>>();
+  for (const t of tables) {
+    idIndexByTable.set(t.name, toRowIndex(t.rows, t.idColumn));
+  }
+
+  // Initial pair scores per table (without boost).
+  const basePairsByTable = new Map<string, ScoredPair[]>();
+  for (const t of tables) {
+    const scorer = options.scorerByTable.get(t.name);
+    if (!scorer) {
+      throw new Error(`Missing scorer for table "${t.name}"`);
+    }
+    basePairsByTable.set(t.name, [...scorer(t.rows)]);
+  }
+
+  let clustersByTable = new Map<string, Map<number, ClusterInfo>>();
+  for (const t of tables) {
+    clustersByTable.set(
+      t.name,
+      clustersFromPairs(t.rows.length, basePairsByTable.get(t.name) ?? [], threshold),
+    );
+  }
+
+  let converged = false;
+  let iter = 0;
+
+  for (; iter < maxIterations; iter++) {
+    const rowToCluster = new Map<string, Map<number, number>>();
+    for (const [name, clusters] of clustersByTable) {
+      rowToCluster.set(name, rowIdToCluster(clusters));
+    }
+
+    const nextClusters = new Map<string, Map<number, ClusterInfo>>();
+    let maxDelta = 0;
+
+    for (const t of tables) {
+      const basePairs = basePairsByTable.get(t.name) ?? [];
+      const boosted = basePairs.map((p) => ({ ...p }));
+
+      // For each relationship where this table is the source, boost pairs
+      // whose FK targets land in the same cluster in the referenced table.
+      for (const rel of relationships) {
+        if (rel.tableA !== t.name) continue;
+        const bClusters = rowToCluster.get(rel.tableB);
+        if (!bClusters) continue;
+        const bIndex = idIndexByTable.get(rel.tableB);
+        if (!bIndex) continue;
+
+        // Build: row index in A (0-based) -> cluster id in B
+        // Keyed by positional index because `pair.idA`/`pair.idB` are 0-based
+        // indices per the GraphERScorer contract. See GraphERScorer JSDoc.
+        const fkClusterByIndex = new Map<number, number>();
+        for (let i = 0; i < t.rows.length; i++) {
+          const fkVal = t.rows[i]![rel.fkColumn];
+          if (fkVal === null || fkVal === undefined) continue;
+          const bRowIdx = bIndex.get(fkVal);
+          if (bRowIdx === undefined) continue;
+          const bCid = bClusters.get(bRowIdx);
+          if (bCid !== undefined) fkClusterByIndex.set(i, bCid);
+        }
+
+        for (const pair of boosted) {
+          const ca = fkClusterByIndex.get(pair.idA);
+          const cb = fkClusterByIndex.get(pair.idB);
+          if (ca !== undefined && cb !== undefined && ca === cb) {
+            const newScore = Math.min(1, pair.score + similarityBoost);
+            (pair as { score: number }).score = newScore;
+          }
+        }
+      }
+
+      const newClusters = clustersFromPairs(t.rows.length, boosted, threshold);
+      const prevClusters = clustersByTable.get(t.name);
+      if (prevClusters) {
+        const delta = clusterSetDelta(prevClusters, newClusters);
+        if (delta > maxDelta) maxDelta = delta;
+      }
+      nextClusters.set(t.name, newClusters);
+    }
+
+    clustersByTable = nextClusters;
+    if (maxDelta < convergenceThreshold) {
+      converged = true;
+      break;
+    }
+  }
+
+  const finalMap = new Map<string, ReadonlyMap<number, ClusterInfo>>();
+  for (const [k, v] of clustersByTable) finalMap.set(k, v);
+
+  return {
+    clustersByTable: finalMap,
+    converged,
+    iterations: iter + (converged ? 1 : 0),
+  };
+}
+
+/**
+ * Compare two cluster assignments over the same row set. Returns fraction of
+ * rows whose cluster signature changed — a rough "delta" proxy. Two rows
+ * have the same signature if they are in the same cluster in both sets.
+ */
+function clusterSetDelta(
+  a: ReadonlyMap<number, ClusterInfo>,
+  b: ReadonlyMap<number, ClusterInfo>,
+): number {
+  const mapA = rowIdToCluster(a);
+  const mapB = rowIdToCluster(b);
+
+  // Align cluster IDs between a and b by finding the most common b-id for
+  // each a-id. Anything mismatched counts as a change.
+  const aToB = new Map<number, Map<number, number>>();
+  for (const [rowId, aCid] of mapA) {
+    const bCid = mapB.get(rowId);
+    if (bCid === undefined) continue;
+    let sub = aToB.get(aCid);
+    if (!sub) {
+      sub = new Map();
+      aToB.set(aCid, sub);
+    }
+    sub.set(bCid, (sub.get(bCid) ?? 0) + 1);
+  }
+
+  const majority = new Map<number, number>();
+  for (const [aCid, counts] of aToB) {
+    let best: [number, number] | null = null;
+    for (const [bCid, count] of counts) {
+      if (best === null || count > best[1]) best = [bCid, count];
+    }
+    if (best) majority.set(aCid, best[0]);
+  }
+
+  let changed = 0;
+  let total = 0;
+  for (const [rowId, aCid] of mapA) {
+    const bCid = mapB.get(rowId);
+    if (bCid === undefined) {
+      changed++;
+      total++;
+      continue;
+    }
+    total++;
+    if (majority.get(aCid) !== bCid) changed++;
+  }
+
+  return total === 0 ? 0 : changed / total;
+}

package/src/core/index.ts

@@ -0,0 +1,314 @@
+/**
+ * index.ts — Core public API surface for GoldenMatch-JS.
+ * Re-exports everything from core modules.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type {
+  Row,
+  ColumnValue,
+  PairKey,
+  MatchkeyConfig,
+  ExactMatchkey,
+  WeightedMatchkey,
+  ProbabilisticMatchkey,
+  MakeMatchkeyConfigInput,
+  MatchkeyField,
+  BlockingConfig,
+  BlockingKeyConfig,
+  SortKeyField,
+  CanopyConfig,
+  GoldenRulesConfig,
+  GoldenFieldRule,
+  StandardizationConfig,
+  ValidationRuleConfig,
+  ValidationConfig,
+  QualityConfig,
+  TransformConfig,
+  BudgetConfig,
+  LLMScorerConfig,
+  DomainConfig,
+  LearningConfig,
+  MemoryConfig,
+  InputFileConfig,
+  InputConfig,
+  OutputConfig,
+  GoldenMatchConfig,
+  ScoredPair,
+  ClusterInfo,
+  DedupeStats,
+  DedupeResult,
+  MatchResult,
+  FieldProvenance,
+  ClusterProvenance,
+  BlockResult,
+} from "./types.js";
+
+export {
+  VALID_SCORERS,
+  VALID_TRANSFORMS,
+  VALID_STRATEGIES,
+  VALID_STANDARDIZERS,
+  makeMatchkeyField,
+  makeMatchkeyConfig,
+  makeBlockingConfig,
+  makeGoldenRulesConfig,
+  makeConfig,
+  makeScoredPair,
+  getMatchkeys,
+} from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Data layer
+// ---------------------------------------------------------------------------
+
+export { TabularData, isNullish, toColumnValue } from "./data.js";
+
+// ---------------------------------------------------------------------------
+// Transforms
+// ---------------------------------------------------------------------------
+
+export { applyTransform, applyTransforms, soundex, metaphone } from "./transforms.js";
+
+// ---------------------------------------------------------------------------
+// Scoring
+// ---------------------------------------------------------------------------
+
+export {
+  scoreField,
+  scorePair,
+  findExactMatches,
+  findFuzzyMatches,
+  scoreBlocksSequential,
+  jaro,
+  jaroWinkler,
+  levenshteinDistance,
+  levenshteinSimilarity,
+  indelDistance,
+  indelSimilarity,
+  tokenSortRatio,
+  soundexMatch,
+  diceCoefficient,
+  jaccardSimilarity,
+  ensembleScore,
+  scoreMatrix,
+  asString,
+} from "./scorer.js";
+
+// ---------------------------------------------------------------------------
+// Matchkey
+// ---------------------------------------------------------------------------
+
+export {
+  computeMatchkeyValue,
+  computeMatchkeys,
+  addRowIds,
+  addSourceColumn,
+} from "./matchkey.js";
+
+// ---------------------------------------------------------------------------
+// Standardization
+// ---------------------------------------------------------------------------
+
+export { applyStandardizer, applyStandardization } from "./standardize.js";
+
+// ---------------------------------------------------------------------------
+// Blocking
+// ---------------------------------------------------------------------------
+
+export {
+  buildBlocks,
+  buildBlocksAsync,
+  buildStaticBlocks,
+  buildMultiPassBlocks,
+  buildAdaptiveBlocks,
+  selectBestBlockingKey,
+} from "./blocker.js";
+
+// ---------------------------------------------------------------------------
+// Embedding + ANN + Cross-encoder
+// ---------------------------------------------------------------------------
+
+export { Embedder, getEmbedder, EmbedderError } from "./embedder.js";
+export type {
+  EmbedderOptions,
+  EmbeddingResult,
+  EmbedderProvider,
+} from "./embedder.js";
+export {
+  ANNBlocker,
+  HNSWANNBlocker,
+  createANNBlocker,
+  buildANNBlocks,
+  buildANNPairBlocks,
+  cosineSim,
+  euclideanDist,
+} from "./ann-blocker.js";
+export type {
+  ANNBlockerOptions,
+  ANNBlockerBase,
+  BuildANNOptions,
+  HNSWOptions,
+  HNSWModule,
+  HNSWIndexLike,
+  CreateANNBlockerOptions,
+} from "./ann-blocker.js";
+export {
+  rerankTopPairs,
+  rerankPair,
+  CrossEncoderHttpError,
+  CrossEncoderModel,
+  _resetCrossEncoderModelCache,
+} from "./cross-encoder.js";
+export type {
+  CrossEncoderOptions,
+  CrossEncoderProvider,
+  CrossEncoderReranker,
+  CrossEncoderModelOptions,
+} from "./cross-encoder.js";
+
+// ---------------------------------------------------------------------------
+// Clustering
+// ---------------------------------------------------------------------------
+
+export {
+  UnionFind,
+  buildClusters,
+  buildMst,
+  splitOversizedCluster,
+  computeClusterConfidence,
+  addToCluster,
+  unmergeRecord,
+  unmergeCluster,
+  pairKey,
+  parsePairKey,
+  getClusterPairScores,
+} from "./cluster.js";
+
+// ---------------------------------------------------------------------------
+// Golden records
+// ---------------------------------------------------------------------------
+
+export {
+  mergeField,
+  buildGoldenRecord,
+  buildGoldenRecordWithProvenance,
+} from "./golden.js";
+
+// ---------------------------------------------------------------------------
+// Pipeline
+// ---------------------------------------------------------------------------
+
+export { runDedupePipeline, runMatchPipeline } from "./pipeline.js";
+
+// ---------------------------------------------------------------------------
+// API
+// ---------------------------------------------------------------------------
+
+export { dedupe, match, scoreStrings, scorePairRecord } from "./api.js";
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export { parseConfig, parseConfigYaml, configToYaml } from "./config/loader.js";
+
+// ---------------------------------------------------------------------------
+// LLM
+// ---------------------------------------------------------------------------
+
+export { BudgetTracker, countTokensApprox } from "./llm/budget.js";
+export type { BudgetSnapshot } from "./llm/budget.js";
+export { llmScorePairs, scoreStringsWithLlm } from "./llm/scorer.js";
+export type { LLMScoreResult } from "./llm/scorer.js";
+export { llmClusterPairs } from "./llm/cluster.js";
+
+// ---------------------------------------------------------------------------
+// Explain
+// ---------------------------------------------------------------------------
+
+export { explainPair, explainCluster } from "./explain.js";
+export type { PairExplanation, ClusterExplanation } from "./explain.js";
+
+// ---------------------------------------------------------------------------
+// Probabilistic (Fellegi-Sunter)
+// ---------------------------------------------------------------------------
+
+export { buildComparisonVector, trainEM, scoreProbabilistic } from "./probabilistic.js";
+export type { EMResult } from "./probabilistic.js";
+
+// ---------------------------------------------------------------------------
+// Evaluation
+// ---------------------------------------------------------------------------
+
+export { evaluatePairs, evaluateClusters, loadGroundTruthPairs } from "./evaluate.js";
+export type { EvalResult } from "./evaluate.js";
+
+// ---------------------------------------------------------------------------
+// Streaming / match-one
+// ---------------------------------------------------------------------------
+
+export { StreamProcessor } from "./streaming.js";
+export { matchOne, findExactMatchesOne } from "./match-one.js";
+
+// ---------------------------------------------------------------------------
+// Cluster comparison + sensitivity
+// ---------------------------------------------------------------------------
+
+export { compareClusters } from "./compare-clusters.js";
+export type { CCMSResult } from "./compare-clusters.js";
+export { runSensitivity, stabilityReport } from "./sensitivity.js";
+export type { SweepParam, SweepPoint, SensitivityResult } from "./sensitivity.js";
+
+// ---------------------------------------------------------------------------
+// Quality, autofix, validation, profiling, ingest
+// ---------------------------------------------------------------------------
+
+export { scanQuality, runQualityCheck } from "./quality.js";
+export type { QualityFinding } from "./quality.js";
+export { autoFixRows } from "./autofix.js";
+export type { AutoFixLog } from "./autofix.js";
+export { validateRows } from "./validate.js";
+export type { ValidationRule, ValidationReport } from "./validate.js";
+export { profileRows } from "./profiler.js";
+export type { ColumnProfile, DatasetProfile } from "./profiler.js";
+export { applyColumnMap, validateColumns, concatRows } from "./ingest.js";
+
+// ---------------------------------------------------------------------------
+// Review queue, autoconfig, domain, lineage, learned blocking, graph ER
+// ---------------------------------------------------------------------------
+
+export { ReviewQueue, gatePairs } from "./review-queue.js";
+export type { ReviewItem, GatedResult } from "./review-queue.js";
+export { autoConfigureRows } from "./autoconfig.js";
+export type { AutoconfigOptions } from "./autoconfig.js";
+export { detectDomain, extractFeatures } from "./domain.js";
+export type { DomainProfile } from "./domain.js";
+export { buildLineage, lineageToJson, lineageFromJson } from "./lineage.js";
+export type { LineageEdge, LineageBundle } from "./lineage.js";
+export { learnBlockingRules, applyLearnedBlocks } from "./learned-blocking.js";
+export type { LearnedPredicate, LearnedRules } from "./learned-blocking.js";
+export { runGraphER } from "./graph-er.js";
+export type { TableSchema, Relationship, GraphERResult } from "./graph-er.js";
+
+// ---------------------------------------------------------------------------
+// Memory (learning corrections)
+// ---------------------------------------------------------------------------
+
+export { MemoryStore } from "./memory/store.js";
+export type { Correction, MemoryStoreConfig } from "./memory/store.js";
+export { applyCorrections, hashRow } from "./memory/corrections.js";
+export { MemoryLearner } from "./memory/learner.js";
+export type { LearnedParams } from "./memory/learner.js";
+
+// ---------------------------------------------------------------------------
+// PPRL (Privacy-Preserving Record Linkage)
+// ---------------------------------------------------------------------------
+
+export { runPPRL, autoConfigurePPRL } from "./pprl/protocol.js";
+export type { PPRLConfig, PPRLResult } from "./pprl/protocol.js";