goldenmatch 0.1.0

package/src/core/pipeline.ts

@@ -0,0 +1,366 @@
+/**
+ * pipeline.ts — Core pipeline orchestrator for GoldenMatch-JS.
+ * Edge-safe: no `node:` imports, pure TypeScript only.
+ *
+ * Ports goldenmatch/core/pipeline.py.
+ * Chains: standardize -> matchkeys -> block -> score -> cluster -> golden.
+ */
+
+import type {
+  Row,
+  GoldenMatchConfig,
+  MatchkeyConfig,
+  DedupeResult,
+  DedupeStats,
+  PairKey,
+  ScoredPair,
+  MatchResult,
+  GoldenRulesConfig,
+  ClusterInfo,
+} from "./types.js";
+import { makeGoldenRulesConfig, getMatchkeys, makeBlockingConfig } from "./types.js";
+import { computeMatchkeys, addRowIds, addSourceColumn } from "./matchkey.js";
+import { applyStandardization } from "./standardize.js";
+import { buildBlocks } from "./blocker.js";
+import {
+  findExactMatches,
+  scoreBlocksSequential,
+} from "./scorer.js";
+import { buildClusters, pairKey } from "./cluster.js";
+import { buildGoldenRecord } from "./golden.js";
+
+// ---------------------------------------------------------------------------
+// Options
+// ---------------------------------------------------------------------------
+
+export interface DedupeOptions {
+  readonly outputGolden?: boolean;
+  readonly outputReport?: boolean;
+  readonly acrossFilesOnly?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/** Build a source lookup map from rows (rowId -> source name). */
+function buildSourceLookup(rows: readonly Row[]): Map<number, string> {
+  const lookup = new Map<number, string>();
+  for (const row of rows) {
+    const id = row.__row_id__ as number;
+    const src = row.__source__ as string | undefined;
+    if (id !== undefined && src !== undefined) {
+      lookup.set(id, src);
+    }
+  }
+  return lookup;
+}
+
+/** Collect all row IDs from rows. */
+function collectRowIds(rows: readonly Row[]): number[] {
+  return rows.map((r) => r.__row_id__ as number);
+}
+
+/** Assign __cluster_id__ to rows based on cluster membership. */
+function assignClusterIds(
+  rows: readonly Row[],
+  clusters: ReadonlyMap<number, ClusterInfo>,
+): Row[] {
+  // Build rowId -> clusterId lookup
+  const rowToCluster = new Map<number, number>();
+  for (const [cid, cinfo] of clusters) {
+    for (const memberId of cinfo.members) {
+      rowToCluster.set(memberId, cid);
+    }
+  }
+
+  return rows.map((row) => {
+    const rowId = row.__row_id__ as number;
+    const cid = rowToCluster.get(rowId);
+    return cid !== undefined ? { ...row, __cluster_id__: cid } : row;
+  });
+}
+
+// ---------------------------------------------------------------------------
+// runDedupePipeline
+// ---------------------------------------------------------------------------
+
+/**
+ * Run the full deduplication pipeline.
+ *
+ * Steps:
+ * 1. Add __row_id__ and __source__ if not present
+ * 2. Apply standardization
+ * 3. Compute matchkeys
+ * 4. Phase 1: Exact matchkeys (hash-based grouping)
+ * 5. Phase 2: Fuzzy matchkeys (block + score)
+ * 6. Phase 3: Cluster (Union-Find with MST splitting)
+ * 7. Phase 4: Build golden records for multi-member clusters
+ * 8. Classify dupes vs unique
+ * 9. Compute stats
+ * 10. Return DedupeResult
+ */
+export function runDedupePipeline(
+  rows: readonly Row[],
+  config: GoldenMatchConfig,
+  options?: DedupeOptions,
+): DedupeResult {
+  if (rows.length === 0) {
+    return _emptyDedupeResult(config);
+  }
+
+  const matchkeys = getMatchkeys(config);
+  const goldenRules = config.goldenRules ?? makeGoldenRulesConfig();
+  const blockingConfig = config.blocking ?? makeBlockingConfig();
+  const acrossFilesOnly = options?.acrossFilesOnly ?? false;
+
+  // ---- Step 1: Add __row_id__ and __source__ ----
+  let processed: Row[] = rows.map((r, i) => {
+    const extra: Record<string, unknown> = {};
+    if (r.__row_id__ === undefined) extra.__row_id__ = i;
+    if (r.__source__ === undefined) extra.__source__ = "default";
+    return Object.keys(extra).length > 0 ? { ...r, ...extra } : (r as Row);
+  });
+
+  // ---- Step 2: Apply standardization ----
+  if (config.standardization) {
+    processed = applyStandardization(processed, config.standardization.rules);
+  }
+
+  // ---- Step 3: Compute matchkeys ----
+  processed = computeMatchkeys(processed, matchkeys);
+
+  // ---- Step 4 & 5: Score exact + fuzzy matchkeys ----
+  const allPairs: ScoredPair[] = [];
+  const matchedPairKeys = new Set<PairKey>();
+  const sourceLookup = buildSourceLookup(processed);
+
+  for (const mk of matchkeys) {
+    if (mk.type === "exact") {
+      // Phase 1: Exact matching via hash grouping
+      let pairs = findExactMatches(processed, mk);
+
+      // Cross-file filter
+      if (acrossFilesOnly) {
+        pairs = pairs.filter((p) => {
+          const srcA = sourceLookup.get(p.idA);
+          const srcB = sourceLookup.get(p.idB);
+          return srcA !== srcB;
+        });
+      }
+
+      for (const p of pairs) {
+        const key = pairKey(p.idA, p.idB);
+        if (!matchedPairKeys.has(key)) {
+          matchedPairKeys.add(key);
+          allPairs.push(p);
+        }
+      }
+    } else {
+      // Phase 2: Fuzzy (weighted/probabilistic) — block then score
+      const blocks = buildBlocks(processed, blockingConfig);
+
+      const pairs = scoreBlocksSequential(blocks, mk, matchedPairKeys, {
+        acrossFilesOnly,
+        sourceLookup,
+      });
+
+      for (const p of pairs) {
+        allPairs.push(p);
+      }
+    }
+  }
+
+  // ---- Step 6: Cluster ----
+  const allIds = collectRowIds(processed);
+  const pairTuples: [number, number, number][] = allPairs.map((p) => [
+    p.idA,
+    p.idB,
+    p.score,
+  ]);
+
+  const clusters = buildClusters(pairTuples, allIds, {
+    maxClusterSize: goldenRules.maxClusterSize,
+    weakClusterThreshold: goldenRules.weakClusterThreshold,
+    autoSplit: goldenRules.autoSplit,
+  });
+
+  // ---- Step 7: Build golden records ----
+  const rowsWithClusters = assignClusterIds(processed, clusters);
+  const goldenRecords: Row[] = [];
+
+  if (options?.outputGolden !== false) {
+    for (const [cid, cinfo] of clusters) {
+      if (cinfo.size < 2) continue; // Only build golden for multi-member clusters
+
+      const clusterRows = rowsWithClusters.filter(
+        (r) => (r.__cluster_id__ as number) === cid,
+      );
+      const golden = buildGoldenRecord(clusterRows, goldenRules);
+
+      const goldenRow: Record<string, unknown> = {
+        __cluster_id__: cid,
+        __golden_confidence__: golden.goldenConfidence,
+      };
+      for (const [col, info] of Object.entries(golden.fields)) {
+        goldenRow[col] = info.value;
+      }
+      goldenRecords.push(goldenRow as Row);
+    }
+  }
+
+  // ---- Step 8: Classify dupes vs unique ----
+  const multiMemberClusterIds = new Set<number>();
+  for (const [cid, cinfo] of clusters) {
+    if (cinfo.size >= 2) multiMemberClusterIds.add(cid);
+  }
+
+  const dupeRowIds = new Set<number>();
+  for (const [, cinfo] of clusters) {
+    if (cinfo.size >= 2) {
+      for (const m of cinfo.members) {
+        dupeRowIds.add(m);
+      }
+    }
+  }
+
+  const dupes: Row[] = [];
+  const unique: Row[] = [];
+  for (const row of rowsWithClusters) {
+    const rowId = row.__row_id__ as number;
+    if (dupeRowIds.has(rowId)) {
+      dupes.push(row);
+    } else {
+      unique.push(row);
+    }
+  }
+
+  // ---- Step 9: Compute stats ----
+  const totalRecords = processed.length;
+  const totalClusters = clusters.size;
+  const matchedRecords = dupes.length;
+  const uniqueRecords = unique.length;
+  const matchRate = totalRecords > 0 ? matchedRecords / totalRecords : 0;
+
+  const stats: DedupeStats = {
+    totalRecords,
+    totalClusters,
+    matchRate,
+    matchedRecords,
+    uniqueRecords,
+  };
+
+  // ---- Step 10: Return result ----
+  return {
+    goldenRecords,
+    clusters,
+    dupes,
+    unique,
+    stats,
+    scoredPairs: allPairs,
+    config,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// runMatchPipeline
+// ---------------------------------------------------------------------------
+
+/**
+ * Run the match pipeline: match target rows against reference rows.
+ *
+ * - Assigns __row_id__ with offset for reference rows
+ * - Assigns __source__ ("target" / "reference")
+ * - Runs same pipeline but filters to cross-source pairs only
+ */
+export function runMatchPipeline(
+  targetRows: readonly Row[],
+  referenceRows: readonly Row[],
+  config: GoldenMatchConfig,
+): MatchResult {
+  if (targetRows.length === 0 || referenceRows.length === 0) {
+    return {
+      matched: [],
+      unmatched: [...targetRows],
+      stats: {
+        totalTarget: targetRows.length,
+        totalReference: referenceRows.length,
+        matchedCount: 0,
+        unmatchedCount: targetRows.length,
+        matchRate: 0,
+      },
+    };
+  }
+
+  // Add row IDs and source labels
+  const target = addSourceColumn(addRowIds(targetRows, 0), "target");
+  const reference = addSourceColumn(
+    addRowIds(referenceRows, targetRows.length),
+    "reference",
+  );
+
+  // Combine and run dedupe pipeline with cross-file filter
+  const combined = [...target, ...reference];
+  const result = runDedupePipeline(combined, config, {
+    acrossFilesOnly: true,
+    outputGolden: false,
+  });
+
+  // Track which target row IDs got matched
+  const targetIds = new Set<number>(
+    target.map((r) => r.__row_id__ as number),
+  );
+  const matchedTargetIds = new Set<number>();
+
+  for (const pair of result.scoredPairs) {
+    if (targetIds.has(pair.idA)) matchedTargetIds.add(pair.idA);
+    if (targetIds.has(pair.idB)) matchedTargetIds.add(pair.idB);
+  }
+
+  // Build matched/unmatched from original target rows
+  const matched: Row[] = [];
+  const unmatched: Row[] = [];
+  for (const row of target) {
+    const rowId = row.__row_id__ as number;
+    if (matchedTargetIds.has(rowId)) {
+      matched.push(row);
+    } else {
+      unmatched.push(row);
+    }
+  }
+
+  return {
+    matched,
+    unmatched,
+    stats: {
+      totalTarget: targetRows.length,
+      totalReference: referenceRows.length,
+      matchedCount: matched.length,
+      unmatchedCount: unmatched.length,
+      matchRate:
+        targetRows.length > 0 ? matched.length / targetRows.length : 0,
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Internal: empty result
+// ---------------------------------------------------------------------------
+
+function _emptyDedupeResult(config: GoldenMatchConfig): DedupeResult {
+  return {
+    goldenRecords: [],
+    clusters: new Map(),
+    dupes: [],
+    unique: [],
+    stats: {
+      totalRecords: 0,
+      totalClusters: 0,
+      matchRate: 0,
+      matchedRecords: 0,
+      uniqueRecords: 0,
+    },
+    scoredPairs: [],
+    config,
+  };
+}

package/src/core/pprl/protocol.ts

@@ -0,0 +1,216 @@
+/**
+ * pprl/protocol.ts — Privacy-preserving record linkage.
+ * Edge-safe: no `node:` imports.
+ *
+ * Ports goldenmatch/pprl/protocol.py. Encodes both datasets as bloom
+ * filters (CLKs) over the selected fields, then scores pairs via Dice or
+ * Jaccard similarity. Two protocol stubs are surfaced: trusted third
+ * party (no crypto beyond the bloom filter itself) and a simple SMC
+ * sketch that adds a salt per party before encoding.
+ */
+
+import type { Row } from "../types.js";
+import { applyTransform } from "../transforms.js";
+import { diceCoefficient } from "../scorer.js";
+import { profileRows, type ColumnProfile } from "../profiler.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface PPRLConfig {
+  readonly fields: readonly string[];
+  readonly securityLevel: "standard" | "high" | "paranoid";
+  readonly protocol: "trusted_third_party" | "smc";
+  readonly threshold: number;
+  /** Optional salt used with "high"/"paranoid" levels. */
+  readonly salt?: string;
+}
+
+export interface PPRLMatch {
+  readonly idA: number;
+  readonly idB: number;
+  readonly score: number;
+}
+
+export interface PPRLResult {
+  readonly matches: readonly PPRLMatch[];
+  readonly stats: Readonly<Record<string, unknown>>;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function rowString(row: Row, fields: readonly string[]): string {
+  const parts: string[] = [];
+  for (const f of fields) {
+    const v = row[f];
+    if (v === null || v === undefined) continue;
+    const s = typeof v === "string" ? v : String(v);
+    const normalized = applyTransform(s, "lowercase") ?? s;
+    const cleaned = applyTransform(normalized, "normalize_whitespace") ?? normalized;
+    parts.push(cleaned);
+  }
+  return parts.join(" ");
+}
+
+function encodeRow(row: Row, config: PPRLConfig): string {
+  const value = rowString(row, config.fields);
+  if (value.length === 0) return "";
+  const transformName =
+    config.salt && (config.securityLevel === "high" || config.securityLevel === "paranoid")
+      ? `bloom_filter:${config.securityLevel}:${config.salt}`
+      : `bloom_filter:${config.securityLevel}`;
+  return applyTransform(value, transformName) ?? "";
+}
+
+// ---------------------------------------------------------------------------
+// Core linkage
+// ---------------------------------------------------------------------------
+
+/**
+ * Encode both row sets as bloom filters and emit pair matches above the
+ * configured threshold.
+ */
+export function runPPRL(
+  rowsA: readonly Row[],
+  rowsB: readonly Row[],
+  config: PPRLConfig,
+): PPRLResult {
+  const encodedA: string[] = rowsA.map((r) => encodeRow(r, config));
+  const encodedB: string[] = rowsB.map((r) => encodeRow(r, config));
+
+  const matches: PPRLMatch[] = [];
+  let compared = 0;
+
+  for (let i = 0; i < encodedA.length; i++) {
+    const a = encodedA[i]!;
+    if (a.length === 0) continue;
+    for (let j = 0; j < encodedB.length; j++) {
+      const b = encodedB[j]!;
+      if (b.length === 0) continue;
+      compared++;
+      const score = diceCoefficient(a, b);
+      if (score >= config.threshold) {
+        matches.push({ idA: i, idB: j, score });
+      }
+    }
+  }
+
+  return {
+    matches,
+    stats: {
+      protocol: config.protocol,
+      securityLevel: config.securityLevel,
+      comparedPairs: compared,
+      matchCount: matches.length,
+      threshold: config.threshold,
+      fields: config.fields,
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Auto-config
+// ---------------------------------------------------------------------------
+
+const MIN_LENGTH = 3;
+const MAX_LENGTH = 15;
+const MAX_FIELDS = 4;
+const MIN_THRESHOLD = 0.85;
+
+/**
+ * Auto-pick PPRL parameters for the given dataset pair. Penalizes
+ * near-unique fields (IDs), over-long fields, and high-null fields.
+ */
+export function autoConfigurePPRL(
+  rowsA: readonly Row[],
+  rowsB: readonly Row[],
+): PPRLConfig {
+  const profileA = profileRows(rowsA);
+  const profileB = profileRows(rowsB);
+
+  const commonCols = new Set<string>();
+  for (const c of profileA.columns) {
+    if (profileB.byName[c.name]) commonCols.add(c.name);
+  }
+
+  interface Candidate {
+    readonly name: string;
+    readonly score: number;
+  }
+
+  const candidates: Candidate[] = [];
+  for (const name of commonCols) {
+    const pa = profileA.byName[name];
+    const pb = profileB.byName[name];
+    if (!pa || !pb) continue;
+
+    const nullRate = Math.max(pa.nullRate, pb.nullRate);
+    if (nullRate > 0.3) continue;
+
+    const avgLen = (pa.avgLength + pb.avgLength) / 2;
+    if (avgLen < MIN_LENGTH) continue;
+    if (avgLen > MAX_LENGTH) continue;
+
+    // Penalize near-unique fields (likely IDs)
+    const card = Math.max(pa.cardinalityRatio, pb.cardinalityRatio);
+    if (card > 0.95) continue;
+
+    // Score: prefer moderate cardinality, low nulls, moderate length.
+    const lenPenalty = Math.abs(avgLen - 8) / 8;
+    const score = (1 - nullRate) * (1 - Math.abs(card - 0.5)) * (1 - lenPenalty);
+
+    candidates.push({ name, score });
+  }
+
+  candidates.sort((a, b) => b.score - a.score);
+  const fields = candidates.slice(0, MAX_FIELDS).map((c) => c.name);
+
+  return {
+    fields,
+    securityLevel: "standard",
+    protocol: "trusted_third_party",
+    threshold: MIN_THRESHOLD,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Protocol wrappers (API-parity stubs)
+// ---------------------------------------------------------------------------
+
+/**
+ * Trusted-third-party linkage: both parties ship encoded CLKs to a
+ * trusted intermediary that runs the similarity scoring. Same mechanics
+ * as `runPPRL`, but callsite is semantically distinct.
+ */
+export function linkTrustedThirdParty(
+  rowsA: readonly Row[],
+  rowsB: readonly Row[],
+  config: PPRLConfig,
+): PPRLResult {
+  return runPPRL(rowsA, rowsB, { ...config, protocol: "trusted_third_party" });
+}
+
+/**
+ * Secure-multiparty-computation linkage (simplified): each party salts
+ * its inputs with a shared secret. Requires a non-empty `salt` in config
+ * and a "high"/"paranoid" security level.
+ */
+export function linkSMC(
+  rowsA: readonly Row[],
+  rowsB: readonly Row[],
+  config: PPRLConfig,
+): PPRLResult {
+  if (!config.salt || config.salt.length === 0) {
+    throw new Error("SMC protocol requires a non-empty `salt`");
+  }
+  if (config.securityLevel === "standard") {
+    throw new Error("SMC protocol requires securityLevel of 'high' or 'paranoid'");
+  }
+  return runPPRL(rowsA, rowsB, { ...config, protocol: "smc" });
+}
+
+// Re-export profile type for consumers that want it alongside.
+export type { ColumnProfile };