goldenmatch 0.1.0

package/src/core/ingest.ts

@@ -0,0 +1,112 @@
+/**
+ * ingest.ts — Edge-safe data-shape transforms for row arrays.
+ * Edge-safe: no `node:` imports, no file I/O.
+ *
+ * Ports goldenmatch/core/ingest.py minus file loading — callers are
+ * expected to bring already-parsed rows (JSON, fetched CSV, etc.).
+ */
+
+import type { Row } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Column renaming
+// ---------------------------------------------------------------------------
+
+/**
+ * Rename columns according to a {oldName: newName} map.
+ *
+ * Keys missing from the map are passed through untouched. If a rename
+ * would collide with an existing column, the mapped column wins
+ * (mirroring Polars behavior).
+ */
+export function applyColumnMap(
+  rows: readonly Row[],
+  columnMap: Readonly<Record<string, string>>,
+): Row[] {
+  if (Object.keys(columnMap).length === 0) {
+    return rows.map((r) => ({ ...r }));
+  }
+  return rows.map((row) => {
+    const newRow: Record<string, unknown> = {};
+    for (const [key, val] of Object.entries(row)) {
+      const newKey = columnMap[key] ?? key;
+      newRow[newKey] = val;
+    }
+    return newRow as Row;
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Column validation
+// ---------------------------------------------------------------------------
+
+/**
+ * Ensure every required column exists on the first row of `rows`.
+ * No-ops on empty input.
+ */
+export function validateColumns(
+  rows: readonly Row[],
+  required: readonly string[],
+): void {
+  if (rows.length === 0) return;
+  const first = rows[0]!;
+  const missing = required.filter((c) => !(c in first));
+  if (missing.length > 0) {
+    throw new Error(`Required columns missing: ${missing.join(", ")}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Row concat (union of schemas)
+// ---------------------------------------------------------------------------
+
+/**
+ * Concatenate several row arrays. Unioned schema: any column present in
+ * any input appears in the output; missing values become null.
+ */
+export function concatRows(rowsArrays: readonly (readonly Row[])[]): Row[] {
+  const allKeys = new Set<string>();
+  let totalLen = 0;
+  for (const arr of rowsArrays) {
+    totalLen += arr.length;
+    for (const row of arr) {
+      for (const k of Object.keys(row)) allKeys.add(k);
+    }
+  }
+
+  const out: Row[] = new Array(totalLen);
+  let idx = 0;
+  for (const arr of rowsArrays) {
+    for (const row of arr) {
+      const merged: Record<string, unknown> = {};
+      for (const k of allKeys) {
+        merged[k] = k in row ? row[k] : null;
+      }
+      out[idx++] = merged as Row;
+    }
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Source tagging
+// ---------------------------------------------------------------------------
+
+/**
+ * Add a `__source__` column to each row. Useful when concatenating rows
+ * from multiple datasets and downstream logic needs to know the origin.
+ */
+export function tagSource(rows: readonly Row[], source: string): Row[] {
+  return rows.map((row) => ({ ...row, __source__: source }));
+}
+
+/**
+ * Add a `__row_id__` column if missing. IDs are assigned sequentially
+ * starting at `startAt` (default 0).
+ */
+export function assignRowIds(rows: readonly Row[], startAt: number = 0): Row[] {
+  return rows.map((row, i) => ({
+    ...row,
+    __row_id__: row["__row_id__"] ?? startAt + i,
+  }));
+}

package/src/core/learned-blocking.ts

@@ -0,0 +1,305 @@
+/**
+ * learned-blocking.ts — Data-driven predicate selection for blocking.
+ * Edge-safe: no `node:` imports.
+ *
+ * Ports goldenmatch/core/learned_blocking.py. Given a labelled set of
+ * matching pairs, greedy-select predicates (equal, soundex, prefix,
+ * qgram) that maximize recall while minimizing candidate pair count.
+ */
+
+import type { Row, BlockResult, ScoredPair } from "./types.js";
+import { applyTransform, soundex } from "./transforms.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface LearnedPredicate {
+  readonly type: "equal" | "soundex" | "first_n_chars" | "qgram";
+  readonly field: string;
+  readonly n?: number;
+  readonly recall: number;
+  readonly reduction: number;
+}
+
+export interface LearnedRules {
+  readonly predicates: readonly LearnedPredicate[];
+  readonly minRecall: number;
+  readonly minReduction: number;
+  readonly learnedAt: string;
+}
+
+export interface LearnBlockingOptions {
+  readonly minRecall?: number;
+  readonly minReduction?: number;
+  readonly predicateDepth?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Predicate evaluation
+// ---------------------------------------------------------------------------
+
+function fieldValue(row: Row, field: string): string | null {
+  const v = row[field];
+  if (v === null || v === undefined) return null;
+  const s = typeof v === "string" ? v : String(v);
+  const trimmed = s.trim();
+  return trimmed.length === 0 ? null : trimmed;
+}
+
+type PredicateFn = (row: Row) => string | null;
+
+function buildPredicateFn(
+  pred: Omit<LearnedPredicate, "recall" | "reduction">,
+): PredicateFn {
+  const { type, field, n } = pred;
+  switch (type) {
+    case "equal":
+      return (row) => {
+        const v = fieldValue(row, field);
+        return v === null ? null : applyTransform(v, "lowercase");
+      };
+    case "soundex":
+      return (row) => {
+        const v = fieldValue(row, field);
+        if (v === null) return null;
+        const code = soundex(v);
+        return code === "0000" ? null : code;
+      };
+    case "first_n_chars": {
+      const len = n ?? 3;
+      return (row) => {
+        const v = fieldValue(row, field);
+        if (v === null) return null;
+        const lc = v.toLowerCase();
+        return lc.length >= len ? lc.slice(0, len) : null;
+      };
+    }
+    case "qgram": {
+      const size = n ?? 3;
+      return (row) => {
+        const v = fieldValue(row, field);
+        if (v === null) return null;
+        const lc = v.toLowerCase();
+        if (lc.length < size) return null;
+        // Use the first q-gram as the block key (simple, deterministic).
+        return lc.slice(0, size);
+      };
+    }
+  }
+}
+
+interface PredicateStats {
+  readonly capturedPairKeys: Set<string>;
+  readonly candidatePairs: number;
+  readonly recall: number;
+  readonly reduction: number;
+  readonly blocks: Map<string, number[]>;
+}
+
+function pairKey(a: number, b: number): string {
+  return a < b ? `${a}|${b}` : `${b}|${a}`;
+}
+
+function evaluatePredicate(
+  rows: readonly Row[],
+  knownPairs: readonly ScoredPair[],
+  fn: PredicateFn,
+  totalPossiblePairs: number,
+): PredicateStats {
+  // Group rows by their block key.
+  const blocks = new Map<string, number[]>();
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i]!;
+    const key = fn(row);
+    if (key === null) continue;
+    const list = blocks.get(key);
+    if (list) list.push(i);
+    else blocks.set(key, [i]);
+  }
+
+  // Count candidate pairs generated by this predicate.
+  let candidatePairs = 0;
+  for (const ids of blocks.values()) {
+    const n = ids.length;
+    candidatePairs += (n * (n - 1)) / 2;
+  }
+
+  // Compute which known pairs are captured: both row indices share a block.
+  // Build row->block-key-set index.
+  const idToKeys = new Map<number, Set<string>>();
+  for (const [key, ids] of blocks) {
+    for (const id of ids) {
+      let set = idToKeys.get(id);
+      if (!set) {
+        set = new Set<string>();
+        idToKeys.set(id, set);
+      }
+      set.add(key);
+    }
+  }
+
+  const capturedPairKeys = new Set<string>();
+  for (const pair of knownPairs) {
+    const keysA = idToKeys.get(pair.idA);
+    const keysB = idToKeys.get(pair.idB);
+    if (!keysA || !keysB) continue;
+    for (const k of keysA) {
+      if (keysB.has(k)) {
+        capturedPairKeys.add(pairKey(pair.idA, pair.idB));
+        break;
+      }
+    }
+  }
+
+  const recall =
+    knownPairs.length === 0 ? 0 : capturedPairKeys.size / knownPairs.length;
+  const reduction =
+    totalPossiblePairs === 0
+      ? 1
+      : Math.max(0, 1 - candidatePairs / totalPossiblePairs);
+
+  return { capturedPairKeys, candidatePairs, recall, reduction, blocks };
+}
+
+// ---------------------------------------------------------------------------
+// Greedy predicate selection
+// ---------------------------------------------------------------------------
+
+/**
+ * Learn blocking rules from known-matching pairs.
+ *
+ * Evaluates candidate predicates for each column, then greedily selects
+ * predicates that add the most new recall per candidate pair introduced,
+ * stopping when minRecall is reached (or when options.predicateDepth is
+ * exceeded).
+ */
+export function learnBlockingRules(
+  rows: readonly Row[],
+  knownPairs: readonly ScoredPair[],
+  columns: readonly string[],
+  options?: LearnBlockingOptions,
+): LearnedRules {
+  const minRecall = options?.minRecall ?? 0.95;
+  const minReduction = options?.minReduction ?? 0.9;
+  const predicateDepth = options?.predicateDepth ?? 3;
+
+  const totalPossible = (rows.length * (rows.length - 1)) / 2;
+
+  const candidates: { pred: LearnedPredicate; stats: PredicateStats }[] = [];
+  for (const field of columns) {
+    const predVariants: {
+      type: LearnedPredicate["type"];
+      n?: number;
+    }[] = [
+      { type: "equal" },
+      { type: "soundex" },
+      { type: "first_n_chars", n: 3 },
+      { type: "first_n_chars", n: 4 },
+      { type: "qgram", n: 3 },
+    ];
+    for (const variant of predVariants) {
+      const predPartial: Omit<LearnedPredicate, "recall" | "reduction"> =
+        variant.n !== undefined
+          ? { type: variant.type, field, n: variant.n }
+          : { type: variant.type, field };
+      const fn = buildPredicateFn(predPartial);
+      const stats = evaluatePredicate(rows, knownPairs, fn, totalPossible);
+      if (stats.recall === 0) continue;
+      // Reject predicates whose reduction falls below threshold unless
+      // they contribute unique recall (caught in the greedy loop below).
+      const pred: LearnedPredicate = {
+        ...predPartial,
+        recall: stats.recall,
+        reduction: stats.reduction,
+      };
+      candidates.push({ pred, stats });
+    }
+  }
+
+  // Greedy selection: at each step pick the predicate that contributes the
+  // most new captured pairs per candidate pair introduced.
+  const selected: LearnedPredicate[] = [];
+  const covered = new Set<string>();
+
+  while (selected.length < predicateDepth) {
+    let best: { pred: LearnedPredicate; gain: number; stats: PredicateStats } | null = null;
+
+    for (const cand of candidates) {
+      if (selected.includes(cand.pred)) continue;
+
+      let newCaptures = 0;
+      for (const pk of cand.stats.capturedPairKeys) {
+        if (!covered.has(pk)) newCaptures++;
+      }
+      if (newCaptures === 0) continue;
+      if (cand.stats.reduction < minReduction && newCaptures === 0) continue;
+
+      // Marginal efficiency: new captures per candidate pair (lower cost
+      // predicates favored). Avoid divide-by-zero.
+      const cost = Math.max(1, cand.stats.candidatePairs);
+      const gain = newCaptures / cost;
+
+      if (best === null || gain > best.gain) {
+        best = { pred: cand.pred, gain, stats: cand.stats };
+      }
+    }
+
+    if (best === null) break;
+    selected.push(best.pred);
+    for (const pk of best.stats.capturedPairKeys) covered.add(pk);
+
+    const achievedRecall =
+      knownPairs.length === 0 ? 0 : covered.size / knownPairs.length;
+    if (achievedRecall >= minRecall) break;
+  }
+
+  return {
+    predicates: selected,
+    minRecall,
+    minReduction,
+    learnedAt: new Date().toISOString(),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Apply learned rules to produce blocks
+// ---------------------------------------------------------------------------
+
+/**
+ * Apply learned rules to rows, returning one BlockResult per non-empty,
+ * non-oversized block. Each predicate is its own pass (OR'd together).
+ */
+export function applyLearnedBlocks(
+  rows: readonly Row[],
+  rules: LearnedRules,
+  maxBlockSize: number,
+): BlockResult[] {
+  const out: BlockResult[] = [];
+
+  for (let p = 0; p < rules.predicates.length; p++) {
+    const pred = rules.predicates[p]!;
+    const fn = buildPredicateFn(pred);
+    const blocks = new Map<string, Row[]>();
+    for (const row of rows) {
+      const key = fn(row);
+      if (key === null) continue;
+      const list = blocks.get(key);
+      if (list) list.push(row);
+      else blocks.set(key, [row]);
+    }
+
+    for (const [key, blockRows] of blocks) {
+      if (blockRows.length < 2) continue;
+      if (blockRows.length > maxBlockSize) continue;
+      out.push({
+        blockKey: `learned:${pred.type}:${pred.field}=${key}`,
+        rows: blockRows,
+        strategy: "learned",
+        depth: p,
+      });
+    }
+  }
+
+  return out;
+}

package/src/core/lineage.ts

@@ -0,0 +1,221 @@
+/**
+ * lineage.ts — Provenance tracking for golden records.
+ * Edge-safe: no `node:` imports.
+ *
+ * Ports goldenmatch/core/lineage.py. Records which source rows contributed
+ * each golden-record field, with the survivorship strategy and confidence.
+ */
+
+import type { ClusterInfo, DedupeResult, Row } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface FieldProvenanceEntry {
+  readonly sourceRowId: number;
+  readonly strategy: string;
+  readonly confidence: number;
+}
+
+export interface LineageEdge {
+  readonly clusterId: number;
+  readonly sourceRowIds: readonly number[];
+  readonly goldenRowId: number;
+  readonly fieldProvenance: Readonly<Record<string, FieldProvenanceEntry>>;
+  readonly timestamp: string;
+  /**
+   * Human-readable summary of this lineage edge. Only populated when
+   * `buildLineage` is called with `{ naturalLanguage: true }`.
+   */
+  readonly naturalLanguage?: string;
+}
+
+export interface LineageBundle {
+  readonly edges: readonly LineageEdge[];
+  readonly timestamp: string;
+  readonly recordCount: number;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function isoTimestamp(): string {
+  // Edge-safe: Date works in browser/workers/edge runtimes.
+  return new Date().toISOString();
+}
+
+function getRowId(row: Row): number | null {
+  const raw = row["__row_id__"];
+  if (typeof raw === "number") return raw;
+  if (typeof raw === "string") {
+    const n = Number(raw);
+    return Number.isFinite(n) ? n : null;
+  }
+  return null;
+}
+
+function getClusterId(row: Row): number | null {
+  const raw = row["__cluster_id__"];
+  if (typeof raw === "number") return raw;
+  if (typeof raw === "string") {
+    const n = Number(raw);
+    return Number.isFinite(n) ? n : null;
+  }
+  return null;
+}
+
+function renderNaturalLanguage(edge: Omit<LineageEdge, "naturalLanguage">): string {
+  const numSources = edge.sourceRowIds.length;
+  const entries = Object.entries(edge.fieldProvenance);
+  const numFields = entries.length;
+  let strongest: [string, FieldProvenanceEntry] | null = null;
+  for (const entry of entries) {
+    if (strongest === null || entry[1].confidence > strongest[1].confidence) {
+      strongest = entry;
+    }
+  }
+  const strongestDesc = strongest
+    ? `${strongest[0]} from row ${strongest[1].sourceRowId} via ${strongest[1].strategy} (conf ${strongest[1].confidence.toFixed(2)})`
+    : "no field provenance";
+  return `Cluster ${edge.clusterId}: merged ${numSources} source records into golden row ${edge.goldenRowId} across ${numFields} fields. Strongest contribution: ${strongestDesc}.`;
+}
+
+function computeConfidence(cluster: ClusterInfo, strategy: string): number {
+  // Base confidence is the cluster confidence; minor bonus for
+  // most_complete / first_non_null which are safer picks.
+  const base = cluster.confidence;
+  if (strategy === "source_priority" || strategy === "most_complete") {
+    return Math.min(1, base + 0.05);
+  }
+  return base;
+}
+
+// ---------------------------------------------------------------------------
+// Build lineage
+// ---------------------------------------------------------------------------
+
+export interface BuildLineageOptions {
+  /**
+   * When `true`, every emitted `LineageEdge` gets a human-readable
+   * `naturalLanguage` summary describing which rows were merged, how many
+   * fields carry provenance, and the strongest contributing field. Zero LLM
+   * cost — purely template-based.
+   */
+  readonly naturalLanguage?: boolean;
+  readonly defaultStrategy?: string;
+}
+
+/**
+ * Build a lineage bundle from a DedupeResult.
+ *
+ * The resulting bundle has one edge per golden record, with field-level
+ * provenance keyed by column name. Source row IDs include every member of
+ * the cluster the golden record came from.
+ *
+ * Pass `{ naturalLanguage: true }` to populate a human-readable summary on
+ * each edge (see {@link LineageEdge.naturalLanguage}).
+ */
+export function buildLineage(
+  result: DedupeResult,
+  options?: BuildLineageOptions,
+): LineageBundle {
+  const defaultStrategy =
+    options?.defaultStrategy ??
+    result.config.goldenRules?.defaultStrategy ??
+    "most_complete";
+
+  const timestamp = isoTimestamp();
+  const edges: LineageEdge[] = [];
+
+  // Pre-index dupes by row_id → row for quick field lookup.
+  const rowById = new Map<number, Row>();
+  for (const r of result.dupes) {
+    const id = getRowId(r);
+    if (id !== null) rowById.set(id, r);
+  }
+  for (const r of result.unique) {
+    const id = getRowId(r);
+    if (id !== null) rowById.set(id, r);
+  }
+
+  for (const golden of result.goldenRecords) {
+    const clusterId = getClusterId(golden);
+    const goldenRowId = getRowId(golden) ?? -1;
+    if (clusterId === null) continue;
+
+    const cluster = result.clusters.get(clusterId);
+    if (!cluster) continue;
+
+    const fieldProvenance: Record<string, FieldProvenanceEntry> = {};
+    const confidence = computeConfidence(cluster, defaultStrategy);
+
+    for (const [key, value] of Object.entries(golden)) {
+      if (key.startsWith("__")) continue;
+      if (value === null || value === undefined) continue;
+
+      // Locate which member row contributed this value.
+      let sourceRowId = goldenRowId;
+      for (const memberId of cluster.members) {
+        const memberRow = rowById.get(memberId);
+        if (memberRow && memberRow[key] === value) {
+          sourceRowId = memberId;
+          break;
+        }
+      }
+
+      const fieldStrategy =
+        result.config.goldenRules?.fieldRules?.[key]?.strategy ??
+        defaultStrategy;
+
+      fieldProvenance[key] = {
+        sourceRowId,
+        strategy: fieldStrategy,
+        confidence,
+      };
+    }
+
+    const base: Omit<LineageEdge, "naturalLanguage"> = {
+      clusterId,
+      sourceRowIds: [...cluster.members],
+      goldenRowId,
+      fieldProvenance,
+      timestamp,
+    };
+    const edge: LineageEdge = options?.naturalLanguage
+      ? { ...base, naturalLanguage: renderNaturalLanguage(base) }
+      : base;
+    edges.push(edge);
+  }
+
+  return {
+    edges,
+    timestamp,
+    recordCount: edges.length,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// (De)serialization
+// ---------------------------------------------------------------------------
+
+/** Serialize a lineage bundle to stable, human-readable JSON. */
+export function lineageToJson(bundle: LineageBundle): string {
+  return JSON.stringify(bundle, null, 2);
+}
+
+/** Parse a lineage bundle from JSON. Does not validate schema. */
+export function lineageFromJson(json: string): LineageBundle {
+  const parsed = JSON.parse(json) as unknown;
+  if (
+    typeof parsed !== "object" ||
+    parsed === null ||
+    !("edges" in parsed) ||
+    !("timestamp" in parsed) ||
+    !("recordCount" in parsed)
+  ) {
+    throw new Error("Invalid lineage bundle: missing required fields");
+  }
+  return parsed as LineageBundle;
+}