goldenmatch 0.1.0

package/tests/unit/evaluate.test.ts

@@ -0,0 +1,85 @@
+import { describe, it, expect } from "vitest";
+import { evaluatePairs, evaluateClusters, buildClusters } from "../../src/core/index.js";
+import type { ScoredPair } from "../../src/core/index.js";
+
+describe("evaluatePairs", () => {
+  it("perfect predictions -> precision=recall=f1=1.0", () => {
+    const predicted: ScoredPair[] = [
+      { idA: 1, idB: 2, score: 0.9 },
+      { idA: 3, idB: 4, score: 0.9 },
+    ];
+    const truth: [number, number][] = [
+      [1, 2],
+      [3, 4],
+    ];
+    const result = evaluatePairs(predicted, truth);
+    expect(result.precision).toBe(1);
+    expect(result.recall).toBe(1);
+    expect(result.f1).toBe(1);
+    expect(result.truePositives).toBe(2);
+    expect(result.falsePositives).toBe(0);
+    expect(result.falseNegatives).toBe(0);
+  });
+
+  it("mix of TP, FP, FN", () => {
+    const predicted: ScoredPair[] = [
+      { idA: 1, idB: 2, score: 0.9 }, // TP
+      { idA: 5, idB: 6, score: 0.9 }, // FP
+    ];
+    const truth: [number, number][] = [
+      [1, 2], // in predicted -> TP
+      [3, 4], // not predicted -> FN
+    ];
+    const result = evaluatePairs(predicted, truth);
+    expect(result.truePositives).toBe(1);
+    expect(result.falsePositives).toBe(1);
+    expect(result.falseNegatives).toBe(1);
+    expect(result.precision).toBe(0.5);
+    expect(result.recall).toBe(0.5);
+    expect(result.f1).toBe(0.5);
+  });
+
+  it("no predictions and no truth -> all zeros", () => {
+    const result = evaluatePairs([], []);
+    expect(result.precision).toBe(0);
+    expect(result.recall).toBe(0);
+    expect(result.f1).toBe(0);
+  });
+
+  it("canonicalizes pair ordering", () => {
+    // predicted (2,1) should match truth (1,2)
+    const predicted: ScoredPair[] = [{ idA: 2, idB: 1, score: 0.9 }];
+    const truth: [number, number][] = [[1, 2]];
+    const result = evaluatePairs(predicted, truth);
+    expect(result.truePositives).toBe(1);
+  });
+});
+
+describe("evaluateClusters", () => {
+  it("expands clusters to pairs then evaluates", () => {
+    // cluster {1,2,3} -> pairs (1,2),(1,3),(2,3)
+    const pairs: [number, number, number][] = [
+      [1, 2, 0.9],
+      [2, 3, 0.9],
+      [1, 3, 0.9],
+    ];
+    const clusters = buildClusters(pairs, [1, 2, 3]);
+    const truth: [number, number][] = [
+      [1, 2],
+      [1, 3],
+      [2, 3],
+    ];
+    const result = evaluateClusters(clusters, truth, [1, 2, 3]);
+    expect(result.precision).toBe(1);
+    expect(result.recall).toBe(1);
+  });
+
+  it("singleton clusters produce no pairs", () => {
+    const clusters = buildClusters([], [1, 2, 3]);
+    // No predicted pairs, but truth has (1,2) -> all false negatives
+    const truth: [number, number][] = [[1, 2]];
+    const result = evaluateClusters(clusters, truth, [1, 2, 3]);
+    expect(result.truePositives).toBe(0);
+    expect(result.falseNegatives).toBe(1);
+  });
+});

package/tests/unit/explain.test.ts

@@ -0,0 +1,73 @@
+import { describe, it, expect } from "vitest";
+import { explainPair, explainCluster, buildClusters, pairKey } from "../../src/core/index.js";
+import type { ClusterInfo, MatchkeyConfig, Row } from "../../src/core/index.js";
+
+const MK: MatchkeyConfig = {
+  name: "name_mk",
+  type: "weighted",
+  threshold: 0.8,
+  fields: [
+    { field: "name", transforms: [], scorer: "jaro_winkler", weight: 1.0 },
+    { field: "city", transforms: [], scorer: "exact", weight: 1.0 },
+  ],
+};
+
+describe("explainPair", () => {
+  it("produces reasoning strings and overall score", () => {
+    const rowA: Row = { name: "John Smith", city: "NYC" };
+    const rowB: Row = { name: "John Smith", city: "NYC" };
+    const exp = explainPair(rowA, rowB, MK);
+    expect(exp.score).toBe(1.0);
+    expect(exp.reasoning.length).toBeGreaterThan(0);
+    expect(exp.confidence).toBe("high");
+    expect(typeof exp.explanation).toBe("string");
+    expect(exp.explanation.length).toBeGreaterThan(0);
+  });
+
+  it("includes fieldScores for each field", () => {
+    const rowA: Row = { name: "John", city: "NYC" };
+    const rowB: Row = { name: "Jon", city: "NYC" };
+    const exp = explainPair(rowA, rowB, MK);
+    expect(exp.fieldScores.name).not.toBe(null);
+    expect(exp.fieldScores.city).toBe(1.0);
+  });
+
+  it("low-confidence when overall score is low", () => {
+    const rowA: Row = { name: "Alice", city: "NYC" };
+    const rowB: Row = { name: "Zeke", city: "LA" };
+    const exp = explainPair(rowA, rowB, MK);
+    expect(exp.confidence).toBe("low");
+  });
+});
+
+describe("explainCluster", () => {
+  it("summarizes multi-member cluster", () => {
+    const pairs: [number, number, number][] = [[1, 2, 0.9], [2, 3, 0.85], [1, 3, 0.88]];
+    const clusters = buildClusters(pairs, [1, 2, 3]);
+    const cid = [...clusters.keys()][0]!;
+    const cinfo = clusters.get(cid)!;
+
+    const rows: Row[] = [
+      { __row_id__: 1, name: "A", city: "X" },
+      { __row_id__: 2, name: "A", city: "X" },
+      { __row_id__: 3, name: "A", city: "X" },
+    ];
+    const exp = explainCluster(cid, cinfo, rows, MK);
+    expect(exp.size).toBe(3);
+    expect(exp.summary).toContain("Cluster of 3");
+  });
+
+  it("singleton cluster has specialized summary", () => {
+    const cinfo: ClusterInfo = {
+      members: [7],
+      size: 1,
+      oversized: false,
+      pairScores: new Map(),
+      confidence: 1.0,
+      bottleneckPair: null,
+      clusterQuality: "strong",
+    };
+    const exp = explainCluster(7, cinfo, [{ __row_id__: 7 }], MK);
+    expect(exp.summary).toContain("Singleton");
+  });
+});

package/tests/unit/golden.test.ts

@@ -0,0 +1,97 @@
+import { describe, it, expect } from "vitest";
+import { mergeField, buildGoldenRecord, makeGoldenRulesConfig } from "../../src/core/index.js";
+import type { GoldenFieldRule, Row } from "../../src/core/index.js";
+
+describe("mergeField strategies", () => {
+  it("most_complete picks longest string", () => {
+    const rule: GoldenFieldRule = { strategy: "most_complete" };
+    const result = mergeField(["Jon", "John Smith", "Joh"], rule);
+    expect(result.value).toBe("John Smith");
+  });
+
+  it("majority_vote picks most frequent", () => {
+    const rule: GoldenFieldRule = { strategy: "majority_vote" };
+    const result = mergeField(["A", "B", "A", "A"], rule);
+    expect(result.value).toBe("A");
+  });
+
+  it("source_priority picks according to priority list", () => {
+    const rule: GoldenFieldRule = {
+      strategy: "source_priority",
+      sourcePriority: ["crm", "erp"],
+    };
+    const result = mergeField(["valA", "valB"], rule, { sources: ["erp", "crm"] });
+    // crm has higher priority, so "valB" wins
+    expect(result.value).toBe("valB");
+  });
+
+  it("most_recent picks latest by date", () => {
+    const rule: GoldenFieldRule = { strategy: "most_recent" };
+    const result = mergeField(["old", "new"], rule, {
+      dates: ["2020-01-01", "2024-01-01"],
+    });
+    expect(result.value).toBe("new");
+  });
+
+  it("first_non_null picks first non-null", () => {
+    const rule: GoldenFieldRule = { strategy: "first_non_null" };
+    const result = mergeField([null, "first", "second"], rule);
+    expect(result.value).toBe("first");
+  });
+
+  it("all-identical values -> confidence 1.0", () => {
+    const rule: GoldenFieldRule = { strategy: "most_complete" };
+    const result = mergeField(["same", "same", "same"], rule);
+    expect(result.confidence).toBe(1.0);
+    expect(result.value).toBe("same");
+  });
+
+  it("all-null -> value null, confidence 0, sourceIndex null", () => {
+    const rule: GoldenFieldRule = { strategy: "most_complete" };
+    const result = mergeField([null, null, null], rule);
+    expect(result.value).toBe(null);
+    expect(result.confidence).toBe(0);
+    expect(result.sourceIndex).toBe(null);
+  });
+});
+
+describe("buildGoldenRecord", () => {
+  it("produces a merged record for the cluster", () => {
+    const clusterRows: Row[] = [
+      { __row_id__: 0, name: "Jon", email: "a@x.com" },
+      { __row_id__: 1, name: "John", email: "a@x.com" },
+    ];
+    const rules = makeGoldenRulesConfig({ defaultStrategy: "most_complete" });
+    const golden = buildGoldenRecord(clusterRows, rules);
+
+    // email is identical -> confidence 1.0
+    expect(golden.fields.email?.value).toBe("a@x.com");
+    expect(golden.fields.email?.confidence).toBe(1.0);
+
+    // name: most_complete picks longest -> "John"
+    expect(golden.fields.name?.value).toBe("John");
+
+    // goldenConfidence averages field confidences
+    expect(golden.goldenConfidence).toBeGreaterThan(0);
+    expect(golden.goldenConfidence).toBeLessThanOrEqual(1);
+  });
+
+  it("empty cluster returns empty record", () => {
+    const rules = makeGoldenRulesConfig();
+    const golden = buildGoldenRecord([], rules);
+    expect(Object.keys(golden.fields).length).toBe(0);
+    expect(golden.goldenConfidence).toBe(0);
+  });
+
+  it("ignores internal columns", () => {
+    const clusterRows: Row[] = [
+      { __row_id__: 0, __cluster_id__: 1, name: "A" },
+    ];
+    const rules = makeGoldenRulesConfig();
+    const golden = buildGoldenRecord(clusterRows, rules);
+    // __row_id__ should NOT appear in output fields
+    expect(golden.fields.__row_id__).toBeUndefined();
+    expect(golden.fields.__cluster_id__).toBeUndefined();
+    expect(golden.fields.name).toBeDefined();
+  });
+});

package/tests/unit/graph-er.test.ts

@@ -0,0 +1,173 @@
+import { describe, it, expect } from "vitest";
+import {
+  runGraphER,
+  type TableSchema,
+  type Relationship,
+  type GraphERScorer,
+} from "../../src/core/graph-er.js";
+import { scorePair } from "../../src/core/scorer.js";
+import { makeMatchkeyField } from "../../src/core/types.js";
+import type { Row, ScoredPair, MatchkeyField } from "../../src/core/types.js";
+
+// Build a scorer that compares every pair in a row list using a weighted
+// matchkey on the supplied fields. runGraphER expects pair idA/idB to be
+// 0-based row indices (it's how clustersFromPairs seeds its Union-Find).
+function allPairsScorer(fields: readonly MatchkeyField[]): GraphERScorer {
+  return (rows: readonly Row[]): readonly ScoredPair[] => {
+    const pairs: ScoredPair[] = [];
+    for (let i = 0; i < rows.length; i++) {
+      for (let j = i + 1; j < rows.length; j++) {
+        const score = scorePair(rows[i]!, rows[j]!, fields);
+        pairs.push({ idA: i, idB: j, score });
+      }
+    }
+    return pairs;
+  };
+}
+
+describe("runGraphER", () => {
+  it("produces clusters per table for 2 tables + 1 relationship", () => {
+    const customers: Row[] = [
+      { id: 1, name: "John Smith", company_id: 100 },
+      { id: 2, name: "Jon Smith", company_id: 100 },
+      { id: 3, name: "Jane Doe", company_id: 200 },
+    ];
+    const companies: Row[] = [
+      { id: 100, name: "Acme Inc" },
+      { id: 200, name: "Widgets LLC" },
+    ];
+
+    const tables: TableSchema[] = [
+      { name: "customers", rows: customers, idColumn: "id" },
+      { name: "companies", rows: companies, idColumn: "id" },
+    ];
+    const relationships: Relationship[] = [
+      { tableA: "customers", tableB: "companies", fkColumn: "company_id" },
+    ];
+
+    const nameField = [
+      makeMatchkeyField({ field: "name", scorer: "jaro_winkler", transforms: ["lowercase"] }),
+    ];
+    const scorerByTable = new Map<string, GraphERScorer>([
+      ["customers", allPairsScorer(nameField)],
+      ["companies", allPairsScorer(nameField)],
+    ]);
+
+    const result = runGraphER(tables, relationships, {
+      scorerByTable,
+      threshold: 0.85,
+      maxIterations: 5,
+    });
+
+    expect(result.clustersByTable.has("customers")).toBe(true);
+    expect(result.clustersByTable.has("companies")).toBe(true);
+    // Companies are distinct -> 2 singleton clusters
+    expect(result.clustersByTable.get("companies")!.size).toBe(2);
+    // John/Jon Smith pair above threshold -> 1 cluster of size 2; Jane is singleton
+    const custClusters = result.clustersByTable.get("customers")!;
+    const sizes = [...custClusters.values()].map((c) => c.size).sort();
+    expect(sizes).toEqual([1, 2]);
+  });
+
+  it("evidence propagation pulls rows with shared FK cluster toward same cluster", () => {
+    // Two customer rows whose names are borderline (below base threshold) but
+    // whose company_id points to the same company row. Evidence boost should
+    // push them over the threshold once companies are clustered.
+    const customers: Row[] = [
+      { id: 1, name: "J Smith", company_id: 100 },
+      { id: 2, name: "John Smyth", company_id: 100 },
+    ];
+    const companies: Row[] = [{ id: 100, name: "Acme Inc" }];
+
+    const tables: TableSchema[] = [
+      { name: "customers", rows: customers, idColumn: "id" },
+      { name: "companies", rows: companies, idColumn: "id" },
+    ];
+    const relationships: Relationship[] = [
+      { tableA: "customers", tableB: "companies", fkColumn: "company_id" },
+    ];
+    const nameField = [makeMatchkeyField({ field: "name", scorer: "jaro_winkler" })];
+    const scorerByTable = new Map<string, GraphERScorer>([
+      ["customers", allPairsScorer(nameField)],
+      ["companies", allPairsScorer(nameField)],
+    ]);
+
+    // Choose threshold right above the raw JW score for "J Smith" vs "John Smyth"
+    // so that only the boost can merge them.
+    const rawScore = scorePair(customers[0]!, customers[1]!, nameField);
+    const threshold = Math.min(0.99, rawScore + 0.05);
+
+    const result = runGraphER(tables, relationships, {
+      scorerByTable,
+      threshold,
+      similarityBoost: 0.5,
+      maxIterations: 5,
+    });
+
+    const custClusters = result.clustersByTable.get("customers")!;
+    // After at least one propagation iteration, the two customers should merge.
+    const merged = [...custClusters.values()].some((c) => c.size === 2);
+    expect(merged).toBe(true);
+  });
+
+  it("converges within maxIterations for a small tractable dataset", () => {
+    const customers: Row[] = [
+      { id: 1, name: "Alice", company_id: 100 },
+      { id: 2, name: "Bob", company_id: 200 },
+    ];
+    const companies: Row[] = [
+      { id: 100, name: "Acme" },
+      { id: 200, name: "Widget" },
+    ];
+    const tables: TableSchema[] = [
+      { name: "customers", rows: customers, idColumn: "id" },
+      { name: "companies", rows: companies, idColumn: "id" },
+    ];
+    const nameField = [makeMatchkeyField({ field: "name", scorer: "jaro_winkler" })];
+    const scorerByTable = new Map<string, GraphERScorer>([
+      ["customers", allPairsScorer(nameField)],
+      ["companies", allPairsScorer(nameField)],
+    ]);
+
+    const result = runGraphER(tables, [], { scorerByTable, maxIterations: 10 });
+    expect(result.converged).toBe(true);
+    expect(result.iterations).toBeLessThanOrEqual(10);
+  });
+
+  it("respects the maxIterations cap when propagation never stabilizes", () => {
+    // Construct inputs that force at least one iteration — we just need to
+    // confirm the loop terminates at the cap even if convergence isn't reached.
+    const customers: Row[] = [
+      { id: 1, name: "A", company_id: 100 },
+      { id: 2, name: "B", company_id: 100 },
+    ];
+    const companies: Row[] = [{ id: 100, name: "Acme" }];
+    const tables: TableSchema[] = [
+      { name: "customers", rows: customers, idColumn: "id" },
+      { name: "companies", rows: companies, idColumn: "id" },
+    ];
+    const nameField = [makeMatchkeyField({ field: "name", scorer: "jaro_winkler" })];
+    const scorerByTable = new Map<string, GraphERScorer>([
+      ["customers", allPairsScorer(nameField)],
+      ["companies", allPairsScorer(nameField)],
+    ]);
+
+    const cap = 2;
+    const result = runGraphER(tables, [], {
+      scorerByTable,
+      maxIterations: cap,
+      convergenceThreshold: 0, // never accept convergence except exact match
+    });
+    // Iterations (+1 if converged) must not exceed the cap + 1.
+    expect(result.iterations).toBeLessThanOrEqual(cap + 1);
+  });
+
+  it("throws when a scorer is missing for a table", () => {
+    const tables: TableSchema[] = [
+      { name: "t", rows: [{ id: 1 }], idColumn: "id" },
+    ];
+    expect(() =>
+      runGraphER(tables, [], { scorerByTable: new Map() }),
+    ).toThrow(/Missing scorer/);
+  });
+});