goldenmatch 0.1.0

package/tests/unit/buildBlocksAsync.test.ts

@@ -0,0 +1,63 @@
+import { describe, it, expect } from "vitest";
+import {
+  buildBlocks,
+  buildBlocksAsync,
+  makeBlockingConfig,
+} from "../../src/core/index.js";
+import type { Row, BlockingConfig } from "../../src/core/index.js";
+
+describe("buildBlocksAsync", () => {
+  it("strategy='static' delegates to buildBlocks (same result)", async () => {
+    const rows: Row[] = [
+      { __row_id__: 0, zip: "12345" },
+      { __row_id__: 1, zip: "12345" },
+      { __row_id__: 2, zip: "67890" },
+      { __row_id__: 3, zip: "67890" },
+    ];
+    const config: BlockingConfig = makeBlockingConfig({
+      strategy: "static",
+      keys: [{ fields: ["zip"], transforms: [] }],
+    });
+    const sync = buildBlocks(rows, config);
+    const async_ = await buildBlocksAsync(rows, config);
+    expect(async_.length).toBe(sync.length);
+    for (let i = 0; i < sync.length; i++) {
+      expect(async_[i]!.blockKey).toBe(sync[i]!.blockKey);
+      expect(async_[i]!.rows.length).toBe(sync[i]!.rows.length);
+    }
+  });
+
+  it("strategy='ann' without apiKey/env throws clear error", async () => {
+    const rows: Row[] = [
+      { __row_id__: 0, name: "Alice" },
+      { __row_id__: 1, name: "Alyce" },
+    ];
+    const config: BlockingConfig = makeBlockingConfig({
+      strategy: "ann",
+      keys: [],
+      annColumn: "name",
+    });
+    // Stash any real API key so the test is deterministic.
+    const env = (globalThis as { process?: { env?: Record<string, string | undefined> } }).process?.env;
+    const saved = env?.OPENAI_API_KEY;
+    if (env) delete env.OPENAI_API_KEY;
+    try {
+      await expect(buildBlocksAsync(rows, config)).rejects.toThrow(/OpenAI|API key|apiKey/);
+    } finally {
+      if (env && saved !== undefined) env.OPENAI_API_KEY = saved;
+    }
+  });
+
+  it("strategy='ann' missing annColumn throws", async () => {
+    const rows: Row[] = [
+      { __row_id__: 0, name: "Alice" },
+      { __row_id__: 1, name: "Alyce" },
+    ];
+    const config: BlockingConfig = makeBlockingConfig({
+      strategy: "ann",
+      keys: [],
+      // annColumn omitted
+    });
+    await expect(buildBlocksAsync(rows, config)).rejects.toThrow(/annColumn/);
+  });
+});

package/tests/unit/cluster.test.ts

@@ -0,0 +1,213 @@
+import { describe, it, expect } from "vitest";
+import {
+  UnionFind,
+  buildClusters,
+  computeClusterConfidence,
+  unmergeRecord,
+  unmergeCluster,
+  addToCluster,
+  pairKey,
+} from "../../src/core/index.js";
+import type { ClusterInfo } from "../../src/core/index.js";
+
+describe("UnionFind", () => {
+  it("add + find returns self", () => {
+    const uf = new UnionFind();
+    uf.add(1);
+    expect(uf.find(1)).toBe(1);
+  });
+
+  it("union joins two elements", () => {
+    const uf = new UnionFind();
+    uf.add(1);
+    uf.add(2);
+    uf.union(1, 2);
+    expect(uf.find(1)).toBe(uf.find(2));
+  });
+
+  it("getClusters returns grouping", () => {
+    const uf = new UnionFind();
+    uf.addMany([1, 2, 3, 4]);
+    uf.union(1, 2);
+    uf.union(3, 4);
+    const clusters = uf.getClusters();
+    expect(clusters.length).toBe(2);
+    const sizes = clusters.map((c) => c.size).sort();
+    expect(sizes).toEqual([2, 2]);
+  });
+
+  it("transitive closure", () => {
+    const uf = new UnionFind();
+    uf.addMany([1, 2, 3]);
+    uf.union(1, 2);
+    uf.union(2, 3);
+    expect(uf.find(1)).toBe(uf.find(3));
+  });
+});
+
+describe("buildClusters", () => {
+  it("simple pairs produce expected clusters", () => {
+    const pairs: [number, number, number][] = [
+      [1, 2, 0.95],
+      [3, 4, 0.9],
+    ];
+    const allIds = [1, 2, 3, 4, 5];
+    const clusters = buildClusters(pairs, allIds);
+    expect(clusters.size).toBe(3); // {1,2}, {3,4}, {5}
+    const sizes = [...clusters.values()].map((c) => c.size).sort();
+    expect(sizes).toEqual([1, 2, 2]);
+  });
+
+  it("cluster with only singletons", () => {
+    const clusters = buildClusters([], [1, 2, 3]);
+    expect(clusters.size).toBe(3);
+    for (const c of clusters.values()) {
+      expect(c.size).toBe(1);
+    }
+  });
+
+  it("weak cluster detection downgrades confidence", () => {
+    // Chain with weak link: edges (1,2)=0.95, (2,3)=0.95, (1,3)=0.3
+    // avg - min = (0.95+0.95+0.3)/3 - 0.3 ~= 0.433, > 0.3 threshold
+    const pairs: [number, number, number][] = [
+      [1, 2, 0.95],
+      [2, 3, 0.95],
+      [1, 3, 0.3],
+    ];
+    const clusters = buildClusters(pairs, [1, 2, 3]);
+    const single = [...clusters.values()][0]!;
+    expect(single.clusterQuality).toBe("weak");
+  });
+
+  it("oversized cluster auto-splits", () => {
+    // With maxClusterSize=2, 3 fully-connected nodes should split
+    const pairs: [number, number, number][] = [
+      [1, 2, 0.9],
+      [2, 3, 0.5], // weakest
+      [1, 3, 0.9],
+    ];
+    const clusters = buildClusters(pairs, [1, 2, 3], { maxClusterSize: 2 });
+    // Should be split into 2+ clusters
+    expect(clusters.size).toBeGreaterThan(1);
+  });
+
+  it("auto-split disabled leaves oversized", () => {
+    const pairs: [number, number, number][] = [
+      [1, 2, 0.9],
+      [2, 3, 0.9],
+      [1, 3, 0.9],
+    ];
+    const clusters = buildClusters(pairs, [1, 2, 3], {
+      maxClusterSize: 2,
+      autoSplit: false,
+    });
+    expect(clusters.size).toBe(1);
+    const single = [...clusters.values()][0]!;
+    expect(single.oversized).toBe(true);
+  });
+});
+
+describe("computeClusterConfidence", () => {
+  it("singleton confidence 1.0", () => {
+    const conf = computeClusterConfidence(new Map(), 1);
+    expect(conf.confidence).toBe(1.0);
+    expect(conf.minEdge).toBe(null);
+  });
+
+  it("confidence formula: 0.4*min + 0.3*avg + 0.3*connectivity", () => {
+    // One pair, size=2 — fully connected so connectivity=1.0
+    const pairs = new Map([[pairKey(1, 2), 0.8]]);
+    const conf = computeClusterConfidence(pairs, 2);
+    expect(conf.minEdge).toBe(0.8);
+    expect(conf.avgEdge).toBe(0.8);
+    expect(conf.connectivity).toBe(1.0);
+    // 0.4*0.8 + 0.3*0.8 + 0.3*1.0 = 0.32 + 0.24 + 0.30 = 0.86
+    expect(conf.confidence).toBeCloseTo(0.86, 5);
+  });
+
+  it("bottleneck pair is weakest edge", () => {
+    const pairs = new Map([
+      [pairKey(1, 2), 0.9],
+      [pairKey(2, 3), 0.5],
+    ]);
+    const conf = computeClusterConfidence(pairs, 3);
+    expect(conf.bottleneckPair).toEqual([2, 3]);
+  });
+});
+
+describe("unmergeRecord", () => {
+  it("removes a record and makes it singleton", () => {
+    const pairs: [number, number, number][] = [
+      [1, 2, 0.95],
+      [2, 3, 0.95],
+      [1, 3, 0.95],
+    ];
+    const clusters = buildClusters(pairs, [1, 2, 3]);
+    // Cluster has {1,2,3}
+    expect(clusters.size).toBe(1);
+
+    const updated = unmergeRecord(1, clusters);
+    // Now record 1 is a singleton; 2,3 may still be together
+    const allMembers: number[] = [];
+    for (const c of updated.values()) {
+      allMembers.push(...c.members);
+    }
+    expect(allMembers.sort()).toEqual([1, 2, 3]);
+
+    // Find cluster containing record 1
+    let foundSingleton = false;
+    for (const c of updated.values()) {
+      if (c.members.length === 1 && c.members[0] === 1) {
+        foundSingleton = true;
+      }
+    }
+    expect(foundSingleton).toBe(true);
+  });
+});
+
+describe("unmergeCluster", () => {
+  it("shatters cluster into singletons", () => {
+    const pairs: [number, number, number][] = [[1, 2, 0.95], [2, 3, 0.95]];
+    const clusters = buildClusters(pairs, [1, 2, 3]);
+    const cid = [...clusters.keys()][0]!;
+    const updated = unmergeCluster(cid, clusters);
+    expect(updated.size).toBe(3);
+    for (const c of updated.values()) {
+      expect(c.size).toBe(1);
+    }
+  });
+});
+
+describe("addToCluster", () => {
+  it("no match -> singleton", () => {
+    const clusters = new Map<number, ClusterInfo>();
+    addToCluster(5, [], clusters);
+    expect(clusters.size).toBe(1);
+    const c = [...clusters.values()][0]!;
+    expect(c.members).toEqual([5]);
+    expect(c.size).toBe(1);
+  });
+
+  it("1 cluster match -> joins that cluster", () => {
+    // Start with cluster {1,2}
+    const pairs: [number, number, number][] = [[1, 2, 0.9]];
+    const clusters = buildClusters(pairs, [1, 2]);
+    addToCluster(3, [[1, 0.9]], clusters);
+    // Should have one cluster of size 3
+    const sizes = [...clusters.values()].map((c) => c.size);
+    expect(sizes).toContain(3);
+  });
+
+  it("2+ cluster matches -> merges clusters", () => {
+    const pairs: [number, number, number][] = [
+      [1, 2, 0.9],
+      [3, 4, 0.9],
+    ];
+    const clusters = buildClusters(pairs, [1, 2, 3, 4]);
+    expect(clusters.size).toBe(2);
+    addToCluster(5, [[1, 0.9], [3, 0.9]], clusters);
+    // Should have merged into one cluster of size 5
+    const sizes = [...clusters.values()].map((c) => c.size);
+    expect(sizes).toContain(5);
+  });
+});

package/tests/unit/compare-clusters.test.ts

@@ -0,0 +1,42 @@
+import { describe, it, expect } from "vitest";
+import { compareClusters, buildClusters } from "../../src/core/index.js";
+
+describe("compareClusters (CCMS)", () => {
+  it("identical clustering -> all unchanged, TWI = 1", () => {
+    const pairs: [number, number, number][] = [[1, 2, 0.9], [3, 4, 0.9]];
+    const a = buildClusters(pairs, [1, 2, 3, 4]);
+    const b = buildClusters(pairs, [1, 2, 3, 4]);
+    const result = compareClusters(a, b);
+    expect(result.unchanged).toBe(a.size);
+    expect(result.merged).toBe(0);
+    expect(result.partitioned).toBe(0);
+    expect(result.overlapping).toBe(0);
+    expect(result.twi).toBeCloseTo(1.0, 5);
+  });
+
+  it("different clustering produces non-zero classifications", () => {
+    // A has {1,2,3}, B has {1,2},{3}
+    const pairsA: [number, number, number][] = [[1, 2, 0.9], [2, 3, 0.9]];
+    const pairsB: [number, number, number][] = [[1, 2, 0.9]];
+    const a = buildClusters(pairsA, [1, 2, 3]);
+    const b = buildClusters(pairsB, [1, 2, 3]);
+    const result = compareClusters(a, b);
+    // A cluster {1,2,3} is partitioned in B
+    expect(result.partitioned).toBeGreaterThanOrEqual(1);
+  });
+
+  it("throws if row id coverage differs", () => {
+    const a = buildClusters([], [1, 2]);
+    const b = buildClusters([], [1, 2, 3]);
+    expect(() => compareClusters(a, b)).toThrow();
+  });
+
+  it("returns cc1, cc2, rc metadata", () => {
+    const a = buildClusters([[1, 2, 0.9]], [1, 2, 3]);
+    const b = buildClusters([[1, 2, 0.9]], [1, 2, 3]);
+    const result = compareClusters(a, b);
+    expect(result.cc1).toBe(a.size);
+    expect(result.cc2).toBe(b.size);
+    expect(result.rc).toBe(3);
+  });
+});

package/tests/unit/config-loader.test.ts

@@ -0,0 +1,301 @@
+import { describe, it, expect } from "vitest";
+import { parseConfig } from "../../src/core/index.js";
+
+describe("parseConfig", () => {
+  it("accepts snake_case keys", () => {
+    const raw = {
+      match_settings: [
+        {
+          name: "email_mk",
+          type: "exact",
+          fields: [{ field: "email", transforms: ["lowercase"], scorer: "exact", weight: 1.0 }],
+        },
+      ],
+      threshold: 0.9,
+    };
+    const config = parseConfig(raw);
+    expect(config.matchkeys?.length).toBe(1);
+    expect(config.matchkeys?.[0]?.name).toBe("email_mk");
+    expect(config.threshold).toBe(0.9);
+  });
+
+  it("accepts camelCase keys", () => {
+    const raw = {
+      matchkeys: [
+        {
+          name: "mk1",
+          type: "weighted",
+          fields: [{ field: "name", transforms: [], scorer: "jaro_winkler", weight: 1.0 }],
+          threshold: 0.85,
+        },
+      ],
+    };
+    const config = parseConfig(raw);
+    const mk0 = config.matchkeys?.[0];
+    expect(mk0?.type).toBe("weighted");
+    if (mk0?.type === "weighted") {
+      expect(mk0.threshold).toBe(0.85);
+    }
+  });
+
+  it("parses matchkeys fields array", () => {
+    const raw = {
+      matchkeys: [
+        {
+          name: "m",
+          type: "weighted",
+          fields: [
+            { field: "first", transforms: ["lowercase"], scorer: "jaro_winkler", weight: 0.5 },
+            { field: "last", transforms: [], scorer: "jaro_winkler", weight: 1.0 },
+          ],
+        },
+      ],
+    };
+    const config = parseConfig(raw);
+    expect(config.matchkeys?.[0]?.fields.length).toBe(2);
+    expect(config.matchkeys?.[0]?.fields[0]?.weight).toBe(0.5);
+  });
+
+  it("parses blocking config", () => {
+    const raw = {
+      blocking: {
+        strategy: "static",
+        keys: [{ fields: ["zip"], transforms: ["lowercase"] }],
+        max_block_size: 1000,
+        skip_oversized: true,
+      },
+    };
+    const config = parseConfig(raw);
+    expect(config.blocking?.strategy).toBe("static");
+    expect(config.blocking?.maxBlockSize).toBe(1000);
+    expect(config.blocking?.skipOversized).toBe(true);
+    expect(config.blocking?.keys.length).toBe(1);
+  });
+
+  it("normalizes golden_rules.default -> defaultStrategy", () => {
+    const raw = {
+      golden_rules: {
+        default: "most_complete",
+      },
+    };
+    const config = parseConfig(raw);
+    expect(config.goldenRules?.defaultStrategy).toBe("most_complete");
+  });
+
+  it("accepts goldenRules.defaultStrategy directly", () => {
+    const raw = {
+      goldenRules: {
+        defaultStrategy: "majority_vote",
+      },
+    };
+    const config = parseConfig(raw);
+    expect(config.goldenRules?.defaultStrategy).toBe("majority_vote");
+  });
+
+  it("throws on invalid config (not an object)", () => {
+    expect(() => parseConfig("not-an-object")).toThrow();
+    expect(() => parseConfig(null)).toThrow();
+  });
+
+  it("throws on invalid nested config (matchkey without name)", () => {
+    const raw = {
+      matchkeys: [{ type: "exact", fields: [] }],
+    };
+    expect(() => parseConfig(raw)).toThrow();
+  });
+
+  // -------------------------------------------------------------------------
+  // String-union validation
+  // -------------------------------------------------------------------------
+
+  describe("string-union validation", () => {
+    it("throws on invalid matchkey type with clear message listing valid options", () => {
+      const raw = {
+        matchkeys: [
+          {
+            name: "bad",
+            type: "garbage",
+            fields: [{ field: "x", transforms: [], scorer: "exact", weight: 1 }],
+          },
+        ],
+      };
+      try {
+        parseConfig(raw);
+        throw new Error("should have thrown");
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        expect(msg).toContain("garbage");
+        expect(msg).toContain("exact");
+        expect(msg).toContain("weighted");
+        expect(msg).toContain("probabilistic");
+      }
+    });
+
+    it("throws on invalid transform with valid options listed", () => {
+      const raw = {
+        matchkeys: [
+          {
+            name: "mk",
+            type: "weighted",
+            fields: [
+              {
+                field: "name",
+                transforms: ["not_a_real_transform"],
+                scorer: "jaro_winkler",
+                weight: 1,
+              },
+            ],
+          },
+        ],
+      };
+      try {
+        parseConfig(raw);
+        throw new Error("should have thrown");
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        expect(msg).toContain("not_a_real_transform");
+        expect(msg).toContain("lowercase");
+        expect(msg).toContain("substring");
+        expect(msg).toContain("qgram");
+      }
+    });
+
+    it("throws on invalid blocking strategy", () => {
+      const raw = {
+        blocking: {
+          strategy: "nonsense",
+          keys: [{ fields: ["zip"], transforms: [] }],
+        },
+      };
+      expect(() => parseConfig(raw)).toThrow(/nonsense/);
+    });
+
+    it("throws on invalid golden_rules field strategy", () => {
+      const raw = {
+        golden_rules: {
+          default: "most_complete",
+          field_rules: {
+            email: { strategy: "pick_worst" },
+          },
+        },
+      };
+      expect(() => parseConfig(raw)).toThrow(/pick_worst/);
+    });
+
+    it("throws on invalid standardizer", () => {
+      const raw = {
+        standardization: {
+          rules: {
+            name: ["scramble"],
+          },
+        },
+      };
+      expect(() => parseConfig(raw)).toThrow(/scramble/);
+    });
+
+    it("throws on invalid memory backend", () => {
+      const raw = {
+        memory: {
+          enabled: true,
+          backend: "redis",
+        },
+      };
+      expect(() => parseConfig(raw)).toThrow(/redis/);
+    });
+
+    it("accepts parametric transforms (substring, qgram, bloom_filter)", () => {
+      const raw = {
+        matchkeys: [
+          {
+            name: "mk",
+            type: "weighted",
+            fields: [
+              {
+                field: "a",
+                transforms: ["substring:0:3", "qgram:3", "bloom_filter"],
+                scorer: "jaro_winkler",
+                weight: 1,
+              },
+              {
+                field: "b",
+                transforms: ["bloom_filter:high"],
+                scorer: "dice",
+                weight: 1,
+              },
+            ],
+          },
+        ],
+      };
+      const config = parseConfig(raw);
+      expect(config.matchkeys?.[0]?.fields[0]?.transforms).toEqual([
+        "substring:0:3",
+        "qgram:3",
+        "bloom_filter",
+      ]);
+      expect(config.matchkeys?.[0]?.fields[1]?.transforms).toEqual([
+        "bloom_filter:high",
+      ]);
+    });
+
+    it("unknown scorer only warns (does not throw)", () => {
+      const warnings: string[] = [];
+      const origWarn = console.warn;
+      console.warn = (msg: unknown) => {
+        warnings.push(String(msg));
+      };
+      try {
+        const raw = {
+          matchkeys: [
+            {
+              name: "mk",
+              type: "weighted",
+              fields: [
+                {
+                  field: "a",
+                  transforms: ["lowercase"],
+                  scorer: "my_plugin_scorer",
+                  weight: 1,
+                },
+              ],
+            },
+          ],
+        };
+        const config = parseConfig(raw);
+        expect(config.matchkeys?.[0]?.fields[0]?.scorer).toBe("my_plugin_scorer");
+        expect(warnings.join(" ")).toContain("my_plugin_scorer");
+      } finally {
+        console.warn = origWarn;
+      }
+    });
+
+    it("accepts valid known scorers without warning", () => {
+      const warnings: string[] = [];
+      const origWarn = console.warn;
+      console.warn = (msg: unknown) => {
+        warnings.push(String(msg));
+      };
+      try {
+        const raw = {
+          matchkeys: [
+            {
+              name: "mk",
+              type: "weighted",
+              fields: [
+                {
+                  field: "a",
+                  transforms: ["lowercase"],
+                  scorer: "jaro_winkler",
+                  weight: 1,
+                },
+              ],
+            },
+          ],
+        };
+        parseConfig(raw);
+        expect(warnings).toHaveLength(0);
+      } finally {
+        console.warn = origWarn;
+      }
+    });
+  });
+});

package/tests/unit/connectors-base.test.ts

@@ -0,0 +1,48 @@
+import { describe, it, expect } from "vitest";
+import {
+  registerConnector,
+  loadConnector,
+  listConnectors,
+  type BaseConnector,
+} from "../../src/node/connectors/base.js";
+
+function makeFakeConnector(name: string): BaseConnector {
+  return {
+    name,
+    async connect() {},
+    async read() {
+      return [];
+    },
+    async close() {},
+  };
+}
+
+describe("connector registry", () => {
+  it("registerConnector + loadConnector round-trips", () => {
+    registerConnector("test_fake_a", () => makeFakeConnector("test_fake_a"));
+    const c = loadConnector("test_fake_a", {});
+    expect(c.name).toBe("test_fake_a");
+  });
+
+  it("loadConnector with unknown name throws including registered list", () => {
+    registerConnector("known_one", () => makeFakeConnector("known_one"));
+    expect(() => loadConnector("does_not_exist", {})).toThrow(/Unknown connector/);
+    expect(() => loadConnector("does_not_exist", {})).toThrow(/known_one/);
+  });
+
+  it("listConnectors includes registered connectors", () => {
+    registerConnector("test_fake_b", () => makeFakeConnector("test_fake_b"));
+    const names = listConnectors();
+    expect(names).toContain("test_fake_b");
+  });
+
+  it("factory receives passed config", async () => {
+    let received: unknown = null;
+    registerConnector("test_fake_c", (cfg) => {
+      received = cfg;
+      return makeFakeConnector("test_fake_c");
+    });
+    loadConnector("test_fake_c", { foo: "bar", n: 42 });
+    expect(received).toEqual({ foo: "bar", n: 42 });
+  });
+});