@mailwoman/match 7.2.0 → 7.3.0

package/em.ts

@@ -0,0 +1,150 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Unsupervised parameter estimation for the Fellegi-Sunter model — the part that makes the matcher
+ *   work with no labeled data.
+ *
+ *   The paradox: to estimate `m`/`u` you need to know which pairs match, but finding matches is the
+ *   whole problem. EM (Winkler 1988) breaks it by treating the match/non-match status as a hidden
+ *   variable and iterating:
+ *
+ *   - **E-step** — under the current parameters, compute each pair's posterior responsibility `g =
+ *       P(match | its agreement pattern)`.
+ *   - **M-step** — re-estimate `λ`, and each level's `m`/`u`, as `g`-weighted (resp. `(1-g)`-weighted)
+ *       fractions of the pairs landing in that level.
+ *
+ *   It converges because true matches agree on most fields and non-matches don't, so the two classes
+ *   pull apart. Assumes conditional independence of the comparisons given match status (the
+ *   standard F-S assumption). Caveat from the literature: EM can land in a local optimum when the
+ *   true match rate is very low — seed from sensible `m`/`u` (the model's existing levels do this)
+ *   and sanity- check that the recovered `m` exceeds `u` on the top agreement level.
+ */
+
+import type { Comparison, FellegiSunterModel } from "./fellegi-sunter.ts"
+
+/**
+ * Tiny floor mixed into the M-step so an unobserved level never produces a zero (→ infinite weight).
+ */
+const EPSILON = 1e-9
+
+/** Reduce a record pair to its agreement pattern — the per-comparison level index (`-1` = missing). */
+export function agreementPattern<R>(comparisons: Comparison<R>[], a: R, b: R): number[] {
+	return comparisons.map((comparison) => comparison.assess(a, b))
+}
+
+/** Options for {@link estimateParameters}. */
+export interface EmOptions {
+	/** Hard iteration cap. Default 100. */
+	maxIterations?: number
+	/** Convergence tolerance on the largest parameter change between iterations. Default 1e-6. */
+	tolerance?: number
+	/** Starting prior match rate. Defaults to the model's `lambda`. */
+	initialLambda?: number
+}
+
+/** The fitted model plus convergence diagnostics. */
+export interface EmResult<R> {
+	/** The input model with every level's `m`/`u` and the prior `lambda` re-estimated. */
+	model: FellegiSunterModel<R>
+	/** The estimated prior match rate. */
+	lambda: number
+	iterations: number
+	converged: boolean
+}
+
+/**
+ * Estimate `m`/`u` and the prior `λ` from unlabeled agreement patterns via EM. The patterns are per-comparison level
+ * indices (as produced by {@link agreementPattern}); a `-1` (missing) field contributes no evidence to either class. The
+ * model's existing level `m`/`u` seed the iteration.
+ */
+export function estimateParameters<R>(
+	model: FellegiSunterModel<R>,
+	patterns: number[][],
+	opts: EmOptions = {}
+): EmResult<R> {
+	const maxIterations = opts.maxIterations ?? 100
+	const tolerance = opts.tolerance ?? 1e-6
+	const comparisons = model.comparisons
+	const levelCounts = comparisons.map((c) => c.levels.length)
+
+	// Per-comparison, per-level m/u, seeded from the model's current levels.
+	const m = comparisons.map((c) => c.levels.map((l) => l.m))
+	const u = comparisons.map((c) => c.levels.map((l) => l.u))
+	let lambda = opts.initialLambda ?? model.lambda
+
+	let iterations = 0
+	let converged = false
+
+	if (patterns.length === 0) {
+		return { model, lambda, iterations, converged }
+	}
+
+	for (; iterations < maxIterations; iterations++) {
+		const mNumerator = comparisons.map((_, i) => new Array<number>(levelCounts[i]!).fill(0))
+		const uNumerator = comparisons.map((_, i) => new Array<number>(levelCounts[i]!).fill(0))
+		const mDenominator = comparisons.map(() => 0)
+		const uDenominator = comparisons.map(() => 0)
+		let responsibilitySum = 0
+
+		// E-step: posterior P(match | pattern) for each pair.
+		for (const pattern of patterns) {
+			let matchLikelihood = lambda
+			let nonMatchLikelihood = 1 - lambda
+
+			for (let i = 0; i < comparisons.length; i++) {
+				const level = pattern[i]!
+
+				if (level < 0) continue
+				matchLikelihood *= m[i]![level]!
+				nonMatchLikelihood *= u[i]![level]!
+			}
+			const total = matchLikelihood + nonMatchLikelihood
+			const g = total > 0 ? matchLikelihood / total : 0
+			responsibilitySum += g
+
+			for (let i = 0; i < comparisons.length; i++) {
+				const level = pattern[i]!
+
+				if (level < 0) continue
+				mNumerator[i]![level]! += g
+				uNumerator[i]![level]! += 1 - g
+				mDenominator[i]! += g
+				uDenominator[i]! += 1 - g
+			}
+		}
+
+		// M-step: re-estimate λ and each level's m/u as (1-)g-weighted fractions.
+		const newLambda = responsibilitySum / patterns.length
+		let maxDelta = Math.abs(newLambda - lambda)
+		lambda = newLambda
+
+		for (let i = 0; i < comparisons.length; i++) {
+			const levels = levelCounts[i]!
+
+			for (let l = 0; l < levels; l++) {
+				const newM =
+					mDenominator[i]! > 0 ? (mNumerator[i]![l]! + EPSILON) / (mDenominator[i]! + EPSILON * levels) : m[i]![l]!
+				const newU =
+					uDenominator[i]! > 0 ? (uNumerator[i]![l]! + EPSILON) / (uDenominator[i]! + EPSILON * levels) : u[i]![l]!
+				maxDelta = Math.max(maxDelta, Math.abs(newM - m[i]![l]!), Math.abs(newU - u[i]![l]!))
+				m[i]![l] = newM
+				u[i]![l] = newU
+			}
+		}
+
+		if (maxDelta < tolerance) {
+			converged = true
+			iterations++
+			break
+		}
+	}
+
+	const fittedComparisons = comparisons.map((c, i) => ({
+		...c,
+		levels: c.levels.map((level, j) => ({ ...level, m: m[i]![j]!, u: u[i]![j]! })),
+	}))
+
+	return { model: { comparisons: fittedComparisons, lambda }, lambda, iterations, converged }
+}

package/fellegi-sunter.ts

@@ -0,0 +1,201 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The Fellegi-Sunter scorer — the matcher's decision layer.
+ *
+ *   Each field comparison lands a record pair in a discrete _agreement level_ (exact / high / low /
+ *   different / missing). Each level carries two probabilities: `m` = P(this level | the pair
+ *   really matches) and `u` = P(this level | it doesn't). Their ratio is a Bayes factor, and its
+ *   log is the level's contribution to the total match weight in bits:
+ *
+ *   ```
+ *   M = log2(λ / (1 - λ))  +  Σ_fields  log2(m_level / u_level)
+ * ```
+ *
+ *   — a prior (how likely any two random records match) plus an additive, per-field-attributable
+ *   stack of evidence. Convert `M` to a probability and threshold it: above the upper bound is a
+ *   link, below the lower bound a non-link, and the band between is _clerical review_ — the
+ *   calibrated abstain zone the whole design leans on.
+ *
+ *   The `m`/`u` numbers here are NOT universal constants. They are estimated from the data — by EM,
+ *   unsupervised (the next increment) — and the term-frequency adjustment that makes a rare-name
+ *   agreement count more than a common one layers on top. This module is the deterministic core
+ *   those build on: given the levels, it produces the weights, the probability, and the decision.
+ */
+
+import { nameSimilarity } from "./comparators.ts"
+
+/** One agreement level of a comparison, with its match / non-match probabilities. */
+export interface ComparisonLevel {
+	/** Human-readable label for debugging (`exact`, `high`, `different`). */
+	label: string
+	/** P(a pair lands in this level | it is a true match). A measure of data quality. */
+	m: number
+	/** P(a pair lands in this level | it is NOT a match). A measure of coincidence / cardinality. */
+	u: number
+	/** For similarity-driven comparisons: the minimum similarity (inclusive) to qualify. */
+	minSimilarity?: number
+	/** For distance-driven comparisons: the maximum distance in km (inclusive) to qualify. */
+	maxKm?: number
+}
+
+/** A per-field comparison: pull a value from each record and assign an agreement level. */
+export interface Comparison<R> {
+	/** Field name, for attribution. */
+	name: string
+	/** Levels ordered highest agreement → lowest (`exact` first, `different` last). */
+	levels: ComparisonLevel[]
+	/** Index into {@link levels}, or `-1` when either value is missing (no evidence → weight 0). */
+	assess(a: R, b: R): number
+	/**
+	 * Optional term-frequency adjustment: on the levels it names, replace the level's average `u` with the agreeing
+	 * value's actual frequency, so agreement on a rare value (`Vijayan`) outweighs agreement on a common one (`Smith`).
+	 * See `withTermFrequency`.
+	 */
+	termFrequency?: TermFrequencyAdjustment<R>
+}
+
+/**
+ * Per-value term-frequency adjustment for a comparison (the Splink/Winkler mechanism). `m` is unchanged; on an
+ * agreement level the effective `u` becomes the value's own frequency, adding `log2(u_level / frequency)` to the weight
+ * — large and positive for rare values, negative for common ones. Floored at
+ * {@link TermFrequencyAdjustment.minimumFrequency} so an ultra-rare value can't produce an unbounded boost.
+ */
+export interface TermFrequencyAdjustment<R> {
+	/** Relative frequency of a value in the data, in (0, 1]. Typically computed on-the-fly. */
+	frequency(value: string): number
+	/** The level indices the adjustment applies to (typically just the exact level). */
+	levels: ReadonlySet<number>
+	/** The agreeing value to look up for a pair (a normalized field value), or null to skip. */
+	value(a: R, b: R): string | null | undefined
+	/** Scale the adjustment in [0, 1]. Default 1. */
+	weight?: number
+	/** Floor for the looked-up frequency, bounding the boost on ultra-rare values. Default 1e-4. */
+	minimumFrequency?: number
+}
+
+/** A Fellegi-Sunter model: the field comparisons plus the prior match rate `λ`. */
+export interface FellegiSunterModel<R> {
+	comparisons: Comparison<R>[]
+	/** Prior probability that two records drawn at random are a match. */
+	lambda: number
+}
+
+/** The scored outcome for one record pair. */
+export interface PairScore {
+	/** Total match weight in bits (`log2` odds). */
+	weight: number
+	/** Match probability in [0, 1]. */
+	probability: number
+	/** Per-field breakdown — what drove the score. */
+	contributions: Array<{ name: string; level: string | null; weight: number }>
+}
+
+/** The terminal decision for a pair under upper / lower match-weight thresholds. */
+export type MatchDecision = "match" | "review" | "non-match"
+
+/** The Bayes-factor weight of a single level, in bits: `log2(m / u)`. */
+export function levelWeight(level: ComparisonLevel): number {
+	if (level.u <= 0) return level.m > 0 ? Infinity : 0
+
+	return Math.log2(level.m / level.u)
+}
+
+/** The prior match weight in bits: `log2(λ / (1 - λ))`. */
+export function priorWeight(lambda: number): number {
+	if (lambda <= 0) return -Infinity
+
+	if (lambda >= 1) return Infinity
+
+	return Math.log2(lambda / (1 - lambda))
+}
+
+/** Convert a total match weight (bits) to a probability, numerically stable for extreme weights. */
+export function probabilityFromWeight(weight: number): number {
+	return 1 / (1 + 2 ** -weight)
+}
+
+/**
+ * A comparison driven by a similarity function and a tier of `minSimilarity` thresholds (the StatCan/Splink recipe).
+ * Levels must be ordered highest → lowest similarity, the last acting as the `different` catch-all (`minSimilarity` 0).
+ * A missing value on either side yields no evidence.
+ */
+export function similarityComparison<R>(config: {
+	name: string
+	extract: (record: R) => string | null | undefined
+	/** Defaults to {@link nameSimilarity}. */
+	similarity?: (a: string, b: string) => number
+	levels: ComparisonLevel[]
+}): Comparison<R> {
+	const similarity = config.similarity ?? nameSimilarity
+
+	return {
+		name: config.name,
+		levels: config.levels,
+		assess(a, b) {
+			const va = config.extract(a)
+			const vb = config.extract(b)
+
+			if (!va || !vb || !va.trim() || !vb.trim()) return -1
+
+			const sim = similarity(va, vb)
+
+			for (let i = 0; i < config.levels.length; i++) {
+				if (sim >= (config.levels[i]!.minSimilarity ?? 0)) return i
+			}
+
+			return config.levels.length - 1
+		},
+	}
+}
+
+/** Score a record pair: total match weight, probability, and the per-field contributions. */
+export function scorePair<R>(model: FellegiSunterModel<R>, a: R, b: R): PairScore {
+	let weight = priorWeight(model.lambda)
+	const contributions: PairScore["contributions"] = []
+
+	for (const comparison of model.comparisons) {
+		const index = comparison.assess(a, b)
+
+		if (index < 0) {
+			contributions.push({ name: comparison.name, level: null, weight: 0 })
+			continue
+		}
+		const level = comparison.levels[index]!
+		let w = levelWeight(level)
+
+		// Term-frequency adjustment: swap the level's average u for the agreeing value's own frequency.
+		const tf = comparison.termFrequency
+
+		if (tf && tf.levels.has(index) && level.u > 0) {
+			const value = tf.value(a, b)
+
+			if (value) {
+				const frequency = Math.max(tf.frequency(value), tf.minimumFrequency ?? 1e-4)
+
+				if (frequency > 0) {
+					w += Math.log2(level.u / frequency) * (tf.weight ?? 1)
+				}
+			}
+		}
+
+		weight += w
+		contributions.push({ name: comparison.name, level: level.label, weight: w })
+	}
+
+	return { weight, probability: probabilityFromWeight(weight), contributions }
+}
+
+/**
+ * Classify a score against upper / lower match-weight thresholds (in bits): at or above `upper` is a link, at or below
+ * `lower` a non-link, and the band between is clerical review (abstain).
+ */
+export function decide(score: PairScore, thresholds: { upper: number; lower: number }): MatchDecision {
+	if (score.weight >= thresholds.upper) return "match"
+
+	if (score.weight <= thresholds.lower) return "non-match"
+
+	return "review"
+}

package/gbt.ts

@@ -0,0 +1,209 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Gradient-boosted shallow regression trees (logistic loss), pure-Node — the learned scorer #603
+ *   names: an offline-trained model (this trainer, or XGBoost/LightGBM exported to the same
+ *   {@link GBT} shape) plus a trivial evaluator, no new runtime dependency. It sits behind the
+ *   matcher's `scorer` hook to replace the Fellegi-Sunter link weight where labels (or a held-out
+ *   truth like an NPI) let a tree learn the over-merge signature the hand-weights miss.
+ *
+ *   This module is feature-agnostic: feature vectors are caller-defined `number[]` (the record
+ *   matcher builds them in `@mailwoman/registry`'s learned-scorer module — one-hot agreement
+ *   levels
+ *
+ *   - Interaction terms + corpus statistics). It only fits ({@link trainGBT}) and scores
+ *       ({@link gbtScore}). The trained {@link GBT} is plain JSON (`{trees, lr, base}`), so a model
+ *       trains offline once and ships as a data file.
+ */
+
+/** A trained tree: an internal split (feature `f` ≤ `thr` → `lo`, else `hi`) or a `leaf` value. */
+export type TreeNode = { leaf: number } | { f: number; thr: number; lo: TreeNode; hi: TreeNode }
+
+const sigmoid = (z: number): number => 1 / (1 + Math.exp(-Math.max(-30, Math.min(30, z))))
+
+/**
+ * Per-feature candidate split thresholds: midpoints for few-valued/binary features, quantiles for continuous.
+ */
+export function buildThresholds(X: number[][]): number[][] {
+	const dim = X[0]?.length ?? 0
+	const out: number[][] = []
+
+	for (let f = 0; f < dim; f++) {
+		const vals = X.map((r) => r[f]!)
+		const uniq = [...new Set(vals)].sort((p, q) => p - q)
+
+		if (uniq.length <= 1) {
+			out.push([])
+		} else if (uniq.length <= 5) {
+			const t: number[] = []
+
+			for (let k = 0; k < uniq.length - 1; k++) {
+				t.push((uniq[k]! + uniq[k + 1]!) / 2)
+			}
+			out.push(t)
+		} else {
+			const sorted = [...vals].sort((p, q) => p - q)
+			const t: number[] = []
+
+			for (let q = 1; q <= 6; q++) {
+				t.push(sorted[Math.floor((q / 7) * (sorted.length - 1))]!)
+			}
+			out.push([...new Set(t)])
+		}
+	}
+
+	return out
+}
+
+/** Weighted SSE of target `g` over `rows` around their weighted mean. */
+function nodeSSE(rows: number[], g: number[], w: number[]): number {
+	let wsum = 0
+	let wg = 0
+
+	for (const i of rows) {
+		wsum += w[i]!
+		wg += w[i]! * g[i]!
+	}
+	const mean = wsum > 0 ? wg / wsum : 0
+	let sse = 0
+
+	for (const i of rows) {
+		const d = g[i]! - mean
+		sse += w[i]! * d * d
+	}
+
+	return sse
+}
+
+/** Greedy depth-limited weighted regression tree on target `g` (the boosting residual). */
+function fitRegTree(
+	rows: number[],
+	X: number[][],
+	g: number[],
+	w: number[],
+	thresholds: number[][],
+	depth: number,
+	minLeaf: number
+): TreeNode {
+	let wsum = 0
+	let wg = 0
+
+	for (const i of rows) {
+		wsum += w[i]!
+		wg += w[i]! * g[i]!
+	}
+	const leaf = wsum > 0 ? wg / wsum : 0
+
+	if (depth === 0 || rows.length < 2 * minLeaf) return { leaf }
+	const parentSSE = nodeSSE(rows, g, w)
+	let bestGain = 1e-12
+	let bestF = -1
+	let bestThr = 0
+	let bestLo: number[] = []
+	let bestHi: number[] = []
+
+	for (let f = 0; f < thresholds.length; f++) {
+		for (const thr of thresholds[f]!) {
+			const lo: number[] = []
+			const hi: number[] = []
+
+			for (const i of rows) {
+				;(X[i]![f]! <= thr ? lo : hi).push(i)
+			}
+
+			if (lo.length < minLeaf || hi.length < minLeaf) continue
+			const gain = parentSSE - (nodeSSE(lo, g, w) + nodeSSE(hi, g, w))
+
+			if (gain > bestGain) {
+				bestGain = gain
+				bestF = f
+				bestThr = thr
+				bestLo = lo
+				bestHi = hi
+			}
+		}
+	}
+
+	if (bestF < 0) return { leaf }
+
+	return {
+		f: bestF,
+		thr: bestThr,
+		lo: fitRegTree(bestLo, X, g, w, thresholds, depth - 1, minLeaf),
+		hi: fitRegTree(bestHi, X, g, w, thresholds, depth - 1, minLeaf),
+	}
+}
+
+function predictTree(t: TreeNode, x: number[]): number {
+	let n = t
+
+	while ("f" in n) {
+		n = x[n.f]! <= n.thr ? n.lo : n.hi
+	}
+
+	return n.leaf
+}
+
+/** A trained gradient-boosted-tree model: an additive ensemble over a base log-odds. Plain JSON. */
+export interface GBT {
+	trees: TreeNode[]
+	lr: number
+	base: number
+}
+
+/** Hyperparameters for {@link trainGBT}. */
+export interface GBTOpts {
+	rounds: number
+	depth: number
+	lr: number
+	minLeaf: number
+}
+
+/** Gradient-boosted regression trees on logistic loss, with per-sample class weights `w`. */
+export function trainGBT(X: number[][], y: number[], w: number[], opts: GBTOpts): GBT {
+	const N = X.length
+	const thresholds = buildThresholds(X)
+	const rowsAll = Array.from({ length: N }, (_, i) => i)
+	let wpos = 0
+	let wtot = 0
+
+	for (let i = 0; i < N; i++) {
+		wtot += w[i]!
+
+		if (y[i] === 1) {
+			wpos += w[i]!
+		}
+	}
+	const base = Math.log((wpos + 1) / (wtot - wpos + 1)) // weighted base log-odds
+	const F = new Array<number>(N).fill(base)
+	const trees: TreeNode[] = []
+
+	for (let m = 0; m < opts.rounds; m++) {
+		const g = new Array<number>(N)
+
+		for (let i = 0; i < N; i++) {
+			g[i] = y[i]! - sigmoid(F[i]!)
+		} // negative gradient of logistic loss
+		const tree = fitRegTree(rowsAll, X, g, w, thresholds, opts.depth, opts.minLeaf)
+
+		for (let i = 0; i < N; i++) {
+			F[i]! += opts.lr * predictTree(tree, X[i]!)
+		}
+		trees.push(tree)
+	}
+
+	return { trees, lr: opts.lr, base }
+}
+
+/** GBT score (logit) for one feature vector. Threshold-comparable like the FS weight. */
+export function gbtScore(m: GBT, x: number[]): number {
+	let f = m.base
+
+	for (const t of m.trees) {
+		f += m.lr * predictTree(t, x)
+	}
+
+	return f
+}

package/index.ts

@@ -0,0 +1,26 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `@mailwoman/match` — the geocode-first record matcher: block → score → cluster.
+ *
+ *   The full three-stage pipeline:
+ *
+ *   1. {@link block Block} — geo-first candidate generation (a spatial-cell union of cheap, high-recall
+ *        keys), so two records at the same place meet regardless of address spelling.
+ *   2. **Score** — string {@link jaroWinkler comparators} → the {@link scorePair Fellegi-Sunter} weight
+ *        model (agreement levels → `log2(m/u)` weights → probability → link / review / non-link),
+ *        with `m`/`u` learned label-free by {@link estimateParameters EM} and rare-value agreement
+ *        up-weighted by {@link withTermFrequency term frequency}.
+ *   3. {@link cluster Cluster} — resolve the non-transitive pairwise link graph into canonical entities.
+ */
+
+export * from "./blocking.ts"
+export * from "./clustering.ts"
+export * from "./comparators.ts"
+export * from "./distance.ts"
+export * from "./em.ts"
+export * from "./fellegi-sunter.ts"
+export * from "./gbt.ts"
+export * from "./tf.ts"