@mailwoman/match 7.1.0 → 7.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/blocking.ts +180 -0
- package/clustering.ts +238 -0
- package/comparators.ts +142 -0
- package/distance.ts +140 -0
- package/em.ts +150 -0
- package/fellegi-sunter.ts +201 -0
- package/gbt.ts +209 -0
- package/index.ts +26 -0
- package/package.json +13 -5
- package/tf.ts +96 -0
package/blocking.ts
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Blocking — candidate generation. Comparing every pair is O(n²) (a million records is a trillion
|
|
7
|
+
* comparisons), so we only score pairs that share a cheap key. This is where the geocode-first
|
|
8
|
+
* bet pays off: two records resolving to the same place land in the same spatial cell regardless
|
|
9
|
+
* of how their address strings are spelled, so geography is the primary block.
|
|
10
|
+
*
|
|
11
|
+
* A {@link BlockingKey} maps a record to zero or more string keys; records sharing any key become
|
|
12
|
+
* candidates. Keys compose as a _union_ (the standard multi-pass approach — high recall from
|
|
13
|
+
* cheap rules): block on the spatial cell OR the canonical key OR the postcode, and a pair that
|
|
14
|
+
* any rule catches is scored. {@link conjunction} builds the AND-style key Geo-ER uses (`name-cell
|
|
15
|
+
* AND geo-cell`) when a single rule is too loose.
|
|
16
|
+
*
|
|
17
|
+
* Recall is the priority — a pair the blocker never proposes can never match, the most dangerous
|
|
18
|
+
* silent failure in record linkage. So the spatial grid is generous and neighbour-expanded by
|
|
19
|
+
* default, and any block too large to scan is _reported_, never silently dropped.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/** Maps a record to zero or more block keys. Two records sharing any key become a candidate pair. */
|
|
23
|
+
export type BlockingKey<R> = (record: R) => string[]
|
|
24
|
+
|
|
25
|
+
/** A geographic coordinate (WGS84 decimal degrees). */
|
|
26
|
+
export interface LatLon {
|
|
27
|
+
latitude: number
|
|
28
|
+
longitude: number
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A spatial-cell block key: a configurable lat/lon grid. `precisionDegrees` sets the cell size (default 0.05° ≈ 5.5 km
|
|
33
|
+
* of latitude — deliberately generous, per the literature, so same-place records reliably co-block). With `neighbors`
|
|
34
|
+
* (default `true`) a record also keys its 8 adjacent cells, so a pair straddling a cell boundary still meets.
|
|
35
|
+
*
|
|
36
|
+
* Note: an equal-_degree_ grid (longitude cells shrink toward the poles) and neighbour expansion inflates block sizes
|
|
37
|
+
* ~9×; an equal-area H3/geohash index with a single-cell + neighbour-query is the refinement. Behaviour — proximity
|
|
38
|
+
* co-blocking — is the same.
|
|
39
|
+
*/
|
|
40
|
+
export function geoCellKey<R>(
|
|
41
|
+
extract: (record: R) => LatLon | null | undefined,
|
|
42
|
+
opts: { precisionDegrees?: number; neighbors?: boolean } = {}
|
|
43
|
+
): BlockingKey<R> {
|
|
44
|
+
const step = opts.precisionDegrees ?? 0.05
|
|
45
|
+
const expand = opts.neighbors ?? true
|
|
46
|
+
|
|
47
|
+
return (record) => {
|
|
48
|
+
const coordinate = extract(record)
|
|
49
|
+
|
|
50
|
+
if (!coordinate || !Number.isFinite(coordinate.latitude) || !Number.isFinite(coordinate.longitude)) return []
|
|
51
|
+
|
|
52
|
+
const latCell = Math.floor(coordinate.latitude / step)
|
|
53
|
+
const lonCell = Math.floor(coordinate.longitude / step)
|
|
54
|
+
|
|
55
|
+
if (!expand) return [`${latCell}:${lonCell}`]
|
|
56
|
+
|
|
57
|
+
const keys: string[] = []
|
|
58
|
+
|
|
59
|
+
for (let dLat = -1; dLat <= 1; dLat++) {
|
|
60
|
+
for (let dLon = -1; dLon <= 1; dLon++) {
|
|
61
|
+
keys.push(`${latCell + dLat}:${lonCell + dLon}`)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return keys
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* An exact-value block key (the canonical address key, a postcode, an email domain…), normalized and optionally
|
|
71
|
+
* truncated to a leading `prefix` of characters (a cheaper, higher-recall rule). A missing or empty value produces no
|
|
72
|
+
* key.
|
|
73
|
+
*/
|
|
74
|
+
export function exactKey<R>(
|
|
75
|
+
extract: (record: R) => string | null | undefined,
|
|
76
|
+
opts: { prefix?: number; normalize?: (value: string) => string } = {}
|
|
77
|
+
): BlockingKey<R> {
|
|
78
|
+
const normalize = opts.normalize ?? ((v: string) => v.trim().toLowerCase().replace(/\s+/g, " "))
|
|
79
|
+
|
|
80
|
+
return (record) => {
|
|
81
|
+
const value = extract(record)
|
|
82
|
+
|
|
83
|
+
if (!value) return []
|
|
84
|
+
const normalized = normalize(value)
|
|
85
|
+
|
|
86
|
+
if (!normalized) return []
|
|
87
|
+
|
|
88
|
+
return [opts.prefix ? normalized.slice(0, opts.prefix) : normalized]
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* A conjunctive block key — the cross-product of its sub-keys, joined (Geo-ER's "name AND distance"). A record is keyed
|
|
94
|
+
* by every combination of one sub-key from each input, so two records co-block only when they agree on _all_ inputs.
|
|
95
|
+
* Tighter blocks, lower recall — use when a single rule is too loose.
|
|
96
|
+
*/
|
|
97
|
+
export function conjunction<R>(...keys: BlockingKey<R>[]): BlockingKey<R> {
|
|
98
|
+
return (record) => {
|
|
99
|
+
let combos = [""]
|
|
100
|
+
|
|
101
|
+
for (const key of keys) {
|
|
102
|
+
const parts = key(record)
|
|
103
|
+
|
|
104
|
+
if (parts.length === 0) return []
|
|
105
|
+
combos = combos.flatMap((prefix) => parts.map((part) => (prefix ? `${prefix}&${part}` : part)))
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return combos
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** The outcome of a blocking pass. */
|
|
113
|
+
export interface BlockResult<R> {
|
|
114
|
+
/** Deduplicated candidate pairs (no self-pairs; a pair caught by multiple keys appears once). */
|
|
115
|
+
pairs: Array<[R, R]>
|
|
116
|
+
/** Blocks that exceeded `maxBlockSize` and were skipped — surfaced so coverage limits are visible. */
|
|
117
|
+
droppedBlocks: Array<{ key: string; size: number }>
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Generate candidate pairs from `records` via one or more blocking keys (their union). Builds an inverted index (key →
|
|
122
|
+
* records) and emits the unique within-block pairs. A block larger than `maxBlockSize` is skipped and reported in
|
|
123
|
+
* `droppedBlocks` rather than blowing up into a quadratic scan — an explicit, visible coverage limit, not a silent
|
|
124
|
+
* drop.
|
|
125
|
+
*/
|
|
126
|
+
export function block<R>(
|
|
127
|
+
records: readonly R[],
|
|
128
|
+
blockingKeys: BlockingKey<R> | BlockingKey<R>[],
|
|
129
|
+
opts: { maxBlockSize?: number } = {}
|
|
130
|
+
): BlockResult<R> {
|
|
131
|
+
const keys = Array.isArray(blockingKeys) ? blockingKeys : [blockingKeys]
|
|
132
|
+
const maxBlockSize = opts.maxBlockSize ?? Infinity
|
|
133
|
+
const index = new Map<string, number[]>()
|
|
134
|
+
|
|
135
|
+
records.forEach((record, i) => {
|
|
136
|
+
const seen = new Set<string>()
|
|
137
|
+
|
|
138
|
+
for (const keyFn of keys) {
|
|
139
|
+
for (const key of keyFn(record)) {
|
|
140
|
+
if (!key || seen.has(key)) continue
|
|
141
|
+
seen.add(key)
|
|
142
|
+
const bucket = index.get(key)
|
|
143
|
+
|
|
144
|
+
if (bucket) {
|
|
145
|
+
bucket.push(i)
|
|
146
|
+
} else {
|
|
147
|
+
index.set(key, [i])
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
const n = records.length
|
|
154
|
+
const emitted = new Set<number>()
|
|
155
|
+
const pairs: Array<[R, R]> = []
|
|
156
|
+
const droppedBlocks: BlockResult<R>["droppedBlocks"] = []
|
|
157
|
+
|
|
158
|
+
for (const [key, bucket] of index) {
|
|
159
|
+
if (bucket.length < 2) continue
|
|
160
|
+
|
|
161
|
+
if (bucket.length > maxBlockSize) {
|
|
162
|
+
droppedBlocks.push({ key, size: bucket.length })
|
|
163
|
+
continue
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
for (let a = 0; a < bucket.length; a++) {
|
|
167
|
+
for (let b = a + 1; b < bucket.length; b++) {
|
|
168
|
+
const lo = Math.min(bucket[a]!, bucket[b]!)
|
|
169
|
+
const hi = Math.max(bucket[a]!, bucket[b]!)
|
|
170
|
+
const id = lo * n + hi
|
|
171
|
+
|
|
172
|
+
if (emitted.has(id)) continue
|
|
173
|
+
emitted.add(id)
|
|
174
|
+
pairs.push([records[lo]!, records[hi]!])
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
return { pairs, droppedBlocks }
|
|
180
|
+
}
|
package/clustering.ts
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Clustering — the third and final matcher stage: resolve scored pairs into canonical entities.
|
|
7
|
+
*
|
|
8
|
+
* The pairwise scorer treats each pair independently, and its scores are NOT transitive: A~B at a
|
|
9
|
+
* high weight and B~C at a high weight does not guarantee A~C is a match. So a distinct stage is
|
|
10
|
+
* required to turn the graph of above-threshold links into coherent groups — skip it and your
|
|
11
|
+
* "entities" silently fracture or fuse.
|
|
12
|
+
*
|
|
13
|
+
* This ships the standard baseline: connected components of the link graph (union-find), with the
|
|
14
|
+
* link threshold as the precision/recall knob — raise it for tighter, purer clusters, lower it
|
|
15
|
+
* for more recall. Its known weakness is over-merging via transitive chains (a string of weak
|
|
16
|
+
* links can pull unrelated records into one component); the principled fix is
|
|
17
|
+
* centroid-/average-linkage hierarchical clustering (Dedupe), which uses the full within-cluster
|
|
18
|
+
* score matrix — a documented refinement, not this first cut. For a geocode-first matcher the
|
|
19
|
+
* over-merge risk is already damped: blocking keeps candidate sets local, so chains can't run
|
|
20
|
+
* across the whole dataset.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** A scored candidate pair: two records and the match weight (bits) the scorer assigned them. */
|
|
24
|
+
export interface ScoredLink<R> {
|
|
25
|
+
a: R
|
|
26
|
+
b: R
|
|
27
|
+
weight: number
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Options for {@link cluster}. */
|
|
31
|
+
export interface ClusterOptions {
|
|
32
|
+
/**
|
|
33
|
+
* Link two records only when their match weight is at or above this (bits) — the precision/recall knob.
|
|
34
|
+
*/
|
|
35
|
+
threshold: number
|
|
36
|
+
/**
|
|
37
|
+
* How the above-threshold link graph resolves into clusters:
|
|
38
|
+
*
|
|
39
|
+
* - `"single"` (default) — connected components (union-find). Fast; ANY above-threshold link fuses two groups, so a
|
|
40
|
+
* single weak link can over-merge unrelated records through a transitive chain.
|
|
41
|
+
* - `"average"` — agglomerative average-linkage refinement WITHIN each connected component: two sub-clusters merge only
|
|
42
|
+
* when the AVERAGE weight of the links between them clears the threshold, so a lone weak bridge no longer fuses two
|
|
43
|
+
* otherwise-dense groups. The documented over-merge fix (Dedupe). Falls back to single-linkage for any component
|
|
44
|
+
* larger than {@link maxAverageLinkageComponent}.
|
|
45
|
+
*/
|
|
46
|
+
linkage?: "single" | "average"
|
|
47
|
+
/**
|
|
48
|
+
* Components larger than this skip the O(k³) average-linkage refine and keep single-linkage. Default 64.
|
|
49
|
+
*/
|
|
50
|
+
maxAverageLinkageComponent?: number
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Refine one connected component by agglomerative average-linkage. Starts with every member a singleton and repeatedly
|
|
55
|
+
* merges the cluster pair with the highest _average_ inter-cluster link weight while that average is at or above
|
|
56
|
+
* `threshold`; clusters with no link between them never merge. O(k³) in the component size, so callers gate it on a
|
|
57
|
+
* size cap.
|
|
58
|
+
*/
|
|
59
|
+
function averageLinkageRefine<R>(members: R[], edges: Array<[number, number, number]>, threshold: number): R[][] {
|
|
60
|
+
const clusters = members.map((_, i) => [i])
|
|
61
|
+
const crossAverage = (a: number[], b: number[]): number | null => {
|
|
62
|
+
const inA = new Set(a)
|
|
63
|
+
const inB = new Set(b)
|
|
64
|
+
let sum = 0
|
|
65
|
+
let count = 0
|
|
66
|
+
|
|
67
|
+
for (const [i, j, w] of edges) {
|
|
68
|
+
if ((inA.has(i) && inB.has(j)) || (inA.has(j) && inB.has(i))) {
|
|
69
|
+
sum += w
|
|
70
|
+
count++
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return count > 0 ? sum / count : null
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
for (;;) {
|
|
78
|
+
let bestAvg = -Infinity
|
|
79
|
+
let bestPair: [number, number] | null = null
|
|
80
|
+
|
|
81
|
+
for (let p = 0; p < clusters.length; p++) {
|
|
82
|
+
for (let q = p + 1; q < clusters.length; q++) {
|
|
83
|
+
const avg = crossAverage(clusters[p]!, clusters[q]!)
|
|
84
|
+
|
|
85
|
+
if (avg !== null && avg > bestAvg) {
|
|
86
|
+
bestAvg = avg
|
|
87
|
+
bestPair = [p, q]
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (!bestPair || bestAvg < threshold) break
|
|
93
|
+
const [p, q] = bestPair
|
|
94
|
+
clusters[p] = clusters[p]!.concat(clusters[q]!)
|
|
95
|
+
clusters.splice(q, 1)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return clusters.map((local) => local.map((i) => members[i]!))
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Cluster records into canonical entities by connected components of the above-threshold link graph. Every input record
|
|
103
|
+
* lands in exactly one cluster — a record with no qualifying link is a singleton. Links referencing a record not in
|
|
104
|
+
* `records` are ignored. Reference identity is used, so pass the same record objects to both arguments.
|
|
105
|
+
*/
|
|
106
|
+
export function cluster<R>(records: readonly R[], links: Iterable<ScoredLink<R>>, opts: ClusterOptions): R[][] {
|
|
107
|
+
const index = new Map<R, number>()
|
|
108
|
+
records.forEach((record, i) => index.set(record, i))
|
|
109
|
+
|
|
110
|
+
const parent = records.map((_, i) => i)
|
|
111
|
+
const rank = new Array<number>(records.length).fill(0)
|
|
112
|
+
|
|
113
|
+
const find = (x: number): number => {
|
|
114
|
+
let root = x
|
|
115
|
+
|
|
116
|
+
while (parent[root] !== root) {
|
|
117
|
+
root = parent[root]!
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Path compression.
|
|
121
|
+
while (parent[x] !== root) {
|
|
122
|
+
const next = parent[x]!
|
|
123
|
+
parent[x] = root
|
|
124
|
+
x = next
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return root
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const union = (x: number, y: number): void => {
|
|
131
|
+
const rx = find(x)
|
|
132
|
+
const ry = find(y)
|
|
133
|
+
|
|
134
|
+
if (rx === ry) return
|
|
135
|
+
|
|
136
|
+
if (rank[rx]! < rank[ry]!) {
|
|
137
|
+
parent[rx] = ry
|
|
138
|
+
} else if (rank[rx]! > rank[ry]!) {
|
|
139
|
+
parent[ry] = rx
|
|
140
|
+
} else {
|
|
141
|
+
parent[ry] = rx
|
|
142
|
+
rank[rx]!++
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Collect ALL valid links (not just above-threshold): connected components form from the
|
|
147
|
+
// above-threshold ones, but the average-linkage refinement needs the full sub-graph — a weak or
|
|
148
|
+
// disagreeing below-threshold edge between two sub-clusters is exactly what should pull them apart.
|
|
149
|
+
const allLinks: ScoredLink<R>[] = []
|
|
150
|
+
|
|
151
|
+
for (const link of links) {
|
|
152
|
+
const ia = index.get(link.a)
|
|
153
|
+
const ib = index.get(link.b)
|
|
154
|
+
|
|
155
|
+
if (ia === undefined || ib === undefined) continue
|
|
156
|
+
allLinks.push(link)
|
|
157
|
+
|
|
158
|
+
if (link.weight >= opts.threshold) {
|
|
159
|
+
union(ia, ib)
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const groups = new Map<number, R[]>()
|
|
164
|
+
records.forEach((record, i) => {
|
|
165
|
+
const root = find(i)
|
|
166
|
+
const group = groups.get(root)
|
|
167
|
+
|
|
168
|
+
if (group) {
|
|
169
|
+
group.push(record)
|
|
170
|
+
} else {
|
|
171
|
+
groups.set(root, [record])
|
|
172
|
+
}
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
if (opts.linkage !== "average") return [...groups.values()]
|
|
176
|
+
|
|
177
|
+
// Average-linkage refinement: split each component where its sub-clusters are joined only by a weak
|
|
178
|
+
// bridge (the average inter-cluster link weight, over ALL edges between them, falls below the threshold).
|
|
179
|
+
const maxComponent = opts.maxAverageLinkageComponent ?? 64
|
|
180
|
+
const localOf = new Map<R, number>()
|
|
181
|
+
|
|
182
|
+
// member → its index within its own group
|
|
183
|
+
for (const members of groups.values()) {
|
|
184
|
+
members.forEach((m, i) => localOf.set(m, i))
|
|
185
|
+
}
|
|
186
|
+
const groupEdges = new Map<number, Array<[number, number, number]>>()
|
|
187
|
+
|
|
188
|
+
for (const link of allLinks) {
|
|
189
|
+
const root = find(index.get(link.a)!)
|
|
190
|
+
|
|
191
|
+
if (root !== find(index.get(link.b)!)) continue // cross-component edge — not part of any refinement
|
|
192
|
+
const list = groupEdges.get(root) ?? []
|
|
193
|
+
list.push([localOf.get(link.a)!, localOf.get(link.b)!, link.weight])
|
|
194
|
+
groupEdges.set(root, list)
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const result: R[][] = []
|
|
198
|
+
|
|
199
|
+
for (const [root, members] of groups) {
|
|
200
|
+
if (members.length <= 1 || members.length > maxComponent) {
|
|
201
|
+
result.push(members)
|
|
202
|
+
continue
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
for (const sub of averageLinkageRefine(members, groupEdges.get(root) ?? [], opts.threshold)) {
|
|
206
|
+
result.push(sub)
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return result
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Pick a cluster's most complete record as its canonical representative — the one with the fewest empty fields (`null`
|
|
215
|
+
* / `undefined` / `""`). Ties keep the earliest. A basic, generic canonicalizer; field-level merging across the cluster
|
|
216
|
+
* is the application's job (it knows which source to trust).
|
|
217
|
+
*/
|
|
218
|
+
export function representative<R extends object>(group: readonly R[]): R | undefined {
|
|
219
|
+
let best: R | undefined
|
|
220
|
+
let bestFilled = -1
|
|
221
|
+
|
|
222
|
+
for (const record of group) {
|
|
223
|
+
let filled = 0
|
|
224
|
+
|
|
225
|
+
for (const value of Object.values(record)) {
|
|
226
|
+
if (value !== null && value !== undefined && value !== "") {
|
|
227
|
+
filled++
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
if (filled > bestFilled) {
|
|
232
|
+
bestFilled = filled
|
|
233
|
+
best = record
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
return best
|
|
238
|
+
}
|
package/comparators.ts
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* String comparators for the matcher's scoring stage.
|
|
7
|
+
*
|
|
8
|
+
* The record-linkage literature (Winkler/Census; Belin 1993) settles on the prefix-weighted Jaro
|
|
9
|
+
* comparator (Jaro-Winkler) as the default for names: it tolerates the typographical error real
|
|
10
|
+
* data is full of better than raw character-edit distance. But J-W has a documented blind spot on
|
|
11
|
+
* compound / double surnames (e.g. Hispanic `Garcia Lopez`): the second half of the compound
|
|
12
|
+
* falls outside J-W's match window, so `Lopez` vs `Garcia Lopez` scores ~0. The fix the
|
|
13
|
+
* literature prescribes is an edit-distance / token fallback for single-vs-compound pairs —
|
|
14
|
+
* implemented in {@link nameSimilarity}.
|
|
15
|
+
*
|
|
16
|
+
* These are pure similarity primitives in [0, 1]. The mapping of a similarity onto discrete
|
|
17
|
+
* Fellegi-Sunter agreement levels (and the m/u weights) is the scorer's job, not theirs.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { distance as levenshteinDistance } from "fastest-levenshtein"
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Jaro similarity in [0, 1]. Two empty strings are identical (1); one empty is 0. Counts matching characters within a
|
|
24
|
+
* sliding window of `floor(max(len)/2) - 1`, discounting half-transpositions.
|
|
25
|
+
*/
|
|
26
|
+
export function jaro(a: string, b: string): number {
|
|
27
|
+
if (a === b) return 1
|
|
28
|
+
const la = a.length
|
|
29
|
+
const lb = b.length
|
|
30
|
+
|
|
31
|
+
if (la === 0 || lb === 0) return 0
|
|
32
|
+
|
|
33
|
+
const window = Math.max(0, Math.floor(Math.max(la, lb) / 2) - 1)
|
|
34
|
+
const aMatched = new Array<boolean>(la).fill(false)
|
|
35
|
+
const bMatched = new Array<boolean>(lb).fill(false)
|
|
36
|
+
|
|
37
|
+
let matches = 0
|
|
38
|
+
|
|
39
|
+
for (let i = 0; i < la; i++) {
|
|
40
|
+
const start = Math.max(0, i - window)
|
|
41
|
+
const end = Math.min(i + window + 1, lb)
|
|
42
|
+
|
|
43
|
+
for (let j = start; j < end; j++) {
|
|
44
|
+
if (bMatched[j] || a[i] !== b[j]) continue
|
|
45
|
+
aMatched[i] = true
|
|
46
|
+
bMatched[j] = true
|
|
47
|
+
matches++
|
|
48
|
+
break
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
if (matches === 0) return 0
|
|
53
|
+
|
|
54
|
+
// Count transpositions: matched chars of `a` and `b`, in order, that disagree (halved).
|
|
55
|
+
let transpositions = 0
|
|
56
|
+
let k = 0
|
|
57
|
+
|
|
58
|
+
for (let i = 0; i < la; i++) {
|
|
59
|
+
if (!aMatched[i]) continue
|
|
60
|
+
|
|
61
|
+
while (!bMatched[k]) {
|
|
62
|
+
k++
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (a[i] !== b[k]) {
|
|
66
|
+
transpositions++
|
|
67
|
+
}
|
|
68
|
+
k++
|
|
69
|
+
}
|
|
70
|
+
transpositions /= 2
|
|
71
|
+
|
|
72
|
+
return (matches / la + matches / lb + (matches - transpositions) / matches) / 3
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Jaro-Winkler similarity in [0, 1]: Jaro with a bonus for a shared prefix — `jw = jaro + prefix * weight * (1 -
|
|
77
|
+
* jaro)`, prefix capped at `maxPrefix` (Winkler's standard 4), `weight` the scaling factor (standard 0.1). Only boosts
|
|
78
|
+
* when `jaro` already clears `boostThreshold` (0.7), per Winkler.
|
|
79
|
+
*/
|
|
80
|
+
export function jaroWinkler(
|
|
81
|
+
a: string,
|
|
82
|
+
b: string,
|
|
83
|
+
opts: { weight?: number; maxPrefix?: number; boostThreshold?: number } = {}
|
|
84
|
+
): number {
|
|
85
|
+
const weight = opts.weight ?? 0.1
|
|
86
|
+
const maxPrefix = opts.maxPrefix ?? 4
|
|
87
|
+
const boostThreshold = opts.boostThreshold ?? 0.7
|
|
88
|
+
|
|
89
|
+
const base = jaro(a, b)
|
|
90
|
+
|
|
91
|
+
if (base < boostThreshold) return base
|
|
92
|
+
|
|
93
|
+
let prefix = 0
|
|
94
|
+
const limit = Math.min(maxPrefix, a.length, b.length)
|
|
95
|
+
|
|
96
|
+
while (prefix < limit && a[prefix] === b[prefix]) {
|
|
97
|
+
prefix++
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return base + prefix * weight * (1 - base)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Normalized Levenshtein similarity in [0, 1]: `1 - editDistance / max(len)`. */
|
|
104
|
+
export function levenshteinSimilarity(a: string, b: string): number {
|
|
105
|
+
if (a === b) return 1
|
|
106
|
+
const longest = Math.max(a.length, b.length)
|
|
107
|
+
|
|
108
|
+
if (longest === 0) return 1
|
|
109
|
+
|
|
110
|
+
return 1 - levenshteinDistance(a, b) / longest
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Name-aware similarity in [0, 1]. Jaro-Winkler by default, with the compound-surname fallback the literature
|
|
115
|
+
* prescribes:
|
|
116
|
+
*
|
|
117
|
+
* - If one name's tokens are a strict subset of the other's (`Lopez` ⊂ `Garcia Lopez`), that is strong partial agreement
|
|
118
|
+
* J-W misses — floor the score at 0.9.
|
|
119
|
+
* - Otherwise return the better of Jaro-Winkler and normalized edit similarity, so a single token that is a substring of
|
|
120
|
+
* a longer compound (`Garcia` vs `Garcialopez`) still scores sensibly.
|
|
121
|
+
*
|
|
122
|
+
* Case- and whitespace-insensitive. Empty input scores 0.
|
|
123
|
+
*/
|
|
124
|
+
export function nameSimilarity(a: string, b: string): number {
|
|
125
|
+
const x = a.trim().toLowerCase().replace(/\s+/g, " ")
|
|
126
|
+
const y = b.trim().toLowerCase().replace(/\s+/g, " ")
|
|
127
|
+
|
|
128
|
+
if (!x || !y) return 0
|
|
129
|
+
|
|
130
|
+
if (x === y) return 1
|
|
131
|
+
|
|
132
|
+
const jw = jaroWinkler(x, y)
|
|
133
|
+
|
|
134
|
+
const xTokens = new Set(x.split(" "))
|
|
135
|
+
const yTokens = new Set(y.split(" "))
|
|
136
|
+
const [small, big] = xTokens.size <= yTokens.size ? [xTokens, yTokens] : [yTokens, xTokens]
|
|
137
|
+
const subset = small.size < big.size && [...small].every((t) => big.has(t))
|
|
138
|
+
|
|
139
|
+
if (subset) return Math.max(jw, 0.9)
|
|
140
|
+
|
|
141
|
+
return Math.max(jw, levenshteinSimilarity(x, y))
|
|
142
|
+
}
|
package/distance.ts
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Geographic distance as a scoring feature — the other half of geocode-first matching.
|
|
7
|
+
*
|
|
8
|
+
* Blocking uses geography to _propose_ candidates; this scores them on it. The research is explicit
|
|
9
|
+
* that an address must be matched as a SPATIAL attribute, not by string similarity (a
|
|
10
|
+
* one-character edit can be 650 m apart), and that distance measurably helps as a comparison
|
|
11
|
+
* feature. So we bucket the great-circle distance between two records' coordinates into ordered
|
|
12
|
+
* Fellegi-Sunter agreement levels (Splink's `DistanceInKMAtThresholds`): "same building" / "same
|
|
13
|
+
* block" / "same area" / far, each with its own m/u and weight.
|
|
14
|
+
*
|
|
15
|
+
* Calibrate the bucket boundaries to the geocoder's OWN error, which is heavy-tailed and density-
|
|
16
|
+
* dependent (≈38 m urban, ≈200 m rural). A weakening of this evidence by geocode quality (a
|
|
17
|
+
* shared interpolated centroid is softer than a shared rooftop point) is the documented
|
|
18
|
+
* refinement.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { haversineKm as greatCircleKm } from "@mailwoman/spatial"
|
|
22
|
+
|
|
23
|
+
import type { LatLon } from "./blocking.ts"
|
|
24
|
+
import type { Comparison, ComparisonLevel } from "./fellegi-sunter.ts"
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Great-circle (haversine) distance in km between two coordinates. The formula's one true home is `@mailwoman/spatial`;
|
|
28
|
+
* this is a thin domain-typed adapter from `match`'s `LatLon` ({ latitude, longitude }) onto the canonical scalar
|
|
29
|
+
* helper — not a second implementation.
|
|
30
|
+
*/
|
|
31
|
+
export const haversineKm = (a: LatLon, b: LatLon): number =>
|
|
32
|
+
greatCircleKm(a.latitude, a.longitude, b.latitude, b.longitude)
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* A geo-distance comparison: bucket the great-circle distance between two records' coordinates into ordered agreement
|
|
36
|
+
* levels. Levels must be ordered NEAREST first by `maxKm`, the last acting as the `far` catch-all (`maxKm` omitted →
|
|
37
|
+
* unbounded). A missing/invalid coordinate on either side yields no evidence.
|
|
38
|
+
*/
|
|
39
|
+
export function distanceComparison<R>(config: {
|
|
40
|
+
name: string
|
|
41
|
+
extract: (record: R) => LatLon | null | undefined
|
|
42
|
+
levels: ComparisonLevel[]
|
|
43
|
+
}): Comparison<R> {
|
|
44
|
+
const valid = (c: LatLon | null | undefined): c is LatLon =>
|
|
45
|
+
!!c && Number.isFinite(c.latitude) && Number.isFinite(c.longitude)
|
|
46
|
+
|
|
47
|
+
return {
|
|
48
|
+
name: config.name,
|
|
49
|
+
levels: config.levels,
|
|
50
|
+
assess(a, b) {
|
|
51
|
+
const ca = config.extract(a)
|
|
52
|
+
const cb = config.extract(b)
|
|
53
|
+
|
|
54
|
+
if (!valid(ca) || !valid(cb)) return -1
|
|
55
|
+
|
|
56
|
+
const km = haversineKm(ca, cb)
|
|
57
|
+
|
|
58
|
+
for (let i = 0; i < config.levels.length; i++) {
|
|
59
|
+
if (km <= (config.levels[i]!.maxKm ?? Infinity)) return i
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return config.levels.length - 1
|
|
63
|
+
},
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Default distance levels, nearest → far, with boundaries at rooftop / block / locality scale. The m/u are illustrative
|
|
69
|
+
* seeds (EM re-estimates them); the boundaries reflect typical geocoder error.
|
|
70
|
+
*/
|
|
71
|
+
export const DEFAULT_DISTANCE_LEVELS: ComparisonLevel[] = [
|
|
72
|
+
{ label: "same-building", maxKm: 0.05, m: 0.7, u: 0.001 },
|
|
73
|
+
{ label: "same-block", maxKm: 0.5, m: 0.2, u: 0.02 },
|
|
74
|
+
{ label: "same-area", maxKm: 5, m: 0.08, u: 0.2 },
|
|
75
|
+
{ label: "far", m: 0.02, u: 0.779 },
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The collapsed spatial-agreement comparison — ONE non-redundant geographic signal.
|
|
80
|
+
*
|
|
81
|
+
* The first matcher carried TWO spatial comparisons: canonical-address-key similarity AND great-circle distance. They
|
|
82
|
+
* double-count — an exact key match implies distance ≈ 0, so a co-located pair banked the same evidence twice, and the
|
|
83
|
+
* redundant vote is exactly what over-merges distinct providers at a shared clinic address. This folds them into one
|
|
84
|
+
* comparison:
|
|
85
|
+
*
|
|
86
|
+
* - **level 0 `same-key`** — an EXACT canonical-key match: the strongest tier, and the one the inverse-address-frequency
|
|
87
|
+
* adjustment rides ({@link withTermFrequency} on level 0), so agreement on a crowded shared key is down-weighted
|
|
88
|
+
* toward worthless while a rare one keeps full weight.
|
|
89
|
+
* - **levels 1…n** — great-circle distance buckets for pairs whose keys DIFFER, so "123 Main St" vs "123 Main Street Apt
|
|
90
|
+
* 2" that geocode to the same rooftop still earns near-agreement (the geo-first point of the whole design).
|
|
91
|
+
* - Keys differ and no usable coordinate → no evidence.
|
|
92
|
+
*
|
|
93
|
+
* Exactly one spatial vote, no redundancy. Pass {@link DEFAULT_SPATIAL_LEVELS} or your own; index 0 must be the
|
|
94
|
+
* exact-key tier, indices 1…n the distance buckets nearest → far by `maxKm` (last = `far`).
|
|
95
|
+
*/
|
|
96
|
+
export function spatialComparison<R>(config: {
|
|
97
|
+
name: string
|
|
98
|
+
key: (record: R) => string | null | undefined
|
|
99
|
+
coordinate: (record: R) => LatLon | null | undefined
|
|
100
|
+
levels: ComparisonLevel[]
|
|
101
|
+
}): Comparison<R> {
|
|
102
|
+
const valid = (c: LatLon | null | undefined): c is LatLon =>
|
|
103
|
+
!!c && Number.isFinite(c.latitude) && Number.isFinite(c.longitude)
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
name: config.name,
|
|
107
|
+
levels: config.levels,
|
|
108
|
+
assess(a, b) {
|
|
109
|
+
const ka = config.key(a)
|
|
110
|
+
const kb = config.key(b)
|
|
111
|
+
|
|
112
|
+
if (ka && kb && ka.trim() && ka === kb) return 0 // exact canonical-key match — one strong vote
|
|
113
|
+
|
|
114
|
+
const ca = config.coordinate(a)
|
|
115
|
+
const cb = config.coordinate(b)
|
|
116
|
+
|
|
117
|
+
if (!valid(ca) || !valid(cb)) return -1 // keys differ and no coordinate → no spatial evidence
|
|
118
|
+
|
|
119
|
+
const km = haversineKm(ca, cb)
|
|
120
|
+
|
|
121
|
+
for (let i = 1; i < config.levels.length; i++) {
|
|
122
|
+
if (km <= (config.levels[i]!.maxKm ?? Infinity)) return i
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return config.levels.length - 1
|
|
126
|
+
},
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Default levels for {@link spatialComparison}: an exact same-key tier on top of the distance buckets. `m`/`u` are
|
|
132
|
+
* EM-estimable seeds (m decreasing, u increasing down the tiers; each column ≈ sums to 1).
|
|
133
|
+
*/
|
|
134
|
+
export const DEFAULT_SPATIAL_LEVELS: ComparisonLevel[] = [
|
|
135
|
+
{ label: "same-key", m: 0.85, u: 0.01 },
|
|
136
|
+
{ label: "same-building", maxKm: 0.05, m: 0.1, u: 0.02 },
|
|
137
|
+
{ label: "same-block", maxKm: 0.5, m: 0.03, u: 0.05 },
|
|
138
|
+
{ label: "same-area", maxKm: 5, m: 0.015, u: 0.2 },
|
|
139
|
+
{ label: "far", m: 0.005, u: 0.72 },
|
|
140
|
+
]
|