@mailwoman/match 8.1.0 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/blocking.ts +18 -7
- package/clustering.ts +12 -2
- package/comparators.ts +9 -3
- package/em.ts +30 -9
- package/fellegi-sunter.ts +83 -27
- package/gbt.ts +44 -12
- package/out/blocking.d.ts +15 -5
- package/out/blocking.d.ts.map +1 -1
- package/out/blocking.js +2 -2
- package/out/blocking.js.map +1 -1
- package/out/clustering.d.ts +6 -2
- package/out/clustering.d.ts.map +1 -1
- package/out/clustering.js.map +1 -1
- package/out/comparators.d.ts +3 -1
- package/out/comparators.d.ts.map +1 -1
- package/out/comparators.js +5 -3
- package/out/comparators.js.map +1 -1
- package/out/em.d.ts +24 -8
- package/out/em.d.ts.map +1 -1
- package/out/em.js +4 -2
- package/out/em.js.map +1 -1
- package/out/fellegi-sunter.d.ts +81 -27
- package/out/fellegi-sunter.d.ts.map +1 -1
- package/out/fellegi-sunter.js +12 -4
- package/out/fellegi-sunter.js.map +1 -1
- package/out/gbt.d.ts +15 -5
- package/out/gbt.d.ts.map +1 -1
- package/out/gbt.js +26 -9
- package/out/gbt.js.map +1 -1
- package/out/tf.d.ts +21 -7
- package/out/tf.d.ts.map +1 -1
- package/out/tf.js +1 -1
- package/out/tf.js.map +1 -1
- package/package.json +2 -2
- package/tf.ts +23 -8
package/blocking.ts
CHANGED
|
@@ -19,10 +19,14 @@
|
|
|
19
19
|
* default, and any block too large to scan is _reported_, never silently dropped.
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
|
-
/**
|
|
22
|
+
/**
|
|
23
|
+
* Maps a record to zero or more block keys. Two records sharing any key become a candidate pair.
|
|
24
|
+
*/
|
|
23
25
|
export type BlockingKey<R> = (record: R) => string[]
|
|
24
26
|
|
|
25
|
-
/**
|
|
27
|
+
/**
|
|
28
|
+
* A geographic coordinate (WGS84 decimal degrees).
|
|
29
|
+
*/
|
|
26
30
|
export interface LatLon {
|
|
27
31
|
latitude: number
|
|
28
32
|
longitude: number
|
|
@@ -75,7 +79,7 @@ export function exactKey<R>(
|
|
|
75
79
|
extract: (record: R) => string | null | undefined,
|
|
76
80
|
opts: { prefix?: number; normalize?: (value: string) => string } = {}
|
|
77
81
|
): BlockingKey<R> {
|
|
78
|
-
const normalize = opts.normalize ?? ((v: string) => v.trim().toLowerCase().
|
|
82
|
+
const normalize = opts.normalize ?? ((v: string) => v.trim().toLowerCase().replaceAll(/\s+/g, " "))
|
|
79
83
|
|
|
80
84
|
return (record) => {
|
|
81
85
|
const value = extract(record)
|
|
@@ -101,7 +105,7 @@ export function conjunction<R>(...keys: BlockingKey<R>[]): BlockingKey<R> {
|
|
|
101
105
|
for (const key of keys) {
|
|
102
106
|
const parts = key(record)
|
|
103
107
|
|
|
104
|
-
if (parts.length
|
|
108
|
+
if (!parts.length) return []
|
|
105
109
|
combos = combos.flatMap((prefix) => parts.map((part) => (prefix ? `${prefix}&${part}` : part)))
|
|
106
110
|
}
|
|
107
111
|
|
|
@@ -109,11 +113,17 @@ export function conjunction<R>(...keys: BlockingKey<R>[]): BlockingKey<R> {
|
|
|
109
113
|
}
|
|
110
114
|
}
|
|
111
115
|
|
|
112
|
-
/**
|
|
116
|
+
/**
|
|
117
|
+
* The outcome of a blocking pass.
|
|
118
|
+
*/
|
|
113
119
|
export interface BlockResult<R> {
|
|
114
|
-
/**
|
|
120
|
+
/**
|
|
121
|
+
* Deduplicated candidate pairs (no self-pairs; a pair caught by multiple keys appears once).
|
|
122
|
+
*/
|
|
115
123
|
pairs: Array<[R, R]>
|
|
116
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* Blocks that exceeded `maxBlockSize` and were skipped — surfaced so coverage limits are visible.
|
|
126
|
+
*/
|
|
117
127
|
droppedBlocks: Array<{ key: string; size: number }>
|
|
118
128
|
}
|
|
119
129
|
|
|
@@ -160,6 +170,7 @@ export function block<R>(
|
|
|
160
170
|
|
|
161
171
|
if (bucket.length > maxBlockSize) {
|
|
162
172
|
droppedBlocks.push({ key, size: bucket.length })
|
|
173
|
+
|
|
163
174
|
continue
|
|
164
175
|
}
|
|
165
176
|
|
package/clustering.ts
CHANGED
|
@@ -20,14 +20,18 @@
|
|
|
20
20
|
* across the whole dataset.
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
|
-
/**
|
|
23
|
+
/**
|
|
24
|
+
* A scored candidate pair: two records and the match weight (bits) the scorer assigned them.
|
|
25
|
+
*/
|
|
24
26
|
export interface ScoredLink<R> {
|
|
25
27
|
a: R
|
|
26
28
|
b: R
|
|
27
29
|
weight: number
|
|
28
30
|
}
|
|
29
31
|
|
|
30
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Options for {@link cluster}.
|
|
34
|
+
*/
|
|
31
35
|
export interface ClusterOptions {
|
|
32
36
|
/**
|
|
33
37
|
* Link two records only when their match weight is at or above this (bits) — the precision/recall knob.
|
|
@@ -58,6 +62,7 @@ export interface ClusterOptions {
|
|
|
58
62
|
*/
|
|
59
63
|
function averageLinkageRefine<R>(members: R[], edges: Array<[number, number, number]>, threshold: number): R[][] {
|
|
60
64
|
const clusters = members.map((_, i) => [i])
|
|
65
|
+
|
|
61
66
|
const crossAverage = (a: number[], b: number[]): number | null => {
|
|
62
67
|
const inA = new Set(a)
|
|
63
68
|
const inB = new Set(b)
|
|
@@ -67,6 +72,7 @@ function averageLinkageRefine<R>(members: R[], edges: Array<[number, number, num
|
|
|
67
72
|
for (const [i, j, w] of edges) {
|
|
68
73
|
if ((inA.has(i) && inB.has(j)) || (inA.has(j) && inB.has(i))) {
|
|
69
74
|
sum += w
|
|
75
|
+
|
|
70
76
|
count++
|
|
71
77
|
}
|
|
72
78
|
}
|
|
@@ -139,6 +145,7 @@ export function cluster<R>(records: readonly R[], links: Iterable<ScoredLink<R>>
|
|
|
139
145
|
parent[ry] = rx
|
|
140
146
|
} else {
|
|
141
147
|
parent[ry] = rx
|
|
148
|
+
|
|
142
149
|
rank[rx]!++
|
|
143
150
|
}
|
|
144
151
|
}
|
|
@@ -161,6 +168,7 @@ export function cluster<R>(records: readonly R[], links: Iterable<ScoredLink<R>>
|
|
|
161
168
|
}
|
|
162
169
|
|
|
163
170
|
const groups = new Map<number, R[]>()
|
|
171
|
+
|
|
164
172
|
records.forEach((record, i) => {
|
|
165
173
|
const root = find(i)
|
|
166
174
|
const group = groups.get(root)
|
|
@@ -183,6 +191,7 @@ export function cluster<R>(records: readonly R[], links: Iterable<ScoredLink<R>>
|
|
|
183
191
|
for (const members of groups.values()) {
|
|
184
192
|
members.forEach((m, i) => localOf.set(m, i))
|
|
185
193
|
}
|
|
194
|
+
|
|
186
195
|
const groupEdges = new Map<number, Array<[number, number, number]>>()
|
|
187
196
|
|
|
188
197
|
for (const link of allLinks) {
|
|
@@ -199,6 +208,7 @@ export function cluster<R>(records: readonly R[], links: Iterable<ScoredLink<R>>
|
|
|
199
208
|
for (const [root, members] of groups) {
|
|
200
209
|
if (members.length <= 1 || members.length > maxComponent) {
|
|
201
210
|
result.push(members)
|
|
211
|
+
|
|
202
212
|
continue
|
|
203
213
|
}
|
|
204
214
|
|
package/comparators.ts
CHANGED
|
@@ -44,7 +44,9 @@ export function jaro(a: string, b: string): number {
|
|
|
44
44
|
if (bMatched[j] || a[i] !== b[j]) continue
|
|
45
45
|
aMatched[i] = true
|
|
46
46
|
bMatched[j] = true
|
|
47
|
+
|
|
47
48
|
matches++
|
|
49
|
+
|
|
48
50
|
break
|
|
49
51
|
}
|
|
50
52
|
}
|
|
@@ -65,8 +67,10 @@ export function jaro(a: string, b: string): number {
|
|
|
65
67
|
if (a[i] !== b[k]) {
|
|
66
68
|
transpositions++
|
|
67
69
|
}
|
|
70
|
+
|
|
68
71
|
k++
|
|
69
72
|
}
|
|
73
|
+
|
|
70
74
|
transpositions /= 2
|
|
71
75
|
|
|
72
76
|
return (matches / la + matches / lb + (matches - transpositions) / matches) / 3
|
|
@@ -100,7 +104,9 @@ export function jaroWinkler(
|
|
|
100
104
|
return base + prefix * weight * (1 - base)
|
|
101
105
|
}
|
|
102
106
|
|
|
103
|
-
/**
|
|
107
|
+
/**
|
|
108
|
+
* Normalized Levenshtein similarity in [0, 1]: `1 - editDistance / max(len)`.
|
|
109
|
+
*/
|
|
104
110
|
export function levenshteinSimilarity(a: string, b: string): number {
|
|
105
111
|
if (a === b) return 1
|
|
106
112
|
const longest = Math.max(a.length, b.length)
|
|
@@ -122,8 +128,8 @@ export function levenshteinSimilarity(a: string, b: string): number {
|
|
|
122
128
|
* Case- and whitespace-insensitive. Empty input scores 0.
|
|
123
129
|
*/
|
|
124
130
|
export function nameSimilarity(a: string, b: string): number {
|
|
125
|
-
const x = a.trim().toLowerCase().
|
|
126
|
-
const y = b.trim().toLowerCase().
|
|
131
|
+
const x = a.trim().toLowerCase().replaceAll(/\s+/g, " ")
|
|
132
|
+
const y = b.trim().toLowerCase().replaceAll(/\s+/g, " ")
|
|
127
133
|
|
|
128
134
|
if (!x || !y) return 0
|
|
129
135
|
|
package/em.ts
CHANGED
|
@@ -29,26 +29,42 @@ import type { Comparison, FellegiSunterModel } from "./fellegi-sunter.ts"
|
|
|
29
29
|
*/
|
|
30
30
|
const EPSILON = 1e-9
|
|
31
31
|
|
|
32
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Reduce a record pair to its agreement pattern — the per-comparison level index (`-1` = missing).
|
|
34
|
+
*/
|
|
33
35
|
export function agreementPattern<R>(comparisons: Comparison<R>[], a: R, b: R): number[] {
|
|
34
36
|
return comparisons.map((comparison) => comparison.assess(a, b))
|
|
35
37
|
}
|
|
36
38
|
|
|
37
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Options for {@link estimateParameters}.
|
|
41
|
+
*/
|
|
38
42
|
export interface EmOptions {
|
|
39
|
-
/**
|
|
43
|
+
/**
|
|
44
|
+
* Hard iteration cap. Default 100.
|
|
45
|
+
*/
|
|
40
46
|
maxIterations?: number
|
|
41
|
-
/**
|
|
47
|
+
/**
|
|
48
|
+
* Convergence tolerance on the largest parameter change between iterations. Default 1e-6.
|
|
49
|
+
*/
|
|
42
50
|
tolerance?: number
|
|
43
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* Starting prior match rate. Defaults to the model's `lambda`.
|
|
53
|
+
*/
|
|
44
54
|
initialLambda?: number
|
|
45
55
|
}
|
|
46
56
|
|
|
47
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* The fitted model plus convergence diagnostics.
|
|
59
|
+
*/
|
|
48
60
|
export interface EmResult<R> {
|
|
49
|
-
/**
|
|
61
|
+
/**
|
|
62
|
+
* The input model with every level's `m`/`u` and the prior `lambda` re-estimated.
|
|
63
|
+
*/
|
|
50
64
|
model: FellegiSunterModel<R>
|
|
51
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* The estimated prior match rate.
|
|
67
|
+
*/
|
|
52
68
|
lambda: number
|
|
53
69
|
iterations: number
|
|
54
70
|
converged: boolean
|
|
@@ -77,7 +93,7 @@ export function estimateParameters<R>(
|
|
|
77
93
|
let iterations = 0
|
|
78
94
|
let converged = false
|
|
79
95
|
|
|
80
|
-
if (patterns.length
|
|
96
|
+
if (!patterns.length) {
|
|
81
97
|
return { model, lambda, iterations, converged }
|
|
82
98
|
}
|
|
83
99
|
|
|
@@ -100,6 +116,7 @@ export function estimateParameters<R>(
|
|
|
100
116
|
matchLikelihood *= m[i]![level]!
|
|
101
117
|
nonMatchLikelihood *= u[i]![level]!
|
|
102
118
|
}
|
|
119
|
+
|
|
103
120
|
const total = matchLikelihood + nonMatchLikelihood
|
|
104
121
|
const g = total > 0 ? matchLikelihood / total : 0
|
|
105
122
|
responsibilitySum += g
|
|
@@ -126,8 +143,10 @@ export function estimateParameters<R>(
|
|
|
126
143
|
for (let l = 0; l < levels; l++) {
|
|
127
144
|
const newM =
|
|
128
145
|
mDenominator[i]! > 0 ? (mNumerator[i]![l]! + EPSILON) / (mDenominator[i]! + EPSILON * levels) : m[i]![l]!
|
|
146
|
+
|
|
129
147
|
const newU =
|
|
130
148
|
uDenominator[i]! > 0 ? (uNumerator[i]![l]! + EPSILON) / (uDenominator[i]! + EPSILON * levels) : u[i]![l]!
|
|
149
|
+
|
|
131
150
|
maxDelta = Math.max(maxDelta, Math.abs(newM - m[i]![l]!), Math.abs(newU - u[i]![l]!))
|
|
132
151
|
m[i]![l] = newM
|
|
133
152
|
u[i]![l] = newU
|
|
@@ -136,7 +155,9 @@ export function estimateParameters<R>(
|
|
|
136
155
|
|
|
137
156
|
if (maxDelta < tolerance) {
|
|
138
157
|
converged = true
|
|
158
|
+
|
|
139
159
|
iterations++
|
|
160
|
+
|
|
140
161
|
break
|
|
141
162
|
}
|
|
142
163
|
}
|
package/fellegi-sunter.ts
CHANGED
|
@@ -27,27 +27,47 @@
|
|
|
27
27
|
|
|
28
28
|
import { nameSimilarity } from "./comparators.ts"
|
|
29
29
|
|
|
30
|
-
/**
|
|
30
|
+
/**
|
|
31
|
+
* One agreement level of a comparison, with its match / non-match probabilities.
|
|
32
|
+
*/
|
|
31
33
|
export interface ComparisonLevel {
|
|
32
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* Human-readable label for debugging (`exact`, `high`, `different`).
|
|
36
|
+
*/
|
|
33
37
|
label: string
|
|
34
|
-
/**
|
|
38
|
+
/**
|
|
39
|
+
* P(a pair lands in this level | it is a true match). A measure of data quality.
|
|
40
|
+
*/
|
|
35
41
|
m: number
|
|
36
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* P(a pair lands in this level | it is NOT a match). A measure of coincidence / cardinality.
|
|
44
|
+
*/
|
|
37
45
|
u: number
|
|
38
|
-
/**
|
|
46
|
+
/**
|
|
47
|
+
* For similarity-driven comparisons: the minimum similarity (inclusive) to qualify.
|
|
48
|
+
*/
|
|
39
49
|
minSimilarity?: number
|
|
40
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* For distance-driven comparisons: the maximum distance in km (inclusive) to qualify.
|
|
52
|
+
*/
|
|
41
53
|
maxKm?: number
|
|
42
54
|
}
|
|
43
55
|
|
|
44
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* A per-field comparison: pull a value from each record and assign an agreement level.
|
|
58
|
+
*/
|
|
45
59
|
export interface Comparison<R> {
|
|
46
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* Field name, for attribution.
|
|
62
|
+
*/
|
|
47
63
|
name: string
|
|
48
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* Levels ordered highest agreement → lowest (`exact` first, `different` last).
|
|
66
|
+
*/
|
|
49
67
|
levels: ComparisonLevel[]
|
|
50
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* Index into {@link levels}, or `-1` when either value is missing (no evidence → weight 0).
|
|
70
|
+
*/
|
|
51
71
|
assess(a: R, b: R): number
|
|
52
72
|
/**
|
|
53
73
|
* Optional term-frequency adjustment: on the levels it names, replace the level's average `u` with the agreeing
|
|
@@ -64,46 +84,74 @@ export interface Comparison<R> {
|
|
|
64
84
|
* {@link TermFrequencyAdjustment.minimumFrequency} so an ultra-rare value can't produce an unbounded boost.
|
|
65
85
|
*/
|
|
66
86
|
export interface TermFrequencyAdjustment<R> {
|
|
67
|
-
/**
|
|
87
|
+
/**
|
|
88
|
+
* Relative frequency of a value in the data, in (0, 1]. Typically computed on-the-fly.
|
|
89
|
+
*/
|
|
68
90
|
frequency(value: string): number
|
|
69
|
-
/**
|
|
91
|
+
/**
|
|
92
|
+
* The level indices the adjustment applies to (typically just the exact level).
|
|
93
|
+
*/
|
|
70
94
|
levels: ReadonlySet<number>
|
|
71
|
-
/**
|
|
95
|
+
/**
|
|
96
|
+
* The agreeing value to look up for a pair (a normalized field value), or null to skip.
|
|
97
|
+
*/
|
|
72
98
|
value(a: R, b: R): string | null | undefined
|
|
73
|
-
/**
|
|
99
|
+
/**
|
|
100
|
+
* Scale the adjustment in [0, 1]. Default 1.
|
|
101
|
+
*/
|
|
74
102
|
weight?: number
|
|
75
|
-
/**
|
|
103
|
+
/**
|
|
104
|
+
* Floor for the looked-up frequency, bounding the boost on ultra-rare values. Default 1e-4.
|
|
105
|
+
*/
|
|
76
106
|
minimumFrequency?: number
|
|
77
107
|
}
|
|
78
108
|
|
|
79
|
-
/**
|
|
109
|
+
/**
|
|
110
|
+
* A Fellegi-Sunter model: the field comparisons plus the prior match rate `λ`.
|
|
111
|
+
*/
|
|
80
112
|
export interface FellegiSunterModel<R> {
|
|
81
113
|
comparisons: Comparison<R>[]
|
|
82
|
-
/**
|
|
114
|
+
/**
|
|
115
|
+
* Prior probability that two records drawn at random are a match.
|
|
116
|
+
*/
|
|
83
117
|
lambda: number
|
|
84
118
|
}
|
|
85
119
|
|
|
86
|
-
/**
|
|
120
|
+
/**
|
|
121
|
+
* The scored outcome for one record pair.
|
|
122
|
+
*/
|
|
87
123
|
export interface PairScore {
|
|
88
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* Total match weight in bits (`log2` odds).
|
|
126
|
+
*/
|
|
89
127
|
weight: number
|
|
90
|
-
/**
|
|
128
|
+
/**
|
|
129
|
+
* Match probability in [0, 1].
|
|
130
|
+
*/
|
|
91
131
|
probability: number
|
|
92
|
-
/**
|
|
132
|
+
/**
|
|
133
|
+
* Per-field breakdown — what drove the score.
|
|
134
|
+
*/
|
|
93
135
|
contributions: Array<{ name: string; level: string | null; weight: number }>
|
|
94
136
|
}
|
|
95
137
|
|
|
96
|
-
/**
|
|
138
|
+
/**
|
|
139
|
+
* The terminal decision for a pair under upper / lower match-weight thresholds.
|
|
140
|
+
*/
|
|
97
141
|
export type MatchDecision = "match" | "review" | "non-match"
|
|
98
142
|
|
|
99
|
-
/**
|
|
143
|
+
/**
|
|
144
|
+
* The Bayes-factor weight of a single level, in bits: `log2(m / u)`.
|
|
145
|
+
*/
|
|
100
146
|
export function levelWeight(level: ComparisonLevel): number {
|
|
101
147
|
if (level.u <= 0) return level.m > 0 ? Infinity : 0
|
|
102
148
|
|
|
103
149
|
return Math.log2(level.m / level.u)
|
|
104
150
|
}
|
|
105
151
|
|
|
106
|
-
/**
|
|
152
|
+
/**
|
|
153
|
+
* The prior match weight in bits: `log2(λ / (1 - λ))`.
|
|
154
|
+
*/
|
|
107
155
|
export function priorWeight(lambda: number): number {
|
|
108
156
|
if (lambda <= 0) return -Infinity
|
|
109
157
|
|
|
@@ -112,7 +160,9 @@ export function priorWeight(lambda: number): number {
|
|
|
112
160
|
return Math.log2(lambda / (1 - lambda))
|
|
113
161
|
}
|
|
114
162
|
|
|
115
|
-
/**
|
|
163
|
+
/**
|
|
164
|
+
* Convert a total match weight (bits) to a probability, numerically stable for extreme weights.
|
|
165
|
+
*/
|
|
116
166
|
export function probabilityFromWeight(weight: number): number {
|
|
117
167
|
return 1 / (1 + 2 ** -weight)
|
|
118
168
|
}
|
|
@@ -125,7 +175,9 @@ export function probabilityFromWeight(weight: number): number {
|
|
|
125
175
|
export function similarityComparison<R>(config: {
|
|
126
176
|
name: string
|
|
127
177
|
extract: (record: R) => string | null | undefined
|
|
128
|
-
/**
|
|
178
|
+
/**
|
|
179
|
+
* Defaults to {@link nameSimilarity}.
|
|
180
|
+
*/
|
|
129
181
|
similarity?: (a: string, b: string) => number
|
|
130
182
|
levels: ComparisonLevel[]
|
|
131
183
|
}): Comparison<R> {
|
|
@@ -151,7 +203,9 @@ export function similarityComparison<R>(config: {
|
|
|
151
203
|
}
|
|
152
204
|
}
|
|
153
205
|
|
|
154
|
-
/**
|
|
206
|
+
/**
|
|
207
|
+
* Score a record pair: total match weight, probability, and the per-field contributions.
|
|
208
|
+
*/
|
|
155
209
|
export function scorePair<R>(model: FellegiSunterModel<R>, a: R, b: R): PairScore {
|
|
156
210
|
let weight = priorWeight(model.lambda)
|
|
157
211
|
const contributions: PairScore["contributions"] = []
|
|
@@ -161,8 +215,10 @@ export function scorePair<R>(model: FellegiSunterModel<R>, a: R, b: R): PairScor
|
|
|
161
215
|
|
|
162
216
|
if (index < 0) {
|
|
163
217
|
contributions.push({ name: comparison.name, level: null, weight: 0 })
|
|
218
|
+
|
|
164
219
|
continue
|
|
165
220
|
}
|
|
221
|
+
|
|
166
222
|
const level = comparison.levels[index]!
|
|
167
223
|
let w = levelWeight(level)
|
|
168
224
|
|
package/gbt.ts
CHANGED
|
@@ -18,7 +18,19 @@
|
|
|
18
18
|
* trains offline once and ships as a data file.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* Distinct values at or below which every split point is tried exactly rather than by quantile.
|
|
23
|
+
*/
|
|
24
|
+
const MAX_EXACT_SPLIT_VALUES = 5
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Quantile split points evaluated for a continuous feature.
|
|
28
|
+
*/
|
|
29
|
+
const QUANTILE_SPLIT_COUNT = 6
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A trained tree: an internal split (feature `f` ≤ `thr` → `lo`, else `hi`) or a `leaf` value.
|
|
33
|
+
*/
|
|
22
34
|
export type TreeNode = { leaf: number } | { f: number; thr: number; lo: TreeNode; hi: TreeNode }
|
|
23
35
|
|
|
24
36
|
const sigmoid = (z: number): number => 1 / (1 + Math.exp(-Math.max(-30, Math.min(30, z))))
|
|
@@ -32,24 +44,26 @@ export function buildThresholds(X: number[][]): number[][] {
|
|
|
32
44
|
|
|
33
45
|
for (let f = 0; f < dim; f++) {
|
|
34
46
|
const vals = X.map((r) => r[f]!)
|
|
35
|
-
const uniq = [...new Set(vals)].
|
|
47
|
+
const uniq = [...new Set(vals)].toSorted((p, q) => p - q)
|
|
36
48
|
|
|
37
49
|
if (uniq.length <= 1) {
|
|
38
50
|
out.push([])
|
|
39
|
-
} else if (uniq.length <=
|
|
51
|
+
} else if (uniq.length <= MAX_EXACT_SPLIT_VALUES) {
|
|
40
52
|
const t: number[] = []
|
|
41
53
|
|
|
42
54
|
for (let k = 0; k < uniq.length - 1; k++) {
|
|
43
55
|
t.push((uniq[k]! + uniq[k + 1]!) / 2)
|
|
44
56
|
}
|
|
57
|
+
|
|
45
58
|
out.push(t)
|
|
46
59
|
} else {
|
|
47
|
-
const sorted = [...vals].
|
|
60
|
+
const sorted = [...vals].toSorted((p, q) => p - q)
|
|
48
61
|
const t: number[] = []
|
|
49
62
|
|
|
50
|
-
for (let q = 1; q <=
|
|
63
|
+
for (let q = 1; q <= QUANTILE_SPLIT_COUNT; q++) {
|
|
51
64
|
t.push(sorted[Math.floor((q / 7) * (sorted.length - 1))]!)
|
|
52
65
|
}
|
|
66
|
+
|
|
53
67
|
out.push([...new Set(t)])
|
|
54
68
|
}
|
|
55
69
|
}
|
|
@@ -57,7 +71,9 @@ export function buildThresholds(X: number[][]): number[][] {
|
|
|
57
71
|
return out
|
|
58
72
|
}
|
|
59
73
|
|
|
60
|
-
/**
|
|
74
|
+
/**
|
|
75
|
+
* Weighted SSE of target `g` over `rows` around their weighted mean.
|
|
76
|
+
*/
|
|
61
77
|
function nodeSSE(rows: number[], g: number[], w: number[]): number {
|
|
62
78
|
let wsum = 0
|
|
63
79
|
let wg = 0
|
|
@@ -66,6 +82,7 @@ function nodeSSE(rows: number[], g: number[], w: number[]): number {
|
|
|
66
82
|
wsum += w[i]!
|
|
67
83
|
wg += w[i]! * g[i]!
|
|
68
84
|
}
|
|
85
|
+
|
|
69
86
|
const mean = wsum > 0 ? wg / wsum : 0
|
|
70
87
|
let sse = 0
|
|
71
88
|
|
|
@@ -77,7 +94,9 @@ function nodeSSE(rows: number[], g: number[], w: number[]): number {
|
|
|
77
94
|
return sse
|
|
78
95
|
}
|
|
79
96
|
|
|
80
|
-
/**
|
|
97
|
+
/**
|
|
98
|
+
* Greedy depth-limited weighted regression tree on target `g` (the boosting residual).
|
|
99
|
+
*/
|
|
81
100
|
function fitRegTree(
|
|
82
101
|
rows: number[],
|
|
83
102
|
X: number[][],
|
|
@@ -94,6 +113,7 @@ function fitRegTree(
|
|
|
94
113
|
wsum += w[i]!
|
|
95
114
|
wg += w[i]! * g[i]!
|
|
96
115
|
}
|
|
116
|
+
|
|
97
117
|
const leaf = wsum > 0 ? wg / wsum : 0
|
|
98
118
|
|
|
99
119
|
if (depth === 0 || rows.length < 2 * minLeaf) return { leaf }
|
|
@@ -146,14 +166,18 @@ function predictTree(t: TreeNode, x: number[]): number {
|
|
|
146
166
|
return n.leaf
|
|
147
167
|
}
|
|
148
168
|
|
|
149
|
-
/**
|
|
169
|
+
/**
|
|
170
|
+
* A trained gradient-boosted-tree model: an additive ensemble over a base log-odds. Plain JSON.
|
|
171
|
+
*/
|
|
150
172
|
export interface GBT {
|
|
151
173
|
trees: TreeNode[]
|
|
152
174
|
lr: number
|
|
153
175
|
base: number
|
|
154
176
|
}
|
|
155
177
|
|
|
156
|
-
/**
|
|
178
|
+
/**
|
|
179
|
+
* Hyperparameters for {@link trainGBT}.
|
|
180
|
+
*/
|
|
157
181
|
export interface GBTOpts {
|
|
158
182
|
rounds: number
|
|
159
183
|
depth: number
|
|
@@ -161,7 +185,9 @@ export interface GBTOpts {
|
|
|
161
185
|
minLeaf: number
|
|
162
186
|
}
|
|
163
187
|
|
|
164
|
-
/**
|
|
188
|
+
/**
|
|
189
|
+
* Gradient-boosted regression trees on logistic loss, with per-sample class weights `w`.
|
|
190
|
+
*/
|
|
165
191
|
export function trainGBT(X: number[][], y: number[], w: number[], opts: GBTOpts): GBT {
|
|
166
192
|
const N = X.length
|
|
167
193
|
const thresholds = buildThresholds(X)
|
|
@@ -176,6 +202,7 @@ export function trainGBT(X: number[][], y: number[], w: number[], opts: GBTOpts)
|
|
|
176
202
|
wpos += w[i]!
|
|
177
203
|
}
|
|
178
204
|
}
|
|
205
|
+
|
|
179
206
|
const base = Math.log((wpos + 1) / (wtot - wpos + 1)) // weighted base log-odds
|
|
180
207
|
const F = new Array<number>(N).fill(base)
|
|
181
208
|
const trees: TreeNode[] = []
|
|
@@ -185,19 +212,24 @@ export function trainGBT(X: number[][], y: number[], w: number[], opts: GBTOpts)
|
|
|
185
212
|
|
|
186
213
|
for (let i = 0; i < N; i++) {
|
|
187
214
|
g[i] = y[i]! - sigmoid(F[i]!)
|
|
188
|
-
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// negative gradient of logistic loss
|
|
189
218
|
const tree = fitRegTree(rowsAll, X, g, w, thresholds, opts.depth, opts.minLeaf)
|
|
190
219
|
|
|
191
220
|
for (let i = 0; i < N; i++) {
|
|
192
221
|
F[i]! += opts.lr * predictTree(tree, X[i]!)
|
|
193
222
|
}
|
|
223
|
+
|
|
194
224
|
trees.push(tree)
|
|
195
225
|
}
|
|
196
226
|
|
|
197
227
|
return { trees, lr: opts.lr, base }
|
|
198
228
|
}
|
|
199
229
|
|
|
200
|
-
/**
|
|
230
|
+
/**
|
|
231
|
+
* GBT score (logit) for one feature vector. Threshold-comparable like the FS weight.
|
|
232
|
+
*/
|
|
201
233
|
export function gbtScore(m: GBT, x: number[]): number {
|
|
202
234
|
let f = m.base
|
|
203
235
|
|
package/out/blocking.d.ts
CHANGED
|
@@ -18,9 +18,13 @@
|
|
|
18
18
|
* silent failure in record linkage. So the spatial grid is generous and neighbour-expanded by
|
|
19
19
|
* default, and any block too large to scan is _reported_, never silently dropped.
|
|
20
20
|
*/
|
|
21
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* Maps a record to zero or more block keys. Two records sharing any key become a candidate pair.
|
|
23
|
+
*/
|
|
22
24
|
export type BlockingKey<R> = (record: R) => string[];
|
|
23
|
-
/**
|
|
25
|
+
/**
|
|
26
|
+
* A geographic coordinate (WGS84 decimal degrees).
|
|
27
|
+
*/
|
|
24
28
|
export interface LatLon {
|
|
25
29
|
latitude: number;
|
|
26
30
|
longitude: number;
|
|
@@ -53,11 +57,17 @@ export declare function exactKey<R>(extract: (record: R) => string | null | unde
|
|
|
53
57
|
* Tighter blocks, lower recall — use when a single rule is too loose.
|
|
54
58
|
*/
|
|
55
59
|
export declare function conjunction<R>(...keys: BlockingKey<R>[]): BlockingKey<R>;
|
|
56
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* The outcome of a blocking pass.
|
|
62
|
+
*/
|
|
57
63
|
export interface BlockResult<R> {
|
|
58
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* Deduplicated candidate pairs (no self-pairs; a pair caught by multiple keys appears once).
|
|
66
|
+
*/
|
|
59
67
|
pairs: Array<[R, R]>;
|
|
60
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* Blocks that exceeded `maxBlockSize` and were skipped — surfaced so coverage limits are visible.
|
|
70
|
+
*/
|
|
61
71
|
droppedBlocks: Array<{
|
|
62
72
|
key: string;
|
|
63
73
|
size: number;
|
package/out/blocking.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blocking.d.ts","sourceRoot":"","sources":["../blocking.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH
|
|
1
|
+
{"version":3,"file":"blocking.d.ts","sourceRoot":"","sources":["../blocking.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;GAEG;AACH,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,EAAE,CAAA;AAEpD;;GAEG;AACH,MAAM,WAAW,MAAM;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;CACjB;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAC3B,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,EACjD,IAAI,GAAE;IAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAO,GAC3D,WAAW,CAAC,CAAC,CAAC,CAwBhB;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EACzB,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,EACjD,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAA;CAAO,GACnE,WAAW,CAAC,CAAC,CAAC,CAahB;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,CAaxE;AAED;;GAEG;AACH,MAAM,WAAW,WAAW,CAAC,CAAC;IAC7B;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACpB;;OAEG;IACH,aAAa,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACnD;AAED;;;;;GAKG;AACH,wBAAgB,KAAK,CAAC,CAAC,EACtB,OAAO,EAAE,SAAS,CAAC,EAAE,EACrB,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,EAC/C,IAAI,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,WAAW,CAAC,CAAC,CAAC,CAmDhB"}
|
package/out/blocking.js
CHANGED
|
@@ -53,7 +53,7 @@ export function geoCellKey(extract, opts = {}) {
|
|
|
53
53
|
* key.
|
|
54
54
|
*/
|
|
55
55
|
export function exactKey(extract, opts = {}) {
|
|
56
|
-
const normalize = opts.normalize ?? ((v) => v.trim().toLowerCase().
|
|
56
|
+
const normalize = opts.normalize ?? ((v) => v.trim().toLowerCase().replaceAll(/\s+/g, " "));
|
|
57
57
|
return (record) => {
|
|
58
58
|
const value = extract(record);
|
|
59
59
|
if (!value)
|
|
@@ -74,7 +74,7 @@ export function conjunction(...keys) {
|
|
|
74
74
|
let combos = [""];
|
|
75
75
|
for (const key of keys) {
|
|
76
76
|
const parts = key(record);
|
|
77
|
-
if (parts.length
|
|
77
|
+
if (!parts.length)
|
|
78
78
|
return [];
|
|
79
79
|
combos = combos.flatMap((prefix) => parts.map((part) => (prefix ? `${prefix}&${part}` : part)));
|
|
80
80
|
}
|