@mailwoman/match 9.4.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/blocking.ts +41 -36
- package/lib/clustering.ts +43 -26
- package/lib/comparators.ts +11 -34
- package/lib/distance.ts +22 -48
- package/lib/em.ts +14 -6
- package/lib/fellegi-sunter.ts +39 -22
- package/lib/gbt.ts +8 -32
- package/lib/tf.ts +26 -14
- package/out/blocking.d.ts +40 -35
- package/out/blocking.d.ts.map +1 -1
- package/out/blocking.js +34 -25
- package/out/blocking.js.map +1 -1
- package/out/clustering.d.ts +30 -17
- package/out/clustering.d.ts.map +1 -1
- package/out/clustering.js +31 -19
- package/out/clustering.js.map +1 -1
- package/out/comparators.d.ts +11 -33
- package/out/comparators.d.ts.map +1 -1
- package/out/comparators.js +11 -34
- package/out/comparators.js.map +1 -1
- package/out/distance.d.ts +18 -43
- package/out/distance.d.ts.map +1 -1
- package/out/distance.js +16 -41
- package/out/distance.js.map +1 -1
- package/out/em.d.ts +14 -6
- package/out/em.d.ts.map +1 -1
- package/out/em.js +5 -3
- package/out/em.js.map +1 -1
- package/out/fellegi-sunter.d.ts +39 -22
- package/out/fellegi-sunter.d.ts.map +1 -1
- package/out/fellegi-sunter.js +17 -12
- package/out/fellegi-sunter.js.map +1 -1
- package/out/gbt.d.ts +7 -17
- package/out/gbt.d.ts.map +1 -1
- package/out/gbt.js +7 -31
- package/out/gbt.js.map +1 -1
- package/out/tf.d.ts +24 -13
- package/out/tf.d.ts.map +1 -1
- package/out/tf.js +17 -11
- package/out/tf.js.map +1 -1
- package/package.json +13 -88
package/out/tf.js
CHANGED
|
@@ -5,24 +5,26 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Term-frequency adjustment — making a rare-value agreement count more than a common one.
|
|
7
7
|
*
|
|
8
|
-
* Two people
|
|
8
|
+
* Two people who share the name "Vijayan" are far stronger evidence of a match than two who share the name "Smith",
|
|
9
9
|
* because "Smith" agreements happen by chance all the time and "Vijayan" agreements don't. The
|
|
10
|
-
* Fellegi-Sunter `m` (how often a true match agrees) is roughly the same either way
|
|
10
|
+
* Fellegi-Sunter `m` (how often a true match agrees) is roughly the same either way. what differs
|
|
11
11
|
* is `u` — the chance a _non_-match agrees — which for an exact agreement on value `v` is just
|
|
12
12
|
* how common `v` is. So we leave `m`, and replace the level's average `u` with `frequency(v)`,
|
|
13
13
|
* adding `log2(u_level / frequency(v))` to the weight: a big positive bump for rare values, a
|
|
14
14
|
* penalty for common ones.
|
|
15
15
|
*
|
|
16
|
-
* Crucially for a label-free matcher: the frequencies are computed
|
|
16
|
+
* Crucially for a label-free matcher: the frequencies are computed on-the-FLY from the input column
|
|
17
17
|
* (the Splink approach) — no external Census table required. Build a {@link TermFrequencyTable}
|
|
18
18
|
* from the values you're matching, then attach it to a comparison with {@link withTermFrequency}.
|
|
19
19
|
*/
|
|
20
|
-
// Local by design: `@mailwoman/record`'s per-field normalizers are the shared home
|
|
20
|
+
// Local by design: `@mailwoman/record`'s per-field normalizers are the shared home.
|
|
21
|
+
// Match takes no record dependency.
|
|
21
22
|
const defaultNormalize = (value) => value.trim().toLowerCase().replaceAll(/\s+/g, " ");
|
|
22
23
|
/**
|
|
23
|
-
* Build a {@link TermFrequencyTable} from an iterable of values (e.g.
|
|
24
|
-
*
|
|
25
|
-
*
|
|
24
|
+
* Build a {@link TermFrequencyTable} from an iterable of values (e.g. Every `given` name in the dataset).
|
|
25
|
+
*
|
|
26
|
+
* Values are normalized (default: trim + lowercase + collapse whitespace) before counting.
|
|
27
|
+
* `frequency()` normalizes its argument the same way, so callers pass raw field values.
|
|
26
28
|
*/
|
|
27
29
|
export function buildTermFrequencyTable(values, opts = {}) {
|
|
28
30
|
const normalize = opts.normalize ?? defaultNormalize;
|
|
@@ -48,10 +50,14 @@ export function buildTermFrequencyTable(values, opts = {}) {
|
|
|
48
50
|
};
|
|
49
51
|
}
|
|
50
52
|
/**
|
|
51
|
-
* Attach a term-frequency adjustment to a comparison.
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
53
|
+
* Attach a term-frequency adjustment to a comparison.
|
|
54
|
+
*
|
|
55
|
+
* By default it applies to the exact level (index 0) and looks up the value via
|
|
56
|
+
* `value(a, b)` — usually the agreeing field extracted from one side.
|
|
57
|
+
* Returns a new comparison.
|
|
58
|
+
*
|
|
59
|
+
* The underlying `assess` and levels are untouched, so this composes with EM
|
|
60
|
+
* (which re-estimates the base `m`/`u` the adjustment sits on top of).
|
|
55
61
|
*/
|
|
56
62
|
export function withTermFrequency(comparison, config) {
|
|
57
63
|
const adjustment = {
|
package/out/tf.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tf.js","sourceRoot":"","sources":["../lib/tf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAsBH,
|
|
1
|
+
{"version":3,"file":"tf.js","sourceRoot":"","sources":["../lib/tf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAsBH,oFAAoF;AACpF,oCAAoC;AACpC,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;AAEtG;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CACtC,MAA2C,EAC3C,IAAI,GAA8C,EAAE;IAEpD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,gBAAgB,CAAA;IACpD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,KAAK,IAAI,IAAI;YAAE,SAAQ;QAC3B,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,CAAC,GAAG;YAAE,SAAQ;QAClB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAE3C,KAAK,EAAE,CAAA;IACR,CAAC;IAED,OAAO;QACN,KAAK;QACL,QAAQ,EAAE,MAAM,CAAC,IAAI;QACrB,SAAS,CAAC,KAAK;YACd,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,CAAC,CAAA;YAEzB,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAA;QACnD,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAChC,UAAyB,EACzB,MAqBC;IAED,MAAM,UAAU,GAA+B;QAC9C,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC;QACnD,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;KACzC,CAAA;IAED,OAAO,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,CAAA;AACpD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/match",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "10.1.0",
|
|
4
4
|
"description": "The geocode-first record matcher: block → score → cluster. This first reduce ships the string comparators (Jaro / Jaro-Winkler + an edit-distance fallback for compound surnames) that the Fellegi-Sunter scorer is built on.",
|
|
5
5
|
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|
|
@@ -13,15 +13,10 @@
|
|
|
13
13
|
"out/**/*.js.map",
|
|
14
14
|
"out/**/*.d.ts",
|
|
15
15
|
"out/**/*.d.ts.map",
|
|
16
|
-
"
|
|
17
|
-
"
|
|
18
|
-
"**/*.ts",
|
|
19
|
-
"**/*.tsx",
|
|
20
|
-
"!*.test.ts",
|
|
21
|
-
"!*.test.tsx",
|
|
16
|
+
"lib/**/*.ts",
|
|
17
|
+
"lib/**/*.tsx",
|
|
22
18
|
"!**/*.test.ts",
|
|
23
|
-
"!**/*.test.tsx"
|
|
24
|
-
"!test/**"
|
|
19
|
+
"!**/*.test.tsx"
|
|
25
20
|
],
|
|
26
21
|
"type": "module",
|
|
27
22
|
"sideEffects": false,
|
|
@@ -39,45 +34,10 @@
|
|
|
39
34
|
"node": "./out/index.js",
|
|
40
35
|
"default": "./out/index.js"
|
|
41
36
|
},
|
|
42
|
-
"
|
|
43
|
-
"types": "./out
|
|
44
|
-
"node": "./out
|
|
45
|
-
"default": "./out
|
|
46
|
-
},
|
|
47
|
-
"./clustering": {
|
|
48
|
-
"types": "./out/clustering.d.ts",
|
|
49
|
-
"node": "./out/clustering.js",
|
|
50
|
-
"default": "./out/clustering.js"
|
|
51
|
-
},
|
|
52
|
-
"./comparators": {
|
|
53
|
-
"types": "./out/comparators.d.ts",
|
|
54
|
-
"node": "./out/comparators.js",
|
|
55
|
-
"default": "./out/comparators.js"
|
|
56
|
-
},
|
|
57
|
-
"./distance": {
|
|
58
|
-
"types": "./out/distance.d.ts",
|
|
59
|
-
"node": "./out/distance.js",
|
|
60
|
-
"default": "./out/distance.js"
|
|
61
|
-
},
|
|
62
|
-
"./fellegi-sunter": {
|
|
63
|
-
"types": "./out/fellegi-sunter.d.ts",
|
|
64
|
-
"node": "./out/fellegi-sunter.js",
|
|
65
|
-
"default": "./out/fellegi-sunter.js"
|
|
66
|
-
},
|
|
67
|
-
"./em": {
|
|
68
|
-
"types": "./out/em.d.ts",
|
|
69
|
-
"node": "./out/em.js",
|
|
70
|
-
"default": "./out/em.js"
|
|
71
|
-
},
|
|
72
|
-
"./tf": {
|
|
73
|
-
"types": "./out/tf.d.ts",
|
|
74
|
-
"node": "./out/tf.js",
|
|
75
|
-
"default": "./out/tf.js"
|
|
76
|
-
},
|
|
77
|
-
"./gbt": {
|
|
78
|
-
"types": "./out/gbt.d.ts",
|
|
79
|
-
"node": "./out/gbt.js",
|
|
80
|
-
"default": "./out/gbt.js"
|
|
37
|
+
"./*": {
|
|
38
|
+
"types": "./out/*.d.ts",
|
|
39
|
+
"node": "./out/*.js",
|
|
40
|
+
"default": "./out/*.js"
|
|
81
41
|
}
|
|
82
42
|
},
|
|
83
43
|
"publishConfig": {
|
|
@@ -89,45 +49,10 @@
|
|
|
89
49
|
"node": "./out/index.js",
|
|
90
50
|
"default": "./out/index.js"
|
|
91
51
|
},
|
|
92
|
-
"
|
|
93
|
-
"types": "./out
|
|
94
|
-
"node": "./out
|
|
95
|
-
"default": "./out
|
|
96
|
-
},
|
|
97
|
-
"./clustering": {
|
|
98
|
-
"types": "./out/clustering.d.ts",
|
|
99
|
-
"node": "./out/clustering.js",
|
|
100
|
-
"default": "./out/clustering.js"
|
|
101
|
-
},
|
|
102
|
-
"./comparators": {
|
|
103
|
-
"types": "./out/comparators.d.ts",
|
|
104
|
-
"node": "./out/comparators.js",
|
|
105
|
-
"default": "./out/comparators.js"
|
|
106
|
-
},
|
|
107
|
-
"./distance": {
|
|
108
|
-
"types": "./out/distance.d.ts",
|
|
109
|
-
"node": "./out/distance.js",
|
|
110
|
-
"default": "./out/distance.js"
|
|
111
|
-
},
|
|
112
|
-
"./fellegi-sunter": {
|
|
113
|
-
"types": "./out/fellegi-sunter.d.ts",
|
|
114
|
-
"node": "./out/fellegi-sunter.js",
|
|
115
|
-
"default": "./out/fellegi-sunter.js"
|
|
116
|
-
},
|
|
117
|
-
"./em": {
|
|
118
|
-
"types": "./out/em.d.ts",
|
|
119
|
-
"node": "./out/em.js",
|
|
120
|
-
"default": "./out/em.js"
|
|
121
|
-
},
|
|
122
|
-
"./tf": {
|
|
123
|
-
"types": "./out/tf.d.ts",
|
|
124
|
-
"node": "./out/tf.js",
|
|
125
|
-
"default": "./out/tf.js"
|
|
126
|
-
},
|
|
127
|
-
"./gbt": {
|
|
128
|
-
"types": "./out/gbt.d.ts",
|
|
129
|
-
"node": "./out/gbt.js",
|
|
130
|
-
"default": "./out/gbt.js"
|
|
52
|
+
"./*": {
|
|
53
|
+
"types": "./out/*.d.ts",
|
|
54
|
+
"node": "./out/*.js",
|
|
55
|
+
"default": "./out/*.js"
|
|
131
56
|
}
|
|
132
57
|
},
|
|
133
58
|
"imports": {
|
|
@@ -139,7 +64,7 @@
|
|
|
139
64
|
}
|
|
140
65
|
},
|
|
141
66
|
"dependencies": {
|
|
142
|
-
"@mailwoman/spatial": "
|
|
67
|
+
"@mailwoman/spatial": "10.1.0",
|
|
143
68
|
"fastest-levenshtein": "^1.0.16"
|
|
144
69
|
},
|
|
145
70
|
"devDependencies": {
|