@mailwoman/match 9.4.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/blocking.ts +41 -36
- package/lib/clustering.ts +43 -26
- package/lib/comparators.ts +11 -34
- package/lib/distance.ts +22 -48
- package/lib/em.ts +14 -6
- package/lib/fellegi-sunter.ts +39 -22
- package/lib/gbt.ts +8 -32
- package/lib/tf.ts +26 -14
- package/out/blocking.d.ts +40 -35
- package/out/blocking.d.ts.map +1 -1
- package/out/blocking.js +34 -25
- package/out/blocking.js.map +1 -1
- package/out/clustering.d.ts +30 -17
- package/out/clustering.d.ts.map +1 -1
- package/out/clustering.js +31 -19
- package/out/clustering.js.map +1 -1
- package/out/comparators.d.ts +11 -33
- package/out/comparators.d.ts.map +1 -1
- package/out/comparators.js +11 -34
- package/out/comparators.js.map +1 -1
- package/out/distance.d.ts +18 -43
- package/out/distance.d.ts.map +1 -1
- package/out/distance.js +16 -41
- package/out/distance.js.map +1 -1
- package/out/em.d.ts +14 -6
- package/out/em.d.ts.map +1 -1
- package/out/em.js +5 -3
- package/out/em.js.map +1 -1
- package/out/fellegi-sunter.d.ts +39 -22
- package/out/fellegi-sunter.d.ts.map +1 -1
- package/out/fellegi-sunter.js +17 -12
- package/out/fellegi-sunter.js.map +1 -1
- package/out/gbt.d.ts +7 -17
- package/out/gbt.d.ts.map +1 -1
- package/out/gbt.js +7 -31
- package/out/gbt.js.map +1 -1
- package/out/tf.d.ts +24 -13
- package/out/tf.d.ts.map +1 -1
- package/out/tf.js +17 -11
- package/out/tf.js.map +1 -1
- package/package.json +13 -88
package/out/distance.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"distance.js","sourceRoot":"","sources":["../lib/distance.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"distance.js","sourceRoot":"","sources":["../lib/distance.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,IAAI,aAAa,EAAsB,MAAM,oBAAoB,CAAA;AAIrF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAgB,EAAE,CAAgB,EAAU,EAAE,CACzE,aAAa,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,SAAS,CAAC,CAAA;AAEhE;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAI,MAIrC;IACA,MAAM,KAAK,GAAG,CAAC,CAAmC,EAAsB,EAAE,CACzE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;IAEnE,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,CAAC,CAAC,EAAE,CAAC;YACV,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;YAE5B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAAE,OAAO,CAAC,CAAC,CAAA;YAEvC,MAAM,EAAE,GAAG,WAAW,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;YAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/C,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,IAAI,QAAQ,CAAC;oBAAE,OAAO,CAAC,CAAA;YAC1D,CAAC;YAED,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;QAChC,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAsB;IACzD,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE;IACzD,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE;IACpD,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE;IACjD,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE;CACnC,CAAA;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAI,MAKpC;IACA,MAAM,KAAK,GAAG,CAAC,CAAmC,EAAsB,EAAE,CACzE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;IAEnE,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,CAAC,CAAC,EAAE,CAAC;YACV,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACxB,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAExB,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE;gBAAE,OAAO,CAAC,CAAA;YAEhD,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;YAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAAE,OAAO,CAAC,CAAC,CAAA;YAEvC,MAAM,EAAE,GAAG,WAAW,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;YAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/C,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,IAAI,QAAQ,CAAC;oBAAE,OAAO,CAAC,CAAA;YAC1D,CAAC;YAED,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;QAChC,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAsB;IACxD,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE;IACvC,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE;IACxD,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE;IACrD,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE;IAClD,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE;CACnC,CAAA"}
|
package/out/em.d.ts
CHANGED
|
@@ -31,15 +31,21 @@ export declare function agreementPattern<R>(comparisons: Comparison<R>[], a: R,
|
|
|
31
31
|
*/
|
|
32
32
|
export interface EmOptions {
|
|
33
33
|
/**
|
|
34
|
-
* Hard iteration cap.
|
|
34
|
+
* Hard iteration cap.
|
|
35
|
+
*
|
|
36
|
+
* Default 100.
|
|
35
37
|
*/
|
|
36
38
|
maxIterations?: number;
|
|
37
39
|
/**
|
|
38
|
-
* Convergence tolerance on the largest parameter change between iterations.
|
|
40
|
+
* Convergence tolerance on the largest parameter change between iterations.
|
|
41
|
+
*
|
|
42
|
+
* Default 1e-6.
|
|
39
43
|
*/
|
|
40
44
|
tolerance?: number;
|
|
41
45
|
/**
|
|
42
|
-
*
|
|
46
|
+
* Prior match rate used to initialize the model.
|
|
47
|
+
*
|
|
48
|
+
* Defaults to the model's `lambda`.
|
|
43
49
|
*/
|
|
44
50
|
initialLambda?: number;
|
|
45
51
|
}
|
|
@@ -59,9 +65,11 @@ export interface EmResult<R> {
|
|
|
59
65
|
converged: boolean;
|
|
60
66
|
}
|
|
61
67
|
/**
|
|
62
|
-
* Estimate `m`/`u` and the prior `λ` from unlabeled agreement patterns via EM.
|
|
63
|
-
*
|
|
64
|
-
*
|
|
68
|
+
* Estimate `m`/`u` and the prior `λ` from unlabeled agreement patterns via EM.
|
|
69
|
+
*
|
|
70
|
+
* The patterns are per-comparison level indices (as produced by {@link agreementPattern});
|
|
71
|
+
* a `-1` (missing) field contributes no evidence to either class.
|
|
72
|
+
* The model's existing level `m`/`u` seed the iteration.
|
|
65
73
|
*/
|
|
66
74
|
export declare function estimateParameters<R>(model: FellegiSunterModel<R>, patterns: number[][], opts?: EmOptions): EmResult<R>;
|
|
67
75
|
//# sourceMappingURL=em.d.ts.map
|
package/out/em.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"em.d.ts","sourceRoot":"","sources":["../lib/em.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAOrE;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAEtF;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB
|
|
1
|
+
{"version":3,"file":"em.d.ts","sourceRoot":"","sources":["../lib/em.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAOrE;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAEtF;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC;IAC1B;;OAEG;IACH,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAA;IAC5B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,OAAO,CAAA;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EACnC,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAC5B,QAAQ,EAAE,MAAM,EAAE,EAAE,EACpB,IAAI,GAAE,SAAc,GAClB,QAAQ,CAAC,CAAC,CAAC,CAyFb"}
|
package/out/em.js
CHANGED
|
@@ -32,9 +32,11 @@ export function agreementPattern(comparisons, a, b) {
|
|
|
32
32
|
return comparisons.map((comparison) => comparison.assess(a, b));
|
|
33
33
|
}
|
|
34
34
|
/**
|
|
35
|
-
* Estimate `m`/`u` and the prior `λ` from unlabeled agreement patterns via EM.
|
|
36
|
-
*
|
|
37
|
-
*
|
|
35
|
+
* Estimate `m`/`u` and the prior `λ` from unlabeled agreement patterns via EM.
|
|
36
|
+
*
|
|
37
|
+
* The patterns are per-comparison level indices (as produced by {@link agreementPattern});
|
|
38
|
+
* a `-1` (missing) field contributes no evidence to either class.
|
|
39
|
+
* The model's existing level `m`/`u` seed the iteration.
|
|
38
40
|
*/
|
|
39
41
|
export function estimateParameters(model, patterns, opts = {}) {
|
|
40
42
|
const maxIterations = opts.maxIterations ?? 100;
|
package/out/em.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"em.js","sourceRoot":"","sources":["../lib/em.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAIH;;GAEG;AACH,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAI,WAA4B,EAAE,CAAI,EAAE,CAAI;IAC3E,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;AAChE,CAAC;
|
|
1
|
+
{"version":3,"file":"em.js","sourceRoot":"","sources":["../lib/em.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAIH;;GAEG;AACH,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAI,WAA4B,EAAE,CAAI,EAAE,CAAI;IAC3E,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;AAChE,CAAC;AA0CD;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CACjC,KAA4B,EAC5B,QAAoB,EACpB,IAAI,GAAc,EAAE;IAEpB,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,GAAG,CAAA;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;IACxC,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,CAAA;IACrC,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAE3D,yEAAyE;IACzE,MAAM,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1D,MAAM,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1D,IAAI,MAAM,GAAG,IAAI,CAAC,aAAa,IAAI,KAAK,CAAC,MAAM,CAAA;IAE/C,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,SAAS,GAAG,KAAK,CAAA;IAErB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAA;IAChD,CAAC;IAED,OAAO,UAAU,GAAG,aAAa,EAAE,UAAU,EAAE,EAAE,CAAC;QACjD,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAS,WAAW,CAAC,CAAC,CAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;QACxF,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAS,WAAW,CAAC,CAAC,CAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;QACxF,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;QAC7C,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;QAC7C,IAAI,iBAAiB,GAAG,CAAC,CAAA;QAEzB,sDAAsD;QACtD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,IAAI,eAAe,GAAG,MAAM,CAAA;YAC5B,IAAI,kBAAkB,GAAG,CAAC,GAAG,MAAM,CAAA;YAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;gBAEzB,IAAI,KAAK,GAAG,CAAC;oBAAE,SAAQ;gBACvB,eAAe,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,KAAK,CAAE,CAAA;gBAChC,kBAAkB,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,KAAK,CAAE,CAAA;YACpC,CAAC;YAED,MAAM,KAAK,GAAG,eAAe,GAAG,kBAAkB,CAAA;YAClD,MAAM,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;YACjD,iBAAiB,IAAI,CAAC,CAAA;YAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;gBAEzB,IAAI,KAAK,GAAG,CAAC;oBAAE,SAAQ;gBACvB,UAAU,CAAC,CAAC,CAAE,CAAC,KAAK,CAAE,IAAI,CAAC,CAAA;gBAC3B,UAAU,CAAC,CAAC,CAAE,CAAC,KAAK,CAAE,IAAI,CAAC,GAAG,CAAC,CAAA;gBAC/B,YAAY,CAAC,CAAC,CAAE,IAAI,CAAC,CAAA;gBACrB,YAAY,CAAC,CAAC,CAAE,IAAI,CAAC,GAAG,CAAC,CAAA;YAC1B,CAAC;QACF,CAAC;QAED,0EAA0E;QAC1E,MAAM,SAAS,GAAG,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAA;QACrD,IAAI,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAA;QAC3C,MAAM,GAAG,SAAS,CAAA;QAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAE,CAAA;YAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjC,MAAM,IAAI,GACT,YAAY,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAE,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,CAAA;gBAE1G,MAAM,IAAI,GACT,YAAY,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAE,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,CAAA;gBAE1G,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACrF,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;gBACf,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;YAChB,CAAC;QACF,CAAC;QAED,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;YAC1B,SAAS,GAAG,IAAI,CAAA;YAEhB,UAAU,EAAE,CAAA;YAEZ,MAAK;QACN,CAAC;IACF,CAAC;IAED,MAAM,iBAAiB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACpD,GAAG,CAAC;QACJ,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;KAC9E,CAAC,CAAC,CAAA;IAEH,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAA;AAC5F,CAAC"}
|
package/out/fellegi-sunter.d.ts
CHANGED
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
*
|
|
6
6
|
* The Fellegi-Sunter scorer — the matcher's decision layer.
|
|
7
7
|
*
|
|
8
|
-
* Each field comparison
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* Each field comparison assigns a record pair to an _agreement level_: exact, high, low, different, or missing.
|
|
9
|
+
* Each level has two probabilities. `m` is P(this level | the pair really matches).
|
|
10
|
+
* `u` is P(this level | the pair does not match). Their ratio is a Bayes factor. Its
|
|
11
11
|
* log is the level's contribution to the total match weight in bits:
|
|
12
12
|
*
|
|
13
13
|
* ```
|
|
@@ -16,13 +16,12 @@
|
|
|
16
16
|
*
|
|
17
17
|
* — a prior (how likely any two random records match) plus an additive, per-field-attributable
|
|
18
18
|
* stack of evidence. Convert `M` to a probability and threshold it: above the upper bound is a
|
|
19
|
-
* link
|
|
20
|
-
* calibrated abstain zone the whole design leans on.
|
|
19
|
+
* link. Below the lower bound, it is a non-link. The band between them is _clerical review_, the calibrated abstain zone.
|
|
21
20
|
*
|
|
22
|
-
* The `m`/`u` numbers here are
|
|
21
|
+
* The `m`/`u` numbers here are not universal constants. They are estimated from the data — by EM,
|
|
23
22
|
* unsupervised (the next increment) — and the term-frequency adjustment that makes a rare-name
|
|
24
23
|
* agreement count more than a common one layers on top. This module is the deterministic core
|
|
25
|
-
* those build on
|
|
24
|
+
* those build on. Given the levels, it produces the weights and probability. It also produces the decision.
|
|
26
25
|
*/
|
|
27
26
|
/**
|
|
28
27
|
* One agreement level of a comparison, with its match / non-match probabilities.
|
|
@@ -37,7 +36,7 @@ export interface ComparisonLevel {
|
|
|
37
36
|
*/
|
|
38
37
|
m: number;
|
|
39
38
|
/**
|
|
40
|
-
* P(a pair lands in this level | it is
|
|
39
|
+
* P(a pair lands in this level | it is not a match). A measure of coincidence / cardinality.
|
|
41
40
|
*/
|
|
42
41
|
u: number;
|
|
43
42
|
/**
|
|
@@ -66,21 +65,29 @@ export interface Comparison<R> {
|
|
|
66
65
|
*/
|
|
67
66
|
assess(a: R, b: R): number;
|
|
68
67
|
/**
|
|
69
|
-
* Optional term-frequency adjustment: on the levels it names, replace the level's
|
|
70
|
-
* value's actual frequency, so agreement on a rare
|
|
68
|
+
* Optional term-frequency adjustment: on the levels it names, replace the level's
|
|
69
|
+
* average `u` with the agreeing value's actual frequency, so agreement on a rare
|
|
70
|
+
* value (`Vijayan`) outweighs agreement on a common one (`Smith`).
|
|
71
|
+
*
|
|
71
72
|
* See `withTermFrequency`.
|
|
72
73
|
*/
|
|
73
74
|
termFrequency?: TermFrequencyAdjustment<R>;
|
|
74
75
|
}
|
|
75
76
|
/**
|
|
76
|
-
* Per-value term-frequency adjustment for a comparison (the Splink/Winkler mechanism).
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
77
|
+
* Per-value term-frequency adjustment for a comparison (the Splink/Winkler mechanism).
|
|
78
|
+
*
|
|
79
|
+
* `m` is unchanged.
|
|
80
|
+
* On an agreement level the effective `u` becomes the value's own frequency, adding
|
|
81
|
+
* `log2(u_level / frequency)` to the weight — large and positive for rare values, negative for common ones.
|
|
82
|
+
*
|
|
83
|
+
* Floored at {@link TermFrequencyAdjustment.minimumFrequency} so an ultra-rare
|
|
84
|
+
* value can't produce an unbounded boost.
|
|
80
85
|
*/
|
|
81
86
|
export interface TermFrequencyAdjustment<R> {
|
|
82
87
|
/**
|
|
83
|
-
* Relative frequency of a value in the data, in (0, 1].
|
|
88
|
+
* Relative frequency of a value in the data, in (0, 1].
|
|
89
|
+
*
|
|
90
|
+
* Typically computed on-the-fly.
|
|
84
91
|
*/
|
|
85
92
|
frequency(value: string): number;
|
|
86
93
|
/**
|
|
@@ -92,11 +99,15 @@ export interface TermFrequencyAdjustment<R> {
|
|
|
92
99
|
*/
|
|
93
100
|
value(a: R, b: R): string | null | undefined;
|
|
94
101
|
/**
|
|
95
|
-
* Scale the adjustment in [0, 1].
|
|
102
|
+
* Scale the adjustment in [0, 1].
|
|
103
|
+
*
|
|
104
|
+
* Default 1.
|
|
96
105
|
*/
|
|
97
106
|
weight?: number;
|
|
98
107
|
/**
|
|
99
|
-
* Floor for the looked-up frequency, bounding the boost on ultra-rare values.
|
|
108
|
+
* Floor for the looked-up frequency, bounding the boost on ultra-rare values.
|
|
109
|
+
*
|
|
110
|
+
* Default 1e-4.
|
|
100
111
|
*/
|
|
101
112
|
minimumFrequency?: number;
|
|
102
113
|
}
|
|
@@ -148,8 +159,11 @@ export declare function priorWeight(lambda: number): number;
|
|
|
148
159
|
*/
|
|
149
160
|
export declare function probabilityFromWeight(weight: number): number;
|
|
150
161
|
/**
|
|
151
|
-
* A comparison driven by a similarity function and a tier of `minSimilarity`
|
|
152
|
-
*
|
|
162
|
+
* A comparison driven by a similarity function and a tier of `minSimilarity`
|
|
163
|
+
* thresholds (the StatCan/Splink recipe).
|
|
164
|
+
*
|
|
165
|
+
* Levels must be ordered highest → lowest similarity, the last acting as the
|
|
166
|
+
* `different` catch-all (`minSimilarity` 0).
|
|
153
167
|
* A missing value on either side yields no evidence.
|
|
154
168
|
*/
|
|
155
169
|
export declare function similarityComparison<R>(config: {
|
|
@@ -162,12 +176,15 @@ export declare function similarityComparison<R>(config: {
|
|
|
162
176
|
levels: ComparisonLevel[];
|
|
163
177
|
}): Comparison<R>;
|
|
164
178
|
/**
|
|
165
|
-
*
|
|
179
|
+
* Scores a record pair and returns its total match weight and probability.
|
|
180
|
+
* The result also includes per-field contributions.
|
|
166
181
|
*/
|
|
167
182
|
export declare function scorePair<R>(model: FellegiSunterModel<R>, a: R, b: R): PairScore;
|
|
168
183
|
/**
|
|
169
|
-
* Classify a score against upper / lower match-weight thresholds (in bits): at or above `upper` is a link
|
|
170
|
-
*
|
|
184
|
+
* Classify a score against upper / lower match-weight thresholds (in bits): at or above `upper` is a link.
|
|
185
|
+
*
|
|
186
|
+
* A score at or below `lower` is a non-link.
|
|
187
|
+
* The band between them means clerical review (abstain).
|
|
171
188
|
*/
|
|
172
189
|
export declare function decide(score: PairScore, thresholds: {
|
|
173
190
|
upper: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fellegi-sunter.d.ts","sourceRoot":"","sources":["../lib/fellegi-sunter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"fellegi-sunter.d.ts","sourceRoot":"","sources":["../lib/fellegi-sunter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAIH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,CAAC,EAAE,MAAM,CAAA;IACT;;OAEG;IACH,CAAC,EAAE,MAAM,CAAA;IACT;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,CAAC;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,MAAM,EAAE,eAAe,EAAE,CAAA;IACzB;;OAEG;IACH,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,MAAM,CAAA;IAC1B;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,uBAAuB,CAAC,CAAC,CAAC,CAAA;CAC1C;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,uBAAuB,CAAC,CAAC;IACzC;;;;OAIG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAChC;;OAEG;IACH,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;IAC3B;;OAEG;IACH,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IAC5C;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACpC,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAA;IAC5B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,aAAa,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAC5E;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,WAAW,CAAA;AAE5D;;GAEG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CAI1D;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE5D;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,MAAM,EAAE;IAC/C,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACjD;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7C,MAAM,EAAE,eAAe,EAAE,CAAA;CACzB,GAAG,UAAU,CAAC,CAAC,CAAC,CAqBhB;AAED;;;GAGG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,SAAS,CAoChF;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,aAAa,CAMpG"}
|
package/out/fellegi-sunter.js
CHANGED
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
*
|
|
6
6
|
* The Fellegi-Sunter scorer — the matcher's decision layer.
|
|
7
7
|
*
|
|
8
|
-
* Each field comparison
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* Each field comparison assigns a record pair to an _agreement level_: exact, high, low, different, or missing.
|
|
9
|
+
* Each level has two probabilities. `m` is P(this level | the pair really matches).
|
|
10
|
+
* `u` is P(this level | the pair does not match). Their ratio is a Bayes factor. Its
|
|
11
11
|
* log is the level's contribution to the total match weight in bits:
|
|
12
12
|
*
|
|
13
13
|
* ```
|
|
@@ -16,13 +16,12 @@
|
|
|
16
16
|
*
|
|
17
17
|
* — a prior (how likely any two random records match) plus an additive, per-field-attributable
|
|
18
18
|
* stack of evidence. Convert `M` to a probability and threshold it: above the upper bound is a
|
|
19
|
-
* link
|
|
20
|
-
* calibrated abstain zone the whole design leans on.
|
|
19
|
+
* link. Below the lower bound, it is a non-link. The band between them is _clerical review_, the calibrated abstain zone.
|
|
21
20
|
*
|
|
22
|
-
* The `m`/`u` numbers here are
|
|
21
|
+
* The `m`/`u` numbers here are not universal constants. They are estimated from the data — by EM,
|
|
23
22
|
* unsupervised (the next increment) — and the term-frequency adjustment that makes a rare-name
|
|
24
23
|
* agreement count more than a common one layers on top. This module is the deterministic core
|
|
25
|
-
* those build on
|
|
24
|
+
* those build on. Given the levels, it produces the weights and probability. It also produces the decision.
|
|
26
25
|
*/
|
|
27
26
|
import { nameSimilarity } from "#comparators";
|
|
28
27
|
/**
|
|
@@ -50,8 +49,11 @@ export function probabilityFromWeight(weight) {
|
|
|
50
49
|
return 1 / (1 + 2 ** -weight);
|
|
51
50
|
}
|
|
52
51
|
/**
|
|
53
|
-
* A comparison driven by a similarity function and a tier of `minSimilarity`
|
|
54
|
-
*
|
|
52
|
+
* A comparison driven by a similarity function and a tier of `minSimilarity`
|
|
53
|
+
* thresholds (the StatCan/Splink recipe).
|
|
54
|
+
*
|
|
55
|
+
* Levels must be ordered highest → lowest similarity, the last acting as the
|
|
56
|
+
* `different` catch-all (`minSimilarity` 0).
|
|
55
57
|
* A missing value on either side yields no evidence.
|
|
56
58
|
*/
|
|
57
59
|
export function similarityComparison(config) {
|
|
@@ -74,7 +76,8 @@ export function similarityComparison(config) {
|
|
|
74
76
|
};
|
|
75
77
|
}
|
|
76
78
|
/**
|
|
77
|
-
*
|
|
79
|
+
* Scores a record pair and returns its total match weight and probability.
|
|
80
|
+
* The result also includes per-field contributions.
|
|
78
81
|
*/
|
|
79
82
|
export function scorePair(model, a, b) {
|
|
80
83
|
let weight = priorWeight(model.lambda);
|
|
@@ -104,8 +107,10 @@ export function scorePair(model, a, b) {
|
|
|
104
107
|
return { weight, probability: probabilityFromWeight(weight), contributions };
|
|
105
108
|
}
|
|
106
109
|
/**
|
|
107
|
-
* Classify a score against upper / lower match-weight thresholds (in bits): at or above `upper` is a link
|
|
108
|
-
*
|
|
110
|
+
* Classify a score against upper / lower match-weight thresholds (in bits): at or above `upper` is a link.
|
|
111
|
+
*
|
|
112
|
+
* A score at or below `lower` is a non-link.
|
|
113
|
+
* The band between them means clerical review (abstain).
|
|
109
114
|
*/
|
|
110
115
|
export function decide(score, thresholds) {
|
|
111
116
|
if (score.weight >= thresholds.upper)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fellegi-sunter.js","sourceRoot":"","sources":["../lib/fellegi-sunter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"fellegi-sunter.js","sourceRoot":"","sources":["../lib/fellegi-sunter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AA+H7C;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,KAAsB;IACjD,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,MAAc;IACzC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,QAAQ,CAAA;IAEjC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAA;IAEhC,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAA;AACxC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAc;IACnD,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAC9B,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAI,MAQvC;IACA,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,cAAc,CAAA;IAEtD,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,CAAC,CAAC,EAAE,CAAC;YACV,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;YAE5B,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE;gBAAE,OAAO,CAAC,CAAC,CAAA;YAErD,MAAM,GAAG,GAAG,UAAU,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;YAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/C,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,aAAa,IAAI,CAAC,CAAC;oBAAE,OAAO,CAAC,CAAA;YAC5D,CAAC;YAED,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;QAChC,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAI,KAA4B,EAAE,CAAI,EAAE,CAAI;IACpE,IAAI,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IACtC,MAAM,aAAa,GAA+B,EAAE,CAAA;IAEpD,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAErC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAA;YAErE,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,CAAE,CAAA;QACvC,IAAI,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAA;QAE1B,gGAAgG;QAChG,MAAM,EAAE,GAAG,UAAU,CAAC,aAAa,CAAA;QAEnC,IAAI,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;YAE5B,IAAI,KAAK,EAAE,CAAC;gBACX,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,gBAAgB,IAAI,IAAI,CAAC,CAAA;gBAE5E,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBACnB,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,CAAA;gBACvD,CAAC;YACF,CAAC;QACF,CAAC;QAED,MAAM,IAAI,CAAC,CAAA;QACX,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAA;IAC7E,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,qBAAqB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAA;AAC7E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,MAAM,CAAC,KAAgB,EAAE,UAA4C;IACpF,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,CAAC,KAAK;QAAE,OAAO,OAAO,CAAA;IAEpD,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,CAAC,KAAK;QAAE,OAAO,WAAW,CAAA;IAExD,OAAO,QAAQ,CAAA;AAChB,CAAC"}
|
package/out/gbt.d.ts
CHANGED
|
@@ -2,20 +2,6 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Gradient-boosted shallow regression trees (logistic loss), pure-Node — the learned scorer #603
|
|
7
|
-
* names: an offline-trained model (this trainer, or XGBoost/LightGBM exported to the same
|
|
8
|
-
* {@link GBT} shape) plus a trivial evaluator, no new runtime dependency. It sits behind the
|
|
9
|
-
* matcher's `scorer` hook to replace the Fellegi-Sunter link weight where labels (or a held-out
|
|
10
|
-
* truth like an NPI) let a tree learn the over-merge signature the hand-weights miss.
|
|
11
|
-
*
|
|
12
|
-
* This module is feature-agnostic: feature vectors are caller-defined `number[]` (the record
|
|
13
|
-
* matcher builds them in `@mailwoman/registry`'s learned-scorer module — one-hot agreement
|
|
14
|
-
* levels
|
|
15
|
-
*
|
|
16
|
-
* - Interaction terms + corpus statistics). It only fits ({@link trainGBT}) and scores
|
|
17
|
-
* ({@link gbtScore}). The trained {@link GBT} is plain JSON (`{trees, lr, base}`), so a model
|
|
18
|
-
* trains offline once and ships as a data file.
|
|
19
5
|
*/
|
|
20
6
|
/**
|
|
21
7
|
* A trained tree: an internal split (feature `f` ≤ `thr` → `lo`, else `hi`) or a `leaf` value.
|
|
@@ -29,11 +15,13 @@ export type TreeNode = {
|
|
|
29
15
|
hi: TreeNode;
|
|
30
16
|
};
|
|
31
17
|
/**
|
|
32
|
-
*
|
|
18
|
+
* Returns each feature's candidate split thresholds: midpoints between values
|
|
19
|
+
* for features with at most five distinct values.
|
|
20
|
+
* It returns six quantiles otherwise.
|
|
33
21
|
*/
|
|
34
22
|
export declare function buildThresholds(X: number[][]): number[][];
|
|
35
23
|
/**
|
|
36
|
-
* A trained gradient-boosted
|
|
24
|
+
* A trained gradient-boosted tree ensemble whose trees add to a base log-odds, stored as plain JSON.
|
|
37
25
|
*/
|
|
38
26
|
export interface GBT {
|
|
39
27
|
trees: TreeNode[];
|
|
@@ -54,7 +42,9 @@ export interface GBTOpts {
|
|
|
54
42
|
*/
|
|
55
43
|
export declare function trainGBT(X: number[][], y: number[], w: number[], opts: GBTOpts): GBT;
|
|
56
44
|
/**
|
|
57
|
-
*
|
|
45
|
+
* Returns the model's logit for one feature vector.
|
|
46
|
+
*
|
|
47
|
+
* Compare it against a threshold like a Fellegi-Sunter match weight.
|
|
58
48
|
*/
|
|
59
49
|
export declare function gbtScore(m: GBT, x: number[]): number;
|
|
60
50
|
//# sourceMappingURL=gbt.d.ts.map
|
package/out/gbt.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gbt.d.ts","sourceRoot":"","sources":["../lib/gbt.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"gbt.d.ts","sourceRoot":"","sources":["../lib/gbt.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,QAAQ,CAAC;IAAC,EAAE,EAAE,QAAQ,CAAA;CAAE,CAAA;AAIhG;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,EAAE,CA+BzD;AA2FD;;GAEG;AACH,MAAM,WAAW,GAAG;IACnB,KAAK,EAAE,QAAQ,EAAE,CAAA;IACjB,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACvB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,GAAG,GAAG,CAoCpF;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAQpD"}
|
package/out/gbt.js
CHANGED
|
@@ -2,33 +2,14 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Gradient-boosted shallow regression trees (logistic loss), pure-Node — the learned scorer #603
|
|
7
|
-
* names: an offline-trained model (this trainer, or XGBoost/LightGBM exported to the same
|
|
8
|
-
* {@link GBT} shape) plus a trivial evaluator, no new runtime dependency. It sits behind the
|
|
9
|
-
* matcher's `scorer` hook to replace the Fellegi-Sunter link weight where labels (or a held-out
|
|
10
|
-
* truth like an NPI) let a tree learn the over-merge signature the hand-weights miss.
|
|
11
|
-
*
|
|
12
|
-
* This module is feature-agnostic: feature vectors are caller-defined `number[]` (the record
|
|
13
|
-
* matcher builds them in `@mailwoman/registry`'s learned-scorer module — one-hot agreement
|
|
14
|
-
* levels
|
|
15
|
-
*
|
|
16
|
-
* - Interaction terms + corpus statistics). It only fits ({@link trainGBT}) and scores
|
|
17
|
-
* ({@link gbtScore}). The trained {@link GBT} is plain JSON (`{trees, lr, base}`), so a model
|
|
18
|
-
* trains offline once and ships as a data file.
|
|
19
|
-
*/
|
|
20
|
-
/**
|
|
21
|
-
* Distinct values at or below which every split point is tried exactly rather than by quantile.
|
|
22
5
|
*/
|
|
23
6
|
const MAX_EXACT_SPLIT_VALUES = 5;
|
|
24
|
-
/**
|
|
25
|
-
* Quantile split points evaluated for a continuous feature.
|
|
26
|
-
*/
|
|
27
7
|
const QUANTILE_SPLIT_COUNT = 6;
|
|
28
|
-
// Local by design: `@mailwoman/registry`'s tools/shared.ts exports this sigmoid; registry depends on match, not the reverse.
|
|
29
8
|
const sigmoid = (z) => 1 / (1 + Math.exp(-Math.max(-30, Math.min(30, z))));
|
|
30
9
|
/**
|
|
31
|
-
*
|
|
10
|
+
* Returns each feature's candidate split thresholds: midpoints between values
|
|
11
|
+
* for features with at most five distinct values.
|
|
12
|
+
* It returns six quantiles otherwise.
|
|
32
13
|
*/
|
|
33
14
|
export function buildThresholds(X) {
|
|
34
15
|
const dim = X[0]?.length ?? 0;
|
|
@@ -57,9 +38,6 @@ export function buildThresholds(X) {
|
|
|
57
38
|
}
|
|
58
39
|
return out;
|
|
59
40
|
}
|
|
60
|
-
/**
|
|
61
|
-
* Weighted SSE of target `g` over `rows` around their weighted mean.
|
|
62
|
-
*/
|
|
63
41
|
function nodeSSE(rows, g, w) {
|
|
64
42
|
let wsum = 0;
|
|
65
43
|
let wg = 0;
|
|
@@ -75,9 +53,6 @@ function nodeSSE(rows, g, w) {
|
|
|
75
53
|
}
|
|
76
54
|
return sse;
|
|
77
55
|
}
|
|
78
|
-
/**
|
|
79
|
-
* Greedy depth-limited weighted regression tree on target `g` (the boosting residual).
|
|
80
|
-
*/
|
|
81
56
|
function fitRegTree(rows, X, g, w, thresholds, depth, minLeaf) {
|
|
82
57
|
let wsum = 0;
|
|
83
58
|
let wg = 0;
|
|
@@ -145,7 +120,7 @@ export function trainGBT(X, y, w, opts) {
|
|
|
145
120
|
wpos += w[i];
|
|
146
121
|
}
|
|
147
122
|
}
|
|
148
|
-
const base = Math.log((wpos + 1) / (wtot - wpos + 1));
|
|
123
|
+
const base = Math.log((wpos + 1) / (wtot - wpos + 1));
|
|
149
124
|
const F = new Array(N).fill(base);
|
|
150
125
|
const trees = [];
|
|
151
126
|
for (let m = 0; m < opts.rounds; m++) {
|
|
@@ -153,7 +128,6 @@ export function trainGBT(X, y, w, opts) {
|
|
|
153
128
|
for (let i = 0; i < N; i++) {
|
|
154
129
|
g[i] = y[i] - sigmoid(F[i]);
|
|
155
130
|
}
|
|
156
|
-
// negative gradient of logistic loss
|
|
157
131
|
const tree = fitRegTree(rowsAll, X, g, w, thresholds, opts.depth, opts.minLeaf);
|
|
158
132
|
for (let i = 0; i < N; i++) {
|
|
159
133
|
F[i] += opts.lr * predictTree(tree, X[i]);
|
|
@@ -163,7 +137,9 @@ export function trainGBT(X, y, w, opts) {
|
|
|
163
137
|
return { trees, lr: opts.lr, base };
|
|
164
138
|
}
|
|
165
139
|
/**
|
|
166
|
-
*
|
|
140
|
+
* Returns the model's logit for one feature vector.
|
|
141
|
+
*
|
|
142
|
+
* Compare it against a threshold like a Fellegi-Sunter match weight.
|
|
167
143
|
*/
|
|
168
144
|
export function gbtScore(m, x) {
|
|
169
145
|
let f = m.base;
|
package/out/gbt.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gbt.js","sourceRoot":"","sources":["../lib/gbt.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"gbt.js","sourceRoot":"","sources":["../lib/gbt.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAEhC,MAAM,oBAAoB,GAAG,CAAC,CAAA;AAO9B,MAAM,OAAO,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AAE1F;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,CAAa;IAC5C,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAA;IAC7B,MAAM,GAAG,GAAe,EAAE,CAAA;IAE1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;QAChC,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAEzD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACb,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,IAAI,sBAAsB,EAAE,CAAC;YAClD,MAAM,CAAC,GAAa,EAAE,CAAA;YAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,CAAA;YACtC,CAAC;YAED,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAClD,MAAM,CAAC,GAAa,EAAE,CAAA;YAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,oBAAoB,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;YAC3D,CAAC;YAED,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,SAAS,OAAO,CAAC,IAAc,EAAE,CAAW,EAAE,CAAW;IACxD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,EAAE,GAAG,CAAC,CAAA;IAEV,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,CAAE,CAAA;QACb,EAAE,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;IACpB,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IACrC,IAAI,GAAG,GAAG,CAAC,CAAA;IAEX,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,GAAG,IAAI,CAAA;QACtB,GAAG,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,GAAG,CAAC,CAAA;IACrB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,SAAS,UAAU,CAClB,IAAc,EACd,CAAa,EACb,CAAW,EACX,CAAW,EACX,UAAsB,EACtB,KAAa,EACb,OAAe;IAEf,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,EAAE,GAAG,CAAC,CAAA;IAEV,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,CAAE,CAAA;QACb,EAAE,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;IACpB,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAErC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,OAAO;QAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IACrC,IAAI,QAAQ,GAAG,KAAK,CAAA;IACpB,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;IACd,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAa,EAAE,CAAA;IACzB,IAAI,MAAM,GAAa,EAAE,CAAA;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,CAAC,CAAE,EAAE,CAAC;YAClC,MAAM,EAAE,GAAa,EAAE,CAAA;YACvB,MAAM,EAAE,GAAa,EAAE,CAAA;YAEvB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtB,CAAC;gBAAA,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACtC,CAAC;YAED,IAAI,EAAE,CAAC,MAAM,GAAG,OAAO,IAAI,EAAE,CAAC,MAAM,GAAG,OAAO;gBAAE,SAAQ;YACxD,MAAM,IAAI,GAAG,SAAS,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAEhE,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;gBACrB,QAAQ,GAAG,IAAI,CAAA;gBACf,KAAK,GAAG,CAAC,CAAA;gBACT,OAAO,GAAG,GAAG,CAAA;gBACb,MAAM,GAAG,EAAE,CAAA;gBACX,MAAM,GAAG,EAAE,CAAA;YACZ,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAE9B,OAAO;QACN,CAAC,EAAE,KAAK;QACR,GAAG,EAAE,OAAO;QACZ,EAAE,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC;QAC/D,EAAE,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC;KAC/D,CAAA;AACF,CAAC;AAED,SAAS,WAAW,CAAC,CAAW,EAAE,CAAW;IAC5C,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC;QACjB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACnC,CAAC;IAED,OAAO,CAAC,CAAC,IAAI,CAAA;AACd,CAAC;AAqBD;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,CAAa,EAAE,CAAW,EAAE,CAAW,EAAE,IAAa;IAC9E,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAA;IAClB,MAAM,UAAU,GAAG,eAAe,CAAC,CAAC,CAAC,CAAA;IACrC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IACtD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,IAAI,CAAC,CAAC,CAAC,CAAE,CAAA;QAEb,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,CAAC,CAAC,CAAE,CAAA;QACd,CAAC;IACF,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAA;IACrD,MAAM,CAAC,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACzC,MAAM,KAAK,GAAe,EAAE,CAAA;IAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAA;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;QAC9B,CAAC;QAED,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;QAE/E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC,CAAE,IAAI,IAAI,CAAC,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;QAC5C,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjB,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,CAAA;AACpC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,CAAM,EAAE,CAAW;IAC3C,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAA;IAEd,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC9B,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC"}
|
package/out/tf.d.ts
CHANGED
|
@@ -5,15 +5,15 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Term-frequency adjustment — making a rare-value agreement count more than a common one.
|
|
7
7
|
*
|
|
8
|
-
* Two people
|
|
8
|
+
* Two people who share the name "Vijayan" are far stronger evidence of a match than two who share the name "Smith",
|
|
9
9
|
* because "Smith" agreements happen by chance all the time and "Vijayan" agreements don't. The
|
|
10
|
-
* Fellegi-Sunter `m` (how often a true match agrees) is roughly the same either way
|
|
10
|
+
* Fellegi-Sunter `m` (how often a true match agrees) is roughly the same either way. what differs
|
|
11
11
|
* is `u` — the chance a _non_-match agrees — which for an exact agreement on value `v` is just
|
|
12
12
|
* how common `v` is. So we leave `m`, and replace the level's average `u` with `frequency(v)`,
|
|
13
13
|
* adding `log2(u_level / frequency(v))` to the weight: a big positive bump for rare values, a
|
|
14
14
|
* penalty for common ones.
|
|
15
15
|
*
|
|
16
|
-
* Crucially for a label-free matcher: the frequencies are computed
|
|
16
|
+
* Crucially for a label-free matcher: the frequencies are computed on-the-FLY from the input column
|
|
17
17
|
* (the Splink approach) — no external Census table required. Build a {@link TermFrequencyTable}
|
|
18
18
|
* from the values you're matching, then attach it to a comparison with {@link withTermFrequency}.
|
|
19
19
|
*/
|
|
@@ -36,32 +36,43 @@ export interface TermFrequencyTable {
|
|
|
36
36
|
readonly distinct: number;
|
|
37
37
|
}
|
|
38
38
|
/**
|
|
39
|
-
* Build a {@link TermFrequencyTable} from an iterable of values (e.g.
|
|
40
|
-
*
|
|
41
|
-
*
|
|
39
|
+
* Build a {@link TermFrequencyTable} from an iterable of values (e.g. Every `given` name in the dataset).
|
|
40
|
+
*
|
|
41
|
+
* Values are normalized (default: trim + lowercase + collapse whitespace) before counting.
|
|
42
|
+
* `frequency()` normalizes its argument the same way, so callers pass raw field values.
|
|
42
43
|
*/
|
|
43
44
|
export declare function buildTermFrequencyTable(values: Iterable<string | null | undefined>, opts?: {
|
|
44
45
|
normalize?: (value: string) => string;
|
|
45
46
|
}): TermFrequencyTable;
|
|
46
47
|
/**
|
|
47
|
-
* Attach a term-frequency adjustment to a comparison.
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
48
|
+
* Attach a term-frequency adjustment to a comparison.
|
|
49
|
+
*
|
|
50
|
+
* By default it applies to the exact level (index 0) and looks up the value via
|
|
51
|
+
* `value(a, b)` — usually the agreeing field extracted from one side.
|
|
52
|
+
* Returns a new comparison.
|
|
53
|
+
*
|
|
54
|
+
* The underlying `assess` and levels are untouched, so this composes with EM
|
|
55
|
+
* (which re-estimates the base `m`/`u` the adjustment sits on top of).
|
|
51
56
|
*/
|
|
52
57
|
export declare function withTermFrequency<R>(comparison: Comparison<R>, config: {
|
|
53
58
|
table: TermFrequencyTable;
|
|
54
59
|
value: (a: R, b: R) => string | null | undefined;
|
|
55
60
|
/**
|
|
56
|
-
* Level indices to adjust.
|
|
61
|
+
* Level indices to adjust.
|
|
62
|
+
*
|
|
63
|
+
* Default `[0]` (the exact level).
|
|
57
64
|
*/
|
|
58
65
|
levels?: Iterable<number>;
|
|
59
66
|
/**
|
|
60
|
-
* Scale in [0, 1].
|
|
67
|
+
* Scale in [0, 1].
|
|
68
|
+
*
|
|
69
|
+
* Default 1.
|
|
61
70
|
*/
|
|
62
71
|
weight?: number;
|
|
63
72
|
/**
|
|
64
|
-
* Frequency floor bounding the boost on ultra-rare values.
|
|
73
|
+
* Frequency floor bounding the boost on ultra-rare values.
|
|
74
|
+
*
|
|
75
|
+
* Default 1e-4.
|
|
65
76
|
*/
|
|
66
77
|
minimumFrequency?: number;
|
|
67
78
|
}): Comparison<R>;
|
package/out/tf.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tf.d.ts","sourceRoot":"","sources":["../lib/tf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAA2B,MAAM,iBAAiB,CAAA;AAE1E;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAChC;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CACzB;
|
|
1
|
+
{"version":3,"file":"tf.d.ts","sourceRoot":"","sources":["../lib/tf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAA2B,MAAM,iBAAiB,CAAA;AAE1E;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAChC;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CACzB;AAMD;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACtC,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,EAC3C,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAA;CAAO,GAClD,kBAAkB,CAwBpB;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAClC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EACzB,MAAM,EAAE;IACP,KAAK,EAAE,kBAAkB,CAAA;IACzB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IAChD;;;;OAIG;IACH,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;IACzB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB,GACC,UAAU,CAAC,CAAC,CAAC,CAUf"}
|