@natlibfi/melinda-record-matching 2.0.1-alpha.1 → 2.0.7-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CODEOWNERS +2 -0
- package/.github/dependabot.yml +40 -0
- package/.github/workflows/melinda-node-tests.yml +70 -0
- package/README.md +1 -1
- package/dist/candidate-search/candidate-search-utils.js +3 -6
- package/dist/candidate-search/candidate-search-utils.js.map +1 -1
- package/dist/candidate-search/index.js +115 -50
- package/dist/candidate-search/index.js.map +1 -1
- package/dist/candidate-search/index.spec.js +41 -48
- package/dist/candidate-search/index.spec.js.map +1 -1
- package/dist/candidate-search/query-list/bib.js +154 -64
- package/dist/candidate-search/query-list/bib.js.map +1 -1
- package/dist/candidate-search/query-list/bib.spec.js +1 -10
- package/dist/candidate-search/query-list/bib.spec.js.map +1 -1
- package/dist/candidate-search/query-list/index.js +4 -10
- package/dist/candidate-search/query-list/index.js.map +1 -1
- package/dist/index.js +498 -50
- package/dist/index.js.map +1 -1
- package/dist/index.spec.js +60 -34
- package/dist/index.spec.js.map +1 -1
- package/dist/match-detection/features/bib/all-source-ids.js +16 -19
- package/dist/match-detection/features/bib/all-source-ids.js.map +1 -1
- package/dist/match-detection/features/bib/authors.js +31 -35
- package/dist/match-detection/features/bib/authors.js.map +1 -1
- package/dist/match-detection/features/bib/bibliographic-level.js +3 -3
- package/dist/match-detection/features/bib/bibliographic-level.js.map +1 -1
- package/dist/match-detection/features/bib/host-component.js +3 -3
- package/dist/match-detection/features/bib/host-component.js.map +1 -1
- package/dist/match-detection/features/bib/index.js +28 -13
- package/dist/match-detection/features/bib/index.js.map +1 -1
- package/dist/match-detection/features/bib/index.spec.js +9 -13
- package/dist/match-detection/features/bib/index.spec.js.map +1 -1
- package/dist/match-detection/features/bib/isbn.js +28 -6
- package/dist/match-detection/features/bib/isbn.js.map +1 -1
- package/dist/match-detection/features/bib/issn.js +0 -4
- package/dist/match-detection/features/bib/issn.js.map +1 -1
- package/dist/match-detection/features/bib/language.js +69 -41
- package/dist/match-detection/features/bib/language.js.map +1 -1
- package/dist/match-detection/features/bib/media-type.js +83 -0
- package/dist/match-detection/features/bib/media-type.js.map +1 -0
- package/dist/match-detection/features/bib/melinda-id.js +0 -4
- package/dist/match-detection/features/bib/melinda-id.js.map +1 -1
- package/dist/match-detection/features/bib/melinda-identifier-factory.js +6 -17
- package/dist/match-detection/features/bib/melinda-identifier-factory.js.map +1 -1
- package/dist/match-detection/features/bib/other-standard-identifier.js +1 -4
- package/dist/match-detection/features/bib/other-standard-identifier.js.map +1 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +256 -0
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +1 -0
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js +71 -0
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map +1 -0
- package/dist/match-detection/features/bib/publication-time.js +9 -8
- package/dist/match-detection/features/bib/publication-time.js.map +1 -1
- package/dist/match-detection/features/bib/record-type.js +6 -3
- package/dist/match-detection/features/bib/record-type.js.map +1 -1
- package/dist/match-detection/features/bib/standard-identifier-factory.js +129 -39
- package/dist/match-detection/features/bib/standard-identifier-factory.js.map +1 -1
- package/dist/match-detection/features/bib/title-version-original.js +79 -0
- package/dist/match-detection/features/bib/title-version-original.js.map +1 -0
- package/dist/match-detection/features/bib/title.js +29 -22
- package/dist/match-detection/features/bib/title.js.map +1 -1
- package/dist/match-detection/features/index.js +0 -4
- package/dist/match-detection/features/index.js.map +1 -1
- package/dist/match-detection/index.js +94 -47
- package/dist/match-detection/index.js.map +1 -1
- package/dist/match-detection/index.spec.js +25 -15
- package/dist/match-detection/index.spec.js.map +1 -1
- package/dist/matching-utils.js +63 -15
- package/dist/matching-utils.js.map +1 -1
- package/package.json +27 -24
- package/src/candidate-search/index.js +93 -31
- package/src/candidate-search/index.spec.js +27 -24
- package/src/candidate-search/query-list/bib.js +149 -22
- package/src/candidate-search/query-list/index.js +1 -1
- package/src/index.js +310 -38
- package/src/index.spec.js +49 -21
- package/src/match-detection/features/bib/all-source-ids.js +3 -3
- package/src/match-detection/features/bib/authors.js +23 -16
- package/src/match-detection/features/bib/bibliographic-level.js +1 -1
- package/src/match-detection/features/bib/host-component.js +1 -1
- package/src/match-detection/features/bib/index.js +4 -0
- package/src/match-detection/features/bib/index.spec.js +10 -2
- package/src/match-detection/features/bib/isbn.js +21 -2
- package/src/match-detection/features/bib/language.js +63 -22
- package/src/match-detection/features/bib/media-type.js +77 -0
- package/src/match-detection/features/bib/melinda-identifier-factory.js +2 -3
- package/src/match-detection/features/bib/other-standard-identifier.js +2 -0
- package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +268 -0
- package/src/match-detection/features/bib/publication-time-allow-cons-years.js +67 -0
- package/src/match-detection/features/bib/publication-time.js +9 -4
- package/src/match-detection/features/bib/record-type.js +5 -1
- package/src/match-detection/features/bib/standard-identifier-factory.js +107 -22
- package/src/match-detection/features/bib/title-version-original.js +78 -0
- package/src/match-detection/features/bib/title.js +27 -7
- package/src/match-detection/index.js +60 -22
- package/src/match-detection/index.spec.js +18 -4
- package/src/matching-utils.js +62 -5
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
/* eslint-disable max-statements */
|
|
1
2
|
/**
|
|
2
3
|
*
|
|
3
4
|
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
5
|
*
|
|
5
6
|
* Melinda record matching modules for Javascript
|
|
6
7
|
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
8
|
+
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
8
9
|
*
|
|
9
10
|
* This file is part of melinda-record-matching-js
|
|
10
11
|
*
|
|
@@ -27,58 +28,142 @@
|
|
|
27
28
|
*/
|
|
28
29
|
|
|
29
30
|
import createDebugLogger from 'debug';
|
|
31
|
+
import {extractSubfieldsFromField, uniqueSubfields} from '../../../matching-utils';
|
|
30
32
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
+
// Note about validity of standardIdentifiers:
|
|
34
|
+
// We have three types of invalid standardIdentifiers:
|
|
35
|
+
// 1. Formally invalid standardIdentifiers (ie. typos either in the resource or the record)
|
|
36
|
+
// 2. Formally valid standardIdentifiers that are used in a wrong resource
|
|
37
|
+
// 3. Canceled standardIdentifiers
|
|
38
|
+
|
|
39
|
+
// Matcher could and should check that a standardIdentifier found in a subfield for a valid identifier is formally valid, and if it's not formally valid, handle it as an invalid standardIdentifier
|
|
40
|
+
// Formally valid standardIdentifiers found in subfield for invalid identifier cannot be handled as valid standardIdentifiers, because they can be a case of type 2) or 3) invalid standardIdentifiers
|
|
41
|
+
// We could also do a separate handling for formally valid an formally invalid standardIdentifiers
|
|
42
|
+
|
|
43
|
+
export default ({pattern, subfieldCodes, identifier, validIdentifierSubfieldCodes = ['a'], invalidIdentifierSubfieldCodes = ['z'], validatorAndNormalizer = undefined}) => {
|
|
44
|
+
const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);
|
|
45
|
+
const debugData = debug.extend('data');
|
|
33
46
|
|
|
34
|
-
export default ({pattern, subfieldCodes}) => {
|
|
35
47
|
return {extract, compare};
|
|
36
48
|
|
|
37
|
-
function extract(record) {
|
|
38
|
-
const
|
|
49
|
+
function extract({record, recordExternal}) {
|
|
50
|
+
const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
|
|
51
|
+
const fields = record.get(pattern);
|
|
52
|
+
debugData(`${label}: ${fields.length} ${identifier}-fields `);
|
|
53
|
+
|
|
54
|
+
// extractIdentifierSubfield normalizes hyphens away from the subfield values
|
|
55
|
+
const identifiersFromFields = fields.map(field => extractSubfieldsFromField(field, subfieldCodes));
|
|
56
|
+
debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);
|
|
57
|
+
const allIdentifiers = identifiersFromFields.flat();
|
|
58
|
+
debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);
|
|
59
|
+
|
|
60
|
+
const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({identifierSubs: allIdentifiers, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes});
|
|
39
61
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
62
|
+
const identifiers = uniqueSubfields(validatedAndNormalizedIdentifiers);
|
|
63
|
+
|
|
64
|
+
debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);
|
|
65
|
+
return identifiers;
|
|
66
|
+
|
|
67
|
+
function validateAndNormalizeIdentifiers({identifierSubs, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {
|
|
68
|
+
if (validatorAndNormalizer) {
|
|
69
|
+
return identifierSubs.map((idSub) => validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}));
|
|
70
|
+
}
|
|
71
|
+
return identifierSubs.map((idSub) => normalizeHyphens(idSub));
|
|
44
72
|
}
|
|
45
73
|
|
|
46
|
-
|
|
74
|
+
function validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {
|
|
75
|
+
const {valid, value} = validatorAndNormalizer(idSub.value);
|
|
76
|
+
if (validIdentifierSubfieldCodes.includes(idSub.code) && valid === false) {
|
|
77
|
+
const [code] = invalidIdentifierSubfieldCodes;
|
|
78
|
+
return {code, value};
|
|
79
|
+
}
|
|
80
|
+
return {code: idSub.code, value};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function normalizeHyphens(idSub) {
|
|
84
|
+
return {code: idSub.code, value: idSub.value.replace(/-/ug, '')};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
|
|
47
88
|
}
|
|
48
89
|
|
|
49
90
|
function compare(a, b) {
|
|
91
|
+
debug(`Comparing A and B`);
|
|
50
92
|
if (a.length === 0 || b.length === 0) {
|
|
51
|
-
debugData(`No standardidentifiers to compare`);
|
|
93
|
+
debugData(`No standardidentifiers (${identifier}) to compare`);
|
|
52
94
|
return 0;
|
|
53
95
|
}
|
|
54
96
|
|
|
97
|
+
debugData(`A: ${JSON.stringify(a)}`);
|
|
98
|
+
debugData(`B: ${JSON.stringify(b)}`);
|
|
99
|
+
|
|
100
|
+
|
|
55
101
|
if (bothHaveValidIdentifiers()) {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
102
|
+
// Compare only valid identifiers, if both have valid idenfiers
|
|
103
|
+
const {maxValues, possibleMatchValues, matchingValues} = getValueCount(true);
|
|
104
|
+
if (matchingValues < 1) {
|
|
105
|
+
debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);
|
|
106
|
+
return -0.75;
|
|
107
|
+
}
|
|
108
|
+
debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues}/${possibleMatchValues} valid identifiers match.`);
|
|
109
|
+
// ignore non-matches if there is mismatching amount of values
|
|
110
|
+
debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);
|
|
111
|
+
//we give some kind of penalty for mismatching amount of values instead of simple divide?
|
|
112
|
+
const penaltyForMissing = 0.1 * (maxValues - possibleMatchValues);
|
|
113
|
+
const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues);
|
|
114
|
+
debug(`\t points: penaltyForMissing: ${penaltyForMissing}`);
|
|
115
|
+
debug(`\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);
|
|
59
116
|
|
|
117
|
+
return 0.75 - penaltyForMisMatch - penaltyForMissing;
|
|
118
|
+
//return matchingValues / possibleMatchValues * 0.75;
|
|
119
|
+
}
|
|
120
|
+
// If both do not have valid identifiers, compare all identifiers
|
|
60
121
|
const {maxValues, matchingValues} = getValueCount();
|
|
122
|
+
debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);
|
|
123
|
+
|
|
61
124
|
return matchingValues / maxValues * 0.2;
|
|
62
125
|
|
|
63
126
|
function bothHaveValidIdentifiers() {
|
|
64
|
-
const aValues = a.filter(({code}) => code
|
|
65
|
-
const bValues = a.filter(({code}) => code
|
|
127
|
+
const aValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));
|
|
128
|
+
const bValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));
|
|
129
|
+
debug(`A: ${aValues.length} valid ${identifier} identifiers`);
|
|
130
|
+
debug(`B: ${bValues.length} valid ${identifier} identifiers`);
|
|
66
131
|
return aValues.length > 0 && bValues.length > 0;
|
|
67
132
|
}
|
|
68
133
|
|
|
69
134
|
function getValueCount(validOnly = false) {
|
|
70
|
-
const aValues = getIdentifiers(a);
|
|
71
|
-
const bValues = getIdentifiers(b);
|
|
135
|
+
const aValues = getIdentifiers(a, validOnly);
|
|
136
|
+
const bValues = getIdentifiers(b, validOnly);
|
|
137
|
+
|
|
138
|
+
const matchingValues = getMatchingValuesAmount(aValues, bValues);
|
|
72
139
|
|
|
73
140
|
return {
|
|
74
141
|
maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,
|
|
75
|
-
|
|
142
|
+
// possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)
|
|
143
|
+
possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,
|
|
144
|
+
matchingValues
|
|
76
145
|
};
|
|
77
146
|
|
|
78
|
-
function
|
|
147
|
+
function getMatchingValuesAmount(aValues, bValues) {
|
|
148
|
+
if (bValues.length > aValues.length) {
|
|
149
|
+
return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
|
|
150
|
+
}
|
|
151
|
+
if (aValues.length > bValues.length) {
|
|
152
|
+
return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// If we have same amount of values, we'll check matches both ways, to avoid mixups in cases
|
|
156
|
+
// there would be duplicate values
|
|
157
|
+
const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
|
|
158
|
+
const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
|
|
159
|
+
|
|
160
|
+
return aToB < bToA ? aToB : bToA;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function getIdentifiers(values, validOnly) {
|
|
79
164
|
if (validOnly) {
|
|
80
165
|
return values
|
|
81
|
-
.filter(({code}) => code
|
|
166
|
+
.filter(({code}) => validIdentifierSubfieldCodes.includes(code))
|
|
82
167
|
.map(({value}) => value);
|
|
83
168
|
}
|
|
84
169
|
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
*
|
|
3
|
+
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
+
*
|
|
5
|
+
* Melinda record matching modules for Javascript
|
|
6
|
+
*
|
|
7
|
+
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
8
|
+
*
|
|
9
|
+
* This file is part of melinda-record-matching-js
|
|
10
|
+
*
|
|
11
|
+
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
+
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
+
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
+
* License, or (at your option) any later version.
|
|
15
|
+
*
|
|
16
|
+
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
+
* GNU Lesser General Public License for more details.
|
|
20
|
+
*
|
|
21
|
+
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
+
*
|
|
24
|
+
* @licend The above is the entire license notice
|
|
25
|
+
* for the JavaScript code in this file.
|
|
26
|
+
*
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import createDebugLogger from 'debug';
|
|
30
|
+
import {LevenshteinDistance as leven} from 'natural';
|
|
31
|
+
|
|
32
|
+
export default ({treshold = 10} = {}) => ({
|
|
33
|
+
name: 'titleVersionOriginal',
|
|
34
|
+
extract: ({record}) => {
|
|
35
|
+
const title = getTitle();
|
|
36
|
+
|
|
37
|
+
if (title) {
|
|
38
|
+
return [title.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return [];
|
|
42
|
+
|
|
43
|
+
function getTitle() {
|
|
44
|
+
const [field] = record.get(/^245$/u);
|
|
45
|
+
|
|
46
|
+
if (field) {
|
|
47
|
+
return field.subfields
|
|
48
|
+
.filter(({code}) => ['a', 'b'].includes(code))
|
|
49
|
+
.map(({value}) => value)
|
|
50
|
+
.join('');
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
compare: (a, b) => {
|
|
55
|
+
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');
|
|
56
|
+
const distance = leven(a[0], b[0]);
|
|
57
|
+
|
|
58
|
+
if (distance === 0) {
|
|
59
|
+
return 0.5;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const maxLength = getMaxLength();
|
|
63
|
+
const percentage = distance / maxLength * 100;
|
|
64
|
+
|
|
65
|
+
debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
|
|
66
|
+
|
|
67
|
+
if (percentage <= treshold) {
|
|
68
|
+
return 0.3;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return -0.5;
|
|
72
|
+
|
|
73
|
+
function getMaxLength() {
|
|
74
|
+
return a[0].length > b[0].length ? a[0].length : b[0].length;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
}
|
|
78
|
+
});
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Melinda record matching modules for Javascript
|
|
6
6
|
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
7
|
+
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
8
8
|
*
|
|
9
9
|
* This file is part of melinda-record-matching-js
|
|
10
10
|
*
|
|
@@ -26,29 +26,49 @@
|
|
|
26
26
|
*
|
|
27
27
|
*/
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
|
|
30
30
|
import {LevenshteinDistance as leven} from 'natural';
|
|
31
|
+
import {testStringOrNumber} from '../../../matching-utils';
|
|
32
|
+
import createDebugLogger from 'debug';
|
|
33
|
+
|
|
34
|
+
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');
|
|
35
|
+
const debugData = debug.extend('data');
|
|
36
|
+
|
|
31
37
|
|
|
32
38
|
export default ({treshold = 10} = {}) => ({
|
|
33
39
|
name: 'Title',
|
|
34
|
-
extract: record => {
|
|
40
|
+
extract: ({record, recordExternal}) => {
|
|
41
|
+
const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
|
|
35
42
|
const title = getTitle();
|
|
43
|
+
debug(`${label} title: ${title}`);
|
|
36
44
|
|
|
37
|
-
if (title) {
|
|
38
|
-
|
|
45
|
+
if (testStringOrNumber(title)) {
|
|
46
|
+
const titleAsNormalizedString = String(title)
|
|
47
|
+
// decompose unicode diacritics
|
|
48
|
+
.normalize('NFD')
|
|
49
|
+
// strip non-letters/numbers
|
|
50
|
+
// - note: combined with decomposing unicode diactics this normalizes both 'saa' and 'sää' as 'saa'
|
|
51
|
+
// - we could precompose the finnish letters back to avoid this
|
|
52
|
+
.replace(/[^\p{Letter}\p{Number}]/gu, '')
|
|
53
|
+
.toLowerCase();
|
|
54
|
+
debug(`${label} titleString: ${titleAsNormalizedString}`);
|
|
55
|
+
return [titleAsNormalizedString];
|
|
39
56
|
}
|
|
40
57
|
|
|
41
58
|
return [];
|
|
42
59
|
|
|
43
60
|
function getTitle() {
|
|
44
61
|
const [field] = record.get(/^245$/u);
|
|
62
|
+
debugData(`${label} titleField: ${JSON.stringify(field)}`);
|
|
45
63
|
|
|
46
64
|
if (field) {
|
|
47
65
|
return field.subfields
|
|
48
|
-
|
|
49
|
-
.
|
|
66
|
+
// get also $n:s and $p:s here
|
|
67
|
+
.filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))
|
|
68
|
+
.map(({value}) => testStringOrNumber(value) ? String(value) : '')
|
|
50
69
|
.join('');
|
|
51
70
|
}
|
|
71
|
+
return false;
|
|
52
72
|
}
|
|
53
73
|
},
|
|
54
74
|
compare: (a, b) => {
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/* eslint-disable no-console */
|
|
2
1
|
/**
|
|
3
2
|
*
|
|
4
3
|
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
@@ -32,40 +31,78 @@ import * as features from './features';
|
|
|
32
31
|
|
|
33
32
|
export {features};
|
|
34
33
|
|
|
35
|
-
|
|
36
|
-
export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
|
|
34
|
+
export default ({strategy, treshold = 0.9}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
|
|
37
35
|
const minProbabilityQuantifier = 0.5;
|
|
38
36
|
|
|
39
37
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');
|
|
40
38
|
const debugData = debug.extend('data');
|
|
41
|
-
const featuresA = extractFeatures(recordA);
|
|
42
|
-
const featuresB = extractFeatures(recordB);
|
|
43
39
|
|
|
44
|
-
debugData(`
|
|
45
|
-
debugData(`
|
|
40
|
+
debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
|
|
41
|
+
debugData(`Records: A: ${recordA}\nB: ${recordB}`);
|
|
42
|
+
debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);
|
|
43
|
+
// We could add here labels for records if we didn't get external labels
|
|
46
44
|
|
|
47
|
-
const
|
|
48
|
-
const similarityVector = generateSimilarityVector();
|
|
45
|
+
const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});
|
|
49
46
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
47
|
+
debug(`We got an array of records: ${Array.isArray(recordB)}`);
|
|
48
|
+
const recordsB = Array.isArray(recordB) ? recordB : [recordB];
|
|
49
|
+
const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];
|
|
50
|
+
|
|
51
|
+
const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));
|
|
52
|
+
|
|
53
|
+
// if we got array of records, we return an array of result
|
|
54
|
+
// if we got a singular record, we return a singular result
|
|
55
|
+
return Array.isArray(recordB) ? detectionResults : detectionResults[0];
|
|
56
|
+
|
|
57
|
+
function actualDetection({featuresA, record, recordExternal, index, localSettings}) {
|
|
58
|
+
const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';
|
|
59
|
+
const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
debug(`Actual detection for record ${index + 1} ${labelB}`);
|
|
63
|
+
const featuresB = extractFeatures({record, recordExternal, localSettings});
|
|
64
|
+
|
|
65
|
+
debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);
|
|
66
|
+
debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);
|
|
67
|
+
|
|
68
|
+
const featurePairs = generateFeaturePairs(featuresA, featuresB);
|
|
69
|
+
const similarityVector = generateSimilarityVector(featurePairs);
|
|
70
|
+
|
|
71
|
+
if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
|
|
72
|
+
const probability = calculateProbability(similarityVector);
|
|
73
|
+
debug(`probability: ${probability} (Treshold: ${treshold})`);
|
|
74
|
+
return returnResult({match: probability >= treshold, probability});
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
|
|
78
|
+
return returnResult({match: false, probability: 0.0});
|
|
54
79
|
}
|
|
55
80
|
|
|
56
|
-
|
|
57
|
-
|
|
81
|
+
function extractFeatures({record, recordExternal}) {
|
|
82
|
+
return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);
|
|
83
|
+
}
|
|
58
84
|
|
|
59
|
-
function
|
|
60
|
-
|
|
61
|
-
|
|
85
|
+
function returnResult(result) {
|
|
86
|
+
if (returnStrategy) {
|
|
87
|
+
debug(`Returning detection strategy with the result`);
|
|
88
|
+
const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold};
|
|
89
|
+
debugData(`${JSON.stringify(resultWithStrategy)}`);
|
|
90
|
+
return resultWithStrategy;
|
|
91
|
+
}
|
|
92
|
+
return result;
|
|
62
93
|
}
|
|
63
94
|
|
|
64
|
-
function
|
|
65
|
-
|
|
95
|
+
function formatStrategy(strategy) {
|
|
96
|
+
const strategyNames = strategy.map(element => element.name);
|
|
97
|
+
return strategyNames || [];
|
|
66
98
|
}
|
|
67
99
|
|
|
68
|
-
function
|
|
100
|
+
function calculateProbability(similarityVector) {
|
|
101
|
+
const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);
|
|
102
|
+
return probability > 1.0 ? 1.0 : probability;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function generateSimilarityVector(featurePairs) {
|
|
69
106
|
const compared = featurePairs.map(({name, a, b}) => {
|
|
70
107
|
const {compare} = strategy.find(({name: featureName}) => name === featureName);
|
|
71
108
|
const points = compare(a, b);
|
|
@@ -76,7 +113,7 @@ export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
|
|
|
76
113
|
return compared.map(({points}) => points);
|
|
77
114
|
}
|
|
78
115
|
|
|
79
|
-
function generateFeaturePairs() {
|
|
116
|
+
function generateFeaturePairs(featuresA, featuresB) {
|
|
80
117
|
const pairs = generatePairs();
|
|
81
118
|
const missingFeatures = findMissing();
|
|
82
119
|
|
|
@@ -105,4 +142,5 @@ export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
|
|
|
105
142
|
.filter(v => pairs.some(({name}) => name === v) === false);
|
|
106
143
|
}
|
|
107
144
|
}
|
|
145
|
+
|
|
108
146
|
};
|
|
@@ -32,6 +32,11 @@ import {expect} from 'chai';
|
|
|
32
32
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
33
33
|
import * as features from './features';
|
|
34
34
|
import createDetectionInterface from '.';
|
|
35
|
+
import {inspect} from 'util';
|
|
36
|
+
import createDebugLogger from 'debug';
|
|
37
|
+
|
|
38
|
+
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:test');
|
|
39
|
+
const debugData = debug.extend('data');
|
|
35
40
|
|
|
36
41
|
describe('match-detection', () => {
|
|
37
42
|
generateTests({
|
|
@@ -41,16 +46,25 @@ describe('match-detection', () => {
|
|
|
41
46
|
fixura: {
|
|
42
47
|
reader: READERS.JSON
|
|
43
48
|
},
|
|
44
|
-
callback: ({getFixture, options, expectedResults, enabled = true}) => {
|
|
49
|
+
callback: ({getFixture, options, expectedResults, array, enabled = true}) => {
|
|
45
50
|
|
|
46
51
|
if (!enabled) {
|
|
52
|
+
debug(`*** DISABLED TEST! ***`);
|
|
47
53
|
return;
|
|
48
54
|
}
|
|
49
55
|
|
|
50
56
|
const detect = createDetectionInterface(formatOptions());
|
|
51
|
-
const recordA = new MarcRecord(getFixture('recordA.json'));
|
|
52
|
-
|
|
53
|
-
|
|
57
|
+
const recordA = new MarcRecord(getFixture('recordA.json'), {subfieldValues: false});
|
|
58
|
+
debugData(inspect(recordA));
|
|
59
|
+
|
|
60
|
+
debug(`Our recordB is an array of records: ${array}`);
|
|
61
|
+
const recordB = array
|
|
62
|
+
? getFixture('recordB.json').map(recordJson => new MarcRecord(recordJson, {subfieldValues: false}))
|
|
63
|
+
: new MarcRecord(getFixture('recordB.json'), {subfieldValues: false});
|
|
64
|
+
debugData(inspect(recordB));
|
|
65
|
+
|
|
66
|
+
const results = detect({recordA, recordB});
|
|
67
|
+
debugData(`${JSON.stringify(results)}`);
|
|
54
68
|
|
|
55
69
|
expect(results).to.eql(expectedResults);
|
|
56
70
|
|
package/src/matching-utils.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Melinda record matching modules for Javascript
|
|
6
6
|
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
7
|
+
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
8
8
|
*
|
|
9
9
|
* This file is part of melinda-record-matching-js
|
|
10
10
|
*
|
|
@@ -31,7 +31,6 @@ import createDebugLogger from 'debug';
|
|
|
31
31
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');
|
|
32
32
|
const debugData = debug.extend('data');
|
|
33
33
|
|
|
34
|
-
// eslint-disable-next-line max-statements
|
|
35
34
|
export function getMelindaIdsF035(record) {
|
|
36
35
|
|
|
37
36
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');
|
|
@@ -58,8 +57,8 @@ export function getMelindaIdsF035(record) {
|
|
|
58
57
|
|
|
59
58
|
return subfields
|
|
60
59
|
.filter(sub => ['a', 'z'].includes(sub.code))
|
|
61
|
-
.filter(sub => melindaIdRegExp.test(sub.value))
|
|
62
|
-
.map(({value}) => value.replace(melindaIdRegExp, '$<id>'));
|
|
60
|
+
.filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))
|
|
61
|
+
.map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');
|
|
63
62
|
|
|
64
63
|
}
|
|
65
64
|
}
|
|
@@ -81,6 +80,64 @@ function countSubfields(field, subfieldCode) {
|
|
|
81
80
|
|
|
82
81
|
export function getSubfieldValues(field, subfieldCode) {
|
|
83
82
|
debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);
|
|
84
|
-
return field.subfields
|
|
83
|
+
return field.subfields
|
|
84
|
+
.filter(({code}) => code === subfieldCode)
|
|
85
|
+
.map(({value}) => testStringOrNumber(value) ? String(value) : '')
|
|
86
|
+
.filter(value => value);
|
|
85
87
|
}
|
|
86
88
|
|
|
89
|
+
export function testStringOrNumber(value) {
|
|
90
|
+
if (typeof value === 'string' || typeof value === 'number') {
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function extractSubfieldsFromField(field, subfieldCodes) {
|
|
97
|
+
if (field === undefined || field.subfields === undefined) {
|
|
98
|
+
return [];
|
|
99
|
+
}
|
|
100
|
+
const resultSubfields = field.subfields
|
|
101
|
+
.filter(({code}) => subfieldCodes.includes(code))
|
|
102
|
+
.map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))
|
|
103
|
+
.filter(value => value);
|
|
104
|
+
return resultSubfields;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function uniqueSubfields(subfields) {
|
|
108
|
+
return subfields.reduce((arr, e) => {
|
|
109
|
+
if (!arr.find(item => item.code === e.code && item.value === e.value)) {
|
|
110
|
+
const newArr = arr.concat(e);
|
|
111
|
+
return newArr;
|
|
112
|
+
}
|
|
113
|
+
return arr;
|
|
114
|
+
}, []);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function getMatchCounts(aValues, bValues) {
|
|
118
|
+
|
|
119
|
+
const matchingValues = getMatchingValuesAmount(aValues, bValues);
|
|
120
|
+
|
|
121
|
+
return {
|
|
122
|
+
maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,
|
|
123
|
+
// possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)
|
|
124
|
+
possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,
|
|
125
|
+
matchingValues
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
function getMatchingValuesAmount(aValues, bValues) {
|
|
129
|
+
if (bValues.length > aValues.length) {
|
|
130
|
+
return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
|
|
131
|
+
}
|
|
132
|
+
if (aValues.length > bValues.length) {
|
|
133
|
+
return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// If we have same amount of values, we'll check matches both ways, to avoid mixups in cases
|
|
137
|
+
// there would be duplicate values
|
|
138
|
+
const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
|
|
139
|
+
const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
|
|
140
|
+
|
|
141
|
+
return aToB < bToA ? aToB : bToA;
|
|
142
|
+
}
|
|
143
|
+
}
|