@natlibfi/melinda-record-matching 2.0.1 → 2.0.7-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/.github/CODEOWNERS +2 -0
  2. package/.github/dependabot.yml +40 -0
  3. package/.github/workflows/melinda-node-tests.yml +70 -0
  4. package/README.md +1 -1
  5. package/dist/candidate-search/candidate-search-utils.js +3 -6
  6. package/dist/candidate-search/candidate-search-utils.js.map +1 -1
  7. package/dist/candidate-search/index.js +115 -50
  8. package/dist/candidate-search/index.js.map +1 -1
  9. package/dist/candidate-search/index.spec.js +41 -48
  10. package/dist/candidate-search/index.spec.js.map +1 -1
  11. package/dist/candidate-search/query-list/bib.js +154 -64
  12. package/dist/candidate-search/query-list/bib.js.map +1 -1
  13. package/dist/candidate-search/query-list/bib.spec.js +1 -10
  14. package/dist/candidate-search/query-list/bib.spec.js.map +1 -1
  15. package/dist/candidate-search/query-list/index.js +4 -10
  16. package/dist/candidate-search/query-list/index.js.map +1 -1
  17. package/dist/index.js +498 -50
  18. package/dist/index.js.map +1 -1
  19. package/dist/index.spec.js +60 -34
  20. package/dist/index.spec.js.map +1 -1
  21. package/dist/match-detection/features/bib/all-source-ids.js +16 -19
  22. package/dist/match-detection/features/bib/all-source-ids.js.map +1 -1
  23. package/dist/match-detection/features/bib/authors.js +31 -35
  24. package/dist/match-detection/features/bib/authors.js.map +1 -1
  25. package/dist/match-detection/features/bib/bibliographic-level.js +3 -3
  26. package/dist/match-detection/features/bib/bibliographic-level.js.map +1 -1
  27. package/dist/match-detection/features/bib/host-component.js +3 -3
  28. package/dist/match-detection/features/bib/host-component.js.map +1 -1
  29. package/dist/match-detection/features/bib/index.js +28 -13
  30. package/dist/match-detection/features/bib/index.js.map +1 -1
  31. package/dist/match-detection/features/bib/index.spec.js +9 -13
  32. package/dist/match-detection/features/bib/index.spec.js.map +1 -1
  33. package/dist/match-detection/features/bib/isbn.js +28 -6
  34. package/dist/match-detection/features/bib/isbn.js.map +1 -1
  35. package/dist/match-detection/features/bib/issn.js +0 -4
  36. package/dist/match-detection/features/bib/issn.js.map +1 -1
  37. package/dist/match-detection/features/bib/language.js +69 -41
  38. package/dist/match-detection/features/bib/language.js.map +1 -1
  39. package/dist/match-detection/features/bib/media-type.js +83 -0
  40. package/dist/match-detection/features/bib/media-type.js.map +1 -0
  41. package/dist/match-detection/features/bib/melinda-id.js +0 -4
  42. package/dist/match-detection/features/bib/melinda-id.js.map +1 -1
  43. package/dist/match-detection/features/bib/melinda-identifier-factory.js +6 -17
  44. package/dist/match-detection/features/bib/melinda-identifier-factory.js.map +1 -1
  45. package/dist/match-detection/features/bib/other-standard-identifier.js +1 -4
  46. package/dist/match-detection/features/bib/other-standard-identifier.js.map +1 -1
  47. package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +256 -0
  48. package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +1 -0
  49. package/dist/match-detection/features/bib/publication-time-allow-cons-years.js +71 -0
  50. package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map +1 -0
  51. package/dist/match-detection/features/bib/publication-time.js +9 -8
  52. package/dist/match-detection/features/bib/publication-time.js.map +1 -1
  53. package/dist/match-detection/features/bib/record-type.js +6 -3
  54. package/dist/match-detection/features/bib/record-type.js.map +1 -1
  55. package/dist/match-detection/features/bib/standard-identifier-factory.js +129 -39
  56. package/dist/match-detection/features/bib/standard-identifier-factory.js.map +1 -1
  57. package/dist/match-detection/features/bib/title-version-original.js +79 -0
  58. package/dist/match-detection/features/bib/title-version-original.js.map +1 -0
  59. package/dist/match-detection/features/bib/title.js +29 -22
  60. package/dist/match-detection/features/bib/title.js.map +1 -1
  61. package/dist/match-detection/features/index.js +0 -4
  62. package/dist/match-detection/features/index.js.map +1 -1
  63. package/dist/match-detection/index.js +94 -47
  64. package/dist/match-detection/index.js.map +1 -1
  65. package/dist/match-detection/index.spec.js +25 -15
  66. package/dist/match-detection/index.spec.js.map +1 -1
  67. package/dist/matching-utils.js +63 -15
  68. package/dist/matching-utils.js.map +1 -1
  69. package/package.json +27 -24
  70. package/src/candidate-search/index.js +93 -31
  71. package/src/candidate-search/index.spec.js +27 -24
  72. package/src/candidate-search/query-list/bib.js +149 -22
  73. package/src/candidate-search/query-list/index.js +1 -1
  74. package/src/index.js +310 -38
  75. package/src/index.spec.js +49 -21
  76. package/src/match-detection/features/bib/all-source-ids.js +3 -3
  77. package/src/match-detection/features/bib/authors.js +23 -16
  78. package/src/match-detection/features/bib/bibliographic-level.js +1 -1
  79. package/src/match-detection/features/bib/host-component.js +1 -1
  80. package/src/match-detection/features/bib/index.js +4 -0
  81. package/src/match-detection/features/bib/index.spec.js +10 -2
  82. package/src/match-detection/features/bib/isbn.js +21 -2
  83. package/src/match-detection/features/bib/language.js +63 -22
  84. package/src/match-detection/features/bib/media-type.js +77 -0
  85. package/src/match-detection/features/bib/melinda-identifier-factory.js +2 -3
  86. package/src/match-detection/features/bib/other-standard-identifier.js +2 -0
  87. package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +268 -0
  88. package/src/match-detection/features/bib/publication-time-allow-cons-years.js +67 -0
  89. package/src/match-detection/features/bib/publication-time.js +9 -4
  90. package/src/match-detection/features/bib/record-type.js +5 -1
  91. package/src/match-detection/features/bib/standard-identifier-factory.js +107 -22
  92. package/src/match-detection/features/bib/title-version-original.js +78 -0
  93. package/src/match-detection/features/bib/title.js +27 -7
  94. package/src/match-detection/index.js +60 -22
  95. package/src/match-detection/index.spec.js +18 -4
  96. package/src/matching-utils.js +62 -5
@@ -1,10 +1,11 @@
1
+ /* eslint-disable max-statements */
1
2
  /**
2
3
  *
3
4
  * @licstart The following is the entire license notice for the JavaScript code in this file.
4
5
  *
5
6
  * Melinda record matching modules for Javascript
6
7
  *
7
- * Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
8
+ * Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
8
9
  *
9
10
  * This file is part of melinda-record-matching-js
10
11
  *
@@ -27,58 +28,142 @@
27
28
  */
28
29
 
29
30
  import createDebugLogger from 'debug';
31
+ import {extractSubfieldsFromField, uniqueSubfields} from '../../../matching-utils';
30
32
 
31
- const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers');
32
- const debugData = debug.extend('data');
33
+ // Note about validity of standardIdentifiers:
34
+ // We have three types of invalid standardIdentifiers:
35
+ // 1. Formally invalid standardIdentifiers (ie. typos either in the resource or the record)
36
+ // 2. Formally valid standardIdentifiers that are used in a wrong resource
37
+ // 3. Canceled standardIdentifiers
38
+
39
+ // Matcher could and should check that a standardIdentifier found in a subfield for a valid identifier is formally valid, and if it's not formally valid, handle it as an invalid standardIdentifier
40
+ // Formally valid standardIdentifiers found in subfield for invalid identifier cannot be handled as valid standardIdentifiers, because they can be a case of type 2) or 3) invalid standardIdentifiers
41
+ // We could also do a separate handling for formally valid an formally invalid standardIdentifiers
42
+
43
+ export default ({pattern, subfieldCodes, identifier, validIdentifierSubfieldCodes = ['a'], invalidIdentifierSubfieldCodes = ['z'], validatorAndNormalizer = undefined}) => {
44
+ const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);
45
+ const debugData = debug.extend('data');
33
46
 
34
- export default ({pattern, subfieldCodes}) => {
35
47
  return {extract, compare};
36
48
 
37
- function extract(record) {
38
- const [field] = record.get(pattern);
49
+ function extract({record, recordExternal}) {
50
+ const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
51
+ const fields = record.get(pattern);
52
+ debugData(`${label}: ${fields.length} ${identifier}-fields `);
53
+
54
+ // extractIdentifierSubfield normalizes hyphens away from the subfield values
55
+ const identifiersFromFields = fields.map(field => extractSubfieldsFromField(field, subfieldCodes));
56
+ debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);
57
+ const allIdentifiers = identifiersFromFields.flat();
58
+ debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);
59
+
60
+ const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({identifierSubs: allIdentifiers, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes});
39
61
 
40
- if (field) {
41
- return field.subfields
42
- .filter(({code}) => subfieldCodes.includes(code))
43
- .map(({code, value}) => ({code, value: value.replace(/-/ug, '')}));
62
+ const identifiers = uniqueSubfields(validatedAndNormalizedIdentifiers);
63
+
64
+ debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);
65
+ return identifiers;
66
+
67
+ function validateAndNormalizeIdentifiers({identifierSubs, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {
68
+ if (validatorAndNormalizer) {
69
+ return identifierSubs.map((idSub) => validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}));
70
+ }
71
+ return identifierSubs.map((idSub) => normalizeHyphens(idSub));
44
72
  }
45
73
 
46
- return [];
74
+ function validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {
75
+ const {valid, value} = validatorAndNormalizer(idSub.value);
76
+ if (validIdentifierSubfieldCodes.includes(idSub.code) && valid === false) {
77
+ const [code] = invalidIdentifierSubfieldCodes;
78
+ return {code, value};
79
+ }
80
+ return {code: idSub.code, value};
81
+ }
82
+
83
+ function normalizeHyphens(idSub) {
84
+ return {code: idSub.code, value: idSub.value.replace(/-/ug, '')};
85
+ }
86
+
87
+
47
88
  }
48
89
 
49
90
  function compare(a, b) {
91
+ debug(`Comparing A and B`);
50
92
  if (a.length === 0 || b.length === 0) {
51
- debugData(`No standardidentifiers to compare`);
93
+ debugData(`No standardidentifiers (${identifier}) to compare`);
52
94
  return 0;
53
95
  }
54
96
 
97
+ debugData(`A: ${JSON.stringify(a)}`);
98
+ debugData(`B: ${JSON.stringify(b)}`);
99
+
100
+
55
101
  if (bothHaveValidIdentifiers()) {
56
- const {maxValues, matchingValues} = getValueCount(true);
57
- return matchingValues / maxValues * 0.75;
58
- }
102
+ // Compare only valid identifiers, if both have valid idenfiers
103
+ const {maxValues, possibleMatchValues, matchingValues} = getValueCount(true);
104
+ if (matchingValues < 1) {
105
+ debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);
106
+ return -0.75;
107
+ }
108
+ debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues}/${possibleMatchValues} valid identifiers match.`);
109
+ // ignore non-matches if there is mismatching amount of values
110
+ debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);
111
+ //we give some kind of penalty for mismatching amount of values instead of simple divide?
112
+ const penaltyForMissing = 0.1 * (maxValues - possibleMatchValues);
113
+ const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues);
114
+ debug(`\t points: penaltyForMissing: ${penaltyForMissing}`);
115
+ debug(`\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);
59
116
 
117
+ return 0.75 - penaltyForMisMatch - penaltyForMissing;
118
+ //return matchingValues / possibleMatchValues * 0.75;
119
+ }
120
+ // If both do not have valid identifiers, compare all identifiers
60
121
  const {maxValues, matchingValues} = getValueCount();
122
+ debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);
123
+
61
124
  return matchingValues / maxValues * 0.2;
62
125
 
63
126
  function bothHaveValidIdentifiers() {
64
- const aValues = a.filter(({code}) => code === 'a');
65
- const bValues = a.filter(({code}) => code === 'a');
127
+ const aValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));
128
+ const bValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));
129
+ debug(`A: ${aValues.length} valid ${identifier} identifiers`);
130
+ debug(`B: ${bValues.length} valid ${identifier} identifiers`);
66
131
  return aValues.length > 0 && bValues.length > 0;
67
132
  }
68
133
 
69
134
  function getValueCount(validOnly = false) {
70
- const aValues = getIdentifiers(a);
71
- const bValues = getIdentifiers(b);
135
+ const aValues = getIdentifiers(a, validOnly);
136
+ const bValues = getIdentifiers(b, validOnly);
137
+
138
+ const matchingValues = getMatchingValuesAmount(aValues, bValues);
72
139
 
73
140
  return {
74
141
  maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,
75
- matchingValues: aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length
142
+ // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)
143
+ possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,
144
+ matchingValues
76
145
  };
77
146
 
78
- function getIdentifiers(values) {
147
+ function getMatchingValuesAmount(aValues, bValues) {
148
+ if (bValues.length > aValues.length) {
149
+ return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
150
+ }
151
+ if (aValues.length > bValues.length) {
152
+ return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
153
+ }
154
+
155
+ // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases
156
+ // there would be duplicate values
157
+ const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
158
+ const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
159
+
160
+ return aToB < bToA ? aToB : bToA;
161
+ }
162
+
163
+ function getIdentifiers(values, validOnly) {
79
164
  if (validOnly) {
80
165
  return values
81
- .filter(({code}) => code === 'a')
166
+ .filter(({code}) => validIdentifierSubfieldCodes.includes(code))
82
167
  .map(({value}) => value);
83
168
  }
84
169
 
@@ -0,0 +1,78 @@
1
+ /**
2
+ *
3
+ * @licstart The following is the entire license notice for the JavaScript code in this file.
4
+ *
5
+ * Melinda record matching modules for Javascript
6
+ *
7
+ * Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
8
+ *
9
+ * This file is part of melinda-record-matching-js
10
+ *
11
+ * melinda-record-matching-js program is free software: you can redistribute it and/or modify
12
+ * it under the terms of the GNU Lesser General Public License as
13
+ * published by the Free Software Foundation, either version 3 of the
14
+ * License, or (at your option) any later version.
15
+ *
16
+ * melinda-record-matching-js is distributed in the hope that it will be useful,
17
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
18
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19
+ * GNU Lesser General Public License for more details.
20
+ *
21
+ * You should have received a copy of the GNU Affero General Public License
22
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
23
+ *
24
+ * @licend The above is the entire license notice
25
+ * for the JavaScript code in this file.
26
+ *
27
+ */
28
+
29
+ import createDebugLogger from 'debug';
30
+ import {LevenshteinDistance as leven} from 'natural';
31
+
32
+ export default ({treshold = 10} = {}) => ({
33
+ name: 'titleVersionOriginal',
34
+ extract: ({record}) => {
35
+ const title = getTitle();
36
+
37
+ if (title) {
38
+ return [title.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
39
+ }
40
+
41
+ return [];
42
+
43
+ function getTitle() {
44
+ const [field] = record.get(/^245$/u);
45
+
46
+ if (field) {
47
+ return field.subfields
48
+ .filter(({code}) => ['a', 'b'].includes(code))
49
+ .map(({value}) => value)
50
+ .join('');
51
+ }
52
+ }
53
+ },
54
+ compare: (a, b) => {
55
+ const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');
56
+ const distance = leven(a[0], b[0]);
57
+
58
+ if (distance === 0) {
59
+ return 0.5;
60
+ }
61
+
62
+ const maxLength = getMaxLength();
63
+ const percentage = distance / maxLength * 100;
64
+
65
+ debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
66
+
67
+ if (percentage <= treshold) {
68
+ return 0.3;
69
+ }
70
+
71
+ return -0.5;
72
+
73
+ function getMaxLength() {
74
+ return a[0].length > b[0].length ? a[0].length : b[0].length;
75
+ }
76
+
77
+ }
78
+ });
@@ -4,7 +4,7 @@
4
4
  *
5
5
  * Melinda record matching modules for Javascript
6
6
  *
7
- * Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
7
+ * Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
8
8
  *
9
9
  * This file is part of melinda-record-matching-js
10
10
  *
@@ -26,29 +26,49 @@
26
26
  *
27
27
  */
28
28
 
29
- import createDebugLogger from 'debug';
29
+
30
30
  import {LevenshteinDistance as leven} from 'natural';
31
+ import {testStringOrNumber} from '../../../matching-utils';
32
+ import createDebugLogger from 'debug';
33
+
34
+ const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');
35
+ const debugData = debug.extend('data');
36
+
31
37
 
32
38
  export default ({treshold = 10} = {}) => ({
33
39
  name: 'Title',
34
- extract: record => {
40
+ extract: ({record, recordExternal}) => {
41
+ const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
35
42
  const title = getTitle();
43
+ debug(`${label} title: ${title}`);
36
44
 
37
- if (title) {
38
- return [title.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
45
+ if (testStringOrNumber(title)) {
46
+ const titleAsNormalizedString = String(title)
47
+ // decompose unicode diacritics
48
+ .normalize('NFD')
49
+ // strip non-letters/numbers
50
+ // - note: combined with decomposing unicode diactics this normalizes both 'saa' and 'sää' as 'saa'
51
+ // - we could precompose the finnish letters back to avoid this
52
+ .replace(/[^\p{Letter}\p{Number}]/gu, '')
53
+ .toLowerCase();
54
+ debug(`${label} titleString: ${titleAsNormalizedString}`);
55
+ return [titleAsNormalizedString];
39
56
  }
40
57
 
41
58
  return [];
42
59
 
43
60
  function getTitle() {
44
61
  const [field] = record.get(/^245$/u);
62
+ debugData(`${label} titleField: ${JSON.stringify(field)}`);
45
63
 
46
64
  if (field) {
47
65
  return field.subfields
48
- .filter(({code}) => ['a', 'b'].includes(code))
49
- .map(({value}) => value)
66
+ // get also $n:s and $p:s here
67
+ .filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))
68
+ .map(({value}) => testStringOrNumber(value) ? String(value) : '')
50
69
  .join('');
51
70
  }
71
+ return false;
52
72
  }
53
73
  },
54
74
  compare: (a, b) => {
@@ -1,4 +1,3 @@
1
- /* eslint-disable no-console */
2
1
  /**
3
2
  *
4
3
  * @licstart The following is the entire license notice for the JavaScript code in this file.
@@ -32,40 +31,78 @@ import * as features from './features';
32
31
 
33
32
  export {features};
34
33
 
35
- // eslint-disable-next-line max-statements
36
- export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
34
+ export default ({strategy, treshold = 0.9}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
37
35
  const minProbabilityQuantifier = 0.5;
38
36
 
39
37
  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');
40
38
  const debugData = debug.extend('data');
41
- const featuresA = extractFeatures(recordA);
42
- const featuresB = extractFeatures(recordB);
43
39
 
44
- debugData(`Features (a): ${JSON.stringify(featuresA)}`);
45
- debugData(`Features (b): ${JSON.stringify(featuresB)}`);
40
+ debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
41
+ debugData(`Records: A: ${recordA}\nB: ${recordB}`);
42
+ debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);
43
+ // We could add here labels for records if we didn't get external labels
46
44
 
47
- const featurePairs = generateFeaturePairs();
48
- const similarityVector = generateSimilarityVector();
45
+ const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});
49
46
 
50
- if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
51
- const probability = calculateprobability();
52
- debug(`probability: ${probability} (Treshold: ${treshold})`);
53
- return {match: probability >= treshold, probability};
47
+ debug(`We got an array of records: ${Array.isArray(recordB)}`);
48
+ const recordsB = Array.isArray(recordB) ? recordB : [recordB];
49
+ const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];
50
+
51
+ const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));
52
+
53
+ // if we got array of records, we return an array of result
54
+ // if we got a singular record, we return a singular result
55
+ return Array.isArray(recordB) ? detectionResults : detectionResults[0];
56
+
57
+ function actualDetection({featuresA, record, recordExternal, index, localSettings}) {
58
+ const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';
59
+ const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';
60
+
61
+
62
+ debug(`Actual detection for record ${index + 1} ${labelB}`);
63
+ const featuresB = extractFeatures({record, recordExternal, localSettings});
64
+
65
+ debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);
66
+ debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);
67
+
68
+ const featurePairs = generateFeaturePairs(featuresA, featuresB);
69
+ const similarityVector = generateSimilarityVector(featurePairs);
70
+
71
+ if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
72
+ const probability = calculateProbability(similarityVector);
73
+ debug(`probability: ${probability} (Treshold: ${treshold})`);
74
+ return returnResult({match: probability >= treshold, probability});
75
+ }
76
+
77
+ debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
78
+ return returnResult({match: false, probability: 0.0});
54
79
  }
55
80
 
56
- debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
57
- return {match: false, probability: 0.0};
81
+ function extractFeatures({record, recordExternal}) {
82
+ return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);
83
+ }
58
84
 
59
- function calculateprobability() {
60
- const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);
61
- return probability > 1.0 ? 1.0 : probability;
85
+ function returnResult(result) {
86
+ if (returnStrategy) {
87
+ debug(`Returning detection strategy with the result`);
88
+ const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold};
89
+ debugData(`${JSON.stringify(resultWithStrategy)}`);
90
+ return resultWithStrategy;
91
+ }
92
+ return result;
62
93
  }
63
94
 
64
- function extractFeatures(record) {
65
- return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract(record)}), []);
95
+ function formatStrategy(strategy) {
96
+ const strategyNames = strategy.map(element => element.name);
97
+ return strategyNames || [];
66
98
  }
67
99
 
68
- function generateSimilarityVector() {
100
+ function calculateProbability(similarityVector) {
101
+ const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);
102
+ return probability > 1.0 ? 1.0 : probability;
103
+ }
104
+
105
+ function generateSimilarityVector(featurePairs) {
69
106
  const compared = featurePairs.map(({name, a, b}) => {
70
107
  const {compare} = strategy.find(({name: featureName}) => name === featureName);
71
108
  const points = compare(a, b);
@@ -76,7 +113,7 @@ export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
76
113
  return compared.map(({points}) => points);
77
114
  }
78
115
 
79
- function generateFeaturePairs() {
116
+ function generateFeaturePairs(featuresA, featuresB) {
80
117
  const pairs = generatePairs();
81
118
  const missingFeatures = findMissing();
82
119
 
@@ -105,4 +142,5 @@ export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
105
142
  .filter(v => pairs.some(({name}) => name === v) === false);
106
143
  }
107
144
  }
145
+
108
146
  };
@@ -32,6 +32,11 @@ import {expect} from 'chai';
32
32
  import {MarcRecord} from '@natlibfi/marc-record';
33
33
  import * as features from './features';
34
34
  import createDetectionInterface from '.';
35
+ import {inspect} from 'util';
36
+ import createDebugLogger from 'debug';
37
+
38
+ const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:test');
39
+ const debugData = debug.extend('data');
35
40
 
36
41
  describe('match-detection', () => {
37
42
  generateTests({
@@ -41,16 +46,25 @@ describe('match-detection', () => {
41
46
  fixura: {
42
47
  reader: READERS.JSON
43
48
  },
44
- callback: ({getFixture, options, expectedResults, enabled = true}) => {
49
+ callback: ({getFixture, options, expectedResults, array, enabled = true}) => {
45
50
 
46
51
  if (!enabled) {
52
+ debug(`*** DISABLED TEST! ***`);
47
53
  return;
48
54
  }
49
55
 
50
56
  const detect = createDetectionInterface(formatOptions());
51
- const recordA = new MarcRecord(getFixture('recordA.json'));
52
- const recordB = new MarcRecord(getFixture('recordB.json'));
53
- const results = detect(recordA, recordB);
57
+ const recordA = new MarcRecord(getFixture('recordA.json'), {subfieldValues: false});
58
+ debugData(inspect(recordA));
59
+
60
+ debug(`Our recordB is an array of records: ${array}`);
61
+ const recordB = array
62
+ ? getFixture('recordB.json').map(recordJson => new MarcRecord(recordJson, {subfieldValues: false}))
63
+ : new MarcRecord(getFixture('recordB.json'), {subfieldValues: false});
64
+ debugData(inspect(recordB));
65
+
66
+ const results = detect({recordA, recordB});
67
+ debugData(`${JSON.stringify(results)}`);
54
68
 
55
69
  expect(results).to.eql(expectedResults);
56
70
 
@@ -4,7 +4,7 @@
4
4
  *
5
5
  * Melinda record matching modules for Javascript
6
6
  *
7
- * Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
7
+ * Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
8
8
  *
9
9
  * This file is part of melinda-record-matching-js
10
10
  *
@@ -31,7 +31,6 @@ import createDebugLogger from 'debug';
31
31
  const debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');
32
32
  const debugData = debug.extend('data');
33
33
 
34
- // eslint-disable-next-line max-statements
35
34
  export function getMelindaIdsF035(record) {
36
35
 
37
36
  const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');
@@ -58,8 +57,8 @@ export function getMelindaIdsF035(record) {
58
57
 
59
58
  return subfields
60
59
  .filter(sub => ['a', 'z'].includes(sub.code))
61
- .filter(sub => melindaIdRegExp.test(sub.value))
62
- .map(({value}) => value.replace(melindaIdRegExp, '$<id>'));
60
+ .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))
61
+ .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');
63
62
 
64
63
  }
65
64
  }
@@ -81,6 +80,64 @@ function countSubfields(field, subfieldCode) {
81
80
 
82
81
  export function getSubfieldValues(field, subfieldCode) {
83
82
  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);
84
- return field.subfields.filter(({code}) => code === subfieldCode).map(({value}) => value);
83
+ return field.subfields
84
+ .filter(({code}) => code === subfieldCode)
85
+ .map(({value}) => testStringOrNumber(value) ? String(value) : '')
86
+ .filter(value => value);
85
87
  }
86
88
 
89
+ export function testStringOrNumber(value) {
90
+ if (typeof value === 'string' || typeof value === 'number') {
91
+ return true;
92
+ }
93
+ return false;
94
+ }
95
+
96
+ export function extractSubfieldsFromField(field, subfieldCodes) {
97
+ if (field === undefined || field.subfields === undefined) {
98
+ return [];
99
+ }
100
+ const resultSubfields = field.subfields
101
+ .filter(({code}) => subfieldCodes.includes(code))
102
+ .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))
103
+ .filter(value => value);
104
+ return resultSubfields;
105
+ }
106
+
107
+ export function uniqueSubfields(subfields) {
108
+ return subfields.reduce((arr, e) => {
109
+ if (!arr.find(item => item.code === e.code && item.value === e.value)) {
110
+ const newArr = arr.concat(e);
111
+ return newArr;
112
+ }
113
+ return arr;
114
+ }, []);
115
+ }
116
+
117
+ export function getMatchCounts(aValues, bValues) {
118
+
119
+ const matchingValues = getMatchingValuesAmount(aValues, bValues);
120
+
121
+ return {
122
+ maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,
123
+ // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)
124
+ possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,
125
+ matchingValues
126
+ };
127
+
128
+ function getMatchingValuesAmount(aValues, bValues) {
129
+ if (bValues.length > aValues.length) {
130
+ return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
131
+ }
132
+ if (aValues.length > bValues.length) {
133
+ return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
134
+ }
135
+
136
+ // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases
137
+ // there would be duplicate values
138
+ const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
139
+ const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
140
+
141
+ return aToB < bToA ? aToB : bToA;
142
+ }
143
+ }