@natlibfi/melinda-record-matching 4.4.0-alpha.1 → 5.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/.github/workflows/melinda-node-tests.yml +13 -9
  2. package/LICENSE.txt +21 -165
  3. package/README.md +2 -2
  4. package/dist/candidate-search/candidate-search-utils.js +6 -43
  5. package/dist/candidate-search/candidate-search-utils.js.map +7 -1
  6. package/dist/candidate-search/choose-queries.js +75 -0
  7. package/dist/candidate-search/choose-queries.js.map +7 -0
  8. package/dist/candidate-search/index.js +89 -225
  9. package/dist/candidate-search/index.js.map +7 -1
  10. package/dist/candidate-search/index.test.js +92 -0
  11. package/dist/candidate-search/index.test.js.map +7 -0
  12. package/dist/candidate-search/query-list/bib.js +141 -320
  13. package/dist/candidate-search/query-list/bib.js.map +7 -1
  14. package/dist/candidate-search/query-list/bib.test.js +34 -0
  15. package/dist/candidate-search/query-list/bib.test.js.map +7 -0
  16. package/dist/candidate-search/query-list/component.js +87 -0
  17. package/dist/candidate-search/query-list/component.js.map +7 -0
  18. package/dist/candidate-search/query-list/index.js +39 -52
  19. package/dist/candidate-search/query-list/index.js.map +7 -1
  20. package/dist/cli.js +138 -0
  21. package/dist/cli.js.map +7 -0
  22. package/dist/index.js +87 -462
  23. package/dist/index.js.map +7 -1
  24. package/dist/index.test.js +69 -0
  25. package/dist/index.test.js.map +7 -0
  26. package/dist/match-detection/features/bib/all-source-ids.js +20 -91
  27. package/dist/match-detection/features/bib/all-source-ids.js.map +7 -1
  28. package/dist/match-detection/features/bib/authors.js +20 -76
  29. package/dist/match-detection/features/bib/authors.js.map +7 -1
  30. package/dist/match-detection/features/bib/bibliographic-level.js +4 -40
  31. package/dist/match-detection/features/bib/bibliographic-level.js.map +7 -1
  32. package/dist/match-detection/features/bib/host-component.js +5 -41
  33. package/dist/match-detection/features/bib/host-component.js.map +7 -1
  34. package/dist/match-detection/features/bib/index.js +17 -119
  35. package/dist/match-detection/features/bib/index.js.map +7 -1
  36. package/dist/match-detection/features/bib/index.test.js +40 -0
  37. package/dist/match-detection/features/bib/index.test.js.map +7 -0
  38. package/dist/match-detection/features/bib/isbn.js +13 -69
  39. package/dist/match-detection/features/bib/isbn.js.map +7 -1
  40. package/dist/match-detection/features/bib/issn.js +5 -50
  41. package/dist/match-detection/features/bib/issn.js.map +7 -1
  42. package/dist/match-detection/features/bib/language.js +25 -86
  43. package/dist/match-detection/features/bib/language.js.map +7 -1
  44. package/dist/match-detection/features/bib/media-type.js +12 -64
  45. package/dist/match-detection/features/bib/media-type.js.map +7 -1
  46. package/dist/match-detection/features/bib/melinda-id.js +5 -47
  47. package/dist/match-detection/features/bib/melinda-id.js.map +7 -1
  48. package/dist/match-detection/features/bib/melinda-identifier-factory.js +16 -67
  49. package/dist/match-detection/features/bib/melinda-identifier-factory.js.map +7 -1
  50. package/dist/match-detection/features/bib/other-standard-identifier.js +5 -51
  51. package/dist/match-detection/features/bib/other-standard-identifier.js.map +7 -1
  52. package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +49 -189
  53. package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +7 -1
  54. package/dist/match-detection/features/bib/publication-time-allow-cons-years.js +10 -53
  55. package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map +7 -1
  56. package/dist/match-detection/features/bib/publication-time.js +8 -46
  57. package/dist/match-detection/features/bib/publication-time.js.map +7 -1
  58. package/dist/match-detection/features/bib/record-type.js +4 -43
  59. package/dist/match-detection/features/bib/record-type.js.map +7 -1
  60. package/dist/match-detection/features/bib/standard-identifier-factory.js +47 -163
  61. package/dist/match-detection/features/bib/standard-identifier-factory.js.map +7 -1
  62. package/dist/match-detection/features/bib/title-version-original.js +11 -53
  63. package/dist/match-detection/features/bib/title-version-original.js.map +7 -1
  64. package/dist/match-detection/features/bib/title.js +20 -72
  65. package/dist/match-detection/features/bib/title.js.map +7 -1
  66. package/dist/match-detection/features/index.js +3 -11
  67. package/dist/match-detection/features/index.js.map +7 -1
  68. package/dist/match-detection/index.js +35 -150
  69. package/dist/match-detection/index.js.map +7 -1
  70. package/dist/match-detection/index.test.js +44 -0
  71. package/dist/match-detection/index.test.js.map +7 -0
  72. package/dist/matching-utils.js +38 -102
  73. package/dist/matching-utils.js.map +7 -1
  74. package/eslint.config.js +52 -0
  75. package/example.env +4 -0
  76. package/package.json +29 -78
  77. package/src/candidate-search/candidate-search-utils.js +1 -28
  78. package/src/candidate-search/choose-queries.js +93 -0
  79. package/src/candidate-search/index.js +85 -119
  80. package/src/candidate-search/{index.spec.js → index.test.js} +30 -43
  81. package/src/candidate-search/query-list/bib.js +54 -55
  82. package/src/candidate-search/query-list/bib.test.js +38 -0
  83. package/src/candidate-search/query-list/component.js +127 -0
  84. package/src/candidate-search/query-list/index.js +31 -35
  85. package/src/cli.js +177 -0
  86. package/src/index.js +27 -68
  87. package/src/{index.spec.js → index.test.js} +11 -42
  88. package/src/match-detection/features/bib/all-source-ids.js +2 -29
  89. package/src/match-detection/features/bib/authors.js +5 -30
  90. package/src/match-detection/features/bib/bibliographic-level.js +0 -27
  91. package/src/match-detection/features/bib/host-component.js +0 -27
  92. package/src/match-detection/features/bib/index.js +16 -43
  93. package/src/match-detection/features/bib/index.test.js +52 -0
  94. package/src/match-detection/features/bib/isbn.js +2 -29
  95. package/src/match-detection/features/bib/issn.js +1 -28
  96. package/src/match-detection/features/bib/language.js +5 -32
  97. package/src/match-detection/features/bib/media-type.js +2 -28
  98. package/src/match-detection/features/bib/melinda-id.js +1 -28
  99. package/src/match-detection/features/bib/melinda-identifier-factory.js +5 -32
  100. package/src/match-detection/features/bib/other-standard-identifier.js +1 -28
  101. package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +7 -34
  102. package/src/match-detection/features/bib/publication-time-allow-cons-years.js +1 -28
  103. package/src/match-detection/features/bib/publication-time.js +2 -28
  104. package/src/match-detection/features/bib/record-type.js +0 -27
  105. package/src/match-detection/features/bib/standard-identifier-factory.js +1 -28
  106. package/src/match-detection/features/bib/title-version-original.js +2 -28
  107. package/src/match-detection/features/bib/title.js +6 -34
  108. package/src/match-detection/features/index.js +1 -28
  109. package/src/match-detection/index.js +1 -28
  110. package/src/match-detection/{index.spec.js → index.test.js} +9 -35
  111. package/src/matching-utils.js +8 -37
  112. package/dist/candidate-search/index.spec.js +0 -144
  113. package/dist/candidate-search/index.spec.js.map +0 -1
  114. package/dist/candidate-search/query-list/bib.spec.js +0 -64
  115. package/dist/candidate-search/query-list/bib.spec.js.map +0 -1
  116. package/dist/index.spec.js +0 -127
  117. package/dist/index.spec.js.map +0 -1
  118. package/dist/match-detection/features/bib/index.spec.js +0 -93
  119. package/dist/match-detection/features/bib/index.spec.js.map +0 -1
  120. package/dist/match-detection/index.spec.js +0 -91
  121. package/dist/match-detection/index.spec.js.map +0 -1
  122. package/src/candidate-search/query-list/bib.spec.js +0 -54
  123. package/src/match-detection/features/bib/index.spec.js +0 -78
package/dist/index.js CHANGED
@@ -1,59 +1,10 @@
1
- "use strict";
2
-
3
- Object.defineProperty(exports, "__esModule", {
4
- value: true
5
- });
6
- exports.matchDetection = exports.default = exports.candidateSearch = void 0;
7
- var _debug = _interopRequireDefault(require("debug"));
8
- var _candidateSearch = _interopRequireWildcard(require("./candidate-search"));
9
- var candidateSearch = _candidateSearch;
10
- exports.candidateSearch = _candidateSearch;
11
- var _matchDetection = _interopRequireWildcard(require("./match-detection"));
12
- var matchDetection = _matchDetection;
13
- exports.matchDetection = _matchDetection;
14
- function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
15
- function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
16
- function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
17
- /**
18
- *
19
- * @licstart The following is the entire license notice for the JavaScript code in this file.
20
- *
21
- * Melinda record matching modules for Javascript
22
- *
23
- * Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
24
- *
25
- * This file is part of melinda-record-matching-js
26
- *
27
- * melinda-record-matching-js program is free software: you can redistribute it and/or modify
28
- * it under the terms of the GNU Lesser General Public License as
29
- * published by the Free Software Foundation, either version 3 of the
30
- * License, or (at your option) any later version.
31
- *
32
- * melinda-record-matching-js is distributed in the hope that it will be useful,
33
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
34
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
35
- * GNU Lesser General Public License for more details.
36
- *
37
- * You should have received a copy of the GNU Affero General Public License
38
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
39
- *
40
- * @licend The above is the entire license notice
41
- * for the JavaScript code in this file.
42
- *
43
- */
44
- //import inspect from 'util';
45
- var _default = ({
46
- detection: detectionOptions,
47
- search: searchOptions,
48
- maxMatches = 1,
49
- maxCandidates = 25,
50
- returnStrategy = false,
51
- returnQuery = false,
52
- returnNonMatches = false,
53
- returnFailures = false
54
- }) => {
55
- const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:index');
56
- const debugData = debug.extend('data');
1
+ import createDebugLogger from "debug";
2
+ import createSearchInterface, * as candidateSearch from "./candidate-search/index.js";
3
+ import createDetectionInterface, * as matchDetection from "./match-detection/index.js";
4
+ export { candidateSearch, matchDetection };
5
+ export default ({ detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false }) => {
6
+ const debug = createDebugLogger("@natlibfi/melinda-record-matching:index");
7
+ const debugData = debug.extend("data");
57
8
  debugData(`DetectionOptions: ${JSON.stringify(detectionOptions)}`);
58
9
  debugData(`SearchOptions: ${JSON.stringify(searchOptions)}`);
59
10
  debugData(`MaxMatches: ${JSON.stringify(maxMatches)}`);
@@ -61,416 +12,134 @@ var _default = ({
61
12
  debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
62
13
  debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);
63
14
  debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);
64
- debugData(`ReturnFailures: ${JSON.stringify(returnFailures)}`);
65
- const detect = (0, _matchDetection.default)(detectionOptions, returnStrategy);
66
- return ({
67
- record,
68
- recordExternal = {
69
- recordSource: 'incomingRecord',
70
- label: 'ic'
71
- }
72
- }) => {
73
- const search = (0, _candidateSearch.default)({
74
- ...searchOptions,
75
- record,
76
- maxCandidates,
77
- recordExternal
78
- });
15
+ const detect = createDetectionInterface(detectionOptions, returnStrategy);
16
+ return prepareSearch;
17
+ async function prepareSearch({ record, recordExternal = { recordSource: "incomingRecord", label: "ic" } }) {
18
+ const { search } = await createSearchInterface({ ...searchOptions, record, maxCandidates, recordExternal });
79
19
  return iterate({});
80
-
81
- // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set
82
- // matches : candidates that have been detected as matches by current matcher job
83
- // nonMatches : candidates that have been detected as non-matches by current matcher job (only if returnNonMatches is 'true')
84
- // duplicateCount : amount of candidate records that were retrieved from the SRU but not handled further because they were already found in the matches/nonMatches
85
-
86
- // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
87
- // state.query : current query (undefined if there was no queries left)
88
- // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
89
- // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)
90
- // state.queriesLeft : amount of queries left
91
- // state.queryCounter : sequence for current query
92
- // state.maxedQueries : queries that resulted in more than serverMaxResults hits
93
-
94
- async function iterate({
95
- initialState = {},
96
- matches = [],
97
- candidateCount = 0,
98
- nonMatches = [],
99
- duplicateCount = 0,
100
- nonMatchCount = 0,
101
- conversionFailures = [],
102
- matchErrors = []
103
- }) {
20
+ async function iterate({ initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, matchErrors = [] }) {
104
21
  debugData(`Starting next matcher iteration.`);
105
- const {
106
- records,
107
- failures,
108
- ...state
109
- } = await search(initialState);
110
- debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, conversionFailures: ${conversionFailures}, matchErrors: ${matchErrors.length}`);
22
+ const { records, ...state } = await search(initialState);
23
+ debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, matchErrors: ${matchErrors.length}`);
111
24
  const recordSetSize = records.length;
112
- const failureSetSize = failures.length;
113
- const newCandidateCount = candidateCount + recordSetSize + failureSetSize;
114
- const newConversionFailures = conversionFailures.concat(failures);
115
- debugData(`Failures: ${failures.length}, ConversionFailures: ${conversionFailures.length}, NewConversionFailures: ${newConversionFailures.length}`);
25
+ const newCandidateCount = candidateCount + recordSetSize;
116
26
  if (recordSetSize > 0) {
117
27
  return handleRecordSet();
118
28
  }
119
29
  if (state.queriesLeft > 0) {
120
30
  debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);
121
- return iterate({
122
- initialState: state,
123
- matches,
124
- candidateCount: newCandidateCount,
125
- nonMatches,
126
- nonMatchCount,
127
- duplicateCount,
128
- conversionFailures: newConversionFailures,
129
- matchErrors
130
- });
31
+ return iterate({ initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, matchErrors });
131
32
  }
132
33
  debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);
133
- return returnResult({
134
- matches,
135
- state,
136
- stopReason: '',
137
- nonMatches,
138
- nonMatchCount,
139
- candidateCount: newCandidateCount,
140
- duplicateCount,
141
- conversionFailures: newConversionFailures,
142
- matchErrors
143
- });
34
+ return returnResult({ matches, state, stopReason: "", nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, matchErrors });
144
35
  function handleRecordSet() {
145
36
  debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);
146
- const matchResult = iterateRecords({
147
- records,
148
- recordSetSize,
149
- maxMatches,
150
- matches,
151
- nonMatches,
152
- nonMatchCount
153
- });
37
+ const matchResult = iterateRecords({ records, recordSetSize, maxMatches, matches, nonMatches, nonMatchCount });
154
38
  const newDuplicateCount = duplicateCount + matchResult.duplicateCount;
155
39
  const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;
156
- const {
157
- newMatches,
158
- newNonMatches,
159
- newMatchErrors
160
- } = handleMatchResult(matchResult, matches, nonMatches, matchErrors);
161
- if (maxMatchesFound({
162
- matches: newMatches,
163
- maxMatches
164
- })) {
165
- return returnResult({
166
- matches: newMatches,
167
- state,
168
- stopReason: 'maxMatches',
169
- nonMatches: newNonMatches,
170
- duplicateCount: newDuplicateCount,
171
- candidateCount: newCandidateCount,
172
- nonMatchCount: newNonMatchCount,
173
- conversionFailures: newConversionFailures,
174
- matchErrors: newMatchErrors
175
- });
40
+ const { newMatches, newNonMatches, newMatchErrors } = handleMatchResult(matchResult, matches, nonMatches, matchErrors);
41
+ if (maxMatchesFound({ matches: newMatches, maxMatches })) {
42
+ return returnResult({ matches: newMatches, state, stopReason: "maxMatches", nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors });
176
43
  }
177
44
  if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
178
- return returnResult({
179
- matches: newMatches,
180
- state,
181
- stopReason: 'maxCandidates',
182
- nonMatches: newNonMatches,
183
- duplicateCount: newDuplicateCount,
184
- candidateCount: newCandidateCount,
185
- nonMatchCount: newNonMatchCount,
186
- conversionFailures: newConversionFailures,
187
- matchErrors: newMatchErrors
188
- });
45
+ return returnResult({ matches: newMatches, state, stopReason: "maxCandidates", nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors });
189
46
  }
190
- return iterate({
191
- initialState: state,
192
- matches: newMatches,
193
- candidateCount: newCandidateCount,
194
- nonMatches: newNonMatches,
195
- duplicateCount: newDuplicateCount,
196
- nonMatchCount: newNonMatchCount,
197
- conversionFailures: newConversionFailures,
198
- matchErrors: newMatchErrors
199
- });
47
+ return iterate({ initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors });
200
48
  }
201
- function handleMatchResult(matchResult, matches, nonMatches, matchErrors) {
49
+ function handleMatchResult(matchResult, matches2, nonMatches2, matchErrors2) {
202
50
  debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);
203
- // eslint-disable-next-line functional/no-conditional-statements
204
51
  if (returnNonMatches) {
205
52
  debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);
206
53
  }
207
- const newMatches = matches.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);
208
- const newNonMatches = returnNonMatches ? nonMatches.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];
209
- const newMatchErrors = matchErrors.concat(matchResult.matchErrors);
54
+ const newMatches = matches2.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);
55
+ const newNonMatches = returnNonMatches ? nonMatches2.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];
56
+ const newMatchErrors = matchErrors2.concat(matchResult.matchErrors);
210
57
  debugData(`- Total amount of matches: ${newMatches.length}`);
211
- // eslint-disable-next-line functional/no-conditional-statements
212
58
  if (returnNonMatches) {
213
59
  debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);
214
60
  }
215
61
  debugData(`MatchResult: ${JSON.stringify(matchResult)}`);
216
- debugData(`Old matchErrors: ${JSON.stringify(matchErrors)}, matchErrors from matchResult: ${JSON.stringify(matchResult.matchErrors)}, New matchErrors: ${JSON.stringify(newMatchErrors)}`);
62
+ debugData(`Old matchErrors: ${JSON.stringify(matchErrors2)}, matchErrors from matchResult: ${JSON.stringify(matchResult.matchErrors)}, New matchErrors: ${JSON.stringify(newMatchErrors)}`);
217
63
  debugData(`- Total amount of matchErrors: ${newMatchErrors.length}`);
218
- return {
219
- newMatches,
220
- newNonMatches,
221
- newMatchErrors
222
- };
64
+ return { newMatches, newNonMatches, newMatchErrors };
223
65
  }
224
- function addQuery(matches) {
66
+ function addQuery(matches2) {
225
67
  debugData(`Adding query ${state.query} to matches`);
226
- return matches.map(match => ({
227
- ...match,
228
- matchQuery: state.query
229
- }));
68
+ return matches2.map((match) => ({ ...match, matchQuery: state.query }));
230
69
  }
231
- function maxCandidatesRetrieved(candidateCount, maxCandidates) {
232
- debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${maxCandidates})`);
233
- if (maxCandidates && candidateCount >= maxCandidates) {
234
- debug(`Stopped matching because maximum number of candidate records ${candidateCount} / ${maxCandidates} have been retrieved`);
70
+ function maxCandidatesRetrieved(candidateCount2, maxCandidates2) {
71
+ debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${maxCandidates2})`);
72
+ if (maxCandidates2 && candidateCount2 >= maxCandidates2) {
73
+ debug(`Stopped matching because maximum number of candidate records ${candidateCount2} / ${maxCandidates2} have been retrieved`);
235
74
  return true;
236
75
  }
237
76
  }
238
77
  }
239
-
240
- // matches : array of matching candidate records
241
- // nonMatches : array of nonMatching candidate records (if returnNonMatches option is true, otherwise empty array)
242
- // - candidate.id
243
- // - candidate.record
244
- // - probability
245
- // - strategy (if returnStrategy option is true)
246
- // - treshold (if returnStrategy option is true)
247
- // - matchQuery (if returnQuery option is true)
248
- // failures: array of conversionFailures from candidate-search and matchErrors from matchDetection in error format {status, payload: {message, id}} if returnFailures is true
249
-
250
- // we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data
251
-
252
- // matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not
253
- // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
254
- // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
255
- // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
256
-
257
- function returnResult({
258
- matches,
259
- state,
260
- stopReason,
261
- nonMatches,
262
- duplicateCount,
263
- candidateCount,
264
- nonMatchCount,
265
- conversionFailures,
266
- matchErrors
267
- }) {
268
- const conversionFailureCount = conversionFailures.length;
78
+ function returnResult({ matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, matchErrors }) {
269
79
  const matchErrorCount = matchErrors.length;
270
- checkCounts({
271
- matches,
272
- nonMatches,
273
- candidateCount,
274
- duplicateCount,
275
- nonMatchCount,
276
- conversionFailureCount,
277
- matchErrorCount
278
- });
279
- const matchStatus = getMatchState(state, stopReason, conversionFailureCount, matchErrorCount);
280
- // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
281
- const matchesResult = returnNonMatches ? {
282
- matches,
283
- matchStatus,
284
- nonMatches,
285
- candidateCount
286
- } : {
287
- matches,
288
- matchStatus,
289
- candidateCount
290
- };
291
- const failures = [...conversionFailures, ...matchErrors];
292
- const result = returnFailures ? {
293
- ...matchesResult,
294
- conversionFailures: failures
295
- } : matchesResult;
296
- debugData(`ReturnFailures ${returnFailures}`);
80
+ checkCounts({ matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, matchErrorCount });
81
+ const matchStatus = getMatchState(state, stopReason, matchErrorCount);
82
+ const matchesResult = returnNonMatches ? { matches, matchStatus, nonMatches, candidateCount } : { matches, matchStatus, candidateCount };
83
+ const result = matchesResult;
297
84
  debugData(`${JSON.stringify(result)}`);
298
85
  return result;
299
-
300
- // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
301
-
302
- function checkCounts({
303
- matches,
304
- nonMatches,
305
- candidateCount,
306
- duplicateCount,
307
- nonMatchCount,
308
- conversionFailureCount,
309
- matchErrorCount
310
- }) {
311
- const matchCount = matches.length;
86
+ function checkCounts({ matches: matches2, nonMatches: nonMatches2, candidateCount: candidateCount2, duplicateCount: duplicateCount2, nonMatchCount: nonMatchCount2, matchErrorCount: matchErrorCount2 }) {
87
+ const matchCount = matches2.length;
312
88
  debugData(`Return nonMatches: ${returnNonMatches}`);
313
- const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
314
- const totalHandled = matchCount + chosenNonMatchCount + duplicateCount;
315
- debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, conversionFailureCount: ${conversionFailureCount}, matchErrorCount: ${matchErrorCount}`);
316
- debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
317
- if (totalHandled !== candidateCount) {
318
- debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);
89
+ const chosenNonMatchCount = returnNonMatches ? nonMatches2.length : nonMatchCount2;
90
+ const totalHandled = matchCount + chosenNonMatchCount + duplicateCount2;
91
+ debug(`candidateCount: ${candidateCount2}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount2}, matchErrorCount: ${matchErrorCount2}`);
92
+ debug(`We got result for ${totalHandled} / ${candidateCount2} retrieved candidates`);
93
+ if (totalHandled !== candidateCount2) {
94
+ debug(`WARNING: Missing results for ${candidateCount2 - totalHandled} candidates`);
319
95
  return;
320
96
  }
321
97
  return;
322
98
  }
323
-
324
- // eslint-disable-next-line max-statements
325
- function getMatchState(state, stopReason, conversionFailuresCount, matchErrorCount) {
326
- debugData(`${JSON.stringify(state)}`);
327
- debug(`We had ${conversionFailuresCount} retrieved candidates that could not be converted.`);
328
- debug(`We had ${matchErrorCount} retrieved candidates that errored in matchDetection.`);
329
- debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
330
- debugData(`StopReason: <${stopReason}>`);
331
- const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;
332
- const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
99
+ function getMatchState(state2, stopReason2, matchErrorCount2) {
100
+ debugData(`${JSON.stringify(state2)}`);
101
+ debug(`We had ${matchErrorCount2} retrieved candidates that errored in matchDetection.`);
102
+ debug(`Queries left ${state2.queriesLeft}, Searches for current query left: ${state2.resultSetOffset && state2.resultSetOffset <= state2.totalRecords}, non-retrieved records: ${state2.totalRecords - state2.queryCandidateCounter}, maxedQueries (${state2.maxedQueries.length}): ${state2.maxedQueries}`);
103
+ debugData(`StopReason: <${stopReason2}>`);
104
+ const searchesLeft = state2.resultSetOffset && state2.resultSetOffset <= state2.totalRecords;
105
+ const nonRetrieved = searchesLeft ? state2.totalRecords - state2.queryCandidateCounter : 0;
333
106
  debugData(`nonRetrieved: ${nonRetrieved}`);
334
-
335
- // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
336
- // 'maxMatches' and 'maxCandidates' are in stopReason, 'maxedQueries', 'conversionFailures' and 'matchErrors' are created here
337
-
338
- if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || conversionFailureCount > 0 || matchErrorCount > 0) {
339
- const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : undefined;
340
- const conversionFailuresStopReason = conversionFailureCount > 0 ? 'conversionFailures' : undefined;
341
- const matchErrorsStopReason = matchErrorCount > 0 ? 'matchErrors' : undefined;
342
- const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || conversionFailuresStopReason || matchErrorsStopReason : stopReason;
107
+ if (state2.queriesLeft > 0 || nonRetrieved > 0 || state2.maxedQueries.length > 0 || matchErrorCount2 > 0) {
108
+ const maxedQueriesStopReason = state2.maxedQueries.length > 0 ? "maxedQueries" : void 0;
109
+ const matchErrorsStopReason = matchErrorCount2 > 0 ? "matchErrors" : void 0;
110
+ const newStopReason = stopReason2 === "" || stopReason2 === void 0 ? maxedQueriesStopReason || matchErrorsStopReason : stopReason2;
343
111
  debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);
344
- debugData(`ConversionFailureStopReason <${conversionFailuresStopReason}>`);
345
112
  debugData(`MatchErrorsStopReason <${matchErrorsStopReason}>`);
346
113
  debugData(`NewStopReason: <${newStopReason}>`);
347
114
  debug(`Match status: false`);
348
- return {
349
- status: false,
350
- stopReason: newStopReason
351
- };
115
+ return { status: false, stopReason: newStopReason };
352
116
  }
353
117
  debug(`Match status: true`);
354
- return {
355
- status: true,
356
- stopReason
357
- };
118
+ return { status: true, stopReason: stopReason2 };
358
119
  }
359
120
  }
360
-
361
- // NOTES:
362
- // - we could optimize by creating the featureSet for the incoming record once and using it for all database/candidateRecords
363
- // - if creating the featureSet for the incoming record fails we have an unprocessable entity
364
- // - if creating the featureSet for a candidate record fails we could skip that candidate - but list the case as a detectionFailure, same as conversionFailures
365
-
366
- function iterateRecords({
367
- records,
368
- recordSetSize,
369
- maxMatches,
370
- matches = [],
371
- nonMatches = [],
372
- recordMatches = [],
373
- recordNonMatches = [],
374
- recordCount = 0,
375
- recordDuplicateCount = 0,
376
- recordNonMatchCount = 0,
377
- recordMatchErrors = []
378
- }) {
379
- // recordSetSize : total amount of records in the current record set
380
- // recordCount : amount of records from the current record set that have been handled
381
- // maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped
382
- // recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results
383
- // recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)
384
-
385
- // records : non-handled records in the current record set
386
- // matches : found matches in the current matcher job
387
- // recordMatches : found matches in the current record set
388
- // recordNonMatches : found nonMatches in the current record set (only if returnNonMatches setting is true)
389
- // recordMatchErrors: errored matchDetection in the current record set
390
-
121
+ function iterateRecords({ records, recordSetSize, maxMatches: maxMatches2, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0, recordMatchErrors = [] }) {
391
122
  const [candidate] = records;
392
123
  const newRecordCount = candidate ? recordCount + 1 : recordCount;
393
-
394
- // The matcher uses same matchDetection strategy for candidates from all candidate-searches -> matchDetection result for the same candidate is always same
395
- // Exceptions would happen if the candidate would have been updated in the database between candidate searches
396
- // Note that if returnNonMatches is false, matcher won't remember candidates that didn't match, so they will be matched again everytime they are retrieved by
397
- // different candidate search queries. Same candidate search query won't have duplicate records.
398
-
399
- /* We could optimize and detect all retrieved candidates at once
400
- const candidateRecords = records.map(record => record.record);
401
- const recordsIsArray = Array.isArray(candidateRecords);
402
- debug(`records is an array: ${recordsIsArray}`);
403
- const result = detect(record, candidateRecords);
404
- debugData(`${JSON.stringify(result)}`);
405
- */
406
-
407
124
  if (candidate) {
408
- // eslint-disable-next-line functional/no-conditional-statements
409
125
  if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {
410
- const {
411
- record: candidateRecord,
412
- id: candidateId
413
- } = candidate;
414
- const recordBExternal = {
415
- id: candidate.id,
416
- recordSource: 'databaseRecord',
417
- label: `db-${candidate.id}`
418
- };
126
+ const { record: candidateRecord, id: candidateId } = candidate;
127
+ const recordBExternal = { id: candidate.id, recordSource: "databaseRecord", label: `db-${candidate.id}` };
419
128
  try {
420
129
  debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);
421
- // we should handle errors from detection somehow - ie. cases where either record or candidateRecord errors
422
- const detectionResult = detect({
423
- recordA: record,
424
- recordB: candidateRecord,
425
- recordAExternal: recordExternal,
426
- recordBExternal
427
- });
130
+ const detectionResult = detect({ recordA: record, recordB: candidateRecord, recordAExternal: recordExternal, recordBExternal });
428
131
  return handleDetectionResult(detectionResult, candidateId, candidateRecord);
429
132
  } catch (error) {
430
133
  debug(`MatchDetection errored: database record ${candidateId}: ${error}`);
431
- const matchError = {
432
- status: 422,
433
- payload: {
434
- message: `Matching errored for database record ${candidateId}. ${error.message}.`,
435
- id: candidateId
436
- }
437
- };
134
+ const matchError = { status: 422, payload: { message: `Matching errored for database record ${candidateId}. ${error.message}.`, id: candidateId } };
438
135
  const newRecordMatchErrors = recordMatchErrors.concat(matchError);
439
- return iterateRecords({
440
- records: records.slice(1),
441
- recordSetSize,
442
- maxMatches,
443
- matches,
444
- recordMatches,
445
- recordCount: newRecordCount,
446
- recordNonMatches,
447
- recordDuplicateCount,
448
- recordNonMatchCount,
449
- recordMatchErrors: newRecordMatchErrors
450
- });
136
+ return iterateRecords({ records: records.slice(1), recordSetSize, maxMatches: maxMatches2, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount, recordMatchErrors: newRecordMatchErrors });
451
137
  }
452
138
  }
453
- return iterateRecords({
454
- records: records.slice(1),
455
- recordSetSize,
456
- maxMatches,
457
- matches,
458
- recordMatches,
459
- recordCount: newRecordCount,
460
- recordNonMatches,
461
- recordDuplicateCount: recordDuplicateCount + 1,
462
- recordNonMatchCount,
463
- recordMatchErrors
464
- });
139
+ return iterateRecords({ records: records.slice(1), recordSetSize, maxMatches: maxMatches2, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount, recordMatchErrors });
465
140
  }
466
141
  debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);
467
- return {
468
- matches: recordMatches,
469
- nonMatches: returnNonMatches ? recordNonMatches : [],
470
- duplicateCount: recordDuplicateCount,
471
- nonMatchCount: recordNonMatchCount,
472
- matchErrors: recordMatchErrors
473
- };
142
+ return { matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount, matchErrors: recordMatchErrors };
474
143
  function handleDetectionResult(detectionResult, candidateId, candidateRecord) {
475
144
  debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);
476
145
  if (detectionResult.match || returnNonMatches) {
@@ -487,93 +156,49 @@ var _default = ({
487
156
  strategy: detectionResult.strategy,
488
157
  treshold: detectionResult.treshold
489
158
  };
490
- const newMatch = returnStrategy ? {
491
- ...matchResult,
492
- ...strategyResult
493
- } : {
494
- ...matchResult
495
- };
159
+ const newMatch = returnStrategy ? { ...matchResult, ...strategyResult } : { ...matchResult };
496
160
  debugData(`${JSON.stringify(newMatch)}`);
497
161
  return handleRecordMatch(detectionResult.match, newMatch);
498
162
  }
499
163
  const newRecordNonMatchCount = recordNonMatchCount + 1;
500
164
  debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);
501
- return iterateRecords({
502
- records: records.slice(1),
503
- recordSetSize,
504
- maxMatches,
505
- matches,
506
- recordMatches,
507
- recordCount: newRecordCount,
508
- recordNonMatches,
509
- recordDuplicateCount,
510
- recordNonMatchCount: newRecordNonMatchCount,
511
- recordMatchErrors
512
- });
165
+ return iterateRecords({ records: records.slice(1), recordSetSize, maxMatches: maxMatches2, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, recordMatchErrors });
513
166
  }
514
167
  function handleRecordMatch(isMatch, newMatch) {
515
168
  const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;
516
169
  const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);
517
170
  const newRecordNonMatchCount = isMatch ? recordNonMatchCount : recordNonMatchCount + 1;
518
- debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);
519
-
520
- // eslint-disable-next-line functional/no-conditional-statements
171
+ debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches2})`);
521
172
  if (returnNonMatches) {
522
173
  debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);
523
174
  }
524
175
  debugData(`- Total nonMatchCount after this detection: ${recordNonMatchCount}`);
525
- if (maxMatchesFound({
526
- matches: matches.concat(newRecordMatches),
527
- maxMatches
528
- })) {
529
- debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
530
- return {
531
- matches: newRecordMatches,
532
- nonMatches: returnNonMatches ? newRecordNonMatches : [],
533
- duplicateCount: recordDuplicateCount,
534
- nonMatchCount: newRecordNonMatchCount,
535
- matchErrors: recordMatchErrors
536
- };
176
+ if (maxMatchesFound({ matches: matches.concat(newRecordMatches), maxMatches: maxMatches2 })) {
177
+ debug(`MaxMatches (${maxMatches2}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
178
+ return { matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors };
537
179
  }
538
- return iterateRecords({
539
- records: records.slice(1),
540
- recordSetSize,
541
- maxMatches,
542
- matches,
543
- recordMatches: newRecordMatches,
544
- recordCount: newRecordCount,
545
- recordNonMatches: returnNonMatches ? newRecordNonMatches : [],
546
- duplicateCount: recordDuplicateCount,
547
- recordNonMatchCount: newRecordNonMatchCount,
548
- matchErrors: recordMatchErrors
549
- });
180
+ return iterateRecords({ records: records.slice(1), recordSetSize, maxMatches: maxMatches2, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors });
550
181
  }
551
- function candidateNotInMatches(matches, candidate) {
552
- debug(`Checking that record ${candidate.id} is not already included in ${matches.length} matches/nonMatches`);
553
- const newCandidateId = candidate.id;
182
+ function candidateNotInMatches(matches2, candidate2) {
183
+ debug(`Checking that record ${candidate2.id} is not already included in ${matches2.length} matches/nonMatches`);
184
+ const newCandidateId = candidate2.id;
554
185
  debugData(`newCandidateId: ${newCandidateId}`);
555
- const result = matches.find(({
556
- candidate
557
- }) => candidate.id === newCandidateId);
186
+ const result = matches2.find(({ candidate: candidate3 }) => candidate3.id === newCandidateId);
558
187
  debugData(`Result: ${result}`);
559
188
  if (result) {
560
- debug(`${candidate.id} was already handled.`);
189
+ debug(`${candidate2.id} was already handled.`);
561
190
  return false;
562
191
  }
563
- debug(`${candidate.id} not found in matches/nonMatches`);
192
+ debug(`${candidate2.id} not found in matches/nonMatches`);
564
193
  return true;
565
194
  }
566
195
  }
567
- function maxMatchesFound({
568
- matches,
569
- maxMatches
570
- }) {
571
- if (maxMatches && matches.length >= maxMatches) {
572
- debug(`Stopping recordSet iteration: maxMatches (${maxMatches}) for matcher job found.`);
196
+ function maxMatchesFound({ matches, maxMatches: maxMatches2 }) {
197
+ if (maxMatches2 && matches.length >= maxMatches2) {
198
+ debug(`Stopping recordSet iteration: maxMatches (${maxMatches2}) for matcher job found.`);
573
199
  return true;
574
200
  }
575
201
  }
576
- };
202
+ }
577
203
  };
578
- exports.default = _default;
579
- //# sourceMappingURL=index.js.map
204
+ //# sourceMappingURL=index.js.map