@natlibfi/melinda-record-matching 4.4.0-alpha.1 → 5.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/melinda-node-tests.yml +13 -9
- package/LICENSE.txt +21 -165
- package/README.md +2 -2
- package/dist/candidate-search/candidate-search-utils.js +6 -43
- package/dist/candidate-search/candidate-search-utils.js.map +7 -1
- package/dist/candidate-search/choose-queries.js +75 -0
- package/dist/candidate-search/choose-queries.js.map +7 -0
- package/dist/candidate-search/index.js +89 -225
- package/dist/candidate-search/index.js.map +7 -1
- package/dist/candidate-search/index.test.js +92 -0
- package/dist/candidate-search/index.test.js.map +7 -0
- package/dist/candidate-search/query-list/bib.js +141 -320
- package/dist/candidate-search/query-list/bib.js.map +7 -1
- package/dist/candidate-search/query-list/bib.test.js +34 -0
- package/dist/candidate-search/query-list/bib.test.js.map +7 -0
- package/dist/candidate-search/query-list/component.js +87 -0
- package/dist/candidate-search/query-list/component.js.map +7 -0
- package/dist/candidate-search/query-list/index.js +39 -52
- package/dist/candidate-search/query-list/index.js.map +7 -1
- package/dist/cli.js +138 -0
- package/dist/cli.js.map +7 -0
- package/dist/index.js +87 -462
- package/dist/index.js.map +7 -1
- package/dist/index.test.js +69 -0
- package/dist/index.test.js.map +7 -0
- package/dist/match-detection/features/bib/all-source-ids.js +20 -91
- package/dist/match-detection/features/bib/all-source-ids.js.map +7 -1
- package/dist/match-detection/features/bib/authors.js +20 -76
- package/dist/match-detection/features/bib/authors.js.map +7 -1
- package/dist/match-detection/features/bib/bibliographic-level.js +4 -40
- package/dist/match-detection/features/bib/bibliographic-level.js.map +7 -1
- package/dist/match-detection/features/bib/host-component.js +5 -41
- package/dist/match-detection/features/bib/host-component.js.map +7 -1
- package/dist/match-detection/features/bib/index.js +17 -119
- package/dist/match-detection/features/bib/index.js.map +7 -1
- package/dist/match-detection/features/bib/index.test.js +40 -0
- package/dist/match-detection/features/bib/index.test.js.map +7 -0
- package/dist/match-detection/features/bib/isbn.js +13 -69
- package/dist/match-detection/features/bib/isbn.js.map +7 -1
- package/dist/match-detection/features/bib/issn.js +5 -50
- package/dist/match-detection/features/bib/issn.js.map +7 -1
- package/dist/match-detection/features/bib/language.js +25 -86
- package/dist/match-detection/features/bib/language.js.map +7 -1
- package/dist/match-detection/features/bib/media-type.js +12 -64
- package/dist/match-detection/features/bib/media-type.js.map +7 -1
- package/dist/match-detection/features/bib/melinda-id.js +5 -47
- package/dist/match-detection/features/bib/melinda-id.js.map +7 -1
- package/dist/match-detection/features/bib/melinda-identifier-factory.js +16 -67
- package/dist/match-detection/features/bib/melinda-identifier-factory.js.map +7 -1
- package/dist/match-detection/features/bib/other-standard-identifier.js +5 -51
- package/dist/match-detection/features/bib/other-standard-identifier.js.map +7 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +49 -189
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +7 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js +10 -53
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map +7 -1
- package/dist/match-detection/features/bib/publication-time.js +8 -46
- package/dist/match-detection/features/bib/publication-time.js.map +7 -1
- package/dist/match-detection/features/bib/record-type.js +4 -43
- package/dist/match-detection/features/bib/record-type.js.map +7 -1
- package/dist/match-detection/features/bib/standard-identifier-factory.js +47 -163
- package/dist/match-detection/features/bib/standard-identifier-factory.js.map +7 -1
- package/dist/match-detection/features/bib/title-version-original.js +11 -53
- package/dist/match-detection/features/bib/title-version-original.js.map +7 -1
- package/dist/match-detection/features/bib/title.js +20 -72
- package/dist/match-detection/features/bib/title.js.map +7 -1
- package/dist/match-detection/features/index.js +3 -11
- package/dist/match-detection/features/index.js.map +7 -1
- package/dist/match-detection/index.js +35 -150
- package/dist/match-detection/index.js.map +7 -1
- package/dist/match-detection/index.test.js +44 -0
- package/dist/match-detection/index.test.js.map +7 -0
- package/dist/matching-utils.js +38 -102
- package/dist/matching-utils.js.map +7 -1
- package/eslint.config.js +52 -0
- package/example.env +4 -0
- package/package.json +29 -78
- package/src/candidate-search/candidate-search-utils.js +1 -28
- package/src/candidate-search/choose-queries.js +93 -0
- package/src/candidate-search/index.js +85 -119
- package/src/candidate-search/{index.spec.js → index.test.js} +30 -43
- package/src/candidate-search/query-list/bib.js +54 -55
- package/src/candidate-search/query-list/bib.test.js +38 -0
- package/src/candidate-search/query-list/component.js +127 -0
- package/src/candidate-search/query-list/index.js +31 -35
- package/src/cli.js +177 -0
- package/src/index.js +27 -68
- package/src/{index.spec.js → index.test.js} +11 -42
- package/src/match-detection/features/bib/all-source-ids.js +2 -29
- package/src/match-detection/features/bib/authors.js +5 -30
- package/src/match-detection/features/bib/bibliographic-level.js +0 -27
- package/src/match-detection/features/bib/host-component.js +0 -27
- package/src/match-detection/features/bib/index.js +16 -43
- package/src/match-detection/features/bib/index.test.js +52 -0
- package/src/match-detection/features/bib/isbn.js +2 -29
- package/src/match-detection/features/bib/issn.js +1 -28
- package/src/match-detection/features/bib/language.js +5 -32
- package/src/match-detection/features/bib/media-type.js +2 -28
- package/src/match-detection/features/bib/melinda-id.js +1 -28
- package/src/match-detection/features/bib/melinda-identifier-factory.js +5 -32
- package/src/match-detection/features/bib/other-standard-identifier.js +1 -28
- package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +7 -34
- package/src/match-detection/features/bib/publication-time-allow-cons-years.js +1 -28
- package/src/match-detection/features/bib/publication-time.js +2 -28
- package/src/match-detection/features/bib/record-type.js +0 -27
- package/src/match-detection/features/bib/standard-identifier-factory.js +1 -28
- package/src/match-detection/features/bib/title-version-original.js +2 -28
- package/src/match-detection/features/bib/title.js +6 -34
- package/src/match-detection/features/index.js +1 -28
- package/src/match-detection/index.js +1 -28
- package/src/match-detection/{index.spec.js → index.test.js} +9 -35
- package/src/matching-utils.js +8 -37
- package/dist/candidate-search/index.spec.js +0 -144
- package/dist/candidate-search/index.spec.js.map +0 -1
- package/dist/candidate-search/query-list/bib.spec.js +0 -64
- package/dist/candidate-search/query-list/bib.spec.js.map +0 -1
- package/dist/index.spec.js +0 -127
- package/dist/index.spec.js.map +0 -1
- package/dist/match-detection/features/bib/index.spec.js +0 -93
- package/dist/match-detection/features/bib/index.spec.js.map +0 -1
- package/dist/match-detection/index.spec.js +0 -91
- package/dist/match-detection/index.spec.js.map +0 -1
- package/src/candidate-search/query-list/bib.spec.js +0 -54
- package/src/match-detection/features/bib/index.spec.js +0 -78
package/dist/index.js
CHANGED
|
@@ -1,59 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
})
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
var _candidateSearch = _interopRequireWildcard(require("./candidate-search"));
|
|
9
|
-
var candidateSearch = _candidateSearch;
|
|
10
|
-
exports.candidateSearch = _candidateSearch;
|
|
11
|
-
var _matchDetection = _interopRequireWildcard(require("./match-detection"));
|
|
12
|
-
var matchDetection = _matchDetection;
|
|
13
|
-
exports.matchDetection = _matchDetection;
|
|
14
|
-
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
|
|
15
|
-
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
|
|
16
|
-
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
17
|
-
/**
|
|
18
|
-
*
|
|
19
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
20
|
-
*
|
|
21
|
-
* Melinda record matching modules for Javascript
|
|
22
|
-
*
|
|
23
|
-
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
24
|
-
*
|
|
25
|
-
* This file is part of melinda-record-matching-js
|
|
26
|
-
*
|
|
27
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
28
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
29
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
30
|
-
* License, or (at your option) any later version.
|
|
31
|
-
*
|
|
32
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
33
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
34
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
35
|
-
* GNU Lesser General Public License for more details.
|
|
36
|
-
*
|
|
37
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
38
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
39
|
-
*
|
|
40
|
-
* @licend The above is the entire license notice
|
|
41
|
-
* for the JavaScript code in this file.
|
|
42
|
-
*
|
|
43
|
-
*/
|
|
44
|
-
//import inspect from 'util';
|
|
45
|
-
var _default = ({
|
|
46
|
-
detection: detectionOptions,
|
|
47
|
-
search: searchOptions,
|
|
48
|
-
maxMatches = 1,
|
|
49
|
-
maxCandidates = 25,
|
|
50
|
-
returnStrategy = false,
|
|
51
|
-
returnQuery = false,
|
|
52
|
-
returnNonMatches = false,
|
|
53
|
-
returnFailures = false
|
|
54
|
-
}) => {
|
|
55
|
-
const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:index');
|
|
56
|
-
const debugData = debug.extend('data');
|
|
1
|
+
import createDebugLogger from "debug";
|
|
2
|
+
import createSearchInterface, * as candidateSearch from "./candidate-search/index.js";
|
|
3
|
+
import createDetectionInterface, * as matchDetection from "./match-detection/index.js";
|
|
4
|
+
export { candidateSearch, matchDetection };
|
|
5
|
+
export default ({ detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false }) => {
|
|
6
|
+
const debug = createDebugLogger("@natlibfi/melinda-record-matching:index");
|
|
7
|
+
const debugData = debug.extend("data");
|
|
57
8
|
debugData(`DetectionOptions: ${JSON.stringify(detectionOptions)}`);
|
|
58
9
|
debugData(`SearchOptions: ${JSON.stringify(searchOptions)}`);
|
|
59
10
|
debugData(`MaxMatches: ${JSON.stringify(maxMatches)}`);
|
|
@@ -61,416 +12,134 @@ var _default = ({
|
|
|
61
12
|
debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
|
|
62
13
|
debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);
|
|
63
14
|
debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
record,
|
|
68
|
-
recordExternal = {
|
|
69
|
-
recordSource: 'incomingRecord',
|
|
70
|
-
label: 'ic'
|
|
71
|
-
}
|
|
72
|
-
}) => {
|
|
73
|
-
const search = (0, _candidateSearch.default)({
|
|
74
|
-
...searchOptions,
|
|
75
|
-
record,
|
|
76
|
-
maxCandidates,
|
|
77
|
-
recordExternal
|
|
78
|
-
});
|
|
15
|
+
const detect = createDetectionInterface(detectionOptions, returnStrategy);
|
|
16
|
+
return prepareSearch;
|
|
17
|
+
async function prepareSearch({ record, recordExternal = { recordSource: "incomingRecord", label: "ic" } }) {
|
|
18
|
+
const { search } = await createSearchInterface({ ...searchOptions, record, maxCandidates, recordExternal });
|
|
79
19
|
return iterate({});
|
|
80
|
-
|
|
81
|
-
// candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set
|
|
82
|
-
// matches : candidates that have been detected as matches by current matcher job
|
|
83
|
-
// nonMatches : candidates that have been detected as non-matches by current matcher job (only if returnNonMatches is 'true')
|
|
84
|
-
// duplicateCount : amount of candidate records that were retrieved from the SRU but not handled further because they were already found in the matches/nonMatches
|
|
85
|
-
|
|
86
|
-
// state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
|
|
87
|
-
// state.query : current query (undefined if there was no queries left)
|
|
88
|
-
// state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
|
|
89
|
-
// state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)
|
|
90
|
-
// state.queriesLeft : amount of queries left
|
|
91
|
-
// state.queryCounter : sequence for current query
|
|
92
|
-
// state.maxedQueries : queries that resulted in more than serverMaxResults hits
|
|
93
|
-
|
|
94
|
-
async function iterate({
|
|
95
|
-
initialState = {},
|
|
96
|
-
matches = [],
|
|
97
|
-
candidateCount = 0,
|
|
98
|
-
nonMatches = [],
|
|
99
|
-
duplicateCount = 0,
|
|
100
|
-
nonMatchCount = 0,
|
|
101
|
-
conversionFailures = [],
|
|
102
|
-
matchErrors = []
|
|
103
|
-
}) {
|
|
20
|
+
async function iterate({ initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, matchErrors = [] }) {
|
|
104
21
|
debugData(`Starting next matcher iteration.`);
|
|
105
|
-
const {
|
|
106
|
-
|
|
107
|
-
failures,
|
|
108
|
-
...state
|
|
109
|
-
} = await search(initialState);
|
|
110
|
-
debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, conversionFailures: ${conversionFailures}, matchErrors: ${matchErrors.length}`);
|
|
22
|
+
const { records, ...state } = await search(initialState);
|
|
23
|
+
debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, matchErrors: ${matchErrors.length}`);
|
|
111
24
|
const recordSetSize = records.length;
|
|
112
|
-
const
|
|
113
|
-
const newCandidateCount = candidateCount + recordSetSize + failureSetSize;
|
|
114
|
-
const newConversionFailures = conversionFailures.concat(failures);
|
|
115
|
-
debugData(`Failures: ${failures.length}, ConversionFailures: ${conversionFailures.length}, NewConversionFailures: ${newConversionFailures.length}`);
|
|
25
|
+
const newCandidateCount = candidateCount + recordSetSize;
|
|
116
26
|
if (recordSetSize > 0) {
|
|
117
27
|
return handleRecordSet();
|
|
118
28
|
}
|
|
119
29
|
if (state.queriesLeft > 0) {
|
|
120
30
|
debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);
|
|
121
|
-
return iterate({
|
|
122
|
-
initialState: state,
|
|
123
|
-
matches,
|
|
124
|
-
candidateCount: newCandidateCount,
|
|
125
|
-
nonMatches,
|
|
126
|
-
nonMatchCount,
|
|
127
|
-
duplicateCount,
|
|
128
|
-
conversionFailures: newConversionFailures,
|
|
129
|
-
matchErrors
|
|
130
|
-
});
|
|
31
|
+
return iterate({ initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, matchErrors });
|
|
131
32
|
}
|
|
132
33
|
debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);
|
|
133
|
-
return returnResult({
|
|
134
|
-
matches,
|
|
135
|
-
state,
|
|
136
|
-
stopReason: '',
|
|
137
|
-
nonMatches,
|
|
138
|
-
nonMatchCount,
|
|
139
|
-
candidateCount: newCandidateCount,
|
|
140
|
-
duplicateCount,
|
|
141
|
-
conversionFailures: newConversionFailures,
|
|
142
|
-
matchErrors
|
|
143
|
-
});
|
|
34
|
+
return returnResult({ matches, state, stopReason: "", nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, matchErrors });
|
|
144
35
|
function handleRecordSet() {
|
|
145
36
|
debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);
|
|
146
|
-
const matchResult = iterateRecords({
|
|
147
|
-
records,
|
|
148
|
-
recordSetSize,
|
|
149
|
-
maxMatches,
|
|
150
|
-
matches,
|
|
151
|
-
nonMatches,
|
|
152
|
-
nonMatchCount
|
|
153
|
-
});
|
|
37
|
+
const matchResult = iterateRecords({ records, recordSetSize, maxMatches, matches, nonMatches, nonMatchCount });
|
|
154
38
|
const newDuplicateCount = duplicateCount + matchResult.duplicateCount;
|
|
155
39
|
const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;
|
|
156
|
-
const {
|
|
157
|
-
|
|
158
|
-
newNonMatches,
|
|
159
|
-
newMatchErrors
|
|
160
|
-
} = handleMatchResult(matchResult, matches, nonMatches, matchErrors);
|
|
161
|
-
if (maxMatchesFound({
|
|
162
|
-
matches: newMatches,
|
|
163
|
-
maxMatches
|
|
164
|
-
})) {
|
|
165
|
-
return returnResult({
|
|
166
|
-
matches: newMatches,
|
|
167
|
-
state,
|
|
168
|
-
stopReason: 'maxMatches',
|
|
169
|
-
nonMatches: newNonMatches,
|
|
170
|
-
duplicateCount: newDuplicateCount,
|
|
171
|
-
candidateCount: newCandidateCount,
|
|
172
|
-
nonMatchCount: newNonMatchCount,
|
|
173
|
-
conversionFailures: newConversionFailures,
|
|
174
|
-
matchErrors: newMatchErrors
|
|
175
|
-
});
|
|
40
|
+
const { newMatches, newNonMatches, newMatchErrors } = handleMatchResult(matchResult, matches, nonMatches, matchErrors);
|
|
41
|
+
if (maxMatchesFound({ matches: newMatches, maxMatches })) {
|
|
42
|
+
return returnResult({ matches: newMatches, state, stopReason: "maxMatches", nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors });
|
|
176
43
|
}
|
|
177
44
|
if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
|
|
178
|
-
return returnResult({
|
|
179
|
-
matches: newMatches,
|
|
180
|
-
state,
|
|
181
|
-
stopReason: 'maxCandidates',
|
|
182
|
-
nonMatches: newNonMatches,
|
|
183
|
-
duplicateCount: newDuplicateCount,
|
|
184
|
-
candidateCount: newCandidateCount,
|
|
185
|
-
nonMatchCount: newNonMatchCount,
|
|
186
|
-
conversionFailures: newConversionFailures,
|
|
187
|
-
matchErrors: newMatchErrors
|
|
188
|
-
});
|
|
45
|
+
return returnResult({ matches: newMatches, state, stopReason: "maxCandidates", nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors });
|
|
189
46
|
}
|
|
190
|
-
return iterate({
|
|
191
|
-
initialState: state,
|
|
192
|
-
matches: newMatches,
|
|
193
|
-
candidateCount: newCandidateCount,
|
|
194
|
-
nonMatches: newNonMatches,
|
|
195
|
-
duplicateCount: newDuplicateCount,
|
|
196
|
-
nonMatchCount: newNonMatchCount,
|
|
197
|
-
conversionFailures: newConversionFailures,
|
|
198
|
-
matchErrors: newMatchErrors
|
|
199
|
-
});
|
|
47
|
+
return iterate({ initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors });
|
|
200
48
|
}
|
|
201
|
-
function handleMatchResult(matchResult,
|
|
49
|
+
function handleMatchResult(matchResult, matches2, nonMatches2, matchErrors2) {
|
|
202
50
|
debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);
|
|
203
|
-
// eslint-disable-next-line functional/no-conditional-statements
|
|
204
51
|
if (returnNonMatches) {
|
|
205
52
|
debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);
|
|
206
53
|
}
|
|
207
|
-
const newMatches =
|
|
208
|
-
const newNonMatches = returnNonMatches ?
|
|
209
|
-
const newMatchErrors =
|
|
54
|
+
const newMatches = matches2.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);
|
|
55
|
+
const newNonMatches = returnNonMatches ? nonMatches2.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];
|
|
56
|
+
const newMatchErrors = matchErrors2.concat(matchResult.matchErrors);
|
|
210
57
|
debugData(`- Total amount of matches: ${newMatches.length}`);
|
|
211
|
-
// eslint-disable-next-line functional/no-conditional-statements
|
|
212
58
|
if (returnNonMatches) {
|
|
213
59
|
debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);
|
|
214
60
|
}
|
|
215
61
|
debugData(`MatchResult: ${JSON.stringify(matchResult)}`);
|
|
216
|
-
debugData(`Old matchErrors: ${JSON.stringify(
|
|
62
|
+
debugData(`Old matchErrors: ${JSON.stringify(matchErrors2)}, matchErrors from matchResult: ${JSON.stringify(matchResult.matchErrors)}, New matchErrors: ${JSON.stringify(newMatchErrors)}`);
|
|
217
63
|
debugData(`- Total amount of matchErrors: ${newMatchErrors.length}`);
|
|
218
|
-
return {
|
|
219
|
-
newMatches,
|
|
220
|
-
newNonMatches,
|
|
221
|
-
newMatchErrors
|
|
222
|
-
};
|
|
64
|
+
return { newMatches, newNonMatches, newMatchErrors };
|
|
223
65
|
}
|
|
224
|
-
function addQuery(
|
|
66
|
+
function addQuery(matches2) {
|
|
225
67
|
debugData(`Adding query ${state.query} to matches`);
|
|
226
|
-
return
|
|
227
|
-
...match,
|
|
228
|
-
matchQuery: state.query
|
|
229
|
-
}));
|
|
68
|
+
return matches2.map((match) => ({ ...match, matchQuery: state.query }));
|
|
230
69
|
}
|
|
231
|
-
function maxCandidatesRetrieved(
|
|
232
|
-
debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${
|
|
233
|
-
if (
|
|
234
|
-
debug(`Stopped matching because maximum number of candidate records ${
|
|
70
|
+
function maxCandidatesRetrieved(candidateCount2, maxCandidates2) {
|
|
71
|
+
debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${maxCandidates2})`);
|
|
72
|
+
if (maxCandidates2 && candidateCount2 >= maxCandidates2) {
|
|
73
|
+
debug(`Stopped matching because maximum number of candidate records ${candidateCount2} / ${maxCandidates2} have been retrieved`);
|
|
235
74
|
return true;
|
|
236
75
|
}
|
|
237
76
|
}
|
|
238
77
|
}
|
|
239
|
-
|
|
240
|
-
// matches : array of matching candidate records
|
|
241
|
-
// nonMatches : array of nonMatching candidate records (if returnNonMatches option is true, otherwise empty array)
|
|
242
|
-
// - candidate.id
|
|
243
|
-
// - candidate.record
|
|
244
|
-
// - probability
|
|
245
|
-
// - strategy (if returnStrategy option is true)
|
|
246
|
-
// - treshold (if returnStrategy option is true)
|
|
247
|
-
// - matchQuery (if returnQuery option is true)
|
|
248
|
-
// failures: array of conversionFailures from candidate-search and matchErrors from matchDetection in error format {status, payload: {message, id}} if returnFailures is true
|
|
249
|
-
|
|
250
|
-
// we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data
|
|
251
|
-
|
|
252
|
-
// matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not
|
|
253
|
-
// matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
|
|
254
|
-
// - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
|
|
255
|
-
// - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
|
|
256
|
-
|
|
257
|
-
function returnResult({
|
|
258
|
-
matches,
|
|
259
|
-
state,
|
|
260
|
-
stopReason,
|
|
261
|
-
nonMatches,
|
|
262
|
-
duplicateCount,
|
|
263
|
-
candidateCount,
|
|
264
|
-
nonMatchCount,
|
|
265
|
-
conversionFailures,
|
|
266
|
-
matchErrors
|
|
267
|
-
}) {
|
|
268
|
-
const conversionFailureCount = conversionFailures.length;
|
|
78
|
+
function returnResult({ matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, matchErrors }) {
|
|
269
79
|
const matchErrorCount = matchErrors.length;
|
|
270
|
-
checkCounts({
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
duplicateCount,
|
|
275
|
-
nonMatchCount,
|
|
276
|
-
conversionFailureCount,
|
|
277
|
-
matchErrorCount
|
|
278
|
-
});
|
|
279
|
-
const matchStatus = getMatchState(state, stopReason, conversionFailureCount, matchErrorCount);
|
|
280
|
-
// add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
|
|
281
|
-
const matchesResult = returnNonMatches ? {
|
|
282
|
-
matches,
|
|
283
|
-
matchStatus,
|
|
284
|
-
nonMatches,
|
|
285
|
-
candidateCount
|
|
286
|
-
} : {
|
|
287
|
-
matches,
|
|
288
|
-
matchStatus,
|
|
289
|
-
candidateCount
|
|
290
|
-
};
|
|
291
|
-
const failures = [...conversionFailures, ...matchErrors];
|
|
292
|
-
const result = returnFailures ? {
|
|
293
|
-
...matchesResult,
|
|
294
|
-
conversionFailures: failures
|
|
295
|
-
} : matchesResult;
|
|
296
|
-
debugData(`ReturnFailures ${returnFailures}`);
|
|
80
|
+
checkCounts({ matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, matchErrorCount });
|
|
81
|
+
const matchStatus = getMatchState(state, stopReason, matchErrorCount);
|
|
82
|
+
const matchesResult = returnNonMatches ? { matches, matchStatus, nonMatches, candidateCount } : { matches, matchStatus, candidateCount };
|
|
83
|
+
const result = matchesResult;
|
|
297
84
|
debugData(`${JSON.stringify(result)}`);
|
|
298
85
|
return result;
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
function checkCounts({
|
|
303
|
-
matches,
|
|
304
|
-
nonMatches,
|
|
305
|
-
candidateCount,
|
|
306
|
-
duplicateCount,
|
|
307
|
-
nonMatchCount,
|
|
308
|
-
conversionFailureCount,
|
|
309
|
-
matchErrorCount
|
|
310
|
-
}) {
|
|
311
|
-
const matchCount = matches.length;
|
|
86
|
+
function checkCounts({ matches: matches2, nonMatches: nonMatches2, candidateCount: candidateCount2, duplicateCount: duplicateCount2, nonMatchCount: nonMatchCount2, matchErrorCount: matchErrorCount2 }) {
|
|
87
|
+
const matchCount = matches2.length;
|
|
312
88
|
debugData(`Return nonMatches: ${returnNonMatches}`);
|
|
313
|
-
const chosenNonMatchCount = returnNonMatches ?
|
|
314
|
-
const totalHandled = matchCount + chosenNonMatchCount +
|
|
315
|
-
debug(`candidateCount: ${
|
|
316
|
-
debug(`We got result for ${totalHandled} / ${
|
|
317
|
-
if (totalHandled !==
|
|
318
|
-
debug(`WARNING: Missing results for ${
|
|
89
|
+
const chosenNonMatchCount = returnNonMatches ? nonMatches2.length : nonMatchCount2;
|
|
90
|
+
const totalHandled = matchCount + chosenNonMatchCount + duplicateCount2;
|
|
91
|
+
debug(`candidateCount: ${candidateCount2}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount2}, matchErrorCount: ${matchErrorCount2}`);
|
|
92
|
+
debug(`We got result for ${totalHandled} / ${candidateCount2} retrieved candidates`);
|
|
93
|
+
if (totalHandled !== candidateCount2) {
|
|
94
|
+
debug(`WARNING: Missing results for ${candidateCount2 - totalHandled} candidates`);
|
|
319
95
|
return;
|
|
320
96
|
}
|
|
321
97
|
return;
|
|
322
98
|
}
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
debugData(`StopReason: <${stopReason}>`);
|
|
331
|
-
const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;
|
|
332
|
-
const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
|
|
99
|
+
function getMatchState(state2, stopReason2, matchErrorCount2) {
|
|
100
|
+
debugData(`${JSON.stringify(state2)}`);
|
|
101
|
+
debug(`We had ${matchErrorCount2} retrieved candidates that errored in matchDetection.`);
|
|
102
|
+
debug(`Queries left ${state2.queriesLeft}, Searches for current query left: ${state2.resultSetOffset && state2.resultSetOffset <= state2.totalRecords}, non-retrieved records: ${state2.totalRecords - state2.queryCandidateCounter}, maxedQueries (${state2.maxedQueries.length}): ${state2.maxedQueries}`);
|
|
103
|
+
debugData(`StopReason: <${stopReason2}>`);
|
|
104
|
+
const searchesLeft = state2.resultSetOffset && state2.resultSetOffset <= state2.totalRecords;
|
|
105
|
+
const nonRetrieved = searchesLeft ? state2.totalRecords - state2.queryCandidateCounter : 0;
|
|
333
106
|
debugData(`nonRetrieved: ${nonRetrieved}`);
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || conversionFailureCount > 0 || matchErrorCount > 0) {
|
|
339
|
-
const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : undefined;
|
|
340
|
-
const conversionFailuresStopReason = conversionFailureCount > 0 ? 'conversionFailures' : undefined;
|
|
341
|
-
const matchErrorsStopReason = matchErrorCount > 0 ? 'matchErrors' : undefined;
|
|
342
|
-
const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || conversionFailuresStopReason || matchErrorsStopReason : stopReason;
|
|
107
|
+
if (state2.queriesLeft > 0 || nonRetrieved > 0 || state2.maxedQueries.length > 0 || matchErrorCount2 > 0) {
|
|
108
|
+
const maxedQueriesStopReason = state2.maxedQueries.length > 0 ? "maxedQueries" : void 0;
|
|
109
|
+
const matchErrorsStopReason = matchErrorCount2 > 0 ? "matchErrors" : void 0;
|
|
110
|
+
const newStopReason = stopReason2 === "" || stopReason2 === void 0 ? maxedQueriesStopReason || matchErrorsStopReason : stopReason2;
|
|
343
111
|
debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);
|
|
344
|
-
debugData(`ConversionFailureStopReason <${conversionFailuresStopReason}>`);
|
|
345
112
|
debugData(`MatchErrorsStopReason <${matchErrorsStopReason}>`);
|
|
346
113
|
debugData(`NewStopReason: <${newStopReason}>`);
|
|
347
114
|
debug(`Match status: false`);
|
|
348
|
-
return {
|
|
349
|
-
status: false,
|
|
350
|
-
stopReason: newStopReason
|
|
351
|
-
};
|
|
115
|
+
return { status: false, stopReason: newStopReason };
|
|
352
116
|
}
|
|
353
117
|
debug(`Match status: true`);
|
|
354
|
-
return {
|
|
355
|
-
status: true,
|
|
356
|
-
stopReason
|
|
357
|
-
};
|
|
118
|
+
return { status: true, stopReason: stopReason2 };
|
|
358
119
|
}
|
|
359
120
|
}
|
|
360
|
-
|
|
361
|
-
// NOTES:
|
|
362
|
-
// - we could optimize by creating the featureSet for the incoming record once and using it for all database/candidateRecords
|
|
363
|
-
// - if creating the featureSet for the incoming record fails we have an unprocessable entity
|
|
364
|
-
// - if creating the featureSet for a candidate record fails we could skip that candidate - but list the case as a detectionFailure, same as conversionFailures
|
|
365
|
-
|
|
366
|
-
function iterateRecords({
|
|
367
|
-
records,
|
|
368
|
-
recordSetSize,
|
|
369
|
-
maxMatches,
|
|
370
|
-
matches = [],
|
|
371
|
-
nonMatches = [],
|
|
372
|
-
recordMatches = [],
|
|
373
|
-
recordNonMatches = [],
|
|
374
|
-
recordCount = 0,
|
|
375
|
-
recordDuplicateCount = 0,
|
|
376
|
-
recordNonMatchCount = 0,
|
|
377
|
-
recordMatchErrors = []
|
|
378
|
-
}) {
|
|
379
|
-
// recordSetSize : total amount of records in the current record set
|
|
380
|
-
// recordCount : amount of records from the current record set that have been handled
|
|
381
|
-
// maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped
|
|
382
|
-
// recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results
|
|
383
|
-
// recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)
|
|
384
|
-
|
|
385
|
-
// records : non-handled records in the current record set
|
|
386
|
-
// matches : found matches in the current matcher job
|
|
387
|
-
// recordMatches : found matches in the current record set
|
|
388
|
-
// recordNonMatches : found nonMatches in the current record set (only if returnNonMatches setting is true)
|
|
389
|
-
// recordMatchErrors: errored matchDetection in the current record set
|
|
390
|
-
|
|
121
|
+
function iterateRecords({ records, recordSetSize, maxMatches: maxMatches2, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0, recordMatchErrors = [] }) {
|
|
391
122
|
const [candidate] = records;
|
|
392
123
|
const newRecordCount = candidate ? recordCount + 1 : recordCount;
|
|
393
|
-
|
|
394
|
-
// The matcher uses same matchDetection strategy for candidates from all candidate-searches -> matchDetection result for the same candidate is always same
|
|
395
|
-
// Exceptions would happen if the candidate would have been updated in the database between candidate searches
|
|
396
|
-
// Note that if returnNonMatches is false, matcher won't remember candidates that didn't match, so they will be matched again everytime they are retrieved by
|
|
397
|
-
// different candidate search queries. Same candidate search query won't have duplicate records.
|
|
398
|
-
|
|
399
|
-
/* We could optimize and detect all retrieved candidates at once
|
|
400
|
-
const candidateRecords = records.map(record => record.record);
|
|
401
|
-
const recordsIsArray = Array.isArray(candidateRecords);
|
|
402
|
-
debug(`records is an array: ${recordsIsArray}`);
|
|
403
|
-
const result = detect(record, candidateRecords);
|
|
404
|
-
debugData(`${JSON.stringify(result)}`);
|
|
405
|
-
*/
|
|
406
|
-
|
|
407
124
|
if (candidate) {
|
|
408
|
-
// eslint-disable-next-line functional/no-conditional-statements
|
|
409
125
|
if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {
|
|
410
|
-
const {
|
|
411
|
-
|
|
412
|
-
id: candidateId
|
|
413
|
-
} = candidate;
|
|
414
|
-
const recordBExternal = {
|
|
415
|
-
id: candidate.id,
|
|
416
|
-
recordSource: 'databaseRecord',
|
|
417
|
-
label: `db-${candidate.id}`
|
|
418
|
-
};
|
|
126
|
+
const { record: candidateRecord, id: candidateId } = candidate;
|
|
127
|
+
const recordBExternal = { id: candidate.id, recordSource: "databaseRecord", label: `db-${candidate.id}` };
|
|
419
128
|
try {
|
|
420
129
|
debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);
|
|
421
|
-
|
|
422
|
-
const detectionResult = detect({
|
|
423
|
-
recordA: record,
|
|
424
|
-
recordB: candidateRecord,
|
|
425
|
-
recordAExternal: recordExternal,
|
|
426
|
-
recordBExternal
|
|
427
|
-
});
|
|
130
|
+
const detectionResult = detect({ recordA: record, recordB: candidateRecord, recordAExternal: recordExternal, recordBExternal });
|
|
428
131
|
return handleDetectionResult(detectionResult, candidateId, candidateRecord);
|
|
429
132
|
} catch (error) {
|
|
430
133
|
debug(`MatchDetection errored: database record ${candidateId}: ${error}`);
|
|
431
|
-
const matchError = {
|
|
432
|
-
status: 422,
|
|
433
|
-
payload: {
|
|
434
|
-
message: `Matching errored for database record ${candidateId}. ${error.message}.`,
|
|
435
|
-
id: candidateId
|
|
436
|
-
}
|
|
437
|
-
};
|
|
134
|
+
const matchError = { status: 422, payload: { message: `Matching errored for database record ${candidateId}. ${error.message}.`, id: candidateId } };
|
|
438
135
|
const newRecordMatchErrors = recordMatchErrors.concat(matchError);
|
|
439
|
-
return iterateRecords({
|
|
440
|
-
records: records.slice(1),
|
|
441
|
-
recordSetSize,
|
|
442
|
-
maxMatches,
|
|
443
|
-
matches,
|
|
444
|
-
recordMatches,
|
|
445
|
-
recordCount: newRecordCount,
|
|
446
|
-
recordNonMatches,
|
|
447
|
-
recordDuplicateCount,
|
|
448
|
-
recordNonMatchCount,
|
|
449
|
-
recordMatchErrors: newRecordMatchErrors
|
|
450
|
-
});
|
|
136
|
+
return iterateRecords({ records: records.slice(1), recordSetSize, maxMatches: maxMatches2, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount, recordMatchErrors: newRecordMatchErrors });
|
|
451
137
|
}
|
|
452
138
|
}
|
|
453
|
-
return iterateRecords({
|
|
454
|
-
records: records.slice(1),
|
|
455
|
-
recordSetSize,
|
|
456
|
-
maxMatches,
|
|
457
|
-
matches,
|
|
458
|
-
recordMatches,
|
|
459
|
-
recordCount: newRecordCount,
|
|
460
|
-
recordNonMatches,
|
|
461
|
-
recordDuplicateCount: recordDuplicateCount + 1,
|
|
462
|
-
recordNonMatchCount,
|
|
463
|
-
recordMatchErrors
|
|
464
|
-
});
|
|
139
|
+
return iterateRecords({ records: records.slice(1), recordSetSize, maxMatches: maxMatches2, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount, recordMatchErrors });
|
|
465
140
|
}
|
|
466
141
|
debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);
|
|
467
|
-
return {
|
|
468
|
-
matches: recordMatches,
|
|
469
|
-
nonMatches: returnNonMatches ? recordNonMatches : [],
|
|
470
|
-
duplicateCount: recordDuplicateCount,
|
|
471
|
-
nonMatchCount: recordNonMatchCount,
|
|
472
|
-
matchErrors: recordMatchErrors
|
|
473
|
-
};
|
|
142
|
+
return { matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount, matchErrors: recordMatchErrors };
|
|
474
143
|
function handleDetectionResult(detectionResult, candidateId, candidateRecord) {
|
|
475
144
|
debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);
|
|
476
145
|
if (detectionResult.match || returnNonMatches) {
|
|
@@ -487,93 +156,49 @@ var _default = ({
|
|
|
487
156
|
strategy: detectionResult.strategy,
|
|
488
157
|
treshold: detectionResult.treshold
|
|
489
158
|
};
|
|
490
|
-
const newMatch = returnStrategy ? {
|
|
491
|
-
...matchResult,
|
|
492
|
-
...strategyResult
|
|
493
|
-
} : {
|
|
494
|
-
...matchResult
|
|
495
|
-
};
|
|
159
|
+
const newMatch = returnStrategy ? { ...matchResult, ...strategyResult } : { ...matchResult };
|
|
496
160
|
debugData(`${JSON.stringify(newMatch)}`);
|
|
497
161
|
return handleRecordMatch(detectionResult.match, newMatch);
|
|
498
162
|
}
|
|
499
163
|
const newRecordNonMatchCount = recordNonMatchCount + 1;
|
|
500
164
|
debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);
|
|
501
|
-
return iterateRecords({
|
|
502
|
-
records: records.slice(1),
|
|
503
|
-
recordSetSize,
|
|
504
|
-
maxMatches,
|
|
505
|
-
matches,
|
|
506
|
-
recordMatches,
|
|
507
|
-
recordCount: newRecordCount,
|
|
508
|
-
recordNonMatches,
|
|
509
|
-
recordDuplicateCount,
|
|
510
|
-
recordNonMatchCount: newRecordNonMatchCount,
|
|
511
|
-
recordMatchErrors
|
|
512
|
-
});
|
|
165
|
+
return iterateRecords({ records: records.slice(1), recordSetSize, maxMatches: maxMatches2, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, recordMatchErrors });
|
|
513
166
|
}
|
|
514
167
|
function handleRecordMatch(isMatch, newMatch) {
|
|
515
168
|
const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;
|
|
516
169
|
const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);
|
|
517
170
|
const newRecordNonMatchCount = isMatch ? recordNonMatchCount : recordNonMatchCount + 1;
|
|
518
|
-
debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${
|
|
519
|
-
|
|
520
|
-
// eslint-disable-next-line functional/no-conditional-statements
|
|
171
|
+
debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches2})`);
|
|
521
172
|
if (returnNonMatches) {
|
|
522
173
|
debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);
|
|
523
174
|
}
|
|
524
175
|
debugData(`- Total nonMatchCount after this detection: ${recordNonMatchCount}`);
|
|
525
|
-
if (maxMatchesFound({
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
})) {
|
|
529
|
-
debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
|
|
530
|
-
return {
|
|
531
|
-
matches: newRecordMatches,
|
|
532
|
-
nonMatches: returnNonMatches ? newRecordNonMatches : [],
|
|
533
|
-
duplicateCount: recordDuplicateCount,
|
|
534
|
-
nonMatchCount: newRecordNonMatchCount,
|
|
535
|
-
matchErrors: recordMatchErrors
|
|
536
|
-
};
|
|
176
|
+
if (maxMatchesFound({ matches: matches.concat(newRecordMatches), maxMatches: maxMatches2 })) {
|
|
177
|
+
debug(`MaxMatches (${maxMatches2}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
|
|
178
|
+
return { matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors };
|
|
537
179
|
}
|
|
538
|
-
return iterateRecords({
|
|
539
|
-
records: records.slice(1),
|
|
540
|
-
recordSetSize,
|
|
541
|
-
maxMatches,
|
|
542
|
-
matches,
|
|
543
|
-
recordMatches: newRecordMatches,
|
|
544
|
-
recordCount: newRecordCount,
|
|
545
|
-
recordNonMatches: returnNonMatches ? newRecordNonMatches : [],
|
|
546
|
-
duplicateCount: recordDuplicateCount,
|
|
547
|
-
recordNonMatchCount: newRecordNonMatchCount,
|
|
548
|
-
matchErrors: recordMatchErrors
|
|
549
|
-
});
|
|
180
|
+
return iterateRecords({ records: records.slice(1), recordSetSize, maxMatches: maxMatches2, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors });
|
|
550
181
|
}
|
|
551
|
-
function candidateNotInMatches(
|
|
552
|
-
debug(`Checking that record ${
|
|
553
|
-
const newCandidateId =
|
|
182
|
+
function candidateNotInMatches(matches2, candidate2) {
|
|
183
|
+
debug(`Checking that record ${candidate2.id} is not already included in ${matches2.length} matches/nonMatches`);
|
|
184
|
+
const newCandidateId = candidate2.id;
|
|
554
185
|
debugData(`newCandidateId: ${newCandidateId}`);
|
|
555
|
-
const result =
|
|
556
|
-
candidate
|
|
557
|
-
}) => candidate.id === newCandidateId);
|
|
186
|
+
const result = matches2.find(({ candidate: candidate3 }) => candidate3.id === newCandidateId);
|
|
558
187
|
debugData(`Result: ${result}`);
|
|
559
188
|
if (result) {
|
|
560
|
-
debug(`${
|
|
189
|
+
debug(`${candidate2.id} was already handled.`);
|
|
561
190
|
return false;
|
|
562
191
|
}
|
|
563
|
-
debug(`${
|
|
192
|
+
debug(`${candidate2.id} not found in matches/nonMatches`);
|
|
564
193
|
return true;
|
|
565
194
|
}
|
|
566
195
|
}
|
|
567
|
-
function maxMatchesFound({
|
|
568
|
-
matches
|
|
569
|
-
|
|
570
|
-
}) {
|
|
571
|
-
if (maxMatches && matches.length >= maxMatches) {
|
|
572
|
-
debug(`Stopping recordSet iteration: maxMatches (${maxMatches}) for matcher job found.`);
|
|
196
|
+
function maxMatchesFound({ matches, maxMatches: maxMatches2 }) {
|
|
197
|
+
if (maxMatches2 && matches.length >= maxMatches2) {
|
|
198
|
+
debug(`Stopping recordSet iteration: maxMatches (${maxMatches2}) for matcher job found.`);
|
|
573
199
|
return true;
|
|
574
200
|
}
|
|
575
201
|
}
|
|
576
|
-
}
|
|
202
|
+
}
|
|
577
203
|
};
|
|
578
|
-
|
|
579
|
-
//# sourceMappingURL=index.js.map
|
|
204
|
+
//# sourceMappingURL=index.js.map
|