@natlibfi/melinda-record-matching 4.3.5 → 5.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/.github/workflows/melinda-node-tests.yml +11 -7
  2. package/eslint.config.js +52 -0
  3. package/example.env +4 -0
  4. package/package.json +22 -90
  5. package/src/candidate-search/choose-queries.js +1 -3
  6. package/src/candidate-search/index.js +63 -89
  7. package/src/candidate-search/{index.spec.js → index.test.js} +19 -18
  8. package/src/candidate-search/query-list/bib.js +10 -4
  9. package/src/candidate-search/query-list/{bib.spec.js → bib.test.js} +8 -7
  10. package/src/candidate-search/query-list/component.js +127 -0
  11. package/src/candidate-search/query-list/index.js +23 -9
  12. package/src/cli.js +177 -0
  13. package/src/index.js +22 -37
  14. package/src/{index.spec.js → index.test.js} +11 -14
  15. package/src/match-detection/features/bib/all-source-ids.js +2 -2
  16. package/src/match-detection/features/bib/authors.js +3 -2
  17. package/src/match-detection/features/bib/index.js +16 -16
  18. package/src/match-detection/features/bib/{index.spec.js → index.test.js} +7 -6
  19. package/src/match-detection/features/bib/isbn.js +2 -2
  20. package/src/match-detection/features/bib/issn.js +1 -1
  21. package/src/match-detection/features/bib/language.js +5 -5
  22. package/src/match-detection/features/bib/media-type.js +1 -1
  23. package/src/match-detection/features/bib/melinda-id.js +1 -1
  24. package/src/match-detection/features/bib/melinda-identifier-factory.js +5 -5
  25. package/src/match-detection/features/bib/other-standard-identifier.js +1 -1
  26. package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +6 -6
  27. package/src/match-detection/features/bib/publication-time-allow-cons-years.js +1 -1
  28. package/src/match-detection/features/bib/publication-time.js +1 -1
  29. package/src/match-detection/features/bib/standard-identifier-factory.js +1 -2
  30. package/src/match-detection/features/bib/title-version-original.js +2 -1
  31. package/src/match-detection/features/bib/title.js +6 -5
  32. package/src/match-detection/features/index.js +1 -1
  33. package/src/match-detection/index.js +1 -1
  34. package/src/match-detection/{index.spec.js → index.test.js} +9 -8
  35. package/src/matching-utils.js +8 -9
  36. package/dist/candidate-search/candidate-search-utils.js +0 -29
  37. package/dist/candidate-search/candidate-search-utils.js.map +0 -1
  38. package/dist/candidate-search/choose-queries.js +0 -122
  39. package/dist/candidate-search/choose-queries.js.map +0 -1
  40. package/dist/candidate-search/index.js +0 -260
  41. package/dist/candidate-search/index.js.map +0 -1
  42. package/dist/candidate-search/index.spec.js +0 -133
  43. package/dist/candidate-search/index.spec.js.map +0 -1
  44. package/dist/candidate-search/query-list/bib.js +0 -458
  45. package/dist/candidate-search/query-list/bib.js.map +0 -1
  46. package/dist/candidate-search/query-list/bib.spec.js +0 -46
  47. package/dist/candidate-search/query-list/bib.spec.js.map +0 -1
  48. package/dist/candidate-search/query-list/index.js +0 -47
  49. package/dist/candidate-search/query-list/index.js.map +0 -1
  50. package/dist/index.js +0 -555
  51. package/dist/index.js.map +0 -1
  52. package/dist/index.spec.js +0 -99
  53. package/dist/index.spec.js.map +0 -1
  54. package/dist/match-detection/features/bib/all-source-ids.js +0 -91
  55. package/dist/match-detection/features/bib/all-source-ids.js.map +0 -1
  56. package/dist/match-detection/features/bib/authors.js +0 -86
  57. package/dist/match-detection/features/bib/authors.js.map +0 -1
  58. package/dist/match-detection/features/bib/bibliographic-level.js +0 -15
  59. package/dist/match-detection/features/bib/bibliographic-level.js.map +0 -1
  60. package/dist/match-detection/features/bib/host-component.js +0 -15
  61. package/dist/match-detection/features/bib/host-component.js.map +0 -1
  62. package/dist/match-detection/features/bib/index.js +0 -119
  63. package/dist/match-detection/features/bib/index.js.map +0 -1
  64. package/dist/match-detection/features/bib/index.spec.js +0 -65
  65. package/dist/match-detection/features/bib/index.spec.js.map +0 -1
  66. package/dist/match-detection/features/bib/isbn.js +0 -49
  67. package/dist/match-detection/features/bib/isbn.js.map +0 -1
  68. package/dist/match-detection/features/bib/issn.js +0 -24
  69. package/dist/match-detection/features/bib/issn.js.map +0 -1
  70. package/dist/match-detection/features/bib/language.js +0 -113
  71. package/dist/match-detection/features/bib/language.js.map +0 -1
  72. package/dist/match-detection/features/bib/media-type.js +0 -56
  73. package/dist/match-detection/features/bib/media-type.js.map +0 -1
  74. package/dist/match-detection/features/bib/melinda-id.js +0 -21
  75. package/dist/match-detection/features/bib/melinda-id.js.map +0 -1
  76. package/dist/match-detection/features/bib/melinda-identifier-factory.js +0 -62
  77. package/dist/match-detection/features/bib/melinda-identifier-factory.js.map +0 -1
  78. package/dist/match-detection/features/bib/other-standard-identifier.js +0 -25
  79. package/dist/match-detection/features/bib/other-standard-identifier.js.map +0 -1
  80. package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +0 -229
  81. package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +0 -1
  82. package/dist/match-detection/features/bib/publication-time-allow-cons-years.js +0 -44
  83. package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map +0 -1
  84. package/dist/match-detection/features/bib/publication-time.js +0 -21
  85. package/dist/match-detection/features/bib/publication-time.js.map +0 -1
  86. package/dist/match-detection/features/bib/record-type.js +0 -18
  87. package/dist/match-detection/features/bib/record-type.js.map +0 -1
  88. package/dist/match-detection/features/bib/standard-identifier-factory.js +0 -188
  89. package/dist/match-detection/features/bib/standard-identifier-factory.js.map +0 -1
  90. package/dist/match-detection/features/bib/title-version-original.js +0 -52
  91. package/dist/match-detection/features/bib/title-version-original.js.map +0 -1
  92. package/dist/match-detection/features/bib/title.js +0 -70
  93. package/dist/match-detection/features/bib/title.js.map +0 -1
  94. package/dist/match-detection/features/index.js +0 -11
  95. package/dist/match-detection/features/index.js.map +0 -1
  96. package/dist/match-detection/index.js +0 -174
  97. package/dist/match-detection/index.js.map +0 -1
  98. package/dist/match-detection/index.spec.js +0 -63
  99. package/dist/match-detection/index.spec.js.map +0 -1
  100. package/dist/matching-utils.js +0 -115
  101. package/dist/matching-utils.js.map +0 -1
package/dist/index.js DELETED
@@ -1,555 +0,0 @@
1
- "use strict";
2
-
3
- Object.defineProperty(exports, "__esModule", {
4
- value: true
5
- });
6
- exports.matchDetection = exports.default = exports.candidateSearch = void 0;
7
- var _debug = _interopRequireDefault(require("debug"));
8
- var _candidateSearch = _interopRequireWildcard(require("./candidate-search"));
9
- var candidateSearch = _candidateSearch;
10
- exports.candidateSearch = _candidateSearch;
11
- var _matchDetection = _interopRequireWildcard(require("./match-detection"));
12
- var matchDetection = _matchDetection;
13
- exports.matchDetection = _matchDetection;
14
- function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
15
- function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
16
- function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
17
- //import inspect from 'util';
18
- var _default = ({
19
- detection: detectionOptions,
20
- search: searchOptions,
21
- maxMatches = 1,
22
- maxCandidates = 25,
23
- returnStrategy = false,
24
- returnQuery = false,
25
- returnNonMatches = false,
26
- returnFailures = false
27
- }) => {
28
- const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:index');
29
- const debugData = debug.extend('data');
30
- debugData(`DetectionOptions: ${JSON.stringify(detectionOptions)}`);
31
- debugData(`SearchOptions: ${JSON.stringify(searchOptions)}`);
32
- debugData(`MaxMatches: ${JSON.stringify(maxMatches)}`);
33
- debugData(`MaxCandidates: ${JSON.stringify(maxCandidates)}`);
34
- debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
35
- debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);
36
- debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);
37
- debugData(`ReturnFailures: ${JSON.stringify(returnFailures)}`);
38
- const detect = (0, _matchDetection.default)(detectionOptions, returnStrategy);
39
- return prepareSearch;
40
- async function prepareSearch({
41
- record,
42
- recordExternal = {
43
- recordSource: 'incomingRecord',
44
- label: 'ic'
45
- }
46
- }) {
47
- const {
48
- search
49
- } = await (0, _candidateSearch.default)({
50
- ...searchOptions,
51
- record,
52
- maxCandidates,
53
- recordExternal
54
- });
55
- return iterate({});
56
-
57
- // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set
58
- // matches : candidates that have been detected as matches by current matcher job
59
- // nonMatches : candidates that have been detected as non-matches by current matcher job (only if returnNonMatches is 'true')
60
- // duplicateCount : amount of candidate records that were retrieved from the SRU but not handled further because they were already found in the matches/nonMatches
61
-
62
- // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
63
- // state.query : current query (undefined if there was no queries left)
64
- // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
65
- // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)
66
- // state.queriesLeft : amount of queries left
67
- // state.queryCounter : sequence for current query
68
- // state.maxedQueries : queries that resulted in more than serverMaxResults hits
69
-
70
- async function iterate({
71
- initialState = {},
72
- matches = [],
73
- candidateCount = 0,
74
- nonMatches = [],
75
- duplicateCount = 0,
76
- nonMatchCount = 0,
77
- conversionFailures = [],
78
- matchErrors = []
79
- }) {
80
- debugData(`Starting next matcher iteration.`);
81
- const {
82
- records,
83
- failures,
84
- ...state
85
- } = await search(initialState);
86
- debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, conversionFailures: ${conversionFailures}, matchErrors: ${matchErrors.length}`);
87
- const recordSetSize = records.length;
88
- const failureSetSize = failures.length;
89
- const newCandidateCount = candidateCount + recordSetSize + failureSetSize;
90
- const newConversionFailures = conversionFailures.concat(failures);
91
- debugData(`Failures: ${failures.length}, ConversionFailures: ${conversionFailures.length}, NewConversionFailures: ${newConversionFailures.length}`);
92
- if (recordSetSize > 0) {
93
- return handleRecordSet();
94
- }
95
- if (state.queriesLeft > 0) {
96
- debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);
97
- return iterate({
98
- initialState: state,
99
- matches,
100
- candidateCount: newCandidateCount,
101
- nonMatches,
102
- nonMatchCount,
103
- duplicateCount,
104
- conversionFailures: newConversionFailures,
105
- matchErrors
106
- });
107
- }
108
- debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);
109
- return returnResult({
110
- matches,
111
- state,
112
- stopReason: '',
113
- nonMatches,
114
- nonMatchCount,
115
- candidateCount: newCandidateCount,
116
- duplicateCount,
117
- conversionFailures: newConversionFailures,
118
- matchErrors
119
- });
120
- function handleRecordSet() {
121
- debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);
122
- const matchResult = iterateRecords({
123
- records,
124
- recordSetSize,
125
- maxMatches,
126
- matches,
127
- nonMatches,
128
- nonMatchCount
129
- });
130
- const newDuplicateCount = duplicateCount + matchResult.duplicateCount;
131
- const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;
132
- const {
133
- newMatches,
134
- newNonMatches,
135
- newMatchErrors
136
- } = handleMatchResult(matchResult, matches, nonMatches, matchErrors);
137
- if (maxMatchesFound({
138
- matches: newMatches,
139
- maxMatches
140
- })) {
141
- return returnResult({
142
- matches: newMatches,
143
- state,
144
- stopReason: 'maxMatches',
145
- nonMatches: newNonMatches,
146
- duplicateCount: newDuplicateCount,
147
- candidateCount: newCandidateCount,
148
- nonMatchCount: newNonMatchCount,
149
- conversionFailures: newConversionFailures,
150
- matchErrors: newMatchErrors
151
- });
152
- }
153
- if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
154
- return returnResult({
155
- matches: newMatches,
156
- state,
157
- stopReason: 'maxCandidates',
158
- nonMatches: newNonMatches,
159
- duplicateCount: newDuplicateCount,
160
- candidateCount: newCandidateCount,
161
- nonMatchCount: newNonMatchCount,
162
- conversionFailures: newConversionFailures,
163
- matchErrors: newMatchErrors
164
- });
165
- }
166
- return iterate({
167
- initialState: state,
168
- matches: newMatches,
169
- candidateCount: newCandidateCount,
170
- nonMatches: newNonMatches,
171
- duplicateCount: newDuplicateCount,
172
- nonMatchCount: newNonMatchCount,
173
- conversionFailures: newConversionFailures,
174
- matchErrors: newMatchErrors
175
- });
176
- }
177
- function handleMatchResult(matchResult, matches, nonMatches, matchErrors) {
178
- debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);
179
- // eslint-disable-next-line functional/no-conditional-statements
180
- if (returnNonMatches) {
181
- debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);
182
- }
183
- const newMatches = matches.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);
184
- const newNonMatches = returnNonMatches ? nonMatches.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];
185
- const newMatchErrors = matchErrors.concat(matchResult.matchErrors);
186
- debugData(`- Total amount of matches: ${newMatches.length}`);
187
- // eslint-disable-next-line functional/no-conditional-statements
188
- if (returnNonMatches) {
189
- debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);
190
- }
191
- debugData(`MatchResult: ${JSON.stringify(matchResult)}`);
192
- debugData(`Old matchErrors: ${JSON.stringify(matchErrors)}, matchErrors from matchResult: ${JSON.stringify(matchResult.matchErrors)}, New matchErrors: ${JSON.stringify(newMatchErrors)}`);
193
- debugData(`- Total amount of matchErrors: ${newMatchErrors.length}`);
194
- return {
195
- newMatches,
196
- newNonMatches,
197
- newMatchErrors
198
- };
199
- }
200
- function addQuery(matches) {
201
- debugData(`Adding query ${state.query} to matches`);
202
- return matches.map(match => ({
203
- ...match,
204
- matchQuery: state.query
205
- }));
206
- }
207
- function maxCandidatesRetrieved(candidateCount, maxCandidates) {
208
- debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${maxCandidates})`);
209
- if (maxCandidates && candidateCount >= maxCandidates) {
210
- debug(`Stopped matching because maximum number of candidate records ${candidateCount} / ${maxCandidates} have been retrieved`);
211
- return true;
212
- }
213
- }
214
- }
215
-
216
- // matches : array of matching candidate records
217
- // nonMatches : array of nonMatching candidate records (if returnNonMatches option is true, otherwise empty array)
218
- // - candidate.id
219
- // - candidate.record
220
- // - probability
221
- // - strategy (if returnStrategy option is true)
222
- // - treshold (if returnStrategy option is true)
223
- // - matchQuery (if returnQuery option is true)
224
- // failures: array of conversionFailures from candidate-search and matchErrors from matchDetection in error format {status, payload: {message, id}} if returnFailures is true
225
-
226
- // we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data
227
-
228
- // matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not
229
- // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
230
- // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
231
- // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
232
-
233
- function returnResult({
234
- matches,
235
- state,
236
- stopReason,
237
- nonMatches,
238
- duplicateCount,
239
- candidateCount,
240
- nonMatchCount,
241
- conversionFailures,
242
- matchErrors
243
- }) {
244
- const conversionFailureCount = conversionFailures.length;
245
- const matchErrorCount = matchErrors.length;
246
- checkCounts({
247
- matches,
248
- nonMatches,
249
- candidateCount,
250
- duplicateCount,
251
- nonMatchCount,
252
- conversionFailureCount,
253
- matchErrorCount
254
- });
255
- const matchStatus = getMatchState(state, stopReason, conversionFailureCount, matchErrorCount);
256
- // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
257
- const matchesResult = returnNonMatches ? {
258
- matches,
259
- matchStatus,
260
- nonMatches,
261
- candidateCount
262
- } : {
263
- matches,
264
- matchStatus,
265
- candidateCount
266
- };
267
- const failures = [...conversionFailures, ...matchErrors];
268
- const result = returnFailures ? {
269
- ...matchesResult,
270
- conversionFailures: failures
271
- } : matchesResult;
272
- debugData(`ReturnFailures ${returnFailures}`);
273
- debugData(`${JSON.stringify(result)}`);
274
- return result;
275
-
276
- // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
277
-
278
- function checkCounts({
279
- matches,
280
- nonMatches,
281
- candidateCount,
282
- duplicateCount,
283
- nonMatchCount,
284
- conversionFailureCount,
285
- matchErrorCount
286
- }) {
287
- const matchCount = matches.length;
288
- debugData(`Return nonMatches: ${returnNonMatches}`);
289
- const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
290
- const totalHandled = matchCount + chosenNonMatchCount + duplicateCount;
291
- debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, conversionFailureCount: ${conversionFailureCount}, matchErrorCount: ${matchErrorCount}`);
292
- debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
293
- if (totalHandled !== candidateCount) {
294
- debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);
295
- return;
296
- }
297
- return;
298
- }
299
-
300
- // eslint-disable-next-line max-statements
301
- function getMatchState(state, stopReason, conversionFailuresCount, matchErrorCount) {
302
- debugData(`${JSON.stringify(state)}`);
303
- debug(`We had ${conversionFailuresCount} retrieved candidates that could not be converted.`);
304
- debug(`We had ${matchErrorCount} retrieved candidates that errored in matchDetection.`);
305
- debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
306
- debugData(`StopReason: <${stopReason}>`);
307
- const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;
308
- const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
309
- debugData(`nonRetrieved: ${nonRetrieved}`);
310
-
311
- // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
312
- // 'maxMatches' and 'maxCandidates' are in stopReason, 'maxedQueries', 'conversionFailures' and 'matchErrors' are created here
313
-
314
- if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || conversionFailureCount > 0 || matchErrorCount > 0) {
315
- const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : undefined;
316
- const conversionFailuresStopReason = conversionFailureCount > 0 ? 'conversionFailures' : undefined;
317
- const matchErrorsStopReason = matchErrorCount > 0 ? 'matchErrors' : undefined;
318
- const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || conversionFailuresStopReason || matchErrorsStopReason : stopReason;
319
- debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);
320
- debugData(`ConversionFailureStopReason <${conversionFailuresStopReason}>`);
321
- debugData(`MatchErrorsStopReason <${matchErrorsStopReason}>`);
322
- debugData(`NewStopReason: <${newStopReason}>`);
323
- debug(`Match status: false`);
324
- return {
325
- status: false,
326
- stopReason: newStopReason
327
- };
328
- }
329
- debug(`Match status: true`);
330
- return {
331
- status: true,
332
- stopReason
333
- };
334
- }
335
- }
336
-
337
- // NOTES:
338
- // - we could optimize by creating the featureSet for the incoming record once and using it for all database/candidateRecords
339
- // - if creating the featureSet for the incoming record fails we have an unprocessable entity
340
- // - if creating the featureSet for a candidate record fails we could skip that candidate - but list the case as a detectionFailure, same as conversionFailures
341
-
342
- function iterateRecords({
343
- records,
344
- recordSetSize,
345
- maxMatches,
346
- matches = [],
347
- nonMatches = [],
348
- recordMatches = [],
349
- recordNonMatches = [],
350
- recordCount = 0,
351
- recordDuplicateCount = 0,
352
- recordNonMatchCount = 0,
353
- recordMatchErrors = []
354
- }) {
355
- // recordSetSize : total amount of records in the current record set
356
- // recordCount : amount of records from the current record set that have been handled
357
- // maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped
358
- // recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results
359
- // recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)
360
-
361
- // records : non-handled records in the current record set
362
- // matches : found matches in the current matcher job
363
- // recordMatches : found matches in the current record set
364
- // recordNonMatches : found nonMatches in the current record set (only if returnNonMatches setting is true)
365
- // recordMatchErrors: errored matchDetection in the current record set
366
-
367
- const [candidate] = records;
368
- const newRecordCount = candidate ? recordCount + 1 : recordCount;
369
-
370
- // The matcher uses same matchDetection strategy for candidates from all candidate-searches -> matchDetection result for the same candidate is always same
371
- // Exceptions would happen if the candidate would have been updated in the database between candidate searches
372
- // Note that if returnNonMatches is false, matcher won't remember candidates that didn't match, so they will be matched again everytime they are retrieved by
373
- // different candidate search queries. Same candidate search query won't have duplicate records.
374
-
375
- /* We could optimize and detect all retrieved candidates at once
376
- const candidateRecords = records.map(record => record.record);
377
- const recordsIsArray = Array.isArray(candidateRecords);
378
- debug(`records is an array: ${recordsIsArray}`);
379
- const result = detect(record, candidateRecords);
380
- debugData(`${JSON.stringify(result)}`);
381
- */
382
-
383
- if (candidate) {
384
- // eslint-disable-next-line functional/no-conditional-statements
385
- if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {
386
- const {
387
- record: candidateRecord,
388
- id: candidateId
389
- } = candidate;
390
- const recordBExternal = {
391
- id: candidate.id,
392
- recordSource: 'databaseRecord',
393
- label: `db-${candidate.id}`
394
- };
395
- try {
396
- debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);
397
- // we should handle errors from detection somehow - ie. cases where either record or candidateRecord errors
398
- const detectionResult = detect({
399
- recordA: record,
400
- recordB: candidateRecord,
401
- recordAExternal: recordExternal,
402
- recordBExternal
403
- });
404
- return handleDetectionResult(detectionResult, candidateId, candidateRecord);
405
- } catch (error) {
406
- debug(`MatchDetection errored: database record ${candidateId}: ${error}`);
407
- const matchError = {
408
- status: 422,
409
- payload: {
410
- message: `Matching errored for database record ${candidateId}. ${error.message}.`,
411
- id: candidateId
412
- }
413
- };
414
- const newRecordMatchErrors = recordMatchErrors.concat(matchError);
415
- return iterateRecords({
416
- records: records.slice(1),
417
- recordSetSize,
418
- maxMatches,
419
- matches,
420
- recordMatches,
421
- recordCount: newRecordCount,
422
- recordNonMatches,
423
- recordDuplicateCount,
424
- recordNonMatchCount,
425
- recordMatchErrors: newRecordMatchErrors
426
- });
427
- }
428
- }
429
- return iterateRecords({
430
- records: records.slice(1),
431
- recordSetSize,
432
- maxMatches,
433
- matches,
434
- recordMatches,
435
- recordCount: newRecordCount,
436
- recordNonMatches,
437
- recordDuplicateCount: recordDuplicateCount + 1,
438
- recordNonMatchCount,
439
- recordMatchErrors
440
- });
441
- }
442
- debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);
443
- return {
444
- matches: recordMatches,
445
- nonMatches: returnNonMatches ? recordNonMatches : [],
446
- duplicateCount: recordDuplicateCount,
447
- nonMatchCount: recordNonMatchCount,
448
- matchErrors: recordMatchErrors
449
- };
450
- function handleDetectionResult(detectionResult, candidateId, candidateRecord) {
451
- debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);
452
- if (detectionResult.match || returnNonMatches) {
453
- debug(`${detectionResult.match ? `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is a match!` : `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is NOT a match!`}`);
454
- debugData(`Strategy: ${JSON.stringify(detectionResult.strategy)}, Treshold: ${JSON.stringify(detectionResult.treshold)}`);
455
- const matchResult = {
456
- probability: detectionResult.probability,
457
- candidate: {
458
- id: candidateId,
459
- record: candidateRecord
460
- }
461
- };
462
- const strategyResult = {
463
- strategy: detectionResult.strategy,
464
- treshold: detectionResult.treshold
465
- };
466
- const newMatch = returnStrategy ? {
467
- ...matchResult,
468
- ...strategyResult
469
- } : {
470
- ...matchResult
471
- };
472
- debugData(`${JSON.stringify(newMatch)}`);
473
- return handleRecordMatch(detectionResult.match, newMatch);
474
- }
475
- const newRecordNonMatchCount = recordNonMatchCount + 1;
476
- debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);
477
- return iterateRecords({
478
- records: records.slice(1),
479
- recordSetSize,
480
- maxMatches,
481
- matches,
482
- recordMatches,
483
- recordCount: newRecordCount,
484
- recordNonMatches,
485
- recordDuplicateCount,
486
- recordNonMatchCount: newRecordNonMatchCount,
487
- recordMatchErrors
488
- });
489
- }
490
- function handleRecordMatch(isMatch, newMatch) {
491
- const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;
492
- const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);
493
- const newRecordNonMatchCount = isMatch ? recordNonMatchCount : recordNonMatchCount + 1;
494
- debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);
495
-
496
- // eslint-disable-next-line functional/no-conditional-statements
497
- if (returnNonMatches) {
498
- debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);
499
- }
500
- debugData(`- Total nonMatchCount after this detection: ${recordNonMatchCount}`);
501
- if (maxMatchesFound({
502
- matches: matches.concat(newRecordMatches),
503
- maxMatches
504
- })) {
505
- debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
506
- return {
507
- matches: newRecordMatches,
508
- nonMatches: returnNonMatches ? newRecordNonMatches : [],
509
- duplicateCount: recordDuplicateCount,
510
- nonMatchCount: newRecordNonMatchCount,
511
- matchErrors: recordMatchErrors
512
- };
513
- }
514
- return iterateRecords({
515
- records: records.slice(1),
516
- recordSetSize,
517
- maxMatches,
518
- matches,
519
- recordMatches: newRecordMatches,
520
- recordCount: newRecordCount,
521
- recordNonMatches: returnNonMatches ? newRecordNonMatches : [],
522
- duplicateCount: recordDuplicateCount,
523
- recordNonMatchCount: newRecordNonMatchCount,
524
- matchErrors: recordMatchErrors
525
- });
526
- }
527
- function candidateNotInMatches(matches, candidate) {
528
- debug(`Checking that record ${candidate.id} is not already included in ${matches.length} matches/nonMatches`);
529
- const newCandidateId = candidate.id;
530
- debugData(`newCandidateId: ${newCandidateId}`);
531
- const result = matches.find(({
532
- candidate
533
- }) => candidate.id === newCandidateId);
534
- debugData(`Result: ${result}`);
535
- if (result) {
536
- debug(`${candidate.id} was already handled.`);
537
- return false;
538
- }
539
- debug(`${candidate.id} not found in matches/nonMatches`);
540
- return true;
541
- }
542
- }
543
- function maxMatchesFound({
544
- matches,
545
- maxMatches
546
- }) {
547
- if (maxMatches && matches.length >= maxMatches) {
548
- debug(`Stopping recordSet iteration: maxMatches (${maxMatches}) for matcher job found.`);
549
- return true;
550
- }
551
- }
552
- }
553
- };
554
- exports.default = _default;
555
- //# sourceMappingURL=index.js.map
package/dist/index.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","names":["_debug","_interopRequireDefault","require","_candidateSearch","_interopRequireWildcard","candidateSearch","exports","_matchDetection","matchDetection","_getRequireWildcardCache","e","WeakMap","r","t","__esModule","default","has","get","n","__proto__","a","Object","defineProperty","getOwnPropertyDescriptor","u","hasOwnProperty","call","i","set","_default","detection","detectionOptions","search","searchOptions","maxMatches","maxCandidates","returnStrategy","returnQuery","returnNonMatches","returnFailures","debug","createDebugLogger","debugData","extend","JSON","stringify","detect","createDetectionInterface","prepareSearch","record","recordExternal","recordSource","label","createSearchInterface","iterate","initialState","matches","candidateCount","nonMatches","duplicateCount","nonMatchCount","conversionFailures","matchErrors","records","failures","state","length","recordSetSize","failureSetSize","newCandidateCount","newConversionFailures","concat","handleRecordSet","queriesLeft","searchCounter","query","returnResult","stopReason","matchResult","iterateRecords","newDuplicateCount","newNonMatchCount","newMatches","newNonMatches","newMatchErrors","handleMatchResult","maxMatchesFound","maxCandidatesRetrieved","addQuery","map","match","matchQuery","conversionFailureCount","matchErrorCount","checkCounts","matchStatus","getMatchState","matchesResult","result","matchCount","chosenNonMatchCount","totalHandled","conversionFailuresCount","resultSetOffset","totalRecords","queryCandidateCounter","maxedQueries","searchesLeft","nonRetrieved","maxedQueriesStopReason","undefined","conversionFailuresStopReason","matchErrorsStopReason","newStopReason","status","recordMatches","recordNonMatches","recordCount","recordDuplicateCount","recordNonMatchCount","recordMatchErrors","candidate","newRecordCount","candidateNotInMatches","candidateRecord","id","candidateId","recordBExternal","detectionResult","recordA","recordB","recordAExternal","handleDetectionResult","error","matchError","payload","message","newRecordMatchErrors","slice","strategy","treshold","probability","strategyResult","newMatch","handleRecordMatch","newRecordNonMatchCount","isMatch","newRecordMatches","newRecordNonMatches","newCandidateId","find"],"sources":["../src/index.js"],"sourcesContent":["import createDebugLogger from 'debug';\nimport createSearchInterface, * as candidateSearch from './candidate-search';\nimport createDetectionInterface, * as matchDetection from './match-detection';\n//import inspect from 'util';\n\nexport {candidateSearch, matchDetection};\n\nexport default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false, returnFailures = false}) => {\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:index');\n const debugData = debug.extend('data');\n\n debugData(`DetectionOptions: ${JSON.stringify(detectionOptions)}`);\n debugData(`SearchOptions: ${JSON.stringify(searchOptions)}`);\n debugData(`MaxMatches: ${JSON.stringify(maxMatches)}`);\n debugData(`MaxCandidates: ${JSON.stringify(maxCandidates)}`);\n debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);\n debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);\n debugData(`ReturnFailures: ${JSON.stringify(returnFailures)}`);\n\n\n const detect = createDetectionInterface(detectionOptions, returnStrategy);\n\n return prepareSearch;\n\n async function prepareSearch({record, recordExternal = {recordSource: 'incomingRecord', label: 'ic'}}) {\n\n const {search} = await createSearchInterface({...searchOptions, record, maxCandidates, recordExternal});\n return iterate({});\n\n // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set\n // matches : candidates that have been detected as matches by current matcher job\n // nonMatches : candidates that have been detected as non-matches by current matcher job (only if returnNonMatches is 'true')\n // duplicateCount : amount of candidate records that were retrieved from the SRU but not handled further because they were already found in the matches/nonMatches\n\n // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)\n // state.query : current query (undefined if there was no queries left)\n // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)\n // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)\n // state.queriesLeft : amount of queries left\n // state.queryCounter : sequence for current query\n // state.maxedQueries : queries that resulted in more than serverMaxResults hits\n\n async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, conversionFailures = [], matchErrors = []}) {\n debugData(`Starting next matcher iteration.`);\n const {records, failures, ...state} = await search(initialState);\n\n debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, conversionFailures: ${conversionFailures}, matchErrors: ${matchErrors.length}`);\n const recordSetSize = records.length;\n const failureSetSize = failures.length;\n const newCandidateCount = candidateCount + recordSetSize + failureSetSize;\n\n const newConversionFailures = conversionFailures.concat(failures);\n debugData(`Failures: ${failures.length}, ConversionFailures: ${conversionFailures.length}, NewConversionFailures: ${newConversionFailures.length}`);\n\n if (recordSetSize > 0) {\n return handleRecordSet();\n }\n\n if (state.queriesLeft > 0) {\n debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);\n return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});\n }\n\n debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);\n return returnResult({matches, state, stopReason: '', nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});\n\n function handleRecordSet() {\n debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);\n\n const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches, nonMatchCount});\n\n const newDuplicateCount = duplicateCount + matchResult.duplicateCount;\n const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;\n const {newMatches, newNonMatches, newMatchErrors} = handleMatchResult(matchResult, matches, nonMatches, matchErrors);\n\n if (maxMatchesFound({matches: newMatches, maxMatches})) {\n return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});\n }\n\n if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {\n return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});\n }\n\n return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});\n }\n\n function handleMatchResult(matchResult, matches, nonMatches, matchErrors) {\n debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);\n // eslint-disable-next-line functional/no-conditional-statements\n if (returnNonMatches) {\n debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);\n }\n\n const newMatches = matches.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);\n const newNonMatches = returnNonMatches ? nonMatches.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];\n const newMatchErrors = matchErrors.concat(matchResult.matchErrors);\n\n debugData(`- Total amount of matches: ${newMatches.length}`);\n // eslint-disable-next-line functional/no-conditional-statements\n if (returnNonMatches) {\n debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);\n }\n\n debugData(`MatchResult: ${JSON.stringify(matchResult)}`);\n debugData(`Old matchErrors: ${JSON.stringify(matchErrors)}, matchErrors from matchResult: ${JSON.stringify(matchResult.matchErrors)}, New matchErrors: ${JSON.stringify(newMatchErrors)}`);\n\n debugData(`- Total amount of matchErrors: ${newMatchErrors.length}`);\n\n return {newMatches, newNonMatches, newMatchErrors};\n }\n\n function addQuery(matches) {\n debugData(`Adding query ${state.query} to matches`);\n return matches.map((match) => ({...match, matchQuery: state.query}));\n }\n\n function maxCandidatesRetrieved(candidateCount, maxCandidates) {\n debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${maxCandidates})`);\n if (maxCandidates && candidateCount >= maxCandidates) {\n debug(`Stopped matching because maximum number of candidate records ${candidateCount} / ${maxCandidates} have been retrieved`);\n return true;\n }\n }\n }\n\n // matches : array of matching candidate records\n // nonMatches : array of nonMatching candidate records (if returnNonMatches option is true, otherwise empty array)\n // - candidate.id\n // - candidate.record\n // - probability\n // - strategy (if returnStrategy option is true)\n // - treshold (if returnStrategy option is true)\n // - matchQuery (if returnQuery option is true)\n // failures: array of conversionFailures from candidate-search and matchErrors from matchDetection in error format {status, payload: {message, id}} if returnFailures is true\n\n // we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data\n\n // matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not\n // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records\n // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)\n // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record\n\n function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, conversionFailures, matchErrors}) {\n const conversionFailureCount = conversionFailures.length;\n const matchErrorCount = matchErrors.length;\n checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount});\n const matchStatus = getMatchState(state, stopReason, conversionFailureCount, matchErrorCount);\n // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered\n const matchesResult = returnNonMatches ? {matches, matchStatus, nonMatches, candidateCount} : {matches, matchStatus, candidateCount};\n const failures = [...conversionFailures, ...matchErrors];\n const result = returnFailures ? {...matchesResult, conversionFailures: failures} : matchesResult;\n debugData(`ReturnFailures ${returnFailures}`);\n debugData(`${JSON.stringify(result)}`);\n return result;\n\n // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match\n\n function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount}) {\n const matchCount = matches.length;\n debugData(`Return nonMatches: ${returnNonMatches}`);\n const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;\n const totalHandled = matchCount + chosenNonMatchCount + duplicateCount;\n debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, conversionFailureCount: ${conversionFailureCount}, matchErrorCount: ${matchErrorCount}`);\n debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);\n if (totalHandled !== candidateCount) {\n debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);\n return;\n }\n return;\n }\n\n // eslint-disable-next-line max-statements\n function getMatchState(state, stopReason, conversionFailuresCount, matchErrorCount) {\n debugData(`${JSON.stringify(state)}`);\n debug(`We had ${conversionFailuresCount} retrieved candidates that could not be converted.`);\n debug(`We had ${matchErrorCount} retrieved candidates that errored in matchDetection.`);\n debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);\n\n debugData(`StopReason: <${stopReason}>`);\n\n const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;\n const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;\n debugData(`nonRetrieved: ${nonRetrieved}`);\n\n // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records\n // 'maxMatches' and 'maxCandidates' are in stopReason, 'maxedQueries', 'conversionFailures' and 'matchErrors' are created here\n\n if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || conversionFailureCount > 0 || matchErrorCount > 0) {\n const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : undefined;\n const conversionFailuresStopReason = conversionFailureCount > 0 ? 'conversionFailures' : undefined;\n const matchErrorsStopReason = matchErrorCount > 0 ? 'matchErrors' : undefined;\n const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || conversionFailuresStopReason || matchErrorsStopReason : stopReason;\n debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);\n debugData(`ConversionFailureStopReason <${conversionFailuresStopReason}>`);\n debugData(`MatchErrorsStopReason <${matchErrorsStopReason}>`);\n debugData(`NewStopReason: <${newStopReason}>`);\n debug(`Match status: false`);\n return {status: false, stopReason: newStopReason};\n }\n\n debug(`Match status: true`);\n return {status: true, stopReason};\n }\n }\n\n // NOTES:\n // - we could optimize by creating the featureSet for the incoming record once and using it for all database/candidateRecords\n // - if creating the featureSet for the incoming record fails we have an unprocessable entity\n // - if creating the featureSet for a candidate record fails we could skip that candidate - but list the case as a detectionFailure, same as conversionFailures\n\n function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0, recordMatchErrors = []}) {\n\n // recordSetSize : total amount of records in the current record set\n // recordCount : amount of records from the current record set that have been handled\n // maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped\n // recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results\n // recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)\n\n // records : non-handled records in the current record set\n // matches : found matches in the current matcher job\n // recordMatches : found matches in the current record set\n // recordNonMatches : found nonMatches in the current record set (only if returnNonMatches setting is true)\n // recordMatchErrors: errored matchDetection in the current record set\n\n const [candidate] = records;\n const newRecordCount = candidate ? recordCount + 1 : recordCount;\n\n // The matcher uses same matchDetection strategy for candidates from all candidate-searches -> matchDetection result for the same candidate is always same\n // Exceptions would happen if the candidate would have been updated in the database between candidate searches\n // Note that if returnNonMatches is false, matcher won't remember candidates that didn't match, so they will be matched again everytime they are retrieved by\n // different candidate search queries. Same candidate search query won't have duplicate records.\n\n /* We could optimize and detect all retrieved candidates at once\n const candidateRecords = records.map(record => record.record);\n const recordsIsArray = Array.isArray(candidateRecords);\n debug(`records is an array: ${recordsIsArray}`);\n const result = detect(record, candidateRecords);\n debugData(`${JSON.stringify(result)}`);\n */\n\n if (candidate) {\n\n // eslint-disable-next-line functional/no-conditional-statements\n if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {\n const {record: candidateRecord, id: candidateId} = candidate;\n const recordBExternal = {id: candidate.id, recordSource: 'databaseRecord', label: `db-${candidate.id}`};\n try {\n debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);\n // we should handle errors from detection somehow - ie. cases where either record or candidateRecord errors\n const detectionResult = detect({recordA: record, recordB: candidateRecord, recordAExternal: recordExternal, recordBExternal});\n\n return handleDetectionResult(detectionResult, candidateId, candidateRecord);\n } catch (error) {\n debug(`MatchDetection errored: database record ${candidateId}: ${error}`);\n\n const matchError = {status: 422, payload: {message: `Matching errored for database record ${candidateId}. ${error.message}.`, id: candidateId}};\n const newRecordMatchErrors = recordMatchErrors.concat(matchError);\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount, recordMatchErrors: newRecordMatchErrors});\n }\n }\n\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount, recordMatchErrors});\n }\n\n debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);\n return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount, matchErrors: recordMatchErrors};\n\n function handleDetectionResult(detectionResult, candidateId, candidateRecord) {\n debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);\n\n if (detectionResult.match || returnNonMatches) {\n debug(`${detectionResult.match ? `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is a match!` : `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is NOT a match!`}`);\n debugData(`Strategy: ${JSON.stringify(detectionResult.strategy)}, Treshold: ${JSON.stringify(detectionResult.treshold)}`);\n\n const matchResult = {\n probability: detectionResult.probability,\n candidate: {\n id: candidateId,\n record: candidateRecord\n }\n };\n const strategyResult = {\n strategy: detectionResult.strategy,\n treshold: detectionResult.treshold\n };\n const newMatch = returnStrategy ? {...matchResult, ...strategyResult} : {...matchResult};\n\n debugData(`${JSON.stringify(newMatch)}`);\n\n return handleRecordMatch(detectionResult.match, newMatch);\n }\n\n const newRecordNonMatchCount = recordNonMatchCount + 1;\n debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);\n\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, recordMatchErrors});\n }\n\n function handleRecordMatch(isMatch, newMatch) {\n const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;\n const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);\n const newRecordNonMatchCount = isMatch ? recordNonMatchCount : recordNonMatchCount + 1;\n\n debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);\n\n // eslint-disable-next-line functional/no-conditional-statements\n if (returnNonMatches) {\n debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);\n }\n debugData(`- Total nonMatchCount after this detection: ${recordNonMatchCount}`);\n\n if (maxMatchesFound({matches: matches.concat(newRecordMatches), maxMatches})) {\n debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);\n return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors};\n }\n\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors});\n }\n\n function candidateNotInMatches(matches, candidate) {\n debug(`Checking that record ${candidate.id} is not already included in ${matches.length} matches/nonMatches`);\n const newCandidateId = candidate.id;\n debugData(`newCandidateId: ${newCandidateId}`);\n const result = matches.find(({candidate}) => candidate.id === newCandidateId);\n debugData(`Result: ${result}`);\n if (result) {\n debug(`${candidate.id} was already handled.`);\n return false;\n }\n debug(`${candidate.id} not found in matches/nonMatches`);\n return true;\n }\n }\n\n function maxMatchesFound({matches, maxMatches}) {\n if (maxMatches && matches.length >= maxMatches) {\n debug(`Stopping recordSet iteration: maxMatches (${maxMatches}) for matcher job found.`);\n return true;\n }\n }\n }\n};\n"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,gBAAA,GAAAC,uBAAA,CAAAF,OAAA;AAA6E,IAAAG,eAAA,GAAAF,gBAAA;AAAAG,OAAA,CAAAD,eAAA,GAAAF,gBAAA;AAC7E,IAAAI,eAAA,GAAAH,uBAAA,CAAAF,OAAA;AAA8E,IAAAM,cAAA,GAAAD,eAAA;AAAAD,OAAA,CAAAE,cAAA,GAAAD,eAAA;AAAA,SAAAE,yBAAAC,CAAA,6BAAAC,OAAA,mBAAAC,CAAA,OAAAD,OAAA,IAAAE,CAAA,OAAAF,OAAA,YAAAF,wBAAA,YAAAA,CAAAC,CAAA,WAAAA,CAAA,GAAAG,CAAA,GAAAD,CAAA,KAAAF,CAAA;AAAA,SAAAN,wBAAAM,CAAA,EAAAE,CAAA,SAAAA,CAAA,IAAAF,CAAA,IAAAA,CAAA,CAAAI,UAAA,SAAAJ,CAAA,eAAAA,CAAA,uBAAAA,CAAA,yBAAAA,CAAA,WAAAK,OAAA,EAAAL,CAAA,QAAAG,CAAA,GAAAJ,wBAAA,CAAAG,CAAA,OAAAC,CAAA,IAAAA,CAAA,CAAAG,GAAA,CAAAN,CAAA,UAAAG,CAAA,CAAAI,GAAA,CAAAP,CAAA,OAAAQ,CAAA,KAAAC,SAAA,UAAAC,CAAA,GAAAC,MAAA,CAAAC,cAAA,IAAAD,MAAA,CAAAE,wBAAA,WAAAC,CAAA,IAAAd,CAAA,oBAAAc,CAAA,OAAAC,cAAA,CAAAC,IAAA,CAAAhB,CAAA,EAAAc,CAAA,SAAAG,CAAA,GAAAP,CAAA,GAAAC,MAAA,CAAAE,wBAAA,CAAAb,CAAA,EAAAc,CAAA,UAAAG,CAAA,KAAAA,CAAA,CAAAV,GAAA,IAAAU,CAAA,CAAAC,GAAA,IAAAP,MAAA,CAAAC,cAAA,CAAAJ,CAAA,EAAAM,CAAA,EAAAG,CAAA,IAAAT,CAAA,CAAAM,CAAA,IAAAd,CAAA,CAAAc,CAAA,YAAAN,CAAA,CAAAH,OAAA,GAAAL,CAAA,EAAAG,CAAA,IAAAA,CAAA,CAAAe,GAAA,CAAAlB,CAAA,EAAAQ,CAAA,GAAAA,CAAA;AAAA,SAAAjB,uBAAAS,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAI,UAAA,GAAAJ,CAAA,KAAAK,OAAA,EAAAL,CAAA;AAC9E;AAAA,IAAAmB,QAAA,GAIeA,CAAC;EAACC,SAAS,EAAEC,gBAAgB;EAAEC,MAAM,EAAEC,aAAa;EAAEC,UAAU,GAAG,CAAC;EAAEC,aAAa,GAAG,EAAE;EAAEC,cAAc,GAAG,KAAK;EAAEC,WAAW,GAAG,KAAK;EAAEC,gBAAgB,GAAG,KAAK;EAAEC,cAAc,GAAG;AAAK,CAAC,KAAK;EAC1M,MAAMC,KAAK,GAAG,IAAAC,cAAiB,EAAC,yCAAyC,CAAC;EAC1E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtCD,SAAS,CAAC,qBAAqBE,IAAI,CAACC,SAAS,CAACd,gBAAgB,CAAC,EAAE,CAAC;EAClEW,SAAS,CAAC,kBAAkBE,IAAI,CAACC,SAAS,CAACZ,aAAa,CAAC,EAAE,CAAC;EAC5DS,SAAS,CAAC,eAAeE,IAAI,CAACC,SAAS,CAACX,UAAU,CAAC,EAAE,CAAC;EACtDQ,SAAS,CAAC,kBAAkBE,IAAI,CAACC,SAAS,CAACV,aAAa,CAAC,EAAE,CAAC;EAC5DO,SAAS,CAAC,mBAAmBE,IAAI,CAACC,SAAS,CAACT,cAAc,CAAC,EAAE,CAAC;EAC9DM,SAAS,CAAC,gBAAgBE,IAAI,CAACC,SAAS,CAACR,WAAW,CAAC,EAAE,CAAC;EACxDK,SAAS,CAAC,qBAAqBE,IAAI,CAACC,SAAS,CAACP,gBAAgB,CAAC,EAAE,CAAC;EAClEI,SAAS,CAAC,mBAAmBE,IAAI,CAACC,SAAS,CAACN,cAAc,CAAC,EAAE,CAAC;EAG9D,MAAMO,MAAM,GAAG,IAAAC,uBAAwB,EAAChB,gBAAgB,EAAEK,cAAc,CAAC;EAEzE,OAAOY,aAAa;EAEpB,eAAeA,aAAaA,CAAC;IAACC,MAAM;IAAEC,cAAc,GAAG;MAACC,YAAY,EAAE,gBAAgB;MAAEC,KAAK,EAAE;IAAI;EAAC,CAAC,EAAE;IAErG,MAAM;MAACpB;IAAM,CAAC,GAAG,MAAM,IAAAqB,wBAAqB,EAAC;MAAC,GAAGpB,aAAa;MAAEgB,MAAM;MAAEd,aAAa;MAAEe;IAAc,CAAC,CAAC;IACvG,OAAOI,OAAO,CAAC,CAAC,CAAC,CAAC;;IAElB;IACA;IACA;IACA;;IAEA;IACA;IACA;IACA;IACA;IACA;IACA;;IAEA,eAAeA,OAAOA,CAAC;MAACC,YAAY,GAAG,CAAC,CAAC;MAAEC,OAAO,GAAG,EAAE;MAAEC,cAAc,GAAG,CAAC;MAAEC,UAAU,GAAG,EAAE;MAAEC,cAAc,GAAG,CAAC;MAAEC,aAAa,GAAG,CAAC;MAAEC,kBAAkB,GAAG,EAAE;MAAEC,WAAW,GAAG;IAAE,CAAC,EAAE;MAC/KpB,SAAS,CAAC,kCAAkC,CAAC;MAC7C,MAAM;QAACqB,OAAO;QAAEC,QAAQ;QAAE,GAAGC;MAAK,CAAC,GAAG,MAAMjC,MAAM,CAACuB,YAAY,CAAC;MAEhEb,SAAS,CAAC,kBAAkBE,IAAI,CAACC,SAAS,CAACoB,KAAK,CAAC,cAAcT,OAAO,CAACU,MAAM,qBAAqBT,cAAc,iBAAiBC,UAAU,CAACQ,MAAM,oBAAoBN,aAAa,yBAAyBC,kBAAkB,kBAAkBC,WAAW,CAACI,MAAM,EAAE,CAAC;MACrQ,MAAMC,aAAa,GAAGJ,OAAO,CAACG,MAAM;MACpC,MAAME,cAAc,GAAGJ,QAAQ,CAACE,MAAM;MACtC,MAAMG,iBAAiB,GAAGZ,cAAc,GAAGU,aAAa,GAAGC,cAAc;MAEzE,MAAME,qBAAqB,GAAGT,kBAAkB,CAACU,MAAM,CAACP,QAAQ,CAAC;MACjEtB,SAAS,CAAC,aAAasB,QAAQ,CAACE,MAAM,yBAAyBL,kBAAkB,CAACK,MAAM,4BAA4BI,qBAAqB,CAACJ,MAAM,EAAE,CAAC;MAEnJ,IAAIC,aAAa,GAAG,CAAC,EAAE;QACrB,OAAOK,eAAe,CAAC,CAAC;MAC1B;MAEA,IAAIP,KAAK,CAACQ,WAAW,GAAG,CAAC,EAAE;QACzBjC,KAAK,CAAC,oBAAoByB,KAAK,CAACS,aAAa,QAAQT,KAAK,CAACU,KAAK,mBAAmBV,KAAK,CAACQ,WAAW,eAAe,CAAC;QACpH,OAAOnB,OAAO,CAAC;UAACC,YAAY,EAAEU,KAAK;UAAET,OAAO;UAAEC,cAAc,EAAEY,iBAAiB;UAAEX,UAAU;UAAEE,aAAa;UAAED,cAAc;UAAEE,kBAAkB,EAAES,qBAAqB;UAAER;QAAW,CAAC,CAAC;MACtL;MAEAtB,KAAK,CAAC,wEAAwEgB,OAAO,CAACU,MAAM,EAAE,CAAC;MAC/F,OAAOU,YAAY,CAAC;QAACpB,OAAO;QAAES,KAAK;QAAEY,UAAU,EAAE,EAAE;QAAEnB,UAAU;QAAEE,aAAa;QAAEH,cAAc,EAAEY,iBAAiB;QAAEV,cAAc;QAAEE,kBAAkB,EAAES,qBAAqB;QAAER;MAAW,CAAC,CAAC;MAE3L,SAASU,eAAeA,CAAA,EAAG;QACzBhC,KAAK,CAAC,0BAA0B2B,aAAa,qDAAqDF,KAAK,CAACS,aAAa,eAAeT,KAAK,CAACU,KAAK,EAAE,CAAC;QAElJ,MAAMG,WAAW,GAAGC,cAAc,CAAC;UAAChB,OAAO;UAAEI,aAAa;UAAEjC,UAAU;UAAEsB,OAAO;UAAEE,UAAU;UAAEE;QAAa,CAAC,CAAC;QAE5G,MAAMoB,iBAAiB,GAAGrB,cAAc,GAAGmB,WAAW,CAACnB,cAAc;QACrE,MAAMsB,gBAAgB,GAAGrB,aAAa,GAAGkB,WAAW,CAAClB,aAAa;QAClE,MAAM;UAACsB,UAAU;UAAEC,aAAa;UAAEC;QAAc,CAAC,GAAGC,iBAAiB,CAACP,WAAW,EAAEtB,OAAO,EAAEE,UAAU,EAAEI,WAAW,CAAC;QAEpH,IAAIwB,eAAe,CAAC;UAAC9B,OAAO,EAAE0B,UAAU;UAAEhD;QAAU,CAAC,CAAC,EAAE;UACtD,OAAO0C,YAAY,CAAC;YAACpB,OAAO,EAAE0B,UAAU;YAAEjB,KAAK;YAAEY,UAAU,EAAE,YAAY;YAAEnB,UAAU,EAAEyB,aAAa;YAAExB,cAAc,EAAEqB,iBAAiB;YAAEvB,cAAc,EAAEY,iBAAiB;YAAET,aAAa,EAAEqB,gBAAgB;YAAEpB,kBAAkB,EAAES,qBAAqB;YAAER,WAAW,EAAEsB;UAAc,CAAC,CAAC;QACvR;QAEA,IAAIG,sBAAsB,CAAClB,iBAAiB,EAAElC,aAAa,CAAC,EAAE;UAC5D,OAAOyC,YAAY,CAAC;YAACpB,OAAO,EAAE0B,UAAU;YAAEjB,KAAK;YAAEY,UAAU,EAAE,eAAe;YAAEnB,UAAU,EAAEyB,aAAa;YAAExB,cAAc,EAAEqB,iBAAiB;YAAEvB,cAAc,EAAEY,iBAAiB;YAAET,aAAa,EAAEqB,gBAAgB;YAAEpB,kBAAkB,EAAES,qBAAqB;YAAER,WAAW,EAAEsB;UAAc,CAAC,CAAC;QAC1R;QAEA,OAAO9B,OAAO,CAAC;UAACC,YAAY,EAAEU,KAAK;UAAET,OAAO,EAAE0B,UAAU;UAAEzB,cAAc,EAAEY,iBAAiB;UAAEX,UAAU,EAAEyB,aAAa;UAAExB,cAAc,EAAEqB,iBAAiB;UAAEpB,aAAa,EAAEqB,gBAAgB;UAAEpB,kBAAkB,EAAES,qBAAqB;UAAER,WAAW,EAAEsB;QAAc,CAAC,CAAC;MACtQ;MAEA,SAASC,iBAAiBA,CAACP,WAAW,EAAEtB,OAAO,EAAEE,UAAU,EAAEI,WAAW,EAAE;QACxEpB,SAAS,CAAC,4CAA4CoC,WAAW,CAACtB,OAAO,CAACU,MAAM,EAAE,CAAC;QACnF;QACA,IAAI5B,gBAAgB,EAAE;UACpBI,SAAS,CAAC,+CAA+CoC,WAAW,CAACpB,UAAU,CAACQ,MAAM,EAAE,CAAC;QAC3F;QAEA,MAAMgB,UAAU,GAAG1B,OAAO,CAACe,MAAM,CAAClC,WAAW,GAAGmD,QAAQ,CAACV,WAAW,CAACtB,OAAO,CAAC,GAAGsB,WAAW,CAACtB,OAAO,CAAC;QACpG,MAAM2B,aAAa,GAAG7C,gBAAgB,GAAGoB,UAAU,CAACa,MAAM,CAAClC,WAAW,GAAGmD,QAAQ,CAACV,WAAW,CAACpB,UAAU,CAAC,GAAGoB,WAAW,CAACpB,UAAU,CAAC,GAAG,EAAE;QACxI,MAAM0B,cAAc,GAAGtB,WAAW,CAACS,MAAM,CAACO,WAAW,CAAChB,WAAW,CAAC;QAElEpB,SAAS,CAAC,8BAA8BwC,UAAU,CAAChB,MAAM,EAAE,CAAC;QAC5D;QACA,IAAI5B,gBAAgB,EAAE;UACpBI,SAAS,CAAC,iCAAiCyC,aAAa,CAACjB,MAAM,EAAE,CAAC;QACpE;QAEAxB,SAAS,CAAC,gBAAgBE,IAAI,CAACC,SAAS,CAACiC,WAAW,CAAC,EAAE,CAAC;QACxDpC,SAAS,CAAC,oBAAoBE,IAAI,CAACC,SAAS,CAACiB,WAAW,CAAC,mCAAmClB,IAAI,CAACC,SAAS,CAACiC,WAAW,CAAChB,WAAW,CAAC,sBAAsBlB,IAAI,CAACC,SAAS,CAACuC,cAAc,CAAC,EAAE,CAAC;QAE1L1C,SAAS,CAAC,kCAAkC0C,cAAc,CAAClB,MAAM,EAAE,CAAC;QAEpE,OAAO;UAACgB,UAAU;UAAEC,aAAa;UAAEC;QAAc,CAAC;MACpD;MAEA,SAASI,QAAQA,CAAChC,OAAO,EAAE;QACzBd,SAAS,CAAC,gBAAgBuB,KAAK,CAACU,KAAK,aAAa,CAAC;QACnD,OAAOnB,OAAO,CAACiC,GAAG,CAAEC,KAAK,KAAM;UAAC,GAAGA,KAAK;UAAEC,UAAU,EAAE1B,KAAK,CAACU;QAAK,CAAC,CAAC,CAAC;MACtE;MAEA,SAASY,sBAAsBA,CAAC9B,cAAc,EAAEtB,aAAa,EAAE;QAC7DO,SAAS,CAAC,gDAAgD2B,iBAAiB,UAAUlC,aAAa,GAAG,CAAC;QACtG,IAAIA,aAAa,IAAIsB,cAAc,IAAItB,aAAa,EAAE;UACpDK,KAAK,CAAC,gEAAgEiB,cAAc,MAAMtB,aAAa,sBAAsB,CAAC;UAC9H,OAAO,IAAI;QACb;MACF;IACF;;IAEA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;;IAEA;;IAEA;IACA;IACA;IACA;;IAEA,SAASyC,YAAYA,CAAC;MAACpB,OAAO;MAAES,KAAK;MAAEY,UAAU;MAAEnB,UAAU;MAAEC,cAAc;MAAEF,cAAc;MAAEG,aAAa;MAAEC,kBAAkB;MAAEC;IAAW,CAAC,EAAE;MAC9I,MAAM8B,sBAAsB,GAAG/B,kBAAkB,CAACK,MAAM;MACxD,MAAM2B,eAAe,GAAG/B,WAAW,CAACI,MAAM;MAC1C4B,WAAW,CAAC;QAACtC,OAAO;QAAEE,UAAU;QAAED,cAAc;QAAEE,cAAc;QAAEC,aAAa;QAAEgC,sBAAsB;QAAEC;MAAe,CAAC,CAAC;MAC1H,MAAME,WAAW,GAAGC,aAAa,CAAC/B,KAAK,EAAEY,UAAU,EAAEe,sBAAsB,EAAEC,eAAe,CAAC;MAC7F;MACA,MAAMI,aAAa,GAAG3D,gBAAgB,GAAG;QAACkB,OAAO;QAAEuC,WAAW;QAAErC,UAAU;QAAED;MAAc,CAAC,GAAG;QAACD,OAAO;QAAEuC,WAAW;QAAEtC;MAAc,CAAC;MACpI,MAAMO,QAAQ,GAAG,CAAC,GAAGH,kBAAkB,EAAE,GAAGC,WAAW,CAAC;MACxD,MAAMoC,MAAM,GAAG3D,cAAc,GAAG;QAAC,GAAG0D,aAAa;QAAEpC,kBAAkB,EAAEG;MAAQ,CAAC,GAAGiC,aAAa;MAChGvD,SAAS,CAAC,kBAAkBH,cAAc,EAAE,CAAC;MAC7CG,SAAS,CAAC,GAAGE,IAAI,CAACC,SAAS,CAACqD,MAAM,CAAC,EAAE,CAAC;MACtC,OAAOA,MAAM;;MAEb;;MAEA,SAASJ,WAAWA,CAAC;QAACtC,OAAO;QAAEE,UAAU;QAAED,cAAc;QAAEE,cAAc;QAAEC,aAAa;QAAEgC,sBAAsB;QAAEC;MAAe,CAAC,EAAE;QAClI,MAAMM,UAAU,GAAG3C,OAAO,CAACU,MAAM;QACjCxB,SAAS,CAAC,sBAAsBJ,gBAAgB,EAAE,CAAC;QACnD,MAAM8D,mBAAmB,GAAG9D,gBAAgB,GAAGoB,UAAU,CAACQ,MAAM,GAAGN,aAAa;QAChF,MAAMyC,YAAY,GAAGF,UAAU,GAAGC,mBAAmB,GAAGzC,cAAc;QACtEnB,KAAK,CAAC,mBAAmBiB,cAAc,cAAc0C,UAAU,iBAAiBC,mBAAmB,qBAAqBzC,cAAc,6BAA6BiC,sBAAsB,sBAAsBC,eAAe,EAAE,CAAC;QACjOrD,KAAK,CAAC,qBAAqB6D,YAAY,MAAM5C,cAAc,uBAAuB,CAAC;QACnF,IAAI4C,YAAY,KAAK5C,cAAc,EAAE;UACnCjB,KAAK,CAAC,gCAAgCiB,cAAc,GAAG4C,YAAY,aAAa,CAAC;UACjF;QACF;QACA;MACF;;MAEA;MACA,SAASL,aAAaA,CAAC/B,KAAK,EAAEY,UAAU,EAAEyB,uBAAuB,EAAET,eAAe,EAAE;QAClFnD,SAAS,CAAC,GAAGE,IAAI,CAACC,SAAS,CAACoB,KAAK,CAAC,EAAE,CAAC;QACrCzB,KAAK,CAAC,UAAU8D,uBAAuB,oDAAoD,CAAC;QAC5F9D,KAAK,CAAC,UAAUqD,eAAe,uDAAuD,CAAC;QACvFrD,KAAK,CAAC,gBAAgByB,KAAK,CAACQ,WAAW,sCAAsCR,KAAK,CAACsC,eAAe,IAAItC,KAAK,CAACsC,eAAe,IAAItC,KAAK,CAACuC,YAAY,4BAA4BvC,KAAK,CAACuC,YAAY,GAAGvC,KAAK,CAACwC,qBAAqB,mBAAmBxC,KAAK,CAACyC,YAAY,CAACxC,MAAM,MAAMD,KAAK,CAACyC,YAAY,EAAE,CAAC;QAEpShE,SAAS,CAAC,gBAAgBmC,UAAU,GAAG,CAAC;QAExC,MAAM8B,YAAY,GAAG1C,KAAK,CAACsC,eAAe,IAAItC,KAAK,CAACsC,eAAe,IAAItC,KAAK,CAACuC,YAAY;QACzF,MAAMI,YAAY,GAAGD,YAAY,GAAG1C,KAAK,CAACuC,YAAY,GAAGvC,KAAK,CAACwC,qBAAqB,GAAG,CAAC;QACxF/D,SAAS,CAAC,iBAAiBkE,YAAY,EAAE,CAAC;;QAE1C;QACA;;QAEA,IAAI3C,KAAK,CAACQ,WAAW,GAAG,CAAC,IAAImC,YAAY,GAAG,CAAC,IAAI3C,KAAK,CAACyC,YAAY,CAACxC,MAAM,GAAG,CAAC,IAAI0B,sBAAsB,GAAG,CAAC,IAAIC,eAAe,GAAG,CAAC,EAAE;UACnI,MAAMgB,sBAAsB,GAAG5C,KAAK,CAACyC,YAAY,CAACxC,MAAM,GAAG,CAAC,GAAG,cAAc,GAAG4C,SAAS;UACzF,MAAMC,4BAA4B,GAAGnB,sBAAsB,GAAG,CAAC,GAAG,oBAAoB,GAAGkB,SAAS;UAClG,MAAME,qBAAqB,GAAGnB,eAAe,GAAG,CAAC,GAAG,aAAa,GAAGiB,SAAS;UAC7E,MAAMG,aAAa,GAAGpC,UAAU,KAAK,EAAE,IAAIA,UAAU,KAAKiC,SAAS,GAAGD,sBAAsB,IAAIE,4BAA4B,IAAIC,qBAAqB,GAAGnC,UAAU;UAClKnC,SAAS,CAAC,4BAA4BmE,sBAAsB,GAAG,CAAC;UAChEnE,SAAS,CAAC,gCAAgCqE,4BAA4B,GAAG,CAAC;UAC1ErE,SAAS,CAAC,0BAA0BsE,qBAAqB,GAAG,CAAC;UAC7DtE,SAAS,CAAC,mBAAmBuE,aAAa,GAAG,CAAC;UAC9CzE,KAAK,CAAC,qBAAqB,CAAC;UAC5B,OAAO;YAAC0E,MAAM,EAAE,KAAK;YAAErC,UAAU,EAAEoC;UAAa,CAAC;QACnD;QAEAzE,KAAK,CAAC,oBAAoB,CAAC;QAC3B,OAAO;UAAC0E,MAAM,EAAE,IAAI;UAAErC;QAAU,CAAC;MACnC;IACF;;IAEA;IACA;IACA;IACA;;IAEA,SAASE,cAAcA,CAAC;MAAChB,OAAO;MAAEI,aAAa;MAAEjC,UAAU;MAAEsB,OAAO,GAAG,EAAE;MAAEE,UAAU,GAAG,EAAE;MAAEyD,aAAa,GAAG,EAAE;MAAEC,gBAAgB,GAAG,EAAE;MAAEC,WAAW,GAAG,CAAC;MAAEC,oBAAoB,GAAG,CAAC;MAAEC,mBAAmB,GAAG,CAAC;MAAEC,iBAAiB,GAAG;IAAE,CAAC,EAAE;MAElO;MACA;MACA;MACA;MACA;;MAEA;MACA;MACA;MACA;MACA;;MAEA,MAAM,CAACC,SAAS,CAAC,GAAG1D,OAAO;MAC3B,MAAM2D,cAAc,GAAGD,SAAS,GAAGJ,WAAW,GAAG,CAAC,GAAGA,WAAW;;MAEhE;MACA;MACA;MACA;;MAEA;AACN;AACA;AACA;AACA;AACA;AACA;;MAEM,IAAII,SAAS,EAAE;QAEb;QACA,IAAIE,qBAAqB,CAACnE,OAAO,CAACe,MAAM,CAACb,UAAU,CAAC,EAAE+D,SAAS,CAAC,EAAE;UAChE,MAAM;YAACxE,MAAM,EAAE2E,eAAe;YAAEC,EAAE,EAAEC;UAAW,CAAC,GAAGL,SAAS;UAC5D,MAAMM,eAAe,GAAG;YAACF,EAAE,EAAEJ,SAAS,CAACI,EAAE;YAAE1E,YAAY,EAAE,gBAAgB;YAAEC,KAAK,EAAE,MAAMqE,SAAS,CAACI,EAAE;UAAE,CAAC;UACvG,IAAI;YACFrF,KAAK,CAAC,qCAAqCsF,WAAW,KAAKJ,cAAc,IAAIvD,aAAa,GAAG,CAAC;YAC9F;YACA,MAAM6D,eAAe,GAAGlF,MAAM,CAAC;cAACmF,OAAO,EAAEhF,MAAM;cAAEiF,OAAO,EAAEN,eAAe;cAAEO,eAAe,EAAEjF,cAAc;cAAE6E;YAAe,CAAC,CAAC;YAE7H,OAAOK,qBAAqB,CAACJ,eAAe,EAAEF,WAAW,EAAEF,eAAe,CAAC;UAC7E,CAAC,CAAC,OAAOS,KAAK,EAAE;YACd7F,KAAK,CAAC,2CAA2CsF,WAAW,KAAKO,KAAK,EAAE,CAAC;YAEzE,MAAMC,UAAU,GAAG;cAACpB,MAAM,EAAE,GAAG;cAAEqB,OAAO,EAAE;gBAACC,OAAO,EAAE,wCAAwCV,WAAW,KAAKO,KAAK,CAACG,OAAO,GAAG;gBAAEX,EAAE,EAAEC;cAAW;YAAC,CAAC;YAC/I,MAAMW,oBAAoB,GAAGjB,iBAAiB,CAACjD,MAAM,CAAC+D,UAAU,CAAC;YACjE,OAAOvD,cAAc,CAAC;cAAChB,OAAO,EAAEA,OAAO,CAAC2E,KAAK,CAAC,CAAC,CAAC;cAAEvE,aAAa;cAAEjC,UAAU;cAAEsB,OAAO;cAAE2D,aAAa;cAAEE,WAAW,EAAEK,cAAc;cAAEN,gBAAgB;cAAEE,oBAAoB;cAAEC,mBAAmB;cAAEC,iBAAiB,EAAEiB;YAAoB,CAAC,CAAC;UAC1O;QACF;QAEA,OAAO1D,cAAc,CAAC;UAAChB,OAAO,EAAEA,OAAO,CAAC2E,KAAK,CAAC,CAAC,CAAC;UAAEvE,aAAa;UAAEjC,UAAU;UAAEsB,OAAO;UAAE2D,aAAa;UAAEE,WAAW,EAAEK,cAAc;UAAEN,gBAAgB;UAAEE,oBAAoB,EAAEA,oBAAoB,GAAG,CAAC;UAAEC,mBAAmB;UAAEC;QAAiB,CAAC,CAAC;MAC9O;MAEAhF,KAAK,CAAC,mCAAmC6E,WAAW,IAAIlD,aAAa,WAAWgD,aAAa,CAACjD,MAAM,mBAAmBoD,oBAAoB,gCAAgChF,gBAAgB,GAAG,GAAG8E,gBAAgB,CAAClD,MAAM,EAAE,GAAG,GAAGqD,mBAAmB,EAAE,oBAAoB,CAAC;MAC1Q,OAAO;QAAC/D,OAAO,EAAE2D,aAAa;QAAEzD,UAAU,EAAEpB,gBAAgB,GAAG8E,gBAAgB,GAAG,EAAE;QAAEzD,cAAc,EAAE2D,oBAAoB;QAAE1D,aAAa,EAAE2D,mBAAmB;QAAEzD,WAAW,EAAE0D;MAAiB,CAAC;MAE/L,SAASY,qBAAqBA,CAACJ,eAAe,EAAEF,WAAW,EAAEF,eAAe,EAAE;QAC5ElF,SAAS,CAAC,8BAA8BoF,WAAW,KAAKJ,cAAc,IAAIvD,aAAa,MAAMvB,IAAI,CAACC,SAAS,CAACmF,eAAe,CAAC,EAAE,CAAC;QAE/H,IAAIA,eAAe,CAACtC,KAAK,IAAIpD,gBAAgB,EAAE;UAC7CE,KAAK,CAAC,GAAGwF,eAAe,CAACtC,KAAK,GAAG,UAAUoC,WAAW,KAAKJ,cAAc,IAAIvD,aAAa,eAAe,GAAG,UAAU2D,WAAW,KAAKJ,cAAc,IAAIvD,aAAa,mBAAmB,EAAE,CAAC;UAC3LzB,SAAS,CAAC,aAAaE,IAAI,CAACC,SAAS,CAACmF,eAAe,CAACW,QAAQ,CAAC,eAAe/F,IAAI,CAACC,SAAS,CAACmF,eAAe,CAACY,QAAQ,CAAC,EAAE,CAAC;UAEzH,MAAM9D,WAAW,GAAG;YAClB+D,WAAW,EAAEb,eAAe,CAACa,WAAW;YACxCpB,SAAS,EAAE;cACTI,EAAE,EAAEC,WAAW;cACf7E,MAAM,EAAE2E;YACV;UACF,CAAC;UACD,MAAMkB,cAAc,GAAG;YACrBH,QAAQ,EAAEX,eAAe,CAACW,QAAQ;YAClCC,QAAQ,EAAEZ,eAAe,CAACY;UAC5B,CAAC;UACD,MAAMG,QAAQ,GAAG3G,cAAc,GAAG;YAAC,GAAG0C,WAAW;YAAE,GAAGgE;UAAc,CAAC,GAAG;YAAC,GAAGhE;UAAW,CAAC;UAExFpC,SAAS,CAAC,GAAGE,IAAI,CAACC,SAAS,CAACkG,QAAQ,CAAC,EAAE,CAAC;UAExC,OAAOC,iBAAiB,CAAChB,eAAe,CAACtC,KAAK,EAAEqD,QAAQ,CAAC;QAC3D;QAEA,MAAME,sBAAsB,GAAG1B,mBAAmB,GAAG,CAAC;QACtD7E,SAAS,CAAC,4CAA4CuG,sBAAsB,EAAE,CAAC;QAE/E,OAAOlE,cAAc,CAAC;UAAChB,OAAO,EAAEA,OAAO,CAAC2E,KAAK,CAAC,CAAC,CAAC;UAAEvE,aAAa;UAAEjC,UAAU;UAAEsB,OAAO;UAAE2D,aAAa;UAAEE,WAAW,EAAEK,cAAc;UAAEN,gBAAgB;UAAEE,oBAAoB;UAAEC,mBAAmB,EAAE0B,sBAAsB;UAAEzB;QAAiB,CAAC,CAAC;MAC5O;MAEA,SAASwB,iBAAiBA,CAACE,OAAO,EAAEH,QAAQ,EAAE;QAC5C,MAAMI,gBAAgB,GAAGD,OAAO,GAAG/B,aAAa,CAAC5C,MAAM,CAACwE,QAAQ,CAAC,GAAG5B,aAAa;QACjF,MAAMiC,mBAAmB,GAAGF,OAAO,GAAG9B,gBAAgB,GAAGA,gBAAgB,CAAC7C,MAAM,CAACwE,QAAQ,CAAC;QAC1F,MAAME,sBAAsB,GAAGC,OAAO,GAAG3B,mBAAmB,GAAGA,mBAAmB,GAAG,CAAC;QAEtF7E,SAAS,CAAC,yCAAyCc,OAAO,CAACe,MAAM,CAAC4E,gBAAgB,CAAC,CAACjF,MAAM,UAAUhC,UAAU,GAAG,CAAC;;QAElH;QACA,IAAII,gBAAgB,EAAE;UACpBI,SAAS,CAAC,4CAA4CgB,UAAU,CAACa,MAAM,CAAC6E,mBAAmB,CAAC,CAAClF,MAAM,EAAE,CAAC;QACxG;QACAxB,SAAS,CAAC,+CAA+C6E,mBAAmB,EAAE,CAAC;QAE/E,IAAIjC,eAAe,CAAC;UAAC9B,OAAO,EAAEA,OAAO,CAACe,MAAM,CAAC4E,gBAAgB,CAAC;UAAEjH;QAAU,CAAC,CAAC,EAAE;UAC5EM,KAAK,CAAC,eAAeN,UAAU,gDAAgDwF,cAAc,yCAAyCvD,aAAa,GAAGuD,cAAc,EAAE,CAAC;UACvK,OAAO;YAAClE,OAAO,EAAE2F,gBAAgB;YAAEzF,UAAU,EAAEpB,gBAAgB,GAAG8G,mBAAmB,GAAG,EAAE;YAAEzF,cAAc,EAAE2D,oBAAoB;YAAE1D,aAAa,EAAEqF,sBAAsB;YAAEnF,WAAW,EAAE0D;UAAiB,CAAC;QAC1M;QAEA,OAAOzC,cAAc,CAAC;UAAChB,OAAO,EAAEA,OAAO,CAAC2E,KAAK,CAAC,CAAC,CAAC;UAAEvE,aAAa;UAAEjC,UAAU;UAAEsB,OAAO;UAAE2D,aAAa,EAAEgC,gBAAgB;UAAE9B,WAAW,EAAEK,cAAc;UAAEN,gBAAgB,EAAE9E,gBAAgB,GAAG8G,mBAAmB,GAAG,EAAE;UAAEzF,cAAc,EAAE2D,oBAAoB;UAAEC,mBAAmB,EAAE0B,sBAAsB;UAAEnF,WAAW,EAAE0D;QAAiB,CAAC,CAAC;MACxU;MAEA,SAASG,qBAAqBA,CAACnE,OAAO,EAAEiE,SAAS,EAAE;QACjDjF,KAAK,CAAC,wBAAwBiF,SAAS,CAACI,EAAE,+BAA+BrE,OAAO,CAACU,MAAM,qBAAqB,CAAC;QAC7G,MAAMmF,cAAc,GAAG5B,SAAS,CAACI,EAAE;QACnCnF,SAAS,CAAC,mBAAmB2G,cAAc,EAAE,CAAC;QAC9C,MAAMnD,MAAM,GAAG1C,OAAO,CAAC8F,IAAI,CAAC,CAAC;UAAC7B;QAAS,CAAC,KAAKA,SAAS,CAACI,EAAE,KAAKwB,cAAc,CAAC;QAC7E3G,SAAS,CAAC,WAAWwD,MAAM,EAAE,CAAC;QAC9B,IAAIA,MAAM,EAAE;UACV1D,KAAK,CAAC,GAAGiF,SAAS,CAACI,EAAE,uBAAuB,CAAC;UAC7C,OAAO,KAAK;QACd;QACArF,KAAK,CAAC,GAAGiF,SAAS,CAACI,EAAE,kCAAkC,CAAC;QACxD,OAAO,IAAI;MACb;IACF;IAEA,SAASvC,eAAeA,CAAC;MAAC9B,OAAO;MAAEtB;IAAU,CAAC,EAAE;MAC9C,IAAIA,UAAU,IAAIsB,OAAO,CAACU,MAAM,IAAIhC,UAAU,EAAE;QAC9CM,KAAK,CAAC,6CAA6CN,UAAU,0BAA0B,CAAC;QACxF,OAAO,IAAI;MACb;IACF;EACF;AACF,CAAC;AAAA5B,OAAA,CAAAS,OAAA,GAAAc,QAAA","ignoreList":[]}