@natlibfi/melinda-record-matching 2.1.0 → 2.2.0-alpha.3

package/src/candidate-search/index.spec.js

@@ -30,6 +30,7 @@ import {expect} from 'chai';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen-http-client';
 import {MarcRecord} from '@natlibfi/marc-record';
+import {Error as MatchingError} from '@natlibfi/melinda-commons';
 import createSearchInterface, {CandidateSearchError} from '.';
 import createDebugLogger from 'debug';
 
@@ -77,7 +78,7 @@ describe('candidate-search', () => {
     }
 
     // eslint-disable-next-line max-statements
-    async function iterate({searchOptions, expectedSearchError, count = 1}) {
+    async function iterate({searchOptions, expectedSearchError, expectedErrorStatus, count = 1}) {
       const expectedResults = getFixture(`expectedResults${count}.json`);
 
       if (expectedSearchError) { // eslint-disable-line functional/no-conditional-statement
@@ -85,8 +86,17 @@ describe('candidate-search', () => {
           await search(searchOptions);
           throw new Error('Expected an error');
         } catch (err) {
+          debug(`Got an error: ${err}`);
           expect(err).to.be.an('error');
-          expect(err.message).to.match(new RegExp(expectedSearchError, 'u'));
+          const errorMessage = err instanceof MatchingError ? err.payload.message : err.message;
+          const errorStatus = err instanceof MatchingError ? err.status : undefined;
+          debug(`errorMessage: ${errorMessage}, errorStatus: ${errorStatus}`);
+          expect(errorMessage).to.match(new RegExp(expectedSearchError, 'u'));
+
+          if (expectedErrorStatus) {
+            expect(errorStatus).to.be(expectedErrorStatus);
+            return;
+          }
           return;
         }
       }
@@ -98,7 +108,7 @@ describe('candidate-search', () => {
       }
 
       function formatResults(results) {
-        // console.log(results); //eslint-disable-line
+        debug(results); //eslint-disable-line
         return {
           ...results,
           records: results.records.map(({record, id}) => ({id, record: record.toObject()}))

package/src/candidate-search/query-list/bib.js

@@ -108,7 +108,7 @@ export function bibSourceIds(record) {
 
         function removeSourcePrefix(subfieldValue) {
           const sourcePrefixRegex = (/^(?<sourcePrefix>\([A-Za-z0-9-]+\))(?<id>.+)$/u);
-          const normalizedValue = subfieldValue.replace(sourcePrefixRegex, '$<id>');
+          const normalizedValue = subfieldValue ? subfieldValue.replace(sourcePrefixRegex, '$<id>') : '';
           debugData(`Normalized ${subfieldValue} to ${normalizedValue}`);
           return normalizedValue;
         }
@@ -116,7 +116,7 @@ export function bibSourceIds(record) {
         function normalizeSidSubfieldValue(subfieldValue) {
           debugData(`Normalizing ${subfieldValue}`);
           const normalizeAwayRegex = (/[- ]/u);
-          return subfieldValue.replace(normalizeAwayRegex, '');
+          return subfieldValue ? subfieldValue.replace(normalizeAwayRegex, '') : '';
         }
 
       }

package/src/index.js

@@ -29,10 +29,11 @@
 import createDebugLogger from 'debug';
 import createSearchInterface, * as candidateSearch from './candidate-search';
 import createDetectionInterface, * as matchDetection from './match-detection';
+//import inspect from 'util';
 
 export {candidateSearch, matchDetection};
 
-export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false}) => {
+export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false, returnFailures = false}) => {
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:index');
   const debugData = debug.extend('data');
 
@@ -43,6 +44,8 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
   debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
   debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);
   debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);
+  debugData(`ReturnFailures: ${JSON.stringify(returnFailures)}`);
+
 
   const detect = createDetectionInterface(detectionOptions, returnStrategy);
 
@@ -63,13 +66,17 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // state.queryCounter : sequence for current query
     // state.maxedQueries : queries that resulted in more than serverMaxResults hits
 
-    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0}) {
+    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, conversionFailures = [], matchErrors = []}) {
       debugData(`Starting next matcher iteration.`);
-      const {records, ...state} = await search(initialState);
+      const {records, failures, ...state} = await search(initialState);
 
-      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}`);
+      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, conversionFailures: ${conversionFailures}, matchErrors: ${matchErrors.length}`);
       const recordSetSize = records.length;
-      const newCandidateCount = candidateCount + recordSetSize;
+      const failureSetSize = failures.length;
+      const newCandidateCount = candidateCount + recordSetSize + failureSetSize;
+
+      const newConversionFailures = conversionFailures.concat(failures);
+      debugData(`Failures: ${failures.length}, ConversionFailures: ${conversionFailures.length}, NewConversionFailures: ${newConversionFailures.length}`);
 
       if (recordSetSize > 0) {
         return handleRecordSet();
@@ -77,32 +84,33 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
 
       if (state.queriesLeft > 0) {
         debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);
-        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, duplicateCount});
+        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});
       }
 
       debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);
-      return returnResult({matches, state, stopReason: '', nonMatches, candidateCount: newCandidateCount, duplicateCount});
+      return returnResult({matches, state, stopReason: '', nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});
 
       function handleRecordSet() {
         debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);
 
-        const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches});
+        const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches, nonMatchCount});
+
         const newDuplicateCount = duplicateCount + matchResult.duplicateCount;
         const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;
-        const {newMatches, newNonMatches} = handleMatchResult(matchResult, matches, nonMatches);
+        const {newMatches, newNonMatches, newMatchErrors} = handleMatchResult(matchResult, matches, nonMatches, matchErrors);
 
         if (maxMatchesFound({matches: newMatches, maxMatches})) {
-          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount});
+          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
         }
 
         if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
-          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount});
+          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
         }
 
-        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount});
+        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
       }
 
-      function handleMatchResult(matchResult, matches, nonMatches) {
+      function handleMatchResult(matchResult, matches, nonMatches, matchErrors) {
         debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);
         // eslint-disable-next-line functional/no-conditional-statement
         if (returnNonMatches) {
@@ -111,6 +119,7 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
 
         const newMatches = matches.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);
         const newNonMatches = returnNonMatches ? nonMatches.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];
+        const newMatchErrors = matchErrors.concat(matchResult.matchErrors);
 
         debugData(`- Total amount of matches: ${newMatches.length}`);
         // eslint-disable-next-line functional/no-conditional-statement
@@ -118,7 +127,12 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
           debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);
         }
 
-        return {newMatches, newNonMatches};
+        debugData(`MatchResult: ${JSON.stringify(matchResult)}`);
+        debugData(`Old matchErrors: ${JSON.stringify(matchErrors)}, matchErrors from matchResult: ${JSON.stringify(matchResult.matchErrors)}, New matchErrors: ${JSON.stringify(newMatchErrors)}`);
+
+        debugData(`- Total amount of matchErrors: ${newMatchErrors.length}`);
+
+        return {newMatches, newNonMatches, newMatchErrors};
       }
 
       function addQuery(matches) {
@@ -143,30 +157,36 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // - strategy (if returnStrategy option is true)
     // - treshold (if returnStrategy option is true)
     // - matchQuery (if returnQuery option is true)
+    // failures: array of conversionFailures from candidate-search and matchErrors from matchDetection in error format {status, payload: {message, id}} if returnFailures is true
 
     // we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data
 
     // matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not
-    // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries',empty string/undefined), reason for stopping retrieving or handling the candidate records
+    // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
     // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
     // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
 
-    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount}) {
-      checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount});
-      const matchStatus = getMatchState(state, stopReason);
+    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, conversionFailures, matchErrors}) {
+      const conversionFailureCount = conversionFailures.length;
+      const matchErrorCount = matchErrors.length;
+      checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount});
+      const matchStatus = getMatchState(state, stopReason, conversionFailureCount, matchErrorCount);
       // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
-      const result = returnNonMatches ? {matches, matchStatus, nonMatches} : {matches, matchStatus};
+      const matchesResult = returnNonMatches ? {matches, matchStatus, nonMatches} : {matches, matchStatus};
+      const failures = [...conversionFailures, ...matchErrors];
+      const result = returnFailures ? {...matchesResult, conversionFailures: failures} : matchesResult;
+      debugData(`ReturnFailures ${returnFailures}`);
       debugData(`${JSON.stringify(result)}`);
       return result;
 
       // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
 
-      function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount}) {
+      function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount}) {
         const matchCount = matches.length;
         debugData(`Return nonMatches: ${returnNonMatches}`);
         const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
-        const totalHandled = matchCount + nonMatchCount + duplicateCount;
-        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}`);
+        const totalHandled = matchCount + chosenNonMatchCount + duplicateCount;
+        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, conversionFailureCount: ${conversionFailureCount}, matchErrorCount: ${matchErrorCount}`);
         debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
         if (totalHandled !== candidateCount) {
           debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);
@@ -175,8 +195,11 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         return;
       }
 
-      function getMatchState(state, stopReason) {
+      // eslint-disable-next-line max-statements
+      function getMatchState(state, stopReason, conversionFailuresCount, matchErrorCount) {
         debugData(`${JSON.stringify(state)}`);
+        debug(`We had ${conversionFailuresCount} retrieved candidates that could not be converted.`);
+        debug(`We had ${matchErrorCount} retrieved candidates that errored in matchDetection.`);
         debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
 
         debugData(`StopReason: <${stopReason}>`);
@@ -185,10 +208,17 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
         debugData(`nonRetrieved: ${nonRetrieved}`);
 
-        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0) {
-          const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : '';
-          const newStopReason = stopReason === '' ? maxedQueriesStopReason : stopReason;
+        // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
+        // 'maxMatches' and 'maxCandidates' are in stopReason, 'maxedQueries', 'conversionFailures' and 'matchErrors' are created here
+
+        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || conversionFailureCount > 0 || matchErrorCount > 0) {
+          const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : undefined;
+          const conversionFailuresStopReason = conversionFailureCount > 0 ? 'conversionFailures' : undefined;
+          const matchErrorsStopReason = matchErrorCount > 0 ? 'matchErrors' : undefined;
+          const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || conversionFailuresStopReason || matchErrorsStopReason : stopReason;
           debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);
+          debugData(`ConversionFailureStopReason <${conversionFailuresStopReason}>`);
+          debugData(`MatchErrorsStopReason <${matchErrorsStopReason}>`);
           debugData(`NewStopReason: <${newStopReason}>`);
           debug(`Match status: false`);
           return {status: false, stopReason: newStopReason};
@@ -199,7 +229,13 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
       }
     }
 
-    function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0}) {
+    // NOTES:
+    // - we could optimize by creating the featureSet for the incoming record once and using it for all database/candidateRecords
+    // - if creating the featureSet for the incoming record fails we have an unprocessable entity
+    // - if creating the featureSet for a candidate record fails we could skip that candidate - but list the case as a detectionFailure, same as conversionFailures
+
+    // eslint-disable-next-line max-statements
+    function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0, recordMatchErrors = []}) {
 
       // recordSetSize : total amount of records in the current record set
       // recordCount : amount of records from the current record set that have been handled
@@ -211,6 +247,7 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
       // matches : found matches in the current matcher job
       // recordMatches : found matches in the current record set
       // recordNonMatches : found nonMatches in the current record set (only if returnNonMatches setting is true)
+      // recordMatchErrors: errored matchDetection in the current record set
 
       const [candidate] = records;
       const newRecordCount = candidate ? recordCount + 1 : recordCount;
@@ -220,19 +257,39 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
       // Note that if returnNonMatches is false, matcher won't remember candidates that didn't match, so they will be matched again everytime they are retrieved by
       // different candidate search queries. Same candidate search query won't have duplicate records.
 
+      /* We could optimize and detect all retrieved candidates at once
+      const candidateRecords = records.map(record => record.record);
+      const recordsIsArray = Array.isArray(candidateRecords);
+      debug(`records is an array: ${recordsIsArray}`);
+      const result = detect(record, candidateRecords);
+      debug(`${JSON.stringify(result)}`);
+      */
+
       if (candidate) {
 
+        // eslint-disable-next-line functional/no-conditional-statement
         if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {
           const {record: candidateRecord, id: candidateId} = candidate;
-          debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);
-          const detectionResult = detect(record, candidateRecord);
-          return handleDetectionResult(detectionResult, candidateId, candidateRecord);
+          try {
+            debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);
+            // we should handle errors from detection somehow - ie. cases where either record or candidateRecord errors
+            const detectionResult = detect(record, candidateRecord);
+
+            return handleDetectionResult(detectionResult, candidateId, candidateRecord);
+          } catch (error) {
+            debug(`MatchDetection errored: database record ${candidateId}: ${error}`);
+
+            const matchError = {status: 422, payload: {message: `Matching errored for database record ${candidateId}. ${error.message}.`, id: candidateId}};
+            const newRecordMatchErrors = recordMatchErrors.concat(matchError);
+            return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount, recordMatchErrors: newRecordMatchErrors});
+          }
         }
-        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount});
+
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount, recordMatchErrors});
       }
 
       debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);
-      return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount};
+      return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount, matchErrors: recordMatchErrors};
 
       function handleDetectionResult(detectionResult, candidateId, candidateRecord) {
         debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);
@@ -262,12 +319,13 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         const newRecordNonMatchCount = recordNonMatchCount + 1;
         debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);
 
-        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount});
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, recordMatchErrors});
       }
 
       function handleRecordMatch(isMatch, newMatch) {
         const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;
         const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);
+        const newRecordNonMatchCount = isMatch ? recordNonMatchCount : recordNonMatchCount + 1;
 
         debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);
 
@@ -275,13 +333,14 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         if (returnNonMatches) {
           debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);
         }
+        debugData(`- Total nonMatchCount after this detection: ${recordNonMatchCount}`);
 
         if (maxMatchesFound({matches: matches.concat(newRecordMatches), maxMatches})) {
           debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
-          return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount};
+          return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors};
         }
 
-        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount});
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors});
       }
 
       function candidateNotInMatches(matches, candidate) {

package/src/index.spec.js

@@ -46,7 +46,7 @@ describe('INDEX', () => {
     }
   });
 
-  async function callback({getFixture, options, enabled = true, expectedMatchStatus, expectedStopReason}) {
+  async function callback({getFixture, options, enabled = true, expectedMatchStatus, expectedStopReason, expectedFailures}) {
 
     if (!enabled) {
       debug(`Disabled test!`);
@@ -59,8 +59,11 @@ describe('INDEX', () => {
 
 
     const match = createMatchInterface(formatOptions());
-    const {matches, matchStatus, nonMatches} = await match(record);
-    debugData(`${matches.length}, ${matchStatus.status}/${matchStatus.stopReason}, ${nonMatches ? nonMatches.length : nonMatches}`);
+    const {matches, matchStatus, nonMatches, conversionFailures} = await match(record);
+    debugData(`Matches: ${matches.length}, Status: ${matchStatus.status}/${matchStatus.stopReason}, NonMatches: ${nonMatches ? nonMatches.length : 'not returned'}, ConversionFailures: ${conversionFailures ? conversionFailures.length : 'not returned'}`);
+
+    expect(matchStatus.status).to.eql(expectedMatchStatus);
+    expect(matchStatus.stopReason).to.eql(expectedStopReason);
 
     const formattedMatchResult = formatRecordResults(matches);
     expect(formattedMatchResult).to.eql(expectedMatches);
@@ -68,9 +71,10 @@ describe('INDEX', () => {
     const formattedNonMatchResult = formatRecordResults(nonMatches);
     expect(formattedNonMatchResult).to.eql(expectedNonMatches);
 
-    expect(matchStatus.status).to.eql(expectedMatchStatus);
-    expect(matchStatus.stopReason).to.eql(expectedStopReason);
-
+    // eslint-disable-next-line functional/no-conditional-statement
+    if (expectedFailures) {
+      expect(conversionFailures).to.eql(expectedFailures);
+    }
 
     function formatOptions() {
       const contextFeatures = matchDetection.features[options.detection.strategy.type];

package/src/match-detection/features/bib/authors.js

@@ -40,7 +40,7 @@ export default ({nameTreshold = 10} = {}) => ({
 
       function toObj({code, value}) {
         if (code === 'a') {
-          return {name: value.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()};
+          return {name: value ? value.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase() : ''};
         }
 
         return {id: value};

package/src/match-detection/features/bib/standard-identifier-factory.js

@@ -40,7 +40,8 @@ export default ({pattern, subfieldCodes}) => {
     if (field) {
       return field.subfields
         .filter(({code}) => subfieldCodes.includes(code))
-        .map(({code, value}) => ({code, value: value.replace(/-/ug, '')}));
+      //  .map(({code, value}) => ({code, value: value.replace(/-/ug, '')}));
+        .map(({code, value}) => ({code, value: value ? value.replace(/-/ug, '') : ''}));
     }
 
     return [];

package/src/match-detection/index.js

@@ -32,7 +32,6 @@ import * as features from './features';
 
 export {features};
 
-// eslint-disable-next-line max-statements
 export default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA, recordB) => {
   const minProbabilityQuantifier = 0.5;
 
@@ -44,22 +43,36 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA,
   debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
 
   const featuresA = extractFeatures(recordA);
-  const featuresB = extractFeatures(recordB);
 
-  debugData(`Features (a): ${JSON.stringify(featuresA)}`);
-  debugData(`Features (b): ${JSON.stringify(featuresB)}`);
+  debug(`We got an array of records: ${Array.isArray(recordB)}`);
+  const recordsB = Array.isArray(recordB) ? recordB : [recordB];
+  const detectionResults = recordsB.map(record => actualDetection(featuresA, record));
 
-  const featurePairs = generateFeaturePairs();
-  const similarityVector = generateSimilarityVector();
+  return Array.isArray(recordB) ? detectionResults : detectionResults[0];
 
-  if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
-    const probability = calculateprobability();
-    debug(`probability: ${probability} (Treshold: ${treshold})`);
-    return returnResult({match: probability >= treshold, probability});
+  function actualDetection(featuresA, record) {
+
+    const featuresB = extractFeatures(record);
+
+    debugData(`Features (a): ${JSON.stringify(featuresA)}`);
+    debugData(`Features (b): ${JSON.stringify(featuresB)}`);
+
+    const featurePairs = generateFeaturePairs(featuresA, featuresB);
+    const similarityVector = generateSimilarityVector(featurePairs);
+
+    if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
+      const probability = calculateprobability(similarityVector);
+      debug(`probability: ${probability} (Treshold: ${treshold})`);
+      return returnResult({match: probability >= treshold, probability});
+    }
+
+    debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
+    return returnResult({match: false, probability: 0.0});
   }
 
-  debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
-  return returnResult({match: false, probability: 0.0});
+  function extractFeatures(record) {
+    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract(record)}), []);
+  }
 
   function returnResult(result) {
     if (returnStrategy) {
@@ -76,16 +89,12 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA,
     return strategyNames || [];
   }
 
-  function calculateprobability() {
+  function calculateprobability(similarityVector) {
     const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);
     return probability > 1.0 ? 1.0 : probability;
   }
 
-  function extractFeatures(record) {
-    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract(record)}), []);
-  }
-
-  function generateSimilarityVector() {
+  function generateSimilarityVector(featurePairs) {
     const compared = featurePairs.map(({name, a, b}) => {
       const {compare} = strategy.find(({name: featureName}) => name === featureName);
       const points = compare(a, b);
@@ -96,7 +105,7 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA,
     return compared.map(({points}) => points);
   }
 
-  function generateFeaturePairs() {
+  function generateFeaturePairs(featuresA, featuresB) {
     const pairs = generatePairs();
     const missingFeatures = findMissing();
 
@@ -125,4 +134,5 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA,
         .filter(v => pairs.some(({name}) => name === v) === false);
     }
   }
+
 };

package/src/match-detection/index.spec.js

@@ -32,6 +32,11 @@ import {expect} from 'chai';
 import {MarcRecord} from '@natlibfi/marc-record';
 import * as features from './features';
 import createDetectionInterface from '.';
+import {inspect} from 'util';
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:test');
+const debugData = debug.extend('data');
 
 describe('match-detection', () => {
   generateTests({
@@ -41,16 +46,25 @@ describe('match-detection', () => {
     fixura: {
       reader: READERS.JSON
     },
-    callback: ({getFixture, options, expectedResults, enabled = true}) => {
+    callback: ({getFixture, options, expectedResults, array, enabled = true}) => {
 
       if (!enabled) {
+        debug(`*** DISABLED TEST! ***`);
         return;
       }
 
       const detect = createDetectionInterface(formatOptions());
-      const recordA = new MarcRecord(getFixture('recordA.json'));
-      const recordB = new MarcRecord(getFixture('recordB.json'));
+      const recordA = new MarcRecord(getFixture('recordA.json'), {subfieldValues: false});
+      debugData(inspect(recordA));
+
+      debug(`Our recordB is an array of records: ${array}`);
+      const recordB = array
+        ? getFixture('recordB.json').map(recordJson => new MarcRecord(recordJson, {subfieldValues: false}))
+        : new MarcRecord(getFixture('recordB.json'), {subfieldValues: false});
+      debugData(inspect(recordB));
+
       const results = detect(recordA, recordB);
+      debugData(`${JSON.stringify(results)}`);
 
       expect(results).to.eql(expectedResults);
 

package/src/matching-utils.js

@@ -59,7 +59,7 @@ export function getMelindaIdsF035(record) {
     return subfields
       .filter(sub => ['a', 'z'].includes(sub.code))
       .filter(sub => melindaIdRegExp.test(sub.value))
-      .map(({value}) => value.replace(melindaIdRegExp, '$<id>'));
+      .map(({value}) => value ? value.replace(melindaIdRegExp, '$<id>') : '');
 
   }
 }