@natlibfi/melinda-record-matching 2.1.0-alpha.1 → 2.2.0-alpha.1

package/src/candidate-search/index.js

@@ -31,6 +31,7 @@ import createClient, {SruSearchError} from '@natlibfi/sru-client';
 import {MarcRecord} from '@natlibfi/marc-record';
 import {MARCXML} from '@natlibfi/marc-record-serializers';
 import generateQueryList from './query-list';
+import {Error as MatchingError} from '@natlibfi/melinda-commons';
 
 export {searchTypes} from './query-list';
 
@@ -39,7 +40,7 @@ export class CandidateSearchError extends Error {}
 // serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000
 
 // eslint-disable-next-line max-statements
-export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {
+export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {
   MarcRecord.setValidationOptions({subfieldValues: false});
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search');
@@ -49,11 +50,16 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxRe
   debugData(`Url: ${url}`);
   debugData(`MaxRecordsPerRequest ${maxRecordsPerRequest}`);
   debugData(`ServerMaxResult: ${serverMaxResult}`);
+  debugData(`MaxCandidates: ${maxCandidates}`);
+
+  // Do not retrieve more candidates than defined in maxCandidates
+  const adjustedMaxRecordsPerRequest = maxRecordsPerRequest >= maxCandidates ? maxCandidates : maxRecordsPerRequest;
 
   const inputRecordId = getRecordId(record);
   const queryList = generateQueryList(record, searchSpec);
   const client = createClient({
-    url, maxRecordsPerRequest,
+    url,
+    maxRecordsPerRequest: adjustedMaxRecordsPerRequest,
     version: '2.0',
     retrieveAll: false
   });
@@ -67,39 +73,39 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxRe
   // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
   // state.query : current query (undefined if there was no queries left)
   // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
-  // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)
+  // state.queryCandidateCounter: amount of candidates (records+failures) retrieved from SRU for matching for current query, including the current record+failure set (undefined if there were no queries left)
   // state.queriesLeft : amount of queries left
   // state.queryCounter : sequence for current query
   // state.maxedQueries : queries that resulted in more than serverMaxResults hits
 
 
-  // eslint-disable-next-line max-statements
   return async ({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) => {
     const query = queryList[queryOffset];
 
     if (query) {
-      const {records, nextOffset, total} = await retrieveRecords();
+      const {records, failures, nextOffset, total} = await retrieveRecords();
 
       // If resultSetOffset === 1 this is the first search for the current query
+      debugData(`ResultSetOffset: ${resultSetOffset}`);
       const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;
       const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;
       const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;
-      const newQueryCandidateCounter = resultSetOffset === 1 ? records.length : queryCandidateCounter + records.length;
+      const newQueryCandidateCounter = resultSetOffset === 1 ? records.length + failures.length : queryCandidateCounter + records.length + failures.length;
 
       const maxedQuery = resultSetOffset === 1 ? checkMaxedQuery(query, total, serverMaxResult) : undefined;
       const newMaxedQueries = maxedQuery ? maxedQueries.concat(maxedQuery) : maxedQueries;
 
       if (typeof nextOffset === 'number') {
         debug(`Next search will be for query ${queryOffset} ${query}, starting from record ${nextOffset}`);
-        return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCandidateCounter, maxedQueries: newMaxedQueries};
+        return {records, failures, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
       }
       debug(`Query ${queryOffset} ${query} done.`);
       debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);
-      return {records, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
+      return {records, failures, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
     }
 
     debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);
-    return {records: [], queriesLeft: 0, queryCounter, maxedQueries};
+    return {records: [], failures: [], queriesLeft: 0, queryCounter, maxedQueries};
 
     function retrieveRecords() {
       return new Promise((resolve, reject) => {
@@ -125,35 +131,39 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxRe
           })
           .on('end', async nextOffset => {
             try {
-              const records = await Promise.all(promises);
-              const filtered = records.filter(r => r);
+              const recordPromises = await Promise.allSettled(promises);
+              debug(`All recordPromises: ${JSON.stringify(recordPromises)}`);
+              const filtered = recordPromises.filter(r => r.status === 'fulfilled').map(r => r.value);
+              const failures = recordPromises.filter(r => r.status === 'rejected').map(r => ({status: r.reason.status, payload: r.reason.payload}));
+
+              debug(`Found ${JSON.stringify(recordPromises)} records`);
+              debug(`Found ${filtered.length} convertable candidates`);
+              debug(`Found ${failures.length} NON-convertable candidates`);
+              debug(`Converted: ${JSON.stringify(filtered)}.`);
+              debug(`Not converted: ${JSON.stringify(failures)}.`);
 
-              debug(`Found ${filtered.length} candidates`);
 
-              resolve({nextOffset, records: filtered, total: totalRecords});
+              resolve({nextOffset, records: filtered, failures, total: totalRecords});
             } catch (err) {
+              debug(`Error caught on END`);
               reject(err);
             }
           })
-          .on('record', foundRecord => {
+          .on('record', foundRecordXML => {
             promises.push(handleRecord()); // eslint-disable-line functional/immutable-data
 
             async function handleRecord() {
               try {
-                const foundRecordMarc = await MARCXML.from(foundRecord, {subfieldValues: false});
+                const foundRecordMarc = await MARCXML.from(foundRecordXML, {subfieldValues: false});
                 const foundRecordId = getRecordId(foundRecordMarc);
 
-                // This does not work and might cause problems:
-                // Record *should* match itself AND in REST the input record is given id 000000001 always
-                debug(`Checking record id's - this does not work ${inputRecordId} vs ${foundRecordId}`);
-                if (inputRecordId === foundRecordId) {
-                  debug(`Input and candidate are the same record per 001. Discarding candidate`);
-                  return;
-                }
-
                 return {record: foundRecordMarc, id: foundRecordId};
               } catch (err) {
-                throw new Error(`Failed converting record: ${err}, record: ${foundRecord}`);
+                // What should this do?
+                const idFromXML = getRecordIdFromXML(foundRecordXML);
+                debugData(`Failed converting record: ${err.message}, id: ${idFromXML}, data: ${foundRecordXML}`);
+                //return {message: `Failed converting record: ${err.message}`, id: idFromXML, data: foundRecordXML};
+                throw new MatchingError(422, {message: `Failed converting record: ${err.message}`, id: idFromXML || '000000000', data: foundRecordXML});
               }
             }
           });
@@ -174,4 +184,11 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxRe
     const [field] = record.get(/^001$/u);
     return field ? field.value : '';
   }
+
+  function getRecordIdFromXML(recordXML) {
+    //<controlfield tag=\"001\">015376846</controlfield
+    debug(`Cannot yet find possible database record id from recordXML (length ${recordXML.length})`);
+    return undefined;
+  }
+
 };

package/src/candidate-search/index.spec.js

@@ -30,6 +30,7 @@ import {expect} from 'chai';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen-http-client';
 import {MarcRecord} from '@natlibfi/marc-record';
+import {Error as MatchingError} from '@natlibfi/melinda-commons';
 import createSearchInterface, {CandidateSearchError} from '.';
 import createDebugLogger from 'debug';
 
@@ -46,7 +47,7 @@ describe('candidate-search', () => {
   });
 
   // eslint-disable-next-line max-statements
-  async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError, expectedSearchError, enabled = true}) {
+  async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError = false, expectedSearchError = false, enabled = true}) {
     const url = 'http://foo.bar';
 
     if (!enabled) {
@@ -64,7 +65,7 @@ describe('candidate-search', () => {
     }
 
     const search = createSearchInterface({...formatFactoryOptions(), url});
-    await iterate({searchOptions});
+    await iterate({searchOptions, expectedSearchError});
 
     function formatFactoryOptions() {
       debug(`Using factoryOptions: ${JSON.stringify(factoryOptions)}`);
@@ -77,7 +78,7 @@ describe('candidate-search', () => {
     }
 
     // eslint-disable-next-line max-statements
-    async function iterate({searchOptions, count = 1}) {
+    async function iterate({searchOptions, expectedSearchError, expectedErrorStatus, count = 1}) {
       const expectedResults = getFixture(`expectedResults${count}.json`);
 
       if (expectedSearchError) { // eslint-disable-line functional/no-conditional-statement
@@ -85,36 +86,35 @@ describe('candidate-search', () => {
           await search(searchOptions);
           throw new Error('Expected an error');
         } catch (err) {
+          debug(`Got an error: ${err}`);
           expect(err).to.be.an('error');
-          expect(err.message).to.match(new RegExp(expectedSearchError, 'u'));
+          const errorMessage = err instanceof MatchingError ? err.payload.message : err.message;
+          const errorStatus = err instanceof MatchingError ? err.status : undefined;
+          debug(`errorMessage: ${errorMessage}, errorStatus: ${errorStatus}`);
+          expect(errorMessage).to.match(new RegExp(expectedSearchError, 'u'));
+
+          if (expectedErrorStatus) {
+            expect(errorStatus).to.be(expectedErrorStatus);
+            return;
+          }
           return;
         }
       }
 
-      const results = await search(searchOptions);
-
-      expect(formatResults(results)).to.eql(expectedResults);
-
-      if (results.records.length > 0) {
-        return iterate({
-          searchOptions: resultsToOptions(results),
-          count: count + 1
-        });
+      // eslint-disable-next-line functional/no-conditional-statement
+      if (!expectedSearchError) {
+        const results = await search(searchOptions);
+        expect(formatResults(results)).to.eql(expectedResults);
       }
 
       function formatResults(results) {
-        // console.log(results); //eslint-disable-line
+        debug(results); //eslint-disable-line
         return {
           ...results,
           records: results.records.map(({record, id}) => ({id, record: record.toObject()}))
         };
       }
 
-      function resultsToOptions(results) {
-        return Object.entries(results)
-          .filter(([k]) => k === 'records' === false) // If key is 'records' return false
-          .reduce((acc, [k, v]) => ({...acc, [k]: v}), {});
-      }
     }
   }
 });

package/src/candidate-search/query-list/bib.js

@@ -175,12 +175,15 @@ export function bibHostComponents(record) {
 
 // SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to
 // (probably) to *headings* index TIT
+// - Aleph cannot currently handle headings searches starting with a boolean - in these cases use word search
+
 // Headings index TIT drops articles etc. from the start of the title according to the filing indicator
 // Currently filing indicator is not implemented - if the title starts with an article and the Melinda
 // record is correctly catalogued using a filing indicator -> dc.title search won't match
 
 export function bibTitle(record) {
   const title = getTitle();
+  const booleanStartWords = ['and', 'or', 'nor', 'not'];
 
   if (title) {
     const formatted = title
@@ -191,8 +194,10 @@ export function bibTitle(record) {
       .slice(0, 30)
       .trim();
 
+    // use word search for titles starting with a boolean
+    const useWordSearch = booleanStartWords.some(word => formatted.toLowerCase().startsWith(word));
     // Prevent too many matches by having a minimum length requirement
-    return formatted.length >= 5 ? [`dc.title="^${formatted}*"`] : [];
+    return formatted.length >= 5 ? [`dc.title="${useWordSearch ? '' : '^'}${formatted}*"`] : [];
   }
 
   return [];

package/src/index.js

@@ -32,7 +32,7 @@ import createDetectionInterface, * as matchDetection from './match-detection';
 
 export {candidateSearch, matchDetection};
 
-export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false}) => {
+export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false, returnFailures}) => {
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:index');
   const debugData = debug.extend('data');
 
@@ -43,11 +43,13 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
   debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
   debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);
   debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);
+  debugData(`ReturnFailures: ${JSON.stringify(returnFailures)}`);
+
 
   const detect = createDetectionInterface(detectionOptions, returnStrategy);
 
   return record => {
-    const search = createSearchInterface({...searchOptions, record});
+    const search = createSearchInterface({...searchOptions, record, maxCandidates});
     return iterate({});
 
     // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set
@@ -63,13 +65,17 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // state.queryCounter : sequence for current query
     // state.maxedQueries : queries that resulted in more than serverMaxResults hits
 
-    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0}) {
+    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, conversionFailures = []}) {
       debugData(`Starting next matcher iteration.`);
-      const {records, ...state} = await search(initialState);
+      const {records, failures, ...state} = await search(initialState);
 
-      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}`);
+      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, conversionFailures: ${conversionFailures}`);
       const recordSetSize = records.length;
-      const newCandidateCount = candidateCount + recordSetSize;
+      const failureSetSize = failures.length;
+      const newCandidateCount = candidateCount + recordSetSize + failureSetSize;
+
+      const newConversionFailures = conversionFailures.concat(failures);
+      debugData(`Failures: ${failures.length}, ConversionFailures: ${conversionFailures.length}, NewConversionFailures: ${newConversionFailures.length}`);
 
       if (recordSetSize > 0) {
         return handleRecordSet();
@@ -77,28 +83,30 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
 
       if (state.queriesLeft > 0) {
         debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);
-        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, duplicateCount});
+        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, conversionFailures: newConversionFailures});
       }
 
       debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);
-      return returnResult({matches, state, stopReason: '', nonMatches, candidateCount: newCandidateCount, duplicateCount});
+      return returnResult({matches, state, stopReason: '', nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, conversionFailures: newConversionFailures});
 
       function handleRecordSet() {
         debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);
 
-        const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches});
+        const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches, nonMatchCount});
+
         const newDuplicateCount = duplicateCount + matchResult.duplicateCount;
+        const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;
         const {newMatches, newNonMatches} = handleMatchResult(matchResult, matches, nonMatches);
 
         if (maxMatchesFound({matches: newMatches, maxMatches})) {
-          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount});
+          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures});
         }
 
         if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
-          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount});
+          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures});
         }
 
-        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount});
+        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures});
       }
 
       function handleMatchResult(matchResult, matches, nonMatches) {
@@ -146,26 +154,29 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data
 
     // matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not
-    // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries',empty string/undefined), reason for stopping retrieving or handling the candidate records
+    // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
     // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
     // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
 
-    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount}) {
-      checkCounts(matches, nonMatches, candidateCount, duplicateCount);
-      const matchStatus = getMatchState(state, stopReason);
+    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, conversionFailures}) {
+      const conversionFailureCount = conversionFailures.length;
+      checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount});
+      const matchStatus = getMatchState(state, stopReason, conversionFailureCount);
       // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
-      const result = returnNonMatches ? {matches, matchStatus, nonMatches} : {matches, matchStatus};
+      const matchesResult = returnNonMatches ? {matches, matchStatus, nonMatches} : {matches, matchStatus};
+      const result = returnFailures ? {...matchesResult, conversionFailures} : matchesResult;
+      debugData(`ReturnFailures ${returnFailures}`);
       debugData(`${JSON.stringify(result)}`);
       return result;
 
-      // we could have a nonMatchCount even if nonMatches won't be returned so that checkCounts would make sense
       // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
 
-      function checkCounts(matches, nonMatches, candidateCount, duplicateCount) {
+      function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount}) {
         const matchCount = matches.length;
-        const nonMatchCount = nonMatches.length;
-        const totalHandled = matchCount + nonMatchCount + duplicateCount;
-        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${nonMatchCount}, duplicateCount: ${duplicateCount}`);
+        debugData(`Return nonMatches: ${returnNonMatches}`);
+        const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
+        const totalHandled = matchCount + chosenNonMatchCount + duplicateCount;
+        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, conversionFailureCount: ${conversionFailureCount}`);
         debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
         if (totalHandled !== candidateCount) {
           debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);
@@ -174,16 +185,26 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         return;
       }
 
-      function getMatchState(state, stopReason) {
-        debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset >= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
+      function getMatchState(state, stopReason, conversionFailuresCount) {
+        debugData(`${JSON.stringify(state)}`);
+        debug(`We had ${conversionFailuresCount} retrieved candidates that could not be converted.`);
+        debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
+
         debugData(`StopReason: <${stopReason}>`);
 
-        const nonRetrieved = state.resultSetOffset >= state.totalRecords ? state.totalRecords - state.queryCandidateCounter : 0;
+        const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;
+        const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
+        debugData(`nonRetrieved: ${nonRetrieved}`);
 
-        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0) {
-          const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : '';
-          const newStopReason = stopReason === '' ? maxedQueriesStopReason : stopReason;
+        // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
+        // 'maxMatches' and 'maxCandidates' are in stopReason, 'maxedQueries' and 'conversionFailures' are created here
+
+        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || conversionFailureCount > 0) {
+          const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : undefined;
+          const conversionFailuresStopReason = conversionFailureCount > 0 ? 'conversionFailures' : undefined;
+          const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || conversionFailuresStopReason : stopReason;
           debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);
+          debugData(`ConversionFailureStopReason <${conversionFailuresStopReason}>`);
           debugData(`NewStopReason: <${newStopReason}>`);
           debug(`Match status: false`);
           return {status: false, stopReason: newStopReason};
@@ -194,12 +215,13 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
       }
     }
 
-    function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0}) {
+    function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0}) {
 
       // recordSetSize : total amount of records in the current record set
       // recordCount : amount of records from the current record set that have been handled
       // maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped
       // recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results
+      // recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)
 
       // records : non-handled records in the current record set
       // matches : found matches in the current matcher job
@@ -222,11 +244,11 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
           const detectionResult = detect(record, candidateRecord);
           return handleDetectionResult(detectionResult, candidateId, candidateRecord);
         }
-        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1});
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount});
       }
 
-      debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled${returnNonMatches ? `, ${recordNonMatches.length} nonMatches found.` : '.'}`);
-      return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount};
+      debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);
+      return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount};
 
       function handleDetectionResult(detectionResult, candidateId, candidateRecord) {
         debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);
@@ -252,12 +274,17 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
 
           return handleRecordMatch(detectionResult.match, newMatch);
         }
-        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount});
+
+        const newRecordNonMatchCount = recordNonMatchCount + 1;
+        debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);
+
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount});
       }
 
       function handleRecordMatch(isMatch, newMatch) {
         const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;
         const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);
+        const newRecordNonMatchCount = isMatch ? recordNonMatchCount : recordNonMatchCount + 1;
 
         debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);
 
@@ -265,13 +292,14 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         if (returnNonMatches) {
           debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);
         }
+        debugData(`- Total nonMatchCount after this detection: ${recordNonMatchCount}`);
 
         if (maxMatchesFound({matches: matches.concat(newRecordMatches), maxMatches})) {
           debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
-          return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount};
+          return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: newRecordNonMatchCount};
         }
 
-        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount});
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount});
       }
 
       function candidateNotInMatches(matches, candidate) {

package/src/index.spec.js

@@ -46,7 +46,7 @@ describe('INDEX', () => {
     }
   });
 
-  async function callback({getFixture, options, enabled = true, expectedMatchStatus, expectedStopReason}) {
+  async function callback({getFixture, options, enabled = true, expectedMatchStatus, expectedStopReason, expectedFailures}) {
 
     if (!enabled) {
       debug(`Disabled test!`);
@@ -59,8 +59,11 @@ describe('INDEX', () => {
 
 
     const match = createMatchInterface(formatOptions());
-    const {matches, matchStatus, nonMatches} = await match(record);
-    debugData(`${matches.length}, ${matchStatus.status}/${matchStatus.stopReason}, ${nonMatches ? nonMatches.length : nonMatches}`);
+    const {matches, matchStatus, nonMatches, conversionFailures} = await match(record);
+    debugData(`Matches: ${matches.length}, Status: ${matchStatus.status}/${matchStatus.stopReason}, NonMatches: ${nonMatches ? nonMatches.length : 'not returned'}, ConversionFailures: ${conversionFailures ? conversionFailures.length : 'not returned'}`);
+
+    expect(matchStatus.status).to.eql(expectedMatchStatus);
+    expect(matchStatus.stopReason).to.eql(expectedStopReason);
 
     const formattedMatchResult = formatRecordResults(matches);
     expect(formattedMatchResult).to.eql(expectedMatches);
@@ -68,9 +71,10 @@ describe('INDEX', () => {
     const formattedNonMatchResult = formatRecordResults(nonMatches);
     expect(formattedNonMatchResult).to.eql(expectedNonMatches);
 
-    expect(matchStatus.status).to.eql(expectedMatchStatus);
-    expect(matchStatus.stopReason).to.eql(expectedStopReason);
-
+    // eslint-disable-next-line functional/no-conditional-statement
+    if (expectedFailures) {
+      expect(conversionFailures).to.eql(expectedFailures);
+    }
 
     function formatOptions() {
       const contextFeatures = matchDetection.features[options.detection.strategy.type];