@natlibfi/melinda-record-matching 4.4.0-alpha.1 → 5.0.0-alpha.1

package/src/candidate-search/query-list/component.js

@@ -0,0 +1,127 @@
+import createDebugLogger from 'debug';
+import {promisify} from 'util';
+import {toQueries} from '../candidate-search-utils.js';
+import {getSubfieldValues, testStringOrNumber, toMelindaIds} from '../../matching-utils.js';
+
+const setTimeoutPromise = promisify(setTimeout); // eslint-disable-line
+
+export function hostIdMelinda(record) {
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:hostIdMelinda');
+  const debugData = debug.extend('data'); // eslint-disable-line
+  debug(`Creating query for the Melinda Id host`);
+
+  const ids = hostId(record);
+  if (ids.length > 0) {
+    return toQueries(ids, 'melinda.partsofhost');
+  }
+
+  debug(`No valid Melinda Id host found`);
+  return [];
+
+  function hostId(record) {
+    // Multi 773 handling
+    const f773s = record.get(/^773$/u)
+      .filter(f773 => f773.subfields.some(sub => sub.code === 'w' && (/\(FI-MELINDA\).*/ui).test(sub.value)));
+
+    if (f773s.length === 0) {
+      return false;
+    }
+
+    // Multi $w handling
+    // $w (prefix)<id> handling
+    // $w <id> & $w (prefix)<id> Match
+    const melindaIds = f773s.map(f773 => toMelindaIds(f773, ['w'])).flat()
+      .filter(value => testStringOrNumber(value)) // drop invalid values
+      .filter((value, index, array) => array.indexOf(value) === index); // unique values;
+    return melindaIds;
+  }
+}
+
+export async function hostIdOtherSource(record, client) {
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:hostIdOtherSource');
+  const debugData = debug.extend('data'); // eslint-disable-line
+  debug(`Creating query for the Other source Id host`);
+
+  const ids = getHostIds(record);
+  const otherSources = getOtherSources(record);
+  if (ids.length > 0 && otherSources.length > 0) {
+    const otherSourceIds = ids.map(id => otherSources.map(source => `${id}${source}`)).flat();
+    const melindaHostQueries = await handleSruCalls(otherSourceIds);
+    debug(JSON.stringify(melindaHostQueries));
+
+    return melindaHostQueries;
+  }
+
+  debug(`No valid other source hosts found`);
+  return [];
+
+
+  function getHostIds(record) {
+    const f773s = record.get(/^773$/u)
+      .filter(f773 => f773.subfields.some(sub => sub.code === 'w' && !(/\(FI-MELINDA\).*/ui).test(sub.value)));
+    if (f773s.length === 0) {
+      return false;
+    }
+
+    // Multi 773 handling
+    const subfieldWs = f773s
+      .map(f773 => {
+        debugData(`f773: ${JSON.stringify(f773)}`);
+        return getSubfieldValues(f773, 'w').flat();
+      }).flat();
+
+    // Multi $w handling
+    // $w (prefix)<id> handling
+    // $w <id> & $w (prefix)<id> Match
+    const ids = subfieldWs.map(value => `${value}`.replace(/\(FI-.*\)/ui, '')) // remove prefixes
+      .filter(value => testStringOrNumber(value)) // drop invalid values
+      .filter((value, index, array) => array.indexOf(value) === index); // unique values
+
+    return ids;
+  }
+
+  function getOtherSources(record) {
+    const fSids = record.get('SID');
+    return fSids.map(field => getSubfieldValues(field, 'b'));
+  }
+
+  async function handleSruCalls(otherSourceIds, ids = []) {
+    const [otherSourceId, ...rest] = otherSourceIds;
+
+    if (otherSourceId === undefined) {
+      debug(`host ids: ${ids}`);
+      const validIds = ids.filter(id => id);
+      return toQueries(validIds, 'melinda.partsofhost');
+    }
+
+    const otherSourceHostQuery = await toQueries([otherSourceId], 'melinda.sourceid');
+    const id = await new Promise((resolve, reject) => {
+      debug(`Searching for hosts with query: ${otherSourceHostQuery}`);
+      let recordId;
+
+      client.searchRetrieve(otherSourceHostQuery)
+        .on('error', err => {
+          debug(`SRU error for query: ${otherSourceHostQuery}: ${err}`);
+          reject(err);
+        })
+        .on('end', async () => {
+          try {
+            debug(`Searching for hosts: done`);
+            await setTimeoutPromise(10);
+            resolve(recordId);
+          } catch (err) {
+            debug(`Error caught on END`);
+            reject(err);
+          }
+        })
+        .on('record', record => {
+          const [field] = record.get(/^001$/u);
+          debug(field);
+          recordId = field.value ? field.value : '';
+        });
+    });
+
+    return handleSruCalls(rest, [...ids, id]);
+  }
+}
+

package/src/candidate-search/query-list/index.js

@@ -1,32 +1,9 @@
-/**
-*
-* @licstart  The following is the entire license notice for the JavaScript code in this file.
-*
-* Melinda record matching modules for Javascript
-*
-* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
-*
-* This file is part of melinda-record-matching-js
-*
-* melinda-record-matching-js program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* melinda-record-matching-js is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Affero General Public License
-* along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*
-* @licend  The above is the entire license notice
-* for the JavaScript code in this file.
-*
-*/
-
-import * as bib from './bib';
+import * as bib from './bib.js';
+import * as component from './component.js';
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:index');
+const debugData = debug.extend('data');
 
 export const searchTypes = {
   bib: {
@@ -35,22 +12,41 @@ export const searchTypes = {
     title: 'bibTitle', // title ( + first author + first publisher if needed)
     titleAuthor: 'bibTitleAuthor', // title + first author (or first publisher if no author)
     titleAuthorYear: 'bibTitleAuthorYear', // title + first author (or first publisher if no author), publishing year
+    titleAuthorYearAlternates: 'bibTitleAuthorYearAlternates', // title + first author (or first publisher if no author), publishing year
     melindaId: 'bibMelindaIds',
     sourceIds: 'bibSourceIds'
     //DEVELOP: bibContent: 'bibContent'
+  },
+  component: {
+    hostIdMelinda: 'hostIdMelinda', // 773 $w (FI-MELINDA)
+    hostIdOtherSource: 'hostIdOtherSource', // 773 $w !(FI-MELINDA)
+    hostIsbn: 'hostIsbn' // 773 $z
   }
 };
 
-export default (record, searchSpec) => {
-  const extractors = {...bib};
+export default async (record, searchSpec, client) => {
+  const extractors = {...bib, ...component};
+  debugData(`extractors: ${JSON.stringify(extractors)}`);
+  debugData(`searchSpec: ${JSON.stringify(searchSpec)}`);
+
+  const qExtractors = searchSpec.map(generateQueryExtractor);
+  const results = await handleQextractors(qExtractors);
+  return results;
+
+  async function handleQextractors(qExtractors, results = []) {
+    const [qExtractor, ...rest] = qExtractors;
 
-  return searchSpec
-    .map(generateQueryExtractor)
-    .map(cb => cb(record))
-    .flat();
+    if (qExtractor === undefined) {
+      return results.flat();
+    }
+
+    const result = await qExtractor(record, client);
+    return handleQextractors(rest, [...results, result]);
+  }
 
   function generateQueryExtractor(type) {
     if (extractors[type]) {
+      //debugData(`${JSON.stringify(extractors[type])}`);
       return extractors[type];
     }
 

package/src/cli.js

@@ -0,0 +1,177 @@
+import fs from 'fs';
+import yargs from 'yargs';
+import createMatchOperator, {candidateSearch, matchDetection} from './index.js';
+import createDebugLogger from 'debug';
+import {MarcRecord} from '@natlibfi/marc-record';
+
+cli();
+
+async function cli() {
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:cli');
+  const args = yargs(process.argv.slice(2))
+    .scriptName('melinda-record-matching-js')
+    .epilog('Copyright (C) 2022-2023 University Of Helsinki (The National Library Of Finland)')
+    .usage('$0 <file> [options] and env variable info in README')
+    .usage('Installed globally: $0 <file> [options] and env variable info in README')
+    .usage('Not installed: npx $0 <file> [options] and env variable info in README')
+    .usage('Build from source: node dist/index.js <file> [options] and env variable info in README')
+    .showHelpOnFail(true)
+    .example([['$ node /dist/cli.js record.json']])
+    .env('MELINDA_RECORD_MATCH')
+    .version()
+    .positional('file', {type: 'string', describe: 'Json file of records to match'})
+    .options({
+      t: {type: 'string', default: 'IDS', alias: 'searchType', describe: 'IDS, STANDARD_IDS, COMPONENT, CONTENT or CONTENTALT'},
+      m: {type: 'number', default: 1, alias: 'maxMatches', describe: ''},
+      c: {type: 'number', default: 1000, alias: 'maxCandidates', describe: ''},
+      s: {type: 'boolean', default: false, alias: 'returnStrategy', describe: ''},
+      q: {type: 'boolean', default: false, alias: 'returnQuery', describe: ''},
+      n: {type: 'boolean', default: false, alias: 'returnNonMatches', describe: ''}
+    })
+    .check((args) => {
+      const [file] = args._;
+      if (file === undefined) {
+        throw new Error('No file argument given');
+      }
+
+      if (!fs.existsSync(file)) {
+        throw new Error(`File ${file} does not exist`);
+      }
+
+      if (args.sruUrl === undefined) {
+        throw new Error('Setup sru url');
+      }
+
+      if (!['IDS', 'STANDARD_IDS', 'COMPONENT', 'CONTENT', 'CONTENTALT'].includes(args.searchType)) {
+        throw new Error('Invalid search type');
+      }
+
+      return true;
+    })
+    .parseSync();
+
+  const [file] = args._;
+  const {searchType} = args;
+  debug(JSON.stringify(args));
+
+  const detection = {
+    treshold: 0.9,
+    strategy: generateStrategy(searchType)
+  };
+
+  const search = {
+    url: args.sruUrl, searchSpec: generateSearchSpec(searchType)
+  };
+
+  const matchOperator = await createMatchOperator({detection, search, ...args});
+
+  const fileRaw = fs.readFileSync(file, 'utf8');
+  const record = new MarcRecord(JSON.parse(fileRaw), {subfieldValues: false});
+
+  const result = await matchOperator({record});
+  debug(JSON.stringify(result));
+
+
+  function generateStrategy(searchType) {
+    if (['IDS'].includes(searchType)) {
+      return [
+        matchDetection.features.bib.melindaId(),
+        matchDetection.features.bib.allSourceIds()
+      ];
+    }
+
+    // We could have differing strategy for STANDARD_IDS
+    // Let's not run title in strategy when we found the candidates through standard_ids search
+
+    if (['STANDARD_IDS'].includes(searchType)) {
+      return [
+        matchDetection.features.bib.hostComponent(),
+        matchDetection.features.bib.isbn(),
+        matchDetection.features.bib.issn(),
+        matchDetection.features.bib.otherStandardIdentifier(),
+        // Let's not use the same title matchDetection here
+        //matchDetection.features.bib.title(),
+        matchDetection.features.bib.authors(),
+        // We probably should have some leeway here for notated music as BK etc.
+        matchDetection.features.bib.recordType(),
+        // Use publicationTimeAllowConsYearsMulti to
+        //  - ignore one year differences in publicationTime
+        //  - extract publicationTimes from f008, f26x and reprint notes in f500
+        //  - do not substract points for mismatching (normal) publicationTime, if there's a match between
+        //       normal publicationTime and a reprintPublication time
+        matchDetection.features.bib.publicationTimeAllowConsYearsMulti(),
+        matchDetection.features.bib.language(),
+        matchDetection.features.bib.bibliographicLevel()
+      ];
+    }
+
+    if (['COMPONENT'].includes(searchType)) {
+      return [
+        matchDetection.features.bib.hostComponent(),
+        matchDetection.features.bib.otherStandardIdentifier(),
+        matchDetection.features.bib.recordType(),
+        matchDetection.features.bib.title(),
+        matchDetection.features.bib.language(),
+        matchDetection.features.bib.authors(),
+        matchDetection.features.bib.bibliographicLevel()
+      ];
+    }
+
+    if (['CONTENT', 'CONTENTALT'].includes(searchType)) {
+      return [
+        matchDetection.features.bib.hostComponent(),
+        matchDetection.features.bib.isbn(),
+        matchDetection.features.bib.issn(),
+        matchDetection.features.bib.otherStandardIdentifier(),
+        matchDetection.features.bib.title(),
+        matchDetection.features.bib.authors(),
+        matchDetection.features.bib.recordType(),
+        matchDetection.features.bib.publicationTime(),
+        matchDetection.features.bib.language(),
+        matchDetection.features.bib.bibliographicLevel()
+      ];
+    }
+
+    throw new Error('Unsupported match validation package');
+  }
+
+
+  function generateSearchSpec(searchType) {
+    if (['IDS'].includes(searchType)) {
+      return [
+        candidateSearch.searchTypes.bib.melindaId,
+        candidateSearch.searchTypes.bib.sourceIds
+      ];
+    }
+
+    if (['STANDARD_IDS'].includes(searchType)) {
+      return [candidateSearch.searchTypes.bib.standardIdentifiers];
+    }
+
+    if (['COMPONENT'].includes(searchType)) {
+      return [
+        //candidateSearch.searchTypes.bib.sourceIds,
+        candidateSearch.searchTypes.component.hostIdMelinda,
+        candidateSearch.searchTypes.component.hostIdOtherSource
+      ];
+    }
+
+    if (['CONTENT'].includes(searchType)) {
+      return [
+        candidateSearch.searchTypes.bib.hostComponents,
+        //candidateSearch.searchTypes.bib.titleAuthor,
+        candidateSearch.searchTypes.bib.title
+      ];
+    }
+    if (['CONTENTALT'].includes(searchType)) {
+      return [
+        candidateSearch.searchTypes.bib.hostComponents,
+        // titleAuthorYearAlternates searches for matchCandidates
+        // with alternate queries, starting from more tight searches
+        candidateSearch.searchTypes.bib.titleAuthorYearAlternates
+      ];
+    }
+
+    throw new Error('Unsupported match validation package');
+  }
+}

package/src/index.js

@@ -1,39 +1,11 @@
-/**
-*
-* @licstart  The following is the entire license notice for the JavaScript code in this file.
-*
-* Melinda record matching modules for Javascript
-*
-* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
-*
-* This file is part of melinda-record-matching-js
-*
-* melinda-record-matching-js program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* melinda-record-matching-js is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Affero General Public License
-* along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*
-* @licend  The above is the entire license notice
-* for the JavaScript code in this file.
-*
-*/
-
 import createDebugLogger from 'debug';
-import createSearchInterface, * as candidateSearch from './candidate-search';
-import createDetectionInterface, * as matchDetection from './match-detection';
+import createSearchInterface, * as candidateSearch from './candidate-search/index.js';
+import createDetectionInterface, * as matchDetection from './match-detection/index.js';
 //import inspect from 'util';
 
 export {candidateSearch, matchDetection};
 
-export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false, returnFailures = false}) => {
+export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false}) => {
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:index');
   const debugData = debug.extend('data');
 
@@ -44,14 +16,15 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
   debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
   debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);
   debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);
-  debugData(`ReturnFailures: ${JSON.stringify(returnFailures)}`);
 
 
   const detect = createDetectionInterface(detectionOptions, returnStrategy);
 
-  return ({record, recordExternal = {recordSource: 'incomingRecord', label: 'ic'}}) => {
+  return prepareSearch;
+
+  async function prepareSearch({record, recordExternal = {recordSource: 'incomingRecord', label: 'ic'}}) {
 
-    const search = createSearchInterface({...searchOptions, record, maxCandidates, recordExternal});
+    const {search} = await createSearchInterface({...searchOptions, record, maxCandidates, recordExternal});
     return iterate({});
 
     // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set
@@ -67,17 +40,14 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // state.queryCounter : sequence for current query
     // state.maxedQueries : queries that resulted in more than serverMaxResults hits
 
-    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, conversionFailures = [], matchErrors = []}) {
+    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, matchErrors = []}) {
       debugData(`Starting next matcher iteration.`);
-      const {records, failures, ...state} = await search(initialState);
+      const {records, ...state} = await search(initialState);
 
-      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, conversionFailures: ${conversionFailures}, matchErrors: ${matchErrors.length}`);
+      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, matchErrors: ${matchErrors.length}`);
       const recordSetSize = records.length;
-      const failureSetSize = failures.length;
-      const newCandidateCount = candidateCount + recordSetSize + failureSetSize;
+      const newCandidateCount = candidateCount + recordSetSize;
 
-      const newConversionFailures = conversionFailures.concat(failures);
-      debugData(`Failures: ${failures.length}, ConversionFailures: ${conversionFailures.length}, NewConversionFailures: ${newConversionFailures.length}`);
 
       if (recordSetSize > 0) {
         return handleRecordSet();
@@ -85,11 +55,11 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
 
       if (state.queriesLeft > 0) {
         debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);
-        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});
+        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, matchErrors});
       }
 
       debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);
-      return returnResult({matches, state, stopReason: '', nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});
+      return returnResult({matches, state, stopReason: '', nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, matchErrors});
 
       function handleRecordSet() {
         debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);
@@ -101,19 +71,18 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         const {newMatches, newNonMatches, newMatchErrors} = handleMatchResult(matchResult, matches, nonMatches, matchErrors);
 
         if (maxMatchesFound({matches: newMatches, maxMatches})) {
-          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
+          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors});
         }
 
         if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
-          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
+          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors});
         }
 
-        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
+        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors});
       }
 
       function handleMatchResult(matchResult, matches, nonMatches, matchErrors) {
         debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);
-        // eslint-disable-next-line functional/no-conditional-statements
         if (returnNonMatches) {
           debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);
         }
@@ -123,7 +92,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         const newMatchErrors = matchErrors.concat(matchResult.matchErrors);
 
         debugData(`- Total amount of matches: ${newMatches.length}`);
-        // eslint-disable-next-line functional/no-conditional-statements
         if (returnNonMatches) {
           debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);
         }
@@ -167,27 +135,24 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
     // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
 
-    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, conversionFailures, matchErrors}) {
-      const conversionFailureCount = conversionFailures.length;
+    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, matchErrors}) {
       const matchErrorCount = matchErrors.length;
-      checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount});
-      const matchStatus = getMatchState(state, stopReason, conversionFailureCount, matchErrorCount);
+      checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, matchErrorCount});
+      const matchStatus = getMatchState(state, stopReason, matchErrorCount);
       // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
       const matchesResult = returnNonMatches ? {matches, matchStatus, nonMatches, candidateCount} : {matches, matchStatus, candidateCount};
-      const failures = [...conversionFailures, ...matchErrors];
-      const result = returnFailures ? {...matchesResult, conversionFailures: failures} : matchesResult;
-      debugData(`ReturnFailures ${returnFailures}`);
+      const result = matchesResult;
       debugData(`${JSON.stringify(result)}`);
       return result;
 
       // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
 
-      function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount}) {
+      function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, matchErrorCount}) {
         const matchCount = matches.length;
         debugData(`Return nonMatches: ${returnNonMatches}`);
         const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
         const totalHandled = matchCount + chosenNonMatchCount + duplicateCount;
-        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, conversionFailureCount: ${conversionFailureCount}, matchErrorCount: ${matchErrorCount}`);
+        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, matchErrorCount: ${matchErrorCount}`);
         debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
         if (totalHandled !== candidateCount) {
           debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);
@@ -197,9 +162,8 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
       }
 
       // eslint-disable-next-line max-statements
-      function getMatchState(state, stopReason, conversionFailuresCount, matchErrorCount) {
+      function getMatchState(state, stopReason, matchErrorCount) {
         debugData(`${JSON.stringify(state)}`);
-        debug(`We had ${conversionFailuresCount} retrieved candidates that could not be converted.`);
         debug(`We had ${matchErrorCount} retrieved candidates that errored in matchDetection.`);
         debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
 
@@ -209,16 +173,14 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
         debugData(`nonRetrieved: ${nonRetrieved}`);
 
-        // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
+        // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries', empty string/undefined), reason for stopping retrieving or handling the candidate records
         // 'maxMatches' and 'maxCandidates' are in stopReason, 'maxedQueries', 'conversionFailures' and 'matchErrors' are created here
 
-        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || conversionFailureCount > 0 || matchErrorCount > 0) {
+        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || matchErrorCount > 0) {
           const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : undefined;
-          const conversionFailuresStopReason = conversionFailureCount > 0 ? 'conversionFailures' : undefined;
           const matchErrorsStopReason = matchErrorCount > 0 ? 'matchErrors' : undefined;
-          const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || conversionFailuresStopReason || matchErrorsStopReason : stopReason;
+          const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || matchErrorsStopReason : stopReason;
           debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);
-          debugData(`ConversionFailureStopReason <${conversionFailuresStopReason}>`);
           debugData(`MatchErrorsStopReason <${matchErrorsStopReason}>`);
           debugData(`NewStopReason: <${newStopReason}>`);
           debug(`Match status: false`);
@@ -266,8 +228,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
       */
 
       if (candidate) {
-
-        // eslint-disable-next-line functional/no-conditional-statements
         if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {
           const {record: candidateRecord, id: candidateId} = candidate;
           const recordBExternal = {id: candidate.id, recordSource: 'databaseRecord', label: `db-${candidate.id}`};
@@ -330,7 +290,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
 
         debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);
 
-        // eslint-disable-next-line functional/no-conditional-statements
         if (returnNonMatches) {
           debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);
         }
@@ -365,5 +324,5 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         return true;
       }
     }
-  };
+  }
 };