@natlibfi/melinda-record-matching 2.0.1-alpha.1 → 2.0.7-alpha.1

package/src/index.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -29,68 +29,340 @@
 import createDebugLogger from 'debug';
 import createSearchInterface, * as candidateSearch from './candidate-search';
 import createDetectionInterface, * as matchDetection from './match-detection';
+//import inspect from 'util';
 
 export {candidateSearch, matchDetection};
 
-export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25}) => {
+export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false, returnFailures = false}) => {
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:index');
-  const detect = createDetectionInterface(detectionOptions);
+  const debugData = debug.extend('data');
 
-  return record => {
-    const search = createSearchInterface({...searchOptions, record});
-    return iterate();
+  debugData(`DetectionOptions: ${JSON.stringify(detectionOptions)}`);
+  debugData(`SearchOptions: ${JSON.stringify(searchOptions)}`);
+  debugData(`MaxMatches: ${JSON.stringify(maxMatches)}`);
+  debugData(`MaxCandidates: ${JSON.stringify(maxCandidates)}`);
+  debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
+  debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);
+  debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);
+  debugData(`ReturnFailures: ${JSON.stringify(returnFailures)}`);
 
-    // eslint-disable-next-line max-statements
-    async function iterate(initialState = {}, matches = [], candidateCount = 0) {
-      const {records, ...state} = await search(initialState);
 
-      if (records.length > 0 || state.queriesLeft > 0) {
-        debug(`Checking ${records.length} candidates for matches`);
+  const detect = createDetectionInterface(detectionOptions, returnStrategy);
 
-        const matchResult = iterateRecords(records);
+  return ({record, recordExternal = {recordSource: 'incomingRecord', label: 'ic'}}) => {
 
-        if (matchResult) {
-          const newMatches = matches.concat(matchResult);
+    const search = createSearchInterface({...searchOptions, record, maxCandidates, recordExternal});
+    return iterate({});
 
-          if (newMatches.length === maxMatches) {
-            return newMatches;
-          }
+    // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set
+    // matches : candidates that have been detected as matches by current matcher job
+    // nonMatches : candidates that have been detected as non-matches by current matcher job (only if returnNonMatches is 'true')
+    // duplicateCount : amount of candidate records that were retrieved from the SRU but not handled further because they were already found in the matches/nonMatches
+
+    // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
+    // state.query : current query (undefined if there was no queries left)
+    // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
+    // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)
+    // state.queriesLeft : amount of queries left
+    // state.queryCounter : sequence for current query
+    // state.maxedQueries : queries that resulted in more than serverMaxResults hits
+
+    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, conversionFailures = [], matchErrors = []}) {
+      debugData(`Starting next matcher iteration.`);
+      const {records, failures, ...state} = await search(initialState);
+
+      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, conversionFailures: ${conversionFailures}, matchErrors: ${matchErrors.length}`);
+      const recordSetSize = records.length;
+      const failureSetSize = failures.length;
+      const newCandidateCount = candidateCount + recordSetSize + failureSetSize;
+
+      const newConversionFailures = conversionFailures.concat(failures);
+      debugData(`Failures: ${failures.length}, ConversionFailures: ${conversionFailures.length}, NewConversionFailures: ${newConversionFailures.length}`);
+
+      if (recordSetSize > 0) {
+        return handleRecordSet();
+      }
+
+      if (state.queriesLeft > 0) {
+        debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);
+        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});
+      }
+
+      debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);
+      return returnResult({matches, state, stopReason: '', nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});
+
+      function handleRecordSet() {
+        debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);
+
+        const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches, nonMatchCount});
+
+        const newDuplicateCount = duplicateCount + matchResult.duplicateCount;
+        const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;
+        const {newMatches, newNonMatches, newMatchErrors} = handleMatchResult(matchResult, matches, nonMatches, matchErrors);
+
+        if (maxMatchesFound({matches: newMatches, maxMatches})) {
+          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
+        }
+
+        if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
+          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
+        }
+
+        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
+      }
 
-          return maxCandidatesRetrieved() ? newMatches : iterate(state, newMatches, candidateCount + records.length);
+      function handleMatchResult(matchResult, matches, nonMatches, matchErrors) {
+        debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);
+        // eslint-disable-next-line functional/no-conditional-statements
+        if (returnNonMatches) {
+          debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);
         }
 
-        return maxCandidatesRetrieved() ? matches : iterate(state, matches, candidateCount + records.length);
+        const newMatches = matches.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);
+        const newNonMatches = returnNonMatches ? nonMatches.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];
+        const newMatchErrors = matchErrors.concat(matchResult.matchErrors);
+
+        debugData(`- Total amount of matches: ${newMatches.length}`);
+        // eslint-disable-next-line functional/no-conditional-statements
+        if (returnNonMatches) {
+          debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);
+        }
+
+        debugData(`MatchResult: ${JSON.stringify(matchResult)}`);
+        debugData(`Old matchErrors: ${JSON.stringify(matchErrors)}, matchErrors from matchResult: ${JSON.stringify(matchResult.matchErrors)}, New matchErrors: ${JSON.stringify(newMatchErrors)}`);
+
+        debugData(`- Total amount of matchErrors: ${newMatchErrors.length}`);
+
+        return {newMatches, newNonMatches, newMatchErrors};
       }
 
-      debug(`No (more) candidate records to check, matches: ${matches.length}`);
-      return matches;
+      function addQuery(matches) {
+        debugData(`Adding query ${state.query} to matches`);
+        return matches.map((match) => ({...match, matchQuery: state.query}));
+      }
 
-      function maxCandidatesRetrieved() {
-        if (candidateCount + records.length > maxCandidates) {
-          debug(`Stopped searching because maximum number of candidates have been retrieved`);
+      function maxCandidatesRetrieved(candidateCount, maxCandidates) {
+        debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${maxCandidates})`);
+        if (maxCandidates && candidateCount >= maxCandidates) {
+          debug(`Stopped matching because maximum number of candidate records ${candidateCount} / ${maxCandidates} have been retrieved`);
           return true;
         }
       }
+    }
+
+    // matches : array of matching candidate records
+    // nonMatches : array of nonMatching candidate records (if returnNonMatches option is true, otherwise empty array)
+    // - candidate.id
+    // - candidate.record
+    // - probability
+    // - strategy (if returnStrategy option is true)
+    // - treshold (if returnStrategy option is true)
+    // - matchQuery (if returnQuery option is true)
+    // failures: array of conversionFailures from candidate-search and matchErrors from matchDetection in error format {status, payload: {message, id}} if returnFailures is true
+
+    // we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data
+
+    // matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not
+    // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
+    // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
+    // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
+
+    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, conversionFailures, matchErrors}) {
+      const conversionFailureCount = conversionFailures.length;
+      const matchErrorCount = matchErrors.length;
+      checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount});
+      const matchStatus = getMatchState(state, stopReason, conversionFailureCount, matchErrorCount);
+      // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
+      const matchesResult = returnNonMatches ? {matches, matchStatus, nonMatches, candidateCount} : {matches, matchStatus, candidateCount};
+      const failures = [...conversionFailures, ...matchErrors];
+      const result = returnFailures ? {...matchesResult, conversionFailures: failures} : matchesResult;
+      debugData(`ReturnFailures ${returnFailures}`);
+      debugData(`${JSON.stringify(result)}`);
+      return result;
+
+      // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
+
+      function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount}) {
+        const matchCount = matches.length;
+        debugData(`Return nonMatches: ${returnNonMatches}`);
+        const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
+        const totalHandled = matchCount + chosenNonMatchCount + duplicateCount;
+        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, conversionFailureCount: ${conversionFailureCount}, matchErrorCount: ${matchErrorCount}`);
+        debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
+        if (totalHandled !== candidateCount) {
+          debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);
+          return;
+        }
+        return;
+      }
+
+      // eslint-disable-next-line max-statements
+      function getMatchState(state, stopReason, conversionFailuresCount, matchErrorCount) {
+        debugData(`${JSON.stringify(state)}`);
+        debug(`We had ${conversionFailuresCount} retrieved candidates that could not be converted.`);
+        debug(`We had ${matchErrorCount} retrieved candidates that errored in matchDetection.`);
+        debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
+
+        debugData(`StopReason: <${stopReason}>`);
+
+        const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;
+        const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
+        debugData(`nonRetrieved: ${nonRetrieved}`);
+
+        // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
+        // 'maxMatches' and 'maxCandidates' are in stopReason, 'maxedQueries', 'conversionFailures' and 'matchErrors' are created here
+
+        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || conversionFailureCount > 0 || matchErrorCount > 0) {
+          const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : undefined;
+          const conversionFailuresStopReason = conversionFailureCount > 0 ? 'conversionFailures' : undefined;
+          const matchErrorsStopReason = matchErrorCount > 0 ? 'matchErrors' : undefined;
+          const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || conversionFailuresStopReason || matchErrorsStopReason : stopReason;
+          debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);
+          debugData(`ConversionFailureStopReason <${conversionFailuresStopReason}>`);
+          debugData(`MatchErrorsStopReason <${matchErrorsStopReason}>`);
+          debugData(`NewStopReason: <${newStopReason}>`);
+          debug(`Match status: false`);
+          return {status: false, stopReason: newStopReason};
+        }
+
+        debug(`Match status: true`);
+        return {status: true, stopReason};
+      }
+    }
+
+    // NOTES:
+    // - we could optimize by creating the featureSet for the incoming record once and using it for all database/candidateRecords
+    // - if creating the featureSet for the incoming record fails we have an unprocessable entity
+    // - if creating the featureSet for a candidate record fails we could skip that candidate - but list the case as a detectionFailure, same as conversionFailures
+
+    function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0, recordMatchErrors = []}) {
+
+      // recordSetSize : total amount of records in the current record set
+      // recordCount : amount of records from the current record set that have been handled
+      // maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped
+      // recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results
+      // recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)
+
+      // records : non-handled records in the current record set
+      // matches : found matches in the current matcher job
+      // recordMatches : found matches in the current record set
+      // recordNonMatches : found nonMatches in the current record set (only if returnNonMatches setting is true)
+      // recordMatchErrors: errored matchDetection in the current record set
+
+      const [candidate] = records;
+      const newRecordCount = candidate ? recordCount + 1 : recordCount;
 
-      function iterateRecords(records) {
-        const [candidate] = records;
+      // The matcher uses same matchDetection strategy for candidates from all candidate-searches -> matchDetection result for the same candidate is always same
+      // Exceptions would happen if the candidate would have been updated in the database between candidate searches
+      // Note that if returnNonMatches is false, matcher won't remember candidates that didn't match, so they will be matched again everytime they are retrieved by
+      // different candidate search queries. Same candidate search query won't have duplicate records.
 
-        if (candidate) {
+      /* We could optimize and detect all retrieved candidates at once
+      const candidateRecords = records.map(record => record.record);
+      const recordsIsArray = Array.isArray(candidateRecords);
+      debug(`records is an array: ${recordsIsArray}`);
+      const result = detect(record, candidateRecords);
+      debugData(`${JSON.stringify(result)}`);
+      */
+
+      if (candidate) {
+
+        // eslint-disable-next-line functional/no-conditional-statements
+        if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {
           const {record: candidateRecord, id: candidateId} = candidate;
-          const {match, probability} = detect(record, candidateRecord);
-
-          if (match) {
-            return {
-              probability,
-              candidate: {
-                id: candidateId,
-                record: candidateRecord
-              }
-            };
+          const recordBExternal = {id: candidate.id, recordSource: 'databaseRecord', label: `db-${candidate.id}`};
+          try {
+            debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);
+            // we should handle errors from detection somehow - ie. cases where either record or candidateRecord errors
+            const detectionResult = detect({recordA: record, recordB: candidateRecord, recordAExternal: recordExternal, recordBExternal});
+
+            return handleDetectionResult(detectionResult, candidateId, candidateRecord);
+          } catch (error) {
+            debug(`MatchDetection errored: database record ${candidateId}: ${error}`);
+
+            const matchError = {status: 422, payload: {message: `Matching errored for database record ${candidateId}. ${error.message}.`, id: candidateId}};
+            const newRecordMatchErrors = recordMatchErrors.concat(matchError);
+            return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount, recordMatchErrors: newRecordMatchErrors});
           }
+        }
+
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount, recordMatchErrors});
+      }
+
+      debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);
+      return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount, matchErrors: recordMatchErrors};
+
+      function handleDetectionResult(detectionResult, candidateId, candidateRecord) {
+        debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);
 
-          return iterateRecords(records.slice(1));
+        if (detectionResult.match || returnNonMatches) {
+          debug(`${detectionResult.match ? `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is a match!` : `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is NOT a match!`}`);
+          debugData(`Strategy: ${JSON.stringify(detectionResult.strategy)}, Treshold: ${JSON.stringify(detectionResult.treshold)}`);
+
+          const matchResult = {
+            probability: detectionResult.probability,
+            candidate: {
+              id: candidateId,
+              record: candidateRecord
+            }
+          };
+          const strategyResult = {
+            strategy: detectionResult.strategy,
+            treshold: detectionResult.treshold
+          };
+          const newMatch = returnStrategy ? {...matchResult, ...strategyResult} : {...matchResult};
+
+          debugData(`${JSON.stringify(newMatch)}`);
+
+          return handleRecordMatch(detectionResult.match, newMatch);
+        }
+
+        const newRecordNonMatchCount = recordNonMatchCount + 1;
+        debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);
+
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, recordMatchErrors});
+      }
+
+      function handleRecordMatch(isMatch, newMatch) {
+        const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;
+        const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);
+        const newRecordNonMatchCount = isMatch ? recordNonMatchCount : recordNonMatchCount + 1;
+
+        debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);
+
+        // eslint-disable-next-line functional/no-conditional-statements
+        if (returnNonMatches) {
+          debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);
         }
+        debugData(`- Total nonMatchCount after this detection: ${recordNonMatchCount}`);
+
+        if (maxMatchesFound({matches: matches.concat(newRecordMatches), maxMatches})) {
+          debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
+          return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors};
+        }
+
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount, matchErrors: recordMatchErrors});
+      }
+
+      function candidateNotInMatches(matches, candidate) {
+        debug(`Checking that record ${candidate.id} is not already included in ${matches.length} matches/nonMatches`);
+        const newCandidateId = candidate.id;
+        debugData(`newCandidateId: ${newCandidateId}`);
+        const result = matches.find(({candidate}) => candidate.id === newCandidateId);
+        debugData(`Result: ${result}`);
+        if (result) {
+          debug(`${candidate.id} was already handled.`);
+          return false;
+        }
+        debug(`${candidate.id} not found in matches/nonMatches`);
+        return true;
+      }
+    }
+
+    function maxMatchesFound({matches, maxMatches}) {
+      if (maxMatches && matches.length >= maxMatches) {
+        debug(`Stopping recordSet iteration: maxMatches (${maxMatches}) for matcher job found.`);
+        return true;
       }
     }
   };

package/src/index.spec.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2023 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -31,6 +31,10 @@ import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen-http-client';
 import {MarcRecord} from '@natlibfi/marc-record';
 import createMatchInterface, {matchDetection} from '.';
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:index:test');
+const debugData = debug.extend('data');
 
 describe('INDEX', () => {
   generateTests({
@@ -42,19 +46,32 @@ describe('INDEX', () => {
     }
   });
 
-  async function callback({getFixture, options, enabled = true}) {
-
-    if (!enabled) {
-      return;
-    }
+  // eslint-disable-next-line max-statements
+  async function callback({getFixture, options, expectedMatchStatus, expectedStopReason, expectedFailures, expectedCandidateCount}) {
 
     const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});
     const expectedMatches = getFixture('expectedMatches.json');
+    const expectedNonMatches = getFixture('expectedNonMatches.json') || [];
+
 
     const match = createMatchInterface(formatOptions());
-    const matches = await match(record);
+    const {matches, matchStatus, nonMatches, conversionFailures, candidateCount} = await match({record});
+    debugData(`Matches: ${matches.length}, Status: ${matchStatus.status}/${matchStatus.stopReason}, NonMatches: ${nonMatches ? nonMatches.length : 'not returned'}, ConversionFailures: ${conversionFailures ? conversionFailures.length : 'not returned'}`);
+
+    expect(matchStatus.status).to.eql(expectedMatchStatus);
+    expect(matchStatus.stopReason).to.eql(expectedStopReason);
+    expect(candidateCount).to.eql(expectedCandidateCount);
 
-    expect(formatResults()).to.eql(expectedMatches);
+    const formattedMatchResult = formatRecordResults(matches);
+    expect(formattedMatchResult).to.eql(expectedMatches);
+
+    const formattedNonMatchResult = formatRecordResults(nonMatches);
+    expect(formattedNonMatchResult).to.eql(expectedNonMatches);
+
+    // eslint-disable-next-line functional/no-conditional-statements
+    if (expectedFailures) {
+      expect(conversionFailures).to.eql(expectedFailures);
+    }
 
     function formatOptions() {
       const contextFeatures = matchDetection.features[options.detection.strategy.type];
@@ -62,26 +79,37 @@ describe('INDEX', () => {
       return {
         ...options,
         search: {
-          ...options.search,
-          maxRecordsPerRequest: 1
+          ...options.search
         },
         detection: {
           ...options.detect,
           strategy: options.detection.strategy.features.map(v => contextFeatures[v]())
-        },
-        maxMatches: 2,
-        maxCandidates: 1
+        }
       };
     }
 
-    function formatResults() {
-      return matches.map(({candidate, probability}) => ({
-        probability,
-        candidate: {
-          id: candidate.id,
-          record: candidate.record.toObject()
-        }
-      }));
+    function formatRecordResults(matches) {
+      if (matches) {
+        debugData(JSON.stringify(matches));
+        return matches.map((match) => ({
+          ...match,
+          candidate: formatCandidate(match.candidate)
+        }));
+      }
+      return [];
     }
+
+    // Format candidate to remove validationOptions from record
+    function formatCandidate({id, record}) {
+      const newId = id;
+      const newRecord = record;
+      return {
+        id: newId,
+        record: newRecord.toObject()
+      };
+    }
+
+
   }
+
 });

package/src/match-detection/features/bib/all-source-ids.js

@@ -1,4 +1,3 @@
-/* eslint-disable max-statements */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -56,8 +55,9 @@ Records with matching local ids for matching local db are a very good match.
 
 export default () => ({
   name: 'All source IDs',
-  extract: record => {
-    debug(`Creating match detection features for all local source id's`);
+  extract: ({record, recordExternal}) => {
+    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
+    debug(`Creating match detection features for all local source id's for ${label}`);
 
     const fSids = record.get('SID');
     debugData(`SID-fields (${fSids.length}): ${JSON.stringify(fSids)}`);

package/src/match-detection/features/bib/authors.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -27,25 +27,32 @@
 */
 
 import {LevenshteinDistance as leven} from 'natural';
+import {testStringOrNumber} from '../../../matching-utils';
+
+// We should extract also organisational authors (110/710)
 
 export default ({nameTreshold = 10} = {}) => ({
   name: 'Authors',
-  extract: record => record.get(/^(?<def>100|700)$/u)
-    .map(({subfields}) => {
-      return subfields
-        .filter(({code, value}) => code && value)
-        .filter(({code}) => ['a', '0'].includes(code))
-        .map(toObj)
-        .reduce((acc, v) => ({...acc, ...v}), {});
-
-      function toObj({code, value}) {
-        if (code === 'a') {
-          return {name: value.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()};
-        }
+  extract: ({record}) => {
+    //const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
+    const authors = record.get(/^(?<def>100|700)$/u)
+      .map(({subfields}) => {
+        return subfields
+          .filter(({code, value}) => code && value)
+          .filter(({code}) => ['a', '0'].includes(code))
+          .map(toObj)
+          .reduce((acc, v) => ({...acc, ...v}), {});
+
+        function toObj({code, value}) {
+          if (code === 'a') {
+            return {name: testStringOrNumber(value) ? String(value).replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase() : ''};
+          }
 
-        return {id: value};
-      }
-    }),
+          return {id: value};
+        }
+      });
+    return authors;
+  },
   compare: (a, b) => {
     const maxAuthors = a.length > b.length ? a.length : b.length;
     const matchingIds = findMatchingIds();

package/src/match-detection/features/bib/bibliographic-level.js

@@ -28,6 +28,6 @@
 
 export default () => ({
   name: 'Bibliographic level',
-  extract: r => r.leader[7] ? [r.leader[7]] : [],
+  extract: ({record}) => record.leader[7] ? [record.leader[7]] : [],
   compare: (a, b) => a[0] === b[0] ? 0.1 : -0.2
 });

package/src/match-detection/features/bib/host-component.js

@@ -28,6 +28,6 @@
 
 export default () => ({
   name: 'Host/Component record',
-  extract: r => r.get(/^773$/u).length > 0 ? ['component'] : ['host'],
+  extract: ({record}) => record.get(/^773$/u).length > 0 ? ['component'] : ['host'],
   compare: (a, b) => a[0] === b[0] ? 0.0 : -1.0
 });

package/src/match-detection/features/bib/index.js

@@ -31,10 +31,14 @@ export {default as isbn} from './isbn';
 export {default as issn} from './issn';
 export {default as otherStandardIdentifier} from './other-standard-identifier';
 export {default as title} from './title';
+export {default as titleVersionOriginal} from './title-version-original';
 export {default as authors} from './authors';
 export {default as recordType} from './record-type';
 export {default as publicationTime} from './publication-time';
+export {default as publicationTimeAllowConsYears} from './publication-time-allow-cons-years';
+export {default as publicationTimeAllowConsYearsMulti} from './publication-time-allow-cons-years-multi';
 export {default as language} from './language';
 export {default as bibliographicLevel} from './bibliographic-level';
 export {default as melindaId} from './melinda-id';
 export {default as allSourceIds} from './all-source-ids';
+export {default as mediaType} from './media-type';

package/src/match-detection/features/bib/index.spec.js

@@ -31,6 +31,11 @@ import {READERS} from '@natlibfi/fixura';
 import {expect} from 'chai';
 import {MarcRecord} from '@natlibfi/marc-record';
 import * as features from '.';
+import createDebugLogger from 'debug';
+
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib:test');
+const debugData = debug.extend('data');
 
 
 describe('match-detection/features/bib/', () => {
@@ -40,19 +45,22 @@ describe('match-detection/features/bib/', () => {
     fixura: {
       reader: READERS.JSON
     },
-    // eslint-disable-next-line max-statements
+
     callback: ({enabled = true, feature, options, type, ...expectations}) => {
 
       if (!enabled) {
         return;
       }
 
+      debug(`Testing: ${feature} ${type}`);
+
       if (type === 'extract') {
         const {expectedFeatures, inputRecord} = expectations;
         const record = new MarcRecord(inputRecord, {subfieldValues: false});
+        debugData(`Record: ${record}`);
         const {extract} = features[feature](options);
 
-        expect(extract(record)).to.eql(expectedFeatures);
+        expect(extract({record})).to.eql(expectedFeatures);
         return;
       }