@natlibfi/melinda-record-matching 2.0.1-alpha.1 → 2.1.0

package/src/index.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -32,65 +32,277 @@ import createDetectionInterface, * as matchDetection from './match-detection';
 
 export {candidateSearch, matchDetection};
 
-export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25}) => {
+export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false}) => {
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:index');
-  const detect = createDetectionInterface(detectionOptions);
+  const debugData = debug.extend('data');
+
+  debugData(`DetectionOptions: ${JSON.stringify(detectionOptions)}`);
+  debugData(`SearchOptions: ${JSON.stringify(searchOptions)}`);
+  debugData(`MaxMatches: ${JSON.stringify(maxMatches)}`);
+  debugData(`MaxCandidates: ${JSON.stringify(maxCandidates)}`);
+  debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
+  debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);
+  debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);
+
+  const detect = createDetectionInterface(detectionOptions, returnStrategy);
 
   return record => {
-    const search = createSearchInterface({...searchOptions, record});
-    return iterate();
+    const search = createSearchInterface({...searchOptions, record, maxCandidates});
+    return iterate({});
+
+    // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set
+    // matches : candidates that have been detected as matches by current matcher job
+    // nonMatches : candidates that have been detected as non-matches by current matcher job (only if returnNonMatches is 'true')
+    // duplicateCount : amount of candidate records that were retrieved from the SRU but not handled further because they were already found in the matches/nonMatches
+
+    // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
+    // state.query : current query (undefined if there was no queries left)
+    // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
+    // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)
+    // state.queriesLeft : amount of queries left
+    // state.queryCounter : sequence for current query
+    // state.maxedQueries : queries that resulted in more than serverMaxResults hits
 
-    // eslint-disable-next-line max-statements
-    async function iterate(initialState = {}, matches = [], candidateCount = 0) {
+    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0}) {
+      debugData(`Starting next matcher iteration.`);
       const {records, ...state} = await search(initialState);
 
-      if (records.length > 0 || state.queriesLeft > 0) {
-        debug(`Checking ${records.length} candidates for matches`);
+      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}`);
+      const recordSetSize = records.length;
+      const newCandidateCount = candidateCount + recordSetSize;
+
+      if (recordSetSize > 0) {
+        return handleRecordSet();
+      }
+
+      if (state.queriesLeft > 0) {
+        debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);
+        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, duplicateCount});
+      }
+
+      debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);
+      return returnResult({matches, state, stopReason: '', nonMatches, candidateCount: newCandidateCount, duplicateCount});
+
+      function handleRecordSet() {
+        debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);
+
+        const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches});
+        const newDuplicateCount = duplicateCount + matchResult.duplicateCount;
+        const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;
+        const {newMatches, newNonMatches} = handleMatchResult(matchResult, matches, nonMatches);
+
+        if (maxMatchesFound({matches: newMatches, maxMatches})) {
+          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount});
+        }
+
+        if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
+          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount});
+        }
 
-        const matchResult = iterateRecords(records);
+        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount});
+      }
 
-        if (matchResult) {
-          const newMatches = matches.concat(matchResult);
+      function handleMatchResult(matchResult, matches, nonMatches) {
+        debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);
+        // eslint-disable-next-line functional/no-conditional-statement
+        if (returnNonMatches) {
+          debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);
+        }
 
-          if (newMatches.length === maxMatches) {
-            return newMatches;
-          }
+        const newMatches = matches.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);
+        const newNonMatches = returnNonMatches ? nonMatches.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];
 
-          return maxCandidatesRetrieved() ? newMatches : iterate(state, newMatches, candidateCount + records.length);
+        debugData(`- Total amount of matches: ${newMatches.length}`);
+        // eslint-disable-next-line functional/no-conditional-statement
+        if (returnNonMatches) {
+          debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);
         }
 
-        return maxCandidatesRetrieved() ? matches : iterate(state, matches, candidateCount + records.length);
+        return {newMatches, newNonMatches};
       }
 
-      debug(`No (more) candidate records to check, matches: ${matches.length}`);
-      return matches;
+      function addQuery(matches) {
+        debugData(`Adding query ${state.query} to matches`);
+        return matches.map((match) => ({...match, matchQuery: state.query}));
+      }
 
-      function maxCandidatesRetrieved() {
-        if (candidateCount + records.length > maxCandidates) {
-          debug(`Stopped searching because maximum number of candidates have been retrieved`);
+      function maxCandidatesRetrieved(candidateCount, maxCandidates) {
+        debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${maxCandidates})`);
+        if (maxCandidates && candidateCount >= maxCandidates) {
+          debug(`Stopped matching because maximum number of candidate records ${candidateCount} / ${maxCandidates} have been retrieved`);
           return true;
         }
       }
+    }
+
+    // matches : array of matching candidate records
+    // nonMatches : array of nonMatching candidate records (if returnNonMatches option is true, otherwise empty array)
+    // - candidate.id
+    // - candidate.record
+    // - probability
+    // - strategy (if returnStrategy option is true)
+    // - treshold (if returnStrategy option is true)
+    // - matchQuery (if returnQuery option is true)
+
+    // we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data
+
+    // matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not
+    // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries',empty string/undefined), reason for stopping retrieving or handling the candidate records
+    // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
+    // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
+
+    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount}) {
+      checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount});
+      const matchStatus = getMatchState(state, stopReason);
+      // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
+      const result = returnNonMatches ? {matches, matchStatus, nonMatches} : {matches, matchStatus};
+      debugData(`${JSON.stringify(result)}`);
+      return result;
+
+      // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
+
+      function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount}) {
+        const matchCount = matches.length;
+        debugData(`Return nonMatches: ${returnNonMatches}`);
+        const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
+        const totalHandled = matchCount + nonMatchCount + duplicateCount;
+        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}`);
+        debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
+        if (totalHandled !== candidateCount) {
+          debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);
+          return;
+        }
+        return;
+      }
+
+      function getMatchState(state, stopReason) {
+        debugData(`${JSON.stringify(state)}`);
+        debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
+
+        debugData(`StopReason: <${stopReason}>`);
+
+        const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;
+        const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
+        debugData(`nonRetrieved: ${nonRetrieved}`);
+
+        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0) {
+          const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : '';
+          const newStopReason = stopReason === '' ? maxedQueriesStopReason : stopReason;
+          debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);
+          debugData(`NewStopReason: <${newStopReason}>`);
+          debug(`Match status: false`);
+          return {status: false, stopReason: newStopReason};
+        }
+
+        debug(`Match status: true`);
+        return {status: true, stopReason};
+      }
+    }
+
+    function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0}) {
+
+      // recordSetSize : total amount of records in the current record set
+      // recordCount : amount of records from the current record set that have been handled
+      // maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped
+      // recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results
+      // recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)
+
+      // records : non-handled records in the current record set
+      // matches : found matches in the current matcher job
+      // recordMatches : found matches in the current record set
+      // recordNonMatches : found nonMatches in the current record set (only if returnNonMatches setting is true)
+
+      const [candidate] = records;
+      const newRecordCount = candidate ? recordCount + 1 : recordCount;
+
+      // The matcher uses same matchDetection strategy for candidates from all candidate-searches -> matchDetection result for the same candidate is always same
+      // Exceptions would happen if the candidate would have been updated in the database between candidate searches
+      // Note that if returnNonMatches is false, matcher won't remember candidates that didn't match, so they will be matched again everytime they are retrieved by
+      // different candidate search queries. Same candidate search query won't have duplicate records.
 
-      function iterateRecords(records) {
-        const [candidate] = records;
+      if (candidate) {
 
-        if (candidate) {
+        if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {
           const {record: candidateRecord, id: candidateId} = candidate;
-          const {match, probability} = detect(record, candidateRecord);
-
-          if (match) {
-            return {
-              probability,
-              candidate: {
-                id: candidateId,
-                record: candidateRecord
-              }
-            };
-          }
-
-          return iterateRecords(records.slice(1));
+          debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);
+          const detectionResult = detect(record, candidateRecord);
+          return handleDetectionResult(detectionResult, candidateId, candidateRecord);
+        }
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount});
+      }
+
+      debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);
+      return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount};
+
+      function handleDetectionResult(detectionResult, candidateId, candidateRecord) {
+        debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);
+
+        if (detectionResult.match || returnNonMatches) {
+          debug(`${detectionResult.match ? `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is a match!` : `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is NOT a match!`}`);
+          debugData(`Strategy: ${JSON.stringify(detectionResult.strategy)}, Treshold: ${JSON.stringify(detectionResult.treshold)}`);
+
+          const matchResult = {
+            probability: detectionResult.probability,
+            candidate: {
+              id: candidateId,
+              record: candidateRecord
+            }
+          };
+          const strategyResult = {
+            strategy: detectionResult.strategy,
+            treshold: detectionResult.treshold
+          };
+          const newMatch = returnStrategy ? {...matchResult, ...strategyResult} : {...matchResult};
+
+          debugData(`${JSON.stringify(newMatch)}`);
+
+          return handleRecordMatch(detectionResult.match, newMatch);
         }
+
+        const newRecordNonMatchCount = recordNonMatchCount + 1;
+        debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);
+
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount});
+      }
+
+      function handleRecordMatch(isMatch, newMatch) {
+        const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;
+        const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);
+
+        debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);
+
+        // eslint-disable-next-line functional/no-conditional-statement
+        if (returnNonMatches) {
+          debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);
+        }
+
+        if (maxMatchesFound({matches: matches.concat(newRecordMatches), maxMatches})) {
+          debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
+          return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount};
+        }
+
+        return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount});
+      }
+
+      function candidateNotInMatches(matches, candidate) {
+        debug(`Checking that record ${candidate.id} is not already included in ${matches.length} matches/nonMatches`);
+        const newCandidateId = candidate.id;
+        debugData(`newCandidateId: ${newCandidateId}`);
+        const result = matches.find(({candidate}) => candidate.id === newCandidateId);
+        debugData(`Result: ${result}`);
+        if (result) {
+          debug(`${candidate.id} was already handled.`);
+          return false;
+        }
+        debug(`${candidate.id} not found in matches/nonMatches`);
+        return true;
+      }
+    }
+
+    function maxMatchesFound({matches, maxMatches}) {
+      if (maxMatches && matches.length >= maxMatches) {
+        debug(`Stopping recordSet iteration: maxMatches (${maxMatches}) for matcher job found.`);
+        return true;
       }
     }
   };

package/src/index.spec.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -31,6 +31,10 @@ import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen-http-client';
 import {MarcRecord} from '@natlibfi/marc-record';
 import createMatchInterface, {matchDetection} from '.';
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:index:test');
+const debugData = debug.extend('data');
 
 describe('INDEX', () => {
   generateTests({
@@ -42,19 +46,31 @@ describe('INDEX', () => {
     }
   });
 
-  async function callback({getFixture, options, enabled = true}) {
+  async function callback({getFixture, options, enabled = true, expectedMatchStatus, expectedStopReason}) {
 
     if (!enabled) {
+      debug(`Disabled test!`);
       return;
     }
 
     const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});
     const expectedMatches = getFixture('expectedMatches.json');
+    const expectedNonMatches = getFixture('expectedNonMatches.json') || [];
+
 
     const match = createMatchInterface(formatOptions());
-    const matches = await match(record);
+    const {matches, matchStatus, nonMatches} = await match(record);
+    debugData(`${matches.length}, ${matchStatus.status}/${matchStatus.stopReason}, ${nonMatches ? nonMatches.length : nonMatches}`);
+
+    const formattedMatchResult = formatRecordResults(matches);
+    expect(formattedMatchResult).to.eql(expectedMatches);
+
+    const formattedNonMatchResult = formatRecordResults(nonMatches);
+    expect(formattedNonMatchResult).to.eql(expectedNonMatches);
+
+    expect(matchStatus.status).to.eql(expectedMatchStatus);
+    expect(matchStatus.stopReason).to.eql(expectedStopReason);
 
-    expect(formatResults()).to.eql(expectedMatches);
 
     function formatOptions() {
       const contextFeatures = matchDetection.features[options.detection.strategy.type];
@@ -62,26 +78,37 @@ describe('INDEX', () => {
       return {
         ...options,
         search: {
-          ...options.search,
-          maxRecordsPerRequest: 1
+          ...options.search
         },
         detection: {
           ...options.detect,
           strategy: options.detection.strategy.features.map(v => contextFeatures[v]())
-        },
-        maxMatches: 2,
-        maxCandidates: 1
+        }
       };
     }
 
-    function formatResults() {
-      return matches.map(({candidate, probability}) => ({
-        probability,
-        candidate: {
-          id: candidate.id,
-          record: candidate.record.toObject()
-        }
-      }));
+    function formatRecordResults(matches) {
+      if (matches) {
+        debugData(JSON.stringify(matches));
+        return matches.map((match) => ({
+          ...match,
+          candidate: formatCandidate(match.candidate)
+        }));
+      }
+      return [];
     }
+
+    // Format candidate to remove validationOptions from record
+    function formatCandidate({id, record}) {
+      const newId = id;
+      const newRecord = record;
+      return {
+        id: newId,
+        record: newRecord.toObject()
+      };
+    }
+
+
   }
+
 });

package/src/match-detection/index.js

@@ -33,11 +33,16 @@ import * as features from './features';
 export {features};
 
 // eslint-disable-next-line max-statements
-export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
+export default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA, recordB) => {
   const minProbabilityQuantifier = 0.5;
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');
   const debugData = debug.extend('data');
+
+  debugData(`Strategy: ${JSON.stringify(strategy)}`);
+  debugData(`Treshold: ${JSON.stringify(treshold)}`);
+  debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
+
   const featuresA = extractFeatures(recordA);
   const featuresB = extractFeatures(recordB);
 
@@ -50,11 +55,26 @@ export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
   if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
     const probability = calculateprobability();
     debug(`probability: ${probability} (Treshold: ${treshold})`);
-    return {match: probability >= treshold, probability};
+    return returnResult({match: probability >= treshold, probability});
   }
 
   debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
-  return {match: false, probability: 0.0};
+  return returnResult({match: false, probability: 0.0});
+
+  function returnResult(result) {
+    if (returnStrategy) {
+      debug(`Returning detection strategy with the result`);
+      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold};
+      debugData(`${JSON.stringify(resultWithStrategy)}`);
+      return resultWithStrategy;
+    }
+    return result;
+  }
+
+  function formatStrategy(strategy) {
+    const strategyNames = strategy.map(element => element.name);
+    return strategyNames || [];
+  }
 
   function calculateprobability() {
     const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);