@natlibfi/melinda-record-matching 4.3.5 → 5.0.0-alpha.1

package/src/candidate-search/query-list/component.js

@@ -0,0 +1,127 @@
+import createDebugLogger from 'debug';
+import {promisify} from 'util';
+import {toQueries} from '../candidate-search-utils.js';
+import {getSubfieldValues, testStringOrNumber, toMelindaIds} from '../../matching-utils.js';
+
+const setTimeoutPromise = promisify(setTimeout); // eslint-disable-line
+
+export function hostIdMelinda(record) {
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:hostIdMelinda');
+  const debugData = debug.extend('data'); // eslint-disable-line
+  debug(`Creating query for the Melinda Id host`);
+
+  const ids = hostId(record);
+  if (ids.length > 0) {
+    return toQueries(ids, 'melinda.partsofhost');
+  }
+
+  debug(`No valid Melinda Id host found`);
+  return [];
+
+  function hostId(record) {
+    // Multi 773 handling
+    const f773s = record.get(/^773$/u)
+      .filter(f773 => f773.subfields.some(sub => sub.code === 'w' && (/\(FI-MELINDA\).*/ui).test(sub.value)));
+
+    if (f773s.length === 0) {
+      return false;
+    }
+
+    // Multi $w handling
+    // $w (prefix)<id> handling
+    // $w <id> & $w (prefix)<id> Match
+    const melindaIds = f773s.map(f773 => toMelindaIds(f773, ['w'])).flat()
+      .filter(value => testStringOrNumber(value)) // drop invalid values
+      .filter((value, index, array) => array.indexOf(value) === index); // unique values;
+    return melindaIds;
+  }
+}
+
+export async function hostIdOtherSource(record, client) {
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:hostIdOtherSource');
+  const debugData = debug.extend('data'); // eslint-disable-line
+  debug(`Creating query for the Other source Id host`);
+
+  const ids = getHostIds(record);
+  const otherSources = getOtherSources(record);
+  if (ids.length > 0 && otherSources.length > 0) {
+    const otherSourceIds = ids.map(id => otherSources.map(source => `${id}${source}`)).flat();
+    const melindaHostQueries = await handleSruCalls(otherSourceIds);
+    debug(JSON.stringify(melindaHostQueries));
+
+    return melindaHostQueries;
+  }
+
+  debug(`No valid other source hosts found`);
+  return [];
+
+
+  function getHostIds(record) {
+    const f773s = record.get(/^773$/u)
+      .filter(f773 => f773.subfields.some(sub => sub.code === 'w' && !(/\(FI-MELINDA\).*/ui).test(sub.value)));
+    if (f773s.length === 0) {
+      return false;
+    }
+
+    // Multi 773 handling
+    const subfieldWs = f773s
+      .map(f773 => {
+        debugData(`f773: ${JSON.stringify(f773)}`);
+        return getSubfieldValues(f773, 'w').flat();
+      }).flat();
+
+    // Multi $w handling
+    // $w (prefix)<id> handling
+    // $w <id> & $w (prefix)<id> Match
+    const ids = subfieldWs.map(value => `${value}`.replace(/\(FI-.*\)/ui, '')) // remove prefixes
+      .filter(value => testStringOrNumber(value)) // drop invalid values
+      .filter((value, index, array) => array.indexOf(value) === index); // unique values
+
+    return ids;
+  }
+
+  function getOtherSources(record) {
+    const fSids = record.get('SID');
+    return fSids.map(field => getSubfieldValues(field, 'b'));
+  }
+
+  async function handleSruCalls(otherSourceIds, ids = []) {
+    const [otherSourceId, ...rest] = otherSourceIds;
+
+    if (otherSourceId === undefined) {
+      debug(`host ids: ${ids}`);
+      const validIds = ids.filter(id => id);
+      return toQueries(validIds, 'melinda.partsofhost');
+    }
+
+    const otherSourceHostQuery = await toQueries([otherSourceId], 'melinda.sourceid');
+    const id = await new Promise((resolve, reject) => {
+      debug(`Searching for hosts with query: ${otherSourceHostQuery}`);
+      let recordId;
+
+      client.searchRetrieve(otherSourceHostQuery)
+        .on('error', err => {
+          debug(`SRU error for query: ${otherSourceHostQuery}: ${err}`);
+          reject(err);
+        })
+        .on('end', async () => {
+          try {
+            debug(`Searching for hosts: done`);
+            await setTimeoutPromise(10);
+            resolve(recordId);
+          } catch (err) {
+            debug(`Error caught on END`);
+            reject(err);
+          }
+        })
+        .on('record', record => {
+          const [field] = record.get(/^001$/u);
+          debug(field);
+          recordId = field.value ? field.value : '';
+        });
+    });
+
+    return handleSruCalls(rest, [...ids, id]);
+  }
+}
+

package/src/candidate-search/query-list/index.js

@@ -1,6 +1,5 @@
-
-
-import * as bib from './bib';
+import * as bib from './bib.js';
+import * as component from './component.js';
 import createDebugLogger from 'debug';
 
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:index');
@@ -17,18 +16,33 @@ export const searchTypes = {
     melindaId: 'bibMelindaIds',
     sourceIds: 'bibSourceIds'
     //DEVELOP: bibContent: 'bibContent'
+  },
+  component: {
+    hostIdMelinda: 'hostIdMelinda', // 773 $w (FI-MELINDA)
+    hostIdOtherSource: 'hostIdOtherSource', // 773 $w !(FI-MELINDA)
+    hostIsbn: 'hostIsbn' // 773 $z
   }
 };
 
-export default (record, searchSpec) => {
-  const extractors = {...bib};
+export default async (record, searchSpec, client) => {
+  const extractors = {...bib, ...component};
   debugData(`extractors: ${JSON.stringify(extractors)}`);
   debugData(`searchSpec: ${JSON.stringify(searchSpec)}`);
 
-  return searchSpec
-    .map(generateQueryExtractor)
-    .map(cb => cb(record))
-    .flat();
+  const qExtractors = searchSpec.map(generateQueryExtractor);
+  const results = await handleQextractors(qExtractors);
+  return results;
+
+  async function handleQextractors(qExtractors, results = []) {
+    const [qExtractor, ...rest] = qExtractors;
+
+    if (qExtractor === undefined) {
+      return results.flat();
+    }
+
+    const result = await qExtractor(record, client);
+    return handleQextractors(rest, [...results, result]);
+  }
 
   function generateQueryExtractor(type) {
     if (extractors[type]) {

package/src/cli.js

@@ -0,0 +1,177 @@
+import fs from 'fs';
+import yargs from 'yargs';
+import createMatchOperator, {candidateSearch, matchDetection} from './index.js';
+import createDebugLogger from 'debug';
+import {MarcRecord} from '@natlibfi/marc-record';
+
+cli();
+
+async function cli() {
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:cli');
+  const args = yargs(process.argv.slice(2))
+    .scriptName('melinda-record-matching-js')
+    .epilog('Copyright (C) 2022-2023 University Of Helsinki (The National Library Of Finland)')
+    .usage('$0 <file> [options] and env variable info in README')
+    .usage('Installed globally: $0 <file> [options] and env variable info in README')
+    .usage('Not installed: npx $0 <file> [options] and env variable info in README')
+    .usage('Build from source: node dist/index.js <file> [options] and env variable info in README')
+    .showHelpOnFail(true)
+    .example([['$ node /dist/cli.js record.json']])
+    .env('MELINDA_RECORD_MATCH')
+    .version()
+    .positional('file', {type: 'string', describe: 'Json file of records to match'})
+    .options({
+      t: {type: 'string', default: 'IDS', alias: 'searchType', describe: 'IDS, STANDARD_IDS, COMPONENT, CONTENT or CONTENTALT'},
+      m: {type: 'number', default: 1, alias: 'maxMatches', describe: ''},
+      c: {type: 'number', default: 1000, alias: 'maxCandidates', describe: ''},
+      s: {type: 'boolean', default: false, alias: 'returnStrategy', describe: ''},
+      q: {type: 'boolean', default: false, alias: 'returnQuery', describe: ''},
+      n: {type: 'boolean', default: false, alias: 'returnNonMatches', describe: ''}
+    })
+    .check((args) => {
+      const [file] = args._;
+      if (file === undefined) {
+        throw new Error('No file argument given');
+      }
+
+      if (!fs.existsSync(file)) {
+        throw new Error(`File ${file} does not exist`);
+      }
+
+      if (args.sruUrl === undefined) {
+        throw new Error('Setup sru url');
+      }
+
+      if (!['IDS', 'STANDARD_IDS', 'COMPONENT', 'CONTENT', 'CONTENTALT'].includes(args.searchType)) {
+        throw new Error('Invalid search type');
+      }
+
+      return true;
+    })
+    .parseSync();
+
+  const [file] = args._;
+  const {searchType} = args;
+  debug(JSON.stringify(args));
+
+  const detection = {
+    treshold: 0.9,
+    strategy: generateStrategy(searchType)
+  };
+
+  const search = {
+    url: args.sruUrl, searchSpec: generateSearchSpec(searchType)
+  };
+
+  const matchOperator = await createMatchOperator({detection, search, ...args});
+
+  const fileRaw = fs.readFileSync(file, 'utf8');
+  const record = new MarcRecord(JSON.parse(fileRaw), {subfieldValues: false});
+
+  const result = await matchOperator({record});
+  debug(JSON.stringify(result));
+
+
+  function generateStrategy(searchType) {
+    if (['IDS'].includes(searchType)) {
+      return [
+        matchDetection.features.bib.melindaId(),
+        matchDetection.features.bib.allSourceIds()
+      ];
+    }
+
+    // We could have differing strategy for STANDARD_IDS
+    // Let's not run title in strategy when we found the candidates through standard_ids search
+
+    if (['STANDARD_IDS'].includes(searchType)) {
+      return [
+        matchDetection.features.bib.hostComponent(),
+        matchDetection.features.bib.isbn(),
+        matchDetection.features.bib.issn(),
+        matchDetection.features.bib.otherStandardIdentifier(),
+        // Let's not use the same title matchDetection here
+        //matchDetection.features.bib.title(),
+        matchDetection.features.bib.authors(),
+        // We probably should have some leeway here for notated music as BK etc.
+        matchDetection.features.bib.recordType(),
+        // Use publicationTimeAllowConsYearsMulti to
+        //  - ignore one year differences in publicationTime
+        //  - extract publicationTimes from f008, f26x and reprint notes in f500
+        //  - do not substract points for mismatching (normal) publicationTime, if there's a match between
+        //       normal publicationTime and a reprintPublication time
+        matchDetection.features.bib.publicationTimeAllowConsYearsMulti(),
+        matchDetection.features.bib.language(),
+        matchDetection.features.bib.bibliographicLevel()
+      ];
+    }
+
+    if (['COMPONENT'].includes(searchType)) {
+      return [
+        matchDetection.features.bib.hostComponent(),
+        matchDetection.features.bib.otherStandardIdentifier(),
+        matchDetection.features.bib.recordType(),
+        matchDetection.features.bib.title(),
+        matchDetection.features.bib.language(),
+        matchDetection.features.bib.authors(),
+        matchDetection.features.bib.bibliographicLevel()
+      ];
+    }
+
+    if (['CONTENT', 'CONTENTALT'].includes(searchType)) {
+      return [
+        matchDetection.features.bib.hostComponent(),
+        matchDetection.features.bib.isbn(),
+        matchDetection.features.bib.issn(),
+        matchDetection.features.bib.otherStandardIdentifier(),
+        matchDetection.features.bib.title(),
+        matchDetection.features.bib.authors(),
+        matchDetection.features.bib.recordType(),
+        matchDetection.features.bib.publicationTime(),
+        matchDetection.features.bib.language(),
+        matchDetection.features.bib.bibliographicLevel()
+      ];
+    }
+
+    throw new Error('Unsupported match validation package');
+  }
+
+
+  function generateSearchSpec(searchType) {
+    if (['IDS'].includes(searchType)) {
+      return [
+        candidateSearch.searchTypes.bib.melindaId,
+        candidateSearch.searchTypes.bib.sourceIds
+      ];
+    }
+
+    if (['STANDARD_IDS'].includes(searchType)) {
+      return [candidateSearch.searchTypes.bib.standardIdentifiers];
+    }
+
+    if (['COMPONENT'].includes(searchType)) {
+      return [
+        //candidateSearch.searchTypes.bib.sourceIds,
+        candidateSearch.searchTypes.component.hostIdMelinda,
+        candidateSearch.searchTypes.component.hostIdOtherSource
+      ];
+    }
+
+    if (['CONTENT'].includes(searchType)) {
+      return [
+        candidateSearch.searchTypes.bib.hostComponents,
+        //candidateSearch.searchTypes.bib.titleAuthor,
+        candidateSearch.searchTypes.bib.title
+      ];
+    }
+    if (['CONTENTALT'].includes(searchType)) {
+      return [
+        candidateSearch.searchTypes.bib.hostComponents,
+        // titleAuthorYearAlternates searches for matchCandidates
+        // with alternate queries, starting from more tight searches
+        candidateSearch.searchTypes.bib.titleAuthorYearAlternates
+      ];
+    }
+
+    throw new Error('Unsupported match validation package');
+  }
+}

package/src/index.js

@@ -1,11 +1,11 @@
 import createDebugLogger from 'debug';
-import createSearchInterface, * as candidateSearch from './candidate-search';
-import createDetectionInterface, * as matchDetection from './match-detection';
+import createSearchInterface, * as candidateSearch from './candidate-search/index.js';
+import createDetectionInterface, * as matchDetection from './match-detection/index.js';
 //import inspect from 'util';
 
 export {candidateSearch, matchDetection};
 
-export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false, returnFailures = false}) => {
+export default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false}) => {
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:index');
   const debugData = debug.extend('data');
 
@@ -16,7 +16,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
   debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
   debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);
   debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);
-  debugData(`ReturnFailures: ${JSON.stringify(returnFailures)}`);
 
 
   const detect = createDetectionInterface(detectionOptions, returnStrategy);
@@ -41,17 +40,14 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // state.queryCounter : sequence for current query
     // state.maxedQueries : queries that resulted in more than serverMaxResults hits
 
-    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, conversionFailures = [], matchErrors = []}) {
+    async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0, matchErrors = []}) {
       debugData(`Starting next matcher iteration.`);
-      const {records, failures, ...state} = await search(initialState);
+      const {records, ...state} = await search(initialState);
 
-      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, conversionFailures: ${conversionFailures}, matchErrors: ${matchErrors.length}`);
+      debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}, nonMatchCount: ${nonMatchCount}, matchErrors: ${matchErrors.length}`);
       const recordSetSize = records.length;
-      const failureSetSize = failures.length;
-      const newCandidateCount = candidateCount + recordSetSize + failureSetSize;
+      const newCandidateCount = candidateCount + recordSetSize;
 
-      const newConversionFailures = conversionFailures.concat(failures);
-      debugData(`Failures: ${failures.length}, ConversionFailures: ${conversionFailures.length}, NewConversionFailures: ${newConversionFailures.length}`);
 
       if (recordSetSize > 0) {
         return handleRecordSet();
@@ -59,11 +55,11 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
 
       if (state.queriesLeft > 0) {
         debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);
-        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});
+        return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, nonMatchCount, duplicateCount, matchErrors});
       }
 
       debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);
-      return returnResult({matches, state, stopReason: '', nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, conversionFailures: newConversionFailures, matchErrors});
+      return returnResult({matches, state, stopReason: '', nonMatches, nonMatchCount, candidateCount: newCandidateCount, duplicateCount, matchErrors});
 
       function handleRecordSet() {
         debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);
@@ -75,19 +71,18 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         const {newMatches, newNonMatches, newMatchErrors} = handleMatchResult(matchResult, matches, nonMatches, matchErrors);
 
         if (maxMatchesFound({matches: newMatches, maxMatches})) {
-          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
+          return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors});
         }
 
         if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
-          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
+          return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors});
         }
 
-        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, conversionFailures: newConversionFailures, matchErrors: newMatchErrors});
+        return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount, matchErrors: newMatchErrors});
       }
 
       function handleMatchResult(matchResult, matches, nonMatches, matchErrors) {
         debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);
-        // eslint-disable-next-line functional/no-conditional-statements
         if (returnNonMatches) {
           debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);
         }
@@ -97,7 +92,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         const newMatchErrors = matchErrors.concat(matchResult.matchErrors);
 
         debugData(`- Total amount of matches: ${newMatches.length}`);
-        // eslint-disable-next-line functional/no-conditional-statements
         if (returnNonMatches) {
           debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);
         }
@@ -141,27 +135,24 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
     // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
 
-    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, conversionFailures, matchErrors}) {
-      const conversionFailureCount = conversionFailures.length;
+    function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount, matchErrors}) {
       const matchErrorCount = matchErrors.length;
-      checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount});
-      const matchStatus = getMatchState(state, stopReason, conversionFailureCount, matchErrorCount);
+      checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, matchErrorCount});
+      const matchStatus = getMatchState(state, stopReason, matchErrorCount);
       // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
       const matchesResult = returnNonMatches ? {matches, matchStatus, nonMatches, candidateCount} : {matches, matchStatus, candidateCount};
-      const failures = [...conversionFailures, ...matchErrors];
-      const result = returnFailures ? {...matchesResult, conversionFailures: failures} : matchesResult;
-      debugData(`ReturnFailures ${returnFailures}`);
+      const result = matchesResult;
       debugData(`${JSON.stringify(result)}`);
       return result;
 
       // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
 
-      function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, conversionFailureCount, matchErrorCount}) {
+      function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount, matchErrorCount}) {
         const matchCount = matches.length;
         debugData(`Return nonMatches: ${returnNonMatches}`);
         const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
         const totalHandled = matchCount + chosenNonMatchCount + duplicateCount;
-        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, conversionFailureCount: ${conversionFailureCount}, matchErrorCount: ${matchErrorCount}`);
+        debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}, matchErrorCount: ${matchErrorCount}`);
         debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
         if (totalHandled !== candidateCount) {
           debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);
@@ -171,9 +162,8 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
       }
 
       // eslint-disable-next-line max-statements
-      function getMatchState(state, stopReason, conversionFailuresCount, matchErrorCount) {
+      function getMatchState(state, stopReason, matchErrorCount) {
         debugData(`${JSON.stringify(state)}`);
-        debug(`We had ${conversionFailuresCount} retrieved candidates that could not be converted.`);
         debug(`We had ${matchErrorCount} retrieved candidates that errored in matchDetection.`);
         debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
 
@@ -183,16 +173,14 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
         const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
         debugData(`nonRetrieved: ${nonRetrieved}`);
 
-        // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries','conversionFailures', empty string/undefined), reason for stopping retrieving or handling the candidate records
+        // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries', empty string/undefined), reason for stopping retrieving or handling the candidate records
         // 'maxMatches' and 'maxCandidates' are in stopReason, 'maxedQueries', 'conversionFailures' and 'matchErrors' are created here
 
-        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || conversionFailureCount > 0 || matchErrorCount > 0) {
+        if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0 || matchErrorCount > 0) {
           const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : undefined;
-          const conversionFailuresStopReason = conversionFailureCount > 0 ? 'conversionFailures' : undefined;
           const matchErrorsStopReason = matchErrorCount > 0 ? 'matchErrors' : undefined;
-          const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || conversionFailuresStopReason || matchErrorsStopReason : stopReason;
+          const newStopReason = stopReason === '' || stopReason === undefined ? maxedQueriesStopReason || matchErrorsStopReason : stopReason;
           debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);
-          debugData(`ConversionFailureStopReason <${conversionFailuresStopReason}>`);
           debugData(`MatchErrorsStopReason <${matchErrorsStopReason}>`);
           debugData(`NewStopReason: <${newStopReason}>`);
           debug(`Match status: false`);
@@ -240,8 +228,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
       */
 
       if (candidate) {
-
-        // eslint-disable-next-line functional/no-conditional-statements
         if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {
           const {record: candidateRecord, id: candidateId} = candidate;
           const recordBExternal = {id: candidate.id, recordSource: 'databaseRecord', label: `db-${candidate.id}`};
@@ -304,7 +290,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
 
         debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);
 
-        // eslint-disable-next-line functional/no-conditional-statements
         if (returnNonMatches) {
           debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);
         }

package/src/{index.spec.js → index.test.js}

@@ -1,9 +1,10 @@
-import {expect} from 'chai';
+import assert from 'node:assert';
+import {describe} from 'node:test';
+import createDebugLogger from 'debug';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen-http-client';
 import {MarcRecord} from '@natlibfi/marc-record';
-import createMatchInterface, {matchDetection} from '.';
-import createDebugLogger from 'debug';
+import createMatchInterface, {matchDetection} from './index.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:index:test');
 const debugData = debug.extend('data');
@@ -11,7 +12,7 @@ const debugData = debug.extend('data');
 describe('INDEX', () => {
   generateTests({
     callback,
-    path: [__dirname, '..', 'test-fixtures', 'index'],
+    path: [import.meta.dirname, '..', 'test-fixtures', 'index'],
     recurse: false,
     fixura: {
       reader: READERS.JSON
@@ -30,19 +31,18 @@ describe('INDEX', () => {
     const {matches, matchStatus, nonMatches, conversionFailures, candidateCount} = await match({record});
     debugData(`Matches: ${matches.length}, Status: ${matchStatus.status}/${matchStatus.stopReason}, NonMatches: ${nonMatches ? nonMatches.length : 'not returned'}, ConversionFailures: ${conversionFailures ? conversionFailures.length : 'not returned'}`);
 
-    expect(matchStatus.status).to.eql(expectedMatchStatus);
-    expect(matchStatus.stopReason).to.eql(expectedStopReason);
-    expect(candidateCount).to.eql(expectedCandidateCount);
+    assert.equal(matchStatus.status, expectedMatchStatus);
+    assert.equal(matchStatus.stopReason, expectedStopReason);
+    assert.equal(candidateCount, expectedCandidateCount);
 
     const formattedMatchResult = formatRecordResults(matches);
-    expect(formattedMatchResult).to.eql(expectedMatches);
+    assert.deepStrictEqual(formattedMatchResult, expectedMatches);
 
     const formattedNonMatchResult = formatRecordResults(nonMatches);
-    expect(formattedNonMatchResult).to.eql(expectedNonMatches);
+    assert.deepStrictEqual(formattedNonMatchResult, expectedNonMatches);
 
-    // eslint-disable-next-line functional/no-conditional-statements
     if (expectedFailures) {
-      expect(conversionFailures).to.eql(expectedFailures);
+      assert.deepStrictEqual(conversionFailures, expectedFailures);
     }
 
     function formatOptions() {
@@ -80,8 +80,5 @@ describe('INDEX', () => {
         record: newRecord.toObject()
       };
     }
-
-
   }
-
 });

package/src/match-detection/features/bib/all-source-ids.js

@@ -1,6 +1,6 @@
 
 import createDebugLogger from 'debug';
-import {validateSidFieldSubfieldCounts, getSubfieldValues} from '../../../matching-utils';
+import {validateSidFieldSubfieldCounts, getSubfieldValues} from '../../../matching-utils.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:all-source-ids');
 const debugData = debug.extend('data');
@@ -71,7 +71,7 @@ export default () => ({
     debugData(`Comparing ${JSON.stringify(a)} and ${JSON.stringify(b)}`);
 
     const matches = a.filter(sidA => b.some(sidB => sidA.sourceDb === sidB.sourceDb &&
-            sidA.sourceId === sidB.sourceId));
+      sidA.sourceId === sidB.sourceId));
     debugData(`Matches (${matches.length}): ${JSON.stringify(matches)}`);
 
     const mismatches = a.filter(sidA => b.some(sidB => sidA.sourceDb === sidB.sourceDb &&

package/src/match-detection/features/bib/authors.js

@@ -1,7 +1,8 @@
 
 
-import {LevenshteinDistance as leven} from 'natural';
-import {testStringOrNumber} from '../../../matching-utils';
+import naturalPkg from 'natural';
+const {LevenshteinDistance: leven} = naturalPkg;
+import {testStringOrNumber} from '../../../matching-utils.js';
 
 // We should extract also organisational authors (110/710)
 

package/src/match-detection/features/bib/index.js

@@ -1,17 +1,17 @@
 
-export {default as hostComponent} from './host-component';
-export {default as isbn} from './isbn';
-export {default as issn} from './issn';
-export {default as otherStandardIdentifier} from './other-standard-identifier';
-export {default as title} from './title';
-export {default as titleVersionOriginal} from './title-version-original';
-export {default as authors} from './authors';
-export {default as recordType} from './record-type';
-export {default as publicationTime} from './publication-time';
-export {default as publicationTimeAllowConsYears} from './publication-time-allow-cons-years';
-export {default as publicationTimeAllowConsYearsMulti} from './publication-time-allow-cons-years-multi';
-export {default as language} from './language';
-export {default as bibliographicLevel} from './bibliographic-level';
-export {default as melindaId} from './melinda-id';
-export {default as allSourceIds} from './all-source-ids';
-export {default as mediaType} from './media-type';
+export {default as hostComponent} from './host-component.js';
+export {default as isbn} from './isbn.js';
+export {default as issn} from './issn.js';
+export {default as otherStandardIdentifier} from './other-standard-identifier.js';
+export {default as title} from './title.js';
+export {default as titleVersionOriginal} from './title-version-original.js';
+export {default as authors} from './authors.js';
+export {default as recordType} from './record-type.js';
+export {default as publicationTime} from './publication-time.js';
+export {default as publicationTimeAllowConsYears} from './publication-time-allow-cons-years.js';
+export {default as publicationTimeAllowConsYearsMulti} from './publication-time-allow-cons-years-multi.js';
+export {default as language} from './language.js';
+export {default as bibliographicLevel} from './bibliographic-level.js';
+export {default as melindaId} from './melinda-id.js';
+export {default as allSourceIds} from './all-source-ids.js';
+export {default as mediaType} from './media-type.js';