@natlibfi/melinda-record-matching 3.0.0-alpha.1 → 3.0.1-alpha.1

package/src/candidate-search/index.spec.js

@@ -46,7 +46,6 @@ describe('candidate-search', () => {
     }
   });
 
-  // eslint-disable-next-line max-statements
   async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError = false, expectedSearchError = false, enabled = true}) {
     const url = 'http://foo.bar';
 
@@ -77,7 +76,6 @@ describe('candidate-search', () => {
       };
     }
 
-    // eslint-disable-next-line max-statements
     async function iterate({searchOptions, expectedSearchError, expectedErrorStatus, count = 1}) {
       const expectedResults = getFixture(`expectedResults${count}.json`);
 
@@ -108,7 +106,7 @@ describe('candidate-search', () => {
       }
 
       function formatResults(results) {
-        debug(results); //eslint-disable-line
+        debug(results);
         return {
           ...results,
           records: results.records.map(({record, id}) => ({id, record: record.toObject()}))

package/src/candidate-search/query-list/bib.js

@@ -217,7 +217,6 @@ export function bibTitle(record) {
   }
 }
 
-// eslint-disable-next-line max-statements
 export function bibStandardIdentifiers(record) {
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibStandardIdentifiers');

package/src/index.js

@@ -235,7 +235,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // - if creating the featureSet for the incoming record fails we have an unprocessable entity
     // - if creating the featureSet for a candidate record fails we could skip that candidate - but list the case as a detectionFailure, same as conversionFailures
 
-    // eslint-disable-next-line max-statements
     function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0, recordMatchErrors = []}) {
 
       // recordSetSize : total amount of records in the current record set

package/src/match-detection/features/bib/all-source-ids.js

@@ -1,4 +1,3 @@
-/* eslint-disable max-statements */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.

package/src/match-detection/features/bib/index.spec.js

@@ -40,7 +40,7 @@ describe('match-detection/features/bib/', () => {
     fixura: {
       reader: READERS.JSON
     },
-    // eslint-disable-next-line max-statements
+
     callback: ({enabled = true, feature, options, type, ...expectations}) => {
 
       if (!enabled) {

package/src/match-detection/features/bib/language.js

@@ -1,4 +1,3 @@
-/* eslint-disable max-statements */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -42,14 +41,14 @@ export default () => ({
     debugData(`${label} 008: ${JSON.stringify(value008)}, 041: ${JSON.stringify(values041)}`);
 
     if (value008 && values041.length > 0) {
-      debugData(`There's both 008 and 041, searching for value in both`);
+      debugData(`${label} There's both 008 and 041, searching for value in both`);
       const correspondingValue = values041.find(v => v === value008);
-      debugData(`Corresponding value: ${correspondingValue}`);
+      debugData(`${label} Corresponding value: ${correspondingValue}`);
       return correspondingValue ? [correspondingValue] : [];
     }
 
     if (!value008 && values041.length < 1) {
-      debugData(`No actual values found`);
+      debugData(`{$label} No actual values found`);
       return [];
     }
 
@@ -57,18 +56,19 @@ export default () => ({
 
     function get008Value() {
       const value = record.get(/^008$/u)?.[0]?.value || undefined;
-      debugData(`008 value: ${value}`);
+      debugData(`${label} 008 value: ${value}`);
 
       if (!value) {
         return undefined;
       }
 
       const code = value.slice(35, 38);
-      debugData(`008 code: ${code}`);
+      debugData(`${label} 008 code: ${code}`);
       return code === '|||' || code === '   ' || code === '^^^' ? undefined : code;
     }
 
     // Main language for the resource: in the first f041 $a or f041 $d
+    // Should we get all $a or $d languages instead of just the first?
     // Uses only f041s that have 2nd ind ' ', which means that the codes used are MARC 21 language codes
 
     function get041Values() {

package/src/match-detection/features/bib/melinda-identifier-factory.js

@@ -62,7 +62,6 @@ export default () => {
 
   }
 
-  // eslint-disable-next-line max-statements
   function compare(a, b) {
 
     if (a.isMelindaRecord && b.isMelindaRecord &&

package/src/match-detection/features/bib/title.js

@@ -26,27 +26,45 @@
 *
 */
 
-import createDebugLogger from 'debug';
+
 import {LevenshteinDistance as leven} from 'natural';
 import {testStringOrNumber} from '../../../matching-utils';
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');
+const debugData = debug.extend('data');
+
 
 export default ({treshold = 10} = {}) => ({
   name: 'Title',
-  extract: ({record}) => {
+  extract: ({record, recordExternal}) => {
+    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
     const title = getTitle();
+    debug(`${label} title: ${title}`);
 
     if (testStringOrNumber(title)) {
-      return [String(title).replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
+      const titleAsNormalizedString = String(title)
+        // decompose unicode diacritics
+        .normalize('NFD')
+        // strip non-letters/numbers
+        // - note: combined with decomposing unicode diactics this normalizes both 'saa' and 'sää' as 'saa'
+        // - we could precompose the finnish letters back to avoid this
+        .replace(/[^\p{Letter}\p{Number}]/gu, '')
+        .toLowerCase();
+      debug(`${label} titleString: ${titleAsNormalizedString}`);
+      return [titleAsNormalizedString];
     }
 
     return [];
 
     function getTitle() {
       const [field] = record.get(/^245$/u);
+      debugData(`${label} titleField: ${JSON.stringify(field)}`);
 
       if (field) {
         return field.subfields
-          .filter(({code}) => ['a', 'b'].includes(code))
+          // get also $n:s and $p:s here
+          .filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))
           .map(({value}) => testStringOrNumber(value) ? String(value) : '')
           .join('');
       }

package/src/match-detection/index.js

@@ -1,4 +1,3 @@
-/* eslint-disable no-console */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -32,7 +31,7 @@ import * as features from './features';
 
 export {features};
 
-export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
+export default ({strategy, treshold = 0.9}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
   const minProbabilityQuantifier = 0.5;
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');
@@ -41,8 +40,9 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA
   debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
   debugData(`Records: A: ${recordA}\nB: ${recordB}`);
   debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);
+  // We could add here labels for records if we didn't get external labels
 
-  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal});
+  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});
 
   debug(`We got an array of records: ${Array.isArray(recordB)}`);
   const recordsB = Array.isArray(recordB) ? recordB : [recordB];
@@ -54,13 +54,13 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA
   // if we got a singular record, we return a singular result
   return Array.isArray(recordB) ? detectionResults : detectionResults[0];
 
-  function actualDetection({featuresA, record, recordExternal, index}) {
+  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {
     const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';
     const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';
 
 
     debug(`Actual detection for record ${index + 1} ${labelB}`);
-    const featuresB = extractFeatures({record, recordExternal});
+    const featuresB = extractFeatures({record, recordExternal, localSettings});
 
     debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);
     debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);
@@ -69,7 +69,7 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA
     const similarityVector = generateSimilarityVector(featurePairs);
 
     if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
-      const probability = calculateprobability(similarityVector);
+      const probability = calculateProbability(similarityVector);
       debug(`probability: ${probability} (Treshold: ${treshold})`);
       return returnResult({match: probability >= treshold, probability});
     }
@@ -97,7 +97,7 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA
     return strategyNames || [];
   }
 
-  function calculateprobability(similarityVector) {
+  function calculateProbability(similarityVector) {
     const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);
     return probability > 1.0 ? 1.0 : probability;
   }

package/src/matching-utils.js

@@ -31,7 +31,6 @@ import createDebugLogger from 'debug';
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');
 const debugData = debug.extend('data');
 
-// eslint-disable-next-line max-statements
 export function getMelindaIdsF035(record) {
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');