@natlibfi/melinda-record-matching 2.2.2 → 3.0.1-alpha.1

package/src/match-detection/features/bib/index.spec.js

@@ -40,7 +40,7 @@ describe('match-detection/features/bib/', () => {
     fixura: {
       reader: READERS.JSON
     },
-    // eslint-disable-next-line max-statements
+
     callback: ({enabled = true, feature, options, type, ...expectations}) => {
 
       if (!enabled) {
@@ -52,7 +52,7 @@ describe('match-detection/features/bib/', () => {
         const record = new MarcRecord(inputRecord, {subfieldValues: false});
         const {extract} = features[feature](options);
 
-        expect(extract(record)).to.eql(expectedFeatures);
+        expect(extract({record})).to.eql(expectedFeatures);
         return;
       }
 

package/src/match-detection/features/bib/language.js

@@ -1,4 +1,3 @@
-/* eslint-disable max-statements */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -34,20 +33,22 @@ const debugData = debug.extend('data');
 
 export default () => ({
   name: 'Language',
-  extract: record => {
+  extract: ({record, recordExternal}) => {
+    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
+
     const value008 = get008Value();
     const values041 = get041Values();
-    debugData(`008: ${JSON.stringify(value008)}, 041: ${JSON.stringify(values041)}`);
+    debugData(`${label} 008: ${JSON.stringify(value008)}, 041: ${JSON.stringify(values041)}`);
 
     if (value008 && values041.length > 0) {
-      debugData(`There's both 008 and 041, searching for value in both`);
+      debugData(`${label} There's both 008 and 041, searching for value in both`);
       const correspondingValue = values041.find(v => v === value008);
-      debugData(`Corresponding value: ${correspondingValue}`);
+      debugData(`${label} Corresponding value: ${correspondingValue}`);
       return correspondingValue ? [correspondingValue] : [];
     }
 
     if (!value008 && values041.length < 1) {
-      debugData(`No actual values found`);
+      debugData(`{$label} No actual values found`);
       return [];
     }
 
@@ -55,18 +56,19 @@ export default () => ({
 
     function get008Value() {
       const value = record.get(/^008$/u)?.[0]?.value || undefined;
-      debugData(`008 value: ${value}`);
+      debugData(`${label} 008 value: ${value}`);
 
       if (!value) {
         return undefined;
       }
 
       const code = value.slice(35, 38);
-      debugData(`008 code: ${code}`);
+      debugData(`${label} 008 code: ${code}`);
       return code === '|||' || code === '   ' || code === '^^^' ? undefined : code;
     }
 
     // Main language for the resource: in the first f041 $a or f041 $d
+    // Should we get all $a or $d languages instead of just the first?
     // Uses only f041s that have 2nd ind ' ', which means that the codes used are MARC 21 language codes
 
     function get041Values() {

package/src/match-detection/features/bib/melinda-identifier-factory.js

@@ -43,7 +43,7 @@ export default () => {
 
   return {extract, compare};
 
-  function extract(record) {
+  function extract({record, recordExternal}) {
 
     const isMelindaRecord = record.get('003').some(f003 => f003.value === 'FI-MELINDA');
     const [f001] = record.get('001').map(field => field.value);
@@ -54,7 +54,7 @@ export default () => {
       f001 === undefined &&
       f035MelindaIds.length < 1) {
 
-      debug(`No Melinda-IDs found`);
+      debug(`${recordExternal.label} No Melinda-IDs found`);
       return {};
     }
 
@@ -62,7 +62,6 @@ export default () => {
 
   }
 
-  // eslint-disable-next-line max-statements
   function compare(a, b) {
 
     if (a.isMelindaRecord && b.isMelindaRecord &&

package/src/match-detection/features/bib/publication-time.js

@@ -33,7 +33,7 @@ import {testStringOrNumber} from '../../../matching-utils';
 
 export default () => ({
   name: 'Publication time',
-  extract: record => {
+  extract: ({record}) => {
     const value = record.get(/^008$/u)?.[0]?.value || undefined;
     return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];
   },

package/src/match-detection/features/bib/record-type.js

@@ -30,6 +30,6 @@
 
 export default () => ({
   name: 'Record type',
-  extract: r => r.leader[6] ? [r.leader[6]] : [],
+  extract: ({record}) => record.leader[6] ? [record.leader[6]] : [],
   compare: (a, b) => a[0] === b[0] ? 0.1 : -0.5
 });

package/src/match-detection/features/bib/standard-identifier-factory.js

@@ -35,7 +35,7 @@ const debugData = debug.extend('data');
 export default ({pattern, subfieldCodes}) => {
   return {extract, compare};
 
-  function extract(record) {
+  function extract({record}) {
     const [field] = record.get(pattern);
 
     if (field) {

package/src/match-detection/features/bib/title.js

@@ -26,27 +26,45 @@
 *
 */
 
-import createDebugLogger from 'debug';
+
 import {LevenshteinDistance as leven} from 'natural';
 import {testStringOrNumber} from '../../../matching-utils';
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');
+const debugData = debug.extend('data');
+
 
 export default ({treshold = 10} = {}) => ({
   name: 'Title',
-  extract: record => {
+  extract: ({record, recordExternal}) => {
+    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
     const title = getTitle();
+    debug(`${label} title: ${title}`);
 
     if (testStringOrNumber(title)) {
-      return [String(title).replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
+      const titleAsNormalizedString = String(title)
+        // decompose unicode diacritics
+        .normalize('NFD')
+        // strip non-letters/numbers
+        // - note: combined with decomposing unicode diactics this normalizes both 'saa' and 'sää' as 'saa'
+        // - we could precompose the finnish letters back to avoid this
+        .replace(/[^\p{Letter}\p{Number}]/gu, '')
+        .toLowerCase();
+      debug(`${label} titleString: ${titleAsNormalizedString}`);
+      return [titleAsNormalizedString];
     }
 
     return [];
 
     function getTitle() {
       const [field] = record.get(/^245$/u);
+      debugData(`${label} titleField: ${JSON.stringify(field)}`);
 
       if (field) {
         return field.subfields
-          .filter(({code}) => ['a', 'b'].includes(code))
+          // get also $n:s and $p:s here
+          .filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))
           .map(({value}) => testStringOrNumber(value) ? String(value) : '')
           .join('');
       }

package/src/match-detection/index.js

@@ -1,4 +1,3 @@
-/* eslint-disable no-console */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -32,36 +31,45 @@ import * as features from './features';
 
 export {features};
 
-export default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA, recordB) => {
+export default ({strategy, treshold = 0.9}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
   const minProbabilityQuantifier = 0.5;
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');
   const debugData = debug.extend('data');
 
-  debugData(`Strategy: ${JSON.stringify(strategy)}`);
-  debugData(`Treshold: ${JSON.stringify(treshold)}`);
-  debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
+  debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
+  debugData(`Records: A: ${recordA}\nB: ${recordB}`);
+  debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);
+  // We could add here labels for records if we didn't get external labels
 
-  const featuresA = extractFeatures(recordA);
+  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});
 
   debug(`We got an array of records: ${Array.isArray(recordB)}`);
   const recordsB = Array.isArray(recordB) ? recordB : [recordB];
-  const detectionResults = recordsB.map(record => actualDetection(featuresA, record));
+  const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];
 
+  const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));
+
+  // if we got array of records, we return an array of result
+  // if we got a singular record, we return a singular result
   return Array.isArray(recordB) ? detectionResults : detectionResults[0];
 
-  function actualDetection(featuresA, record) {
+  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {
+    const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';
+    const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';
+
 
-    const featuresB = extractFeatures(record);
+    debug(`Actual detection for record ${index + 1} ${labelB}`);
+    const featuresB = extractFeatures({record, recordExternal, localSettings});
 
-    debugData(`Features (a): ${JSON.stringify(featuresA)}`);
-    debugData(`Features (b): ${JSON.stringify(featuresB)}`);
+    debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);
+    debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);
 
     const featurePairs = generateFeaturePairs(featuresA, featuresB);
     const similarityVector = generateSimilarityVector(featurePairs);
 
     if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
-      const probability = calculateprobability(similarityVector);
+      const probability = calculateProbability(similarityVector);
       debug(`probability: ${probability} (Treshold: ${treshold})`);
       return returnResult({match: probability >= treshold, probability});
     }
@@ -70,8 +78,8 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA,
     return returnResult({match: false, probability: 0.0});
   }
 
-  function extractFeatures(record) {
-    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract(record)}), []);
+  function extractFeatures({record, recordExternal}) {
+    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);
   }
 
   function returnResult(result) {
@@ -89,7 +97,7 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA,
     return strategyNames || [];
   }
 
-  function calculateprobability(similarityVector) {
+  function calculateProbability(similarityVector) {
     const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);
     return probability > 1.0 ? 1.0 : probability;
   }

package/src/match-detection/index.spec.js

@@ -63,7 +63,7 @@ describe('match-detection', () => {
         : new MarcRecord(getFixture('recordB.json'), {subfieldValues: false});
       debugData(inspect(recordB));
 
-      const results = detect(recordA, recordB);
+      const results = detect({recordA, recordB});
       debugData(`${JSON.stringify(results)}`);
 
       expect(results).to.eql(expectedResults);

package/src/matching-utils.js

@@ -31,7 +31,6 @@ import createDebugLogger from 'debug';
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');
 const debugData = debug.extend('data');
 
-// eslint-disable-next-line max-statements
 export function getMelindaIdsF035(record) {
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');