@natlibfi/melinda-record-matching 1.0.6 → 2.0.0-alpha.1

package/src/match-detection/features/bib/language.js

@@ -1,3 +1,4 @@
+/* eslint-disable max-statements */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -26,34 +27,67 @@
 *
 */
 
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:language');
+const debugData = debug.extend('data');
+
 export default () => ({
   name: 'Language',
   extract: record => {
     const value008 = get008Value();
     const values041 = get041Values();
+    debugData(`008: ${JSON.stringify(value008)}, 041: ${JSON.stringify(values041)}`);
 
     if (value008 && values041.length > 0) {
+      debugData(`There's both 008 and 041, searching for value in both`);
       const correspondingValue = values041.find(v => v === value008);
+      debugData(`Corresponding value: ${correspondingValue}`);
       return correspondingValue ? [correspondingValue] : [];
     }
 
+    if (!value008 && values041.length < 1) {
+      debugData(`No actual values found`);
+      return [];
+    }
+
     return value008 ? [value008] : [values041[0]];
 
     function get008Value() {
-      const value = record.get(/^008$/u)?.[0]?.value || [];
+      const value = record.get(/^008$/u)?.[0]?.value || undefined;
+      debugData(`008 value: ${value}`);
+
+      if (!value) {
+        return undefined;
+      }
+
       const code = value.slice(35, 38);
-      return code === '|||' ? undefined : code;
+      debugData(`008 code: ${code}`);
+      return code === '|||' || code === '   ' || code === '^^^' ? undefined : code;
     }
 
+    // Main language for the resource: in the first f041 $a or f041 $d
+    // Uses only f041s that have 2nd ind ' ', which means that the codes used are MARC 21 language codes
+
     function get041Values() {
       return record.get(/^041$/u)
+        .filter(({ind2}) => ind2 === ' ')
         .map(({subfields}) => subfields)
         .flat()
-        .filter(({code}) => code === 'a')
+        .filter(({code}) => code === 'a' || code === 'd')
         .map(({value}) => value);
     }
   },
   compare: (a, b) => {
+    debugData(`Comparing ${JSON.stringify(a[0])} and ${JSON.stringify(b[0])}`);
+
+    if (a.length === 0 || b.length === 0) {
+      debugData(`No language to compare`);
+      return 0;
+    }
+
+    debugData(`There area languages to compare`);
+
     if (a[0] === b[0]) {
       return 0.1;
     }

package/src/match-detection/features/bib/melinda-id.js

@@ -0,0 +1,34 @@
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+
+import createInterface from './melinda-identifier-factory';
+
+export default () => {
+  const {extract, compare} = createInterface();
+  return {extract, compare, name: 'melinda-id'};
+};

package/src/match-detection/features/bib/melinda-identifier-factory.js

@@ -0,0 +1,94 @@
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+
+import createDebugLogger from 'debug';
+import {getMelindaIdsF035} from '../../../matching-utils';
+
+// 003+001 FI-MELINDA <melinda-id>
+// 035 $a (FI-MELINDA)<melinda-id>
+// 035 $z (FI-MELINDA)<melinda-id>
+// 035 $a FCC<melinda-id>
+// 035 $z FCC<melinda-id>
+// melinda-id = 001234567
+
+export default () => {
+
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/melinda-id');
+  const debugData = debug.extend('data');
+
+  return {extract, compare};
+
+  function extract(record) {
+
+    const isMelindaRecord = record.get('003').some(f003 => f003.value === 'FI-MELINDA');
+    const [f001] = record.get('001').map(field => field.value);
+    const f035MelindaIds = getMelindaIdsF035(record);
+
+    if (
+      isMelindaRecord === undefined &&
+      f001 === undefined &&
+      f035MelindaIds.length < 1) {
+
+      debug(`No Melinda-IDs found`);
+      return {};
+    }
+
+    return {isMelindaRecord, f001, f035MelindaIds};
+
+  }
+
+  // eslint-disable-next-line max-statements
+  function compare(a, b) {
+
+    if (a.isMelindaRecord && b.isMelindaRecord &&
+        a.f001 === b.f001) {
+      debugData(`Melinda record's A f001 ${a.f001} matches Melinda record's B f001 ${a.f001}`);
+      return 1;
+    }
+
+    if (a.isMelindaRecord && typeof b.f035MelindaIds !== 'undefined' &&
+        b.f035MelindaIds.some(id => id === a.f001)) {
+      debugData(`Melinda record's A f001 ${a.f001} matches record B f035 ${JSON.stringify(b.f035MelindaIds)}`);
+      return 1;
+    }
+
+    if (b.isMelindaRecord && typeof a.f035MelindaIds !== 'undefined' &&
+        a.f035MelindaIds.some(id => id === b.f001)) {
+      debugData(`Melinda record's B f001 ${b.f001} matches record A f035 ${JSON.stringify(a.f035MelindaIds)}`);
+      return 1;
+    }
+
+    if (typeof a.f035MelindaIds !== 'undefined' && typeof b.f035MelindaIds !== 'undefined' &&
+         a.f035MelindaIds.some(idA => b.f035MelindaIds.some(idB => idB === idA))) {
+      debugData(`Record A f035 ${JSON.stringify(a.f035MelindaIds)} matches record B f035 ${JSON.stringify(b.f035MelindaIds)}`);
+      return 1;
+    }
+    debug(`No matching Melinda-IDs.`);
+    return 0;
+  }
+};

package/src/match-detection/features/bib/standard-identifier-factory.js

@@ -26,6 +26,11 @@
 *
 */
 
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers');
+const debugData = debug.extend('data');
+
 export default ({pattern, subfieldCodes}) => {
   return {extract, compare};
 
@@ -43,6 +48,7 @@ export default ({pattern, subfieldCodes}) => {
 
   function compare(a, b) {
     if (a.length === 0 || b.length === 0) {
+      debugData(`No standardidentifiers to compare`);
       return 0;
     }
 

package/src/match-detection/index.js

@@ -1,3 +1,4 @@
+/* eslint-disable no-console */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -31,15 +32,17 @@ import * as features from './features';
 
 export {features};
 
+// eslint-disable-next-line max-statements
 export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
   const minProbabilityQuantifier = 0.5;
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');
+  const debugData = debug.extend('data');
   const featuresA = extractFeatures(recordA);
   const featuresB = extractFeatures(recordB);
 
-  debug(`Features (a): ${JSON.stringify(featuresA)}`);
-  debug(`Features (b): ${JSON.stringify(featuresB)}`);
+  debugData(`Features (a): ${JSON.stringify(featuresA)}`);
+  debugData(`Features (b): ${JSON.stringify(featuresB)}`);
 
   const featurePairs = generateFeaturePairs();
   const similarityVector = generateSimilarityVector();
@@ -50,7 +53,7 @@ export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
     return {match: probability >= treshold, probability};
   }
 
-  debug(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
+  debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
   return {match: false, probability: 0.0};
 
   function calculateprobability() {
@@ -69,7 +72,7 @@ export default ({strategy, treshold = 0.9}) => (recordA, recordB) => {
       return {name, points};
     });
 
-    debug(`Points: ${JSON.stringify(compared)}`);
+    debugData(`Points: ${JSON.stringify(compared)}`);
     return compared.map(({points}) => points);
   }
 

package/src/match-detection/index.spec.js

@@ -33,28 +33,35 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import * as features from './features';
 import createDetectionInterface from '.';
 
-generateTests({
-  path: [__dirname, '..', '..', 'test-fixtures', 'match-detection', 'index'],
-  useMetadataFile: true,
-  recurse: false,
-  fixura: {
-    reader: READERS.JSON
-  },
-  callback: ({getFixture, options, expectedResults}) => {
-    const detect = createDetectionInterface(formatOptions());
-    const recordA = new MarcRecord(getFixture('recordA.json'));
-    const recordB = new MarcRecord(getFixture('recordB.json'));
-    const results = detect(recordA, recordB);
+describe('match-detection', () => {
+  generateTests({
+    path: [__dirname, '..', '..', 'test-fixtures', 'match-detection', 'index'],
+    useMetadataFile: true,
+    recurse: false,
+    fixura: {
+      reader: READERS.JSON
+    },
+    callback: ({getFixture, options, expectedResults, enabled = true}) => {
 
-    expect(results).to.eql(expectedResults);
+      if (!enabled) {
+        return;
+      }
 
-    function formatOptions() {
-      const contextFeatures = features[options.strategy.type];
+      const detect = createDetectionInterface(formatOptions());
+      const recordA = new MarcRecord(getFixture('recordA.json'));
+      const recordB = new MarcRecord(getFixture('recordB.json'));
+      const results = detect(recordA, recordB);
 
-      return {
-        ...options,
-        strategy: options.strategy.features.map(v => contextFeatures[v]())
-      };
+      expect(results).to.eql(expectedResults);
+
+      function formatOptions() {
+        const contextFeatures = features[options.strategy.type];
+
+        return {
+          ...options,
+          strategy: options.strategy.features.map(v => contextFeatures[v]())
+        };
+      }
     }
-  }
+  });
 });

package/src/matching-utils.js

@@ -0,0 +1,86 @@
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');
+const debugData = debug.extend('data');
+
+// eslint-disable-next-line max-statements
+export function getMelindaIdsF035(record) {
+
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');
+  const debugData = debug.extend('data');
+
+  const f035s = record.getFields('035');
+
+  if (f035s.length < 1) {
+    debug(`No f035s found.`);
+    return [];
+  }
+
+  const allF035MelindaIds = [].concat(...f035s.map(toMelindaIds));
+  const f035MelindaIds = [...new Set(allF035MelindaIds)];
+
+  debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);
+  debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);
+  debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);
+
+  return f035MelindaIds;
+
+  function toMelindaIds({subfields}) {
+    const melindaIdRegExp = /^(?<prefix>\(FI-MELINDA\)|FCC)(?<id>\d{9})$/u;
+
+    return subfields
+      .filter(sub => ['a', 'z'].includes(sub.code))
+      .filter(sub => melindaIdRegExp.test(sub.value))
+      .map(({value}) => value.replace(melindaIdRegExp, '$<id>'));
+
+  }
+}
+
+export function validateSidFieldSubfieldCounts(field) {
+  // Valid SID-fields have just one $c and one $b
+  debugData(`Validating SID field ${JSON.stringify(field)}`);
+  const countC = countSubfields(field, 'c');
+  const countB = countSubfields(field, 'b');
+  debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);
+
+  return countC === 1 && countB === 1;
+}
+
+function countSubfields(field, subfieldCode) {
+  // debug(`Counting subfields ${subfieldCode}`);
+  return field.subfields.filter(({code}) => code === subfieldCode).length;
+}
+
+export function getSubfieldValues(field, subfieldCode) {
+  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);
+  return field.subfields.filter(({code}) => code === subfieldCode).map(({value}) => value);
+}
+