@natlibfi/melinda-record-matching 5.0.1-alpha.1 → 5.0.2-alpha.1

package/src/match-detection/features/bib/issn.js

@@ -1,7 +1,89 @@
+/*
+//import createInterface from './standard-identifier-factory.js';
 
-import createInterface from './standard-identifier-factory.js';
 
 export default () => {
   const {extract, compare} = createInterface({pattern: /^022$/u, subfieldCodes: ['a', 'z', 'y']});
   return {extract, compare, name: 'ISSN'};
 };
+*/
+
+
+import createDebugLogger from 'debug';
+
+import {isComponentRecord} from '@natlibfi/melinda-commons';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:issn');
+const debugData = debug.extend('data');
+
+export default () => ({
+  name: 'ISSN',
+  extract: ({record/*, recordExternal*/}) => {
+    //const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
+
+    const isHost = !isComponentRecord(record, false, []);
+
+    // debug(`\t Is HOST: '${isHost}'`);
+    return getIssns();
+
+    function getIssns() {
+      const fields = getRelevantFields();
+      if (fields.length === 0) {
+        return [];
+      }
+      debug(`\t ${fields.length} potential ISSN fields (${fields[0].tag})`);
+      const subfieldCodes = getRelevantSubfieldCodes();
+      //debug(`\t subfield codes: '${subfieldCodes.join("', '")}'`);
+      const subfieldValues = fields.flatMap(f => f.subfields.filter(sf => subfieldCodes.includes(sf.code)).map(sf => sf.value));
+      //debug(`\t cand values: '${subfieldValues.join("', '")}'`);
+      // Stripping punctuaction with substring here is pretty quick and dirty approach...
+      const validSubfieldValues = subfieldValues?.map(val => normalizeSubfieldValue(val)).filter(val => isValidIssn(val));
+      if (!validSubfieldValues) {
+        return [];
+      }
+      return uniqArray(validSubfieldValues);
+    }
+
+    function getRelevantFields() {
+      if (isHost) {
+        return record.get(/^022$/u);
+      }
+      return record.get(/^[79]73$/u);
+    }
+
+    function normalizeSubfieldValue(val) {
+      return val.replace(/[., :;].*$/u, '');
+    }
+
+    function isValidIssn(val) {
+      return /^[0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9X]$/u.test(val);
+    }
+
+    function getRelevantSubfieldCodes() {
+      if (isHost) {
+        return ['a', 'y', 'z'];
+      }
+      return ['x'];
+    }
+  },
+  // eslint-disable-next-line max-statements
+  compare: (aa, bb) => {
+    debugData(`Comparing ISSN sets ${JSON.stringify(aa)} and ${JSON.stringify(bb)}`);
+    if (aa.length === 0 || bb.length === 0) {
+      // No data for decision
+      return 0;
+    }
+    const firstSharedIssn = aa.find(val => bb.includes(val));
+    if (firstSharedIssn) {
+      debug(`\t Shared ISSN found: '${firstSharedIssn}'`);
+      // Maybe less for comps?
+      return 0.2;
+    }
+    return -0.2;
+
+  }
+});
+
+export function uniqArray(arr) {
+  return arr.filter((val, i) => arr.indexOf(val) === i);
+}

package/src/match-detection/features/bib/language.js

@@ -1,111 +1,299 @@
 
 import createDebugLogger from 'debug';
+
+import {MarcRecord} from '@natlibfi/marc-record';
+
+import {FixSami041, Remove041zxx} from '@natlibfi/marc-record-validators-melinda';
+
 import {getMatchCounts} from '../../../matching-utils.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:language');
 const debugData = debug.extend('data');
 
+// NB! 2025-12-17: I changed the logic drastically. However, I tried to keep the scores as same as possible.
+// 'extract' no longer extracts anything. Instead it clones the record, and does some preprocessing (using validators) instead.
+// 'compare' had multiple changes
+// - f041 ind1 differences ('0' vs '1') now result in a small penalty
+// - Two sami languages related changes:
+// -- validator adds a 'smi' subfield before a corresponding sma/sme/...subfield, if needed (national)
+// -- 'smi' only vs 'smi'+'sma' does not cause penalty
+// - 'mul' vs 'fin'+'swe' does not cause a penalty. However, 'mul' vs 'fin' alone triggers penalty.
+// - 008/35-37 and f041$a/$d are calculated separately
+// - a threshold is applied: return value is always between -1.0 and +0.1 (as it was before)
+// - we try to handle 041 $2 ISO 639-2 and ISO 639-3 as well.
+// - 'und' vs one other language does not cause a penalty in certain contexts. In other contexts it's treated as a normal "language code". (Tune/alleviate penalties later on.)
+
 export default () => ({
   name: 'Language',
   extract: ({record, recordExternal}) => {
     const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
+    const clonedRecord = new MarcRecord(record, record._validationOptions); // clone(record); // NB! This loses record.get()...
+
+    FixSami041().fix(clonedRecord); // Handle 'smi' adding if needed
+    Remove041zxx().fix(clonedRecord); // Remove 'zxx' from f041s
+    // Add other language code normalizations?
 
-    const value008 = get008Value();
-    const values041 = get041Values();
-    debugData(`${label}: 008: ${JSON.stringify(value008)}, 041: ${JSON.stringify(values041)}`);
+    // Should we return all the fields, or just the relevant fields?
+    return [{leader: clonedRecord.leader, fields: clonedRecord.fields}, label];
+  },
+  // eslint-disable-next-line max-statements
+  compare: (aa, bb) => {
+    const [a, aLabel] = aa;
+    const [b, bLabel] = bb;
+    debugData(`Comparing language ${JSON.stringify(a)} and ${JSON.stringify(b)}`);
+
+    const score008 = compare008();
+    const score041 = compare041();
+    return applyLimits(score008 + score041);
 
-    if (!value008 && values041.length < 1) {
-      debugData(`${label}: No actual values found`);
-      return [];
+    function applyLimits(score) {
+      if (score > 0.1) { // Keeps the original max, we might have 0.10 (041) + 0.05 (008/35-37) = 0.15 now
+        return 0.1;
+      }
+      if (score < -1) {
+        return -1.0;
+      }
+      return score;
     }
 
-    const allValues = value008 === undefined ? values041 : values041.concat(value008);
-    const uniqueSortedValues = [...new Set(allValues)].sort();
+    function compare008() {
+      const a008 = get008Value(a, aLabel);
+      const b008 = get008Value(b, bLabel);
+      // Something seriously wrong with 008:
+      if (a008 === undefined || b008 === undefined) {
+        return 0.0; // Punish for generic badness?
+      }
 
-    return uniqueSortedValues;
+      if (containsNoData(a008) || containsNoData(b008)){
+        // Nothing to compare
+        return 0.0;
+      }
 
-    function get008Value() {
-      const value = record.get(/^008$/u)?.[0]?.value || undefined;
-      debugData(`${label}: 008 value: ${value}`);
+      if (a008 === b008) {
+        return 0.05;
+      }
 
-      if (!value) {
-        return undefined;
+      if (a008 === 'mul' || b008 === 'mul') {
+        return 0.0;
       }
 
-      const code = value.slice(35, 38);
-      debugData(`${label}: 008 code: ${code}`);
-      return isLangCodeForALanguage(code) ? code : undefined;
+      return -0.2;
     }
 
-    // Uses only f041s that have 2nd ind ' ', which means that the codes used are MARC 21 language codes
 
-    function get041Values() {
-      return record.get(/^041$/u)
-        .filter(({ind2}) => ind2 === ' ')
-        .map(({subfields}) => subfields)
-        .flat()
-        .filter(({code}) => code === 'a' || code === 'd')
-        .filter(({value}) => value && isLangCodeForALanguage(value))
-        .map(({value}) => value);
-    }
 
-    // Check if a string is a possible, validly formed language code for a single language
-    // Currently accept also codes in capitals
-    function isLangCodeForALanguage(code) {
-      if (code.length !== 3) {
-        debugData(`Code ${code} is not correct length (3) for a language code.`);
-        return false;
+
+
+    function compare041() {
+      const a041s = getFields041(a);
+      const b041s = getFields041(b);
+      if (a041s.length === 0 || b041s.length === 0) {
+        return 0.0; // Should we punish these for badness?
       }
-      if (code === '|||' || code === '   ' || code === '^^^' || code === 'mul' || code === 'zxx') {
-        debugData(`Code ${code} is not code for a spesific language.`);
-        return false;
+      if (a041s.length === 1 && b041s.length === 1) {
+        // The language codes are typically same between different sources. Incur a small penalty for differences though...
+        const sourcePenalty = getSourceOfLanguageCode(a041s[0]) === getSourceOfLanguageCode(b041s[0]) ? 0.0 : -0.05;
+        const mainPenalty = compareLanguageCodeLists(getFieldsLanguageCodes(a041s[0]), getFieldsLanguageCodes(b041s[0]));
+        const scoreInd1 = mainPenalty === 0.0 ? 0.0 : indicator1Penalty(a041s[0], b041s[0]);
+        return mainPenalty + sourcePenalty + scoreInd1;
       }
-      const langCodePattern = /^[a-z][a-z][a-z]$/ui;
-      if (!langCodePattern.test(code)) {
-        debugData(`Code ${code} is not valid as a language code`);
-        return false;
+      // Things are already complicated (likely to pe penalized, so no nedd to worrty about language code sources, I daresay.
+      return compareLanguageCodeLists(getLanguageCodesFrom041Fields(a), getLanguageCodesFrom041Fields(b));
+
+      function getFields041(record) {
+        if (!record.fields) {
+          return [];
+        }
+        return record.fields.filter(f => f.tag === '041');
       }
-      return true;
+
     }
 
-  },
-  // eslint-disable-next-line max-statements
-  compare: (a, b) => {
-    debugData(`Comparing ${JSON.stringify(a)} and ${JSON.stringify(b)}`);
+    function getSourceOfLanguageCode(field) {
+      const sf2 = field.subfields.find(sf => sf.code === '2');
+      if (!sf2) {
+        return 'MARC';
+      }
+      // Normalize the two relevant language code names:
+      if (sf2.value.match(/639.2/u)) {
+        return 'ISO 639-2';
+      }
+      if (sf2.value.match(/639.3/u)) {
+        return 'ISO 639-3';
+      }
+      return sf2.value; // We don't actually have anything else in Melinda though
+    }
 
-    if (a.length === 0 || b.length === 0) {
-      debugData(`No language to compare`);
-      return 0;
+    function indicator1Penalty(aField, bField) {
+      if (aField.ind1 === ' ' || bField.ind1 === ' '|| aField.ind1 === bField.ind1 ) {
+        return 0.0;
+      }
+      debug(`\t Indicator penalty: '${aField.ind1}' vs '${bField.ind1}'`);
+      return -0.1;
     }
 
-    if (a.length === b.length && a.every((element, index) => element === b[index])) {
-      debugData(`All languages match`);
-      return 0.1;
+
+
+    function containsNoData(languageCode) {
+      return ['   ', '|||', 'und'].includes(languageCode);
     }
 
-    const {matchingValues, possibleMatchValues, maxValues} = getMatchCounts(a, b);
+    function compareLanguageCodeLists(a, b) {
+      if (a.length === 0 || b.length === 0) {
+        debugData(`No language to compare`);
+        return 0;
+      }
+
+      // If either one of the sets has only the generic 'smi', remove the specific sami languages codes from the other as well:
+      const samiLanguageCodes = ['sma', 'sme', 'smj', 'smn', 'sms']; // NB! Don't put generic 'smi' here!
+
+      if (a.includes('smi') && b.includes('smi')) {
+        if (a.some(code => samiLanguageCodes.includes(code)) && !b.some(code => samiLanguageCodes.includes(code))) {
+          return compareLanguageCodeLists(a.filter(val => !samiLanguageCodes.includes(val)), b); // recurse
+        }
+        if (b.some(code => samiLanguageCodes.includes(code)) && !a.some(code => samiLanguageCodes.includes(code))) {
+          return compareLanguageCodeLists(a, b.filter(val => !samiLanguageCodes.includes(val))); // recurse
+        }
+      }
+
+
+
+
+      if (a.length === b.length && a.every((element, index) => element === b[index])) {
+        debugData(`All languages match`);
+        return 0.1;
+      }
+
+      // Handle 'mul'. (Should we check whether the other array contains mul as well, probably not...)
+      // (If 'mul' does not appear alone, then it is very iffy... )
+      if (a.length === 1 && a[0] === 'mul' && b.length > 1) {
+        return 0;
+      }
+      if (b.length === 1 && b[0] === 'mul' && a.length > 1) {
+        return 0;
+      }
+
+
+
+      // Damage control:
+
+      // Not using the generic solution here as eg. 'und' needs a special treatment
+      const sharedValues = getSharedValues(a, b);
+      const aOnly = a.filter(val => !sharedValues.includes(val));
+      const bOnly = b.filter(val => !sharedValues.includes(val));
+      const hasUnd = [...aOnly, ...bOnly].includes('und');
+
+      if (sharedValues.length < 1) {
+        console.info(`NV: ${sharedValues.join(", ")}`);
+        if (aOnly.length === 1 && bOnly.length === 1 && hasUnd) {
+          debug(`Both have languages, but none of these match. However, the benefit of doubt is given: '${aOnly[0]}' and '${bOnly[0]}' might mean the same}`);
+          return 0;
+        }
+        debug(`Both have languages, but none of these match.`); // includes 'mul' vs a single language code
+        return -1.0;
+      }
 
-    if (matchingValues < 1) {
-      debug(`Both have languages, but none of these match.`);
-      return -1.0;
+      const {matchingValues, possibleMatchValues, maxValues} = getMatchCounts(a, b);
+
+      debug(`Both have languages, ${matchingValues}/${possibleMatchValues} valid languages match.`);
+      // ignore non-matches if there is mismatching amount of values
+      debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);
+      // we give some kind of penalty for mismatching amount of values instead of simple divide?
+      const missingCount = maxValues - possibleMatchValues;
+      const misMatchCount = possibleMatchValues - matchingValues;
+      debug(`\t missing: ${missingCount}`);
+      debug(`\t mismatches: ${misMatchCount}`);
+
+      const penaltyForMissing = 0.02 * (maxValues - possibleMatchValues);
+      const penaltyForMisMatch = 0.05 * (possibleMatchValues - matchingValues);
+      debug(`\t points: penaltyForMissing: ${penaltyForMissing}`);
+      debug(`\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);
+
+      const points = Number(Number(0.1 - penaltyForMisMatch - penaltyForMissing).toFixed(2));
+      debug(`Total points: ${points}`);
+
+      return points;
     }
-    debug(`Both have languages, ${matchingValues}/${possibleMatchValues} valid languages match.`);
-    // ignore non-matches if there is mismatching amount of values
-    debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);
-    //we give some kind of penalty for mismatching amount of values instead of simple divide?
-    const missingCount = maxValues - possibleMatchValues;
-    const misMatchCount = possibleMatchValues - matchingValues;
-    debug(`\t missing: ${missingCount}`);
-    debug(`\t mismatches: ${misMatchCount}`);
-
-    const penaltyForMissing = 0.02 * (maxValues - possibleMatchValues);
-    const penaltyForMisMatch = 0.05 * (possibleMatchValues - matchingValues);
-    debug(`\t points: penaltyForMissing: ${penaltyForMissing}`);
-    debug(`\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);
-
-    const points = Number(Number(0.1 - penaltyForMisMatch - penaltyForMissing).toFixed(2));
-    debug(`Total points: ${points}`);
-
-    return points;
   }
+
 });
+
+function get008Value(record, label = 'record') {
+  if (!record || !record.fields) {
+    return;
+  }
+  const f008 = record.fields.find(f => f.tag === '008');
+  if (!f008) {
+    return undefined;
+  }
+  const value = f008.value || defaultValue;
+
+  if (!value) {
+    debugData(`${label}: Failed to extact 008/35-37 value from '${value}'`);
+    return defaultValue;
+  }
+
+  const code = value.slice(35, 38);
+  debugData(`${label}: 008 code: '${code}'`);
+  return code;
+}
+
+// Check if a string is a possible, validly formed language code for a single language
+// Currently accept also codes in capitals
+function isLangCodeForALanguage(code, label = 'record', encoding = undefined) {
+  if (!code) {
+    return false;
+  }
+
+  if ([undefined, 'ISO 639-2', 'ISO-639-3'].includes(encoding) && code.length !== 3) {
+    debugData(`${label}: Code ${code} is not correct length (3) for a language code.`);
+    return false;
+  }
+  if (!encoding) {
+    // 'mul' should be passed as 'mul' can match 'fin' + 'swe' etc.
+    // 'zxx' should be removed by a validator
+    // '^^^' is Aleph-specific corruption, not supporting it anymore
+    if (code === '|||' || code === '   ' ) { // || code === '^^^' || code === 'mul' || code === 'zxx') {
+      debugData(`${label}: Code ${code} is not code for a spesific language.`);
+      return false;
+    }
+  }
+  // Marc, ISO 639-2 and ISO 639-3 are all three-letters long. Not other language codes seen in Melinda:
+  const langCodePattern = /^[a-z][a-z][a-z]$/ui;
+  if (!langCodePattern.test(code)) {
+    debugData(`${label}: Code ${code} is not valid as a language code`);
+    return false;
+  }
+  return true;
+}
+
+
+
+function getLanguageCodesFrom041Fields(record) {
+  // NB! We brutally don't check $2 (language code source) as marc's language codes is practically a subset of ISO 639-2,
+  // and also ISO 639-2 and ISO 639-3 overlap to a degree. If we ever run into a trouble with some ISO 639-2 vs 639-3 mismatch, then we'll work it out.
+  // Also we could write a validator that converts ISO 639-2 to marc if applicable and maybe even partial support for ISO-639-3.
+  // Note that ISO 639-2-B values correspond with marc beteer that ISO 639-2-T. Eg. code for Chinese 'zho' should/code be normalized to 'chi'!
+  const values = record.fields.filter(f => f.tag === '041').flatMap(f => getFieldsLanguageCodes(f));
+  /*
+        // .filter(({ind2}) => ind2 === ' ')
+        .map(({subfields}) => subfields)
+        .flat()
+        .filter(({code}) => code === 'a' || code === 'd')
+        .filter(({value}) => value && isLangCodeForALanguage(value))
+        .map(({value}) => value);
+    */
+  return [...new Set(values)].sort();
+}
+
+function getFieldsLanguageCodes(field) {
+  const relevantSubfields = field.subfields.filter(sf => sf.code === 'a' || sf.code === 'd').filter(sf => isLangCodeForALanguage(sf.value, 'field'));
+
+  const values = relevantSubfields.map(sf => sf.value).flat();
+
+  return [...new Set(values)].sort();
+}
+
+function getSharedValues(list1, list2) {
+  return list1.filter(val => list2.includes(val));
+}

package/src/match-detection/features/bib/title-version-original.js

@@ -3,7 +3,7 @@ import createDebugLogger from 'debug';
 import naturalPkg from 'natural';
 const {LevenshteinDistance: leven} = naturalPkg;
 
-export default ({treshold = 10} = {}) => ({
+export default ({threshold = 10} = {}) => ({
   name: 'titleVersionOriginal',
   extract: ({record}) => {
     const title = getTitle();
@@ -38,7 +38,7 @@ export default ({treshold = 10} = {}) => ({
 
     debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
 
-    if (percentage <= treshold) {
+    if (percentage <= threshold) {
       return 0.3;
     }
 

package/src/match-detection/features/bib/title.js

@@ -7,7 +7,7 @@ const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detecti
 const debugData = debug.extend('data');
 
 
-export default ({treshold = 10} = {}) => ({
+export default ({threshold = 10} = {}) => ({
   name: 'Title',
   extract: ({record, recordExternal}) => {
     const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
@@ -19,8 +19,9 @@ export default ({treshold = 10} = {}) => ({
         // decompose unicode diacritics
         .normalize('NFD')
         // strip non-letters/numbers
-        // - note: combined with decomposing unicode diactics this normalizes both 'saa' and 'sää' as 'saa'
-        // - we could precompose the finnish letters back to avoid this
+        // - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 'sää' as 'saa'
+        // - we could precompose the Finnish letters back to avoid this
+        // - see validator normalize-utf8-diacritics for details
         .replace(/[^\p{Letter}\p{Number}]/gu, '')
         .toLowerCase();
       debug(`${label}: titleString: ${titleAsNormalizedString}`);
@@ -56,7 +57,7 @@ export default ({treshold = 10} = {}) => ({
 
     debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
 
-    if (percentage <= treshold) {
+    if (percentage <= threshold) {
       return 0.3;
     }
 

package/src/match-detection/index.js

@@ -4,13 +4,13 @@ import * as features from './features/index.js';
 
 export {features};
 
-export default ({strategy, treshold = 0.9}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
+export default ({strategy, threshold = 0.8999}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
   const minProbabilityQuantifier = 0.5;
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');
   const debugData = debug.extend('data');
 
-  debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
+  debugData(`Strategy: ${JSON.stringify(strategy)}, Threshold: ${JSON.stringify(threshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
   debugData(`Records: A: ${recordA}\nB: ${recordB}`);
   debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);
   // We could add here labels for records if we didn't get external labels
@@ -43,8 +43,8 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false, localSetting
 
     if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
       const probability = calculateProbability(similarityVector);
-      debug(`probability: ${probability} (Treshold: ${treshold})`);
-      return returnResult({match: probability >= treshold, probability});
+      debug(`probability: ${probability} (Threshold: ${threshold})`);
+      return returnResult({match: probability >= threshold, probability});
     }
 
     debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
@@ -58,7 +58,7 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false, localSetting
   function returnResult(result) {
     if (returnStrategy) {
       debug(`Returning detection strategy with the result`);
-      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold};
+      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), threshold};
       debugData(`${JSON.stringify(resultWithStrategy)}`);
       return resultWithStrategy;
     }