@natlibfi/melinda-record-matching 5.0.3 → 5.0.5-alpha.2

package/src/match-detection/features/bib/f773.js

@@ -0,0 +1,118 @@
+// NB! This checks identifiers ($w=ID, $x=ISSN, $z=ISBN and $o=others) to check whether the host is the same.
+// BB! $g is not checked *yet*. It could very strongly indicate that the records are the same... (I doubt $q could be useful here)
+// Rationale:
+// 773$w indicates sameness. However, non-Melinda records probably refer to non-melinda hosts.
+// Thus check other identifiers subfields ($x, $z and $o as well)
+//
+
+
+import createDebugLogger from 'debug';
+
+import {isComponentRecord} from '@natlibfi/melinda-commons';
+import {uniqArray} from './issn.js';
+import {parse773g} from '../../../candidate-search/query-list/component.js';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:issn');
+const debugData = debug.extend('data');
+
+const MAX_IDENTIFIER = 0.1; // This should be pretty low: it only says something about the host
+const MAX_G = 0.5; // $g is about the comp itself, so score can be high here
+
+export default () => ({
+  name: 'f773 ',
+  extract: ({record/*, recordExternal*/}) => {
+    //const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
+
+    if (!isComponentRecord(record, false, [])) {
+      return [];
+    }
+    const f773s = record.get(/^773$/u); // Identifier extract handles multiple 773...
+
+    // I think it's ok, if, say, $o and $x values match, so I'm not keeping the subfield codes
+    const values = f773s.map(f => f.subfields)
+      .flat()
+      .filter(sf => ['w', 'o', 'x', 'z'].includes(sf.code))
+      .map(sf =>  normalizeValue(sf.value));
+
+    // $g: only on $g is supported
+    const gData = parse773g(f773s[0]);
+
+    return [uniqArray(values), gData];
+
+    function normalizeValue(value) {
+      return value.replace(/(?:\. -|\.)$/u, '');
+    }
+
+
+  },
+
+  compare: (aa, bb) => {
+    const [aIdentifiers, ag] = aa;
+    const [bIdentifiers, bg] = bb;
+
+    const identifierScore = scoreIdentifiers();
+    const gScore = scoreG();
+
+    if (identifierScore === MAX_IDENTIFIER && gScore === MAX_G) { // Pretty impressive hit, even if title matches not
+      return 1.0;
+    }
+
+    return identifierScore + gScore;
+
+    function scoreG() {
+      // NB! $g contents are very noise, so wrong values may be extracted. Thus do not overpenalize.
+
+      // All exist match: things must be pretty good:
+      if (ag.number && ag.number === bg.number && ag.pages && ag.pages === bg.pages && ag.year && ag.year === bg.year) {
+        return MAX_G;
+      }
+      // Not comparing volume. It correlates with year.
+      return scoreYear() + scoreNumber() + scorePages();
+    }
+
+    function scoreNumber() {
+      if (!ag.number || !bg.number) {
+        return 0.0;
+      }
+      if (ag.number === bg.number) {
+        return 0.05;
+      }
+      return -0.02;
+    }
+
+    function scorePages() {
+      if (!ag.pages || !bg.pages) {
+        return 0.0;
+      }
+      if (ag.pages === bg.pages) { // If pages match, things must be pretty good
+        return 0.1;
+      }
+      return -0.05;
+    }
+
+    function scoreYear() { // publication-time.js also uses this, so don't score heavily here
+      if (!ag.year || !bg.year) {
+        return 0.0;
+      }
+      if (ag.year === bg.year) {
+        return 0.02;
+      }
+      return -0.02;
+    }
+
+    function scoreIdentifiers() {
+      debugData(`Comparing ISSN sets ${JSON.stringify(aIdentifiers)} and ${JSON.stringify(bIdentifiers)}`);
+      if (aIdentifiers.length === 0 || bIdentifiers.length === 0) {
+        // No data for decision
+        return 0;
+      }
+      const firstSharedIdentifier = aIdentifiers.find(val => bIdentifiers.includes(val));
+      if (firstSharedIdentifier) {
+        debug(`\t Shared identifier found: '${firstSharedIdentifier}'`);
+        return MAX_IDENTIFIER;
+      }
+      return -0.5;
+    }
+  }
+});
+

package/src/match-detection/features/bib/index.js

@@ -4,7 +4,6 @@ export {default as isbn} from './isbn.js';
 export {default as issn} from './issn.js';
 export {default as otherStandardIdentifier} from './other-standard-identifier.js';
 export {default as title} from './title.js';
-export {default as titleVersionOriginal} from './title-version-original.js';
 export {default as authors} from './authors.js';
 export {default as recordType} from './record-type.js';
 export {default as publicationTime} from './publication-time.js';
@@ -15,3 +14,4 @@ export {default as bibliographicLevel} from './bibliographic-level.js';
 export {default as melindaId} from './melinda-id.js';
 export {default as allSourceIds} from './all-source-ids.js';
 export {default as mediaType} from './media-type.js';
+export {default as f773} from './f773.js';

package/src/match-detection/features/bib/index.test.js

@@ -42,7 +42,7 @@ describe('match-detection/features/bib/', () => {
         const {featuresA, featuresB, expectedPoints} = expectations;
         const {compare} = features[feature](options);
 
-        assert.equal(compare(featuresA, featuresB), expectedPoints);
+        assert.equal(Math.round(compare(featuresA, featuresB) *100)/100, expectedPoints);
         return;
       }
 

package/src/match-detection/features/bib/isbn.js

@@ -3,8 +3,6 @@ import createDebugLogger from 'debug';
 import {parse as isbnParse} from 'isbn3';
 
 import {uniqArray} from './issn.js';
-import {isComponentRecord} from '@natlibfi/melinda-commons';
-
 
 const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:ISBN`);
 const debugData = debug.extend('data');
@@ -16,12 +14,7 @@ export default () => ({
   extract: ({record/*, recordExternal*/}) => {
     //const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
 
-    const host = !isComponentRecord(record, false, []);
-
-    if (host) {
-      return record.get('020').filter(f => f.subfields?.some(sf => ['a', 'z'].includes(sf.code) && sf.value));
-    }
-    return record.get('773').filter(f => f.subfields?.some(sf => ['z'].includes(sf.code) && sf.value));
+    return record.get('020').filter(f => f.subfields?.some(sf => ['a', 'z'].includes(sf.code) && sf.value));
   },
   // eslint-disable-next-line max-statements
   compare: (aa, bb) => {
@@ -31,52 +24,72 @@ export default () => ({
       return 0;
     }
 
-    const [subfieldCodeForGoodValues, subfieldCodeForBadValues] = getSubfieldCodes(aa[0].tag);
+    const subfieldCodeForGoodValues = 'a';
+    const subfieldCodeForBadValues = 'z';
 
-    const [goodValuesA, badValuesA] = getValues(aa);
-    if (goodValuesA.length) {
-      debug(`GOOD VALUES (A): ${goodValuesA.join(', ')}`);
-    }
-    if (badValuesA.length) {
-      debug(`BAD VALUES (A): ${badValuesA.join(', ')}`);
-    }
+    const [aValidValuesA, aInvalidValuesA, zValidValuesA, zInvalidValuesA] = getValuesWrapper(aa, 'AA'); // initial 'a' and 'z' refer to 020 subfields codes
+    const [aValidValuesB, aInvalidValuesB, zValidValuesB, zInvalidValuesB] = getValuesWrapper(bb, 'BB');
 
-    const [goodValuesB, badValuesB] = getValues(bb);
-    if (goodValuesB.length) {
-      debug(`GOOD VALUES (B): ${goodValuesB.join(', ')}`);
-    }
-    if (badValuesB.length) {
-      debug(`BAD VALUES (B): ${badValuesB.join(', ')}`);
+    function getValuesWrapper(data, prefix) {
+      const [aValidValues, aInvalidValues, zValidValues, zInvalidValues] = getValues(data);
+      if (aValidValues.length) {
+        debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForGoodValues} VALID: ${aValidValues.join(', ')}`);
+      }
+      if (aInvalidValues.length) {
+        debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForGoodValues} INVALID: ${aInvalidValues.join(', ')}`);
+      }
+      if (zValidValues.length) {
+        debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForBadValues} VALID: ${zValidValues.join(', ')}`);
+      }
+      if (zInvalidValues.length) {
+        debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForBadValues} INVALID: ${zInvalidValues.join(', ')}`);
+      }
+      return [aValidValues, aInvalidValues, zValidValues, zInvalidValues];
     }
 
-    const [sharedGoodValues, goodValuesAOnly, goodValuesBOnly] = getUnionData(goodValuesA, goodValuesB);
+    const [sharedGoodValues, goodValuesAOnly, goodValuesBOnly] = getUnionData(aValidValuesA, aValidValuesB);
 
-    //debug(`GOOD\tBOTH: ${sharedGoodValues.length}, A only: ${goodValuesAOnly.length}, B only: ${goodValuesBOnly.length}`);
+    debug(`GOOD\tBOTH: ${sharedGoodValues.length}, A only: ${goodValuesAOnly.length}, B only: ${goodValuesBOnly.length}`);
 
-    const hitScore = scoreHit();
+    if (sharedGoodValues.length > 0) {
+      // Third argument (aka 'N times') is >= 0
+      return scoreData(MAX_SCORE, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);
+    }
 
-    function scoreHit() {
-      if (sharedGoodValues.length > 0) {
-        return MAX_SCORE;
-      }
+    const hitScore = scoreSuboptimalHit();
+
+    function scoreSuboptimalHit() {
       // One record consider ISBN good and the other record considered it's canceled:
-      if (goodValuesA.some(valA => badValuesB.includes(valA)) || goodValuesB.some(valB => badValuesA.includes(valB))) {
+      if (aValidValuesA.some(valA => zValidValuesB.includes(valA)) || aValidValuesB.some(valB => zValidValuesA.includes(valB))) {
         return MAX_SCORE;
       }
 
-      // Value is bad, but looks isbn-ish to a human eye (not validating the isbn again, note than invalid isbns in 020$a are considered bad):
+      // Subfield is for cancelled/whatever values, but the value is syntactically valid:
       // Could happen for two canceled ISBNs for example. I'll give this two thirds of the full score
-      if (badValuesA.some(valA => looksGood(valA) && badValuesB.includes(valA)) || badValuesB.some(valB => looksGood(valB) && badValuesA.includes(valB))) {
+      const zzValid = zValidValuesA.find(valA => zValidValuesB.includes(valA));
+      if (zzValid) {
+        debug(`Both contain a valid value in 020$z: ${zzValid}`);
+        return MAX_SCORE * 2 / 3;
+      }
+      // Shared invalid identifiers:
+      const aaInvalid = aInvalidValuesA.find(valA => aInvalidValuesB.includes(valA) || zInvalidValuesB.includes(valA)) || aInvalidValuesB.find(valB => zInvalidValuesA.includes(valB));
+      if (aaInvalid) {
+        debug(`Shared invalid value in 020$a and 020$a-or-$z subfields: ${aaInvalid}`);
         return MAX_SCORE * 2 / 3;
       }
 
+      /* // Currently I think that paired invalid idenfiers in 020$z are meaningless...
+      const zzInvalid = zInvalidValuesB.find(valB => zInvalidValuesA.includes(valB)) || zInvalidValuesA.find(valA => zInvalidValuesB.includes(valA));
+      if (zzInvalid) {
+        debug(`Shared invalid value in 020$z subfields: ${zzInvalid}`);
+        return MAX_SCORE / 3;
+      }
+      */
+
       return 0;
     }
 
-    if (sharedGoodValues.length > 0) {
-      // Third argument (aka 'N times') is >= 0
-      return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);
-    }
+
 
     if (hitScore === MAX_SCORE) {
       // -1 is needed to make the third argument >= 0 (otherwise min val would be 0)
@@ -90,41 +103,34 @@ export default () => ({
 
     // No match:
 
-    if (goodValuesA.length === 0 || goodValuesB === 0) { // At least one record did not have any good ISBNs, so not penalizing here! (Invalid 020$as are counted a bad.)
+    if (aValidValuesA.length + aInvalidValuesA.length === 0 || aValidValuesB.length + aInvalidValuesB.length === 0) { // At least one record did not have any good ISBNs, so not penalizing here! (Invalid 020$as are counted a bad.)
       return 0.0;
     }
+    // We have same matching invalid identifiers. Don't penalize:
+    const allA = [...aValidValuesA, ...aInvalidValuesA, ...zValidValuesA, ...zInvalidValuesA];
+    const allB = [...aValidValuesB, ...aInvalidValuesB, ...zValidValuesB, ...zInvalidValuesB];
+    const [sharedValues, tmp1, tmp2] = getUnionData(allA, allB);
+    debug(`WHATEVER\tBOTH: ${sharedGoodValues.length}, A only: ${tmp1.length}, B only: ${tmp2.length}`);
 
-    return -0.75; // Has good ISBNs on both records, but they did not match
-
-
-    function getSubfieldCodes(tag) {
-      if (tag === '773') {
-        return ['z', undefined];
-      }
-      return ['a', 'z'];
-    }
-
-    function looksGood(val) {
-      // isbn10 can end in X:
-      if (/^([0-9]-?){9}[0-9X]$/u.test(val)) {
-        return true;
-      }
-      // isbn13 can not:
-      if (/^([0-9]-?){12}[0-9]$/u.test(val)) {
-        return true;
-      }
-      return false;
+    if (sharedValues.length > 0) {
+      return 0;
     }
+    // We have values but they disagree:
+    return -0.75; // Has good ISBNs on both records, but they did not match
 
     function getValues(fields) {
+      // Valid values are normalized to their isbn-13 form. Invalid values get their '-'s removed.
       const goodValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForGoodValues)).map(sf => validatorAndNormalizer(sf.value));
       const trueGoodValues = goodValues.filter(val => val.valid).map(val => val.value);
       const wannabeGoodValues = goodValues.filter(val => !val.valid).map(val => val.value);
       if (!subfieldCodeForBadValues) { // 773
-        return [trueGoodValues, wannabeGoodValues];
+        return [trueGoodValues, wannabeGoodValues, [], []];
       }
-      const badValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForBadValues)).map(sf => sf.value);
-      return [uniqArray(trueGoodValues), uniqArray([...wannabeGoodValues, ...badValues])];
+      const badValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForBadValues)).map(sf => validatorAndNormalizer(sf.value));
+      const validBadValues = badValues.filter(val => val.valid).map(val => val.value);
+      const invalidBadValues = badValues.filter(val => !val.valid).map(val => val.value);
+      //const badValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForBadValues)).map(sf => validatorAndNormalizer(sf.value).value);
+      return [uniqArray(trueGoodValues), uniqArray(wannabeGoodValues), uniqArray(validBadValues), uniqArray(invalidBadValues)];
     }
 
     function validatorAndNormalizer(string) {

package/src/match-detection/features/bib/issn.js

@@ -1,18 +1,5 @@
-/*
-//import createInterface from './standard-identifier-factory.js';
-
-
-export default () => {
-  const {extract, compare} = createInterface({pattern: /^022$/u, subfieldCodes: ['a', 'z', 'y']});
-  return {extract, compare, name: 'ISSN'};
-};
-*/
-
-
 import createDebugLogger from 'debug';
 
-import {isComponentRecord} from '@natlibfi/melinda-commons';
-
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:issn');
 const debugData = debug.extend('data');
 
@@ -21,18 +8,16 @@ export default () => ({
   extract: ({record/*, recordExternal*/}) => {
     //const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
 
-    const isHost = !isComponentRecord(record, false, []);
-
-    // debug(`\t Is HOST: '${isHost}'`);
     return getIssns();
 
     function getIssns() {
-      const fields = getRelevantFields();
+      const fields = record.get(/^022$/u);
       if (fields.length === 0) {
         return [];
       }
+
       debug(`\t ${fields.length} potential ISSN fields (${fields[0].tag})`);
-      const subfieldCodes = getRelevantSubfieldCodes();
+      const subfieldCodes = ['a', 'y', 'z']
       //debug(`\t subfield codes: '${subfieldCodes.join("', '")}'`);
       const subfieldValues = fields.flatMap(f => f.subfields.filter(sf => subfieldCodes.includes(sf.code)).map(sf => sf.value));
       //debug(`\t cand values: '${subfieldValues.join("', '")}'`);
@@ -44,13 +29,6 @@ export default () => ({
       return uniqArray(validSubfieldValues);
     }
 
-    function getRelevantFields() {
-      if (isHost) {
-        return record.get(/^022$/u);
-      }
-      return record.get(/^[79]73$/u);
-    }
-
     function normalizeSubfieldValue(val) {
       return val.replace(/[., :;].*$/u, '');
     }
@@ -58,13 +36,6 @@ export default () => ({
     function isValidIssn(val) {
       return /^[0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9X]$/u.test(val);
     }
-
-    function getRelevantSubfieldCodes() {
-      if (isHost) {
-        return ['a', 'y', 'z'];
-      }
-      return ['x'];
-    }
   },
   // eslint-disable-next-line max-statements
   compare: (aa, bb) => {

package/src/match-detection/features/bib/language.js

@@ -31,10 +31,11 @@ export default () => ({
 
     FixSami041().fix(clonedRecord); // Handle 'smi' adding if needed
     Remove041zxx().fix(clonedRecord); // Remove 'zxx' from f041s
+    clonedRecord.fields = clonedRecord.fields.filter(f => ['008', '041'].includes(f.tag));
     // Add other language code normalizations?
 
     // Should we return all the fields, or just the relevant fields?
-    return [{leader: clonedRecord.leader, fields: clonedRecord.fields}, label];
+    return [clonedRecord.fields, label];
   },
   // eslint-disable-next-line max-statements
   compare: (aa, bb) => {
@@ -50,9 +51,6 @@ export default () => ({
       if (score > 0.1) { // Keeps the original max, we might have 0.10 (041) + 0.05 (008/35-37) = 0.15 now
         return 0.1;
       }
-      if (score < -1) {
-        return -1.0;
-      }
       return score;
     }
 
@@ -73,7 +71,7 @@ export default () => ({
         return 0.05;
       }
 
-      if (a008 === 'mul' || b008 === 'mul') {
+      if (a008 === 'mul' || b008 === 'mul') { // We'll let 041 decide
         return 0.0;
       }
 
@@ -87,6 +85,7 @@ export default () => ({
     function compare041() {
       const a041s = getFields041(a);
       const b041s = getFields041(b);
+
       if (a041s.length === 0 || b041s.length === 0) {
         return 0.0; // Should we punish these for badness?
       }
@@ -97,14 +96,14 @@ export default () => ({
         const scoreInd1 = mainPenalty === 0.0 ? 0.0 : indicator1Penalty(a041s[0], b041s[0]);
         return mainPenalty + sourcePenalty + scoreInd1;
       }
-      // Things are already complicated (likely to pe penalized, so no nedd to worrty about language code sources, I daresay.
+      // Things are already complicated (likely to pe penalized, so no nedd to worry about language code sources, I daresay.
       return compareLanguageCodeLists(getLanguageCodesFrom041Fields(a), getLanguageCodesFrom041Fields(b));
 
-      function getFields041(record) {
-        if (!record.fields) {
+      function getFields041(fields) {
+        if (!fields) {
           return [];
         }
-        return record.fields.filter(f => f.tag === '041');
+        return fields.filter(f => f.tag === '041');
       }
 
     }
@@ -139,7 +138,7 @@ export default () => ({
     }
 
     function compareLanguageCodeLists(a, b) {
-      if (a.length === 0 || b.length === 0) {
+      if (a.length === 0 || b.length === 0 || a.every(val => containsNoData(val)) || b.every(val => containsNoData(val))) {
         debugData(`No language to compare`);
         return 0;
       }
@@ -176,19 +175,9 @@ export default () => ({
 
 
       // Damage control:
-
-      // Not using the generic solution here as eg. 'und' needs a special treatment
-      const sharedValues = getSharedValues(a, b);
-      const aOnly = a.filter(val => !sharedValues.includes(val));
-      const bOnly = b.filter(val => !sharedValues.includes(val));
-      const hasUnd = [...aOnly, ...bOnly].includes('und');
+      const sharedValues = getSharedValues(a, b).filter(val => !containsNoData(val));
 
       if (sharedValues.length < 1) {
-        console.info(`NV: ${sharedValues.join(", ")}`);
-        if (aOnly.length === 1 && bOnly.length === 1 && hasUnd) {
-          debug(`Both have languages, but none of these match. However, the benefit of doubt is given: '${aOnly[0]}' and '${bOnly[0]}' might mean the same}`);
-          return 0;
-        }
         debug(`Both have languages, but none of these match.`); // includes 'mul' vs a single language code
         return -1.0;
       }
@@ -218,11 +207,11 @@ export default () => ({
 
 });
 
-function get008Value(record, label = 'record') {
-  if (!record || !record.fields) {
+function get008Value(fields, label = 'record') {
+  if (!fields) {
     return;
   }
-  const f008 = record.fields.find(f => f.tag === '008');
+  const f008 = fields.find(f => f.tag === '008');
   if (!f008) {
     return undefined;
   }
@@ -269,12 +258,12 @@ function isLangCodeForALanguage(code, label = 'record', encoding = undefined) {
 
 
 
-function getLanguageCodesFrom041Fields(record) {
+function getLanguageCodesFrom041Fields(fields) {
   // NB! We brutally don't check $2 (language code source) as marc's language codes is practically a subset of ISO 639-2,
   // and also ISO 639-2 and ISO 639-3 overlap to a degree. If we ever run into a trouble with some ISO 639-2 vs 639-3 mismatch, then we'll work it out.
   // Also we could write a validator that converts ISO 639-2 to marc if applicable and maybe even partial support for ISO-639-3.
   // Note that ISO 639-2-B values correspond with marc beteer that ISO 639-2-T. Eg. code for Chinese 'zho' should/code be normalized to 'chi'!
-  const values = record.fields.filter(f => f.tag === '041').flatMap(f => getFieldsLanguageCodes(f));
+  const values = fields.filter(f => f.tag === '041').flatMap(f => getFieldsLanguageCodes(f));
   /*
         // .filter(({ind2}) => ind2 === ' ')
         .map(({subfields}) => subfields)

package/src/match-detection/features/bib/publication-time-allow-cons-years.js

@@ -1,6 +1,7 @@
 
 import createDebugLogger from 'debug';
 import {testStringOrNumber} from '../../../matching-utils.js';
+import {matchingYears} from './publication-time.js';
 
 // We should also get copyright time and copyright/publication times from 26x
 
@@ -16,14 +17,16 @@ export default () => ({
 
     const [firstA] = a;
     const [firstB] = b;
+    // Sanity check: If either of years is a non string/number, values are not comparable
+    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {
+      return 0.0;
+    }
 
     if (firstA === firstB) {
       return 0.1;
     }
-
-    // If either of years is a non string/number, values are not comparable
-    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {
-      return 0;
+    if (matchingYears(firstA, firstB)) { // Handle 'u' Eg. '18uu' vs '1812'
+      return 0.05;
     }
 
     const firstANumber = parseInt(firstA, 10);