@natlibfi/melinda-record-matching 5.0.3 → 5.0.5-alpha.2

package/src/match-detection/features/bib/publication-time.js

@@ -1,18 +1,23 @@
+// We should also get copyright time and copyright/publication times from 26x
+// see publication-time-allow-cons-years for a version allowing consequent years to match
 
+import {extractPublicationYearFrom773} from '../../../candidate-search/query-list/component.js';
+
+const MAX = 0.1;
+const MIN = -1.0;
 
-import {testStringOrNumber} from '../../../matching-utils.js';
 
-// We should also get copyright time and copyright/publication times from 26x
-// see publication-time-allow-cons-years for a version allowing consequent years to match
 
 export default () => ({
   name: 'Publication time',
   extract: ({record}) => {
-    return getDateData();
-    //const value = record.get(/^008$/u)?.[0]?.value || undefined;
-    //return testStringOrNumber(value) ? [String(value).slice(6, 15)] : [];
+    let dateData = getDataFrom008();
+    const [f773] = record.get(/^773$/u);
+    const hostYear = f773 && extractPublicationYearFrom773(f773) || null;
+    dateData.hostYear = hostYear;
+    return dateData;
 
-    function getDateData() {
+    function getDataFrom008() {
       const [f008] = record.get(/^008$/u);
       if (!f008 || f008.value.length < 16) {
         return splitDateData('|        ');
@@ -29,50 +34,91 @@ export default () => ({
   },
 
   compare: (aa, bb) => {
-    if (aa.typeOfDate === 'b') { // Berfore Christ. No really makes sense in our domain, though.
+    // Be happy with a f773$g match:
+    if (aa.hostYear && bb.hostYear && aa.hostYear === bb.hostYear) {
+      return MAX;
+    }
+    // Check 008
+    if (aa.typeOfDate === 'b') { // 008/06: Before Christ. No really makes sense in our domain, though.
       if (bb.typeOfDate === 'b') {
         return 0;
       }
-      return -1.0;
+      return MIN;
     }
     if (aa.typeOfDate === 'n' || bb.typeOfDate === 'n') { // n=unknown
       return 0;
     }
 
+    // Try to handle questionable dates:
+    if (aa.typeOfDate === 'q') { // questionable data
+      if (bb.typeOfDate !== 'q') {
+        if (yearInBetween(aa.date1, bb.date1, aa.date2)) {
+          return MAX;
+        }
+        return MIN;
+      }
+
+      // What if there are questionable dates on both sides?
+      if (aa.date1 === bb.date1 && bb.date1 === bb.date2) {
+        return MAX;
+      }
+      // Lazily return 0. (We could check for overlap etc. but not really worth the effort)
+      return 0.0;
+    }
+    if (bb.typeOfDate === 'q') {
+      if (yearInBetween(bb.date1, aa.date1, bb.date2)) {
+        return MAX;
+      }
+      return MIN;
+    }
+
     const skipList = ['    ', '||||', 'uuuu'];
     if (skipList.includes(aa.date1) || skipList.includes(bb.date1)) { // 008/07-10 carries no information
       return 0;
     }
 
     if (matchingYears(aa.date1, bb.date1)) { // 'u' support
-      return 0.1;
+      return MAX;
     }
 
-    // TODO: add $q support here
-
-    return -1.0;
-
-    function matchingYears(yyyy1, yyyy2) {
-      if (yyyy1.length === 0) { // All digits have been succesfully consumed -> success
-        return true;
-      }
-      if (yyyy1[0] === 'u' || yyyy2[0] === 'u') { // Ignore 'u' (it refers to unknown millenia, century, decade or year)
-        return matchingYears(yyyy1.slice(1), yyyy2.slice(1));
-      }
-      if (yyyy1[0] !== yyyy2[0]) {
-        return false;
+    if (isValidYear(aa.date1) && bb.date1) {
+      const difference = Math.abs(aa.date1 - bb.date1);
+      if (difference === 1) {
+        return MIN/8;
       }
-      // Should we require that yyyy[0] is a digit at this point?
-      return matchingYears(yyyy1.slice(1), yyyy2.slice(1));
     }
 
-    /*
-    function hasFourDigits(yyyy) { // Will be needed by 'q' support
-      return yyyy.match(/^[0-9]{4}$/u);
-    }
-      */
+    return MIN;
   }
 
 });
 
+function yearInBetween(start, curr, end) {
+  if (!isValidYear(start) || !isValidYear(curr) || !isValidYear(end)) {
+    return false;
+  }
+  return start <= curr && curr <= end;
+}
+
+export function matchingYears(yyyy1, yyyy2) {
+  if (yyyy1.length === 0) { // All digits have been succesfully consumed -> success
+    return true;
+  }
+  if (yyyy1[0] === 'u' || yyyy2[0] === 'u') { // Ignore 'u' (it refers to unknown millenia, century, decade or year)
+    return matchingYears(yyyy1.slice(1), yyyy2.slice(1));
+  }
+  if (yyyy1[0] !== yyyy2[0]) {
+    return false;
+  }
+  if (yyyy1[0] >= '0' && yyyy1[0] <= '9') { // Require that yyyy[0] is a digit at this point? (What if year is 983?)
+    return matchingYears(yyyy1.slice(1), yyyy2.slice(1));
+  }
+  return false;
+}
+
+
+const validYearRegexp = /^(?:1[89][0-9][0-9]|20[012][0-9])$/u;
 
+function isValidYear(yyyy) {
+  return validYearRegexp.test(yyyy); // Currently supports 1800-2029
+}

package/src/match-detection/features/bib/title.js

@@ -3,19 +3,28 @@ import naturalPkg from 'natural';
 const {LevenshteinDistance: leven} = naturalPkg;
 import {testStringOrNumber} from '../../../matching-utils.js';
 
-const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');
-const debugData = debug.extend('data');
 
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');
+const debugData = debug.extend('data');
 
-export default ({threshold = 10} = {}) => ({
+export default ({threshold = 0.9} = {}) => ({
   name: 'Title',
   extract: ({record, recordExternal}) => {
     const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
-    const title = getTitle();
-    debug(`${label}: title: ${title}`);
+    const a = getTitle(record, ['a']);
+    debug(`${label}: title: ${a}`);
+
+    if (a) {
+      const b = normalizeTitle(getTitle(record, ['b'])) || '';
+      const n = normalizeTitle(getTitle(record, ['n'])) || '';
+      const p = normalizeTitle(getTitle(record, ['p'])) || '';
+      return [normalizeTitle(a), b, n, p];
+    }
+
+    return ['', '', '', ''];
 
-    if (testStringOrNumber(title)) {
-      const titleAsNormalizedString = String(title)
+    function normalizeTitle(title) {
+      return title
         // decompose unicode diacritics
         .normalize('NFD')
         // strip non-letters/numbers
@@ -24,48 +33,117 @@ export default ({threshold = 10} = {}) => ({
         // - see validator normalize-utf8-diacritics for details
         .replace(/[^\p{Letter}\p{Number}]/gu, '')
         .toLowerCase();
-      debug(`${label}: titleString: ${titleAsNormalizedString}`);
-      return [titleAsNormalizedString];
     }
 
-    return [];
-
-    function getTitle() {
-      const [field] = record.get(/^245$/u);
-      debugData(`${label}: titleField: ${JSON.stringify(field)}`);
-
-      if (field) {
-        return field.subfields
-          // get also $n:s and $p:s here
-          .filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))
-          .map(({value}) => testStringOrNumber(value) ? String(value) : '')
-          .join('');
-      }
-      return false;
-    }
   },
   compare: (a, b) => {
-    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');
-    const distance = leven(a[0], b[0]);
+    const [aa, ab, an, ap] = a;
+    const [ba, bb, bn, bp] = b;
+
+    if (isEmpty(aa) || isEmpty(ba)) {
+      return -1.0;
+    }
+    // F245$n information is critical; it can not mismatch at all:
+    if (an.length && bn.length && an !== bn) { // If these exists, they must be the same (we might convert Roman numbers to Arabic numbers though)
+      return -1.0;
+    }
+
+    const aFull = toFullTitle(a);
+    const bFull = toFullTitle(b);
+
+    const [distance, maxLength, correctness] = doLevenshtein(aFull, bFull);
+
+    debug(`'${aFull}' vs '${bFull}': Max length = ${maxLength}, distance = ${distance}, correctness = ${correctness}`);
 
     if (distance === 0) {
       return 0.5;
     }
 
-    const maxLength = getMaxLength();
-    const percentage = distance / maxLength * 100;
+    if (correctness >= threshold) {
+      return 0.4;
+    }
+
+    if (an && bn) {
+      // There seems to be some wobble between $b and $p, for example:
+      if (ab && isEmpty(ap) && bp && isEmpty(bb)) {
+        return compare([aa, ap, an, ab], [ba, bb, bn, bp]);
+      }
+      if (ap && isEmpty(ab) && bb && isEmpty(bp)) {
+        return compare([aa, ab, an, ap], [ba, bp, bn, bb]);
+      }
+    }
+
+    // Try the same without $p:
+    if (localXor(ap, bp)) {
+      const result = compare([aa, ab, an, ''], [ba, bb, bn, '']);
+      return result > 0.0 ? result * 0.8 : result;
+    }
+
+    if (isEmpty(ap) && isEmpty(bp) && localXor(ab, bb)) {
+      // Try the same without $b ($p is not here)
+      const result = compare([aa, '', an, ''], [ba, '', bn, '']);
+      return result > 0.0 ? result * 0.8 : result;
+    }
+
+    return -0.5; // Not likely
+
+    function isEmpty(x) {
+      return !x || x.length === 0;
+    }
 
-    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
+    function localXor(x, y) {
+      if (isEmpty(x)) {
+        return !isEmpty(y);
+      }
+      // 'x' exists, thus 'y' can not exist:
+      return isEmpty(y);
+    }
 
-    if (percentage <= threshold) {
-      return 0.3;
+    function doLevenshtein(string1, string2) {
+      const distance = leven(string1, string2);
+      const len = getMaxLength(string1, string2);
+      const correctness = 1.0 - (distance / len);
+      return [distance, len, correctness];
     }
 
-    return -0.5;
+    function toFullTitle(arr) {
+      const relevant = arr.filter(val => typeof val === 'string' && val.length);
+      return relevant.join(' ');
+    }
 
-    function getMaxLength() {
-      return a[0].length > b[0].length ? a[0].length : b[0].length;
+    function getMaxLength(str1, str2) {
+      return str1.length > str2.length ? str1.length : str2.length;
     }
 
   }
 });
+
+export function getTitle(record, targetSubfieldCodes = ['a', 'b', 'n', 'p']) {
+  const [field] = record.get(/^245$/u);
+  debugData(`titleField: ${JSON.stringify(field)}`);
+
+  if (field) {
+    const title = field.subfields
+      // get also $n:s and $p:s here
+      .filter(({code}) => targetSubfieldCodes.includes(code))
+      // Would be nice to normalize $n values...
+      .map(({value}) => testStringOrNumber(value) ? String(value) : '')
+      .join(' ')
+      .replace(/\[[^\]]*\]/ug, ' ') // Remove [whatever] stuff. ADD TEST FOR THIS
+      .replace(/ [=\/:](?:$| )/ug, ' ') // Strip punctuation
+      // Also đ vs d pairs seen:
+      //.replace(/(?:đ|ȧ|t̄|ǩ|ǧ|s̆|c̆)/ug, '') // Hack. Saamelaisbibliografia has often dropped this wierd characters (oft old articles, no longer used in sami either)
+      // trim:
+      .replace(/ +/ug, ' ')
+      .replace(/^ +/u, '')
+      .replace(/ +$/u, '');
+
+    // Skip non-filing indicator (note that '9' is a magic indicator value, so we don't do it):
+    if (/^[1-8]$/u.test(field.ind2)) { // Skip non-filing characters
+      return title.slice(parseInt(field.ind2));
+    }
+    return title;
+
+  }
+  return false;
+}

package/src/match-detection/index.js

@@ -5,7 +5,8 @@ import * as features from './features/index.js';
 export {features};
 
 export default ({strategy, threshold = 0.8999}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
-  const minProbabilityQuantifier = 0.5;
+  const MIN_PROPABILITY_QUALIFIER = 0.4; // Title perfect match: 0.5
+  const FORCE_FAIL = -1.0;
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');
   const debugData = debug.extend('data');
@@ -41,13 +42,18 @@ export default ({strategy, threshold = 0.8999}, returnStrategy = false, localSet
     const featurePairs = generateFeaturePairs(featuresA, featuresB);
     const similarityVector = generateSimilarityVector(featurePairs);
 
-    if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
+    if (similarityVector.some(v => v >= MIN_PROPABILITY_QUALIFIER)) {
       const probability = calculateProbability(similarityVector);
       debug(`probability: ${probability} (Threshold: ${threshold})`);
+      if (similarityVector.some(v => v <= FORCE_FAIL)) {
+        debug(`Some feature resulted in utter failure (${FORCE_FAIL} or less)`);
+        return returnResult({match: false, probability: 0.0});
+      }
+
       return returnResult({match: probability >= threshold, probability});
     }
 
-    debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
+    debugData(`No feature yielded minimum probability amount of points (${MIN_PROPABILITY_QUALIFIER})`);
     return returnResult({match: false, probability: 0.0});
   }
 

package/src/matching-utils.js

@@ -112,3 +112,20 @@ export function getMatchCounts(aValues, bValues) {
     return aToB < bToA ? aToB : bToA;
   }
 }
+
+
+export function stringAfter(string, substring) {
+  const pos = string.indexOf(substring);
+  if (pos === -1) {
+    return string;
+  }
+  return string.substring(pos+substring.length);
+}
+
+export function stringBefore(string, substring) {
+  const pos = string.indexOf(substring);
+  if (pos === -1) {
+    return string;
+  }
+  return string.substring(0, pos);
+}

package/dist/match-detection/features/bib/title-version-original.js

@@ -1,37 +0,0 @@
-import createDebugLogger from "debug";
-import naturalPkg from "natural";
-const { LevenshteinDistance: leven } = naturalPkg;
-export default ({ threshold = 10 } = {}) => ({
-  name: "titleVersionOriginal",
-  extract: ({ record }) => {
-    const title = getTitle();
-    if (title) {
-      return [title.replace(/[^\p{Letter}\p{Number}]/gu, "").toLowerCase()];
-    }
-    return [];
-    function getTitle() {
-      const [field] = record.get(/^245$/u);
-      if (field) {
-        return field.subfields.filter(({ code }) => ["a", "b"].includes(code)).map(({ value }) => value).join("");
-      }
-    }
-  },
-  compare: (a, b) => {
-    const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original");
-    const distance = leven(a[0], b[0]);
-    if (distance === 0) {
-      return 0.5;
-    }
-    const maxLength = getMaxLength();
-    const percentage = distance / maxLength * 100;
-    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
-    if (percentage <= threshold) {
-      return 0.3;
-    }
-    return -0.5;
-    function getMaxLength() {
-      return a[0].length > b[0].length ? a[0].length : b[0].length;
-    }
-  }
-});
-//# sourceMappingURL=title-version-original.js.map

package/dist/match-detection/features/bib/title-version-original.js.map

@@ -1,7 +0,0 @@
-{
-  "version": 3,
-  "sources": ["../../../../src/match-detection/features/bib/title-version-original.js"],
-  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\n\nexport default ({threshold = 10} = {}) => ({\n  name: 'titleVersionOriginal',\n  extract: ({record}) => {\n    const title = getTitle();\n\n    if (title) {\n      return [title.replace(/[^\\p{Letter}\\p{Number}]/gu, '').toLowerCase()];\n    }\n\n    return [];\n\n    function getTitle() {\n      const [field] = record.get(/^245$/u);\n\n      if (field) {\n        return field.subfields\n          .filter(({code}) => ['a', 'b'].includes(code))\n          .map(({value}) => value)\n          .join('');\n      }\n    }\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');\n    const distance = leven(a[0], b[0]);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    const maxLength = getMaxLength();\n    const percentage = distance / maxLength * 100;\n\n    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);\n\n    if (percentage <= threshold) {\n      return 0.3;\n    }\n\n    return -0.5;\n\n    function getMaxLength() {\n      return a[0].length > b[0].length ? a[0].length : b[0].length;\n    }\n\n  }\n});\n"],
-  "mappings": "AACA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AAErC,eAAe,CAAC,EAAC,YAAY,GAAE,IAAI,CAAC,OAAO;AAAA,EACzC,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,UAAM,QAAQ,SAAS;AAEvB,QAAI,OAAO;AACT,aAAO,CAAC,MAAM,QAAQ,6BAA6B,EAAE,EAAE,YAAY,CAAC;AAAA,IACtE;AAEA,WAAO,CAAC;AAER,aAAS,WAAW;AAClB,YAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AAEnC,UAAI,OAAO;AACT,eAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,CAAC,KAAK,GAAG,EAAE,SAAS,IAAI,CAAC,EAC5C,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,KAAK,EAAE;AAAA,MACZ;AAAA,IACF;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,QAAQ,kBAAkB,uFAAuF;AACvH,UAAM,WAAW,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;AAEjC,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,aAAa;AAC/B,UAAM,aAAa,WAAW,YAAY;AAE1C,UAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,SAAS,gBAAgB,QAAQ,kBAAkB,UAAU,EAAE;AAEvG,QAAI,cAAc,WAAW;AAC3B,aAAO;AAAA,IACT;AAEA,WAAO;AAEP,aAAS,eAAe;AACtB,aAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;AAAA,IACxD;AAAA,EAEF;AACF;",
-  "names": []
-}

package/src/match-detection/features/bib/title-version-original.js

@@ -1,52 +0,0 @@
-
-import createDebugLogger from 'debug';
-import naturalPkg from 'natural';
-const {LevenshteinDistance: leven} = naturalPkg;
-
-export default ({threshold = 10} = {}) => ({
-  name: 'titleVersionOriginal',
-  extract: ({record}) => {
-    const title = getTitle();
-
-    if (title) {
-      return [title.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
-    }
-
-    return [];
-
-    function getTitle() {
-      const [field] = record.get(/^245$/u);
-
-      if (field) {
-        return field.subfields
-          .filter(({code}) => ['a', 'b'].includes(code))
-          .map(({value}) => value)
-          .join('');
-      }
-    }
-  },
-  compare: (a, b) => {
-    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');
-    const distance = leven(a[0], b[0]);
-
-    if (distance === 0) {
-      return 0.5;
-    }
-
-    const maxLength = getMaxLength();
-    const percentage = distance / maxLength * 100;
-
-    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
-
-    if (percentage <= threshold) {
-      return 0.3;
-    }
-
-    return -0.5;
-
-    function getMaxLength() {
-      return a[0].length > b[0].length ? a[0].length : b[0].length;
-    }
-
-  }
-});