@natlibfi/marc-record-validators-melinda 12.0.0-alpha.6 → 12.0.0-alpha.8

package/src/ending-punctuation-conf.js

@@ -1,10 +1,11 @@
 // Const finnishTerms = ['ysa', 'yso', 'kassu', 'seko', 'valo', 'kulo', 'puho', 'oiko', 'mero', 'liito', 'fast', 'allars', 'kaunokki'];
 const finnishTerms = [
   /^(?:allars|bella|fast|juho|jupo|kassu|kauno|kaunokki|keko|koko|kulo|liiko|liito|local|mero|mts|musa|oiko|puho|seko|woto|valo|ysa|yso)$/u,
-  /^(?:kauno|slm|yso)\//u
+  /^(?:kauno|slm|yso)\//u // <= yso/* etc
 ];
 
-const validPuncMarks = '?"-!,)]';
+const validPuncMarks = '?-!.'; // NB! ')' and ']' are only valid for some fields!
+const validQuoteChars = "\"'";
 // Configuration specification
 const confSpec = [
   { // 010-035 EI
@@ -500,7 +501,7 @@ const confSpec = [
     index: null,
     punc: false,
     special: {
-      termField: '2',
+      termSubfieldCode: '2',
       finnishTerms,
       else: true
     }
@@ -516,7 +517,7 @@ const confSpec = [
     index: null,
     punc: false,
     special: {
-      termField: '2',
+      termSubfieldCode: '2',
       finnishTerms,
       else: true
     }
@@ -666,4 +667,4 @@ const confSpec = [
   }
 ];
 
-export {finnishTerms, validPuncMarks, confSpec};
+export {finnishTerms, validPuncMarks, validQuoteChars, confSpec};

package/src/ending-punctuation.js

@@ -27,7 +27,7 @@
 */
 
 // Import {validPuncMarks, finnishTerms, confSpec} from './ending-punctuation-conf.js';
-import {validPuncMarks, finnishTerms, confSpec} from './ending-punctuation-conf.js';
+import {validPuncMarks, validQuoteChars, finnishTerms, confSpec} from './ending-punctuation-conf.js';
 import createDebugLogger from 'debug';
 
 const debug = createDebugLogger('@natlibfi/marc-record-validator-melinda/ending-punctuation');
@@ -65,40 +65,65 @@ export default function () {
 
 // Field validation with punctuation rules for normal and special cases in subfunction (to reduce complexity to please travisci)
 function validateField(field, linkedTag, fix, message) {
-  // This is used to find last subfield that should have punctuation
-  function findLastSubfield(field) {
-    const subfields = field.subfields.filter(sf => isNaN(sf.code) && 'value' in sf);
-    return subfields.slice(-1).shift();
+  function getDefaultPuncMarks(tag) {
+    if (tag.match(/^[1678](?:00|10|11|30)/u) || tag === '740') { // As defined in Loppupisteohje
+      return `${validPuncMarks})`;
+    }
+    // We don't want ').' here either. However, Loppupisteohje is a bit iffy here.
+    // BUG: Note that our generic rules will remove dot from Finnish terms such as https://finto.fi/yso-aika/fi/page/p1069910600
+    if (['647', '648', '650', '651', '654', '655', '656', '657', '658', '662'].includes(tag)) {
+       return `${validPuncMarks})`;
+    }
+    if(['260'].includes(tag)) {
+      return `${validPuncMarks})]`;
+    }
+    return validPuncMarks;
   }
 
   // Punctuation rule (Boolean), Check no ending dot strict (Boolean)
   function normalPuncRules(subfield, punc, tag, checkEnd, overrideValidPuncMarks) {
-    const puncMarks = overrideValidPuncMarks || validPuncMarks;
-    const lastPuncMark = puncMarks.includes(subfield.value.slice(-1)); // If string ends to punctuation char
-    const lastPuncDot = '.'.includes(subfield.value.slice(-1)); // If string ends to dot
+    const puncMarks = overrideValidPuncMarks || getDefaultPuncMarks(tag);
+    const lastChar = subfield.value.slice(-1);
+    const lastPuncMark = puncMarks.includes(lastChar); // If string ends to punctuation char
+    const lastPuncDot = '.'.includes(lastChar); // If string ends to dot
+    const penultimateCharacter = subfield.value.length >= 2 ? subfield.value.charAt(subfield.value.length - 2) : undefined;
+    const antePenultimateCharacter = subfield.value.length >= 3 ? subfield.value.charAt(subfield.value.length - 3) : undefined;
+
 
-    // Last char should be punc, but its not one of marks nor dot
+    // Last char should be punc, but it's not one of listed punctuation marks nor dot
     if (punc && !(lastPuncMark || lastPuncDot)) {
-      // Console.log("1. Invalid punctuation - missing")
-      message.message.push(`Field ${tag} has invalid ending punctuation`);
-      if (fix) {
-        subfield.value = subfield.value.concat('.');
-        message.fix.push(`Field ${tag} - Added punctuation to $${subfield.code}`);
+      //console.log(puncMarks)
+      if (penultimateCharacter && validQuoteChars.includes(lastChar) && puncMarks.includes(penultimateCharacter)) {
+        // Exception: do nothing! Ending in punc+quote combo is all right, and does not imply a missing punc
+      }
+      else {
+        // Console.log("1. Invalid punctuation - missing")
+        message.message.push(`Field ${tag} requires ending punctuation, ends in '${lastChar}'`);
+        if (fix) {
+          subfield.value = subfield.value.concat('.');
+          message.fix.push(`Field ${tag} - Added punctuation to $${subfield.code}`);
+        }
       }
 
       // Last char is dot, but previous char is one of punc marks, like 'Question?.'
-    } else if (lastPuncDot && subfield.value.length > 1 && puncMarks.includes(subfield.value.charAt(subfield.value.length - 2))) {
+    } else if (lastPuncDot && penultimateCharacter && puncMarks.includes(penultimateCharacter)) {
       // Console.log("2. Invalid punctuation - duplicate, like '?.'")
-      message.message.push(`Field ${tag} has invalid ending punctuation`);
+      message.message.push(`Field ${tag} has an extra dot after '${penultimateCharacter}'`);
       if (fix) {
         subfield.value = subfield.value.slice(0, -1);
-        message.fix.push(`Field ${tag} - Removed double punctuation from $${subfield.code}`);
+        message.fix.push(`Field ${tag} - Removed dot after punctuation from $${subfield.code}`);
+      }
+      // Last char is dot, but previous two cars are punc+quote, like 'Lorum "Ipsum.".'
+    } else if (lastPuncDot && antePenultimateCharacter && validQuoteChars.includes(penultimateCharacter) && puncMarks.includes(antePenultimateCharacter)) {
+      message.message.push(`Field ${tag} has an extra dot in '${antePenultimateCharacter}${penultimateCharacter}${lastChar}'`);
+      if (fix) {
+        subfield.value = subfield.value.slice(0, -1);
+        message.fix.push(`Field ${tag} - Removed '${lastChar}' after '${antePenultimateCharacter}${penultimateCharacter}'`);
       }
-
       // Last char shouldn't be dot !! This is behind checkEnd boolean, because of dots at end of abbreviations, so this is checked only in special cases !!//
     } else if (checkEnd && (!punc && lastPuncDot)) {
       // Console.log("3. Invalid punctuation - Shouldn't be dot, is")
-      message.message.push(`Field ${tag} has invalid ending punctuation`);
+      message.message.push(`Field ${tag} has unwanted ending punctuation '${lastChar}'`);
       if (fix) {
         subfield.value = subfield.value.slice(0, -1);
         message.fix.push(`Field ${tag} - Removed punctuation from $${subfield.code}`);
@@ -136,7 +161,7 @@ function validateField(field, linkedTag, fix, message) {
         if (res.special.ifInd2 && res.special.ifInd2.includes(field.ind2)) {
           normalPuncRules(lastSubField, res.special.ifBoth, tag, true, res.special.ifLastCharNot);
 
-          // Matches execption to special rule, noPuncIfInd2 (likely with value 4, that indicates copyright mark)
+          // Matches exception to special rule, noPuncIfInd2 (likely with value 4, that indicates copyright mark)
         } else if (res.special.noPuncIfInd2 && field.ind2 && res.special.noPuncIfInd2.includes(field.ind2)) {
           normalPuncRules(lastSubField, !res.special.ifBoth, tag, true, res.special.ifLastCharNot);
 
@@ -158,12 +183,17 @@ function validateField(field, linkedTag, fix, message) {
       normalPuncRules(lastSubField, res.punc, tag, false, false);
 
       // Search for Finnish terms
-    } else if (res.special.termField) {
+    } else if (res.special.termSubfieldCode) {
       lastSubField = findLastSubfield(field);
 
       if (lastSubField) {
-        const languageField = field.subfields.find(({code}) => code === res.special.termField);
-        if (languageField && languageField.value && finnishTerms.some(p => p.test(languageField.value))) {
+        const lexicon = getLexicon(field, res.special.termSubfieldCode);
+        const proceed = !finnishException(field, res.special.termSubfieldCode, false);
+
+
+        //const languageField = field.subfields.find(({code}) => code === res.special.termSubfieldCode);
+        //if (languageField && languageField.value && finnishTerms.some(p => p.test(languageField.value))) {
+        if (lexicon && finnishTerms.some(p => p.test(lexicon)) && proceed) {
           // If (languageField && languageField.value && finnishTerms.indexOf(languageField.value) > -1) {
           normalPuncRules(lastSubField, res.punc, tag, true, false);
         } else {
@@ -199,6 +229,15 @@ function validateField(field, linkedTag, fix, message) {
 
       validateField(field, linkedTag, fix, message);
     }
+    // fallback
+    else {
+      debug(`special is definedm but no rule applies`);
+      const lastSubField = findLastSubfield(field);
+
+      if (lastSubField) {
+        normalPuncRules(lastSubField, res.punc, field.tag, false, false, fix, message);
+      }
+    }
   }
 
   let res = null;
@@ -232,8 +271,12 @@ function validateField(field, linkedTag, fix, message) {
     return;
   }
 
+  const forceNormal = res.special ? finnishException(field, res.special.termSubfieldCode, true) : false;
   // Normal rules
-  if (typeof res.special === 'undefined' || res.special === null) {
+  if (typeof res.special === 'undefined' || res.special === null || forceNormal) {
+    if (forceNormal) {
+      console.info("EXCEPTION. SKIP FINNISH RULES");
+    }
     lastSubField = findLastSubfield(field);
 
     if (lastSubField) {
@@ -260,3 +303,51 @@ export function validateSingleField(field, linkedTag, fix) {
   return message;
 }
 
+function getLexicon(field, subfieldCode) {
+  const languageSubfield = field.subfields.find(({code}) => code === subfieldCode); // res.special.termSubfieldCode);
+  if (!languageSubfield || !languageSubfield.value) {
+    return undefined;
+  }
+  if (finnishTerms.find(p => p.test(languageSubfield.value))) {
+    return languageSubfield.value;
+  }
+  return undefined;
+}
+
+function finnishException(field, termSubfieldCode, hasDot = true) {
+  const lexicon = getLexicon(field, termSubfieldCode);
+  if (!lexicon) {
+    return false;
+  }
+
+  const lastSubfield = findLastSubfield(field);
+  if (!lastSubfield || !lastSubfield.value) {
+    return false;
+  }
+  // Some terms can end in '.' that we want to keep
+  if (field.tag === '648') { // Yso-aika checks
+    //console.log(`Finnish Exception? '${lastSubfield.value}', '${lexicon}', '${field.tag}'`);
+    if (lexicon === 'yso/fin') { // 'eaa.' appears in prefLAbels and 'eKr.' in altLabels
+      if (hasDot) {
+        return lastSubfield.value.match(/ (?:eaa|[ej]Kr|jaa)\.$/u); // Finnish term from which the dot is not to be removed
+      }
+      return lastSubfield.value.match(/ (?:eaa|[ej]Kr)|jaa$/u); // Finnish word that needs a dot
+    }
+
+    if (lexicon === 'yso/swe') {
+      if (hasDot) {
+        return lastSubfield.value.match(/ (?:[ef]\.Kr|f\.v\.t)\.$/u);
+      }
+      return lastSubfield.value.match(/ (?:[ef]\.Kr|f\.v\.t)$/u);
+    }
+  }
+  // yso has 'MODEL.LA.' and 'Corel R.A.V.E.' but these are so rare I'm not listing them
+
+  return false;
+}
+
+  // This is used to find last subfield that should have punctuation
+  function findLastSubfield(field) {
+    const subfields = field.subfields.filter(sf => isNaN(sf.code) && 'value' in sf);
+    return subfields.slice(-1).shift();
+  }