@natlibfi/marc-record-validators-melinda 10.16.0 → 10.16.1-alpha.1

package/dist/punctuation2.js

@@ -9,6 +9,7 @@ exports.fieldGetFixedString = fieldGetFixedString;
 exports.fieldNeedsModification = fieldNeedsModification;
 exports.fieldStripPunctuation = fieldStripPunctuation;
 var _endingPunctuation = require("./ending-punctuation");
+var _subfield6Utils = require("./subfield6Utils");
 var _utils = require("./utils");
 var _clone = _interopRequireDefault(require("clone"));
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
@@ -28,14 +29,15 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 
 //const debug = createDebugLogger('debug/punctuation2');
 
+const descriptionString = 'Remove invalid and add valid punctuation to data fields';
 function _default() {
   return {
-    description: 'Add punctuation to data fields',
+    description: descriptionString,
     validate,
     fix
   };
   function fix(record) {
-    (0, _utils.nvdebug)('Add punctuation to data fields: fixer');
+    (0, _utils.nvdebug)(`${descriptionString}: fixer`);
     const res = {
       message: [],
       fix: [],
@@ -45,7 +47,7 @@ function _default() {
     return res;
   }
   function validate(record) {
-    (0, _utils.nvdebug)('Add punctuation to data fields: validate');
+    (0, _utils.nvdebug)(`${descriptionString}: validate`);
     const fieldsNeedingModification = record.fields.filter(f => fieldNeedsModification(f, true));
     const values = fieldsNeedingModification.map(f => (0, _utils.fieldToString)(f));
     const newValues = fieldsNeedingModification.map(f => fieldGetFixedString(f, true));
@@ -88,9 +90,9 @@ function fieldNeedsModification(field, add = true) {
 /////////////////////////////////////////////////////////////////////////////////////
 
 //const stripCrap = / *[-;:,+]+$/u;
-const commaNeedsPuncAfter = /(?:[a-z0-9A-Z]|å|ä|ö|Å|Ä|Ö|\))$/u;
 const defaultNeedsPuncAfter = /(?:[a-z0-9A-Z]|å|ä|ö|Å|Ä|Ö)$/u;
 const defaultNeedsPuncAfter2 = /(?:[\]a-zA-Z0-9)]|ä|å|ö|Å|Ä|Ö)$/u;
+const doesNotEndInPunc = /[^!?.:;,]$/u; // non-punc for pre-240/700/XXX $, note that '.' comes if preceded by ')'
 const blocksPuncRHS = /^(?:\()/u;
 const allowsPuncRHS = /^(?:[A-Za-z0-9]|å|ä|ö|Å|Ä|Ö)/u;
 const dotIsProbablyPunc = /(?:[a-z0-9)]|å|ä|ö|(?:[A-Za-z0-9]|Å|Ä|Ö)(?:[A-Z]|Å|Ä|Ö))\.$/u;
@@ -103,8 +105,8 @@ const removeColons = {
   'remove': / *[;:]$/u
 };
 const removeX00Comma = {
-  'code': 'abcqde',
-  'followedBy': 'abcqde#',
+  'code': 'abcdenqt',
+  'followedBy': 'abcdenqtv#',
   'context': /.,$/u,
   'remove': /,$/u
 };
@@ -138,6 +140,14 @@ const cleanX00eDot = {
   'context': /(?:[ai]ja|jä)[.,]$/u,
   'remove': /\.$/u
 };
+const removeCommaBeforeLanguageSubfieldL = {
+  'followedBy': 'l',
+  'remove': /,$/u
+};
+const removeCommaBeforeTitleSubfieldT = {
+  'followedBy': 't',
+  'remove': /,$/u
+};
 const X00RemoveDotAfterBracket = {
   'code': 'cq',
   'context': /\)\.$/u,
@@ -152,9 +162,17 @@ const cleanPuncBeforeLanguage = {
 };
 const addX00aComma = {
   'add': ',',
-  'code': 'abcqdej',
+  'code': 'abcqej',
+  'followedBy': 'cdeg',
+  'context': doesNotEndInPunc,
+  'contextRHS': allowsPuncRHS
+};
+const addX00dComma = {
+  'name': 'X00$d ending in "-" does not get comma',
+  'add': ',',
+  'code': 'd',
   'followedBy': 'cdeg',
-  'context': commaNeedsPuncAfter,
+  'context': /[^-,.!]$/u,
   'contextRHS': allowsPuncRHS
 };
 const addX00aComma2 = {
@@ -164,10 +182,10 @@ const addX00aComma2 = {
   'context': /(?:[A-Z]|Å|Ä|Ö)\.$/u,
   'contextRHS': allowsPuncRHS
 };
-const addX00aDot = {
+const addX00Dot = {
   'add': '.',
-  'code': 'abcdet',
-  'followedBy': '#tu',
+  'code': 'abcdetv',
+  'followedBy': '#fklptu',
   'context': defaultNeedsPuncAfter
 };
 
@@ -192,19 +210,19 @@ const addX10Dot = {
   'followedBy': 'tu#',
   'context': defaultNeedsPuncAfter
 };
-const addLanguageComma = {
-  'name': 'Add comma before 810$l',
-  'add': ',',
-  'code': 'tv',
-  'followedBy': 'l',
-  'context': defaultNeedsPuncAfter2
-};
 const addColonToRelationshipInformation = {
   'name': 'Add \':\' to 7X0 $i relationship info',
   'add': ':',
   'code': 'i',
   'context': defaultNeedsPuncAfter2
 };
+const addDotBeforeLanguageSubfieldL = {
+  'name': 'Add dot before $l',
+  'add': '.',
+  'code': 'abepst',
+  'followedBy': 'l',
+  'context': doesNotEndInPunc
+};
 
 // 490:
 const addSemicolonBeforeVolumeDesignation = {
@@ -222,8 +240,11 @@ const REMOVE_AND_ADD = 3;
 // Crappy punctuation consists of various crap that is somewhat common.
 // We strip crap for merge decisions. We are not trying to actively remove crap here.
 
-const removeX00Whatever = [removeX00Comma, cleanX00aDot, cleanX00eDot, cleanCorruption, cleanX00dCommaOrDot, cleanRHS, X00RemoveDotAfterBracket, removeColons, cleanPuncBeforeLanguage];
-const removeX10Whatever = [removeX00Comma, cleanX00aDot, cleanX00eDot, cleanCorruption, removeColons, cleanPuncBeforeLanguage];
+const removeCrapFromAllEntryFields = [removeCommaBeforeLanguageSubfieldL, removeCommaBeforeTitleSubfieldT];
+const removeX00Whatever = [removeX00Comma, cleanX00aDot, cleanX00eDot, cleanCorruption, cleanX00dCommaOrDot, cleanRHS, X00RemoveDotAfterBracket, removeColons, cleanPuncBeforeLanguage, ...removeCrapFromAllEntryFields];
+const removeX10Whatever = [removeX00Comma, cleanX00aDot, cleanX00eDot, cleanCorruption, removeColons, cleanPuncBeforeLanguage, ...removeCrapFromAllEntryFields];
+const removeX11Whatever = removeCrapFromAllEntryFields;
+const removeX30Whatever = removeCrapFromAllEntryFields;
 const remove490And830Whatever = [{
   'code': 'axyzv',
   'followedBy': 'axyzv',
@@ -273,11 +294,14 @@ const crappy24X = [{
   'followedBy': 'pc',
   'remove': /\.$/u,
   'context': dotIsProbablyPunc
-} // MELINDA-8817
-];
+},
+// MELINDA-8817
+removeCommaBeforeLanguageSubfieldL];
 const cleanCrappyPunctuationRules = {
   '100': removeX00Whatever,
   '110': removeX10Whatever,
+  '111': removeX11Whatever,
+  '130': removeX30Whatever,
   '240': crappy24X,
   '245': crappy24X,
   '246': crappy24X,
@@ -308,8 +332,12 @@ const cleanCrappyPunctuationRules = {
   '490': remove490And830Whatever,
   '600': removeX00Whatever,
   '610': removeX10Whatever,
+  '611': removeX11Whatever,
+  '630': removeX30Whatever,
   '700': removeX00Whatever,
   '710': removeX10Whatever,
+  '711': removeX11Whatever,
+  '730': removeX30Whatever,
   '773': linkingEntryWhatever,
   '774': linkingEntryWhatever,
   '776': linkingEntryWhatever,
@@ -342,14 +370,14 @@ const cleanLegalX00Dot = {
   'context': /(?:[a-z0-9)]|å|ä|ö)\.$/u,
   'remove': /\.$/u
 };
-const cleanLanguageComma = {
-  'name': 'language comma',
-  'code': 'tv',
+const cleanDotBeforeLanguageSubfieldL = {
+  'name': 'pre-language-$l dot',
   'followedBy': 'l',
-  'context': /.,$/u,
-  'remove': /,$/u
+  'context': /.\.$/u,
+  'remove': /\.$/u
 };
-const legalX00punc = [cleanLegalX00Comma, cleanLegalX00iColon, cleanLegalX00bDot, cleanLegalX00Dot, cleanLanguageComma];
+const legalEntryField = [cleanDotBeforeLanguageSubfieldL];
+const legalX00punc = [cleanLegalX00Comma, cleanLegalX00iColon, cleanLegalX00bDot, cleanLegalX00Dot, ...legalEntryField];
 const cleanLegalX10Comma = {
   'name': 'X10comma',
   'code': 'abe',
@@ -364,7 +392,7 @@ const cleanLegalX10Dot = {
   'context': /.\.$/u,
   'remove': /\.$/u
 };
-const legalX10punc = [cleanLegalX10Comma, cleanLegalX10Dot, cleanX00eDot, cleanLanguageComma];
+const legalX10punc = [cleanLegalX10Comma, cleanLegalX10Dot, cleanX00eDot, ...legalEntryField];
 const cleanLegalSeriesTitle = [
 // 490 and 830
 {
@@ -421,16 +449,14 @@ const clean24X = [{
   'code': 'n',
   'followedBy': 'p',
   'remove': /,$/u
-}];
+}, cleanDotBeforeLanguageSubfieldL];
 const cleanValidPunctuationRules = {
   '100': legalX00punc,
   '110': legalX10punc,
-  '600': legalX00punc,
-  '610': legalX10punc,
-  '700': legalX00punc,
-  '710': legalX10punc,
-  '800': legalX00punc,
-  '810': legalX10punc,
+  '111': legalEntryField,
+  '130': legalEntryField,
+  '240': clean24X,
+  '243': clean24X,
   '245': clean24X,
   '246': clean24X,
   '260': [{
@@ -493,6 +519,10 @@ const cleanValidPunctuationRules = {
     'followedBy': 'c',
     'remove': /:$/u
   }],
+  '600': legalX00punc,
+  '610': legalX10punc,
+  '611': legalEntryField,
+  '630': legalEntryField,
   // Experimental, MET366-ish (end punc in internationally valid, but we don't use it here in Finland):
   '648': [{
     'code': 'a',
@@ -500,68 +530,63 @@ const cleanValidPunctuationRules = {
     'ind2': ['4'],
     'remove': /\.$/u
   }],
-  '830': cleanLegalSeriesTitle,
+  '700': legalX00punc,
+  '710': legalX10punc,
+  '711': legalEntryField,
+  '730': legalEntryField,
+  '800': legalX00punc,
+  '810': legalX10punc,
+  '811': legalEntryField,
+  '830': [...legalEntryField, ...cleanLegalSeriesTitle],
   '946': clean24X
 };
 
-// addColonToRelationshipInformation only applies to 700/710 but as others don't have $i, it's fine
-const addX00 = [addX00aComma, addX00aComma2, addX00aDot, addLanguageComma, addSemicolonBeforeVolumeDesignation, addColonToRelationshipInformation];
-const addX10 = [addX10bDot, addX10eComma, addX10Dot, addLanguageComma, addSemicolonBeforeVolumeDesignation, addColonToRelationshipInformation];
-const add245 = [
-// Blah! Also "$a = $b" and "$a ; $b" can be valid... But ' :' is better than nothing, I guess...
-{
+// Overgeneralizes a bit: eg. addColonToRelationshipInformation only applies to 700/710 but as others don't have $i, it's fine.
+const addToAllEntryFields = [addDotBeforeLanguageSubfieldL, addSemicolonBeforeVolumeDesignation, addColonToRelationshipInformation];
+const addX00 = [addX00aComma, addX00aComma2, addX00Dot, addX00dComma, ...addToAllEntryFields];
+const addX10 = [addX10bDot, addX10eComma, addX10Dot, ...addToAllEntryFields];
+const addX11 = [...addToAllEntryFields];
+const addX30 = [...addToAllEntryFields];
+const add24X = [{
+  'code': 'i',
+  'followedBy': 'a',
+  'add': ':',
+  'context': defaultNeedsPuncAfter
+}, {
   'code': 'a',
   'followedBy': 'b',
   'add': ' :',
   'context': defaultNeedsPuncAfter
-}, {
-  'code': 'ab',
-  'followedBy': 'n',
-  'add': '.',
-  'context': defaultNeedsPuncAfter
 }, {
   'code': 'abk',
   'followedBy': 'f',
   'add': ',',
   'context': defaultNeedsPuncAfter
-}, {
-  'code': 'n',
-  'followedBy': 'p',
-  'add': ',',
-  'context': defaultNeedsPuncAfter
 }, {
   'code': 'abfnp',
   'followedBy': 'c',
   'add': ' /',
   'context': defaultNeedsPuncAfter
-}, {
-  'code': 'abc',
-  'followedBy': '#',
+}, addDotBeforeLanguageSubfieldL];
+const add245 = [...add24X,
+// Blah! Also "$a = $b" and "$a ; $b" can be valid... But ' :' is better than nothing, I guess...
+{
+  'code': 'ab',
+  'followedBy': 'n',
   'add': '.',
   'context': defaultNeedsPuncAfter
-} // Stepping on "punctuation validaror's" toes
-];
-const add246 = [{
-  'code': 'i',
-  'followedBy': 'a',
-  'add': ':',
-  'context': defaultNeedsPuncAfter
-}, {
-  'code': 'a',
-  'followedBy': 'b',
-  'add': ' :',
-  'context': defaultNeedsPuncAfter
 }, {
-  'code': 'abk',
-  'followedBy': 'f',
+  'code': 'n',
+  'followedBy': 'p',
   'add': ',',
   'context': defaultNeedsPuncAfter
 }, {
-  'code': 'abfnp',
-  'followedBy': 'c',
-  'add': ' /',
+  'code': 'abc',
+  'followedBy': '#',
+  'add': '.',
   'context': defaultNeedsPuncAfter
-}];
+} // Stepping on "punctuation validator's" toes
+];
 const addSeriesTitle = [
 // 490 and 830
 {
@@ -579,9 +604,12 @@ const addSeriesTitle = [
 const addPairedPunctuationRules = {
   '100': addX00,
   '110': addX10,
-  '240': add246,
+  '111': addX11,
+  '130': addX30,
+  '240': add24X,
+  '243': add24X,
   '245': add245,
-  '246': add246,
+  '246': add24X,
   '260': [{
     'code': 'a',
     'followedBy': 'b',
@@ -658,11 +686,16 @@ const addPairedPunctuationRules = {
   }],
   '600': addX00,
   '610': addX10,
+  '611': addX11,
+  '630': addX30,
   '700': addX00,
   '710': addX10,
+  '711': addX11,
+  '730': addX30,
   '800': addX00,
   '810': addX10,
-  '830': addSeriesTitle,
+  '811': addX11,
+  '830': [...addX30, ...addSeriesTitle],
   '946': [{
     'code': 'i',
     'followedBy': 'a',
@@ -690,6 +723,10 @@ function debugRule(rule) {
 */
 
 function ruleAppliesToSubfieldCode(targetSubfieldCodes, currSubfieldCode) {
+  if (!targetSubfieldCodes) {
+    // We are not interested in what subfield precedes 240$l, ',' is removed anyway
+    return true;
+  }
   const negation = targetSubfieldCodes.includes('!');
   if (negation) {
     return !targetSubfieldCodes.includes(currSubfieldCode);
@@ -769,36 +806,41 @@ function checkRule(rule, field, subfield1, subfield2) {
   return true;
 }
 function applyPunctuationRules(field, subfield1, subfield2, ruleArray = null, operation = NONE) {
-  if (!(`${field.tag}` in ruleArray) || ruleArray === null || operation === NONE) {
-    /*
-    if (!['020', '650'].includes(tag) || !isControlSubfieldCode(subfield1.code)) { // eslint-disable-line functional/no-conditional-statements
-      nvdebug(`No punctuation rules found for ${tag} (looking for: ‡${subfield1.code})`, debug);
-     }
-    */
+  if (operation === NONE || ruleArray === null) {
+    // !fieldIsApplicable(field, ruleArray)) {
+    return;
+  }
+  const tag2 = field.tag === '880' ? (0, _subfield6Utils.fieldGetUnambiguousTag)(field) : field.tag;
+  if (!tag2) {
+    return;
+  }
+  if (!(`${tag2}` in ruleArray)) {
     return;
   }
-  (0, _utils.nvdebug)(`PUNCTUATE ${field.tag} '${(0, _utils.subfieldToString)(subfield1)}' XXX '${subfield2 ? (0, _utils.subfieldToString)(subfield2) : '#'} }`);
 
-  //nvdebug(`OP=${operation} ${tag}: '${subfield1.code}: ${subfield1.value}' ??? '${subfield2 ? subfield2.code : '#'}'`, debug);
-  const candRules = ruleArray[field.tag];
+  //nvdebug(`PUNCTUATE ${field.tag}/${tag2} '${subfieldToString(subfield1)}' XXX '${subfield2 ? subfieldToString(subfield2) : '#'} }`);
+
+  //nvdebug(`OP=${operation} ${tag2}: '${subfield1.code}: ${subfield1.value}' ??? '${subfield2 ? subfield2.code : '#'}'`);
+  const candRules = ruleArray[tag2];
   candRules.forEach(rule => {
     //debugRule(rule);
-
+    //nvdebug(' WP1');
     if (!checkRule(rule, field, subfield1, subfield2)) {
       return;
     }
+    //nvdebug(' WP2');
 
     //const originalValue = subfield1.value;
     if (rule.remove && [REMOVE, REMOVE_AND_ADD].includes(operation) && subfield1.value.match(rule.remove)) {
       // eslint-disable-line functional/no-conditional-statements
       //nvdebug(`    PUNC REMOVAL TO BE PERFORMED FOR $${subfield1.code} '${subfield1.value}'`, debug);
       subfield1.value = subfield1.value.replace(rule.remove, ''); // eslint-disable-line functional/immutable-data
-      //nvdebug(`    PUNC REMOVAL PERFORMED FOR '${subfield1.value}'`, debug);
+      //nvdebug(`    PUNC REMOVAL PERFORMED FOR '${subfield1.value}'`);
     }
     if (rule.add && [ADD, REMOVE_AND_ADD].includes(operation)) {
       // eslint-disable-line functional/no-conditional-statements
       subfield1.value += rule.add; // eslint-disable-line functional/immutable-data
-      //nvdebug(`    ADDED '${rule.add}' TO FORM '${subfield1.value}'`, debug);
+      //nvdebug(`    ADDED '${rule.add}' TO FORM '${subfield1.value}'`);
     }
 
     /*