@natlibfi/marc-record-validators-melinda 10.15.1 → 10.15.2-alpha.3

package/package.json

@@ -14,7 +14,7 @@
     "url": "git@github.com:natlibfi/marc-record-validators-melinda.git"
   },
   "license": "MIT",
-  "version": "10.15.1",
+  "version": "10.15.2-alpha.3",
   "main": "./dist/index.js",
   "publishConfig": {
     "access": "public"
@@ -37,8 +37,8 @@
   "dependencies": {
     "@babel/register": "^7.22.15",
     "@natlibfi/issn-verify": "^1.0.3",
-    "@natlibfi/marc-record": "^8.0.0",
-    "@natlibfi/marc-record-validate": "^8.0.3",
+    "@natlibfi/marc-record": "^8.0.2",
+    "@natlibfi/marc-record-validate": "^8.0.4",
     "cld3-asm": "^3.1.1",
     "clone": "^2.1.2",
     "debug": "^4.3.4",
@@ -48,22 +48,22 @@
     "xml2js": "^0.6.2"
   },
   "peerDependencies": {
-    "@natlibfi/marc-record-validate": "^8.0.3"
+    "@natlibfi/marc-record-validate": "^8.0.4"
   },
   "devDependencies": {
-    "@babel/cli": "^7.23.0",
-    "@babel/core": "^7.23.3",
+    "@babel/cli": "^7.23.4",
+    "@babel/core": "^7.23.6",
     "@babel/eslint-parser": "^7.23.3",
-    "@babel/preset-env": "^7.23.3",
+    "@babel/preset-env": "^7.23.6",
     "@natlibfi/eslint-config-melinda-backend": "^3.0.3",
-    "@natlibfi/fixugen": "^2.0.2",
-    "@natlibfi/fixura": "^3.0.2",
+    "@natlibfi/fixugen": "^2.0.3",
+    "@natlibfi/fixura": "^3.0.3",
     "babel-plugin-istanbul": "^6.1.1",
     "babel-plugin-rewire": "^1.2.0",
     "chai": "^4.3.10",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
-    "eslint": "^8.53.0",
+    "eslint": "^8.55.0",
     "fetch-mock": "^9.11.0",
     "mocha": "^10.2.0",
     "nyc": "^15.1.0"

package/src/normalize-utf8-diacritics.js

@@ -77,7 +77,7 @@ export default function () {
 
 
 // Traditionally these six are precomposed and all the rest decomposed
-function precomposeFinnishLetters(value = '') {
+export function precomposeFinnishLetters(value = '') {
   return value.
     replace(/å/gu, 'å').
     replace(/ä/gu, 'ä').
@@ -87,7 +87,7 @@ function precomposeFinnishLetters(value = '') {
     replace(/Ö/gu, 'Ö');
 }
 
-function fixComposition(value = '') {
+export function fixComposition(value = '') {
   // Target: Diacritics use Melinda internal notation.
   // General solution: Decompose everything and then compose 'å', 'ä', 'ö', 'Å', 'Ä' and 'Ö'.
   // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/normalize

package/src/punctuation2.js

@@ -111,11 +111,12 @@ const cleanPuncBeforeLanguage = {'code': 'atvxyz', 'followedBy': 'l', 'context':
 
 const addX00aComma = {'add': ',', 'code': 'abcqdej', 'followedBy': 'cdeg', 'context': commaNeedsPuncAfter, 'contextRHS': allowsPuncRHS};
 const addX00aComma2 = {'add': ',', 'code': 'abcdej', 'followedBy': 'cdeg', 'context': /(?:[A-Z]|Å|Ä|Ö)\.$/u, 'contextRHS': allowsPuncRHS};
-const addX00aDot = {'add': '.', 'code': 'abcde', 'followedBy': '#tu', 'context': defaultNeedsPuncAfter};
+const addX00aDot = {'add': '.', 'code': 'abcdet', 'followedBy': '#tu', 'context': defaultNeedsPuncAfter};
 
+//const addX10iaComma = {'name': 'Punctuate relationship information', 'code': 'i', 'followedBy': 'a', 'context': defaultNeedsPuncAfter2};
 const addX10bDot = {'name': 'Add X10 pre-$b dot', 'add': '.', 'code': 'ab', 'followedBy': 'b', 'context': defaultNeedsPuncAfter};
 const addX10eComma = {'add': ',', 'code': 'abe', 'followedBy': 'e', 'context': defaultNeedsPuncAfter};
-const addX10Dot = {'name': 'Add X10 final dot', 'add': '.', 'code': 'abe', 'followedBy': '#', 'context': defaultNeedsPuncAfter};
+const addX10Dot = {'name': 'Add X10 final dot', 'add': '.', 'code': 'abet', 'followedBy': 'tu#', 'context': defaultNeedsPuncAfter};
 const addLanguageComma = {'name': 'Add comma before 810$l', 'add': ',', 'code': 'tv', 'followedBy': 'l', 'context': defaultNeedsPuncAfter2};
 const addColonToRelationshipInformation = {'name': 'Add \':\' to 7X0 $i relationship info', 'add': ':', 'code': 'i', 'context': defaultNeedsPuncAfter2};
 
@@ -194,7 +195,7 @@ const cleanLanguageComma = {'name': 'language comma', 'code': 'tv', 'followedBy'
 const legalX00punc = [cleanLegalX00Comma, cleanLegalX00iColon, cleanLegalX00bDot, cleanLegalX00Dot, cleanLanguageComma];
 
 const cleanLegalX10Comma = {'name': 'X10comma', 'code': 'abe', 'followedBy': 'e', 'context': /.,$/u, 'remove': /,$/u};
-const cleanLegalX10Dot = {'name': 'X10dot', 'code': 'ab', 'followedBy': 'b#', 'context': /.\.$/u, 'remove': /\.$/u};
+const cleanLegalX10Dot = {'name': 'X10dot', 'code': 'abt', 'followedBy': 'bst#', 'context': /.\.$/u, 'remove': /\.$/u};
 
 const legalX10punc = [cleanLegalX10Comma, cleanLegalX10Dot, cleanX00eDot, cleanLanguageComma];
 

package/src/removeInferiorDataFields.js

@@ -5,6 +5,7 @@ import {fieldHasSubfield, fieldsToString, fieldToString, nvdebug, uniqArray} fro
 import {fieldHasValidSubfield8} from './subfield8Utils';
 import {encodingLevelIsBetterThanPrepublication, fieldRefersToKoneellisestiTuotettuTietue, getEncodingLevel} from './prepublicationUtils';
 import {cloneAndNormalizeFieldForComparison} from './normalizeFieldForComparison';
+import {fixComposition, precomposeFinnishLetters} from './normalize-utf8-diacritics';
 
 // Relocated from melinda-marc-record-merge-reducers (and renamed)
 
@@ -241,14 +242,21 @@ function deriveIndividualDeletables(record) {
       return deletables;
     }
 
+    const accentless = getAccentlessVersion(currString);
+    const d490 = deriveIndividualDeletables490([currString]);
+    const subsets = getIdentifierlessAndKeeplessSubsets(currString); // eslint-disable-line no-param-reassign
+    const moreToDo = [...accentless, ...d490, ...subsets];
+
+
     if (currString.match(/^[1678]00/u)) {
-      // Proof-of-concpet rule. Should be improved eventually...
+      // Proof-of-concept rule. Should be improved eventually...
       if (currString.match(/, ‡e [^‡]+\.$/u)) {
         const tmp = currString.replace(/, ‡e [^‡]+\.$/u, '.');
-        return processTodoList([tmp, ...stillToDo], [...deletables, tmp]);
+        return processTodoList([tmp, ...stillToDo, ...moreToDo], [...deletables, tmp]);
       }
     }
 
+
     if (currString.match(/^505 .0.*-- ‡t/u)) { // MRA-413-ish
       const tmp = currString.replace(/ -- ‡t /gu, ' -- '). // remove non-initial $t subfields
         replace(/ ‡[rg] /gu, ' '). // remove $r and $g subfields
@@ -256,7 +264,7 @@ function deriveIndividualDeletables(record) {
         // ind2: '1' => '#':
         replace(/^505 (.)0/u, '505 $1#'); // eslint-disable-line prefer-named-capture-group
       if (tmp !== currString) {
-        return processTodoList([tmp, ...stillToDo], [...deletables, tmp]);
+        return processTodoList([tmp, ...stillToDo, ...moreToDo], [...deletables, tmp]);
       }
       //nvdebug(`505 ORIGINAL: '${fieldAsString}'`)
       //nvdebug(`505 DERIVATE: '${tmp}'`)
@@ -270,23 +278,35 @@ function deriveIndividualDeletables(record) {
       if (tmp.match(/ ‡6 [0-9][0-9][0-9]-00\/[^ ]+ /u)) {
         const tmp2 = tmp.replace(/( ‡6 [0-9][0-9][0-9]-00)[^ ]+/u, '$1'); // eslint-disable-line prefer-named-capture-group
         //nvdebug(`MET-381: ADD TO DELETABLES: '${tmp2}'`);
-        return processTodoList(stillToDo, [...deletables, tmp, tmp2]);
+        return processTodoList([...stillToDo, ...moreToDo], [...deletables, tmp, tmp2]);
       }
-      return processTodoList(stillToDo, [...deletables, tmp]);
+      return processTodoList([...stillToDo, ...moreToDo], [...deletables, tmp]);
     }
 
-    const d490 = deriveIndividualDeletables490([currString]);
-    if (d490.length) {
-      return processTodoList([...stillToDo, ...d490], [...deletables, ...d490]);
-    }
-    // d490.forEach(str => deletables.push(str)); // eslint-disable-line functional/immutable-data
 
-    const subsets = getIdentifierlessAndKeeplessSubsets(currString); // eslint-disable-line no-param-reassign
+    const ennakkotieto653 = currString.match(/^653./u) ? [`${currString} ‡g ENNAKKOTIETO`] : []; // MET-528
+
+    const newDeletables = [...deletables, ...subsets, ...accentless, ...d490, ...ennakkotieto653];
+
     if (subsets.length) {
-      return processTodoList([...stillToDo, ...subsets], [...deletables, ...subsets]);
+      return processTodoList([...stillToDo, ...moreToDo], newDeletables);
     }
 
-    return processTodoList(stillToDo, deletables);
+    return processTodoList([...stillToDo, ...moreToDo], newDeletables);
+  }
+
+  function getAccentlessVersion(string) { // MET-527
+    //nvdebug(`START: '${string}`);
+    // This is a sanity check: if precomposition does something, there's something wrong, and we don't want to proceed..
+    if (string !== precomposeFinnishLetters(string)) {
+      return [];
+    }
+    const accentless = String(fixComposition(string)).replace(/\p{Diacritic}/gu, '');
+    //nvdebug(`FROM '${string}'\n  TO '${accentless}'`);
+    if (accentless === string) { // Don't self-destruct
+      return [];
+    }
+    return [accentless];
   }
 
 }

package/test-fixtures/punctuation2/96/expectedResult.json

@@ -0,0 +1,26 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "001", "value": "000000001" },
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Pulun Yliopisto." },
+        { "code": "b", "value": "Yksikkö." },
+        { "code": "b", "value": "Alayksikkö." }
+    ]},
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+      { "code": "a", "value": "Pulun Yliopisto." },
+      { "code": "b", "value": "Yksikkö," },
+      { "code": "e", "value": "julkaisija." }
+  ]},
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+      { "code": "i", "value": "Sisältää (teos):" },
+      { "code": "a", "value": "Whatever." },
+      { "code": "t", "value": "Bulebule." }
+  ]}
+
+  ]
+}

package/test-fixtures/punctuation2/96/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description":"Fix 96 - fix punctuation of X10 fields.",
+  "enabled": true,
+  "fix": true,
+  "only": false
+}

package/test-fixtures/punctuation2/96/record.json

@@ -0,0 +1,25 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "001", "value": "000000001" },
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Pulun Yliopisto" },
+        { "code": "b", "value": "Yksikkö" },
+        { "code": "b", "value": "Alayksikkö" }
+    ]},
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+      { "code": "a", "value": "Pulun Yliopisto" },
+      { "code": "b", "value": "Yksikkö" },
+      { "code": "e", "value": "julkaisija" }
+  ]},
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+      { "code": "i", "value": "Sisältää (teos)" },
+      { "code": "a", "value": "Whatever" },
+      { "code": "t", "value": "Bulebule" }
+  ]}
+
+  ]
+}

package/test-fixtures/remove-inferior-datafields/f12/expectedResult.json

@@ -0,0 +1,15 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "001", "value": "f13" },
+    { "tag": "490", "ind1": "0", "ind2": " ", "subfields": [
+      { "code": "a", "value": "Mortuí non silent ;"},
+      { "code": "v", "value": "v2"}
+    ]},
+    { "tag": "600", "ind1": "1", "ind2": "4", "subfields": [
+      { "code": "a", "value": "Sirén, Matti"}
+    ]}
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}

package/test-fixtures/remove-inferior-datafields/f12/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "f12/MET-527: diacriticful > diacriticless",
+  "enabled": true,
+  "fix": true,
+  "only": false
+}

package/test-fixtures/remove-inferior-datafields/f12/record.json

@@ -0,0 +1,22 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "001", "value": "f13" },
+    { "tag": "490", "ind1": "0", "ind2": " ", "subfields": [
+      { "code": "a", "value": "Mortuí non silent ;"},
+      { "code": "v", "value": "v2"}
+    ]},
+    { "tag": "490", "ind1": "0", "ind2": " ", "subfields": [
+      { "code": "a", "value": "Mortui non silent ;"},
+      { "code": "v", "value": "v2"}
+    ]},
+    { "tag": "600", "ind1": "1", "ind2": "4", "subfields": [
+      { "code": "a", "value": "Sirén, Matti"}
+    ]},
+    { "tag": "600", "ind1": "1", "ind2": "4", "subfields": [
+      { "code": "a", "value": "Siren, Matti"}
+    ]}
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}

package/test-fixtures/remove-inferior-datafields/f13/expectedResult.json

@@ -0,0 +1,11 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "001", "value": "f13" },
+    { "tag": "653", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "taikasana"}
+    ]}
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}

package/test-fixtures/remove-inferior-datafields/f13/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "f13/MET-528-ish: remove ENNAKKOTIETO version, ennakkotietoless pair exists",
+  "enabled": true,
+  "fix": true,
+  "only": false
+}

package/test-fixtures/remove-inferior-datafields/f13/record.json

@@ -0,0 +1,15 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "001", "value": "f13" },
+    { "tag": "653", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "taikasana"}
+    ]},
+    { "tag": "653", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "taikasana"},
+      { "code": "g", "value": "ENNAKKOTIETO"}
+    ]}
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}

package/test-fixtures/strip-punctuation/96/expectedResult.json

@@ -0,0 +1,26 @@
+{
+  "_validationOptions": {},
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "001", "value": "000000001" },
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Pulun Yliopisto" },
+        { "code": "b", "value": "Yksikkö" },
+        { "code": "b", "value": "Alayksikkö" }
+    ]},
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+      { "code": "a", "value": "Pulun Yliopisto" },
+      { "code": "b", "value": "Yksikkö" },
+      { "code": "e", "value": "julkaisija" }
+  ]},
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+      { "code": "i", "value": "Sisältää (teos)" },
+      { "code": "a", "value": "Whatever" },
+      { "code": "t", "value": "Bulebule" }
+  ]}
+
+  ]
+}

package/test-fixtures/strip-punctuation/96/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description":"Fix 96 - fix punctuation of X10 fields.",
+  "enabled": true,
+  "fix": true,
+  "only": false
+}

package/test-fixtures/strip-punctuation/96/record.json

@@ -0,0 +1,25 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "001", "value": "000000001" },
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Pulun Yliopisto." },
+        { "code": "b", "value": "Yksikkö." },
+        { "code": "b", "value": "Alayksikkö." }
+    ]},
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+      { "code": "a", "value": "Pulun Yliopisto." },
+      { "code": "b", "value": "Yksikkö," },
+      { "code": "e", "value": "julkaisija." }
+  ]},
+
+    { "tag": "710", "ind1": "2", "ind2": " ", "subfields": [
+      { "code": "i", "value": "Sisältää (teos):" },
+      { "code": "a", "value": "Whatever." },
+      { "code": "t", "value": "Bulebule." }
+  ]}
+
+  ]
+}