@natlibfi/marc-record-validators-melinda 12.0.0-alpha.7 → 12.0.0-alpha.8

package/src/ending-punctuation.js

@@ -65,12 +65,6 @@ export default function () {
 
 // Field validation with punctuation rules for normal and special cases in subfunction (to reduce complexity to please travisci)
 function validateField(field, linkedTag, fix, message) {
-  // This is used to find last subfield that should have punctuation
-  function findLastSubfield(field) {
-    const subfields = field.subfields.filter(sf => isNaN(sf.code) && 'value' in sf);
-    return subfields.slice(-1).shift();
-  }
-
   function getDefaultPuncMarks(tag) {
     if (tag.match(/^[1678](?:00|10|11|30)/u) || tag === '740') { // As defined in Loppupisteohje
       return `${validPuncMarks})`;
@@ -98,7 +92,7 @@ function validateField(field, linkedTag, fix, message) {
 
     // Last char should be punc, but it's not one of listed punctuation marks nor dot
     if (punc && !(lastPuncMark || lastPuncDot)) {
-      console.log(puncMarks)
+      //console.log(puncMarks)
       if (penultimateCharacter && validQuoteChars.includes(lastChar) && puncMarks.includes(penultimateCharacter)) {
         // Exception: do nothing! Ending in punc+quote combo is all right, and does not imply a missing punc
       }
@@ -189,12 +183,17 @@ function validateField(field, linkedTag, fix, message) {
       normalPuncRules(lastSubField, res.punc, tag, false, false);
 
       // Search for Finnish terms
-    } else if (res.special.termField) {
+    } else if (res.special.termSubfieldCode) {
       lastSubField = findLastSubfield(field);
 
       if (lastSubField) {
-        const languageField = field.subfields.find(({code}) => code === res.special.termField);
-        if (languageField && languageField.value && finnishTerms.some(p => p.test(languageField.value))) {
+        const lexicon = getLexicon(field, res.special.termSubfieldCode);
+        const proceed = !finnishException(field, res.special.termSubfieldCode, false);
+
+
+        //const languageField = field.subfields.find(({code}) => code === res.special.termSubfieldCode);
+        //if (languageField && languageField.value && finnishTerms.some(p => p.test(languageField.value))) {
+        if (lexicon && finnishTerms.some(p => p.test(lexicon)) && proceed) {
           // If (languageField && languageField.value && finnishTerms.indexOf(languageField.value) > -1) {
           normalPuncRules(lastSubField, res.punc, tag, true, false);
         } else {
@@ -272,8 +271,12 @@ function validateField(field, linkedTag, fix, message) {
     return;
   }
 
+  const forceNormal = res.special ? finnishException(field, res.special.termSubfieldCode, true) : false;
   // Normal rules
-  if (typeof res.special === 'undefined' || res.special === null) {
+  if (typeof res.special === 'undefined' || res.special === null || forceNormal) {
+    if (forceNormal) {
+      console.info("EXCEPTION. SKIP FINNISH RULES");
+    }
     lastSubField = findLastSubfield(field);
 
     if (lastSubField) {
@@ -300,3 +303,51 @@ export function validateSingleField(field, linkedTag, fix) {
   return message;
 }
 
+function getLexicon(field, subfieldCode) {
+  const languageSubfield = field.subfields.find(({code}) => code === subfieldCode); // res.special.termSubfieldCode);
+  if (!languageSubfield || !languageSubfield.value) {
+    return undefined;
+  }
+  if (finnishTerms.find(p => p.test(languageSubfield.value))) {
+    return languageSubfield.value;
+  }
+  return undefined;
+}
+
+function finnishException(field, termSubfieldCode, hasDot = true) {
+  const lexicon = getLexicon(field, termSubfieldCode);
+  if (!lexicon) {
+    return false;
+  }
+
+  const lastSubfield = findLastSubfield(field);
+  if (!lastSubfield || !lastSubfield.value) {
+    return false;
+  }
+  // Some terms can end in '.' that we want to keep
+  if (field.tag === '648') { // Yso-aika checks
+    //console.log(`Finnish Exception? '${lastSubfield.value}', '${lexicon}', '${field.tag}'`);
+    if (lexicon === 'yso/fin') { // 'eaa.' appears in prefLAbels and 'eKr.' in altLabels
+      if (hasDot) {
+        return lastSubfield.value.match(/ (?:eaa|[ej]Kr|jaa)\.$/u); // Finnish term from which the dot is not to be removed
+      }
+      return lastSubfield.value.match(/ (?:eaa|[ej]Kr)|jaa$/u); // Finnish word that needs a dot
+    }
+
+    if (lexicon === 'yso/swe') {
+      if (hasDot) {
+        return lastSubfield.value.match(/ (?:[ef]\.Kr|f\.v\.t)\.$/u);
+      }
+      return lastSubfield.value.match(/ (?:[ef]\.Kr|f\.v\.t)$/u);
+    }
+  }
+  // yso has 'MODEL.LA.' and 'Corel R.A.V.E.' but these are so rare I'm not listing them
+
+  return false;
+}
+
+  // This is used to find last subfield that should have punctuation
+  function findLastSubfield(field) {
+    const subfields = field.subfields.filter(sf => isNaN(sf.code) && 'value' in sf);
+    return subfields.slice(-1).shift();
+  }

package/src/ending-punctuation.test.js

@@ -1585,7 +1585,7 @@ describe('ending-punctuation', () => {
         ]
       });
 
-      const recordVali648dFinNo = new MarcRecord({
+      const recordValid648FinNo = new MarcRecord({
         leader: '',
         fields: [
           {
@@ -1596,6 +1596,16 @@ describe('ending-punctuation', () => {
               {code: 'a', value: '1900-luku'},
               {code: '2', value: 'yso/swe'}
             ]
+          },
+          { // Exception: term ending in dot:
+            tag : '648',
+            ind1: ' ',
+            ind2: '7',
+            subfields: [
+              {code: 'a', value: '1200-luku eaa.'},
+              {code: '2', value: 'yso/fin'},
+              {code: '0', value: 'http://www.yso.fi/onto/yso/p1129911200'}
+            ]
           }
         ]
       });
@@ -1668,7 +1678,7 @@ describe('ending-punctuation', () => {
 
       it('Finds record valid - 648 Finnish, without punc', async () => {
         const validator = await validatorFactory();
-        const result = await validator.validate(recordVali648dFinNo);
+        const result = await validator.validate(recordValid648FinNo);
         assert.equal(result.valid, true);
       });
 
@@ -1713,7 +1723,7 @@ describe('ending-punctuation', () => {
         ]
       });
 
-      const recordInvali648dFinYes = new MarcRecord({
+      const recordInvalid648FinYes = new MarcRecord({
         leader: '',
         fields: [
           {
@@ -1724,6 +1734,15 @@ describe('ending-punctuation', () => {
               {code: 'a', value: '1900-luku.'},
               {code: '2', value: 'yso/swe'}
             ]
+          }, { // Exception:
+            tag: '648',
+            ind1: ' ',
+            ind2: '7',
+            subfields: [
+              {code: 'a', value: '1200-luku eaa'},
+              {code: '2', value: 'yso/fin'},
+              {code: '0', value: 'http://www.yso.fi/onto/yso/p1129911200'}
+            ]
           }
         ]
       });
@@ -1799,9 +1818,9 @@ describe('ending-punctuation', () => {
 
       it('Finds record invalid - 648 Finnish, with punc', async () => {
         const validator = await validatorFactory();
-        const result = await validator.validate(recordInvali648dFinYes);
+        const result = await validator.validate(recordInvalid648FinYes);
         assert.deepEqual(result, {
-          message: ['Field 648 has unwanted ending punctuation \'.\''],
+          message: ['Field 648 has unwanted ending punctuation \'.\'', "Field 648 requires ending punctuation, ends in 'a'"],
           valid: false
         });
       });
@@ -1857,11 +1876,16 @@ describe('ending-punctuation', () => {
 
       it('Repairs the invalid record - 648 Finnish, removes punc $a', async () => {
         const validator = await validatorFactory();
-        const result = await validator.fix(recordInvali648dFinYes);
-        assert.equal(recordInvali648dFinYes.equalsTo(recordVali648dFinNo), true);
+        const result = await validator.fix(recordInvalid648FinYes);
+        //console.info("NV-------");
+        //console.info(JSON.stringify(recordInvalid648FinYes));
+        //console.info(JSON.stringify(recordValid648FinNo));
+        //assert.equal(recordInvalid648FinYes.equalsTo(recordValid648FinNo), true);
+        assert.deepEqual(recordInvalid648FinYes, recordValid648FinNo);
+        //console.info(JSON.stringify(result));
         assert.deepEqual(result, {
-          message: ['Field 648 has unwanted ending punctuation \'.\''],
-          fix: ['Field 648 - Removed punctuation from $a'],
+          message: ['Field 648 has unwanted ending punctuation \'.\'', "Field 648 requires ending punctuation, ends in 'a'"],
+          fix: ['Field 648 - Removed punctuation from $a', 'Field 648 - Added punctuation to $a'],
           valid: false
         });
       });

package/src/indicator-fixes.js

@@ -171,6 +171,18 @@ function normalize245Indicator1(field, record) {
   field.ind1 = field1XX.length === 0 ? '0' : '1';
 }
 
+function noDisplayConstantGenerated520Indicator1(field) {
+  if (field.tag !== '520') {
+    return;
+  }
+  const as = field.subfields.filter(sf => sf.code === 'a');
+  // Set ind1=8 "no display constant generated" fro certain values (part of MELKEHITYS-2579):
+  if (as.length === 1 && ['Abstract.', 'Abstrakt.', 'Abstrakti.', 'Abstract.', 'English Summary.', 'Sammandrag.', 'Tiivistelmä.'].includes(field.subfields[0].value)) {
+    field.ind1 = '8';
+  }
+
+}
+
 function normalize776Indicator2(field) {
   if (field.tag !== '776') {
     return;
@@ -242,6 +254,7 @@ function recordNormalizeIndicators(record) {
 function fieldNormalizeIndicators(field, record, languages) {
   normalize084Indicator1(field);
   normalize245Indicator1(field, record);
+  noDisplayConstantGenerated520Indicator1(field);
   normalizeNonFilingIndicator1(field, languages);
   normalizeNonFilingIndicator2(field, languages);
   normalize776Indicator2(field);

package/src/merge-fields/dataProvenance.js

@@ -2,7 +2,7 @@
 
 import {subfieldArraysContainSameData} from "../utils.js";
 
-function tagToDataProvenanceSubfieldCode(tag) {
+export function tagToDataProvenanceSubfieldCode(tag) {
     if ( ['533', '800', '810', '811', '830'].includes(tag)) {
         return 'y';
     }

package/src/punctuation2.js

@@ -10,9 +10,10 @@
 *          (They are jumped over when looking for next (non-controlfield subfield)
 */
 import {validateSingleField} from './ending-punctuation.js';
+import {tagToDataProvenanceSubfieldCode} from './merge-fields/dataProvenance.js';
 import {fieldGetUnambiguousTag} from './subfield6Utils.js';
 //import createDebugLogger from 'debug';
-import {fieldToString, nvdebug} from './utils.js';
+import {fieldToString, isControlSubfieldCode, nvdebug} from './utils.js';
 import clone from 'clone';
 
 //const debug = createDebugLogger('debug/punctuation2');
@@ -49,12 +50,19 @@ export default function () {
   }
 }
 
-function isControlSubfield(subfield) {
-  return ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'].includes(subfield.code);
+
+
+function isIrrelevantSubfield(subfield, tag) {
+  const dataProvenanceSubfieldCode = tagToDataProvenanceSubfieldCode(tag);
+  if (subfield.code === dataProvenanceSubfieldCode) {
+    return true;
+  }
+  return isControlSubfieldCode(subfield.code); // Currently this contains other stuff as well ($3, $4, $7, $9...)
 }
 
+
 function getNextRelevantSubfield(field, currSubfieldIndex) {
-  return field.subfields.find((subfield, index) => index > currSubfieldIndex && !isControlSubfield(subfield));
+  return field.subfields.find((subfield, index) => index > currSubfieldIndex && !isIrrelevantSubfield(subfield, field.tag));
 }
 
 export function fieldGetFixedString(field, add = true) {
@@ -155,7 +163,8 @@ const remove490And830Whatever = [{'code': 'axyzv', 'followedBy': 'axyzv', 'remov
 const linkingEntryRemoveWhatever = [
   {'code': 'i', 'followedBy': 'at', 'remove': / ?:$/u}, // ':'
   {'code': 'at', 'remove': /\.$/u},
-  {'code': 'abdghiklmnopqrstuwxyz', 'followedBy': 'abdghiklmnopqrstuwxyz', 'remove': /\. -$/u}
+  // Only ". -" separator is still used in music. We can strip it, but can only create the non-music punctuation!
+  {'code': 'abdghiklmnopqrstuwxyz', 'followedBy': 'abdghiklmnopqrstuwxyz#', 'remove': /\. -$/u}
 ];
 
 

package/src/translate-terms.test.js

@@ -1,9 +1,23 @@
 import assert from 'node:assert';
-import {MarcRecord} from '@natlibfi/marc-record';
+import createDebugLogger from 'debug';
+import fetchMock from 'fetch-mock';
+
 import validatorFactory from './translate-terms.js';
+
+import {MarcRecord} from '@natlibfi/marc-record';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+import {fakeData} from '../test-fixtures/translate-terms-data.js';
+
+const uris = [
+  'http://www.yso.fi/onto/yso/p13299',
+  'http://www.yso.fi/onto/yso/p111739',
+  'http://www.yso.fi/onto/yso/p6197061979',
+  'http://www.yso.fi/onto/yso/p6196061969',
+  'http://urn.fi/URN:NBN:fi:au:slm:s161'
+];
+
+
 
 generateTests({
   callback,
@@ -15,6 +29,15 @@ generateTests({
   },
   hooks: {
     before: async () => {
+
+      fetchMock.mockGlobal()
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[0]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData})
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[1]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData})
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[2]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData})
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[3]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData})
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[4]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData});
+
+
       testValidatorFactory();
     }
   }

package/src/utils.js

@@ -103,6 +103,12 @@ export function nvdebugFieldArray(fields, prefix = '  ', func = undefined) {
 }
 
 export function isControlSubfieldCode(subfieldCode) {
+  // NB! Only $w, $0, $1, $5, $6 and $8 are really control subfields. In Finland $9 is oft a control subfield
+  // $3 material (part of the whole thing)
+  // $4 means 'relationship' (similar to relator terms at least in X00 and similar)
+  // $7 is usually provinance subfield. However, it can be stored in other subfields as well. See merge-fields/dataProvenance.js for details
+  // However, change this only if needed. Maybe all provinance subfields should return true?
+  // This may become relevant when AI starts to create stuff...
   if (['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'w'].includes(subfieldCode)) {
     return true;
   }

package/test-fixtures/indicator-fixes/10/expectedResult.json

@@ -0,0 +1,11 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "005", "value": "20220202020202.0" },
+    { "tag": "520", "ind1": "8", "ind2": " ", "subfields": [ {"code": "a", "value": "Abstrakti."}, {"code": "9", "value": "TESTI<KEEP>"} ]},
+    { "tag": "520", "ind1": " ", "ind2": " ", "subfields": [ {"code": "a", "value": "Whatever."}]},
+    { "tag": "520", "ind1": "8", "ind2": " ", "subfields": [ {"code": "a", "value": "Tiivistelmä."}]}
+
+  ],
+  "leader": ""
+}

package/test-fixtures/indicator-fixes/10/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "10: certain value have ind1=8 as per MELKEHITYS-2579",
+  "fix": true
+}

package/test-fixtures/indicator-fixes/10/record.json

@@ -0,0 +1,11 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "005", "value": "20220202020202.0" },
+    { "tag": "520", "ind1": " ", "ind2": " ", "subfields": [ {"code": "a", "value": "Abstrakti."}, {"code": "9", "value": "TESTI<KEEP>"} ]},
+    { "tag": "520", "ind1": " ", "ind2": " ", "subfields": [ {"code": "a", "value": "Whatever."}]},
+    { "tag": "520", "ind1": "3", "ind2": " ", "subfields": [ {"code": "a", "value": "Tiivistelmä."}]}
+
+  ],
+  "leader": ""
+}

package/test-fixtures/translate-terms-data.js

@@ -0,0 +1,42 @@
+const fakeTerms = {
+    graph: [
+        {
+            uri: 'http://www.yso.fi/onto/yso/p13299',
+            prefLabel: [
+                { lang: 'fi', value: 'laiturit' },
+                { lang: 'sv', value: 'bryggor'}
+            ]
+        },
+        {
+            uri: 'http://www.yso.fi/onto/yso/p111739',
+            prefLabel: [
+                { lang: 'fi', value: 'Ivalo (Inari)' },
+                { lang: 'sv', value: 'Ivalo (Enare)'}
+            ]
+        },
+        {
+            uri: 'http://www.yso.fi/onto/yso/p6197061979',
+            prefLabel: [
+                { lang: 'fi', value: '1970-luku' },
+                { lang: 'sv', value: '1970-talet' }
+            ]
+        },
+        {
+            uri: 'http://www.yso.fi/onto/yso/p6196061969',
+            prefLabel: [
+                { lang: 'fi', value: '1960-luku' },
+                { lang: 'sv', value: '1960-talet' }
+            ]
+        },
+        {
+            uri: 'http://urn.fi/URN:NBN:fi:au:slm:s161',
+            prefLabel: [
+                { lang: 'fi', value: 'naistenlehdet' },
+                { lang: 'sv', value: 'damtidningar' }
+            ]
+        }
+    ]
+
+};
+
+export const fakeData = JSON.stringify(fakeTerms);