@natlibfi/marc-record-validators-melinda 12.1.0-alpha.1 → 13.0.0

package/src/mergeRelatorTermFields.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './mergeRelatorTermFields.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -19,7 +19,7 @@ generateTests({
     }
   }
 });
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/mergeRelatorTermFields:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/mergeRelatorTermFields:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -29,12 +29,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/multiple-subfield-0.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './multiple-subfield-0.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -20,7 +20,7 @@ generateTests({
   }
 });
 
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/multiple-subfield-0:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/multiple-subfield-0:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -30,12 +30,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/normalize-dashes.js

@@ -1,6 +1,12 @@
-//import createDebugLogger from 'debug';
 import clone from 'clone';
-import {fieldToString, isControlSubfieldCode, nvdebug} from './utils.js';
+import {fieldToString, isContentSubfieldCode, nvdebug} from './utils.js';
+
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:normaliza-dashes');
+//const debugData = debug.extend('data');
+const debugDev = debug.extend('dev');
+
 
 // Author(s): Nicholas Volk
 export default function () {
@@ -11,7 +17,7 @@ export default function () {
   };
 
   function fix(record) {
-    nvdebug(`FIX ME`);
+    nvdebug(`FIX ME`, debugDev);
     record.fields.forEach(field => {
       fixDashes(field);
     });
@@ -23,7 +29,7 @@ export default function () {
   function validate(record) {
     const res = {message: []};
 
-    nvdebug(`VALIDATE ME`);
+    nvdebug(`VALIDATE ME`, debugDev);
     record.fields?.forEach(field => {
       validateField(field, res);
     });
@@ -34,7 +40,7 @@ export default function () {
 
   function validateField(field, res) {
     const orig = fieldToString(field);
-    nvdebug(` VALIDATE FIELD '${orig}'`);
+    nvdebug(` VALIDATE FIELD '${orig}'`, debugDev);
 
     const normalizedField = fixDashes(clone(field));
     const mod = fieldToString(normalizedField);
@@ -52,14 +58,14 @@ function fixDashes(field) {
     return field;
   }
 
-  nvdebug(`Dashing ${fieldToString(field)}`);
+  nvdebug(`Dashing ${fieldToString(field)}`, debugDev);
 
   field.subfields.forEach(sf => subfieldFixDashes(sf));
 
   return field;
 
   function subfieldFixDashes(subfield) {
-    if (isControlSubfieldCode(subfield.code)) {
+    if (!isContentSubfieldCode(subfield.code, field.tag)) {
       return;
     }
     // Normalize dashes U+2010 ... U+2015 to '-':

package/src/normalize-dashes.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './normalize-dashes.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -19,7 +19,7 @@ generateTests({
     }
   }
 });
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-dashes:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-dashes:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -29,12 +29,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/normalize-identifiers.js

@@ -1,17 +1,10 @@
 // Relocated from melinda-marc-record-merge-reducers (and renamed)
-//import createDebugLogger from 'debug';
 import clone from 'clone';
-import {fieldToString} from './utils.js';
-//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:normalize-identifiers');
+import {fieldToString, /*nvdebug*/} from './utils.js';
 
-/*
-function nvdebug(message, func) {
-  if (func) {
-    func(message);
-  }
-  console.info(message); // eslint-disable-line no-console
-}
-*/
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:normalize-identifiers');
+//const debugData = debug.extend('data');
+//const debugDev = debug.extend('dev');
 
 export default function () {
 
@@ -32,9 +25,9 @@ export default function () {
       }
       */
 
-    //nvdebug(`NORMALIZE CONTROL NUMBER FIX`, debug);
+    //nvdebug(`NORMALIZE CONTROL NUMBER FIX`, debugDev);
     record.fields.forEach(field => {
-      //nvdebug(` NORMALIZE CONTROL NUMBER FIX ${fieldToString(field)}`, debug);
+      //nvdebug(` NORMALIZE CONTROL NUMBER FIX ${fieldToString(field)}`, debugDev);
 
       fieldNormalizeControlNumbers(field);
       //validateField(field, true, message);
@@ -46,7 +39,7 @@ export default function () {
 
   function validate(record) {
     const res = {message: []};
-    //nvdebug(`NORMALIZE CONTROL NUMBER VALIDATE`, debug);
+    //nvdebug(`NORMALIZE CONTROL NUMBER VALIDATE`, debugDev);
     // Actual parsing of all fields
     /*
       if (!record.fields) {
@@ -55,7 +48,7 @@ export default function () {
       */
 
     record.fields.forEach(field => {
-      //nvdebug(` NORMALIZE CONTROL NUMBER VALIDATE ${fieldToString(field)}`, debug);
+      //nvdebug(` NORMALIZE CONTROL NUMBER VALIDATE ${fieldToString(field)}`, debugDev);
       validateField(field, res);
     });
 
@@ -126,8 +119,8 @@ function normalizeNineDigitIDs(value, targetFormat = 'ALEPH_INTERNAL') {
   const currPrefix = value.slice(0, -9);
 
   if (currPrefix in mappings) {
-    //nvdebug(`${currPrefix}, TF:${targetFormat}...`);
-    //nvdebug(`${JSON.stringify(mappings[currPrefix])}`);
+    //nvdebug(`${currPrefix}, TF:${targetFormat}...`, debugDev);
+    //nvdebug(`${JSON.stringify(mappings[currPrefix])}`, debugDev);
     return `${mappings[currPrefix][targetFormat]}${nineDigitTail}`;
   }
   return value;
@@ -167,7 +160,7 @@ export function normalizeControlSubfieldValue(value = '', targetFormat = 'ALEPH_
 
 //export function normalizableSubfieldPrefix(tag, sf) {
 export function normalizeAs(tag, subfieldCode) {
-  //nvdebug(`nAs ${tag}, ${subfieldCode}`);
+  //nvdebug(`nAs ${tag}, ${subfieldCode}`, debugDev);
   if (subfieldCode === '0' || subfieldCode === '1' || subfieldCode === 'w') {
     return 'ALEPH_INTERNAL';
   }
@@ -189,7 +182,7 @@ export function fieldNormalizeControlNumbers(field) {
   field.subfields.forEach(sf => {
     const targetFormat = normalizeAs(field.tag, sf.code);
     if (targetFormat !== undefined) {
-      //nvdebug(`NORMALIZE SUBFIELD $${sf.code} IN FIELD: '${fieldToString(field)}' TO ${targetFormat}`);
+      //nvdebug(`NORMALIZE SUBFIELD $${sf.code} IN FIELD: '${fieldToString(field)}' TO ${targetFormat}`, debugDev);
       sf.value = normalizeControlSubfieldValue(sf.value, targetFormat);
       return;
     }

package/src/normalize-identifiers.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './normalize-identifiers.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -19,7 +19,7 @@ generateTests({
     }
   }
 });
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-identifiers:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-identifiers:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -29,12 +29,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/normalize-qualifying-information.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './normalize-qualifying-information.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -19,7 +19,7 @@ generateTests({
     }
   }
 });
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-qualifying-information:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-qualifying-information:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -29,12 +29,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/normalize-utf8-diacritics.js

@@ -1,12 +1,15 @@
 //import createDebugLogger from 'debug';
 import clone from 'clone';
 import {convert as nongenericNormalization} from './unicode-decomposition.js';
-import {fieldToString} from './utils.js';
+import {fieldToString /*, nvdebug */} from './utils.js';
 
 // Note that https://github.com/NatLibFi/marc-record-validators-melinda/blob/master/src/unicode-decomposition.js contains
 // similar functionalities. It's less generic and lacks diacritic removal but has it advantages as well.
 
 //const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-utf-diacritics');
+//const debugData = debug.extend('data');
+//const debugDev = debug.extend('dev');
+
 
 // See also https://github.com/NatLibFi/marc-record-validators-melinda/blob/master/src/unicode-decomposition.js .
 // It uses a list of convertable characters whilst this uses a generic stuff as well.
@@ -111,13 +114,13 @@ export function fieldFixComposition(field) {
     return field;
   }
   //const originalValue = fieldToString(field);
-  //nvdebug(`fFC: '${originalValue}'`, debug);
+  //nvdebug(`fFC: '${originalValue}'`, debugDev);
   field.subfields.forEach((subfield, index) => {
     field.subfields[index].value = fixComposition(subfield.value);
   });
   //const newValue = fieldToString(field);
   //if (originalValue !== newValue) {
-  //  debug(`FIXCOMP: '${originalValue}' => '${newValue}'`);
+  //  nvdebug(`FIXCOMP: '${originalValue}' => '${newValue}'`, debugDev);
   //}
   return field;
 }

package/src/normalize-utf8-diacritics.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './normalize-utf8-diacritics.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -20,7 +20,7 @@ generateTests({
   }
 });
 
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-utf8-diacritics:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-utf8-diacritics:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -30,12 +30,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/normalizeFieldForComparison.js

@@ -8,26 +8,18 @@
 */
 import clone from 'clone';
 import {fieldStripPunctuation} from './punctuation2.js';
-import {fieldToString, isControlSubfieldCode} from './utils.js';
+import {fieldToString, isContentSubfieldCode} from './utils.js';
 
 import {fieldNormalizeControlNumbers/*, normalizeControlSubfieldValue*/} from './normalize-identifiers.js';
 import createDebugLogger from 'debug';
 import {normalizePartData, subfieldContainsPartData} from './normalizeSubfieldValueForComparison.js';
+import {isEnnakkotietoSubfield} from './prepublicationUtils.js';
+import {getSynonym} from './merge-fields/worldKnowledge.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:normalizeFieldForComparison');
 //const debugData = debug.extend('data');
 const debugDev = debug.extend('dev');
 
-export function isEnnakkotietoSubfieldG(subfield) {
-  if (valuelessSubfield(subfield)) {
-    return false;
-  }
-  if (subfield.code !== 'g') {
-    return false;
-  }
-  return subfield.value.match(/^ENNAKKOTIETO\.?$/gui);
-}
-
 function debugFieldComparison(oldField, newField) { // NB: Debug-only function!
   /*
   // We may drop certain subfields:
@@ -60,7 +52,7 @@ function containsHumanName(tag = '???', subfieldCode = undefined) {
 }
 
 function containsCorporateName(tag = '???', subfieldCode = undefined) {
-  // NB! This set is for bibs! Auth has 400... What else...
+  // NB! This set is for bibs! Auth has 410... What else...
   if (['110', '610', '710', '810'].includes(tag)) {
     if (subfieldCode === undefined || subfieldCode === 'a') {
       return true;
@@ -72,15 +64,15 @@ function containsCorporateName(tag = '???', subfieldCode = undefined) {
 
 function skipAllSubfieldNormalizations(value, subfieldCode, tag) {
 
-  if (isEnnakkotietoSubfieldG({'code': subfieldCode, value})) {
+  if (isEnnakkotietoSubfield({'code': subfieldCode, value})) {
     return true;
   }
 
-  if (tag === '035' && ['a', 'z'].includes(subfieldCode)) { // A
+  if (tag === '035' && ['a', 'z'].includes(subfieldCode)) {
     return true;
   }
 
-  if (isControlSubfieldCode(subfieldCode)) {
+  if (!isContentSubfieldCode(subfieldCode, tag)) {
     return true;
   }
   return false;
@@ -145,6 +137,23 @@ function fieldLowercase(field) {
   }
 }
 
+function normalize300SubfieldA(field) {
+  if (field.tag !== '300') {
+    return;
+  }
+  field.subfields.forEach(sf => normalizePages(sf));
+
+  function normalizePages(subfield) {
+    if (valuelessSubfield(subfield)) {
+      return;
+    }
+
+    if (subfield.code !== 'a') {
+      return;
+    }
+    subfield.value = subfield.value.replace(/(?:\b(?:pages?|pp\.|Seite|sida|sidor|sivu|sivua)\b|\bs$)/g, 's.');
+  }
+}
 
 function hack490SubfieldA(field) {
   if (field.tag !== '490') {
@@ -193,7 +202,7 @@ function normalizeISBN(field) {
     return;
   }
 
-  //nvdebug(`ISBN-field? ${fieldToString(field)}`);
+  //nvdebug(`ISBN-field? ${fieldToString(field)}`, debugDev);
   const relevantSubfields = field.subfields.filter(sf => tagAndSubfieldCodeReferToIsbn(field.tag, sf.code) && looksLikeIsbn(sf.value));
   relevantSubfields.forEach(sf => normalizeIsbnSubfield(sf));
 
@@ -201,7 +210,7 @@ function normalizeISBN(field) {
     if (valuelessSubfield(sf)) {
       return;
     }
-    //nvdebug(` ISBN-subfield? ${subfieldToString(sf)}`);
+    //nvdebug(` ISBN-subfield? ${subfieldToString(sf)}`, debugDev);
     sf.value = sf.value.replace(/-/ug, '');
     sf.value = sf.value.replace(/x/u, 'X');
   }
@@ -210,6 +219,7 @@ function normalizeISBN(field) {
 
 function fieldSpecificHacks(field) {
   normalizeISBN(field); // 020$a, not $z!
+  normalize300SubfieldA(field);
   hack490SubfieldA(field);
 }
 
@@ -244,17 +254,20 @@ function removeDecomposedDiacritics(value = '') {
 
 function normalizeSubfieldValue(value, subfieldCode, tag) {
   // NB! For comparison of values only
-  /* eslint-disable */
+  /* beslint-disable */
+  value = removeCharsThatDontCarryMeaning(value, tag, subfieldCode);
+  value = getSynonym(tag, subfieldCode, value); // Must be done before punc stripping and lowercasing...
   value = subfieldValueLowercase(value, subfieldCode, tag);
 
+
   // Normalize: s. = sivut = pp.
   value = normalizePartData(value, subfieldCode, tag);
-  value = value.replace(/^\[([^[\]]+)\]/gu, '$1'); // eslint-disable-line functional/immutable-data
+  value = value.replace(/^\[([^[\]]+)\]/gu, '$1');
 
   if (['130', '730'].includes(tag) && subfieldCode === 'a') {
     value = value.replace(' : ', ', '); // "Halloween ends (elokuva, 2022)" vs "Halloween ends (elokuva : 2023)"
   }
-  /* eslint-enable */
+  /* beslint-enable */
 
   // Not going to do these in the foreseeable future, but keeping them here for discussion:
   // Possible normalizations include but are not limited to:
@@ -311,7 +324,8 @@ function normalizeField(field) {
 export function cloneAndNormalizeFieldForComparison(field) {
   // NB! This new field is for comparison purposes only.
   // Some of the normalizations might be considered a bit overkill for other purposes.
-  const clonedField = clone(field);
+  const clonedField = cloneAndRemovePunctuation(field); // was only clone(field)
+
   if (fieldSkipNormalization(field)) {
     return clonedField;
   }
@@ -319,8 +333,11 @@ export function cloneAndNormalizeFieldForComparison(field) {
     if (valuelessSubfield(sf)) {
       return;
     }
+
     sf.value = normalizeSubfieldValue(sf.value, sf.code, field.tag);
-    sf.value = removeCharsThatDontCarryMeaning(sf.value, field.tag, sf.code);
+    //sf.value = normalizeForSamenessCheck(field.tag, sf.code, sf.value);
+
+
   });
 
   normalizeField(clonedField);

package/src/normalizeSubfieldValueForComparison.js

@@ -105,8 +105,8 @@ export function partsAgree(value1, value2, tag, subfieldCode) {
   }
   const [partType1, partYear1, partNumber1] = splitAndNormalizePartData(value1);
   const [partType2, partYear2, partNumber2] = splitAndNormalizePartData(value2);
-  //nvdebug(`P1: ${partType1} | ${partYear1} | ${partNumber1}`);
-  //nvdebug(`P2: ${partType2} | ${partYear2} | ${partNumber2}`);
+  //nvdebug(`P1: ${partType1} | ${partYear1} | ${partNumber1}`, debugDev);
+  //nvdebug(`P2: ${partType2} | ${partYear2} | ${partNumber2}`, debugDev);
   if (partNumber1 !== partNumber2) {
     return false;
   }

package/src/prepublicationUtils.js

@@ -203,7 +203,7 @@ export function removeWorsePrepubField594s(record) {
 
 
 export function isEnnakkotietoSubfield(subfield) {
-  if (subfield.code !== '9' && subfield.code !== 'g') {
+  if (!['g', '9', '7'].includes(subfield.code)) {
     return false;
   }
   // Length <= 13 allows punctuation, but does not require it:

package/src/punctuation.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './punctuation/index.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -19,7 +19,7 @@ generateTests({
     }
   }
 });
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/punctuation:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/punctuation:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -29,12 +29,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');