@natlibfi/marc-record-validators-melinda 11.6.7 → 12.0.0-alpha.12

package/src/{normalize-qualifying-information.spec.js → normalize-qualifying-information.test.js}

@@ -1,20 +1,22 @@
-import {expect} from 'chai';
+import assert from 'node:assert';
 import {MarcRecord} from '@natlibfi/marc-record';
-import validatorFactory from './normalize-qualifying-information';
+import validatorFactory from './normalize-qualifying-information.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
 import createDebugLogger from 'debug';
 
 generateTests({
   callback,
-  path: [__dirname, '..', 'test-fixtures', 'normalize-qualifying-information'],
+  path: [import.meta.dirname, '..', 'test-fixtures', 'normalize-qualifying-information'],
   useMetadataFile: true,
   recurse: false,
   fixura: {
     reader: READERS.JSON
   },
-  mocha: {
-    before: () => testValidatorFactory()
+  hooks: {
+    before: async () => {
+      testValidatorFactory();
+    }
   }
 });
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-qualifying-information:test');
@@ -22,12 +24,9 @@ const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normal
 async function testValidatorFactory() {
   const validator = await validatorFactory();
 
-  expect(validator)
-    .to.be.an('object')
-    .that.has.any.keys('description', 'validate');
-
-  expect(validator.description).to.be.a('string');
-  expect(validator.validate).to.be.a('function');
+  assert.equal(typeof validator, 'object');
+  assert.equal(typeof validator.description, 'string');
+  assert.equal(typeof validator.validate, 'function');
 }
 
 async function callback({getFixture, enabled = true, fix = false}) {
@@ -43,10 +42,10 @@ async function callback({getFixture, enabled = true, fix = false}) {
 
   if (!fix) {
     const result = await validator.validate(record);
-    expect(result).to.eql(expectedResult);
+    assert.deepEqual(result, expectedResult);
     return;
   }
 
   await validator.fix(record);
-  expect(record).to.eql(expectedResult);
+  assert.deepEqual(record, expectedResult);
 }

package/src/normalize-utf8-diacritics.js

@@ -1,7 +1,7 @@
 //import createDebugLogger from 'debug';
 import clone from 'clone';
-import {convert as nongenericNormalization} from './unicode-decomposition';
-import {fieldToString} from './utils';
+import {convert as nongenericNormalization} from './unicode-decomposition.js';
+import {fieldToString} from './utils.js';
 
 // Note that https://github.com/NatLibFi/marc-record-validators-melinda/blob/master/src/unicode-decomposition.js contains
 // similar functionalities. It's less generic and lacks diacritic removal but has it advantages as well.

package/src/{normalize-utf8-diacritics.spec.js → normalize-utf8-diacritics.test.js}

@@ -1,33 +1,33 @@
-import {expect} from 'chai';
+import assert from 'node:assert';
 import {MarcRecord} from '@natlibfi/marc-record';
-import validatorFactory from './normalize-utf8-diacritics';
+import validatorFactory from './normalize-utf8-diacritics.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
 import createDebugLogger from 'debug';
 
 generateTests({
   callback,
-  path: [__dirname, '..', 'test-fixtures', 'normalize-utf8-diacritics'],
+  path: [import.meta.dirname, '..', 'test-fixtures', 'normalize-utf8-diacritics'],
   useMetadataFile: true,
   recurse: false,
   fixura: {
     reader: READERS.JSON
   },
-  mocha: {
-    before: () => testValidatorFactory()
+  hooks: {
+    before: async () => {
+      testValidatorFactory();
+    }
   }
 });
+
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-utf8-diacritics:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
 
-  expect(validator)
-    .to.be.an('object')
-    .that.has.any.keys('description', 'validate');
-
-  expect(validator.description).to.be.a('string');
-  expect(validator.validate).to.be.a('function');
+  assert.equal(typeof validator, 'object');
+  assert.equal(typeof validator.description, 'string');
+  assert.equal(typeof validator.validate, 'function');
 }
 
 async function callback({getFixture, enabled = true, fix = false}) {
@@ -43,10 +43,10 @@ async function callback({getFixture, enabled = true, fix = false}) {
 
   if (!fix) {
     const result = await validator.validate(record);
-    expect(result).to.eql(expectedResult);
+    assert.deepEqual(result, expectedResult);
     return;
   }
 
   await validator.fix(record);
-  expect(record).to.eql(expectedResult);
+  assert.deepEqual(record, expectedResult);
 }

package/src/normalizeFieldForComparison.js

@@ -7,18 +7,21 @@
   Thus it is here. However, most of the testing is done via merge-reducers...
 */
 import clone from 'clone';
-import {fieldStripPunctuation} from './punctuation2';
+import {fieldStripPunctuation} from './punctuation2.js';
 import {fieldToString, isControlSubfieldCode} from './utils.js';
 
-import {fieldNormalizeControlNumbers/*, normalizeControlSubfieldValue*/} from './normalize-identifiers';
+import {fieldNormalizeControlNumbers/*, normalizeControlSubfieldValue*/} from './normalize-identifiers.js';
 import createDebugLogger from 'debug';
-import {normalizePartData, subfieldContainsPartData} from './normalizeSubfieldValueForComparison';
+import {normalizePartData, subfieldContainsPartData} from './normalizeSubfieldValueForComparison.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:normalizeFieldForComparison');
 //const debugData = debug.extend('data');
 const debugDev = debug.extend('dev');
 
 export function isEnnakkotietoSubfieldG(subfield) {
+  if (valuelessSubfield(subfield)) {
+    return false;
+  }
   if (subfield.code !== 'g') {
     return false;
   }
@@ -116,6 +119,9 @@ function subfieldValueLowercase(value, subfieldCode, tag) {
 }
 
 function subfieldLowercase(sf, tag) {
+  if (valuelessSubfield(sf)) {
+    return;
+  }
   sf.value = subfieldValueLowercase(sf.value, sf.code, tag);
 }
 
@@ -124,7 +130,7 @@ function fieldLowercase(field) {
     return;
   }
 
-  field.subfields.forEach(sf => subfieldLowercase(sf, field.tag)); // eslint-disable-line array-callback-return
+  field.subfields.forEach(sf => subfieldLowercase(sf, field.tag));
 
   function skipFieldLowercase(field) {
     if (skipAllFieldNormalizations(field.tag)) {
@@ -144,10 +150,14 @@ function hack490SubfieldA(field) {
   if (field.tag !== '490') {
     return;
   }
-  field.subfields.forEach(sf => removeSarja(sf)); // eslint-disable-line array-callback-return
+  field.subfields.forEach(sf => removeSarja(sf));
 
   // NB! This won't work, if the punctuation has not been stripped beforehand!
   function removeSarja(subfield) {
+    if (valuelessSubfield(subfield)) {
+      return;
+    }
+
     if (subfield.code !== 'a') {
       return;
     }
@@ -185,9 +195,12 @@ function normalizeISBN(field) {
 
   //nvdebug(`ISBN-field? ${fieldToString(field)}`);
   const relevantSubfields = field.subfields.filter(sf => tagAndSubfieldCodeReferToIsbn(field.tag, sf.code) && looksLikeIsbn(sf.value));
-  relevantSubfields.forEach(sf => normalizeIsbnSubfield(sf)); // eslint-disable-line array-callback-return
+  relevantSubfields.forEach(sf => normalizeIsbnSubfield(sf));
 
   function normalizeIsbnSubfield(sf) {
+    if (valuelessSubfield(sf)) {
+      return;
+    }
     //nvdebug(` ISBN-subfield? ${subfieldToString(sf)}`);
     sf.value = sf.value.replace(/-/ug, '');
     sf.value = sf.value.replace(/x/u, 'X');
@@ -202,6 +215,9 @@ function fieldSpecificHacks(field) {
 
 export function fieldTrimSubfieldValues(field) {
   field.subfields?.forEach((sf) => {
+    if (valuelessSubfield(sf)) {
+      return;
+    }
     sf.value = sf.value.replace(/^[ \t\n]+/u, '');
     sf.value = sf.value.replace(/[ \t\n]+$/u, '');
     sf.value = sf.value.replace(/[ \t\n]+/gu, ' ');
@@ -212,6 +228,9 @@ function fieldRemoveDecomposedDiacritics(field) {
   // Raison d'être/motivation: "Sirén" and diacriticless "Siren" might refer to a same surname, so this normalization
   // allows us to compare authors and avoid duplicate fields.
   field.subfields.forEach((sf) => {
+    if (valuelessSubfield(sf)) {
+        return;
+    }
     sf.value = removeDecomposedDiacritics(sf.value);
   });
 }
@@ -297,6 +316,9 @@ export function cloneAndNormalizeFieldForComparison(field) {
     return clonedField;
   }
   clonedField.subfields.forEach((sf) => { // Do this for all fields or some fields?
+    if (valuelessSubfield(sf)) {
+      return;
+    }
     sf.value = normalizeSubfieldValue(sf.value, sf.code, field.tag);
     sf.value = removeCharsThatDontCarryMeaning(sf.value, field.tag, sf.code);
   });
@@ -318,3 +340,7 @@ function fieldSkipNormalization(field) {
   }
   return false;
 }
+
+function valuelessSubfield(sf) {
+  return sf.value === undefined;
+}

package/src/normalizeSubfieldValueForComparison.js

@@ -1,4 +1,4 @@
-import {nvdebug} from './utils';
+import {nvdebug} from './utils.js';
 import createDebugLogger from 'debug';
 
 // Normalizes at least 490$v and 773$g which contain information such as "Raita 5" vs "5", and "Osa 3" vs "Osa III".

package/src/prepublicationUtils.js

@@ -1,4 +1,4 @@
-import {fieldHasSubfield, nvdebug, nvdebugFieldArray} from './utils';
+import {fieldHasSubfield, nvdebug, nvdebugFieldArray} from './utils.js';
 import createDebugLogger from 'debug';
 
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:prepublicationUtils');
@@ -178,7 +178,7 @@ export function deleteAllPrepublicationNotesFromField500InNonPubRecord(record) {
 
 
   nvdebug(`Delete all ${f500.length} instance(s) of field 500`, debugDev);
-  f500.forEach(field => record.removeField(field)); // eslint-disable-line array-callback-return
+  f500.forEach(field => record.removeField(field));
 }
 
 
@@ -188,7 +188,7 @@ export function removeWorsePrepubField500s(record) {
   nvdebugFieldArray(fields, '  Candidates for non-best 500 b4 filtering: ', debugDev);
   const nonBest = fields.filter(field => fields.some(field2 => firstFieldHasBetterPrepubEncodingLevel(field2, field)));
   nvdebugFieldArray(nonBest, '  Remove non-best 500: ', debugDev);
-  nonBest.forEach(field => record.removeField(field)); // eslint-disable-line array-callback-return
+  nonBest.forEach(field => record.removeField(field));
 }
 
 
@@ -198,7 +198,7 @@ export function removeWorsePrepubField594s(record) {
   nvdebugFieldArray(fields594, '  Candidates for non-best 594 b4 filtering: ', debugDev);
   const nonBest = fields594.filter(field => fields594.some(field2 => firstFieldHasBetterPrepubEncodingLevel(field2, field)));
   nvdebugFieldArray(nonBest, '  Remove non-best 594: ', debugDev);
-  nonBest.forEach(field => record.removeField(field)); // eslint-disable-line array-callback-return
+  nonBest.forEach(field => record.removeField(field));
 }
 
 

package/src/punctuation/index.js

@@ -1,6 +1,6 @@
 import {MarcRecord} from '@natlibfi/marc-record';
 import createDebug from 'debug';
-import {autRules, bibRules} from './rules';
+import {autRules, bibRules} from './rules/index.js';
 
 const debug = createDebug('@natlibfi/marc-record-validator-melinda/punctuation:dev');
 

package/src/punctuation/rules/index.js

@@ -1,5 +1,5 @@
-import bibRules from './bib';
-import autRules from './aut';
+import bibRules from './bib.js';
+import autRules from './aut.js';
 
 export {
   bibRules,

package/src/{punctuation.spec.js → punctuation.test.js}

@@ -1,20 +1,22 @@
-import {expect} from 'chai';
+import assert from 'node:assert';
 import {MarcRecord} from '@natlibfi/marc-record';
-import validatorFactory from './punctuation/';
+import validatorFactory from './punctuation/index.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
 import createDebugLogger from 'debug';
 
 generateTests({
   callback,
-  path: [__dirname, '..', 'test-fixtures', 'punctuation'],
+  path: [import.meta.dirname, '..', 'test-fixtures', 'punctuation'],
   useMetadataFile: true,
   recurse: false,
   fixura: {
     reader: READERS.JSON
   },
-  mocha: {
-    before: () => testValidatorFactory()
+  hooks: {
+    before: async () => {
+      testValidatorFactory();
+    }
   }
 });
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/punctuation:test');
@@ -22,12 +24,9 @@ const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/punctu
 async function testValidatorFactory() {
   const validator = await validatorFactory();
 
-  expect(validator)
-    .to.be.an('object')
-    .that.has.any.keys('description', 'validate');
-
-  expect(validator.description).to.be.a('string');
-  expect(validator.validate).to.be.a('function');
+  assert.equal(typeof validator, 'object');
+  assert.equal(typeof validator.description, 'string');
+  assert.equal(typeof validator.validate, 'function');
 }
 
 async function callback({getFixture, enabled = true, fix = false}) {
@@ -43,10 +42,10 @@ async function callback({getFixture, enabled = true, fix = false}) {
 
   if (!fix) {
     const result = await validator.validate(record);
-    expect(result).to.eql(expectedResult);
+    assert.deepEqual(result, expectedResult);
     return;
   }
 
   await validator.fix(record);
-  expect(record).to.eql(expectedResult);
+  assert.deepEqual(record, expectedResult);
 }

package/src/punctuation2.js

@@ -9,10 +9,11 @@
 * NOTE #3: As of 2023-06-05 control subfields ($0...$9) are obsolete. Don't use them in rules.
 *          (They are jumped over when looking for next (non-controlfield subfield)
 */
-import {validateSingleField} from './ending-punctuation';
-import {fieldGetUnambiguousTag} from './subfield6Utils';
+import {validateSingleField} from './ending-punctuation.js';
+import {tagToDataProvenanceSubfieldCode} from './merge-fields/dataProvenance.js';
+import {fieldGetUnambiguousTag} from './subfield6Utils.js';
 //import createDebugLogger from 'debug';
-import {fieldToString, nvdebug} from './utils';
+import {fieldToString, isControlSubfieldCode, nvdebug} from './utils.js';
 import clone from 'clone';
 
 //const debug = createDebugLogger('debug/punctuation2');
@@ -27,7 +28,7 @@ export default function () {
   function fix(record) {
     nvdebug(`${descriptionString}: fixer`);
     const res = {message: [], fix: [], valid: true};
-    record.fields.forEach(f => fieldFixPunctuation(f)); // eslint-disable-line array-callback-return
+    record.fields.forEach(f => fieldFixPunctuation(f));
     return res;
   }
 
@@ -49,12 +50,19 @@ export default function () {
   }
 }
 
-function isControlSubfield(subfield) {
-  return ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'].includes(subfield.code);
+
+
+function isIrrelevantSubfield(subfield, tag) {
+  const dataProvenanceSubfieldCode = tagToDataProvenanceSubfieldCode(tag);
+  if (subfield.code === dataProvenanceSubfieldCode) {
+    return true;
+  }
+  return isControlSubfieldCode(subfield.code); // Currently this contains other stuff as well ($3, $4, $7, $9...)
 }
 
+
 function getNextRelevantSubfield(field, currSubfieldIndex) {
-  return field.subfields.find((subfield, index) => index > currSubfieldIndex && !isControlSubfield(subfield));
+  return field.subfields.find((subfield, index) => index > currSubfieldIndex && !isIrrelevantSubfield(subfield, field.tag));
 }
 
 export function fieldGetFixedString(field, add = true) {
@@ -155,7 +163,8 @@ const remove490And830Whatever = [{'code': 'axyzv', 'followedBy': 'axyzv', 'remov
 const linkingEntryRemoveWhatever = [
   {'code': 'i', 'followedBy': 'at', 'remove': / ?:$/u}, // ':'
   {'code': 'at', 'remove': /\.$/u},
-  {'code': 'abdghiklmnopqrstuwxyz', 'followedBy': 'abdghiklmnopqrstuwxyz', 'remove': /\. -$/u}
+  // Only ". -" separator is still used in music. We can strip it, but can only create the non-music punctuation!
+  {'code': 'abdghiklmnopqrstuwxyz', 'followedBy': 'abdghiklmnopqrstuwxyz#', 'remove': /\. -$/u}
 ];
 
 

package/src/{punctuation2.spec.js → punctuation2.test.js}

@@ -1,20 +1,22 @@
-import {expect} from 'chai';
+import assert from 'node:assert';
 import {MarcRecord} from '@natlibfi/marc-record';
-import validatorFactory from './punctuation2';
+import validatorFactory from '../src/punctuation2.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
 import createDebugLogger from 'debug';
 
 generateTests({
   callback,
-  path: [__dirname, '..', 'test-fixtures', 'punctuation2'],
+  path: [import.meta.dirname, '..', 'test-fixtures', 'punctuation2'],
   useMetadataFile: true,
   recurse: false,
   fixura: {
     reader: READERS.JSON
   },
-  mocha: {
-    before: () => testValidatorFactory()
+  hooks: {
+    before: async () => {
+      testValidatorFactory();
+    }
   }
 });
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/punctuation2:test');
@@ -22,12 +24,9 @@ const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/punctu
 async function testValidatorFactory() {
   const validator = await validatorFactory();
 
-  expect(validator)
-    .to.be.an('object')
-    .that.has.any.keys('description', 'validate');
-
-  expect(validator.description).to.be.a('string');
-  expect(validator.validate).to.be.a('function');
+  assert.equal(typeof validator, 'object');
+  assert.equal(typeof validator.description, 'string');
+  assert.equal(typeof validator.validate, 'function');
 }
 
 async function callback({getFixture, enabled = true, fix = false}) {
@@ -43,10 +42,10 @@ async function callback({getFixture, enabled = true, fix = false}) {
 
   if (!fix) {
     const result = await validator.validate(record);
-    expect(result).to.eql(expectedResult);
+    assert.deepEqual(result, expectedResult);
     return;
   }
 
   await validator.fix(record);
-  expect(record).to.eql(expectedResult);
+  assert.deepEqual(record, expectedResult);
 }

package/src/reindexSubfield6OccurenceNumbers.js

@@ -1,8 +1,8 @@
 import createDebugLogger from 'debug';
-import {fieldHasSubfield, fieldToString, nvdebug} from './utils';
+import {fieldHasSubfield, fieldToString, nvdebug} from './utils.js';
 import {fieldGetOccurrenceNumberPairs, fieldGetUnambiguousOccurrenceNumber, fieldResetOccurrenceNumber, intToOccurrenceNumberString, isValidSubfield6,
   recordGetMaxSubfield6OccurrenceNumberAsInteger,
-  subfield6GetOccurrenceNumber, subfield6GetOccurrenceNumberAsInteger, subfield6ResetOccurrenceNumber} from './subfield6Utils';
+  subfield6GetOccurrenceNumber, subfield6GetOccurrenceNumberAsInteger, subfield6ResetOccurrenceNumber} from './subfield6Utils.js';
 
 // Relocated from melinda-marc-record-merge-reducers (and renamed)
 
@@ -91,7 +91,7 @@ function recordDisambiguateSharedSubfield6OccurrenceNumbers(record) {
     return;
   }
   nvdebug(`Disambiguate occurrence numbers (N=${sharedOccurrenceNumberFields.length}) in...`, debug);
-  sharedOccurrenceNumberFields.forEach(field => disambiguateOccurrenceNumber(field)); // eslint-disable-line array-callback-return
+  sharedOccurrenceNumberFields.forEach(field => disambiguateOccurrenceNumber(field));
 
   function disambiguateable(field) {
     if (field.tag === '880') { // Not needed, already filtered...
@@ -131,7 +131,7 @@ function recordDisambiguateSharedSubfield6OccurrenceNumbers(record) {
     nvdebug(` Reindex '${fieldToString(field)}' occurrence number and it's ${pairedFields.length} pair(s) using '${newOccurrenceNumber}'`, debug);
 
     fieldResetOccurrenceNumber(field, newOccurrenceNumber, occurrenceNumber);
-    pairedFields.forEach(pairedField => fieldResetOccurrenceNumber(pairedField, newOccurrenceNumber, occurrenceNumber)); // eslint-disable-line array-callback-return
+    pairedFields.forEach(pairedField => fieldResetOccurrenceNumber(pairedField, newOccurrenceNumber, occurrenceNumber));
 
   }
 
@@ -139,7 +139,6 @@ function recordDisambiguateSharedSubfield6OccurrenceNumbers(record) {
 }
 function recordGetNumberOfUniqueSubfield6OccurrenceNumbers(record) {
   // Calculates the number of used different occurrence numbers
-  /* eslint-disable */
   let indexArray = [];
   record.fields.forEach(field => gatherFieldData(field));
 
@@ -161,8 +160,7 @@ function recordGetNumberOfUniqueSubfield6OccurrenceNumbers(record) {
     indexArray[i] = 1;
   }
   let n = 0;
-  indexArray.forEach(elem => n+= elem); 
-  /* eslint-enable */
+  indexArray.forEach(elem => n+= elem);
   return n;
 }
 

package/src/{reindexSubfield6OccurenceNumbers.spec.js → reindexSubfield6OccurenceNumbers.test.js}

@@ -1,20 +1,22 @@
-import {expect} from 'chai';
+import assert from 'node:assert';
 import {MarcRecord} from '@natlibfi/marc-record';
-import validatorFactory from './reindexSubfield6OccurenceNumbers';
+import validatorFactory from './reindexSubfield6OccurenceNumbers.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
 import createDebugLogger from 'debug';
 
 generateTests({
   callback,
-  path: [__dirname, '..', 'test-fixtures', 'reindex-sf6-occurence-numbers'],
+  path: [import.meta.dirname, '..', 'test-fixtures', 'reindex-sf6-occurence-numbers'],
   useMetadataFile: true,
   recurse: false,
   fixura: {
     reader: READERS.JSON
   },
-  mocha: {
-    before: () => testValidatorFactory()
+  hooks: {
+    before: async () => {
+      testValidatorFactory();
+    }
   }
 });
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/reindexSubfield6OccurrenceNumbers:test');
@@ -22,12 +24,9 @@ const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/reinde
 async function testValidatorFactory() {
   const validator = await validatorFactory();
 
-  expect(validator)
-    .to.be.an('object')
-    .that.has.any.keys('description', 'validate');
-
-  expect(validator.description).to.be.a('string');
-  expect(validator.validate).to.be.a('function');
+  assert.equal(typeof validator, 'object');
+  assert.equal(typeof validator.description, 'string');
+  assert.equal(typeof validator.validate, 'function');
 }
 
 async function callback({getFixture, enabled = true, fix = false}) {
@@ -43,10 +42,10 @@ async function callback({getFixture, enabled = true, fix = false}) {
 
   if (!fix) {
     const result = await validator.validate(record);
-    expect(result).to.eql(expectedResult);
+    assert.deepEqual(result, expectedResult);
     return;
   }
 
   await validator.fix(record);
-  expect(record).to.eql(expectedResult);
+  assert.deepEqual(record, expectedResult);
 }