@natlibfi/marc-record-validators-melinda 12.0.0-alpha.9 → 12.0.1-alpha.1

package/src/sortFields.js

@@ -211,7 +211,7 @@ export function fieldOrderComparator(fieldA, fieldB) {
 
   //const sorterFunctions = [sortByTag, sortByIndexTerms, sortAlphabetically, sortByRelatorTerm, sortByOccurrenceNumber, preferFenniKeep, sortByFieldLinkAndSequenceNumber];
 
-  const sorterFunctions = [sortByTag, sortByIndexTerms, sortAlphabetically, sortByRelatorTerm, sortBySubfield6, preferFenniKeep, sortByFieldLinkAndSequenceNumber];
+  const sorterFunctions = [sortByTag, sortByIndexTerms, sortAlphabetically, sortByRelatorTerm, sortBySubfield6, preferFenniKeep, sortByFieldLinkAndSequenceNumber, sortByHacks];
   //const sorterFunctions = [sortByIndexTerms, sortByRelatorTerm, sortByOccurrenceNumber, preferFenniKeep, sortByFieldLinkAndSequenceNumber];
 
   return globalFieldOrderComparator(fieldA, fieldB, sorterFunctions);
@@ -390,6 +390,25 @@ function sortByFieldLinkAndSequenceNumber(fieldA, fieldB) { // Sort by subfield
   return -1;
 }
 
+function sortByHacks(fieldA, fieldB) {
+  // Place for various hackier rules
+
+  if (fieldA.tag === fieldB.tag) {
+    if (fieldA.tag === '041') {
+      // If we have multiple f041, we want to put the $2-less first; Values from $2 whatever can not be copied to 008/35-37. (See sync-language.js.)
+      const a2 = fieldA.subfields.some(sf => sf.code === '2');
+      const b2 = fieldB.subfields.some(sf => sf.code === '2');
+      if (a2 && !b2) {
+        return 1;
+      }
+      if (b2 && !a2) {
+        return -1;
+      }
+    }
+
+  }
+  return 0;
+}
 
 function sortBySubfield6(fieldA, fieldB) { // Sort by subfield $6, ex-sortByOccurrenceNumber...
   if (fieldA.tag !== '880' || fieldB.tag !== '880') {

package/src/sortFields.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './sortFields.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -19,7 +19,7 @@ generateTests({
     }
   }
 });
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sortFields:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sortFields:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -29,12 +29,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = true}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = true}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('input.json'));
   const expectedResult = getFixture('result.json');

package/src/sortRelatorTerms.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './sortRelatorTerms.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -20,7 +20,7 @@ generateTests({
   }
 });
 
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sortRelatorTerms:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sortRelatorTerms:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -30,12 +30,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/sortSubfields.js

@@ -136,7 +136,9 @@ function getSubfieldSortOrder(field) {
     debugDev(`sort order for ${field.tag}: ${entry[0].sortOrder}`);
     return entry[0].sortOrder;
   }
-  nvdebug(`WARNING!\tNo subfield order found for ${field.tag}.`);
+  if (!['300'].includes(field.tag)) { // Lis tags which use normal $a...$z order here!
+    nvdebug(`WARNING!\tNo subfield order found for ${field.tag}.`);
+  }
   return [];
 }
 

package/src/sortSubfields.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './sortSubfields.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -20,7 +20,7 @@ generateTests({
   }
 });
 
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sortSubfields:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sortSubfields:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -30,12 +30,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false, tagPattern = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false, tagPattern = false}) {
   const validator = await validatorFactory(tagPattern);
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/stripPunctuation.js

@@ -6,26 +6,29 @@
 */
 
 import {fieldGetFixedString, fieldNeedsModification, fieldStripPunctuation} from './punctuation2.js';
-// import createDebugLogger from 'debug';
-import {fieldToString, nvdebug} from './utils.js';
+import createDebugLogger from 'debug';
 
-// const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/punctuation2');
+import {fieldToString} from './utils.js';
+
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/punctuation2');
+
+const description =  'Strip punctuation in data fields';
 
 export default function () {
   return {
-    description: 'Strip punctuation to data fields',
+    description: description,
     validate, fix
   };
 
   function fix(record) {
-    nvdebug('Strip punctuation to data fields: fixer');
+    debug(`${description}: fixer`);
     const res = {message: [], fix: [], valid: true};
     record.fields.forEach(f => fieldStripPunctuation(f));
     return res;
   }
 
   function validate(record) {
-    nvdebug('Strip punctuation to data fields: validate');
+    debug(`${description}: validate`);
 
     const fieldsNeedingModification = record.fields.filter(f => fieldNeedsModification(f, false));
 

package/src/stripPunctuation.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './stripPunctuation.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -20,7 +20,7 @@ generateTests({
   }
 });
 
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/stripPunctuation:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/stripPunctuation:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -30,12 +30,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/subfield-exclusion.test.js

@@ -1,6 +1,6 @@
 import assert from 'node:assert';
 import {MarcRecord} from '@natlibfi/marc-record';
-import validatorFactory from '../src/subfield-exclusion.js';
+import validatorFactory from './subfield-exclusion.js';
 import {describe, it} from 'node:test';
 
 //chai.use(chaiAsPromised);

package/src/subfield6Utils.js

@@ -1,12 +1,12 @@
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 // const debug = createDebugLogger('@natlibfi/marc-record-validator-melinda/subfield6Utils');
 
 import {add8s, fieldsGetAllSubfield8LinkingNumbers, getSubfield8LinkingNumber, isValidSubfield8} from './subfield8Utils.js';
-import {fieldHasSubfield, fieldToString, fieldsToString, nvdebug, subfieldToString} from './utils.js';
+import {fieldHasSubfield, fieldToString, /* fieldsToString, */ nvdebug, subfieldToString} from './utils.js';
 
-const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:subfield6Utils');
+//const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:subfield6Utils');
 //const debugData = debug.extend('data');
-const debugDev = debug.extend('dev');
+//const debugDev = debug.extend('dev');
 
 // NB! Subfield 6 is non-repeatable and it should always comes first!
 // NB! Index size should always be 2 (preceding 0 required for 01..09) However, support for 100+ was added on 2023-02-27.
@@ -71,7 +71,7 @@ export function subfield6HasWantedTagAndOccurrenceNumber(subfield, tagAndOccurre
   }
   // We could also use generic code and go getTag()+'-'+getIndex() instead of regexp...
   const key = subfield.value.replace(/^([0-9][0-9][0-9]-[0-9][0-9]+).*$/u, '$1');
-  nvdebug(` Compare '${key}' vs '${tagAndOccurrenceNumber}'`);
+  //nvdebug(` Compare '${key}' vs '${tagAndOccurrenceNumber}'`);
   return key === tagAndOccurrenceNumber;
 }
 
@@ -222,7 +222,7 @@ export function fieldGetOccurrenceNumberPairs(field, candFields) {
     nvdebug(`NO PAIRS FOUND FOR '${fieldToString(field)}'`);
     return pairs;
   }
-  nvdebug(`${pairs.length} PAIR(S) FOUND FOR '${fieldToString(field)}'`);
+  //nvdebug(`${pairs.length} PAIR(S) FOUND FOR '${fieldToString(field)}'`);
   pairs.forEach(pairedField => nvdebug(`  '${fieldToString(pairedField)}'`));
   return pairs;
 }
@@ -357,7 +357,7 @@ function guessTargetLinkingNumber(fields, defaultTargetLinkingNumber) {
 export function fieldsToNormalizedString(fields, defaultTargetLinkingNumber = 0, normalizeOccurrenceNumber = false, normalizeEntryTag = false) {
   const targetLinkingNumber = guessTargetLinkingNumber(fields, defaultTargetLinkingNumber);
 
-  nvdebug(`fieldsToNormalizedString: OCC: ${normalizeOccurrenceNumber}`);
+  //nvdebug(`fieldsToNormalizedString: OCC: ${normalizeOccurrenceNumber}`);
   const strings = fields.map(field => fieldToNormalizedString(field, targetLinkingNumber, normalizeOccurrenceNumber, normalizeEntryTag));
   strings.sort();
   return strings.join('\t__SEPARATOR__\t');
@@ -445,7 +445,7 @@ export function isFirstLinkedSubfield6Field(field, record) {
   }
   const chain = getAllLinkedSubfield6Fields(field, record);
   if (!isRelevantSubfield6Chain(chain)) {
-    nvdebug(`Rejected 6: ${fieldsToString(chain)}`);
+    //nvdebug(`Rejected 6: ${fieldsToString(chain)}`);
     return false;
   }
 
@@ -472,13 +472,13 @@ export function get6s(field, candidateFields) { // NB! Convert field to fields!!
   if (sixes.length === 0) {
     return [field];
   }
-  nvdebug(`SIXES: ${sixes.length}`);
+  //nvdebug(`SIXES: ${sixes.length}`);
   const occurrenceNumbers = sixes.map(sf => subfield6GetOccurrenceNumber(sf)).filter(value => value !== undefined && value !== '00');
-  nvdebug(occurrenceNumbers.join(' -- '));
+  //nvdebug(occurrenceNumbers.join(' -- '));
 
   const relevantFields = candidateFields.filter(f => occurrenceNumbers.some(o => fieldHasOccurrenceNumber(f, o)));
-  nvdebug(`${fieldToString(field)}: $6-RELFIELDS FOUND: ${relevantFields.length}...`);
-  relevantFields.forEach(f => nvdebug(fieldToString(f)));
+  //nvdebug(`${fieldToString(field)}: $6-RELFIELDS FOUND: ${relevantFields.length}...`);
+  //relevantFields.forEach(f => nvdebug(fieldToString(f)));
   return relevantFields;
 }
 
@@ -488,6 +488,6 @@ export function resetSubfield6Tag(subfield, tag) {
   }
   // NB! mainly for 1XX<->7XX transfers
   const newValue = `${tag}-${subfield.value.substring(4)}`;
-  nvdebug(`Set subfield $6 value from ${subfieldToString(subfield)} to ${newValue}`, debugDev);
+  //nvdebug(`Set subfield $6 value from ${subfieldToString(subfield)} to ${newValue}`, debugDev);
   subfield.value = newValue;
 }

package/src/subfieldValueNormalizations.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './subfieldValueNormalizations.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -20,7 +20,7 @@ generateTests({
   }
 });
 
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/subfieldValueNormalizations:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/subfieldValueNormalizations:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -30,12 +30,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
 
   const recordFixture = getFixture('record.json');

package/src/sync-007-and-300.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './sync-007-and-300.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -20,7 +20,7 @@ generateTests({
   }
 
 });
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sync-007-and-300:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sync-007-and-300:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -30,12 +30,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/sync-language.js

@@ -0,0 +1,148 @@
+// Author(s): Nicholas Volk
+
+//import createDebugLogger from 'debug';
+import clone from 'clone';
+
+import {fieldToString} from './utils.js';
+
+// const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:sync-language');
+
+export default function () {
+
+  return {
+    description: 'Sync 008/35-37 and 041$a/$d',
+    validate, fix
+  };
+
+  function getSubfieldCodeFor041(record) {
+    const c = record?.leader[6];
+    if (c === 'i' || c === 'j') {
+      return 'd';
+    }
+    return 'a';
+  }
+
+  function transferableValue(val) {
+    if (val.match(/^[a-z]{3}$/u)) {
+      // 'zxx' is an exception to the otherwise relatively transitive 008/35-37 <=> 041$a relationship...
+      // https://wiki.helsinki.fi/xwiki/bin/view/rdasovellusohje/RDA-kuvailu%20MARC%2021%20-formaatilla/RDA-sovellusohje/008%20Kontrollikentt%C3%A4/#Hmp.35-37Kieli
+      // "008-kielikoodi on sama kuin ensimmäinen 041-kentän a- (tai d-) osakenttä. Poikkeuksen muodostaa koodi zxx:
+      //  jos tietueen 008ssa on kielikoodi zxx (ei kielellistä sisältöä, esim. soitinmusiikki), tietueessa ei voi olla 041 a- eikä d-osakenttää."
+      // So 'zxx' ain't transferable (done here), and it should be removed from 041 ($a and $d at least, not done here nor elsewhere at the moment...)
+      // Note that 'mul' and 'und' are treated (in this validator) the same as normal values.
+      if (val === 'zxx') {
+        return false;
+      }
+      return true;
+    }
+    return false;
+  }
+
+
+  function zxxRemoval041(record, field, validateMode) {
+    const fields041 = record.fields.filter(f => f.tag === '041');
+
+    return handleZxx(fields041);
+
+    function handleZxx(fields, result = []) {
+      const [currField, ...remainingFields] = fields;
+      if ( !currField) {
+        return result;
+      }
+      // Theoretically 'zxx' might mean something in ISO-639-3 or some other language code list:
+      if (!field.subfields || field.subfields.some(sf => sf.code === '2')) {
+        return handleZxx(remainingFields, result);
+      }
+      // Presumable 'zxx' is bad in any data subfield, not just 'a' and 'd':
+      const otherSubfields = field.subfields.some(sf => !sf.code.match(/^[a-z]$/u || sf.value !== 'zxx'));
+      if (otherSubfields.length === field.subfields.length) {
+        return handleZxx(remainingFields, result);
+      }
+      const originalString = fieldToString(field);
+      if (otherSubfields.length === 0) {
+        const message = `Remove '${originalString}'`;
+        if (!validateMode) {
+          record.removeField(field);
+        }
+        return handleZxx(remainingFields, [...result, message]);
+      }
+      // Some subfields are removed:
+      if (validateMode) {
+        // NB! In validation field is not really deleted, and the non-deleted field might trigger other fixes (than won't be done to a deleted field) later on
+        const clonedField = clone(field);
+        clonedField.subfields = otherSubfields;
+        const modifiedString = fieldToString(clonedField);
+        const message = `Modify '${originalString}' => '${modifiedString}`;
+        return handleZxx(remainingFields, [...result, message]);
+      }
+      field.subfields = otherSubfields;
+      const modifiedString = fieldToString(field);
+      const message = `Modify '${originalString}' => '${modifiedString}`;
+      return handleZxx(remainingFields, [...result, message]);
+    }
+
+
+  }
+
+  function sync008And041(record, validateMode) {
+    const f008 = record.fields.find(f => f.tag === '008');
+    const f041 = record.fields.find(f => f.tag === '041');
+
+    if (!f008 || f008.value.length !== 40) { // Some sanity checks
+      return [];
+    }
+
+    const subfieldCode = getSubfieldCodeFor041(record);
+    const lang008 = f008.value.substring(35, 38);
+
+    if (!f041) {
+      // Insert missing 041
+      if (transferableValue(lang008)) {
+        const newField = {'tag': '041', 'ind1': ' ', 'ind2': ' ', 'subfields': [ {'code': subfieldCode, 'value': lang008}]};
+        if (!validateMode) {
+          record.insertField(newField);
+        }
+        return [ `Add '${fieldToString(newField)}'` ];
+      }
+      // Can't do anything, and we only report this we can fix...
+      return [];
+    }
+
+    const firstRelevantSubfield = f041.subfields.find(sf => sf.code === subfieldCode);
+
+    // NB! If $2 is set, *never* copy it's value to 008/35-37... Otherwise we might a loop in Aleph.
+    // Note that if $2 is used, 008/35-37 should actually be '|||'. Now we just aggressively leave it alone.
+    if (!firstRelevantSubfield || !transferableValue(firstRelevantSubfield.value) || f041.subfields.some(sf => sf.code === '2') || lang008 === firstRelevantSubfield.value) {
+      return []
+    }
+
+    // Update 008/35-37:
+    const cloned008 = clone(f008);
+    cloned008.value = `${f008.value.substring(0, 35)}${firstRelevantSubfield.value}${f008.value.substring(38)}`;
+
+
+    if (!validateMode) {
+      f008.value = cloned008.value;
+    }
+    return [ `Modify '${f008.value}' to '${cloned008.value}'` ];
+  }
+
+  function fix(record, validateMode = false) {
+    const zxxMessages = zxxRemoval041(record, validateMode);
+
+    const syncMessages = sync008And041(record, validateMode);
+
+    const messages = [...zxxMessages, ...syncMessages];
+
+    if (validateMode) {
+       return {message: messages, valid: messages.length === 0};
+    }
+    return {message: [], fix: messages, valid: true};
+  }
+
+
+  function validate(record) {
+    return fix(record, true);
+  }
+}
+

package/src/sync-language.test.js

@@ -0,0 +1,52 @@
+import assert from 'node:assert/strict';
+//import createDebugLogger from 'debug';
+
+import validatorFactory from './sync-language.js';
+
+import {MarcRecord} from '@natlibfi/marc-record';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+
+generateTests({
+  callback,
+  path: [import.meta.dirname, '..', 'test-fixtures', 'sync-language'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  hooks: {
+    before: async () => {
+      testValidatorFactory();
+    }
+  }
+});
+
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/drop-terms:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  assert.equal(typeof validator, 'object');
+  assert.equal(typeof validator.description, 'string');
+  assert.equal(typeof validator.validate, 'function');
+}
+
+async function callback({getFixture, fix = false}) {
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  //const expectedResult = new MarcRecord(getFixture('expectedResult.json'));
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    assert.deepEqual(result, expectedResult);
+    const originalRecord = new MarcRecord(getFixture('record.json'));
+    assert.deepEqual(record, originalRecord); // The record should now change in validation-only
+    return;
+  }
+
+  await validator.fix(record);
+  assert.deepEqual(record, new MarcRecord(expectedResult));
+}