@natlibfi/marc-record-validators-melinda 12.1.0-alpha.1 → 13.0.0

package/src/removeDuplicateDataFields.js

@@ -1,5 +1,5 @@
 import createDebugLogger from 'debug';
-import {fieldsToString, fieldToString, nvdebug} from './utils.js';
+import {fieldToString, nvdebug} from './utils.js';
 import {fieldHasValidSubfield6, fieldsGetOccurrenceNumbers, fieldsToNormalizedString, fieldToNormalizedString, get6s} from './subfield6Utils.js';
 import {add8s, fieldHasLinkingNumber, fieldHasValidSubfield8, fieldsGetAllSubfield8LinkingNumbers, getSubfield8LinkingNumber, recordGetAllSubfield8LinkingNumbers, recordGetFieldsWithSubfield8LinkingNumber} from './subfield8Utils.js';
 
@@ -11,6 +11,8 @@ const LINKED_NOT_PROCESSED = 1;
 // Relocated from melinda-marc-record-merge-reducers (and renamed)
 
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:removeDuplicateDataFields');
+//const debugData = debug.extend('data');
+const debugDev = debug.extend('dev');
 
 export default function () {
   return {
@@ -19,7 +21,7 @@ export default function () {
   };
 
   function fix(record) {
-    nvdebug('Remove duplicate data fields');
+    nvdebug('Remove duplicate data fields', debugDev);
     const res = {message: [], fix: [], valid: true};
     removeDuplicateDatafields(record, true);
     // This can not really fail...
@@ -28,7 +30,7 @@ export default function () {
 
   function validate(record) {
     // Check max, and check number of different indexes
-    nvdebug('Validate record: duplicate data fields cause (t)error', debug);
+    nvdebug('Validate record: duplicate data fields cause (t)error', debugDev);
 
     const duplicates = removeDuplicateDatafields(record, false);
 
@@ -49,7 +51,7 @@ export default function () {
 
 /*
 function numberOfLinkageSubfields(field) {
-  nvdebug(`N of Linkage Subs(${fieldToString(field)})`);
+  nvdebug(`N of Linkage Subs(${fieldToString(field)})`, debugDev);
   const subfields = field.subfields.filter(sf => sf.code === '6' || sf.code === '8');
   return subfields.length;
 }
@@ -139,21 +141,21 @@ export function removeDuplicateSubfield8Chains(record, fix = true) {
 
   let removables = []; // for validation
 
-  nvdebug("CHAIN-8");
+  //nvdebug("CHAIN-8", debugDev);
   const seenLinkingNumbers = recordGetAllSubfield8LinkingNumbers(record);
   if (seenLinkingNumbers.length === 0) {
     return removables;
   }
 
-  nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);
+  //nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debugDev);
 
   seenLinkingNumbers.forEach(currLinkingNumber => {
     const linkedFields = recordGetFieldsWithSubfield8LinkingNumber(record, currLinkingNumber) //getFieldsWithSubfield8Index(base, baseIndex);
     // As/If there's just one occurrence number it should be fine to use normalizeOccurrenceNumber = true
     const normalizeOccurrenceNumber = true;
     const linkedFieldsAsString = fieldsToNormalizedString(linkedFields, currLinkingNumber, normalizeOccurrenceNumber, true);
-    nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);
-    nvdebug(`${linkedFieldsAsString}`, debug);
+    //nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debugDev);
+    //nvdebug(`${linkedFieldsAsString}`, debugDev);
 
     if (linkedFieldsAsString in seen)  {
       if (!removables.includes(linkedFieldsAsString)) {
@@ -161,15 +163,15 @@ export function removeDuplicateSubfield8Chains(record, fix = true) {
       }
 
       if (fix) {
-        nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);
+        //nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debugDev);
         linkedFields.forEach(field => recordRemoveFieldOrSubfield8(record, field, currLinkingNumber));
         return;
       }
 
-      nvdebug(`$8 VALIDATION: DUPLICATE DETECTED ${linkedFieldsAsString}`, debug);
+      //nvdebug(`$8 VALIDATION: DUPLICATE DETECTED ${linkedFieldsAsString}`, debugDev);
       return;
     }
-    nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);
+    //nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debugDev);
     seen[linkedFieldsAsString] = 1;
     return;
   });
@@ -190,28 +192,28 @@ export function handleDuplicateSubfield8Chains(record, fix) {
 
   let seen = {};
 
-  nvdebug("CHAIN-8");
+  //nvdebug("CHAIN-8", debugDev);
   const seenLinkingNumbers = recordGetAllSubfield8LinkingNumbers(record);
   if (seenLinkingNumbers.length === 0) {
     return;
   }
 
-  nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);
+  //nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debugDev);
 
   seenLinkingNumbers.forEach(currLinkingNumber => {
     const linkedFields = recordGetFieldsWithSubfield8LinkingNumber(record, currLinkingNumber) //getFieldsWithSubfield8Index(base, baseIndex);
     // As/If there's just one occurrence number it should be fine to use normalizeOccurrenceNumber = true
     const normalizeOccurrenceNumber = false; //true;
     const linkedFieldsAsString = fieldsToNormalizedString(linkedFields, currLinkingNumber, normalizeOccurrenceNumber, true);
-    nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);
-    nvdebug(`${linkedFieldsAsString}`, debug);
+    //nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debugDev);
+    //nvdebug(`${linkedFieldsAsString}`, debugDev);
 
     if (linkedFieldsAsString in seen)  {
-      nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);
+      //nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debugDev);
       linkedFields.forEach(field => newRecordRemoveFieldOrSubfield8(record, field, currLinkingNumber, fix));
       return;
     }
-    nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);
+    //nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debugDev);
     seen[linkedFieldsAsString] = 1;
     return;
   });
@@ -223,12 +225,12 @@ function markIdenticalSubfield6Chains(chain, record) {
   const normalizeTag = chain.some(field => field.tag.substring(0, 1) === '1'); // 1XX can delete 7XX as well!
   const chainAsString = fieldsToNormalizedString(chain, 0, normalizeOccurrenceNumber, normalizeTag);
 
-  nvdebug(`markIdenticalSubfield6Chains: ${chainAsString}`);
+  //nvdebug(`markIdenticalSubfield6Chains: ${chainAsString}`, debugDev);
   record.fields.forEach(f => compareWithChain(f));
 
 
   function compareWithChain(f) {
-    nvdebug(`FIELD2CHAIN ${fieldToString(f)}`);
+    //nvdebug(`FIELD2CHAIN ${fieldToString(f)}`, debugDev);
     const otherChain = fieldToChain(f, record);
     // Not a lone field or chain (head) or ... or is-same-chain
     if (otherChain.length === 0 || sameField(chain[0], otherChain[0])) {
@@ -239,7 +241,7 @@ function markIdenticalSubfield6Chains(chain, record) {
     // Mark other chain as deleted:
     if (chainAsString === otherChainAsString) {
       otherChain.forEach(f => {
-        nvdebug(` mark ${fieldToString(f)} as deleted ($6-chain)...`);
+        //nvdebug(` mark ${fieldToString(f)} as deleted ($6-chain)...`, debugDev);
         f.deleted = 1;
       });
       return;
@@ -260,7 +262,7 @@ function markIdenticalLoneFieldsAsDeletable(field, record) {
 
   // Mark fields as deleted:
   identicalLoneFields.forEach(f => {
-    nvdebug(` mark ${fieldToString(f)} as deleted (lone field)...`);
+    //nvdebug(` mark ${fieldToString(f)} as deleted (lone field)...`, debugDev);
     f.deleted = 1;
   });
 
@@ -300,7 +302,7 @@ function acceptFieldsWithSubfield8(fieldsWithSubfield8, requireSingleTag = false
 
   // If linking number
   function anomaly8(linkingNumber) {
-    nvdebug(`  Looking for anomalies in linkin number ${linkingNumber}`);
+    //nvdebug(`  Looking for anomalies in linkin number ${linkingNumber}`, debugDev);
     const relevantFields = fieldsWithSubfield8.filter(f => fieldHasLinkingNumber(f, linkingNumber));
     if (requireSingleTag) {
       return !isSingleTagLinkingNumber(linkingNumber, relevantFields, relevantFields[0].tag);
@@ -332,7 +334,7 @@ export function fieldToChain(field, record) {
   }
   const chain = newGetAllLinkedFields(field, record, true, true);
 
-  nvdebug(` Chain contains ${chain.length} field(s)`);
+  // nvdebug(` Chain contains ${chain.length} field(s)`, debugDev);
   if (!isChainHead(field, chain)) { // newGetAllLinkedFields() marks relevant record.fields!
     return [];
   }
@@ -357,7 +359,7 @@ export function fieldToChain(field, record) {
     return [];
   }
 
-  //nvdebug(`Proceed with ${fieldsToString(chain)}`);
+  //nvdebug(`Proceed with ${fieldsToString(chain)}`, debugDev);
 
 
   return chain;
@@ -366,7 +368,7 @@ export function fieldToChain(field, record) {
 
 function fieldHandleDuplicateDatafields(field, record) {
   const chain = fieldToChain(field, record);
-  nvdebug(` TRY TO HANDLE DUPLICATES OF '${fieldsToString(chain)}'`);
+  //nvdebug(` TRY TO HANDLE DUPLICATES OF '${fieldsToString(chain)}'`, debugDev);
 
   if (chain.length === 0) {
     return;
@@ -383,7 +385,7 @@ function fieldHandleDuplicateDatafields(field, record) {
   if (fieldsWithSubfield6.length === 0) {
 
     if (fieldsWithSubfield8.length === 0) { // chain.length === 1?
-      nvdebug(` Trying to find duplicates of single field '${fieldToString(chain[0])}'`);
+      //nvdebug(` Trying to find duplicates of single field '${fieldToString(chain[0])}'`, debugDev);
       markIdenticalLoneFieldsAsDeletable(chain[0], record);
       return;
     }
@@ -401,8 +403,8 @@ function fieldHandleDuplicateDatafields(field, record) {
   }
 
 
-  nvdebug(` NO HANDLER FOUND FOR '${fieldsToString(chain)}'`);
-  nvdebug(`  N8s: ${fieldsWithSubfield6.length}`);
+  //nvdebug(` NO HANDLER FOUND FOR '${fieldsToString(chain)}'`, debugDev);
+  //nvdebug(`  N8s: ${fieldsWithSubfield6.length}`, debugDev);
 
 }
 

package/src/removeDuplicateDataFields.test.js

@@ -3,7 +3,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './removeDuplicateDataFields.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -19,7 +19,7 @@ generateTests({
     }
   }
 });
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/removeDuplicateDataFields:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/removeDuplicateDataFields:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -29,12 +29,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate, 'function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');

package/src/removeInferiorDataFields.js

@@ -1,7 +1,7 @@
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 import {fieldToChain, sameField} from './removeDuplicateDataFields.js';
 import {fieldGetOccurrenceNumberPairs, fieldHasValidSubfield6, fieldSevenToOneOccurrenceNumber, fieldsToNormalizedString} from './subfield6Utils.js';
-import {fieldHasSubfield, fieldsToString, fieldToString, nvdebug, uniqArray} from './utils.js';
+import {fieldHasSubfield, fieldsToString, fieldToString, /*nvdebug,*/ uniqArray} from './utils.js';
 import {fieldHasValidSubfield8} from './subfield8Utils.js';
 import {encodingLevelIsBetterThanPrepublication, fieldRefersToKoneellisestiTuotettuTietue, getEncodingLevel} from './prepublicationUtils.js';
 import {cloneAndNormalizeFieldForComparison} from './normalizeFieldForComparison.js';
@@ -12,7 +12,10 @@ import {fixComposition, precomposeFinnishLetters} from './normalize-utf8-diacrit
 // NB! This validator handles only full fields, and does not support subfield $8 removal.
 // Also, having multiple $8 subfields in same fields is not supported.
 // If this functionality is needed, see removeDuplicateDatafields.js for examples of subfield-only stuff.
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:removeInferiorDataFields');
+
+// const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:removeInferiorDataFields');
+//const debugData = debug.extend('data');
+//const debugDev = debug.extend('dev');
 
 export default function () {
   return {
@@ -21,7 +24,7 @@ export default function () {
   };
 
   function fix(record) {
-    nvdebug('Fix record: remove inferior (eg. subset) data fields', debug);
+    //nvdebug('Fix record: remove inferior (eg. subset) data fields', debugDev);
     const res = {message: [], fix: [], valid: true};
     removeInferiorDatafields(record, true);
     // This can not really fail...
@@ -30,7 +33,7 @@ export default function () {
 
   function validate(record) {
     // Check max, and check number of different indexes
-    nvdebug('Validate record: remove inferior (eg. subset) data fields', debug);
+    //nvdebug('Validate record: remove inferior (eg. subset) data fields', debugDev);
 
     const duplicates = removeInferiorDatafields(record, false);
 
@@ -43,14 +46,14 @@ export default function () {
 
 
 function deriveInferiorChains(fields, record) {
-  //nvdebug(`======= GOT ${fields.length} FIELDS TO CHAINIFY`);
+  //nvdebug(`======= GOT ${fields.length} FIELDS TO CHAINIFY`, debugDev);
   const hash = {};
 
   fields.forEach(f => fieldToChainToDeletables(f));
 
   return hash;
 
-  //nvdebug(`WP1: GOT ${todoList.length} CHAINS`);
+  //nvdebug(`WP1: GOT ${todoList.length} CHAINS`, debugDev);
 
 
   // here we map deletableStringObject[str] => field. The idea is to help debugging. We don't actually need the field object...
@@ -63,7 +66,7 @@ function deriveInferiorChains(fields, record) {
     }
     const chainAsString = fieldsToNormalizedString(chain, 0, true, true);
     const arr = deriveChainDeletables([chainAsString]);
-    //nvdebug(`GOT ${arr.length} DELETABLES FOR ${chainAsString}`);
+    //nvdebug(`GOT ${arr.length} DELETABLES FOR ${chainAsString}`, debugDev);
     arr.forEach(val => {
       if (!(val in hash)) {
         hash[val] = field;
@@ -116,16 +119,16 @@ function isRelevantChain6(field, record) {
 
 export function removeInferiorChains(record, fix = true) {
   const fields = record.fields.filter(f => isRelevantChain6(f, record));
-  //nvdebug(`WP2.0: GOT ${fields.length} chain(s)`);
+  //nvdebug(`WP2.0: GOT ${fields.length} chain(s)`, debugDev);
 
   const deletableChainsAsKeys = deriveInferiorChains(fields, record);
   const nChains = Object.keys(deletableChainsAsKeys).length;
-  //nvdebug(`WP2: GOT ${nChains} chain(s)`);
+  //nvdebug(`WP2: GOT ${nChains} chain(s)`, debugDev);
   if (nChains === 0) {
     return [];
   }
 
-  //nvdebug(`removeInferiorChains() has ${fields.length} fields-in-chain(s), and a list of ${nChains} deletable(s)`);
+  //nvdebug(`removeInferiorChains() has ${fields.length} fields-in-chain(s), and a list of ${nChains} deletable(s)`, debugDev);
 
   return innerRemoveInferiorChains(fields);
 
@@ -153,11 +156,11 @@ export function removeInferiorChains(record, fix = true) {
     if (chainContains1XX(chain)) {
       triggeringChain.forEach(f => sevenToOne(f, triggeringChain));
     }
-    //nvdebug(`iRIS6C: ${chainAsString}`);
+    //nvdebug(`iRIS6C: ${chainAsString}`, debugDev);
     const deletedString = fieldsToString(chain);
     const message = `DEL: '${deletedString}'  REASON: '${fieldsToString(triggeringChain)}'`;
     if (fix) {
-      //nvdebug(`INFERIOR $6 CHAIN REMOVAL: ${message}}`, debug);
+      //nvdebug(`INFERIOR $6 CHAIN REMOVAL: ${message}}`, debugDev);
       chain.forEach(field => record.removeField(field));
     }
     return innerRemoveInferiorChains(remainingFields, [...deletedStringsArray, message]);
@@ -197,7 +200,7 @@ function deriveIndividualDeletables490(todoList, deletables = []) {
   if (fieldAsString === undefined) {
     return deletables;
   }
-  //nvdebug(`PROCESS ${fieldAsString}`);
+  //nvdebug(`PROCESS ${fieldAsString}`, debugDev);
   if (!fieldAsString.match(/^490/u)) {
     return deriveIndividualDeletables490(stillToDo, deletables);
   }
@@ -220,13 +223,14 @@ function deriveIndividualDeletables490(todoList, deletables = []) {
 
   /*
   if (arr.length) {
-    nvdebug(`${arr.length} derivation(s) for ${fieldAsString}`);
-    nvdebug(arr.join('\n'));
+    nvdebug(`${arr.length} derivation(s) for ${fieldAsString}`, debugDev);
+    nvdebug(arr.join('\n'), debugDev);
   }
   */
   return arr;
 }
 
+// eslint-disable-next-line max-lines-per-function
 function deriveIndividualDeletables(record) {
   const todoList = record.fields.map(f => fieldToString(f));
   //const finishedRecord = encodingLevelIsBetterThanPrepublication(getEncodingLevel(record));
@@ -274,28 +278,6 @@ function deriveIndividualDeletables(record) {
       }
     }
 
-    if (currString.match(/^500 ## ‡a Lisäpainokset: Lisäpainos /u)) { // MET-569
-      const tmp1 = currString.replace(' Lisäpainos ', ' [Lisäpainos] ');
-      const tmp2 = currString.replace(' Lisäpainos ', ' ');
-      if (tmp1 !== currString && tmp2 !== currString) {
-        return processTodoList([...stillToDo, ...moreToDo], [...deletables, tmp1, tmp2]);
-      }
-    }
-
-    if (currString.match(/^500 ## ‡a Lisäpainokset: \[Lisäpainos\] /u)) { // MET-569
-      const tmp = currString.replace(' [Lisäpainos] ', ' ');
-      if (tmp !== currString) {
-        return processTodoList([...stillToDo, ...moreToDo], [...deletables, tmp]);
-      }
-    }
-
-    if (currString.match(/^500 ## ‡a Ei vastaanotettu\.$/u)) { // MELKEHITYS-3147
-      return processTodoList([...stillToDo, ...moreToDo], [...deletables, '500 ## ‡a EI VASTAANOTETTU.']);
-    }
-    if (currString.match(/^500 ## ‡a Ei ilmesty\.$/u)) { // MELKEHITYS-3147
-      return processTodoList([...stillToDo, ...moreToDo], [...deletables, '500 ## ‡a EI ILMESTY.']);
-    }
-
     if (currString.match(/^505 .0.*-- ‡t/u)) { // MRA-413-ish
       const tmp = currString.replace(/ -- ‡t /gu, ' -- '). // remove non-initial $t subfields
         replace(/ ‡[rg] /gu, ' '). // remove $r and $g subfields
@@ -305,53 +287,115 @@ function deriveIndividualDeletables(record) {
       if (tmp !== currString) {
         return processTodoList([tmp, ...stillToDo, ...moreToDo], [...deletables, tmp]);
       }
-      //nvdebug(`505 ORIGINAL: '${fieldAsString}'`)
-      //nvdebug(`505 DERIVATE: '${tmp}'`)
+      //nvdebug(`505 ORIGINAL: '${fieldAsString}'`, debugDev)
+      //nvdebug(`505 DERIVATE: '${tmp}'`, debugDev)
     }
 
-    if (currString.match(/^594 ## ‡a Ei vastaanotettu ‡5 FENNI$/u)) { // MELKEHITYS-3147
-      return processTodoList([...stillToDo, ...moreToDo], [...deletables, '594 ## ‡a EI VASTAANOTETTU ‡5 FENNI']);
-    }
-    if (currString.match(/^594 ## ‡a Ei ilmesty ‡5 FENNI$/u)) { // MELKEHITYS-3147
-      return processTodoList([...stillToDo, ...moreToDo], [...deletables, '594 ## ‡a EI ILMESTY ‡5 FENNI']);
+    const inferiorFunctions = [ getPrepublicationTerms, getInferior341, getAiBased, getMelkehitys3147, getMet831, getMet569, getCorrespondingDeprecated502s ];
+
+    const inferiorTerms = getInferiorTerms(inferiorFunctions); //getPrepublicationTerms(currString);
+
+    const newDeletables = [...deletables, ...subsets, ...accentless, ...d490, ...inferiorTerms];
+
+    return processTodoList([...stillToDo, ...moreToDo], newDeletables);
+
+    function getInferiorTerms(functions, results = []) {
+      const [currFunction, ...remainingFunctions] = functions;
+      if (!currFunction) {
+        return results
+      }
+      const newDeletables = currFunction(currString);
+      return getInferiorTerms(remainingFunctions, [...results, ...newDeletables]);
     }
 
-    // MET-381: remove occurence number TAG-00, if TAG-NN existists
-    if (currString.match(/^880.* ‡6 [0-9][0-9][0-9]-(?:[1-9][0-9]|0[1-9])/u)) {
-      const tmp = currString.replace(/( ‡6 [0-9][0-9][0-9])-[0-9]+/u, '$1-00');
-      //nvdebug(`MET-381: ADD TO DELETABLES: '${tmp}'`);
-      //deletableStringsArray.push(tmp);
-      if (tmp.match(/ ‡6 [0-9][0-9][0-9]-00\/[^ ]+ /u)) {
-        const tmp2 = tmp.replace(/( ‡6 [0-9][0-9][0-9]-00)[^ ]+/u, '$1');
-        //nvdebug(`MET-381: ADD TO DELETABLES: '${tmp2}'`);
-        return processTodoList([...stillToDo, ...moreToDo], [...deletables, tmp, tmp2]);
+    function getMet569(string) {
+      if (string.match(/^500 ## ‡a Lisäpainokset: Lisäpainos /u)) { // MET-569
+        const tmp1 = string.replace(' Lisäpainos ', ' [Lisäpainos] ');
+        const tmp2 = string.replace(' Lisäpainos ', ' ');
+        if (tmp1 !== string && tmp2 !== string) { // Should not happen, just a sanity check
+          return [tmp1, tmp2];
+        }
+      }
+
+      if (string.match(/^500 ## ‡a Lisäpainokset: \[Lisäpainos\] /u)) { // MET-569
+        const tmp = string.replace(' [Lisäpainos] ', ' ');
+        if (tmp !== string) { // Should not happen, just a sanity check
+          return [tmp];
+        }
       }
-      return processTodoList([...stillToDo, ...moreToDo], [...deletables, tmp]);
+      return [];
     }
 
-    // MET-575 (merge: applies in postprocessing)
-    const inferiorTerms = getPrepublicationTerms(currString);
+    function getCorrespondingDeprecated502s(string, results = []) { // MET-801-ish
+      if (!string.match(/^502/u)) {
+        return results;
+      }
 
-    // MELKEHITYS-3277-ish: non-AI is better than AI (a rare case where longer version is inferior):
-    const aiBased = `${currString} ‡7 (dpenmw)AI`;
+      // 502 ## $a Väitöskirja : $c Opinahjo, $d 1234. => 502 ## $a Väitöskirja : $c Opinahjo.
+      let tmp = string.replace(/, ‡d [^‡]+\.($| ‡)/u, '.$1');
+      if (tmp !== string) {
+        return getCorrespondingDeprecated502s(tmp, [...results, tmp]);
+      }
+      // 502 ## $a Väitöskirja : $c Opinahjo. => 502 ## $a Väitöskirja.
+      tmp = string.replace(/ : ‡c [^‡]+\.($| ‡)/u, '.');
+      if (tmp !== string) {
+        return getCorrespondingDeprecated502s(tmp, [...results, tmp]);
+      }
 
-    const newDeletables = [...deletables, ...subsets, ...accentless, ...d490, ...inferiorTerms, aiBased];
+      // 502 ## $a Väitöskirja--Opinahjo, YYYY. => 502 ## $a Väitöskirja : $c Opinahjo, $d YYYY.
+      if (string.match(/^502 ## ‡a [^‡]+--[^‡]+\.$/u)) {
+        tmp = string
+          .replace(/--(.*,) ([12][0-9][0-9][0-9]\.)$/u, ' : ‡c $1 ‡d $2')
+          .replace(/--/u, ': ‡c ');
+        return getCorrespondingDeprecated502s(tmp, [...results, tmp]);
+      }
+      return results;
+    }
 
-    if (subsets.length) {
-      return processTodoList([...stillToDo, ...moreToDo], newDeletables);
+    function getMet831(string) { // MET-381
+      // MET-381: remove occurence number TAG-00, if TAG-NN existists
+      if (string.match(/^880.* ‡6 [0-9][0-9][0-9]-(?:[1-9][0-9]|0[1-9])/u)) {
+        const tmp = string.replace(/( ‡6 [0-9][0-9][0-9])-[0-9]+/u, '$1-00');
+        if (tmp.match(/ ‡6 [0-9][0-9][0-9]-00\/[^ ]+ /u)) {
+          const tmp2 = tmp.replace(/( ‡6 [0-9][0-9][0-9]-00)[^ ]+/u, '$1');
+          return [tmp, tmp2];
+        }
+        return [tmp];
+      }
+      return [];
+    }
+
+    function getMelkehitys3147(string) {
+      if (string.match(/^500 ## ‡a Ei vastaanotettu\.$/u)) { // MELKEHITYS-3147
+        return ['500 ## ‡a EI VASTAANOTETTU.'];
+      }
+      if (string.match(/^500 ## ‡a Ei ilmesty\.$/u)) { // MELKEHITYS-3147
+        return ['500 ## ‡a EI ILMESTY.'];
+      }
+
+      if (string.match(/^594 ## ‡a Ei vastaanotettu ‡5 FENNI$/u)) { // MELKEHITYS-3147
+        return ['594 ## ‡a EI VASTAANOTETTU ‡5 FENNI'];
+      }
+      if (string.match(/^594 ## ‡a Ei ilmesty ‡5 FENNI$/u)) { // MELKEHITYS-3147
+        return ['594 ## ‡a EI ILMESTY ‡5 FENNI'];
+      }
+      return [];
     }
 
-    return processTodoList([...stillToDo, ...moreToDo], newDeletables);
   }
 
+  function getAiBased(string) { // MELKEHITYS-3277-ish: non-AI is better than AI (a rare case where longer version is inferior):
+    return [`${string} ‡7 (dpenmw)AI`];
+  }
+  
   function getAccentlessVersion(string) { // MET-527
-    //nvdebug(`START: '${string}`);
+    //nvdebug(`START: '${string}`, debugDev);
     // This is a sanity check: if precomposition does something, there's something wrong, and we don't want to proceed..
     if (string !== precomposeFinnishLetters(string)) {
       return [];
     }
     const accentless = String(fixComposition(string)).replace(/\p{Diacritic}/gu, '');
-    //nvdebug(`FROM '${string}'\n  TO '${accentless}'`);
+    //nvdebug(`FROM '${string}'\n  TO '${accentless}'`, debugDev);
     if (accentless === string) { // Don't self-destruct
       return [];
     }
@@ -359,11 +403,22 @@ function deriveIndividualDeletables(record) {
   }
 
   function getPrepublicationTerms(fieldAsString) {
+    const subfield7Prepub = `${fieldAsString} ‡7 Ennakkotieto`
     if (fieldAsString.match(/^653./u)) {
       // MET-528 (extented by MET-575)
-      return [`${fieldAsString} ‡g ENNAKKOTIETO`, `${fieldAsString} ‡g Ennakkotieto`, `${fieldAsString} ‡g ennakkotieto`, `${fieldAsString} ‡g ENNAKKOTIETO.`, `${fieldAsString} ‡g Ennakkotieto.`, `${fieldAsString} ‡g ennakkotieto.`];
+      return [subfield7Prepub, `${fieldAsString} ‡g ENNAKKOTIETO`, `${fieldAsString} ‡g Ennakkotieto`, `${fieldAsString} ‡g ennakkotieto`, `${fieldAsString} ‡g ENNAKKOTIETO.`, `${fieldAsString} ‡g Ennakkotieto.`, `${fieldAsString} ‡g ennakkotieto.`];
     }
 
+    return [subfield7Prepub];
+  }
+
+  function getInferior341(fieldAsString) { // MET-783
+    if (fieldAsString.match(/^341 .. ‡a textual ‡[bcdef].* ‡2 sapdv(?:$| ‡)/u)) {
+      return ['341 ## ‡a textual ‡2 sapdv', '341 1# ‡a textual ‡2 sapdv', '341 0# ‡a textual ‡2 sapdv'];
+    }
+    if (fieldAsString.match(/^341 .. ‡a auditory ‡[bcdef].* ‡2 sapdv(?:$| ‡)/u)) {
+      return ['341 ## ‡a auditory ‡2 sapdv', '341 1# ‡a auditory ‡2 sapdv', '341 0# ‡a auditory ‡2 sapdv'];
+    }
     return [];
   }
 
@@ -407,8 +462,8 @@ export function removeIndividualInferiorDatafields(record, fix = true) { // No $
   const deletableFieldsAsStrings = deriveIndividualDeletables(record);
   const deletableFieldsAsNormalizedStrings = deriveIndividualNormalizedDeletables(record);
 
-  // nvdebug(`Deletables:\n ${deletableFieldsAsStrings.join('\n ')}`);
-  // nvdebug(`Normalized deletables:\n ${deletableFieldsAsNormalizedStrings.join('\n ')}`);
+  // nvdebug(`Deletables:\n ${deletableFieldsAsStrings.join('\n ')}`, debugDev);
+  // nvdebug(`Normalized deletables:\n ${deletableFieldsAsNormalizedStrings.join('\n ')}`, debugDev);
 
   const hits = record.fields.filter(field => isDeletableField(field));
 
@@ -416,7 +471,7 @@ export function removeIndividualInferiorDatafields(record, fix = true) { // No $
 
   if (fix) {
     hits.forEach(field => {
-      //nvdebug(`Remove inferior field: ${fieldToString(field)}`, debug);
+      //nvdebug(`Remove inferior field: ${fieldToString(field)}`, debugDev);
       record.removeField(field);
     });
   }
@@ -444,8 +499,8 @@ export function removeInferiorDatafields(record, fix = true) {
   const removables6 = removeInferiorChains(record, fix); // Lone subfield $6 chains
   // HOW TO HANDLE $6+$8 combos? Skipping is relatively OK.
 
-  nvdebug(`REMOVABLES:\n  ${removables.join('\n  ')}`, debug);
-  nvdebug(`REMOVABLES 6:\n  ${removables6.join('\n  ')}`, debug);
+  //nvdebug(`REMOVABLES:\n  ${removables.join('\n  ')}`, debugDev);
+  //nvdebug(`REMOVABLES 6:\n  ${removables6.join('\n  ')}`, debugDev);
 
   const removablesAll = removables.concat(removables6); //.concat(removables8);
 

package/src/removeInferiorDataFields.test.js

@@ -1,9 +1,9 @@
-import assert from 'node:assert';
+import assert from 'node:assert/strict';
 import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './removeInferiorDataFields.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+//import createDebugLogger from 'debug';
 
 generateTests({
   callback,
@@ -20,7 +20,7 @@ generateTests({
   }
 });
 
-const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/removeInferiorDataFields:test');
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/removeInferiorDataFields:test');
 
 async function testValidatorFactory() {
   const validator = await validatorFactory();
@@ -30,12 +30,7 @@ async function testValidatorFactory() {
   assert.equal(typeof validator.validate).to.be.a('function');
 }
 
-async function callback({getFixture, enabled = true, fix = false}) {
-  if (enabled === false) {
-    debug('TEST SKIPPED!');
-    return;
-  }
-
+async function callback({getFixture, fix = false}) {
   const validator = await validatorFactory();
   const record = new MarcRecord(getFixture('record.json'));
   const expectedResult = getFixture('expectedResult.json');
@@ -44,9 +39,11 @@ async function callback({getFixture, enabled = true, fix = false}) {
   if (!fix) {
     const result = await validator.validate(record);
     assert.deepEqual(result, expectedResult);
+    const originalRecord = new MarcRecord(getFixture('record.json'));
+    assert.deepEqual(record, originalRecord); // The record should now change in validation-only
     return;
   }
 
   await validator.fix(record);
-  assert.deepEqual(record, expectedResult);
+  assert.deepEqual(record, new MarcRecord(expectedResult));
 }