@natlibfi/marc-record-validators-melinda 12.0.6 → 12.0.7

package/src/punctuation2.js

@@ -10,10 +10,10 @@
 *          (They are jumped over when looking for next (non-controlfield subfield)
 */
 import {validateSingleField} from './ending-punctuation.js';
-import {tagToDataProvenanceSubfieldCode} from './merge-fields/dataProvenance.js';
+import {tagToDataProvenanceSubfieldCode} from './dataProvenanceUtils.js';
 import {fieldGetUnambiguousTag} from './subfield6Utils.js';
 //import createDebugLogger from 'debug';
-import {fieldToString, isControlSubfieldCode, nvdebug} from './utils.js';
+import {fieldToString, isContentSubfieldCode, nvdebug} from './utils.js';
 import clone from 'clone';
 
 //const debug = createDebugLogger('debug/punctuation2');
@@ -57,7 +57,7 @@ function isIrrelevantSubfield(subfield, tag) {
   if (subfield.code === dataProvenanceSubfieldCode) {
     return true;
   }
-  return isControlSubfieldCode(subfield.code); // Currently this contains other stuff as well ($3, $4, $7, $9...)
+  return !isContentSubfieldCode(subfield.code); // Currently this contains other stuff as well ($3, $4, $7, $9...)
 }
 
 
@@ -123,16 +123,16 @@ const X00RemoveDotAfterBracket = {'code': 'cq', 'context': /\)\.$/u, 'remove': /
 // 390, 800, 810, 830...
 const cleanPuncBeforeLanguage = {'code': 'atvxyz', 'followedBy': 'l', 'context': puncIsProbablyPunc, 'remove': / *[.,:;]$/u};
 
-const addX00aComma = {'add': ',', 'code': 'abcqej', 'followedBy': 'cdeg', 'context': doesNotEndInPunc, 'contextRHS': allowsPuncRHS};
+const addX00aComma = {'add': ',', 'code': 'abcqejt', 'followedBy': 'cdegnr', 'context': doesNotEndInPunc, 'contextRHS': allowsPuncRHS};
 const addX00dComma = {'name': 'X00$d ending in "-" does not get comma', 'add': ',', 'code': 'd', 'followedBy': 'cdeg', 'context': /[^-,.!]$/u, 'contextRHS': allowsPuncRHS};
 const addX00aComma2 = {'add': ',', 'code': 'abcdej', 'followedBy': 'cdeg', 'context': /(?:[A-Z]|Å|Ä|Ö)\.$/u, 'contextRHS': allowsPuncRHS};
 const addX00Dot = {'add': '.', 'code': 'abcdetv', 'followedBy': 'fklptu', 'context': needsPuncAfterAlphanumeric};
 const addEntryFieldFinalDot = {'name': 'X00 final dot', 'add': '.', 'code': 'abcdefghijklmnopqrstuvwxyz', 'followedBy': '#', 'context': /[^.)!?-]$/u};
 
 
-const addX10iColon = {name: 'Punctuate relationship information', add: ':', code: 'i', context: defaultNeedsPuncAfter2};
+const addXX0iColon = {name: 'Punctuate relationship information', add: ':', code: 'i', context: defaultNeedsPuncAfter2}; // Not explicitly checking it, but this should always be followed by 'a' or 't'
 const addX10bDot = {'name': 'Add X10 pre-$b dot', 'add': '.', 'code': 'ab', 'followedBy': 'b', 'context': defaultNeedsPuncAfter2};
-const addX10eComma = {'add': ',', 'code': 'abe', 'followedBy': 'e', 'context': defaultNeedsPuncAfter2};
+const addX10Comma = {'add': ',', 'code': 'abet', 'followedBy': 'en', 'context': defaultNeedsPuncAfter2};
 const addX10Dot = {'name': 'Add X10 final dot', 'add': '.', 'code': 'abet', 'followedBy': 'tu#', 'context': needsPuncAfterAlphanumeric};
 const addColonToRelationshipInformation = {'name': 'Add \':\' to 7X0 $i relationship info', 'add': ':', 'code': 'i', 'context': defaultNeedsPuncAfter2};
 
@@ -223,11 +223,11 @@ const cleanCrappyPunctuationRules = {
   '946': crappy24X
 };
 
-const cleanLegalX00Comma = {'code': 'abcde', 'followedBy': 'cdegj', 'context': /.,$/u, 'remove': /,$/u};
+const cleanLegalX00Comma = {'code': 'abcdetn', 'followedBy': 'cdegjnr', 'context': /.,$/u, 'remove': /,$/u};
 // Accept upper case letters in X00$b, since they are probably Roman numerals.
 const cleanLegalX00bDot = {'code': 'b', 'followedBy': 't#', context: /^[IVXLCDM]+\.$/u, 'remove': /\.$/u};
 const cleanLegalX00iColon = {'code': 'i', 'followedBy': 'a', 'remove': / *:$/u}; // NB! context is not needed
-const cleanLegalX00Dot = {'code': 'abcdetvl', 'followedBy': 'tu#', 'context': /(?:[a-z0-9)]|å|ä|ö)\.$/u, 'remove': /\.$/u};
+const cleanLegalX00Dot = {'code': 'abcdetkvl', 'followedBy': 'tklu#', 'context': /(?:[a-z0-9)]|å|ä|ö)\.$/u, 'remove': /\.$/u};
 const cleanDotBeforeLanguageSubfieldL = {'name': 'pre-language-$l dot', 'followedBy': 'l', 'context': /.\.$/u, 'remove': /\.$/u};
 
 const legalEntryField = [cleanDotBeforeLanguageSubfieldL];
@@ -313,8 +313,8 @@ const cleanValidPunctuationRules = {
 const addToAllEntryFields = [addDotBeforeLanguageSubfieldL, addSemicolonBeforeVolumeDesignation, addColonToRelationshipInformation, addEntryFieldFinalDot];
 
 
-const addX00 = [addX00aComma, addX00aComma2, addX00Dot, addX00dComma, ...addToAllEntryFields];
-const addX10 = [addX10iColon, addX10bDot, addX10eComma, addX10Dot, ...addToAllEntryFields];
+const addX00 = [addXX0iColon, addX00aComma, addX00aComma2, addX00Dot, addX00dComma, ...addToAllEntryFields];
+const addX10 = [addXX0iColon, addX10bDot, addX10Comma, addX10Dot, ...addToAllEntryFields];
 const addX11 = [...addToAllEntryFields, addX11Spacecolon];
 const addX30 = [...addToAllEntryFields];
 

package/src/removeDuplicateDataFields.js

@@ -1,5 +1,5 @@
 import createDebugLogger from 'debug';
-import {fieldsToString, fieldToString, nvdebug} from './utils.js';
+import {fieldToString, nvdebug} from './utils.js';
 import {fieldHasValidSubfield6, fieldsGetOccurrenceNumbers, fieldsToNormalizedString, fieldToNormalizedString, get6s} from './subfield6Utils.js';
 import {add8s, fieldHasLinkingNumber, fieldHasValidSubfield8, fieldsGetAllSubfield8LinkingNumbers, getSubfield8LinkingNumber, recordGetAllSubfield8LinkingNumbers, recordGetFieldsWithSubfield8LinkingNumber} from './subfield8Utils.js';
 
@@ -139,21 +139,21 @@ export function removeDuplicateSubfield8Chains(record, fix = true) {
 
   let removables = []; // for validation
 
-  nvdebug("CHAIN-8");
+  //nvdebug("CHAIN-8");
   const seenLinkingNumbers = recordGetAllSubfield8LinkingNumbers(record);
   if (seenLinkingNumbers.length === 0) {
     return removables;
   }
 
-  nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);
+  //nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);
 
   seenLinkingNumbers.forEach(currLinkingNumber => {
     const linkedFields = recordGetFieldsWithSubfield8LinkingNumber(record, currLinkingNumber) //getFieldsWithSubfield8Index(base, baseIndex);
     // As/If there's just one occurrence number it should be fine to use normalizeOccurrenceNumber = true
     const normalizeOccurrenceNumber = true;
     const linkedFieldsAsString = fieldsToNormalizedString(linkedFields, currLinkingNumber, normalizeOccurrenceNumber, true);
-    nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);
-    nvdebug(`${linkedFieldsAsString}`, debug);
+    //nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);
+    //nvdebug(`${linkedFieldsAsString}`, debug);
 
     if (linkedFieldsAsString in seen)  {
       if (!removables.includes(linkedFieldsAsString)) {
@@ -161,15 +161,15 @@ export function removeDuplicateSubfield8Chains(record, fix = true) {
       }
 
       if (fix) {
-        nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);
+        //nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);
         linkedFields.forEach(field => recordRemoveFieldOrSubfield8(record, field, currLinkingNumber));
         return;
       }
 
-      nvdebug(`$8 VALIDATION: DUPLICATE DETECTED ${linkedFieldsAsString}`, debug);
+      //nvdebug(`$8 VALIDATION: DUPLICATE DETECTED ${linkedFieldsAsString}`, debug);
       return;
     }
-    nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);
+    //nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);
     seen[linkedFieldsAsString] = 1;
     return;
   });
@@ -190,28 +190,28 @@ export function handleDuplicateSubfield8Chains(record, fix) {
 
   let seen = {};
 
-  nvdebug("CHAIN-8");
+  //nvdebug("CHAIN-8");
   const seenLinkingNumbers = recordGetAllSubfield8LinkingNumbers(record);
   if (seenLinkingNumbers.length === 0) {
     return;
   }
 
-  nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);
+  //nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);
 
   seenLinkingNumbers.forEach(currLinkingNumber => {
     const linkedFields = recordGetFieldsWithSubfield8LinkingNumber(record, currLinkingNumber) //getFieldsWithSubfield8Index(base, baseIndex);
     // As/If there's just one occurrence number it should be fine to use normalizeOccurrenceNumber = true
     const normalizeOccurrenceNumber = false; //true;
     const linkedFieldsAsString = fieldsToNormalizedString(linkedFields, currLinkingNumber, normalizeOccurrenceNumber, true);
-    nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);
-    nvdebug(`${linkedFieldsAsString}`, debug);
+    //nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);
+    //nvdebug(`${linkedFieldsAsString}`, debug);
 
     if (linkedFieldsAsString in seen)  {
-      nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);
+      //nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);
       linkedFields.forEach(field => newRecordRemoveFieldOrSubfield8(record, field, currLinkingNumber, fix));
       return;
     }
-    nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);
+    //nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);
     seen[linkedFieldsAsString] = 1;
     return;
   });
@@ -223,12 +223,12 @@ function markIdenticalSubfield6Chains(chain, record) {
   const normalizeTag = chain.some(field => field.tag.substring(0, 1) === '1'); // 1XX can delete 7XX as well!
   const chainAsString = fieldsToNormalizedString(chain, 0, normalizeOccurrenceNumber, normalizeTag);
 
-  nvdebug(`markIdenticalSubfield6Chains: ${chainAsString}`);
+  //nvdebug(`markIdenticalSubfield6Chains: ${chainAsString}`);
   record.fields.forEach(f => compareWithChain(f));
 
 
   function compareWithChain(f) {
-    nvdebug(`FIELD2CHAIN ${fieldToString(f)}`);
+    //nvdebug(`FIELD2CHAIN ${fieldToString(f)}`);
     const otherChain = fieldToChain(f, record);
     // Not a lone field or chain (head) or ... or is-same-chain
     if (otherChain.length === 0 || sameField(chain[0], otherChain[0])) {
@@ -239,7 +239,7 @@ function markIdenticalSubfield6Chains(chain, record) {
     // Mark other chain as deleted:
     if (chainAsString === otherChainAsString) {
       otherChain.forEach(f => {
-        nvdebug(` mark ${fieldToString(f)} as deleted ($6-chain)...`);
+        //nvdebug(` mark ${fieldToString(f)} as deleted ($6-chain)...`);
         f.deleted = 1;
       });
       return;
@@ -260,7 +260,7 @@ function markIdenticalLoneFieldsAsDeletable(field, record) {
 
   // Mark fields as deleted:
   identicalLoneFields.forEach(f => {
-    nvdebug(` mark ${fieldToString(f)} as deleted (lone field)...`);
+    //nvdebug(` mark ${fieldToString(f)} as deleted (lone field)...`);
     f.deleted = 1;
   });
 
@@ -300,7 +300,7 @@ function acceptFieldsWithSubfield8(fieldsWithSubfield8, requireSingleTag = false
 
   // If linking number
   function anomaly8(linkingNumber) {
-    nvdebug(`  Looking for anomalies in linkin number ${linkingNumber}`);
+    //nvdebug(`  Looking for anomalies in linkin number ${linkingNumber}`);
     const relevantFields = fieldsWithSubfield8.filter(f => fieldHasLinkingNumber(f, linkingNumber));
     if (requireSingleTag) {
       return !isSingleTagLinkingNumber(linkingNumber, relevantFields, relevantFields[0].tag);
@@ -332,7 +332,7 @@ export function fieldToChain(field, record) {
   }
   const chain = newGetAllLinkedFields(field, record, true, true);
 
-  nvdebug(` Chain contains ${chain.length} field(s)`);
+  // nvdebug(` Chain contains ${chain.length} field(s)`);
   if (!isChainHead(field, chain)) { // newGetAllLinkedFields() marks relevant record.fields!
     return [];
   }
@@ -366,7 +366,7 @@ export function fieldToChain(field, record) {
 
 function fieldHandleDuplicateDatafields(field, record) {
   const chain = fieldToChain(field, record);
-  nvdebug(` TRY TO HANDLE DUPLICATES OF '${fieldsToString(chain)}'`);
+  //nvdebug(` TRY TO HANDLE DUPLICATES OF '${fieldsToString(chain)}'`);
 
   if (chain.length === 0) {
     return;
@@ -383,7 +383,7 @@ function fieldHandleDuplicateDatafields(field, record) {
   if (fieldsWithSubfield6.length === 0) {
 
     if (fieldsWithSubfield8.length === 0) { // chain.length === 1?
-      nvdebug(` Trying to find duplicates of single field '${fieldToString(chain[0])}'`);
+      //nvdebug(` Trying to find duplicates of single field '${fieldToString(chain[0])}'`);
       markIdenticalLoneFieldsAsDeletable(chain[0], record);
       return;
     }
@@ -401,8 +401,8 @@ function fieldHandleDuplicateDatafields(field, record) {
   }
 
 
-  nvdebug(` NO HANDLER FOUND FOR '${fieldsToString(chain)}'`);
-  nvdebug(`  N8s: ${fieldsWithSubfield6.length}`);
+  //nvdebug(` NO HANDLER FOUND FOR '${fieldsToString(chain)}'`);
+  //nvdebug(`  N8s: ${fieldsWithSubfield6.length}`);
 
 }
 

package/src/removeInferiorDataFields.js

@@ -359,12 +359,13 @@ function deriveIndividualDeletables(record) {
   }
 
   function getPrepublicationTerms(fieldAsString) {
+    const subfield7Prepub = `${fieldAsString} ‡7 Ennakkotieto`
     if (fieldAsString.match(/^653./u)) {
       // MET-528 (extented by MET-575)
-      return [`${fieldAsString} ‡g ENNAKKOTIETO`, `${fieldAsString} ‡g Ennakkotieto`, `${fieldAsString} ‡g ennakkotieto`, `${fieldAsString} ‡g ENNAKKOTIETO.`, `${fieldAsString} ‡g Ennakkotieto.`, `${fieldAsString} ‡g ennakkotieto.`];
+      return [subfield7Prepub, `${fieldAsString} ‡g ENNAKKOTIETO`, `${fieldAsString} ‡g Ennakkotieto`, `${fieldAsString} ‡g ennakkotieto`, `${fieldAsString} ‡g ENNAKKOTIETO.`, `${fieldAsString} ‡g Ennakkotieto.`, `${fieldAsString} ‡g ennakkotieto.`];
     }
 
-    return [];
+    return [subfield7Prepub];
   }
 
 }

package/src/sortSubfields.js

@@ -8,8 +8,8 @@ const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:sortSu
 //const debugData = debug.extend('data');
 const debugDev = debug.extend('dev');
 
-const defaultSortOrderStringFinns = '8673abcdefghijklmnopqrstuvwxyz420159'; // NB! We Finns like $2 before $0 in 6XX...
-const defaultSortOrderStringOthers = '8673abcdefghijklmnopqrstuvwxyz402159';
+const defaultSortOrderStringFinns = '863abcdefghijklmnopqrstuvwxyz4201759'; // NB! We Finns like $2 before $0 in 6XX...
+const defaultSortOrderStringOthers = '863abcdefghijklmnopqrstuvwxyz4021759'; // NB 760-788 have '7' in different position...
 
 const defaultSortOrderFinns = defaultSortOrderStringFinns.split('');
 const defaultSortOrderOthers = defaultSortOrderStringOthers.split('');
@@ -70,7 +70,7 @@ export default function (defaultTagPattern) {
 const sortOrderForX00 = ['i', 'a', 'b', 'q', 'c', 'd', 'e', 't', 'u', 'l', 'f', 'x', 'y', 'z', '0', '1', '5', '9']; // skip $g. Can't remember why, though...
 const sortOrderForX10 = ['i', 'a', 'b', 't', 'n', 'c', 'e', 'v', 'w', 'x', 'y', 'z', '0', '1', '5', '9']; // somewhat iffy
 const sortOrderForX11 = ['a', 'n', 'd', 'c', 'e', 'g', 'j', '0', '1', '5', '9'];
-const sortOrderFor7XX = ['8', '7', 'i', 'a', 's', 't', 'b', 'c', 'd', 'm', 'h', 'k', 'o', 'x', 'z', 'g', 'q', 'w'];
+const sortOrderFor7XXLinkingEntries = ['8', '7', 'i', 'a', 's', 't', 'b', 'c', 'd', 'm', 'h', 'k', 'o', 'x', 'z', 'g', 'q', 'w', '0', '1', 'l', '9', '5'];
 const sortOrderFor246 = ['i', 'a', 'b', 'n', 'p', 'f', '5', '9']; // Used by field 946 as well
 
 // List *only* exceptional order here. Otherwise default order is used.
@@ -107,22 +107,22 @@ const subfieldSortOrder = [
   {'tag': '700', 'sortOrder': sortOrderForX00},
   {'tag': '710', 'sortOrder': sortOrderForX10},
   {'tag': '711', 'sortOrder': sortOrderForX11},
-  {'tag': '760', 'sortOrder': sortOrderFor7XX},
-  {'tag': '762', 'sortOrder': sortOrderFor7XX},
-  {'tag': '765', 'sortOrder': sortOrderFor7XX},
-  {'tag': '767', 'sortOrder': sortOrderFor7XX},
-  {'tag': '770', 'sortOrder': sortOrderFor7XX},
-  {'tag': '772', 'sortOrder': sortOrderFor7XX},
-  {'tag': '773', 'sortOrder': sortOrderFor7XX},
-  {'tag': '774', 'sortOrder': sortOrderFor7XX},
-  {'tag': '775', 'sortOrder': sortOrderFor7XX},
-  {'tag': '776', 'sortOrder': sortOrderFor7XX},
-  {'tag': '777', 'sortOrder': sortOrderFor7XX},
-  {'tag': '780', 'sortOrder': sortOrderFor7XX},
-  {'tag': '785', 'sortOrder': sortOrderFor7XX},
-  {'tag': '786', 'sortOrder': sortOrderFor7XX},
-  {'tag': '787', 'sortOrder': sortOrderFor7XX},
-  {'tag': '788', 'sortOrder': sortOrderFor7XX},
+  {'tag': '760', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '762', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '765', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '767', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '770', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '772', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '773', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '774', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '775', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '776', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '777', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '780', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '785', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '786', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '787', 'sortOrder': sortOrderFor7XXLinkingEntries},
+  {'tag': '788', 'sortOrder': sortOrderFor7XXLinkingEntries},
   {'tag': '800', 'sortOrder': sortOrderForX00},
   {'tag': '810', 'sortOrder': sortOrderForX10},
   {'tag': '811', 'sortOrder': sortOrderForX11},

package/src/subfield6Utils.js

@@ -219,7 +219,7 @@ export function fieldGetOccurrenceNumberPairs(field, candFields) {
   //nvdebug(`  Trying to finds pair for ${fieldToString(field)} in ${candFields.length} fields`);
   const pairs = candFields.filter(otherField => isSubfield6Pair(field, otherField));
   if (pairs.length === 0) {
-    nvdebug(`NO PAIRS FOUND FOR '${fieldToString(field)}'`);
+    //nvdebug(`NO PAIRS FOUND FOR '${fieldToString(field)}'`);
     return pairs;
   }
   //nvdebug(`${pairs.length} PAIR(S) FOUND FOR '${fieldToString(field)}'`);

package/src/subfield8Utils.js

@@ -1,7 +1,7 @@
 // import createDebugLogger from 'debug';
 // const debug = createDebugLogger('@natlibfi/marc-record-validator-melinda/subfield8Utils');
 
-import {fieldToString, nvdebug} from './utils.js';
+// import {fieldToString, nvdebug} from './utils.js';
 
 const sf8Regexp = /^([1-9][0-9]*)(?:\.[0-9]+)?(?:\\[acprux])?$/u;
 
@@ -56,7 +56,7 @@ export function fieldsGetAllSubfield8LinkingNumbers(fields) {
     field.subfields.forEach(sf => {
       const linkingNumber = getSubfield8LinkingNumber(sf);
       if (linkingNumber > 0 && !subfield8LinkingNumbers.includes(linkingNumber)) {
-        nvdebug(` LINK8: Add subfield \$8 ${linkingNumber} to seen values list`);
+        //nvdebug(` LINK8: Add subfield \$8 ${linkingNumber} to seen values list`);
         subfield8LinkingNumbers.push(linkingNumber);
       }
     });
@@ -76,10 +76,10 @@ export function add8s(fields, record) {
     return fields;
   }
 
-  nvdebug(`Linking number(s): ${linkingNumbers.join(', ')}`);
+  //nvdebug(`Linking number(s): ${linkingNumbers.join(', ')}`);
   linkingNumbers.forEach(number => collectLinkingNumberFields(number));
 
-  fields.forEach(f => nvdebug(`AFTER ADDING 8s: '${fieldToString(f)}'`));
+  //fields.forEach(f => nvdebug(`AFTER ADDING 8s: '${fieldToString(f)}'`));
 
   return fields;
 
@@ -88,7 +88,7 @@ export function add8s(fields, record) {
     fields = fields.filter(f => !fieldHasLinkingNumber(f, linkingNumber));
     // Add them and their "sisters" back:
     const addableFields = record.fields.filter(f => fieldHasLinkingNumber(f, linkingNumber));
-    addableFields.forEach(f => nvdebug(`(RE-?)ADD ${fieldToString(f)}`));
+    //addableFields.forEach(f => nvdebug(`(RE-?)ADD ${fieldToString(f)}`));
     fields = fields.concat(addableFields);
 
   }

package/src/utils.js

@@ -1,13 +1,12 @@
 import createDebugLogger from 'debug';
-
-//import fs from 'fs';
-//import path from 'path';
+import {melindaFieldSpecs} from './melindaCustomMergeFields.js';
+import {isDataProvenanceSubfieldCode} from './dataProvenanceUtils.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:utils');
 //const debugData = debug.extend('data');
 const debugDev = debug.extend('dev');
 
-import {melindaFieldSpecs} from './melindaCustomMergeFields.js';
+
 
 //JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'src', 'melindaCustomMergeFields.json'), 'utf8'));
 
@@ -102,19 +101,38 @@ export function nvdebugFieldArray(fields, prefix = '  ', func = undefined) {
   fields.forEach(field => nvdebug(`${prefix}${fieldToString(field)}`, func));
 }
 
-export function isControlSubfieldCode(subfieldCode) {
+function isControlSubfieldCode(subfieldCode, tag = undefined) {
   // NB! Only $w, $0, $1, $5, $6 and $8 are really control subfields. In Finland $9 is oft a control subfield
-  // $3 material (part of the whole thing)
-  // $4 means 'relationship' (similar to relator terms at least in X00 and similar)
-  // $7 is usually provinance subfield. However, it can be stored in other subfields as well. See merge-fields/dataProvenance.js for details
-  // However, change this only if needed. Maybe all provinance subfields should return true?
-  // This may become relevant when AI starts to create stuff...
-  if (['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'w'].includes(subfieldCode)) {
+  if (['0', '1', '2', '5', '6', '8', 'w'].includes(subfieldCode)) {
     return true;
   }
+  // Subfield '7' is control subfield for some tags:
+  if (tag && subfieldCode === '7') {
+    if ( tag.match(/^7[678]/u) || ['800', '810', '811', '830'].includes(tag) ) {
+      return true;
+    }
+  }
   return false;
 }
 
+export function isContentSubfieldCode(subfieldCode, tag = undefined) {
+  if (isControlSubfieldCode(subfieldCode, tag)) { // 'w', '0', '1', '5', '6' and '8'. (Also '7' for ....)
+    return false;
+  }
+  if (tag && isDataProvenanceSubfieldCode(subfieldCode, tag)) { // Note that default '7' is handled below
+    return false;
+  }
+  // $7 contains typically data provenance, sometimes it's a control field, and for f533, f856 ja f857 it's something else, but it's never content!
+  if (['2', '3', '4', '7', '9'].includes(subfieldCode)) {
+    return false;
+  }
+
+
+  return true;
+}
+
+
+
 export function getCatalogingLanguage(record, defaultCatalogingLanguage = undefined) {
   const [field040] = record.get(/^040$/u);
   if (!field040) {
@@ -247,4 +265,13 @@ export function subfieldArraysContainSameData(arr1, arr2) {
   }
 
   return arr2.every(sf2 => arr1.some(sf => subfieldsAreIdentical(sf, sf2)));
-}
+}
+
+export function tagIsRepeatable(tag) {
+  const fieldSpecs = melindaFieldSpecs.fields.filter(field => field.tag === tag);
+  if (fieldSpecs.length !== 1) {
+    debugDev(` WARNING! Getting field ${tag} data failed! Default to repeatable field.`);
+    return true;
+  }
+  return fieldSpecs[0].repeatable;
+}

package/test-fixtures/cyrillux/f14/expectedResult.json

@@ -0,0 +1,32 @@
+{
+  "_validationOptions": {},
+  "leader": "12345cam  22123454i 4500",
+  "fields": [
+    { "tag": "008", "value": "01234567890123456789012345678901234ukr89"},
+    { "tag": "240", "ind1": " ", "ind2": " ", "subfields": [
+        { "code": "6", "value": "880-01" },
+        { "code": "a", "value": "Hohol." },
+        { "code": "l", "value": "Hohol." },
+        { "code": "9", "value": "SFS4900 <TRANS>" }
+    ]},
+    { "tag": "776", "ind1": " ", "ind2": " ", "subfields": [
+        { "code": "6", "value": "880-02" },
+        { "code": "t", "value": "Hohol." },
+        { "code": "l", "value": "Гоголь." },
+        { "code": "9", "value": "SFS4900 <TRANS>" }
+    ]},
+    { "tag": "880", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "6", "value": "240-01" },
+      { "code": "a", "value": "Гоголь." },
+      { "code": "l", "value": "Гоголь." },
+      { "code": "9", "value": "CYRILLIC <TRANS>" }
+    ]},
+    { "tag": "880", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "6", "value": "776-02" },
+      { "code": "t", "value": "Гоголь." },
+      { "code": "l", "value": "Гоголь." },
+      { "code": "9", "value": "CYRILLIC <TRANS>" }
+    ]}
+  ]
+}
+

package/test-fixtures/cyrillux/f14/metadata.json

@@ -0,0 +1,10 @@
+{
+  "description": "14: transliteration(SFS-4900 only) does not apply the provenance subfield $l",
+  "comment": "The point in this test is that 776$l is not transliterated as it is \"untouchable\" provenance subfield, while 240$l is transliterated.",
+  "only": false,
+  "fix": true,
+  "config": {
+    "doISO9Transliteration": false,
+    "doSFS4900Transliteration": true
+  }
+}

package/test-fixtures/cyrillux/f14/record.json

@@ -0,0 +1,14 @@
+{
+  "leader": "12345cam  22123454i 4500",
+  "fields": [
+    { "tag": "008", "value": "01234567890123456789012345678901234ukr89"},
+    { "tag": "240", "ind1": " ", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Гоголь." },
+        { "code": "l", "value": "Гоголь." }
+    ]},
+    { "tag": "776", "ind1": " ", "ind2": " ", "subfields": [
+        { "code": "t", "value": "Гоголь." },
+        { "code": "l", "value": "Гоголь." }
+    ]}
+  ]
+}

package/test-fixtures/merge-fields/f042_01/expectedResult.json

@@ -0,0 +1,12 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "042", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "lc" },
+      { "code": "a", "value": "nsdp" },
+      { "code": "a", "value": "lcnuc" }
+    ]}
+  ],
+
+  "_validationOptions": {}
+}

package/test-fixtures/merge-fields/f042_01/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "f042/01: merge f042, rare case where we don't need a pair of any kind",
+  "fix": true,
+  "tagPattern": "^042$",
+  "only": false
+}

package/test-fixtures/merge-fields/f042_01/record.json

@@ -0,0 +1,13 @@
+{
+  "leader": "01331cam a22003494i 4500",
+    "fields": [
+    { "tag": "042", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "lc" }
+    ]},
+    { "tag": "042", "ind1": " ", "ind2": " ", "subfields": [
+        { "code": "a", "value": "nsdp" },
+        { "code": "a", "value": "lcnuc" }
+    ]}
+  ],
+  "_validationOptions": {}
+}

package/test-fixtures/merge-fields/f06/expectedResult.json

@@ -0,0 +1,42 @@
+{
+  "_validationOptions": {},
+  "fields": [
+        { "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Wuolijoki, Hella." },
+        { "code": "t", "value": "Heta Niskavuori." },
+        { "code": "k", "value": "Valikoima." },
+        { "code": "0", "value": "(FIN11)000063570"
+        }
+      ]
+    },
+    { "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Wuolijoki, Hella." },
+        { "code": "t", "value": "Heta Niskavuori." },
+        { "code": "k", "value": "Valikoima." },
+        { "code": "l", "value": "Ruotsi." },
+        { "code": "0", "value": "(FIN11)000063570"
+        }
+      ]
+    },
+    { "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Nuolijoki, Nella." },
+        { "code": "t", "value": "Beta Tiskivuori." },
+        { "code": "l", "value": "Suomi." },
+        { "code": "0", "value": "(FIN11)900063570"
+        }
+      ]
+    },
+    { "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Nuolijoki, Nella." },
+        { "code": "t", "value": "Beta Tiskivuori." },
+        { "code": "k", "value": "Valikoima." },
+        { "code": "l", "value": "Ruotsi." },
+        { "code": "0", "value": "(FIN11)900063570"
+        }
+      ]
+    }
+
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}

package/test-fixtures/merge-fields/f06/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "Don't merge name-title with missing language, and don't merge different languages",
+  "fix": true,
+  "tagPattern": "^(700)$",
+  "only": false
+}

package/test-fixtures/merge-fields/f06/record.json

@@ -0,0 +1,41 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Wuolijoki, Hella." },
+        { "code": "t", "value": "Heta Niskavuori." },
+        { "code": "k", "value": "Valikoima." },
+        { "code": "0", "value": "(FIN11)000063570"
+        }
+      ]
+    },
+    { "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Wuolijoki, Hella." },
+        { "code": "t", "value": "Heta Niskavuori." },
+        { "code": "k", "value": "Valikoima." },
+        { "code": "l", "value": "Ruotsi." },
+        { "code": "0", "value": "(FIN11)000063570"
+        }
+      ]
+    },
+    { "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Nuolijoki, Nella." },
+        { "code": "t", "value": "Beta Tiskivuori." },
+        { "code": "l", "value": "Suomi." },
+        { "code": "0", "value": "(FIN11)900063570"
+        }
+      ]
+    },
+    { "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
+        { "code": "a", "value": "Nuolijoki, Nella." },
+        { "code": "t", "value": "Beta Tiskivuori." },
+        { "code": "k", "value": "Valikoima." },
+        { "code": "l", "value": "Ruotsi." },
+        { "code": "0", "value": "(FIN11)900063570"
+        }
+      ]
+    }
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}