@natlibfi/marc-record-validators-melinda 12.0.6 → 12.0.7

package/src/merge-fields/dataProvenance.js

@@ -1,41 +1,24 @@
 // See https://www.loc.gov/marc/bibliographic/bdapndxj.html for details
 
-import {subfieldArraysContainSameData} from "../utils.js";
-
-export function tagToDataProvenanceSubfieldCode(tag) {
-    if ( ['533', '800', '810', '811', '830'].includes(tag)) {
-        return 'y';
-    }
-    if ( tag === '856' || tag === '857' ) {
-        return 'e';
-    }
-
-    if ( tag.match(/^7[678]/u) ) {
-        return 'l'
-    }
-
-    if ( tag.match(/^00/u)) {
-        return undefined;
-    }
-    return '7';
-}
-
+import {subfieldArraysContainSameData} from '../utils.js';
+import {tagToDataProvenanceSubfieldCode} from '../dataProvenanceUtils.js';
 
 export function provenanceSubfieldsPermitMerge(baseField, sourceField) {
-    const provinanceSubfieldCode = tagToDataProvenanceSubfieldCode(baseField.tag);
     if (!baseField.subfields) {
         return true;
     }
-    if (provinanceSubfieldCode === undefined) {
+    const provenanceSubfieldCode = tagToDataProvenanceSubfieldCode(baseField.tag);
+
+    if (provenanceSubfieldCode === undefined) {
         return false;
     }
 
-    const baseProvinanceSubfields = baseField.subfields.filter(sf => sf.code === provinanceSubfieldCode);
-    const sourceProvinanceSubfields = sourceField.subfields.filter(sf => sf.code === provinanceSubfieldCode);
+    const baseProvenanceSubfields = baseField.subfields.filter(sf => sf.code === provenanceSubfieldCode);
+    const sourceProvenanceSubfields = sourceField.subfields.filter(sf => sf.code === provenanceSubfieldCode);
 
     // Currently we just compare two arrays. Later on we might do something more sophisticated with specific $7 data provenance category/relationship codes,
     // or actual values.
 
-    return subfieldArraysContainSameData(baseProvinanceSubfields, sourceProvinanceSubfields);
+    return subfieldArraysContainSameData(baseProvenanceSubfields, sourceProvenanceSubfields);
 
 }

package/src/merge-fields/index.js

@@ -49,7 +49,7 @@ export default function (defaultTagPattern = undefined) {
   }
 
   function fix(record, config = undefined) {
-    const config2 = config || defaultConfig;
+    const config2 = config || defaultConfig.mergeConfiguration;
     record.internalMerge = true;
     mergeFieldsWithinRecord(record, config2);
     delete record.internalMerge;

package/src/merge-fields/mergableIndicator.js

@@ -1,5 +1,5 @@
 import createDebugLogger from 'debug';
-import {marc21GetTagsLegalInd1Value, marc21GetTagsLegalInd2Value, nvdebug} from '../utils.js';
+import {marc21GetTagsLegalInd1Value, marc21GetTagsLegalInd2Value} from '../utils.js';
 
 // Specs: https://workgroups.helsinki.fi/x/K1ohCw (though we occasionally differ from them)...
 
@@ -20,7 +20,6 @@ function marc21NoNeedToCheckInd1(tag) {
 
 function marc21NoNeedToCheckInd2(tag) {
   const cands = marc21GetTagsLegalInd2Value(tag);
-  nvdebug(`CHECK IND2 ${typeof cands} FOR ${tag}`, debugDev);
   if (typeof cands === 'string') { // single cand
     return true;
   }

package/src/merge-fields/mergeConfig.js

@@ -30,9 +30,10 @@ export const mergeConfig = {
       '246': {'0': 1, '1': 1, '2': 1, '3': 1, '4': 1, '5': 1, '6': 1, '7': 1, '8': 1, ' ': 2},
       '363': {'0': 1, '1': 1, ' ': 2},
       '382': {'0': 1, '1': 1, ' ': 2},
-      '730': ['2', ' ']
+      '730': ['2', ' '] // If this is ok, it ok for 700, 710 and 711 as well...
     },
     'comment #4': 'List indicators that do not block merge here. Non-filing indicators do not prevent field merge (their support is hard-coded). They are mainly listed here as an example.',
+    // Actually I wouldn't want to ignore X00ind1=3 vs other values such as IND1=1... However, this is not currently supported.
     'ignoreIndicator1': ['100', '110', '111', '130', '210', '242', '245', '246', '247', '307', '490', '505', '506', '510', '511', '516', '520', '521', '522', '524', '526', '583', '586', '600', '610', '630', '650', '651', '655', '700', '710', '730', '740', '760', '762', '765', '767', '770', '772', '773', '774', '775', '776', '777', '780', '785', '786', '787', '788', '800', '810'],
     'ignoreIndicator2': ['017', '222', '240', '242', '243', '245', '760', '762', '765', '767', '770', '773', '774', '775', '776', '777', '786', '787', '788', '830'],
     'comment #5': 'If one indicator has value, and the other has not, it does not necessarily mean mismatch',

package/src/merge-fields/mergeConstraints.js

@@ -3,19 +3,20 @@ const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:me
 //const debugData = debug.extend('data');
 const debugDev = debug.extend('dev');
 
-// Specs: https://workgroups.helsinki.fi/x/K1ohCw (though we occasionally differ from them)...
-
-// "key" is an unique key that must match (be absent or exist+be identical) in both.
-// "paired" refers to a field that must either exist in both or be absent in both (negative XOR). Typically it's not defined.
+// "required": both fields bust have the subfield and the value must be same.
+// "paired": subfield bust either exists (with value match) or not exist in both fields (negative XOR). Sort of an optional version of 'required'
+// "key": one or both fields lack the subfield OR both fields have it and theor values match.
+// There are typically NR subfields but the system should support multival repeatable subfields. However, (R) support is considered experimental.
 // NB: key+paired with identical values is an attempt to prevent copy for (ET) fields, and to force separate fields on (T) fields.
 // NB! If base has eg. no 264, two+ 264 fields can be copied from the source.
 
 // NB! not all X00 fields have, say, $x subfield. However, we can still share them...
 // $h is non-1XX?, $i is 7XX only, $w is 8XX only...
-const keyX00 = 'abcjloqrtuwx'; // Shared: $abcdefg...
-const keyX10 = 'abcdfghlnoprstuwx';
-const keyX11 = 'acdefghlnpqstuwx';
+const keyX00 = 'abcjklnoqrstuwx'; // Shared: $abcdefg...
+const keyX10 = 'abcdfghklnoprstuwx';
+const keyX11 = 'acdefghklnpqstuwx';
 const keyX30 = 'adfghklmnoprstvwxyz';
+const pairedTitleSubfields = 'iklmnoprst'
 
 const mergeConstraints = [
   {'tag': '010', 'required': 'a', 'key': 'a'},
@@ -41,8 +42,8 @@ const mergeConstraints = [
   {'tag': '037', 'required': 'b', 'key': 'ab'},
   {'tag': '039', 'required': 'a'},
   {'tag': '040', 'required': '', 'key': ''},
-  {'tag': '041', 'required': '', 'paired': '2', 'key': ''}, // Don't put $2 in 'key'! hasCommonNominator() would get into trouble with it...
-  {'tag': '042', 'required': 'a', 'key': ''}, // NB: preprocessor hacks applied
+  {'tag': '041', 'required': '', 'paired': '2', 'key': 'ad'}, // Don't put $2 in 'key'! hasCommonNominator() would get into trouble with it...
+  {'tag': '042', 'required': '', 'key': ''}, // Contents (supposedly $a subfields) can be anything, and still merge... (042 $a foo + 042 $b bar is perfectly mergeable)
   {'tag': '043', 'required': 'a', 'key': 'abc'},
   {'tag': '044', 'required': '', 'key': 'abc', 'paired': 'abc'},
   {'tag': '045', 'required': '', 'key': 'abc', 'paired': 'abc'}, // (ET) // 045 is problematic either-$a or $b or $c...
@@ -69,11 +70,11 @@ const mergeConstraints = [
   {'tag': '086', 'required': '', 'paired': 'a', 'key': 'a'},
   {'tag': '088', 'required': '', 'paired': 'a', 'key': 'a'},
   // NB! 100, 110 and 111 may have title parts that are handled elsewhere
-  {'tag': '100', 'required': 'a', 'paired': 't', 'key': keyX00},
-  {'tag': '110', 'required': 'a', 'paired': 'bt', 'key': keyX10},
-  {'tag': '111', 'required': 'a', 'paired': 't', 'key': keyX11},
+  {'tag': '100', 'required': 'a', 'paired': `b${pairedTitleSubfields}`, 'key': keyX00},
+  {'tag': '110', 'required': 'a', 'paired': `b${pairedTitleSubfields}`, 'key': keyX10},
+  {'tag': '111', 'required': 'a', 'paired': pairedTitleSubfields, 'key': keyX11},
   // NB! 130 has no name part, key is used for title part
-  {'tag': '130', 'required': 'a', 'key': keyX30},
+  {'tag': '130', 'required': 'a', paired: pairedTitleSubfields, 'key': keyX30},
   {'tag': '210', 'required': 'a', 'key': 'ab'},
   {'tag': '222', 'required': 'a', 'key': 'ab'},
   {'tag': '240', 'required': 'a', 'key': 'adfghklmnoprs'},
@@ -95,7 +96,7 @@ const mergeConstraints = [
   //{'tag': '264', 'required': '', 'paired': 'abc', 'key': 'abc'}, // NB "S.l." normalizations?" not implemented
   {'tag': '264', 'required': '', 'key': 'abc'}, // NB "S.l." normalizations?" not implemented
   // SKIP TAG 270 ON PURPOSE! Melinda's N=43.
-  {'tag': '300', 'required': 'a', 'key': 'abcefg'},
+  {'tag': '300', 'required': '', 'key': 'abcefg'},
   {'tag': '306', 'required': 'a', 'key': 'a'},
   // SKIP TAG 307 ON PURPOSE! N=0
   {'tag': '310', 'required': 'a', 'key': 'ab'},
@@ -132,7 +133,7 @@ const mergeConstraints = [
   {'tag': '385', 'required': 'a', 'paired': 'abmn', 'key': 'abmn'},
   {'tag': '386', 'required': 'a', 'paired': 'abmn', 'key': 'abmn'},
   {'tag': '388', 'required': 'a', 'key': 'a'},
-  {'tag': '490', 'required': 'a', 'key': 'axvl'},
+  {'tag': '490', 'required': '', 'key': 'axvl'},
   {'tag': '500', 'required': 'a', 'key': 'a'},
   {'tag': '501', 'required': 'a', 'key': 'a'},
   {'tag': '502', 'required': 'a', 'key': 'abcdgo'},
@@ -195,10 +196,10 @@ const mergeConstraints = [
   {'tag': '597', 'required': ''},
   {'tag': '598', 'required': ''},
   {'tag': '599', 'required': ''},
-  {'tag': '600', 'required': 'a', 'paired': 'tvxyz', 'key': keyX00},
-  {'tag': '610', 'required': 'a', 'paired': 'btvxyz', 'key': keyX10},
-  {'tag': '611', 'required': 'a', 'paired': 'tvxyz', 'key': keyX11},
-  {'tag': '630', 'required': 'a', 'paired': 'atvxyz', 'key': keyX30},
+  {'tag': '600', 'required': 'a', 'paired': `b${pairedTitleSubfields}vxyz`, 'key': keyX00},
+  {'tag': '610', 'required': 'a', 'paired': `b${pairedTitleSubfields}vxyz`, 'key': keyX10},
+  {'tag': '611', 'required': 'a', 'paired': `${pairedTitleSubfields}vxyz`, 'key': keyX11},
+  {'tag': '630', 'required': 'a', 'paired': `${pairedTitleSubfields}vxyz`, 'key': keyX30},
   // NB! 700, 710 and 711 may have title parts that are handled elsewhere
   {'tag': '647', 'required': 'a', 'paired': 'avxyz', 'key': 'acdgvxyz02'},
   {'tag': '648', 'required': 'a', 'paired': 'avxyz', 'key': 'avxyz02'},
@@ -212,12 +213,12 @@ const mergeConstraints = [
   {'tag': '658', 'required': 'a', 'paired': 'abcd'}, // N=0
   {'tag': '662', 'required': '', 'paired': 'abcdefgh'}, // N=0
   {'tag': '688', 'required': 'a'}, // N=0
-  {'tag': '700', 'required': 'a', 'paired': 't', 'key': keyX00}, // h/i/m/o/r/s/x are missing from 100
-  {'tag': '710', 'required': 'a', 'paired': 'bt', 'key': keyX10}, // h/j/m/o/r/s/x are missing from 110
-  {'tag': '711', 'required': 'a', 'paired': 'cdeflns', 'key': keyX11}, // h/i/s/x are missing from 711
+  {'tag': '700', 'required': 'a', 'paired': `b${pairedTitleSubfields}x`, 'key': keyX00}, // h/i/m/o/r/s/x are missing from 100, NB! 's' is repeatable, but we are not merging them here!
+  {'tag': '710', 'required': 'a', 'paired': `b${pairedTitleSubfields}x`, 'key': keyX10}, // h/j/m/o/r/s/x are missing from 110
+  {'tag': '711', 'required': 'a', 'paired': `cdef${pairedTitleSubfields}x`, 'key': keyX11}, // h/i/s/x are missing from 711
   {'tag': '720', 'required': 'a', 'key': 'a'},
   // NB! 730 has no name part, key is used for title part
-  {'tag': '730', 'required': 'a', 'key': keyX30}, // NB: 130->730 magic subfields might not agree...
+  {'tag': '730', 'required': 'a', 'paired': `df${pairedTitleSubfields}x`, 'key': keyX30}, // NB: 130->730 magic subfields might not agree...
   {'tag': '740', 'required': 'a', 'key': 'ahnp'},
   {'tag': '751', 'required': 'a', 'key': 'a'}, // N=11, kaikissa pelkkä $a
   {'tag': '752', 'required': '', 'key': 'abcdefgh'}, // N=12234
@@ -242,7 +243,7 @@ const mergeConstraints = [
   {'tag': '786', 'required': '', 'paired': 'abcrstuxyz', 'key': 'abcdhijmprstuxyz4'},
   {'tag': '787', 'required': '', 'paired': 'abcdhmstuxyz4'},
   {'tag': '788', 'required': '', 'paired': 'stx', 'key': 'abdestx'},
-  {'tag': '800', 'required': 'a', 'paired': 't', 'key': keyX00},
+  {'tag': '800', 'required': 'a', 'paired': 'bt', 'key': keyX00},
   {'tag': '810', 'required': 'a', 'paired': 'bt', 'key': keyX10},
   {'tag': '811', 'required': 'a', 'paired': 't', 'key': keyX11},
   {'tag': '830', 'required': 'a', 'key': keyX30},
@@ -289,22 +290,26 @@ const mergeConstraints = [
   {'tag': 'SID', 'required': ''}
 ];
 
-function constraintToValue(tagsConstraints, constraintName) {
-  if (constraintName in tagsConstraints) {
-    return tagsConstraints[constraintName];
-  }
-  return null; // NB! "" might mean "apply to everything" (eg. 040.key) while null means that it is not applied.
-}
 
-export function getMergeConstraintsForTag(tag, constraintName) {
+export function getMergeConstraintsForTag(tag, constraintName = undefined) {
   const tagsConstraintsArray = mergeConstraints.filter(entry => tag === entry.tag);
   if (tagsConstraintsArray.length === 0) {
-    debugDev(`WARNING\tNo key found for ${tag}. Returning NULL!`);
-    return null;
+    debugDev(`WARNING\tNo key found for ${tag}!`);
+  }
+  if (!constraintName) {
+    return tagsConstraintsArray;
   }
   // NB! should we support multiple contains for a field? Eg. 505$a vs 505($tg)+
   if (tagsConstraintsArray.length > 1) {
     debugDev(`WARNING\tMultiple values for '${constraintName}' (N=${tagsConstraintsArray.length}) found in ${tag}. Using first values.`);
   }
-  return constraintToValue(tagsConstraintsArray[0], constraintName);
+  //return constraintToValue(tagsConstraintsArray[0], constraintName);
+  return tagsConstraintsArray.map(c => constraintToValue(c, constraintName));
+
+  function constraintToValue(tagsConstraints, constraintName) {
+    if (constraintName in tagsConstraints) {
+      return tagsConstraints[constraintName];
+    }
+    return null; // NB! "" might mean "apply to everything" (eg. 040.key) while null means that it is not applied.
+  }
 }

package/src/merge-fields/mergeField.js

@@ -2,19 +2,16 @@
 import createDebugLogger from 'debug';
 import {fieldToString, fieldsToString, fieldsAreIdentical, nvdebug, hasCopyright, removeCopyright, subfieldToString} from '../utils.js';
 import {fieldGetOccurrenceNumberPairs} from '../subfield6Utils.js';
-import {cloneAndNormalizeFieldForComparison, cloneAndRemovePunctuation, isEnnakkotietoSubfieldG} from '../normalizeFieldForComparison.js';
+import {cloneAndNormalizeFieldForComparison, cloneAndRemovePunctuation} from '../normalizeFieldForComparison.js';
 import {mergeOrAddSubfield} from './mergeOrAddSubfield.js';
 import {mergeIndicators} from './mergeIndicator.js';
 import {mergableTag} from './mergableTag.js';
 import {getCounterpart} from './counterpartField.js';
+import {isEnnakkotietoSubfield} from '../prepublicationUtils.js';
 //import {default as normalizeEncoding} from '@natlibfi/marc-record-validators-melinda/dist/normalize-utf8-diacritics';
 //import {postprocessRecords} from './mergeOrAddPostprocess.js';
 //import {preprocessBeforeAdd} from './processFilter.js';
 
-//import fs from 'fs';
-//import path from 'path';
-
-
 //const defaultConfig = JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'reducers', 'config.json'), 'utf8'));
 
 // Specs: https://workgroups.helsinki.fi/x/K1ohCw (though we occasionally differ from them)...
@@ -25,7 +22,7 @@ const debugDev = debug.extend('dev');
 
 // NB! Can we do this via config.json?
 function removeEnnakkotieto(field) {
-  const tmp = field.subfields.filter(subfield => !isEnnakkotietoSubfieldG(subfield));
+  const tmp = field.subfields.filter(subfield => !isEnnakkotietoSubfield(subfield));
   // remove only iff some other subfield remains
   if (tmp.length > 0) {
     field.subfields = tmp;
@@ -59,7 +56,7 @@ function mergeField2(baseRecord, baseField, sourceField, config, candFieldPairs8
   // If a base ennakkotieto is merged with real data, remove ennakkotieto subfield:
   // (If our prepub normalizations are ok, this should not be needed.
   //  However, it's simple and works well enough, so let's keep it here.)
-  if (baseField.subfields?.find(sf => isEnnakkotietoSubfieldG(sf)) && !sourceField.subfields?.find(sf => isEnnakkotietoSubfieldG(sf))) {
+  if (baseField.subfields?.find(sf => isEnnakkotietoSubfield(sf)) && !sourceField.subfields?.find(sf => isEnnakkotietoSubfield(sf))) {
     removeEnnakkotieto(baseField);
     baseField.merged = 1;
   }

package/src/merge-fields/mergeOrAddSubfield.js

@@ -1,12 +1,13 @@
 import createDebugLogger from 'debug';
-import {cloneAndNormalizeFieldForComparison, isEnnakkotietoSubfieldG} from '../normalizeFieldForComparison.js';
+import {cloneAndNormalizeFieldForComparison} from '../normalizeFieldForComparison.js';
 import {normalizeAs, normalizeControlSubfieldValue} from '../normalize-identifiers.js';
-import {fieldHasSubfield, fieldToString, isControlSubfieldCode, nvdebug, subfieldIsRepeatable, subfieldToString} from '../utils.js';
+import {fieldHasSubfield, fieldToString, isContentSubfieldCode, nvdebug, subfieldIsRepeatable, subfieldToString} from '../utils.js';
 import {mergeSubfield} from './mergeSubfield.js';
 import {sortAdjacentSubfields} from '../sortSubfields.js';
 
 import {valueCarriesMeaning} from './worldKnowledge.js';
 import {resetSubfield6Tag} from '../subfield6Utils.js';
+import {isEnnakkotietoSubfield} from '../prepublicationUtils.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:mergeOrAddSubfield');
 //const debugData = debug.extend('data');
@@ -25,11 +26,11 @@ function catalogingSourceModifyingAgencyCandIsOriginalCatalogingSourceAgencyInTa
   return false;
 }
 
-function ennakkotietoInSubfieldG(candSubfieldData) {
-  if (isEnnakkotietoSubfieldG({'code': candSubfieldData.code, 'value': candSubfieldData.originalValue})) {
+function dataContainsPrepublicationSubfield(candSubfieldData) {
+  if (isEnnakkotietoSubfield({'code': candSubfieldData.code, 'value': candSubfieldData.originalValue})) {
     // Skip just ‡g subfield or the whole field?
     // We decided to skip just this subfield. We want at least $0 and maybe even more from ennakkotieto.
-    debugDev(`Skip '‡g ${candSubfieldData.originalValue}'`);
+    debugDev(`Skip '‡${candSubfieldData.code} ${candSubfieldData.originalValue}'`);
     return true;
   }
   return false;
@@ -75,7 +76,7 @@ function skipNormalizedComparison(tag, subfieldCode, subfieldValue) {
 }
 
 function mergeOrAddSubfieldNotRequired(targetField, candSubfieldData) {
-  if (catalogingSourceModifyingAgencyCandIsOriginalCatalogingSourceAgencyInTargetField(targetField, candSubfieldData) || ennakkotietoInSubfieldG(candSubfieldData)) {
+  if (catalogingSourceModifyingAgencyCandIsOriginalCatalogingSourceAgencyInTargetField(targetField, candSubfieldData) || dataContainsPrepublicationSubfield(candSubfieldData)) {
     return true;
   }
 
@@ -126,7 +127,7 @@ function addSubfield(targetField, candSubfield) {
 }
 
 function setPunctuationFlag(field, addedSubfield) {
-  if (isControlSubfieldCode(addedSubfield.code)) { // These are never punctuation related
+  if (!isContentSubfieldCode(addedSubfield.code, field.tag)) { // These are never punctuation related
     return;
   }
   field.useExternalEndPunctuation = 1;

package/src/merge-fields/mergeSubfield.js

@@ -1,6 +1,6 @@
 import createDebugLogger from 'debug';
 import {partsAgree, subfieldContainsPartData} from '../normalizeSubfieldValueForComparison.js';
-import {valueCarriesMeaning} from './worldKnowledge.js';
+import {getSynonyms, valueCarriesMeaning} from './worldKnowledge.js';
 import {nvdebug} from '../utils.js';
 import {tagAndSubfieldCodeReferToIsbn} from '../normalizeFieldForComparison.js';
 import {canContainOptionalQualifier, splitToNameAndQualifier} from './counterpartField.js';
@@ -88,7 +88,7 @@ function isPehmeakantinen(value) {
   return ['mjuka pärmar', 'paperback', 'pehmeäkantinen', 'softcover'].includes(value);
 }
 
-function isItsenainenJatkoOsa(value) {
+function isItsenainenJatkoOsa(value) { // Probably movable to synonyms...
   if (value.match(/^Fristående fortsättning på verket[^a-z]*$/ui)) {
     return true;
   }
@@ -142,6 +142,15 @@ function pairHttpAndHttps(candSubfield, relevantSubfields) {
 }
 
 function isSynonym(field, candSubfield, relevantSubfields) {
+
+  const finnishForm = getSynonyms(candSubfield.value, field.tag, candSubfield.code, 'fin');
+  if (finnishForm && finnishForm === relevantSubfields.some(sf => finnishForm === getSynonyms(sf.value, field.tag, candSubfield.code, 'fin'))) {
+    // NB! There's currently no intelligence (such as checking cat language from 040$b): the preferred value is the one in base.
+    // We might later on create a separate translation validator/fixer...
+    return true;
+  }
+
+
   if (candSubfield.code === 'q' && ['015', '020', '024', '028'].includes(field.tag)) {
     return coverTypesMatch(candSubfield, relevantSubfields);
   }

package/src/merge-fields/worldKnowledge.js

@@ -1,4 +1,7 @@
-//import {nvdebug} from '../utils';
+//import {nvdebug} from '../utils.js';
+
+// NB! This file (or at least synonyms) should eventually be moved away from merge to '..'.
+
 
 export function valueCarriesMeaning(tag, subfieldCode, value) {
   // Some data is pretty meaningless and as meaningless is pretty close to nothing, this meaningless data should no prevent merge.
@@ -20,11 +23,77 @@ export function valueCarriesMeaning(tag, subfieldCode, value) {
   return true;
 }
 
+const synonyms = [
+  {tags: ['700', '710', '711', '730'], code: 'i', 'fin': 'Sisältää (ekspressio)', 'swe': 'Innehåller (uttryck)'},
+  {tags: ['700', '710', '711', '730'], code: 'i', 'fin': 'Sisältää (teos)', 'swe': 'Innehåller (verk)'},
+  {tags: ['700', '710', '711', '730'], code: 'l', 'fin': 'Englanti', 'swe': 'Engelska'},
+  {tags: ['700', '710', '711', '730'], code: 'l', 'fin': 'Ruotsi', 'swe': 'Svenska'},
+  {tags: ['700', '710', '711', '730'], code: 'l', 'fin': 'Suomi', 'swe': 'Finska'}
+  // There might eventually be need for a list of terms is given language (eg. engl. paperback and softcover)
+];
+
+export function getSynonyms(term, tag = undefined, subfieldCode = undefined, preferredLanguage = undefined, ignoreCase = true, relevantLanguagesString = 'fin swe',) {
+  if (!term) {
+    return [];
+  }
+  //nvdebug(`WP1 CANDS: ${synonyms.length} FOR '${term}'`);
+  const relevantLanguges = relevantLanguagesString.split(/\s+/u);
+  const normalizedTerm = ignoreCase ? term.toLowerCase() : term;
+  const synonymsWithTag = tag ? synonyms.filter(s => s.tags.includes(tag)) : synonyms;
+  if (synonymsWithTag.length === 0) {
+    return [];
+  }
+  //nvdebug(`WP2 (FILTER ${tag}) CANDS: ${synonymsWithTag.length}`);
+  const synonymsWithTagAndCode = subfieldCode ? synonymsWithTag.filter(s => s.code === subfieldCode) : synonymsWithTag;
+  //nvdebug(`WP3 (FILTER $${subfieldCode}) CANDS: ${synonymsWithTagAndCode.length}:\n${JSON.stringify(synonymsWithTagAndCode)}`);
+  const matchingSynonyms = synonymsWithTagAndCode.filter(s => termAndLangMatch(s));
+
+  if (preferredLanguage && matchingSynonyms.length > 0) {
+    //console.log(`USING PREFERRED LANG '${preferredLanguage}' for TERM '${term}':\n${JSON.stringify(matchingSynonyms)}`);
+    return matchingSynonyms.map(s => s[preferredLanguage]);
+  }
+  return matchingSynonyms;
+
+  function termAndLangMatch(synonym) {
+    if (relevantLanguges.includes('fin')) {
+      if (ignoreCase && synonym.fin.toLowerCase() === normalizedTerm ) {
+        return true;
+      }
+      if (!ignoreCase && synonym.fin === term) {
+        return true;
+      }
+    }
+
+    if (relevantLanguges.includes('swe')) {
+      if (ignoreCase && synonym.swe.toLowerCase() === normalizedTerm ) {
+        return true;
+      }
+      if (!ignoreCase && synonym.swe === term) {
+        return true;
+      }
+    }
+    return false;
+  }
+}
+
+export function getSynonym(tag, subfieldCode, originalValue) {
+  const finnishForm = getSynonyms(originalValue, tag, subfieldCode, 'fin');
+  if (finnishForm.length === 1) {
+    //nvdebug(`FINNISH FORM FOR ${tag}$${subfieldCode}: '${finnishForm[0]}'`);
+    return finnishForm[0];
+  }
+  return originalValue;
+}
+
 export function normalizeForSamenessCheck(tag, subfieldCode, originalValue) {
   // NB! These work only for non-repeatable subfields!
   // Repeatable subfields are currently handled in mergeSubfields.js. Only non-repeatable subfields block field merge,
-  // (This split is suboptiomal... Minimum fix: make this disctinction cleaner...)
-  if (subfieldCode === 'a' && ['100', '600', '700', '800'].includes(tag)) {
+  // (This split is suboptiomal... Minimum fix: make this distinction cleaner...)
+
+  //nvdebug(`TRYING TO DO ${tag}$${subfieldCode} '${originalValue}'`);
+  originalValue = getSynonym(tag, subfieldCode, originalValue);
+
+  if (subfieldCode === 'a' && ['100', '600', '700', '800'].includes(tag)) { // "Etunimi Sukunimi"...
     return normalizePersonalName(originalValue);
   }
 

package/src/merge-fields.test.js

@@ -1,4 +1,4 @@
-import assert from 'node:assert';
+import assert from 'node:assert/strict';
 import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from './merge-fields/index.js';
 import {READERS} from '@natlibfi/fixura';
@@ -46,5 +46,5 @@ async function callback({getFixture, fix = false, tagPattern = false}) {
   }
 
   await validator.fix(record);
-  assert.deepEqual(record, expectedResult);
+  assert.deepEqual(record, new MarcRecord(expectedResult));
 }

package/src/normalize-dashes.js

@@ -1,6 +1,6 @@
 //import createDebugLogger from 'debug';
 import clone from 'clone';
-import {fieldToString, isControlSubfieldCode, nvdebug} from './utils.js';
+import {fieldToString, isContentSubfieldCode, nvdebug} from './utils.js';
 
 // Author(s): Nicholas Volk
 export default function () {
@@ -59,7 +59,7 @@ function fixDashes(field) {
   return field;
 
   function subfieldFixDashes(subfield) {
-    if (isControlSubfieldCode(subfield.code)) {
+    if (!isContentSubfieldCode(subfield.code, field.tag)) {
       return;
     }
     // Normalize dashes U+2010 ... U+2015 to '-':

package/src/normalizeFieldForComparison.js

@@ -8,26 +8,18 @@
 */
 import clone from 'clone';
 import {fieldStripPunctuation} from './punctuation2.js';
-import {fieldToString, isControlSubfieldCode} from './utils.js';
+import {fieldToString, isContentSubfieldCode} from './utils.js';
 
 import {fieldNormalizeControlNumbers/*, normalizeControlSubfieldValue*/} from './normalize-identifiers.js';
 import createDebugLogger from 'debug';
 import {normalizePartData, subfieldContainsPartData} from './normalizeSubfieldValueForComparison.js';
+import {isEnnakkotietoSubfield} from './prepublicationUtils.js';
+import {getSynonym} from './merge-fields/worldKnowledge.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:normalizeFieldForComparison');
 //const debugData = debug.extend('data');
 const debugDev = debug.extend('dev');
 
-export function isEnnakkotietoSubfieldG(subfield) {
-  if (valuelessSubfield(subfield)) {
-    return false;
-  }
-  if (subfield.code !== 'g') {
-    return false;
-  }
-  return subfield.value.match(/^ENNAKKOTIETO\.?$/gui);
-}
-
 function debugFieldComparison(oldField, newField) { // NB: Debug-only function!
   /*
   // We may drop certain subfields:
@@ -60,7 +52,7 @@ function containsHumanName(tag = '???', subfieldCode = undefined) {
 }
 
 function containsCorporateName(tag = '???', subfieldCode = undefined) {
-  // NB! This set is for bibs! Auth has 400... What else...
+  // NB! This set is for bibs! Auth has 410... What else...
   if (['110', '610', '710', '810'].includes(tag)) {
     if (subfieldCode === undefined || subfieldCode === 'a') {
       return true;
@@ -72,15 +64,15 @@ function containsCorporateName(tag = '???', subfieldCode = undefined) {
 
 function skipAllSubfieldNormalizations(value, subfieldCode, tag) {
 
-  if (isEnnakkotietoSubfieldG({'code': subfieldCode, value})) {
+  if (isEnnakkotietoSubfield({'code': subfieldCode, value})) {
     return true;
   }
 
-  if (tag === '035' && ['a', 'z'].includes(subfieldCode)) { // A
+  if (tag === '035' && ['a', 'z'].includes(subfieldCode)) {
     return true;
   }
 
-  if (isControlSubfieldCode(subfieldCode)) {
+  if (!isContentSubfieldCode(subfieldCode, tag)) {
     return true;
   }
   return false;
@@ -244,17 +236,20 @@ function removeDecomposedDiacritics(value = '') {
 
 function normalizeSubfieldValue(value, subfieldCode, tag) {
   // NB! For comparison of values only
-  /* eslint-disable */
+  /* beslint-disable */
+  value = removeCharsThatDontCarryMeaning(value, tag, subfieldCode);
+  value = getSynonym(tag, subfieldCode, value); // Must be done before punc stripping and lowercasing...
   value = subfieldValueLowercase(value, subfieldCode, tag);
 
+
   // Normalize: s. = sivut = pp.
   value = normalizePartData(value, subfieldCode, tag);
-  value = value.replace(/^\[([^[\]]+)\]/gu, '$1'); // eslint-disable-line functional/immutable-data
+  value = value.replace(/^\[([^[\]]+)\]/gu, '$1');
 
   if (['130', '730'].includes(tag) && subfieldCode === 'a') {
     value = value.replace(' : ', ', '); // "Halloween ends (elokuva, 2022)" vs "Halloween ends (elokuva : 2023)"
   }
-  /* eslint-enable */
+  /* beslint-enable */
 
   // Not going to do these in the foreseeable future, but keeping them here for discussion:
   // Possible normalizations include but are not limited to:
@@ -311,7 +306,8 @@ function normalizeField(field) {
 export function cloneAndNormalizeFieldForComparison(field) {
   // NB! This new field is for comparison purposes only.
   // Some of the normalizations might be considered a bit overkill for other purposes.
-  const clonedField = clone(field);
+  const clonedField = cloneAndRemovePunctuation(field); // was only clone(field)
+
   if (fieldSkipNormalization(field)) {
     return clonedField;
   }
@@ -319,8 +315,11 @@ export function cloneAndNormalizeFieldForComparison(field) {
     if (valuelessSubfield(sf)) {
       return;
     }
+
     sf.value = normalizeSubfieldValue(sf.value, sf.code, field.tag);
-    sf.value = removeCharsThatDontCarryMeaning(sf.value, field.tag, sf.code);
+    //sf.value = normalizeForSamenessCheck(field.tag, sf.code, sf.value);
+
+
   });
 
   normalizeField(clonedField);

package/src/prepublicationUtils.js

@@ -203,7 +203,7 @@ export function removeWorsePrepubField594s(record) {
 
 
 export function isEnnakkotietoSubfield(subfield) {
-  if (subfield.code !== '9' && subfield.code !== 'g') {
+  if (!['g', '9', '7'].includes(subfield.code)) {
     return false;
   }
   // Length <= 13 allows punctuation, but does not require it: