@natlibfi/marc-record-validators-melinda 12.0.6 → 12.0.7

package/dist/merge-fields/counterpartField.js

@@ -1,16 +1,15 @@
 import createDebugLogger from "debug";
-import { fieldHasSubfield, fieldHasNSubfields, fieldHasMultipleSubfields, fieldToString, nvdebug, removeCopyright } from "../utils.js";
+import { fieldHasSubfield, fieldHasNSubfields, fieldHasMultipleSubfields, fieldToString, nvdebug, removeCopyright, tagIsRepeatable } from "../utils.js";
 import { cloneAndNormalizeFieldForComparison, cloneAndRemovePunctuation } from "../normalizeFieldForComparison.js";
 import { normalizeControlSubfieldValue } from "../normalize-identifiers.js";
 import { getMergeConstraintsForTag } from "./mergeConstraints.js";
 import { controlSubfieldsPermitMerge } from "./controlSubfields.js";
 import { mergableIndicator1, mergableIndicator2 } from "./mergableIndicator.js";
 import { partsAgree } from "../normalizeSubfieldValueForComparison.js";
-import { normalizeForSamenessCheck, valueCarriesMeaning } from "./worldKnowledge.js";
+import { getSynonym, normalizeForSamenessCheck, valueCarriesMeaning } from "./worldKnowledge.js";
 import { provenanceSubfieldsPermitMerge } from "./dataProvenance.js";
 const debug = createDebugLogger("@natlibfi/marc-record-validators-melinda:mergeField:counterpart");
 const debugDev = debug.extend("dev");
-const irrelevantSubfieldsInNameAndTitlePartComparison = "5689";
 const counterpartRegexps = {
   // NB! tag is from source!
   // Note that in the normal case, all source 1XX fields have been converted to 7XX fields.
@@ -47,7 +46,7 @@ export function splitToNameAndQualifier(value) {
   }
   return [value, void 0];
 }
-export function splitToNameAndQualifierAndProcessName(name) {
+function splitToNameAndQualifierAndProcessName(name) {
   const [qualifierlessName, qualifier] = splitToNameAndQualifier(name);
   const [prefix, basename, suffix] = stripPrefixAndSuffix(qualifierlessName);
   return { name: getBestName(basename).toLowerCase(), prefix, suffix, qualifier };
@@ -120,78 +119,22 @@ function corporateNamesAgree(value1, value2, tag, subfieldCode) {
   }
   return true;
 }
-function pairableValue(tag, subfieldCode, value1, value2) {
-  if (withAndWithoutQualifierAgree(value1, value2, tag, subfieldCode)) {
-    return value1;
-  }
-  if (partsAgree(value1, value2, tag, subfieldCode) || corporateNamesAgree(value1, value2, tag, subfieldCode)) {
-    return value1;
-  }
-  return void 0;
-}
 function counterpartExtraNormalize(tag, subfieldCode, value) {
   value = value.replace(/(\S)(?:,|\.|\?|!|\. -| *:| *;| =| \/)$/u, "$1");
   value = value.replace(/^\(([^()]+)\)$/u, "$1");
   value = value.replace(/^\[([^[\]]+)\]$/u, "$1");
-  value = removeCopyright(value);
+  if (tag === "260" && subfieldCode === "c") {
+    value = removeCopyright(value);
+  }
   value = value.replace(/http:\/\//ug, "https://");
+  value = getSynonym(tag, subfieldCode, value);
   value = normalizeForSamenessCheck(tag, subfieldCode, value);
   return value;
 }
-function uniqueKeyMatches(baseField, sourceField, forcedKeyString = null) {
-  const keySubfieldsAsString = forcedKeyString || getMergeConstraintsForTag(baseField.tag, "key");
-  return optionalSubfieldComparison(baseField, sourceField, keySubfieldsAsString);
-}
-function optionalSubfieldComparison(originalBaseField, originalSourceField, keySubfieldsAsString) {
-  const field1 = cloneAndNormalizeFieldForComparison(originalBaseField);
-  const field2 = cloneAndNormalizeFieldForComparison(originalSourceField);
-  if (keySubfieldsAsString === null) {
-    return fieldToString(field1).substring(6) === fieldToString(field2).substring(6);
-  }
-  const subfieldArray = keySubfieldsAsString.split("");
-  if (subfieldArray.length > 0 && !subfieldArray.some((sfCode) => hasCommonNominator(sfCode))) {
-    return false;
-  }
-  return subfieldArray.every((subfieldCode) => testOptionalSubfield(originalBaseField.tag, subfieldCode));
-  function hasCommonNominator(subfieldCode) {
-    const subfields1 = field1.subfields.filter((subfield) => subfield.code === subfieldCode && valueCarriesMeaning(field1.tag, subfield.code, subfield.value));
-    const subfields2 = field2.subfields.filter((subfield) => subfield.code === subfieldCode && valueCarriesMeaning(field2.tag, subfield.code, subfield.value));
-    return subfields1.length > 0 && subfields2.length > 0;
-  }
-  function testOptionalSubfield(tag, subfieldCode) {
-    const subfields1 = field1.subfields.filter((subfield) => subfield.code === subfieldCode && valueCarriesMeaning(field1.tag, subfield.code, subfield.value));
-    const subfields2 = field2.subfields.filter((subfield) => subfield.code === subfieldCode && valueCarriesMeaning(field2.tag, subfield.code, subfield.value));
-    if (subfields1.length === 0 || subfields2.length === 0) {
-      return true;
-    }
-    const subfieldValues1 = subfields1.map((sf) => counterpartExtraNormalize(tag, subfieldCode, sf.value));
-    const subfieldValues2 = subfields2.map((sf) => counterpartExtraNormalize(tag, subfieldCode, sf.value));
-    if (subfieldValues1.every((val) => subfieldValues2.includes(val)) || subfieldValues2.every((val) => subfieldValues1.includes(val))) {
-      return true;
-    }
-    if (subfieldValues1.length === 1 && subfieldValues2.length === 1) {
-      return pairableValue(field1.tag, subfieldCode, subfieldValues1[0], subfieldValues2[0]) !== void 0;
-    }
-    return false;
-  }
-}
-function mandatorySubfieldComparison(originalField1, originalField2, keySubfieldsAsString) {
-  const field1 = cloneAndNormalizeFieldForComparison(originalField1);
-  const field2 = cloneAndNormalizeFieldForComparison(originalField2);
-  if (keySubfieldsAsString === null) {
-    return fieldToString(field1) === fieldToString(field2);
-  }
-  const subfieldArray = keySubfieldsAsString.split("");
-  return subfieldArray.every((subfieldCode) => mandatorySingleSubfieldComparison(subfieldCode));
-  function mandatorySingleSubfieldComparison(subfieldCode) {
-    const subfieldValues1 = field1.subfields.filter((subfield) => subfield.code === subfieldCode).map((sf) => sf.value);
-    const subfieldValues2 = field2.subfields.filter((subfield) => subfield.code === subfieldCode).map((sf) => sf.value);
-    if (subfieldValues1.length !== subfieldValues2.length) {
-      debugDev(`mSC: Unique key: subfield ${subfieldCode} issues...`);
-      return false;
-    }
-    return subfieldValues1.every((value) => subfieldValues2.includes(value));
-  }
+function hasCommonNominator(field1, field2, subfieldCode) {
+  const subfields1 = field1.subfields.filter((subfield) => subfield.code === subfieldCode && valueCarriesMeaning(field1.tag, subfield.code, subfield.value));
+  const subfields2 = field2.subfields.filter((subfield) => subfield.code === subfieldCode && valueCarriesMeaning(field2.tag, subfield.code, subfield.value));
+  return subfields1.length > 0 && subfields2.length > 0;
 }
 function tagToRegexp(tag, internalMerge = false) {
   if (internalMerge && tag in counterpartRegexpsSingle) {
@@ -204,7 +147,7 @@ function tagToRegexp(tag, internalMerge = false) {
   return new RegExp(`^${tag}$`, "u");
 }
 function areRequiredSubfieldsPresent(field) {
-  const subfieldString = getMergeConstraintsForTag(field.tag, "required");
+  const subfieldString = getMergeConstraintsForTag(field.tag, "required").join("");
   if (subfieldString === null) {
     return true;
   }
@@ -218,15 +161,22 @@ function areRequiredSubfieldsPresent(field) {
     return true;
   });
 }
-function arePairedSubfieldsInBalance(field1, field2) {
-  const subfieldString = getMergeConstraintsForTag(field1.tag, "paired");
-  if (subfieldString === null) {
-    return true;
+function getUnbalancedPairedSubfieldCode(field1, field2) {
+  const fullSubfieldString = getMergeConstraintsForTag(field1.tag, "paired").join("") || "";
+  if (fullSubfieldString === "") {
+    return false;
+  }
+  const pairable = pairableIdentifier(field1, field2, "(FIN11)");
+  const subfieldString = pairable ? removeNameRelatedSubfieldCodes(fullSubfieldString, field1.tag) : fullSubfieldString;
+  debug(`CHECK ${pairable ? "PAIRABLE " : ""}${field1.tag} PAIRS: '${fullSubfieldString}' => '${subfieldString}'`);
+  if (subfieldString === "") {
+    return false;
   }
   const subfieldArray = subfieldString.split("");
-  return subfieldArray.every((sfcode) => fieldHasNSubfields(field1, sfcode) === fieldHasNSubfields(field2, sfcode));
+  return subfieldArray.find((sfcode) => fieldHasNSubfields(field1, sfcode) !== fieldHasNSubfields(field2, sfcode));
 }
 function syntacticallyMergablePair(baseField, sourceField, config) {
+  nvdebug("CHECK SYNTAX");
   if (!mergableIndicator1(baseField, sourceField, config)) {
     nvdebug(`non-mergable (reason: indicator1): ${JSON.stringify(config)}`, debugDev);
     return false;
@@ -247,14 +197,16 @@ function syntacticallyMergablePair(baseField, sourceField, config) {
     nvdebug("non-mergable (reason: missing subfields)", debugDev);
     return false;
   }
-  if (!arePairedSubfieldsInBalance(baseField, sourceField)) {
-    nvdebug("required subfield pair check failed.", debugDev);
+  const subfieldCodeThatFailsToPair = getUnbalancedPairedSubfieldCode(baseField, sourceField);
+  if (subfieldCodeThatFailsToPair) {
+    nvdebug(`non-mergable (reason: required subfield pair check failed for code: '${subfieldCodeThatFailsToPair}')`, debugDev);
     return false;
   }
   return true;
 }
 function mergablePair(baseField, sourceField, config) {
   if (!syntacticallyMergablePair(baseField, sourceField, config)) {
+    nvdebug("non-mergable (reason: syntax)", debugDev);
     return false;
   }
   if (!semanticallyMergablePair(baseField, sourceField)) {
@@ -266,21 +218,41 @@ function mergablePair(baseField, sourceField, config) {
   S: ${fieldToString(sourceField)}`, debugDev);
   return true;
 }
-function pairableAsteriIDs(baseField, sourceField) {
-  const fin11a = getAsteriIDs(baseField);
-  if (fin11a.length === 0) {
-    return false;
+function removeNameRelatedSubfieldCodes(codestring, tag) {
+  const removables = getNameRelatedSubfieldCodes(tag);
+  return removeCharsFromString(codestring, removables);
+  function removeCharsFromString(string, removableCharsAsString) {
+    const removableChars = removableCharsAsString.split("");
+    return string.split("").filter((c) => !removableChars.includes(c)).join("");
+  }
+  function getNameRelatedSubfieldCodes(tag2) {
+    if (["100", "600", "700", "800"].includes(tag2)) {
+      return "abcdq";
+    }
+    if (["110", "610", "710", "810"].includes(tag2)) {
+      return "abcdn";
+    }
+    if (["111", "611", "711", "811"].includes(tag2)) {
+      return "acden";
+    }
+    return "";
   }
-  const fin11b = getAsteriIDs(sourceField);
-  if (fin11b.length === 0) {
+}
+function pairableIdentifier(field1, field2, prefix) {
+  const normalizedPrefix = prefix;
+  nvdebug(`PREF '${prefix}' => '${normalizedPrefix}'`);
+  const prefixLength = normalizedPrefix.length;
+  const identifiers1 = getIdentifiers(field1);
+  if (identifiers1.length !== 1) {
     return false;
   }
-  if (!controlSubfieldsPermitMerge(baseField, sourceField)) {
+  const identifiers2 = getIdentifiers(field2);
+  if (identifiers2.length !== 1) {
     return false;
   }
-  return true;
-  function getAsteriIDs(field) {
-    return field.subfields.filter((sf) => sf.code === "0").map((sf) => normalizeControlSubfieldValue(sf.value)).filter((val) => val.substring(0, 7) === "(FIN11)");
+  return identifiers1[0] === identifiers2[0];
+  function getIdentifiers(field) {
+    return field.subfields.filter((sf) => sf.code === "0").map((sf) => normalizeControlSubfieldValue(sf.value)).filter((val) => val.substring(0, prefixLength) === normalizedPrefix);
   }
 }
 function hasRepeatableSubfieldThatShouldBeTreatedAsNonRepeatable(field) {
@@ -295,84 +267,110 @@ function hasRepeatableSubfieldThatShouldBeTreatedAsNonRepeatable(field) {
   }
   return false;
 }
-function pairableName(baseField, sourceField) {
-  const reducedField1 = fieldToNamePart(baseField);
-  const reducedField2 = fieldToNamePart(sourceField);
-  const string1 = fieldToString(reducedField1);
-  const string2 = fieldToString(reducedField2);
-  if (string1 === string2) {
+function getRelevantSubfieldValues(field, subfieldCode) {
+  const values = field.subfields.filter((sf) => sf.code === subfieldCode).map((sf) => counterpartExtraNormalize(field.tag, subfieldCode, sf.value));
+  return values.filter((v) => valueCarriesMeaning(field.tag, subfieldCode, v));
+}
+function pairableValue(tag, subfieldCode, value1, value2) {
+  if (value1 === value2) {
+    return true;
+  }
+  if (withAndWithoutQualifierAgree(value1, value2, tag, subfieldCode)) {
+    return true;
+  }
+  if (partsAgree(value1, value2, tag, subfieldCode) || corporateNamesAgree(value1, value2, tag, subfieldCode)) {
     return true;
   }
-  if (hasRepeatableSubfieldThatShouldBeTreatedAsNonRepeatable(reducedField1) || hasRepeatableSubfieldThatShouldBeTreatedAsNonRepeatable(reducedField2)) {
+  return false;
+}
+function pairableValueInArray(tag, subfieldCode, val, arr) {
+  return arr.some((val2) => pairableValue(tag, subfieldCode, val, val2));
+}
+function tightSubfieldMatch(field1, field2, subfieldCode, mustHave = false) {
+  nvdebug(`${subfieldCode} F1: ${fieldToString(field1)}`);
+  nvdebug(`${subfieldCode} F2: ${fieldToString(field2)}`);
+  const values1 = getRelevantSubfieldValues(field1, subfieldCode);
+  const values2 = getRelevantSubfieldValues(field2, subfieldCode);
+  if (!mustHave) {
+    if (values1.length === 0 || values2.length === 0) {
+      return true;
+    }
+  }
+  if (values1.length !== values2.length) {
     return false;
   }
-  if (uniqueKeyMatches(reducedField1, reducedField2)) {
-    nvdebug(`    name match: '${fieldToString(reducedField1)}'`, debugDev);
+  nvdebug(`Compare $${subfieldCode} contents:
+  '${values1.join("'\n  '")}' vs
+  '${values2.join("'\n  '")}'`);
+  return values1.every((v) => pairableValueInArray(field1.tag, subfieldCode, v, values2)) && values2.every((v) => pairableValueInArray(field1.tag, subfieldCode, v, values1));
+}
+function looseSubfieldMatch(field1, field2, subfieldCode) {
+  const values1 = getRelevantSubfieldValues(field1, subfieldCode);
+  const values2 = getRelevantSubfieldValues(field2, subfieldCode);
+  if (values1.length === 0 || values2.length === 0) {
+    return true;
+  }
+  if (values1.every((v) => pairableValueInArray(field1.tag, subfieldCode, v, values2))) {
     return true;
   }
-  if (pairableAsteriIDs(baseField, sourceField)) {
+  if (values2.every((v) => pairableValueInArray(field1.tag, subfieldCode, v, values1))) {
     return true;
   }
-  nvdebug(`    name mismatch:`, debugDev);
-  nvdebug(`     '${fieldToString(reducedField1)}' vs`, debugDev);
-  nvdebug(`     '${fieldToString(reducedField2)}'`, debugDev);
   return false;
 }
 function semanticallyMergablePair(baseField, sourceField) {
-  if (!titlePartsMatch(baseField, sourceField)) {
-    nvdebug(` ${baseField.tag} is unmergable: Title part mismatch.`, debugDev);
+  const field1 = cloneAndNormalizeFieldForComparison(baseField);
+  const field2 = cloneAndNormalizeFieldForComparison(sourceField);
+  const string1 = fieldToString(field1);
+  const string2 = fieldToString(field2);
+  nvdebug(`IN ${baseField.tag}: pairableName():
+ '${string1}' vs
+ '${string2}'`, debugDev);
+  if (string1 === string2) {
+    return true;
+  }
+  const mergeConstraints = getMergeConstraintsForTag(field1.tag);
+  if (mergeConstraints.length === 0) {
     return false;
   }
-  if (!pairableName(baseField, sourceField)) {
-    nvdebug("Unmergable: Name part mismatch", debugDev);
+  if (hasRepeatableSubfieldThatShouldBeTreatedAsNonRepeatable(field1) || hasRepeatableSubfieldThatShouldBeTreatedAsNonRepeatable(field2)) {
+    nvdebug(`Unmergable: data is too complex to be automatically safely merged`, debugDev);
     return false;
   }
-  return true;
-}
-function namePartThreshold(field) {
-  if (!/[10]0$/u.test(field.tag)) {
-    return -1;
+  const asteriMatch = pairableIdentifier(field1, field2, "(FIN11)");
+  const allRequired = mergeConstraints[0].required || "";
+  const reallyRequired = asteriMatch ? removeNameRelatedSubfieldCodes(allRequired, field1.tag) : allRequired;
+  if (!reallyRequired.split("").every((c) => tightSubfieldMatch(field1, field2, c, true))) {
+    return false;
   }
-  const t = field.subfields.findIndex((currSubfield) => currSubfield.code === "t");
-  const u = t;
-  if (t === -1) {
-    return u;
+  const allPaired = mergeConstraints[0].paired || "";
+  const reallyPaired = asteriMatch ? removeNameRelatedSubfieldCodes(allPaired, field1.tag) : allPaired;
+  if (!reallyPaired.split("").every((c) => tightSubfieldMatch(field1, field2, c, false))) {
+    return false;
   }
-  if (u === -1) {
-    return t;
+  const allKeys = mergeConstraints[0].key || "";
+  const relevantKeys = asteriMatch ? removeNameRelatedSubfieldCodes(allKeys, field1.tag) : allKeys;
+  if (!relevantKeys.split("").every((c) => looseSubfieldMatch(field1, field2, c))) {
+    return false;
   }
-  return t > u ? u : t;
-}
-function fieldToNamePart(field) {
-  const index = namePartThreshold(field);
-  const relevantSubfields = field.subfields.filter((sf, i) => i < index || index === -1).filter((sf) => !irrelevantSubfieldsInNameAndTitlePartComparison.includes(sf.code));
-  const subsetField = { "tag": field.tag, "ind1": field.ind1, "ind2": field.ind2, subfields: relevantSubfields };
-  return subsetField;
-}
-function fieldToTitlePart(field) {
-  const index = field.subfields.findIndex((currSubfield) => currSubfield.code === "t");
-  const relevantSubfields = field.subfields.filter((sf, i) => i >= index).filter((sf) => !irrelevantSubfieldsInNameAndTitlePartComparison.includes(sf.code));
-  const subsetField = { "tag": field.tag, "ind1": field.ind1, "ind2": field.ind2, subfields: relevantSubfields };
-  debugDev(`Title subset: ${fieldToString(subsetField)}`);
-  return subsetField;
-}
-function containsTitlePart(field) {
-  return fieldCanHaveTitlePart(field) && fieldHasSubfield(field, "t");
-  function fieldCanHaveTitlePart(field2) {
-    return ["100", "110", "111", "700", "710", "711"].includes(field2.tag);
+  if (allRequired.length > 0) {
+    return true;
   }
-}
-function titlePartsMatch(field1, field2) {
-  if (!containsTitlePart(field1)) {
-    return !containsTitlePart(field2);
+  if (reallyPaired.length > 0 && field1.subfields.some((sf) => reallyPaired.includes(sf.code))) {
+    return true;
   }
-  if (!containsTitlePart(field2)) {
-    return false;
+  if (!tagIsRepeatable(field1.tag) || relevantKeys.length == 0) {
+    return true;
+  }
+  if (relevantKeys.length > 0) {
+    if (field1.subfields.some((sf) => relevantKeys.includes(sf.code)) || field2.subfields.some((sf) => relevantKeys.includes(sf.code))) {
+      return relevantKeys.split("").some((code) => hasCommonNominator(field1, field2, code));
+    }
   }
-  debugDev(`TITLE PARTS NEED TO BE COMPARED`);
-  const subset1 = fieldToTitlePart(field1);
-  const subset2 = fieldToTitlePart(field2);
-  return mandatorySubfieldComparison(subset1, subset2, "dfhklmnoprstxvg");
+  nvdebug(`    name mismatch (${keys}):`, debugDev);
+  nvdebug(`     '${fieldToString(baseField)}' vs`, debugDev);
+  nvdebug(`     '${fieldToString(sourceField)}'`, debugDev);
+  return false;
 }
 function getAlternativeNamesFrom9XX(record, field) {
   if (!field.tag.match(/^(?:100|110|111|600|610|611|700|710|711)$/u)) {
@@ -418,7 +416,6 @@ function getCounterpartIndex(field, counterpartCands, altNames, config) {
   return normalizedCounterpartCands.findIndex((normCandField) => altNames.some((altName) => mergablePairWithAltName(normCandField, normalizedField, altName, config)));
 }
 function field264Exception(baseField, sourceRecord, sourceField, config) {
-  nvdebug("Field 264 exception as per MET-456", debugDev);
   if (baseField.tag !== "264") {
     return false;
   }
@@ -459,15 +456,8 @@ export function getCounterpart(baseRecord, sourceRecord, field, config) {
   }
   nvdebug(`Compare incoming '${fieldToString(field)}' with (up to) ${counterpartCands.length} existing field(s)`, debugDev);
   const normalizedField = cloneAndNormalizeFieldForComparison(field);
-  nvdebug(`Norm to: '${fieldToString(normalizedField)}'`, debugDev);
+  nvdebug(` Normalize incoming field to: '${fieldToString(normalizedField)}'`, debugDev);
   const uniqueAlternativeNames = getUniqueAlernativeNames();
-  function getUniqueAlernativeNames() {
-    if (baseIsSource(baseRecord, sourceRecord)) {
-      return [];
-    }
-    const alternativeNames = getAlternativeNamesFrom9XX(baseRecord, field).concat(getAlternativeNamesFrom9XX(sourceRecord, field));
-    return alternativeNames.filter((name, i) => alternativeNames.indexOf(name) === i);
-  }
   const index = getCounterpartIndex(normalizedField, counterpartCands, uniqueAlternativeNames, config);
   if (index > -1) {
     return counterpartCands[index];
@@ -476,5 +466,12 @@ export function getCounterpart(baseRecord, sourceRecord, field, config) {
     return counterpartCands[0];
   }
   return null;
+  function getUniqueAlernativeNames() {
+    if (baseIsSource(baseRecord, sourceRecord)) {
+      return [];
+    }
+    const alternativeNames = getAlternativeNamesFrom9XX(baseRecord, field).concat(getAlternativeNamesFrom9XX(sourceRecord, field));
+    return alternativeNames.filter((name, i) => alternativeNames.indexOf(name) === i);
+  }
 }
 //# sourceMappingURL=counterpartField.js.map