@natlibfi/marc-record-validators-melinda 10.2.0 → 10.2.1

package/dist/removeDuplicateDataFields.js

@@ -136,41 +136,40 @@ function removeIndividualDuplicateDatafields(record, fix = true) {
   // No $6 nor $8 in field
   /* eslint-disable */
   let seen = {};
-  let removables = []; // for validation
-
-  //record.fields.forEach(field => nvdebug(`DUPL-1 CHECK SINGLE ${fieldToString(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));
+  record.fields.forEach(field => (0, _utils.nvdebug)(`DUPL-1 CHECK SINGLE ${(0, _utils.fieldToString)(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));
+  const removableFields = record.fields.filter(field => removableIndividualDuplicateDatafield(field));
+  const removableFieldsAsStrings = removableFields.map(field => (0, _utils.fieldToString)(field));
+  if (fix) {
+    removableFields.forEach(field => record.removeField(field));
+  }
+  return removableFieldsAsStrings;
+  function removableIndividualDuplicateDatafield(field) {
+    if (!field.subfields) {
+      // Not a datafield
+      //nvdebug(`SKIP subfieldless ${fieldToString(field)}`);
+      return false;
+    }
+    // There's actually no reason to check whether individual fields contain a $6 or an $8...
+    // If everything incl. occurence/xxxx numbers match it's still deletable, regardless of chains.
 
-  const fields = record.fields;
-  fields.forEach(field => removeIndividualDuplicateDatafield(field));
-  function removeIndividualDuplicateDatafield(field) {
-    //nvdebug(`removeIndividualDuplicateDatafield? ${fieldToString(field)} (and friends)`);
+    //nvdebug(`removeIndividualDuplicateDatafield? ${fieldToString(field)}`);
 
     // We are in trouble if $9 ^ and $9 ^^ style chains appear here...
     const fieldAsString = (0, _utils.fieldToString)(field); // Never normalize!
 
     //nvdebug(` step 2 ${fieldAsString}`);
     if (fieldAsString in seen) {
-      // nvdebug(` step 3 ${fieldAsString}`);
-      // There's actually no reason to check whether individual fields contain a $6 or an $8...
-
-      if (!removables.includes(fieldAsString)) {
-        removables.push(fieldAsString);
-      }
-      if (fix) {
-        //nvdebug(`DOUBLE REMOVAL: REMOVE ${fieldAsString}`, debug);
-        fields.forEach(currField => record.removeField(currField));
-        return;
-      }
-      (0, _utils.nvdebug)(`VALIDATION-1: DUPLICATE DETECTED ${fieldAsString}`, debug);
-      return;
+      (0, _utils.nvdebug)(`DUPLICATE SINGLETON DETECTED: ${fieldAsString}`);
+      return true;
     }
-    (0, _utils.nvdebug)(`ADD2SEEN-1 ${fieldAsString}`, debug);
+    (0, _utils.nvdebug)(`MARK SINGLETON AS SEEN: ${fieldAsString}`, debug);
     seen[fieldAsString] = 1;
     return;
   }
+
   /* eslint-enable */
-  return removables;
 }
+
 function recordRemoveFieldOrSubfield8(record, field, currLinkingNumber) {
   const eights = field.subfields.filter(sf => sf.code === '8');
   if (eights.length < 2) {

package/dist/removeDuplicateDataFields.js.map

@@ -1 +1 @@
-{"version":3,"file":"removeDuplicateDataFields.js","names":["debug","createDebugLogger","description","validate","fix","record","nvdebug","res","message","valid","removeDuplicateDatafields","duplicates","length","add6s","field","sixes","subfields","filter","sf","isValidSubfield6","occurrenceNumbers","map","subfield6GetOccurrenceNumber","value","undefined","join","relevantFields","fields","f","some","o","fieldHasOccurrenceNumber","forEach","fieldToString","add8s","fieldHasSubfield","numberOfLinkageSubfields","code","getAllLinkedSubfield6Fields","moreFields","getFirstField","fieldsAsStrings","i","findIndex","includes","isFirstLinkedSubfield6Field","chain","firstField","removeIndividualDuplicateDatafields","seen","removables","removeIndividualDuplicateDatafield","fieldAsString","push","currField","removeField","recordRemoveFieldOrSubfield8","currLinkingNumber","eights","getSubfield8LinkingNumber","removeSubfield","removeDuplicateSubfield8Chains","seenLinkingNumbers","recordGetAllSubfield8LinkingNumbers","linkedFields","recordGetFieldsWithSubfield8LinkingNumber","linkedFieldsAsString","fieldsToNormalizedString","fieldsToString","removeDuplicateSubfield6Chains","removeDuplicateDatafield","fieldsAsString","removables8","removables6","removablesAll","concat"],"sources":["../src/removeDuplicateDataFields.js"],"sourcesContent":["import createDebugLogger from 'debug';\nimport {fieldHasSubfield, fieldsToString, fieldToString, nvdebug} from './utils';\nimport {fieldHasOccurrenceNumber, fieldsToNormalizedString, isValidSubfield6, subfield6GetOccurrenceNumber} from './subfield6Utils';\nimport {getSubfield8LinkingNumber, recordGetAllSubfield8LinkingNumbers, recordGetFieldsWithSubfield8LinkingNumber} from './subfield8Utils';\n\n// Relocated from melinda-marc-record-merge-reducers (and renamed)\n\nconst debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:removeDuplicateDataFields');\n\nexport default function () {\n  return {\n    description: 'Remove duplicate data fields. Certain exceptions apply, mainly too complited chained fields',\n    validate, fix\n  };\n\n  function fix(record) {\n    nvdebug('Remove duplicate data fields');\n    const res = {message: [], fix: [], valid: true};\n    removeDuplicateDatafields(record, true);\n    // This can not really fail...\n    return res;\n  }\n\n  function validate(record) {\n    // Check max, and check number of different indexes\n    nvdebug('Validate record: duplicate data fields cause (t)error', debug);\n\n    const duplicates = removeDuplicateDatafields(record, false);\n\n    //const orphanedFields = getOrphanedFields(fieldsContainingSubfield6);\n\n    const res = {message: duplicates};\n\n    /*\n    if (orphanedFields.length > 0) { // eslint-disable-line functional/no-conditional-statement\n      res.message = [`${orphanedFields.length} orphaned occurrence number field(s) detected`]; // eslint-disable-line functional/immutable-data\n    }\n    */\n    res.valid = res.message.length < 1; // eslint-disable-line functional/immutable-data\n    return res;\n  }\n}\n\nfunction add6s(field, record) {\n\n  /*\n  // Can't rely on nice pairs...\n  if (fieldHasSubfield(field, '6')) {\n\n    const pairs = fieldGetOccurrenceNumberPairs(field, record.fields);\n    if (pairs) {\n      return [field].concat(pairs);\n    }\n\n  }\n  */\n\n  // Get all fields with given occurence number\n  const sixes = field.subfields.filter(sf => isValidSubfield6(sf));\n\n  if (sixes.length === 0) {\n    return [field];\n  }\n  nvdebug(`SIXES: ${sixes.length}`);\n  const occurrenceNumbers = sixes.map(sf => subfield6GetOccurrenceNumber(sf)).filter(value => value !== undefined && value !== '00');\n  nvdebug(occurrenceNumbers.join(' -- '));\n\n  const relevantFields = record.fields.filter(f => occurrenceNumbers.some(o => fieldHasOccurrenceNumber(f, o)));\n  nvdebug(`RELFIELDS FOUND: ${relevantFields.length}...`);\n  relevantFields.forEach(f => nvdebug(fieldToString(f)));\n  return relevantFields;\n}\n\nfunction add8s(fields, record) {\n  // Not implemented yet:\n  if (fields && fields.some(f => fieldHasSubfield(f, '8'))) {\n    return [];\n  }\n  return record ? fields : fields;\n}\n\nfunction numberOfLinkageSubfields(field) {\n  const subfields = field.subfields.filter(sf => sf.code === '6' || sf.code === '8');\n  return subfields.length;\n}\n\n\nfunction getAllLinkedSubfield6Fields(field, record) {\n  const fields = add6s(field, record);\n  const moreFields = add8s(fields, record);\n\n  // Currently we don't handle fields with more than one $6 and/or $8 subfield.\n  if (moreFields.length === 0 || moreFields.some(f => numberOfLinkageSubfields(f) > 1)) {\n    return []; // Don't fix!\n  }\n  return moreFields;\n}\n\nfunction getFirstField(record, fields) {\n  const fieldsAsStrings = fields.map(field => fieldToString(field));\n  record.fields.forEach((field, i) => nvdebug(`${i}:\\t${fieldToString(field)}`));\n  nvdebug(`INCOMING: ${fieldsAsStrings.join('\\t')}`);\n  const i = record.fields.findIndex(field => fieldsAsStrings.includes(fieldToString(field)));\n  if (i > -1) {\n    const field = record.fields[i];\n    nvdebug(`1st F: ${i + 1}/${record.fields.length} ${fieldToString(field)}`);\n    return field;\n  }\n  return undefined;\n}\n\n\nfunction isFirstLinkedSubfield6Field(field, record) {\n  if (!field.subfields) { // Is not a datafield\n    return false;\n  }\n  const chain = getAllLinkedSubfield6Fields(field, record);\n  if (chain.length < 2) {\n    return false;\n  }\n\n  // Interpretation of first: position of field in record (however, we might have a duplicate field. See tests...)\n  const firstField = getFirstField(record, chain);\n  if (firstField) {\n    return fieldToString(field) === fieldToString(firstField);\n  }\n  return false;\n\n  // Fallback:\n  //return fieldToString(field) === fieldToString(chain[0]);\n}\n\nexport function removeIndividualDuplicateDatafields(record, fix = true) { // No $6 nor $8 in field\n  /* eslint-disable */\n  let seen = {};\n\n  let removables = []; // for validation\n\n  //record.fields.forEach(field => nvdebug(`DUPL-1 CHECK SINGLE ${fieldToString(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));\n  \n  const fields = record.fields;\n  \n  fields.forEach(field => removeIndividualDuplicateDatafield(field));\n\n  function removeIndividualDuplicateDatafield(field) {\n    //nvdebug(`removeIndividualDuplicateDatafield? ${fieldToString(field)} (and friends)`);\n\n    // We are in trouble if $9 ^ and $9 ^^ style chains appear here...\n    const fieldAsString = fieldToString(field); // Never normalize!\n\n    //nvdebug(` step 2 ${fieldAsString}`);\n    if (fieldAsString in seen)  {\n      // nvdebug(` step 3 ${fieldAsString}`);\n      // There's actually no reason to check whether individual fields contain a $6 or an $8...\n\n      if (!removables.includes(fieldAsString)) {\n        removables.push(fieldAsString);\n      }\n\n      if (fix) {\n        //nvdebug(`DOUBLE REMOVAL: REMOVE ${fieldAsString}`, debug);\n        fields.forEach(currField => record.removeField(currField));\n        return;\n      }\n      nvdebug(`VALIDATION-1: DUPLICATE DETECTED ${fieldAsString}`, debug);\n      return;\n    }\n    nvdebug(`ADD2SEEN-1 ${fieldAsString}`, debug);\n    seen[fieldAsString] = 1;\n    return;\n  }\n  /* eslint-enable */\n  return removables;\n}\n\n\nfunction recordRemoveFieldOrSubfield8(record, field, currLinkingNumber) {\n  const eights = field.subfields.filter(sf => sf.code === '8');\n  if (eights.length < 2) {\n    record.removeField(field);\n    return;\n  }\n  const subfields = field.subfields.filter(sf => getSubfield8LinkingNumber(sf) === currLinkingNumber);\n  subfields.forEach(sf => record.removeSubfield(sf, field));\n}\n\n\nexport function removeDuplicateSubfield8Chains(record, fix = true) {\n  /* eslint-disable */\n  let seen = {};\n\n  let removables = []; // for validation\n\n  nvdebug(\"CHAIN-8\");\n  const seenLinkingNumbers = recordGetAllSubfield8LinkingNumbers(record);\n  if (seenLinkingNumbers.length === 0) {\n    return removables;\n  }\n\n  nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);\n\n  seenLinkingNumbers.forEach(currLinkingNumber => {\n    const linkedFields = recordGetFieldsWithSubfield8LinkingNumber(record, currLinkingNumber) //getFieldsWithSubfield8Index(base, baseIndex);\n    const linkedFieldsAsString = fieldsToNormalizedString(linkedFields, currLinkingNumber);\n    nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);\n    nvdebug(`${linkedFieldsAsString}`, debug);\n\n    if (linkedFieldsAsString in seen)  {\n      if (!removables.includes(linkedFieldsAsString)) {\n        removables.push(linkedFieldsAsString);\n      }\n\n      if (fix) {\n        nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);\n        linkedFields.forEach(field => recordRemoveFieldOrSubfield8(record, field, currLinkingNumber));\n        return;\n      }\n\n      nvdebug(`$8 VALIDATION: DUPLICATE DETECTED ${linkedFieldsAsString}`, debug);\n      return;\n    }\n    nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);\n    seen[linkedFieldsAsString] = 1;\n    return;\n  });\n\n  /* eslint-enable */\n  return removables;\n}\n\nexport function removeDuplicateSubfield6Chains(record, fix = true) {\n  /* eslint-disable */\n  let seen = {};\n\n  let removables = []; // for validation\n\n  record.fields.forEach(field => nvdebug(`DUPL-CHECK $CHAIN ${fieldToString(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));\n  \n  const fields = record.fields.filter(field => isFirstLinkedSubfield6Field(field, record)); // Well a\n  \n  fields.forEach(field => removeDuplicateDatafield(field));\n\n  function removeDuplicateDatafield(field) {\n    nvdebug(`removeDuplicateDatafield? $6 ${fieldToString(field)} (and friends)`);\n    const fields = getAllLinkedSubfield6Fields(field, record);\n    if(fields.length === 0) {\n      return;\n    }\n\n    const fieldsAsString = fieldsToNormalizedString(fields);\n    nvdebug(` step 2 ${fieldsAsString}`);\n    if (fieldsAsString in seen)  {\n      nvdebug(` step 3 ${fieldsAsString}`);\n\n      removables.push(fieldsAsString);\n\n      if (fix) {\n        nvdebug(`DOUBLE REMOVAL: REMOVE ${fieldsAsString}`, debug);\n        fields.forEach(currField => record.removeField(currField));\n        return;\n      }\n      nvdebug(`VALIDATION: DUPLICATE DETECTED ${fieldsAsString}`, debug);\n      \n    }\n    nvdebug(`DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${fieldsAsString}`, debug);\n    seen[fieldsAsString] = 1;\n    return;\n  }\n\n\n  /* eslint-enable */\n  return removables;\n}\n\nexport function removeDuplicateDatafields(record, fix = true) {\n  const removables = removeIndividualDuplicateDatafields(record, fix); // Lone fields\n  const removables8 = removeDuplicateSubfield8Chains(record, fix); // Lone subfield $8 chains\n  const removables6 = removeDuplicateSubfield6Chains(record, fix); // Lone subfield $6 chains\n  // HOW TO HANDLE $6+$8 combos?\n\n  const removablesAll = removables.concat(removables8).concat(removables6);\n\n  return removablesAll;\n}\n"],"mappings":";;;;;;;;;;AAAA;AACA;AACA;AACA;AAA2I;AAE3I;;AAEA,MAAMA,KAAK,GAAG,IAAAC,cAAiB,EAAC,oEAAoE,CAAC;AAEtF,oBAAY;EACzB,OAAO;IACLC,WAAW,EAAE,6FAA6F;IAC1GC,QAAQ;IAAEC;EACZ,CAAC;EAED,SAASA,GAAG,CAACC,MAAM,EAAE;IACnB,IAAAC,cAAO,EAAC,8BAA8B,CAAC;IACvC,MAAMC,GAAG,GAAG;MAACC,OAAO,EAAE,EAAE;MAAEJ,GAAG,EAAE,EAAE;MAAEK,KAAK,EAAE;IAAI,CAAC;IAC/CC,yBAAyB,CAACL,MAAM,EAAE,IAAI,CAAC;IACvC;IACA,OAAOE,GAAG;EACZ;EAEA,SAASJ,QAAQ,CAACE,MAAM,EAAE;IACxB;IACA,IAAAC,cAAO,EAAC,uDAAuD,EAAEN,KAAK,CAAC;IAEvE,MAAMW,UAAU,GAAGD,yBAAyB,CAACL,MAAM,EAAE,KAAK,CAAC;;IAE3D;;IAEA,MAAME,GAAG,GAAG;MAACC,OAAO,EAAEG;IAAU,CAAC;;IAEjC;AACJ;AACA;AACA;AACA;IACIJ,GAAG,CAACE,KAAK,GAAGF,GAAG,CAACC,OAAO,CAACI,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,OAAOL,GAAG;EACZ;AACF;AAEA,SAASM,KAAK,CAACC,KAAK,EAAET,MAAM,EAAE;EAE5B;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;EAIE;EACA,MAAMU,KAAK,GAAGD,KAAK,CAACE,SAAS,CAACC,MAAM,CAACC,EAAE,IAAI,IAAAC,gCAAgB,EAACD,EAAE,CAAC,CAAC;EAEhE,IAAIH,KAAK,CAACH,MAAM,KAAK,CAAC,EAAE;IACtB,OAAO,CAACE,KAAK,CAAC;EAChB;EACA,IAAAR,cAAO,EAAE,UAASS,KAAK,CAACH,MAAO,EAAC,CAAC;EACjC,MAAMQ,iBAAiB,GAAGL,KAAK,CAACM,GAAG,CAACH,EAAE,IAAI,IAAAI,4CAA4B,EAACJ,EAAE,CAAC,CAAC,CAACD,MAAM,CAACM,KAAK,IAAIA,KAAK,KAAKC,SAAS,IAAID,KAAK,KAAK,IAAI,CAAC;EAClI,IAAAjB,cAAO,EAACc,iBAAiB,CAACK,IAAI,CAAC,MAAM,CAAC,CAAC;EAEvC,MAAMC,cAAc,GAAGrB,MAAM,CAACsB,MAAM,CAACV,MAAM,CAACW,CAAC,IAAIR,iBAAiB,CAACS,IAAI,CAACC,CAAC,IAAI,IAAAC,wCAAwB,EAACH,CAAC,EAAEE,CAAC,CAAC,CAAC,CAAC;EAC7G,IAAAxB,cAAO,EAAE,oBAAmBoB,cAAc,CAACd,MAAO,KAAI,CAAC;EACvDc,cAAc,CAACM,OAAO,CAACJ,CAAC,IAAI,IAAAtB,cAAO,EAAC,IAAA2B,oBAAa,EAACL,CAAC,CAAC,CAAC,CAAC;EACtD,OAAOF,cAAc;AACvB;AAEA,SAASQ,KAAK,CAACP,MAAM,EAAEtB,MAAM,EAAE;EAC7B;EACA,IAAIsB,MAAM,IAAIA,MAAM,CAACE,IAAI,CAACD,CAAC,IAAI,IAAAO,uBAAgB,EAACP,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE;IACxD,OAAO,EAAE;EACX;EACA,OAAOvB,MAAM,GAAGsB,MAAM,GAAGA,MAAM;AACjC;AAEA,SAASS,wBAAwB,CAACtB,KAAK,EAAE;EACvC,MAAME,SAAS,GAAGF,KAAK,CAACE,SAAS,CAACC,MAAM,CAACC,EAAE,IAAIA,EAAE,CAACmB,IAAI,KAAK,GAAG,IAAInB,EAAE,CAACmB,IAAI,KAAK,GAAG,CAAC;EAClF,OAAOrB,SAAS,CAACJ,MAAM;AACzB;AAGA,SAAS0B,2BAA2B,CAACxB,KAAK,EAAET,MAAM,EAAE;EAClD,MAAMsB,MAAM,GAAGd,KAAK,CAACC,KAAK,EAAET,MAAM,CAAC;EACnC,MAAMkC,UAAU,GAAGL,KAAK,CAACP,MAAM,EAAEtB,MAAM,CAAC;;EAExC;EACA,IAAIkC,UAAU,CAAC3B,MAAM,KAAK,CAAC,IAAI2B,UAAU,CAACV,IAAI,CAACD,CAAC,IAAIQ,wBAAwB,CAACR,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE;IACpF,OAAO,EAAE,CAAC,CAAC;EACb;;EACA,OAAOW,UAAU;AACnB;AAEA,SAASC,aAAa,CAACnC,MAAM,EAAEsB,MAAM,EAAE;EACrC,MAAMc,eAAe,GAAGd,MAAM,CAACN,GAAG,CAACP,KAAK,IAAI,IAAAmB,oBAAa,EAACnB,KAAK,CAAC,CAAC;EACjET,MAAM,CAACsB,MAAM,CAACK,OAAO,CAAC,CAAClB,KAAK,EAAE4B,CAAC,KAAK,IAAApC,cAAO,EAAE,GAAEoC,CAAE,MAAK,IAAAT,oBAAa,EAACnB,KAAK,CAAE,EAAC,CAAC,CAAC;EAC9E,IAAAR,cAAO,EAAE,aAAYmC,eAAe,CAAChB,IAAI,CAAC,IAAI,CAAE,EAAC,CAAC;EAClD,MAAMiB,CAAC,GAAGrC,MAAM,CAACsB,MAAM,CAACgB,SAAS,CAAC7B,KAAK,IAAI2B,eAAe,CAACG,QAAQ,CAAC,IAAAX,oBAAa,EAACnB,KAAK,CAAC,CAAC,CAAC;EAC1F,IAAI4B,CAAC,GAAG,CAAC,CAAC,EAAE;IACV,MAAM5B,KAAK,GAAGT,MAAM,CAACsB,MAAM,CAACe,CAAC,CAAC;IAC9B,IAAApC,cAAO,EAAE,UAASoC,CAAC,GAAG,CAAE,IAAGrC,MAAM,CAACsB,MAAM,CAACf,MAAO,IAAG,IAAAqB,oBAAa,EAACnB,KAAK,CAAE,EAAC,CAAC;IAC1E,OAAOA,KAAK;EACd;EACA,OAAOU,SAAS;AAClB;AAGA,SAASqB,2BAA2B,CAAC/B,KAAK,EAAET,MAAM,EAAE;EAClD,IAAI,CAACS,KAAK,CAACE,SAAS,EAAE;IAAE;IACtB,OAAO,KAAK;EACd;EACA,MAAM8B,KAAK,GAAGR,2BAA2B,CAACxB,KAAK,EAAET,MAAM,CAAC;EACxD,IAAIyC,KAAK,CAAClC,MAAM,GAAG,CAAC,EAAE;IACpB,OAAO,KAAK;EACd;;EAEA;EACA,MAAMmC,UAAU,GAAGP,aAAa,CAACnC,MAAM,EAAEyC,KAAK,CAAC;EAC/C,IAAIC,UAAU,EAAE;IACd,OAAO,IAAAd,oBAAa,EAACnB,KAAK,CAAC,KAAK,IAAAmB,oBAAa,EAACc,UAAU,CAAC;EAC3D;EACA,OAAO,KAAK;;EAEZ;EACA;AACF;;AAEO,SAASC,mCAAmC,CAAC3C,MAAM,EAAED,GAAG,GAAG,IAAI,EAAE;EAAE;EACxE;EACA,IAAI6C,IAAI,GAAG,CAAC,CAAC;EAEb,IAAIC,UAAU,GAAG,EAAE,CAAC,CAAC;;EAErB;;EAEA,MAAMvB,MAAM,GAAGtB,MAAM,CAACsB,MAAM;EAE5BA,MAAM,CAACK,OAAO,CAAClB,KAAK,IAAIqC,kCAAkC,CAACrC,KAAK,CAAC,CAAC;EAElE,SAASqC,kCAAkC,CAACrC,KAAK,EAAE;IACjD;;IAEA;IACA,MAAMsC,aAAa,GAAG,IAAAnB,oBAAa,EAACnB,KAAK,CAAC,CAAC,CAAC;;IAE5C;IACA,IAAIsC,aAAa,IAAIH,IAAI,EAAG;MAC1B;MACA;;MAEA,IAAI,CAACC,UAAU,CAACN,QAAQ,CAACQ,aAAa,CAAC,EAAE;QACvCF,UAAU,CAACG,IAAI,CAACD,aAAa,CAAC;MAChC;MAEA,IAAIhD,GAAG,EAAE;QACP;QACAuB,MAAM,CAACK,OAAO,CAACsB,SAAS,IAAIjD,MAAM,CAACkD,WAAW,CAACD,SAAS,CAAC,CAAC;QAC1D;MACF;MACA,IAAAhD,cAAO,EAAE,oCAAmC8C,aAAc,EAAC,EAAEpD,KAAK,CAAC;MACnE;IACF;IACA,IAAAM,cAAO,EAAE,cAAa8C,aAAc,EAAC,EAAEpD,KAAK,CAAC;IAC7CiD,IAAI,CAACG,aAAa,CAAC,GAAG,CAAC;IACvB;EACF;EACA;EACA,OAAOF,UAAU;AACnB;AAGA,SAASM,4BAA4B,CAACnD,MAAM,EAAES,KAAK,EAAE2C,iBAAiB,EAAE;EACtE,MAAMC,MAAM,GAAG5C,KAAK,CAACE,SAAS,CAACC,MAAM,CAACC,EAAE,IAAIA,EAAE,CAACmB,IAAI,KAAK,GAAG,CAAC;EAC5D,IAAIqB,MAAM,CAAC9C,MAAM,GAAG,CAAC,EAAE;IACrBP,MAAM,CAACkD,WAAW,CAACzC,KAAK,CAAC;IACzB;EACF;EACA,MAAME,SAAS,GAAGF,KAAK,CAACE,SAAS,CAACC,MAAM,CAACC,EAAE,IAAI,IAAAyC,yCAAyB,EAACzC,EAAE,CAAC,KAAKuC,iBAAiB,CAAC;EACnGzC,SAAS,CAACgB,OAAO,CAACd,EAAE,IAAIb,MAAM,CAACuD,cAAc,CAAC1C,EAAE,EAAEJ,KAAK,CAAC,CAAC;AAC3D;AAGO,SAAS+C,8BAA8B,CAACxD,MAAM,EAAED,GAAG,GAAG,IAAI,EAAE;EACjE;EACA,IAAI6C,IAAI,GAAG,CAAC,CAAC;EAEb,IAAIC,UAAU,GAAG,EAAE,CAAC,CAAC;;EAErB,IAAA5C,cAAO,EAAC,SAAS,CAAC;EAClB,MAAMwD,kBAAkB,GAAG,IAAAC,mDAAmC,EAAC1D,MAAM,CAAC;EACtE,IAAIyD,kBAAkB,CAAClD,MAAM,KAAK,CAAC,EAAE;IACnC,OAAOsC,UAAU;EACnB;EAEA,IAAA5C,cAAO,EAAE,8BAA6BwD,kBAAkB,CAACrC,IAAI,CAAC,IAAI,CAAE,EAAC,EAAEzB,KAAK,CAAC;EAE7E8D,kBAAkB,CAAC9B,OAAO,CAACyB,iBAAiB,IAAI;IAC9C,MAAMO,YAAY,GAAG,IAAAC,yDAAyC,EAAC5D,MAAM,EAAEoD,iBAAiB,CAAC,EAAC;IAC1F,MAAMS,oBAAoB,GAAG,IAAAC,wCAAwB,EAACH,YAAY,EAAEP,iBAAiB,CAAC;IACtF,IAAAnD,cAAO,EAAE,8BAA6BmD,iBAAkB,GAAE,EAAEzD,KAAK,CAAC;IAClE,IAAAM,cAAO,EAAE,GAAE4D,oBAAqB,EAAC,EAAElE,KAAK,CAAC;IAEzC,IAAIkE,oBAAoB,IAAIjB,IAAI,EAAG;MACjC,IAAI,CAACC,UAAU,CAACN,QAAQ,CAACsB,oBAAoB,CAAC,EAAE;QAC9ChB,UAAU,CAACG,IAAI,CAACa,oBAAoB,CAAC;MACvC;MAEA,IAAI9D,GAAG,EAAE;QACP,IAAAE,cAAO,EAAE,kCAAiC,IAAA8D,qBAAc,EAACJ,YAAY,CAAE,EAAC,EAAEhE,KAAK,CAAC;QAChFgE,YAAY,CAAChC,OAAO,CAAClB,KAAK,IAAI0C,4BAA4B,CAACnD,MAAM,EAAES,KAAK,EAAE2C,iBAAiB,CAAC,CAAC;QAC7F;MACF;MAEA,IAAAnD,cAAO,EAAE,qCAAoC4D,oBAAqB,EAAC,EAAElE,KAAK,CAAC;MAC3E;IACF;IACA,IAAAM,cAAO,EAAE,6CAA4C4D,oBAAqB,EAAC,EAAElE,KAAK,CAAC;IACnFiD,IAAI,CAACiB,oBAAoB,CAAC,GAAG,CAAC;IAC9B;EACF,CAAC,CAAC;;EAEF;EACA,OAAOhB,UAAU;AACnB;AAEO,SAASmB,8BAA8B,CAAChE,MAAM,EAAED,GAAG,GAAG,IAAI,EAAE;EACjE;EACA,IAAI6C,IAAI,GAAG,CAAC,CAAC;EAEb,IAAIC,UAAU,GAAG,EAAE,CAAC,CAAC;;EAErB7C,MAAM,CAACsB,MAAM,CAACK,OAAO,CAAClB,KAAK,IAAI,IAAAR,cAAO,EAAE,qBAAoB,IAAA2B,oBAAa,EAACnB,KAAK,CAAE,UAASV,GAAG,GAAG,KAAK,GAAG,UAAW,EAAC,CAAC,CAAC;EAEtH,MAAMuB,MAAM,GAAGtB,MAAM,CAACsB,MAAM,CAACV,MAAM,CAACH,KAAK,IAAI+B,2BAA2B,CAAC/B,KAAK,EAAET,MAAM,CAAC,CAAC,CAAC,CAAC;;EAE1FsB,MAAM,CAACK,OAAO,CAAClB,KAAK,IAAIwD,wBAAwB,CAACxD,KAAK,CAAC,CAAC;EAExD,SAASwD,wBAAwB,CAACxD,KAAK,EAAE;IACvC,IAAAR,cAAO,EAAE,gCAA+B,IAAA2B,oBAAa,EAACnB,KAAK,CAAE,gBAAe,CAAC;IAC7E,MAAMa,MAAM,GAAGW,2BAA2B,CAACxB,KAAK,EAAET,MAAM,CAAC;IACzD,IAAGsB,MAAM,CAACf,MAAM,KAAK,CAAC,EAAE;MACtB;IACF;IAEA,MAAM2D,cAAc,GAAG,IAAAJ,wCAAwB,EAACxC,MAAM,CAAC;IACvD,IAAArB,cAAO,EAAE,WAAUiE,cAAe,EAAC,CAAC;IACpC,IAAIA,cAAc,IAAItB,IAAI,EAAG;MAC3B,IAAA3C,cAAO,EAAE,WAAUiE,cAAe,EAAC,CAAC;MAEpCrB,UAAU,CAACG,IAAI,CAACkB,cAAc,CAAC;MAE/B,IAAInE,GAAG,EAAE;QACP,IAAAE,cAAO,EAAE,0BAAyBiE,cAAe,EAAC,EAAEvE,KAAK,CAAC;QAC1D2B,MAAM,CAACK,OAAO,CAACsB,SAAS,IAAIjD,MAAM,CAACkD,WAAW,CAACD,SAAS,CAAC,CAAC;QAC1D;MACF;MACA,IAAAhD,cAAO,EAAE,kCAAiCiE,cAAe,EAAC,EAAEvE,KAAK,CAAC;IAEpE;IACA,IAAAM,cAAO,EAAE,0CAAyCiE,cAAe,EAAC,EAAEvE,KAAK,CAAC;IAC1EiD,IAAI,CAACsB,cAAc,CAAC,GAAG,CAAC;IACxB;EACF;;EAGA;EACA,OAAOrB,UAAU;AACnB;AAEO,SAASxC,yBAAyB,CAACL,MAAM,EAAED,GAAG,GAAG,IAAI,EAAE;EAC5D,MAAM8C,UAAU,GAAGF,mCAAmC,CAAC3C,MAAM,EAAED,GAAG,CAAC,CAAC,CAAC;EACrE,MAAMoE,WAAW,GAAGX,8BAA8B,CAACxD,MAAM,EAAED,GAAG,CAAC,CAAC,CAAC;EACjE,MAAMqE,WAAW,GAAGJ,8BAA8B,CAAChE,MAAM,EAAED,GAAG,CAAC,CAAC,CAAC;EACjE;;EAEA,MAAMsE,aAAa,GAAGxB,UAAU,CAACyB,MAAM,CAACH,WAAW,CAAC,CAACG,MAAM,CAACF,WAAW,CAAC;EAExE,OAAOC,aAAa;AACtB"}
+{"version":3,"file":"removeDuplicateDataFields.js","names":["debug","createDebugLogger","description","validate","fix","record","nvdebug","res","message","valid","removeDuplicateDatafields","duplicates","length","add6s","field","sixes","subfields","filter","sf","isValidSubfield6","occurrenceNumbers","map","subfield6GetOccurrenceNumber","value","undefined","join","relevantFields","fields","f","some","o","fieldHasOccurrenceNumber","forEach","fieldToString","add8s","fieldHasSubfield","numberOfLinkageSubfields","code","getAllLinkedSubfield6Fields","moreFields","getFirstField","fieldsAsStrings","i","findIndex","includes","isFirstLinkedSubfield6Field","chain","firstField","removeIndividualDuplicateDatafields","seen","removableFields","removableIndividualDuplicateDatafield","removableFieldsAsStrings","removeField","fieldAsString","recordRemoveFieldOrSubfield8","currLinkingNumber","eights","getSubfield8LinkingNumber","removeSubfield","removeDuplicateSubfield8Chains","removables","seenLinkingNumbers","recordGetAllSubfield8LinkingNumbers","linkedFields","recordGetFieldsWithSubfield8LinkingNumber","linkedFieldsAsString","fieldsToNormalizedString","push","fieldsToString","removeDuplicateSubfield6Chains","removeDuplicateDatafield","fieldsAsString","currField","removables8","removables6","removablesAll","concat"],"sources":["../src/removeDuplicateDataFields.js"],"sourcesContent":["import createDebugLogger from 'debug';\nimport {fieldHasSubfield, fieldsToString, fieldToString, nvdebug} from './utils';\nimport {fieldHasOccurrenceNumber, fieldsToNormalizedString, isValidSubfield6, subfield6GetOccurrenceNumber} from './subfield6Utils';\nimport {getSubfield8LinkingNumber, recordGetAllSubfield8LinkingNumbers, recordGetFieldsWithSubfield8LinkingNumber} from './subfield8Utils';\n\n// Relocated from melinda-marc-record-merge-reducers (and renamed)\n\nconst debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:removeDuplicateDataFields');\n\nexport default function () {\n  return {\n    description: 'Remove duplicate data fields. Certain exceptions apply, mainly too complited chained fields',\n    validate, fix\n  };\n\n  function fix(record) {\n    nvdebug('Remove duplicate data fields');\n    const res = {message: [], fix: [], valid: true};\n    removeDuplicateDatafields(record, true);\n    // This can not really fail...\n    return res;\n  }\n\n  function validate(record) {\n    // Check max, and check number of different indexes\n    nvdebug('Validate record: duplicate data fields cause (t)error', debug);\n\n    const duplicates = removeDuplicateDatafields(record, false);\n\n    //const orphanedFields = getOrphanedFields(fieldsContainingSubfield6);\n\n    const res = {message: duplicates};\n\n    /*\n    if (orphanedFields.length > 0) { // eslint-disable-line functional/no-conditional-statement\n      res.message = [`${orphanedFields.length} orphaned occurrence number field(s) detected`]; // eslint-disable-line functional/immutable-data\n    }\n    */\n    res.valid = res.message.length < 1; // eslint-disable-line functional/immutable-data\n    return res;\n  }\n}\n\nfunction add6s(field, record) {\n\n  /*\n  // Can't rely on nice pairs...\n  if (fieldHasSubfield(field, '6')) {\n\n    const pairs = fieldGetOccurrenceNumberPairs(field, record.fields);\n    if (pairs) {\n      return [field].concat(pairs);\n    }\n\n  }\n  */\n\n  // Get all fields with given occurence number\n  const sixes = field.subfields.filter(sf => isValidSubfield6(sf));\n\n  if (sixes.length === 0) {\n    return [field];\n  }\n  nvdebug(`SIXES: ${sixes.length}`);\n  const occurrenceNumbers = sixes.map(sf => subfield6GetOccurrenceNumber(sf)).filter(value => value !== undefined && value !== '00');\n  nvdebug(occurrenceNumbers.join(' -- '));\n\n  const relevantFields = record.fields.filter(f => occurrenceNumbers.some(o => fieldHasOccurrenceNumber(f, o)));\n  nvdebug(`RELFIELDS FOUND: ${relevantFields.length}...`);\n  relevantFields.forEach(f => nvdebug(fieldToString(f)));\n  return relevantFields;\n}\n\nfunction add8s(fields, record) {\n  // Not implemented yet:\n  if (fields && fields.some(f => fieldHasSubfield(f, '8'))) {\n    return [];\n  }\n  return record ? fields : fields;\n}\n\nfunction numberOfLinkageSubfields(field) {\n  const subfields = field.subfields.filter(sf => sf.code === '6' || sf.code === '8');\n  return subfields.length;\n}\n\n\nfunction getAllLinkedSubfield6Fields(field, record) {\n  const fields = add6s(field, record);\n  const moreFields = add8s(fields, record);\n\n  // Currently we don't handle fields with more than one $6 and/or $8 subfield.\n  if (moreFields.length === 0 || moreFields.some(f => numberOfLinkageSubfields(f) > 1)) {\n    return []; // Don't fix!\n  }\n  return moreFields;\n}\n\nfunction getFirstField(record, fields) {\n  const fieldsAsStrings = fields.map(field => fieldToString(field));\n  record.fields.forEach((field, i) => nvdebug(`${i}:\\t${fieldToString(field)}`));\n  nvdebug(`INCOMING: ${fieldsAsStrings.join('\\t')}`);\n  const i = record.fields.findIndex(field => fieldsAsStrings.includes(fieldToString(field)));\n  if (i > -1) {\n    const field = record.fields[i];\n    nvdebug(`1st F: ${i + 1}/${record.fields.length} ${fieldToString(field)}`);\n    return field;\n  }\n  return undefined;\n}\n\n\nfunction isFirstLinkedSubfield6Field(field, record) {\n  if (!field.subfields) { // Is not a datafield\n    return false;\n  }\n  const chain = getAllLinkedSubfield6Fields(field, record);\n  if (chain.length < 2) {\n    return false;\n  }\n\n  // Interpretation of first: position of field in record (however, we might have a duplicate field. See tests...)\n  const firstField = getFirstField(record, chain);\n  if (firstField) {\n    return fieldToString(field) === fieldToString(firstField);\n  }\n  return false;\n\n  // Fallback:\n  //return fieldToString(field) === fieldToString(chain[0]);\n}\n\nexport function removeIndividualDuplicateDatafields(record, fix = true) { // No $6 nor $8 in field\n  /* eslint-disable */\n  let seen = {};\n\n  record.fields.forEach(field => nvdebug(`DUPL-1 CHECK SINGLE ${fieldToString(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));\n  \n  const removableFields = record.fields.filter(field => removableIndividualDuplicateDatafield(field));\n  const removableFieldsAsStrings = removableFields.map(field => fieldToString(field));\n\n  if (fix) {\n    removableFields.forEach(field => record.removeField(field));\n  }\n\n  return removableFieldsAsStrings;\n\n  function removableIndividualDuplicateDatafield(field) {\n    if (!field.subfields) { // Not a datafield\n      //nvdebug(`SKIP subfieldless ${fieldToString(field)}`);\n      return false;\n    }\n    // There's actually no reason to check whether individual fields contain a $6 or an $8...\n    // If everything incl. occurence/xxxx numbers match it's still deletable, regardless of chains.\n\n    //nvdebug(`removeIndividualDuplicateDatafield? ${fieldToString(field)}`);\n\n    // We are in trouble if $9 ^ and $9 ^^ style chains appear here...\n    const fieldAsString = fieldToString(field); // Never normalize!\n\n    //nvdebug(` step 2 ${fieldAsString}`);\n    if (fieldAsString in seen)  {\n      nvdebug(`DUPLICATE SINGLETON DETECTED: ${fieldAsString}`);\n      return true;\n    }\n    nvdebug(`MARK SINGLETON AS SEEN: ${fieldAsString}`, debug);\n    seen[fieldAsString] = 1;\n    return;\n  }\n\n  /* eslint-enable */\n\n}\n\n\nfunction recordRemoveFieldOrSubfield8(record, field, currLinkingNumber) {\n  const eights = field.subfields.filter(sf => sf.code === '8');\n  if (eights.length < 2) {\n    record.removeField(field);\n    return;\n  }\n  const subfields = field.subfields.filter(sf => getSubfield8LinkingNumber(sf) === currLinkingNumber);\n  subfields.forEach(sf => record.removeSubfield(sf, field));\n}\n\n\nexport function removeDuplicateSubfield8Chains(record, fix = true) {\n  /* eslint-disable */\n  let seen = {};\n\n  let removables = []; // for validation\n\n  nvdebug(\"CHAIN-8\");\n  const seenLinkingNumbers = recordGetAllSubfield8LinkingNumbers(record);\n  if (seenLinkingNumbers.length === 0) {\n    return removables;\n  }\n\n  nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);\n\n  seenLinkingNumbers.forEach(currLinkingNumber => {\n    const linkedFields = recordGetFieldsWithSubfield8LinkingNumber(record, currLinkingNumber) //getFieldsWithSubfield8Index(base, baseIndex);\n    const linkedFieldsAsString = fieldsToNormalizedString(linkedFields, currLinkingNumber);\n    nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);\n    nvdebug(`${linkedFieldsAsString}`, debug);\n\n    if (linkedFieldsAsString in seen)  {\n      if (!removables.includes(linkedFieldsAsString)) {\n        removables.push(linkedFieldsAsString);\n      }\n\n      if (fix) {\n        nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);\n        linkedFields.forEach(field => recordRemoveFieldOrSubfield8(record, field, currLinkingNumber));\n        return;\n      }\n\n      nvdebug(`$8 VALIDATION: DUPLICATE DETECTED ${linkedFieldsAsString}`, debug);\n      return;\n    }\n    nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);\n    seen[linkedFieldsAsString] = 1;\n    return;\n  });\n\n  /* eslint-enable */\n  return removables;\n}\n\nexport function removeDuplicateSubfield6Chains(record, fix = true) {\n  /* eslint-disable */\n  let seen = {};\n\n  let removables = []; // for validation\n\n  record.fields.forEach(field => nvdebug(`DUPL-CHECK $CHAIN ${fieldToString(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));\n  \n  const fields = record.fields.filter(field => isFirstLinkedSubfield6Field(field, record)); // Well a\n  \n  fields.forEach(field => removeDuplicateDatafield(field));\n\n  function removeDuplicateDatafield(field) {\n    nvdebug(`removeDuplicateDatafield? $6 ${fieldToString(field)} (and friends)`);\n    const fields = getAllLinkedSubfield6Fields(field, record);\n    if(fields.length === 0) {\n      return;\n    }\n\n    const fieldsAsString = fieldsToNormalizedString(fields);\n    nvdebug(` step 2 ${fieldsAsString}`);\n    if (fieldsAsString in seen)  {\n      nvdebug(` step 3 ${fieldsAsString}`);\n\n      removables.push(fieldsAsString);\n\n      if (fix) {\n        nvdebug(`DOUBLE REMOVAL: REMOVE ${fieldsAsString}`, debug);\n        fields.forEach(currField => record.removeField(currField));\n        return;\n      }\n      nvdebug(`VALIDATION: DUPLICATE DETECTED ${fieldsAsString}`, debug);\n      \n    }\n    nvdebug(`DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${fieldsAsString}`, debug);\n    seen[fieldsAsString] = 1;\n    return;\n  }\n\n\n  /* eslint-enable */\n  return removables;\n}\n\nexport function removeDuplicateDatafields(record, fix = true) {\n  const removables = removeIndividualDuplicateDatafields(record, fix); // Lone fields\n  const removables8 = removeDuplicateSubfield8Chains(record, fix); // Lone subfield $8 chains\n  const removables6 = removeDuplicateSubfield6Chains(record, fix); // Lone subfield $6 chains\n  // HOW TO HANDLE $6+$8 combos?\n\n  const removablesAll = removables.concat(removables8).concat(removables6);\n\n  return removablesAll;\n}\n"],"mappings":";;;;;;;;;;AAAA;AACA;AACA;AACA;AAA2I;AAE3I;;AAEA,MAAMA,KAAK,GAAG,IAAAC,cAAiB,EAAC,oEAAoE,CAAC;AAEtF,oBAAY;EACzB,OAAO;IACLC,WAAW,EAAE,6FAA6F;IAC1GC,QAAQ;IAAEC;EACZ,CAAC;EAED,SAASA,GAAG,CAACC,MAAM,EAAE;IACnB,IAAAC,cAAO,EAAC,8BAA8B,CAAC;IACvC,MAAMC,GAAG,GAAG;MAACC,OAAO,EAAE,EAAE;MAAEJ,GAAG,EAAE,EAAE;MAAEK,KAAK,EAAE;IAAI,CAAC;IAC/CC,yBAAyB,CAACL,MAAM,EAAE,IAAI,CAAC;IACvC;IACA,OAAOE,GAAG;EACZ;EAEA,SAASJ,QAAQ,CAACE,MAAM,EAAE;IACxB;IACA,IAAAC,cAAO,EAAC,uDAAuD,EAAEN,KAAK,CAAC;IAEvE,MAAMW,UAAU,GAAGD,yBAAyB,CAACL,MAAM,EAAE,KAAK,CAAC;;IAE3D;;IAEA,MAAME,GAAG,GAAG;MAACC,OAAO,EAAEG;IAAU,CAAC;;IAEjC;AACJ;AACA;AACA;AACA;IACIJ,GAAG,CAACE,KAAK,GAAGF,GAAG,CAACC,OAAO,CAACI,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,OAAOL,GAAG;EACZ;AACF;AAEA,SAASM,KAAK,CAACC,KAAK,EAAET,MAAM,EAAE;EAE5B;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;EAIE;EACA,MAAMU,KAAK,GAAGD,KAAK,CAACE,SAAS,CAACC,MAAM,CAACC,EAAE,IAAI,IAAAC,gCAAgB,EAACD,EAAE,CAAC,CAAC;EAEhE,IAAIH,KAAK,CAACH,MAAM,KAAK,CAAC,EAAE;IACtB,OAAO,CAACE,KAAK,CAAC;EAChB;EACA,IAAAR,cAAO,EAAE,UAASS,KAAK,CAACH,MAAO,EAAC,CAAC;EACjC,MAAMQ,iBAAiB,GAAGL,KAAK,CAACM,GAAG,CAACH,EAAE,IAAI,IAAAI,4CAA4B,EAACJ,EAAE,CAAC,CAAC,CAACD,MAAM,CAACM,KAAK,IAAIA,KAAK,KAAKC,SAAS,IAAID,KAAK,KAAK,IAAI,CAAC;EAClI,IAAAjB,cAAO,EAACc,iBAAiB,CAACK,IAAI,CAAC,MAAM,CAAC,CAAC;EAEvC,MAAMC,cAAc,GAAGrB,MAAM,CAACsB,MAAM,CAACV,MAAM,CAACW,CAAC,IAAIR,iBAAiB,CAACS,IAAI,CAACC,CAAC,IAAI,IAAAC,wCAAwB,EAACH,CAAC,EAAEE,CAAC,CAAC,CAAC,CAAC;EAC7G,IAAAxB,cAAO,EAAE,oBAAmBoB,cAAc,CAACd,MAAO,KAAI,CAAC;EACvDc,cAAc,CAACM,OAAO,CAACJ,CAAC,IAAI,IAAAtB,cAAO,EAAC,IAAA2B,oBAAa,EAACL,CAAC,CAAC,CAAC,CAAC;EACtD,OAAOF,cAAc;AACvB;AAEA,SAASQ,KAAK,CAACP,MAAM,EAAEtB,MAAM,EAAE;EAC7B;EACA,IAAIsB,MAAM,IAAIA,MAAM,CAACE,IAAI,CAACD,CAAC,IAAI,IAAAO,uBAAgB,EAACP,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE;IACxD,OAAO,EAAE;EACX;EACA,OAAOvB,MAAM,GAAGsB,MAAM,GAAGA,MAAM;AACjC;AAEA,SAASS,wBAAwB,CAACtB,KAAK,EAAE;EACvC,MAAME,SAAS,GAAGF,KAAK,CAACE,SAAS,CAACC,MAAM,CAACC,EAAE,IAAIA,EAAE,CAACmB,IAAI,KAAK,GAAG,IAAInB,EAAE,CAACmB,IAAI,KAAK,GAAG,CAAC;EAClF,OAAOrB,SAAS,CAACJ,MAAM;AACzB;AAGA,SAAS0B,2BAA2B,CAACxB,KAAK,EAAET,MAAM,EAAE;EAClD,MAAMsB,MAAM,GAAGd,KAAK,CAACC,KAAK,EAAET,MAAM,CAAC;EACnC,MAAMkC,UAAU,GAAGL,KAAK,CAACP,MAAM,EAAEtB,MAAM,CAAC;;EAExC;EACA,IAAIkC,UAAU,CAAC3B,MAAM,KAAK,CAAC,IAAI2B,UAAU,CAACV,IAAI,CAACD,CAAC,IAAIQ,wBAAwB,CAACR,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE;IACpF,OAAO,EAAE,CAAC,CAAC;EACb;;EACA,OAAOW,UAAU;AACnB;AAEA,SAASC,aAAa,CAACnC,MAAM,EAAEsB,MAAM,EAAE;EACrC,MAAMc,eAAe,GAAGd,MAAM,CAACN,GAAG,CAACP,KAAK,IAAI,IAAAmB,oBAAa,EAACnB,KAAK,CAAC,CAAC;EACjET,MAAM,CAACsB,MAAM,CAACK,OAAO,CAAC,CAAClB,KAAK,EAAE4B,CAAC,KAAK,IAAApC,cAAO,EAAE,GAAEoC,CAAE,MAAK,IAAAT,oBAAa,EAACnB,KAAK,CAAE,EAAC,CAAC,CAAC;EAC9E,IAAAR,cAAO,EAAE,aAAYmC,eAAe,CAAChB,IAAI,CAAC,IAAI,CAAE,EAAC,CAAC;EAClD,MAAMiB,CAAC,GAAGrC,MAAM,CAACsB,MAAM,CAACgB,SAAS,CAAC7B,KAAK,IAAI2B,eAAe,CAACG,QAAQ,CAAC,IAAAX,oBAAa,EAACnB,KAAK,CAAC,CAAC,CAAC;EAC1F,IAAI4B,CAAC,GAAG,CAAC,CAAC,EAAE;IACV,MAAM5B,KAAK,GAAGT,MAAM,CAACsB,MAAM,CAACe,CAAC,CAAC;IAC9B,IAAApC,cAAO,EAAE,UAASoC,CAAC,GAAG,CAAE,IAAGrC,MAAM,CAACsB,MAAM,CAACf,MAAO,IAAG,IAAAqB,oBAAa,EAACnB,KAAK,CAAE,EAAC,CAAC;IAC1E,OAAOA,KAAK;EACd;EACA,OAAOU,SAAS;AAClB;AAGA,SAASqB,2BAA2B,CAAC/B,KAAK,EAAET,MAAM,EAAE;EAClD,IAAI,CAACS,KAAK,CAACE,SAAS,EAAE;IAAE;IACtB,OAAO,KAAK;EACd;EACA,MAAM8B,KAAK,GAAGR,2BAA2B,CAACxB,KAAK,EAAET,MAAM,CAAC;EACxD,IAAIyC,KAAK,CAAClC,MAAM,GAAG,CAAC,EAAE;IACpB,OAAO,KAAK;EACd;;EAEA;EACA,MAAMmC,UAAU,GAAGP,aAAa,CAACnC,MAAM,EAAEyC,KAAK,CAAC;EAC/C,IAAIC,UAAU,EAAE;IACd,OAAO,IAAAd,oBAAa,EAACnB,KAAK,CAAC,KAAK,IAAAmB,oBAAa,EAACc,UAAU,CAAC;EAC3D;EACA,OAAO,KAAK;;EAEZ;EACA;AACF;;AAEO,SAASC,mCAAmC,CAAC3C,MAAM,EAAED,GAAG,GAAG,IAAI,EAAE;EAAE;EACxE;EACA,IAAI6C,IAAI,GAAG,CAAC,CAAC;EAEb5C,MAAM,CAACsB,MAAM,CAACK,OAAO,CAAClB,KAAK,IAAI,IAAAR,cAAO,EAAE,uBAAsB,IAAA2B,oBAAa,EAACnB,KAAK,CAAE,UAASV,GAAG,GAAG,KAAK,GAAG,UAAW,EAAC,CAAC,CAAC;EAExH,MAAM8C,eAAe,GAAG7C,MAAM,CAACsB,MAAM,CAACV,MAAM,CAACH,KAAK,IAAIqC,qCAAqC,CAACrC,KAAK,CAAC,CAAC;EACnG,MAAMsC,wBAAwB,GAAGF,eAAe,CAAC7B,GAAG,CAACP,KAAK,IAAI,IAAAmB,oBAAa,EAACnB,KAAK,CAAC,CAAC;EAEnF,IAAIV,GAAG,EAAE;IACP8C,eAAe,CAAClB,OAAO,CAAClB,KAAK,IAAIT,MAAM,CAACgD,WAAW,CAACvC,KAAK,CAAC,CAAC;EAC7D;EAEA,OAAOsC,wBAAwB;EAE/B,SAASD,qCAAqC,CAACrC,KAAK,EAAE;IACpD,IAAI,CAACA,KAAK,CAACE,SAAS,EAAE;MAAE;MACtB;MACA,OAAO,KAAK;IACd;IACA;IACA;;IAEA;;IAEA;IACA,MAAMsC,aAAa,GAAG,IAAArB,oBAAa,EAACnB,KAAK,CAAC,CAAC,CAAC;;IAE5C;IACA,IAAIwC,aAAa,IAAIL,IAAI,EAAG;MAC1B,IAAA3C,cAAO,EAAE,iCAAgCgD,aAAc,EAAC,CAAC;MACzD,OAAO,IAAI;IACb;IACA,IAAAhD,cAAO,EAAE,2BAA0BgD,aAAc,EAAC,EAAEtD,KAAK,CAAC;IAC1DiD,IAAI,CAACK,aAAa,CAAC,GAAG,CAAC;IACvB;EACF;;EAEA;AAEF;;AAGA,SAASC,4BAA4B,CAAClD,MAAM,EAAES,KAAK,EAAE0C,iBAAiB,EAAE;EACtE,MAAMC,MAAM,GAAG3C,KAAK,CAACE,SAAS,CAACC,MAAM,CAACC,EAAE,IAAIA,EAAE,CAACmB,IAAI,KAAK,GAAG,CAAC;EAC5D,IAAIoB,MAAM,CAAC7C,MAAM,GAAG,CAAC,EAAE;IACrBP,MAAM,CAACgD,WAAW,CAACvC,KAAK,CAAC;IACzB;EACF;EACA,MAAME,SAAS,GAAGF,KAAK,CAACE,SAAS,CAACC,MAAM,CAACC,EAAE,IAAI,IAAAwC,yCAAyB,EAACxC,EAAE,CAAC,KAAKsC,iBAAiB,CAAC;EACnGxC,SAAS,CAACgB,OAAO,CAACd,EAAE,IAAIb,MAAM,CAACsD,cAAc,CAACzC,EAAE,EAAEJ,KAAK,CAAC,CAAC;AAC3D;AAGO,SAAS8C,8BAA8B,CAACvD,MAAM,EAAED,GAAG,GAAG,IAAI,EAAE;EACjE;EACA,IAAI6C,IAAI,GAAG,CAAC,CAAC;EAEb,IAAIY,UAAU,GAAG,EAAE,CAAC,CAAC;;EAErB,IAAAvD,cAAO,EAAC,SAAS,CAAC;EAClB,MAAMwD,kBAAkB,GAAG,IAAAC,mDAAmC,EAAC1D,MAAM,CAAC;EACtE,IAAIyD,kBAAkB,CAAClD,MAAM,KAAK,CAAC,EAAE;IACnC,OAAOiD,UAAU;EACnB;EAEA,IAAAvD,cAAO,EAAE,8BAA6BwD,kBAAkB,CAACrC,IAAI,CAAC,IAAI,CAAE,EAAC,EAAEzB,KAAK,CAAC;EAE7E8D,kBAAkB,CAAC9B,OAAO,CAACwB,iBAAiB,IAAI;IAC9C,MAAMQ,YAAY,GAAG,IAAAC,yDAAyC,EAAC5D,MAAM,EAAEmD,iBAAiB,CAAC,EAAC;IAC1F,MAAMU,oBAAoB,GAAG,IAAAC,wCAAwB,EAACH,YAAY,EAAER,iBAAiB,CAAC;IACtF,IAAAlD,cAAO,EAAE,8BAA6BkD,iBAAkB,GAAE,EAAExD,KAAK,CAAC;IAClE,IAAAM,cAAO,EAAE,GAAE4D,oBAAqB,EAAC,EAAElE,KAAK,CAAC;IAEzC,IAAIkE,oBAAoB,IAAIjB,IAAI,EAAG;MACjC,IAAI,CAACY,UAAU,CAACjB,QAAQ,CAACsB,oBAAoB,CAAC,EAAE;QAC9CL,UAAU,CAACO,IAAI,CAACF,oBAAoB,CAAC;MACvC;MAEA,IAAI9D,GAAG,EAAE;QACP,IAAAE,cAAO,EAAE,kCAAiC,IAAA+D,qBAAc,EAACL,YAAY,CAAE,EAAC,EAAEhE,KAAK,CAAC;QAChFgE,YAAY,CAAChC,OAAO,CAAClB,KAAK,IAAIyC,4BAA4B,CAAClD,MAAM,EAAES,KAAK,EAAE0C,iBAAiB,CAAC,CAAC;QAC7F;MACF;MAEA,IAAAlD,cAAO,EAAE,qCAAoC4D,oBAAqB,EAAC,EAAElE,KAAK,CAAC;MAC3E;IACF;IACA,IAAAM,cAAO,EAAE,6CAA4C4D,oBAAqB,EAAC,EAAElE,KAAK,CAAC;IACnFiD,IAAI,CAACiB,oBAAoB,CAAC,GAAG,CAAC;IAC9B;EACF,CAAC,CAAC;;EAEF;EACA,OAAOL,UAAU;AACnB;AAEO,SAASS,8BAA8B,CAACjE,MAAM,EAAED,GAAG,GAAG,IAAI,EAAE;EACjE;EACA,IAAI6C,IAAI,GAAG,CAAC,CAAC;EAEb,IAAIY,UAAU,GAAG,EAAE,CAAC,CAAC;;EAErBxD,MAAM,CAACsB,MAAM,CAACK,OAAO,CAAClB,KAAK,IAAI,IAAAR,cAAO,EAAE,qBAAoB,IAAA2B,oBAAa,EAACnB,KAAK,CAAE,UAASV,GAAG,GAAG,KAAK,GAAG,UAAW,EAAC,CAAC,CAAC;EAEtH,MAAMuB,MAAM,GAAGtB,MAAM,CAACsB,MAAM,CAACV,MAAM,CAACH,KAAK,IAAI+B,2BAA2B,CAAC/B,KAAK,EAAET,MAAM,CAAC,CAAC,CAAC,CAAC;;EAE1FsB,MAAM,CAACK,OAAO,CAAClB,KAAK,IAAIyD,wBAAwB,CAACzD,KAAK,CAAC,CAAC;EAExD,SAASyD,wBAAwB,CAACzD,KAAK,EAAE;IACvC,IAAAR,cAAO,EAAE,gCAA+B,IAAA2B,oBAAa,EAACnB,KAAK,CAAE,gBAAe,CAAC;IAC7E,MAAMa,MAAM,GAAGW,2BAA2B,CAACxB,KAAK,EAAET,MAAM,CAAC;IACzD,IAAGsB,MAAM,CAACf,MAAM,KAAK,CAAC,EAAE;MACtB;IACF;IAEA,MAAM4D,cAAc,GAAG,IAAAL,wCAAwB,EAACxC,MAAM,CAAC;IACvD,IAAArB,cAAO,EAAE,WAAUkE,cAAe,EAAC,CAAC;IACpC,IAAIA,cAAc,IAAIvB,IAAI,EAAG;MAC3B,IAAA3C,cAAO,EAAE,WAAUkE,cAAe,EAAC,CAAC;MAEpCX,UAAU,CAACO,IAAI,CAACI,cAAc,CAAC;MAE/B,IAAIpE,GAAG,EAAE;QACP,IAAAE,cAAO,EAAE,0BAAyBkE,cAAe,EAAC,EAAExE,KAAK,CAAC;QAC1D2B,MAAM,CAACK,OAAO,CAACyC,SAAS,IAAIpE,MAAM,CAACgD,WAAW,CAACoB,SAAS,CAAC,CAAC;QAC1D;MACF;MACA,IAAAnE,cAAO,EAAE,kCAAiCkE,cAAe,EAAC,EAAExE,KAAK,CAAC;IAEpE;IACA,IAAAM,cAAO,EAAE,0CAAyCkE,cAAe,EAAC,EAAExE,KAAK,CAAC;IAC1EiD,IAAI,CAACuB,cAAc,CAAC,GAAG,CAAC;IACxB;EACF;;EAGA;EACA,OAAOX,UAAU;AACnB;AAEO,SAASnD,yBAAyB,CAACL,MAAM,EAAED,GAAG,GAAG,IAAI,EAAE;EAC5D,MAAMyD,UAAU,GAAGb,mCAAmC,CAAC3C,MAAM,EAAED,GAAG,CAAC,CAAC,CAAC;EACrE,MAAMsE,WAAW,GAAGd,8BAA8B,CAACvD,MAAM,EAAED,GAAG,CAAC,CAAC,CAAC;EACjE,MAAMuE,WAAW,GAAGL,8BAA8B,CAACjE,MAAM,EAAED,GAAG,CAAC,CAAC,CAAC;EACjE;;EAEA,MAAMwE,aAAa,GAAGf,UAAU,CAACgB,MAAM,CAACH,WAAW,CAAC,CAACG,MAAM,CAACF,WAAW,CAAC;EAExE,OAAOC,aAAa;AACtB"}

package/package.json

@@ -14,7 +14,7 @@
     "url": "git@github.com:natlibfi/marc-record-validators-melinda.git"
   },
   "license": "MIT",
-  "version": "10.2.0",
+  "version": "10.2.1",
   "main": "./dist/index.js",
   "publishConfig": {
     "access": "public"

package/src/removeDuplicateDataFields.js

@@ -134,43 +134,42 @@ export function removeIndividualDuplicateDatafields(record, fix = true) { // No
   /* eslint-disable */
   let seen = {};
 
-  let removables = []; // for validation
-
-  //record.fields.forEach(field => nvdebug(`DUPL-1 CHECK SINGLE ${fieldToString(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));
-  
-  const fields = record.fields;
+  record.fields.forEach(field => nvdebug(`DUPL-1 CHECK SINGLE ${fieldToString(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));
   
-  fields.forEach(field => removeIndividualDuplicateDatafield(field));
+  const removableFields = record.fields.filter(field => removableIndividualDuplicateDatafield(field));
+  const removableFieldsAsStrings = removableFields.map(field => fieldToString(field));
+
+  if (fix) {
+    removableFields.forEach(field => record.removeField(field));
+  }
 
-  function removeIndividualDuplicateDatafield(field) {
-    //nvdebug(`removeIndividualDuplicateDatafield? ${fieldToString(field)} (and friends)`);
+  return removableFieldsAsStrings;
+
+  function removableIndividualDuplicateDatafield(field) {
+    if (!field.subfields) { // Not a datafield
+      //nvdebug(`SKIP subfieldless ${fieldToString(field)}`);
+      return false;
+    }
+    // There's actually no reason to check whether individual fields contain a $6 or an $8...
+    // If everything incl. occurence/xxxx numbers match it's still deletable, regardless of chains.
+
+    //nvdebug(`removeIndividualDuplicateDatafield? ${fieldToString(field)}`);
 
     // We are in trouble if $9 ^ and $9 ^^ style chains appear here...
     const fieldAsString = fieldToString(field); // Never normalize!
 
     //nvdebug(` step 2 ${fieldAsString}`);
     if (fieldAsString in seen)  {
-      // nvdebug(` step 3 ${fieldAsString}`);
-      // There's actually no reason to check whether individual fields contain a $6 or an $8...
-
-      if (!removables.includes(fieldAsString)) {
-        removables.push(fieldAsString);
-      }
-
-      if (fix) {
-        //nvdebug(`DOUBLE REMOVAL: REMOVE ${fieldAsString}`, debug);
-        fields.forEach(currField => record.removeField(currField));
-        return;
-      }
-      nvdebug(`VALIDATION-1: DUPLICATE DETECTED ${fieldAsString}`, debug);
-      return;
+      nvdebug(`DUPLICATE SINGLETON DETECTED: ${fieldAsString}`);
+      return true;
     }
-    nvdebug(`ADD2SEEN-1 ${fieldAsString}`, debug);
+    nvdebug(`MARK SINGLETON AS SEEN: ${fieldAsString}`, debug);
     seen[fieldAsString] = 1;
     return;
   }
+
   /* eslint-enable */
-  return removables;
+
 }
 
 

package/test-fixtures/remove-duplicate-datafields/f07/expectedResult.json

@@ -0,0 +1,15 @@
+{ "_validationOptions": {},
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "001", "value": "base"},
+    { "tag": "110", "ind1": "2", "ind2": " ", "subfields": [
+      {"code": "a", "value": "Yhdysvallat"},
+      {"code": "b", "value": "Office of Strategic Services,"},
+      {"code": "e", "value": "kirjoittaja."}
+    ]},
+    { "tag": "856", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "u", "value": "https://www.novitaknits.com/fi/fazer-x-novita/" },
+      { "code": "y", "value": "Linkki julkaisun verkkosivuille" }
+    ]}
+  ]
+}

package/test-fixtures/remove-duplicate-datafields/f07/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "Fix-07: single duplicate datafield fixed (bug found via reducer)",
+  "enabled": true,
+  "fix": true,
+  "only": false
+}

package/test-fixtures/remove-duplicate-datafields/f07/record.json

@@ -0,0 +1,21 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "001", "value": "base"},
+    { "tag": "110", "ind1": "2", "ind2": " ", "subfields": [
+      {"code": "a", "value": "Yhdysvallat"},
+      {"code": "b", "value": "Office of Strategic Services,"},
+      {"code": "e", "value": "kirjoittaja."}
+    ]},
+    {
+      "tag": "856", "ind1": " ", "ind2": " ", "subfields": [
+        { "code": "u", "value": "https://www.novitaknits.com/fi/fazer-x-novita/" },
+        { "code": "y", "value": "Linkki julkaisun verkkosivuille" }
+    ]},
+    { "tag": "856", "ind1": " ", "ind2": " ", "subfields": [
+        { "code": "u", "value": "https://www.novitaknits.com/fi/fazer-x-novita/" },
+        { "code": "y", "value": "Linkki julkaisun verkkosivuille" }
+      ]
+    }
+  ]
+}

package/test-fixtures/remove-duplicate-datafields/f08/expectedResult.json

@@ -0,0 +1,21 @@
+{ "_validationOptions": {},
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "020", "ind1": " ", "ind2": " ",
+      "subfields": [
+        { "code": "a", "value": "978-951-583-508-6"},
+        { "code": "q", "value": "Svenska litteratursällskapet i Finland" },
+        { "code": "q", "value": "inbunden" }
+    ]},
+    { "tag": "020", "ind1": " ", "ind2": " ",
+      "subfields": [
+        { "code": "a", "value": "978-91-984961-7-8" },
+        { "code": "q", "value": "Appell Förlag" },
+        { "code": "q", "value": "inbunden" }
+      ]}, 
+    { "tag": "040", "ind1": " ", "ind2": " ",
+      "subfields": [
+        { "code": "b", "value": "swe" }
+      ]}
+  ]
+}

package/test-fixtures/remove-duplicate-datafields/f08/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "Fix-08: record that caused issues on the reducer side",
+  "enabled": true,
+  "fix": true,
+  "only": false
+}

package/test-fixtures/remove-duplicate-datafields/f08/record.json

@@ -0,0 +1,33 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "020", "ind1": " ", "ind2": " ",
+      "subfields": [
+        { "code": "a", "value": "978-951-583-508-6"},
+        { "code": "q", "value": "Svenska litteratursällskapet i Finland" },
+        { "code": "q", "value": "inbunden" }
+    ]},
+    { "tag": "020", "ind1": " ", "ind2": " ", "subfields": [
+        { "code": "a", "value": "978-91-984961-7-8" },
+        { "code": "q", "value": "Appell Förlag" },
+        { "code": "q", "value": "inbunden" }
+      ]},
+    { "tag": "020", "ind1": " ", "ind2": " ", "subfields": [
+        { "code": "a", "value": "978-951-583-508-6"},
+        { "code": "q", "value": "Svenska litteratursällskapet i Finland" },
+        { "code": "q", "value": "inbunden" }
+    ]},
+    { "tag": "020", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "978-91-984961-7-8" },
+      { "code": "q", "value": "Appell Förlag" },
+      { "code": "q", "value": "inbunden" }
+    ]}, 
+    { "tag": "020", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "978-91-984961-7-8" },
+      { "code": "q", "value": "Appell Förlag" },
+      { "code": "q", "value": "inbunden" }
+    ]}, 
+ 
+    { "tag": "040", "ind1": " ", "ind2": " ", "subfields": [ { "code": "b", "value": "swe" } ]}
+  ]
+}