@natlibfi/marc-record-validators-melinda 3.2.0 → 3.3.14-alpha.3

package/src/removeDuplicateDataFields.js

@@ -0,0 +1,447 @@
+import createDebugLogger from 'debug';
+import {fieldsToString, fieldToString, nvdebug} from './utils';
+import {fieldHasValidSubfield6, fieldsGetOccurrenceNumbers, fieldsToNormalizedString, fieldToNormalizedString, get6s} from './subfield6Utils';
+import {add8s, fieldHasLinkingNumber, fieldHasValidSubfield8, fieldsGetAllSubfield8LinkingNumbers, getSubfield8LinkingNumber, recordGetAllSubfield8LinkingNumbers, recordGetFieldsWithSubfield8LinkingNumber} from './subfield8Utils';
+
+const LINK_ROOT = 4;
+const LINKED_AND_PROCESSED = 2;
+const LINKED_NOT_PROCESSED = 1;
+
+
+// Relocated from melinda-marc-record-merge-reducers (and renamed)
+
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:removeDuplicateDataFields');
+
+export default function () {
+  return {
+    description: 'Remove duplicate data fields. Certain exceptions apply, mainly too complited chained fields',
+    validate, fix
+  };
+
+  function fix(record) {
+    nvdebug('Remove duplicate data fields');
+    const res = {message: [], fix: [], valid: true};
+    removeDuplicateDatafields(record, true);
+    // This can not really fail...
+    return res;
+  }
+
+  function validate(record) {
+    // Check max, and check number of different indexes
+    nvdebug('Validate record: duplicate data fields cause (t)error', debug);
+
+    const duplicates = removeDuplicateDatafields(record, false);
+
+    //const orphanedFields = getOrphanedFields(fieldsContainingSubfield6);
+
+    const res = {message: duplicates};
+
+    /*
+    if (orphanedFields.length > 0) { // eslint-disable-line functional/no-conditional-statements
+      res.message = [`${orphanedFields.length} orphaned occurrence number field(s) detected`]; // eslint-disable-line functional/immutable-data
+    }
+    */
+    res.valid = res.message.length < 1; // eslint-disable-line functional/immutable-data
+    return res;
+  }
+}
+
+
+/*
+function numberOfLinkageSubfields(field) {
+  nvdebug(`N of Linkage Subs(${fieldToString(field)})`);
+  const subfields = field.subfields.filter(sf => sf.code === '6' || sf.code === '8');
+  return subfields.length;
+}
+*/
+
+function removeLinkNotes(record) {
+  record.fields.forEach(f => delete f.linkNote); // eslint-disable-line functional/immutable-data
+}
+
+
+function newGetAllLinkedFields(field, record, useSixes = true, useEights = true) {
+  removeLinkNotes(record); // should be clear, but let's play safe
+
+  /* eslint-disable */
+  field.linkNote = LINK_ROOT;
+
+  let currField = field;
+
+  // Loop until all linked fields have been processed:
+  while (currField !== undefined) {
+    if (useSixes) {
+      const related6s = get6s(currField, record.fields)
+      related6s.forEach(f => linkField(f));
+    }
+    if (useEights) {
+      const related8s = add8s([currField], record);
+      related8s.forEach(f => linkField(f));
+    }
+    if (currField.linkNote !== LINK_ROOT) {
+      currField.linkNote = LINKED_AND_PROCESSED;
+    }
+    currField = record.fields.find(f => f.linkNote === LINKED_NOT_PROCESSED);
+  }
+
+  // Collect relevant fields:
+  const linkedFields = record.fields.filter(f => f.linkNote);
+
+  removeLinkNotes(record);
+
+  return linkedFields;
+
+  function linkField(f) {
+    if (!f.linkNote) {
+      f.linkNote = LINKED_NOT_PROCESSED;
+    }
+  }
+
+
+  /* eslint-enable */
+}
+
+
+function recordRemoveFieldOrSubfield8(record, field, currLinkingNumber) {
+  const eights = field.subfields.filter(sf => sf.code === '8');
+  if (eights.length < 2) {
+    record.removeField(field);
+    return;
+  }
+  const subfields = field.subfields.filter(sf => getSubfield8LinkingNumber(sf) === currLinkingNumber);
+  subfields.forEach(sf => record.removeSubfield(sf, field));
+}
+
+function newRecordRemoveFieldOrSubfield8(record, field, currLinkingNumber, fix) {
+  const eights = field.subfields.filter(sf => sf.code === '8');
+  if (eights.length < 2) {
+    field.deleted = 1; // eslint-disable-line functional/immutable-data
+    return;
+  }
+  const subfields = field.subfields.filter(sf => getSubfield8LinkingNumber(sf) === currLinkingNumber);
+  subfields.forEach(sf => {
+    field.modified = 1; // eslint-disable-line functional/immutable-data
+    if (fix) { // eslint-disable-line functional/no-conditional-statements
+      record.removeSubfield(sf, field);
+    }
+  });
+}
+
+
+export function removeDuplicateSubfield8Chains(record, fix = true) {
+
+  // Seen $8 subsfields in various fields:
+  // 161 700
+  // 17 710
+  // 11 110
+  // 8 730
+  // 1 100
+  // Given these stats, there's no need to check for 1XX-vs-7XX removals
+
+  /* eslint-disable */
+  let seen = {};
+
+  let removables = []; // for validation
+
+  nvdebug("CHAIN-8");
+  const seenLinkingNumbers = recordGetAllSubfield8LinkingNumbers(record);
+  if (seenLinkingNumbers.length === 0) {
+    return removables;
+  }
+
+  nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);
+
+  seenLinkingNumbers.forEach(currLinkingNumber => {
+    const linkedFields = recordGetFieldsWithSubfield8LinkingNumber(record, currLinkingNumber) //getFieldsWithSubfield8Index(base, baseIndex);
+    // As/If there's just one occurrence number it should be fine to use normalizeOccurrenceNumber = true
+    const normalizeOccurrenceNumber = true;
+    const linkedFieldsAsString = fieldsToNormalizedString(linkedFields, currLinkingNumber, normalizeOccurrenceNumber, true);
+    nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);
+    nvdebug(`${linkedFieldsAsString}`, debug);
+
+    if (linkedFieldsAsString in seen)  {
+      if (!removables.includes(linkedFieldsAsString)) {
+        removables.push(linkedFieldsAsString);
+      }
+
+      if (fix) {
+        nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);
+        linkedFields.forEach(field => recordRemoveFieldOrSubfield8(record, field, currLinkingNumber));
+        return;
+      }
+
+      nvdebug(`$8 VALIDATION: DUPLICATE DETECTED ${linkedFieldsAsString}`, debug);
+      return;
+    }
+    nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);
+    seen[linkedFieldsAsString] = 1;
+    return;
+  });
+
+  /* eslint-enable */
+  return removables;
+}
+
+
+export function handleDuplicateSubfield8Chains(record, fix) {
+
+  // Seen $8 subsfields in various fields:
+  // 161 700
+  // 17 710
+  // 11 110
+  // 8 730
+  // 1 100
+  // Given these stats, there's no need to check for 1XX-vs-7XX removals
+
+  /* eslint-disable */
+  let seen = {};
+
+  nvdebug("CHAIN-8");
+  const seenLinkingNumbers = recordGetAllSubfield8LinkingNumbers(record);
+  if (seenLinkingNumbers.length === 0) {
+    return;
+  }
+
+  nvdebug(`seen linking numbers ($8): ${seenLinkingNumbers.join(', ')}`, debug);
+
+  seenLinkingNumbers.forEach(currLinkingNumber => {
+    const linkedFields = recordGetFieldsWithSubfield8LinkingNumber(record, currLinkingNumber) //getFieldsWithSubfield8Index(base, baseIndex);
+    // As/If there's just one occurrence number it should be fine to use normalizeOccurrenceNumber = true
+    const normalizeOccurrenceNumber = false; //true;
+    const linkedFieldsAsString = fieldsToNormalizedString(linkedFields, currLinkingNumber, normalizeOccurrenceNumber, true);
+    nvdebug(`Results for LINKING NUMBER ${currLinkingNumber}:`, debug);
+    nvdebug(`${linkedFieldsAsString}`, debug);
+
+    if (linkedFieldsAsString in seen)  {
+      nvdebug(`$8 CHAIN FIX: REMOVE $8 GROUP: ${fieldsToString(linkedFields)}`, debug);
+      linkedFields.forEach(field => newRecordRemoveFieldOrSubfield8(record, field, currLinkingNumber, fix));
+      return;
+    }
+    nvdebug(`$8 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${linkedFieldsAsString}`, debug);
+    seen[linkedFieldsAsString] = 1;
+    return;
+  });
+
+  /* eslint-enable */
+
+}
+
+function markIdenticalSubfield6Chains(chain, record) {
+  const normalizeOccurrenceNumber = true;
+  const normalizeTag = chain.some(field => field.tag.substring(0, 1) === '1'); // 1XX can delete 7XX as well!
+  const chainAsString = fieldsToNormalizedString(chain, 0, normalizeOccurrenceNumber, normalizeTag);
+
+  nvdebug(`markIdenticalSubfield6Chains: ${chainAsString}`);
+  record.fields.forEach(f => compareWithChain(f));
+
+
+  function compareWithChain(f) {
+    nvdebug(`FIELD2CHAIN ${fieldToString(f)}`);
+    const otherChain = fieldToChain(f, record);
+    // Not a lone field or chain (head) or ... or is-same-chain
+    if (otherChain.length === 0 || sameField(chain[0], otherChain[0])) {
+      return;
+    }
+    const otherChainAsString = fieldsToNormalizedString(otherChain, 0, normalizeOccurrenceNumber, normalizeTag);
+
+    // Mark other chain as deleted:
+    if (chainAsString === otherChainAsString) {
+      otherChain.forEach(f => {
+        nvdebug(` mark ${fieldToString(f)} as deleted ($6-chain)...`);
+        f.deleted = 1; // eslint-disable-line functional/immutable-data
+      });
+      return;
+    }
+  }
+
+}
+
+function markIdenticalLoneFieldsAsDeletable(field, record) {
+  if (field.deleted) {
+    return;
+  }
+  // targetLinkingNumber = 0, normalizedOccurenceNumber = false, normalizeTag = true)
+  const normalizeTag = field.tag.substring(0, 1) === '1'; // 1XX can delete 7XX as well!
+  const fieldAsString = fieldToNormalizedString(field, 0, false, normalizeTag);
+
+  const identicalLoneFields = record.fields.filter(f => !sameField(f, field) && fieldToNormalizedString(f, 0, false, normalizeTag) === fieldAsString);
+
+  // Mark fields as deleted:
+  identicalLoneFields.forEach(f => {
+    nvdebug(` mark ${fieldToString(f)} as deleted (lone field)...`);
+    f.deleted = 1; // eslint-disable-line functional/immutable-data
+  });
+
+}
+
+function acceptFieldsWithSubfield6(fieldsWithSubfield6) {
+  // There can be only one non-880 field:
+  const non880 = fieldsWithSubfield6.filter(f => f.tag !== '880');
+  if (non880.length > 1) {
+    return false;
+  }
+
+  const occurrenceNumbers = fieldsGetOccurrenceNumbers(fieldsWithSubfield6);
+  // Chain can contain only single occurrence number:
+  if (occurrenceNumbers.length > 1) {
+    return false;
+  }
+
+  return true;
+}
+
+
+function isSingleTagLinkingNumber(linkingNumber, fields, tag) {
+  const relevantFields = fields.filter(f => fieldHasLinkingNumber(f, linkingNumber));
+  if (relevantFields.some(f => f.tag !== tag)) {
+    return false;
+  }
+  return true;
+}
+
+function acceptFieldsWithSubfield8(fieldsWithSubfield8, requireSingleTag = false) {
+  const linkingNumbers = fieldsGetAllSubfield8LinkingNumbers(fieldsWithSubfield8);
+  if (linkingNumbers.some(linkingNumber => anomaly8(linkingNumber))) {
+    return false;
+  }
+  return true;
+
+  // If linking number
+  function anomaly8(linkingNumber) {
+    nvdebug(`  Looking for anomalies in linkin number ${linkingNumber}`);
+    const relevantFields = fieldsWithSubfield8.filter(f => fieldHasLinkingNumber(f, linkingNumber));
+    if (requireSingleTag) {
+      return !isSingleTagLinkingNumber(linkingNumber, relevantFields, relevantFields[0].tag);
+    }
+
+    const f880 = relevantFields.filter(f => f.tag === '880');
+    if (f880.length === 0 || f880.length === relevantFields.length) {
+      return false;
+    }
+    return true;
+  }
+}
+
+
+export function sameField(field1, field2) {
+  /* eslint-disable */
+    field1.tmpMyId = 666;
+    const result = field2.tmpMyId === 666 ? true : false;
+    delete field1.tmpMyId;
+    /* eslint-enable */
+  return result;
+}
+
+export function isChainHead(field, chain) {
+  return sameField(field, chain[0]);
+}
+
+export function fieldToChain(field, record) {
+  if (field.deleted || !field.subfields) {
+    return [];
+  }
+  const chain = newGetAllLinkedFields(field, record, true, true);
+
+  nvdebug(` Chain contains ${chain.length} field(s)`);
+  if (!isChainHead(field, chain)) { // newGetAllLinkedFields() marks relevant record.fields!
+    return [];
+  }
+
+  const fieldsWithSubfield6 = chain.filter(f => fieldHasValidSubfield6(f));
+  // Hack: multiple $6 fields, but either all are non-880 or all are 880: treat field as a single entry
+  if (fieldsWithSubfield6.length > 0) {
+    const non880 = fieldsWithSubfield6.filter(f => f.tag !== '880');
+    if (non880.length === 0 || non880.length === fieldsWithSubfield6.length) {
+      return [field];
+    }
+    if (non880.length !== 1) {
+      return [field];
+    }
+  }
+
+  if (!acceptFieldsWithSubfield6(fieldsWithSubfield6)) { // Check tag subfield $6s are legal(ish)
+    return [];
+  }
+  const fieldsWithSubfield8 = chain.filter(f => fieldHasValidSubfield8(f));
+  if (!acceptFieldsWithSubfield8(fieldsWithSubfield8, false)) {
+    return [];
+  }
+
+  //nvdebug(`Proceed with ${fieldsToString(chain)}`);
+
+
+  return chain;
+}
+
+
+function fieldHandleDuplicateDatafields(field, record) {
+  const chain = fieldToChain(field, record);
+  nvdebug(` TRY TO HANDLE DUPLICATES OF '${fieldsToString(chain)}'`);
+
+  if (chain.length === 0) {
+    return;
+  }
+
+  const fieldsWithSubfield6 = chain.filter(f => fieldHasValidSubfield6(f));
+  const fieldsWithSubfield8 = chain.filter(f => fieldHasValidSubfield8(f));
+
+  // Lone fields:
+  if (chain.length === 1) {
+    markIdenticalLoneFieldsAsDeletable(chain[0], record);
+    return;
+  }
+  if (fieldsWithSubfield6.length === 0) {
+
+    if (fieldsWithSubfield8.length === 0) { // chain.length === 1?
+      nvdebug(` Trying to find duplicates of single field '${fieldToString(chain[0])}'`);
+      markIdenticalLoneFieldsAsDeletable(chain[0], record);
+      return;
+    }
+    const linkingNumbers = fieldsGetAllSubfield8LinkingNumbers(fieldsWithSubfield8);
+    if (linkingNumbers.length < 2) {
+      markIdenticalSubfield6Chains(chain, record);
+      return;
+    }
+  }
+
+  if (fieldsWithSubfield6.length > 0 && acceptFieldsWithSubfield8(fieldsWithSubfield8, true)) { // Checks that non-880 tags are all same
+    // Chain is removable
+    markIdenticalSubfield6Chains(chain, record);
+    return;
+  }
+
+
+  nvdebug(` NO HANDLER FOUND FOR '${fieldsToString(chain)}'`);
+  nvdebug(`  N8s: ${fieldsWithSubfield6.length}`);
+
+}
+
+
+export function removeDuplicateDatafields(record, fix = true) {
+  // Sometimes only $8 subfield (vs the whole field) is removed. Thus they are handled separately:
+  handleDuplicateSubfield8Chains(record, fix);
+
+  const dataFields = record.fields.filter(f => f.subfields !== undefined);
+
+  dataFields.forEach(f => fieldHandleDuplicateDatafields(f, record));
+
+  const deletableFields = dataFields.filter(f => f.deleted);
+  const modifiedFields = dataFields.filter(f => f.modified && !f.deleted);
+
+  const result = deletableFields.map(f => `DEL: ${fieldToString(f)}`);
+  if (modifiedFields.length) { // eslint-disable-line functional/no-conditional-statements
+    modifiedFields.forEach(f => delete f.modified); // eslint-disable-line functional/immutable-data
+    result.push(modifiedFields.map(f => `MOD: ${fieldToString(f)}`)); // eslint-disable-line functional/immutable-data
+  }
+
+  if (fix) {
+    deletableFields.forEach(f => record.removeField(f));
+    return result;
+  }
+
+  deletableFields.forEach(f => delete f.deleted); // eslint-disable-line functional/immutable-data
+  deletableFields.forEach(f => delete f.modified); // eslint-disable-line functional/immutable-data
+
+
+  return result;
+}

package/src/removeDuplicateDataFields.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './removeDuplicateDataFields';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'remove-duplicate-datafields'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/removeDuplicateDataFields:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}