@natlibfi/marc-record-validators-melinda 3.2.0 → 3.3.14-alpha.3

package/src/removeInferiorDataFields.js

@@ -0,0 +1,259 @@
+import createDebugLogger from 'debug';
+import {fieldToChain, sameField} from './removeDuplicateDataFields';
+import {fieldGetOccurrenceNumberPairs, fieldHasValidSubfield6, fieldSevenToOneOccurrenceNumber, fieldsToNormalizedString} from './subfield6Utils';
+import {fieldsToString, fieldToString, nvdebug} from './utils';
+import {fieldHasValidSubfield8} from './subfield8Utils';
+
+// Relocated from melinda-marc-record-merge-reducers (and renamed)
+
+// NB! This validator handles only full fields, and does not support subfield $8 removal.
+// Also, having multiple $8 subfields in same fields is not supported.
+// If this functionality is needed, see removeDuplicateDatafields.js for examples of subfield-only stuff.
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:removeSubsetDataFields');
+
+export default function () {
+  return {
+    description: 'Remove subset data fields. Certain exceptions apply, mainly too complited chained fields',
+    validate, fix
+  };
+
+  function fix(record) {
+    nvdebug('Fix record: remove subset data fields', debug);
+    const res = {message: [], fix: [], valid: true};
+    removeInferiorDatafields(record, true);
+    // This can not really fail...
+    return res;
+  }
+
+  function validate(record) {
+    // Check max, and check number of different indexes
+    nvdebug('Validate record: remove subset data fields', debug);
+
+    const duplicates = removeInferiorDatafields(record, false);
+
+    const res = {message: duplicates};
+
+    res.valid = res.message.length < 1; // eslint-disable-line functional/immutable-data
+    return res;
+  }
+}
+
+
+function deriveInferiorChains(fields, record) {
+  /* eslint-disable */
+  let deletableStringsObject = {};
+
+  nvdebug(`WP1: GOT ${fields.length} field(s) for potential deletable chain derivation`);
+  fields.forEach(field => fieldDeriveChainDeletables(field));
+
+  function fieldDeriveChainDeletables(field) {
+    const chain = fieldToChain(field, record);
+    if (chain.length === 0) {
+      return;
+    }
+    const chainAsString = fieldsToNormalizedString(chain, 0, true, true);
+
+    //nvdebug(`666: ${chainAsString}`);
+
+    // Fix MRA-476 (part 1): one $6 value can be worse than the other
+    let tmp = chainAsString;
+    while (tmp.match(/ ‡6 [0-9X][0-9][0-9]-(?:XX|[0-9]+)\/[^ ]+/u)) {
+      tmp = tmp.replace(/( ‡6 [0-9X][0-9][0-9]-(?:XX|[0-9]+))\/[^ ]+/u, '$1');
+      //nvdebug(`FFS: ${tmp}`);
+      deletableStringsObject[tmp] = field;
+    }
+
+    // Remove keepless versions:
+    tmp = chainAsString;
+    while (tmp.match(/ ‡9 [A-Z]+<KEEP>/)) {
+      tmp = tmp.replace(/ ‡9 [A-Z]+<KEEP>/, '');
+      deletableStringsObject[tmp] = field;
+      //nvdebug(`FFS: ${tmp}`);
+    }
+  }
+
+
+  /* eslint-enable */
+  return deletableStringsObject;
+}
+
+function isRelevantChain6(field, record) {
+  //Can't be a chain:
+  if (!fieldHasValidSubfield6(field) && !fieldHasValidSubfield8(field)) {
+    return false;
+  }
+  // Too short to be a chain:
+  const chain = fieldToChain(field, record);
+  if (chain.length < 2) {
+    return false;
+  }
+  // No field can contains no more than one subfield $6
+  if (chain.some(f => f.subfields.filter(sf => sf.code === '6').length > 1)) {
+    return false;
+  }
+
+  // Chainwise non-initial fields are not relevant as chains is handled through the initial/head field
+  /* eslint-disable */
+  field.tmpInferiorId = 666;
+  const result = chain[0].tmpInferiorId === 666 ? true : false;
+  delete field.tmpInferiorId;
+  /* eslint-enable */
+  return result;
+}
+
+export function removeInferiorChains(record, fix = true) {
+  const fields = record.fields.filter(f => isRelevantChain6(f, record));
+  //nvdebug(`WP2.0: GOT ${fields.length} chain(s)`);
+
+  const deletableChainsAsKeys = deriveInferiorChains(fields, record);
+  const nChains = Object.keys(deletableChainsAsKeys).length;
+  //nvdebug(`WP2: GOT ${nChains} chain(s)`);
+  if (nChains === 0) {
+    return [];
+  }
+
+  nvdebug(`removeInferiorChains() has ${fields.length} fields-in-chain(s), and a list of ${nChains} deletable(s)`);
+
+
+  /* eslint-disable */
+
+  let deletedStringsArray = [];
+  fields.forEach(f => innerRemoveInferiorChain(f));
+
+  function chainContains1XX(chain) {
+    return chain.some(f => f.tag.substring(0, 1) === '1');
+  }
+
+  function sevenToOne(field, chain) {
+    if (!['700', '710', '711', '730'].includes(field.tag)) {
+      return;
+    }
+    // Retag field 7XX as 1XX and fix corresponding occurrence numbers as well:
+    const pairs = fieldGetOccurrenceNumberPairs(field, chain);
+    field.tag = `1${field.tag.substring(1)}`; // eslint-disable-line functional/immutable-data
+    // There should always be one pair, but I'm not sanity-checking this
+    pairs.forEach(pairedField => fieldSevenToOneOccurrenceNumber(pairedField));
+  }
+
+  function innerRemoveInferiorChain(field) {
+    const chain = fieldToChain(field, record);
+    if (chain.length === 0 || !sameField(field, chain[0])) {
+      return;
+    }
+
+    // Better to keep inferior 1XX (vs better 7XX) than to delete 1XX!
+    if(chain.some(f => f.tag.substring(0, 1) === '1')) {
+      return;
+    }
+
+    const chainAsString = fieldsToNormalizedString(chain, 0, true, true);
+    if (chainAsString in deletableChainsAsKeys) {
+      const triggeringField = deletableChainsAsKeys[chainAsString];
+      const triggeringChain = fieldToChain(triggeringField, record);
+
+      // 1XX may be converted to XXX. However, it should not be removed.
+      // Better to keep inferior 1XX (vs better 7XX) than to delete 1XX!
+      if(chainContains1XX(chain)) {
+        if (chainContains1XX(triggeringChain)) {
+          // This should *never* happen, but keep this as a sanity check
+          return;
+        }
+        triggeringChain.forEach(f => sevenToOne(f, triggeringChain));
+      }
+      nvdebug(`iRIS6C: ${chainAsString}`);
+      const deletedString = fieldsToString(chain);
+      const message = `DEL: '${deletedString}'  REASON: '${fieldsToString(triggeringChain)}'`;
+      deletedStringsArray.push(message);
+      if (fix) {
+        nvdebug(`INFERIOR $6 CHAIN REMOVAL: ${message}}`, debug);
+        chain.forEach(currField => record.removeField(currField));
+      }
+    }
+  }
+
+  /* eslint-enable */
+  return deletedStringsArray;
+}
+
+function deriveIndividualDeletables(record) {
+  /* eslint-disable */
+  let deletableStringsArray = [];
+
+  record.fields.forEach(field => fieldDeriveIndividualDeletables(field));
+
+  function fieldDeriveIndividualDeletables(field) {
+    const fieldAsString = fieldToString(field);
+
+    // Proof-of-concept rule:
+    let tmp = fieldAsString;
+    if (field.tag.match(/^[1678]00$/u)) {
+      while (tmp.match(/, ‡e [^‡]+\.$/)) {
+        tmp = tmp.replace(/, ‡e [^‡]+\.$/, '.');
+        deletableStringsArray.push(tmp);
+      }
+    }
+    // MET-381: remove occurence number TAG-00, if TAG-NN existists
+    if (field.tag === '880') {
+      tmp = fieldAsString;
+      if (tmp.match(/ ‡6 [0-9][0-9][0-9]-([?:1-9][0-9]|0[1-9])/)) {
+        tmp = tmp.replace(/( ‡6 [0-9][0-9][0-9])-[0-9]+/, '$1-00');
+        nvdebug(`MET-381: ADD TO DELETABLES: ${tmp}`);
+        deletableStringsArray.push(tmp);
+        if (tmp.match(/ ‡6 [0-9][0-9][0-9]-00\/[^ ]+ /)) {
+          tmp = tmp.replace(/( ‡6 [0-9][0-9][0-9]-00)[^ ]+/, '$1');
+          nvdebug(`MET-381: ADD TO DELETABLES: ${tmp}`);
+          deletableStringsArray.push(tmp);
+        }
+      }
+    }
+
+
+
+    // Remove keepless versions:
+    tmp = fieldAsString;
+    while (tmp.match(/ ‡9 [A-Z]+<KEEP>/)) {
+      tmp = tmp.replace(/ ‡9 [A-Z]+<KEEP>/, '');
+      deletableStringsArray.push(tmp);
+    }
+  }
+  /* eslint-enable */
+  return deletableStringsArray; // we should do uniq!
+
+}
+
+
+export function removeIndividualInferiorDatafields(record, fix = true) { // No $6 nor $8 in field
+  const deletableFieldsAsStrings = deriveIndividualDeletables(record);
+  const hits = record.fields.filter(field => isDeletableField(field));
+
+  const deletedFieldsAsStrings = hits.map(f => fieldToString(f));
+
+  if (fix) { // eslint-disable-line functional/no-conditional-statements
+    hits.forEach(field => {
+      nvdebug(`Remove inferior field: ${fieldToString(field)}`);
+      record.removeField(field);
+    });
+  }
+
+  return deletedFieldsAsStrings;
+
+  function isDeletableField(field) {
+    const fieldAsString = fieldToString(field);
+    return deletableFieldsAsStrings.includes(fieldAsString);
+  }
+}
+
+
+export function removeInferiorDatafields(record, fix = true) {
+  const removables = removeIndividualInferiorDatafields(record, fix); // Lone fields
+  //const removables8 = removeDuplicateSubfield8Chains(record, fix); // Lone subfield $8 chains
+  const removables6 = removeInferiorChains(record, fix); // Lone subfield $6 chains
+  // HOW TO HANDLE $6+$8 combos?
+
+  nvdebug(`REMOVABLES:\n  ${removables.join('\n  ')}`);
+  nvdebug(`REMOVABLES 6:\n  ${removables6.join('\n  ')}`);
+
+  const removablesAll = removables.concat(removables6); //.concat(removables8);
+
+  return removablesAll;
+}

package/src/removeInferiorDataFields.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './removeInferiorDataFields';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'remove-inferior-datafields'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/removeInferiorDataFields:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}

package/src/resolvable-ext-references-melinda.js

@@ -1,125 +1,92 @@
-/**
- *
- * @licstart  The following is the entire license notice for the JavaScript code in this file.
- *
- * MARC record validators used in Melinda
- *
- * Copyright (C) 2014-2018 University Of Helsinki (The National Library Of Finland)
- *
- * This file is part of marc-record-validators-melinda
- *
- * marc-record-validators-melinda program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * marc-record-validators-melinda is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * @licend  The above is the entire license notice
- * for the JavaScript code in this file.
- *
- */
-
-/* eslint-disable require-await */
 import {parseString} from 'xml2js';
 import fetch from 'node-fetch';
-import {last} from 'lodash';
-
-export default async function ({endpoint, prefixPattern, fields}) {
-	if (typeof endpoint === 'string' && prefixPattern instanceof RegExp && typeof fields === 'object') {
-		return {
-			description: 'Checks if Melinda entity references are resolvable',
-			validate
-		};
-	}
-	throw new Error('Error in validation parameters');
-
-	async function validate(record) {
-		const validateResult = await validateRecord(record);
-		return validateResult;
-	}
-
-	function validateRecord(record) {
-		const removedPrefixes = [];
-
-		// Filter matching field keys from record.fields
-		const subfields = record.fields.reduce((prev, current) => {
-			Object.keys(fields).forEach(key => {
-				if (key === current.tag) {
-					prev.push(current);
-				}
-			});
-			return prev;
-		}, []);
-
-		// Filter matching objects from subfields
-		const matchingTags = [...subfields].reduce((prev, current) => {
-			Object.keys(fields).forEach(key => {
-				if (key === current.tag) {
-					current.subfields.filter(item => {
-						if (Object.values(fields[key]).filter(value => value === item.code)[0]) {
-							prev.push({tag: current.tag, code: item.code, value: item.value});
-						}
-						return prev;
-					});
-				}
-			});
-			return prev;
-		}, []);
-
-		// Matching prefixPattern is removed from object value field.
-		matchingTags.forEach(obj => {
-			if (prefixPattern.test(obj.value)) {
-				obj.value = obj.value.replace(prefixPattern, '');
-				removedPrefixes.push(obj);
-			}
-		});
-		return resolveValidation(removedPrefixes);
-	}
-
-	function resolveValidation(removedPrefixes) {
-		// If matching prefixPatterns found make an API call
-		if (removedPrefixes.length > 0) {
-			return validateMatchingTags(removedPrefixes).then(result => {
-				return result;
-			});
-		}
-		return {valid: true, messages: []};
-	}
-
-	async function validateMatchingTags(tags) {
-		const resolved = await Promise.all(tags.map(obj => {
-			return getData(obj.value).then(valid => {
-				return Object.assign({valid}, obj);
-			});
-		}));
-
-		if (resolved.every(value => value.valid === true)) {
-			return {valid: true, messages: []};
-		}
-
-		return {valid: false, messages: resolved.map(obj => `Field ${obj.tag}$${obj.code} with value ${obj.value} is not resolvable`)};
-	}
-
-	async function getData(recID) {
-		const queryParam = '?operation=searchRetrieve&maximumRecords=2&version=1&query=rec.id=';
-
-		const response = await fetch(`${endpoint}${queryParam}${recID}`);
-
-		const xml = await response.text();
-
-		return new Promise(resolve => {
-			parseString(xml, (err, result) => {
-				const record = last(result['zs:searchRetrieveResponse']['zs:records']);
-				const position = parseInt(last(record['zs:record'])['zs:recordPosition'][0], 10);
-				resolve(position === 1 && !err);
-			});
-		});
-	}
+
+export default function ({endpoint, prefixPattern, fields}) {
+  if (typeof endpoint === 'string' && prefixPattern instanceof RegExp && typeof fields === 'object') {
+    return {
+      description: 'Checks if Melinda entity references are resolvable',
+      validate
+    };
+  }
+
+  throw new Error('Error in validation parameters');
+
+  async function validate(record) {
+    const validateResult = await validateRecord(record);
+    return validateResult;
+  }
+
+  function validateRecord(record) {
+    const removedPrefixes = [];
+
+    // Filter matching field keys from record.fields
+    const subfields = record.fields.reduce((prev, current) => {
+      Object.keys(fields).forEach(key => {
+        if (key === current.tag) { // eslint-disable-line functional/no-conditional-statements
+          prev.push(current); // eslint-disable-line functional/immutable-data
+        }
+      });
+      return prev;
+    }, []);
+
+    // Filter matching objects from subfields
+    const matchingTags = [...subfields].reduce((prev, current) => {
+      Object.keys(fields).forEach(key => {
+        if (key === current.tag) { // eslint-disable-line functional/no-conditional-statements
+          current.subfields.filter(item => {
+            if (Object.values(fields[key]).filter(value => value === item.code)[0]) { // eslint-disable-line functional/no-conditional-statements
+              prev.push({tag: current.tag, code: item.code, value: item.value}); // eslint-disable-line functional/immutable-data
+            }
+
+            return prev;
+          });
+        }
+      });
+      return prev;
+    }, []);
+
+    // Matching prefixPattern is removed from object value field.
+    matchingTags.forEach(obj => {
+      if (prefixPattern.test(obj.value)) { // eslint-disable-line functional/no-conditional-statements
+        obj.value = obj.value.replace(prefixPattern, ''); // eslint-disable-line functional/immutable-data
+        removedPrefixes.push(obj); // eslint-disable-line functional/immutable-data
+      }
+    });
+    return resolveValidation(removedPrefixes);
+  }
+
+  function resolveValidation(removedPrefixes) {
+    // If matching prefixPatterns found make an API call
+    if (removedPrefixes.length > 0) {
+      return validateMatchingTags(removedPrefixes).then(result => result);
+    }
+
+    return {valid: true, messages: []};
+  }
+
+  async function validateMatchingTags(tags) {
+    const resolved = await Promise.all(tags.map(obj => getData(obj.value).then(valid => ({valid, ...obj}))));
+
+    if (resolved.every(value => value.valid === true)) {
+      return {valid: true, messages: []};
+    }
+
+    return {valid: false, messages: resolved.map(obj => `Field ${obj.tag}$${obj.code} with value ${obj.value} is not resolvable`)};
+  }
+
+  async function getData(recID) {
+    const queryParam = '?operation=searchRetrieve&maximumRecords=2&version=1&query=rec.id=';
+
+    const response = await fetch(`${endpoint}${queryParam}${recID}`);
+
+    const xml = await response.text();
+
+    return new Promise(resolve => {
+      parseString(xml, (err, result) => {
+        const record = result['zs:searchRetrieveResponse']['zs:records'].slice(-1)?.[0];
+        const position = parseInt(record?.['zs:record'].slice(-1)?.[0]['zs:recordPosition'][0], 10);
+        resolve(position === 1 && !err);
+      });
+    });
+  }
 }