@natlibfi/marc-record-validators-melinda 3.2.0 → 3.3.14-alpha.2

package/src/mergeField500Lisapainokset.js

@@ -0,0 +1,153 @@
+export default function () {
+
+  return {
+    description: 'Merge 500 $a Lisäpainokset fields',
+    validate, fix
+  };
+
+  function fix(record) {
+    const res = {message: [], fix: [], valid: true};
+    mergeLisapainokset(record);
+    return res;
+  }
+
+  function validate(record) {
+    const relevantFields = getRelevantFields(record);
+    if (relevantFields.length < 2) {
+      return {message: [], 'valid': true}; // No action required
+    }
+
+    const printData = extractAllPrintData(relevantFields);
+    if (printData.length === 0) {
+      return {message: ['There are issues, but the fixer can not fix them!'], 'valid': false}; // No action required
+    }
+
+    return {message: [`Fixer can merge ${relevantFields.length} 500 $a Lisäpainokset fields into one`], 'valid': false};
+  }
+}
+
+
+function getRelevantFields(record) {
+  return record.fields.filter(field => validLisapainosField(field));
+}
+
+function validLisapainosField(field) {
+  // We are only interested in field 500 with a lone $a subfield.
+  // Especially $9 FENNI<KEEP> should not be merged!
+  if (field.tag !== '500' || field.subfields.length !== 1 || field.subfields[0].code !== 'a') {
+    return false;
+  }
+  return field.subfields[0].value.match(/^(?:Lisäpainokset|Lisäpainos): (?:[1-9][0-9]*\. (?:p\.|painos|uppl\.) [0-9]+\.)(?: - [1-9][0-9]*\. (?:p\.|painos|uppl\.) \[?[0-9]+\]?\.)*$/u);
+}
+
+function fieldToPrintsString(field) {
+  // Could this just be something on the lines of s/^\S+ // ?
+  return field.subfields[0].value.replace(/^(?:Lisäpainokset|Lisäpainos): /u, '').replace(/\.$/u, '');
+}
+
+const printPreference = ['painos', 'p.', 'upplaga', 'uppl.'];
+function getPrintPreference(value) {
+  return printPreference.findIndex(pp => pp === value);
+}
+
+function mergePrintData(value1, value2) {
+  const [index1, print1, year1] = value1.split(' ');
+  const [index2, print2, year2] = value2.split(' ');
+
+  const betterIndex = index1 ? index1 : index2; // just to cheat eslint...
+
+  // merge print1 and print2
+  const betterPrint = getBetterPrint(print1, print2);
+  if (!betterPrint) {
+    return null;
+  }
+
+  const betterYear = getBetterYear(year1, year2);
+  if (!betterYear) {
+    return null;
+  }
+
+  return `${betterIndex} ${betterPrint} ${betterYear}`;
+
+  function getBetterYear(y1, y2) {
+    if (y1 === y2 || y2 === `[${y1}]`) {
+      return y1;
+    }
+    if (y1 === `[${y2}]`) {
+      return y2;
+    }
+    return null;
+  }
+
+  function getBetterPrint(print1, print2) {
+    if (print1 === print2) {
+      return print1;
+    }
+
+    const i1 = getPrintPreference(print1);
+    const i2 = getPrintPreference(print2);
+    if (i1 === -1 || i2 === -1) {
+      return null;
+    }
+    if (i1 <= i2) {
+      return printPreference[i1];
+    }
+    return printPreference[i2];
+  }
+
+}
+
+
+function extractAllPrintData(relevantFields) {
+  /* eslint-disable */
+  // Gather data about 500 $a Lisäpainokset.*
+  let allPrintData = [];
+  let i;
+  let j;
+  for (i=0; i < relevantFields.length; i++) {
+    const value = fieldToPrintsString(relevantFields[i]);
+    const fieldsPrintData = value.split('. - ');
+    for (j=0; j < fieldsPrintData.length; j++) {
+      let currPrintData = fieldsPrintData[j];
+      // Example value: "2. p. 2020"
+      const [ printIndex ] = currPrintData.split('.');
+      if (allPrintData[printIndex] !== undefined) {
+        if (allPrintData[printIndex] !== currPrintData ) {
+          const mergedPrintData = mergePrintData(allPrintData[printIndex], currPrintData);
+          if (!mergedPrintData) {
+            return []; // reason for for-loops: exit function from within nested loops
+          }
+          currPrintData = mergedPrintData;
+        }
+      }
+      allPrintData[printIndex] = currPrintData;
+    }
+  };
+  return allPrintData.filter(p => p !== undefined);
+}
+
+
+export function mergeLisapainokset(record) {
+  const relevantFields = getRelevantFields(record);
+  if (relevantFields.length < 2) {
+    return;
+  }
+
+  /* eslint-disable */
+  const collapsedArray = extractAllPrintData(relevantFields);
+  if (collapsedArray.length === 0) {
+    return;
+  }
+
+  const content = "Lisäpainokset: " + collapsedArray.join('. - ') + ".";
+
+  relevantFields[0].subfields[0].value = content; // Keep the place
+
+  relevantFields.forEach((field, index) => {
+    if (index > 0) {
+      record.removeField(field);
+      return;
+    }
+  });
+  /* eslint-enable */
+}

package/src/mergeField500Lisapainokset.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './mergeField500Lisapainokset';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'lisapainokset'],
+  useMetadataFile: true,
+  recurse: true,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/mergeField500Lisapainokset:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}

package/src/mergeRelatorTermFields.js

@@ -0,0 +1,143 @@
+// Merge author/agent fields
+//
+// Rationale: Same author can appear in one 1XX and multiple 7XX fields having only different $e subfields.
+// These fields can be merged (and $e-subfields can then be sorted)...
+//
+// Author(s): Nicholas Volk
+
+
+import clone from 'clone';
+import {fieldFixPunctuation, fieldStripPunctuation} from './punctuation2';
+import {fieldToString, nvdebug} from './utils';
+import {sortAdjacentSubfields} from './sortSubfields';
+import {sortAdjacentESubfields} from './sortRelatorTerms';
+//import createDebugLogger from 'debug';
+/*
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:mergeRelatorTermFields');
+//const debugData = debug.extend('data');
+*/
+
+export default function () {
+
+  return {
+    description: 'Merge author fields that only differ in $e relator terms',
+    validate, fix
+  };
+
+  function fix(record) {
+    const msg = mergeRelatorTermFields(record, true);
+    const res = {message: msg, fix: msg, valid: true};
+    return res;
+  }
+
+  function validate(record) {
+    const msg = mergeRelatorTermFields(record, false);
+    const res = {message: msg};
+
+    res.valid = !(res.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+}
+
+function createNormalizedClone(field) {
+  const clonedField = clone(field);
+  // Normalize
+  fieldStripPunctuation(clonedField);
+  return clonedField;
+}
+
+function createNormalizedCloneWithoutRelatorTerms(field) {
+  const clonedField = createNormalizedClone(field);
+  // Remove relator terms $e subfi:
+  clonedField.subfields = clonedField.subfields.filter(sf => sf.code !== 'e'); // eslint-disable-line functional/immutable-data
+  return clonedField;
+}
+
+function fieldToRelatorTermSubfieldCode(field) {
+  if (['100', '110', '700', '710', '720', '751', '752'].includes(field.tag)) {
+    return 'e';
+  }
+  if (field.tag === '111' || field.tag === '711') {
+    return 'j';
+  }
+  return '?'; // No need to complain. Nothing is found.
+}
+
+function getRelatorTermStrings(relatorTermSubfieldCode, field) {
+  return field.subfields.filter(sf => sf.code === relatorTermSubfieldCode).map(sf => sf.value);
+
+}
+
+function extractAddableRelatorTerms(fromField, toField) {
+  const relatorTermSubfieldCode = fieldToRelatorTermSubfieldCode(fromField);
+  const normalizedFromFieldRelatorTerms = getRelatorTermStrings(relatorTermSubfieldCode, fromField);
+  if (normalizedFromFieldRelatorTerms.length === 0) {
+    return [];
+  }
+  // Remove values that already exist:
+  const normalizedToFieldRelatorTerms = getRelatorTermStrings(relatorTermSubfieldCode, toField);
+  return normalizedFromFieldRelatorTerms.filter(str => !normalizedToFieldRelatorTerms.includes(str));
+}
+
+
+function copyRelatorSubfields(fromField, toField) {
+  const relatorTermSubfieldCode = fieldToRelatorTermSubfieldCode(fromField);
+  const newRelatorTerms = extractAddableRelatorTerms(fromField, toField);
+
+  newRelatorTerms.forEach(term => toField.subfields.push({code: relatorTermSubfieldCode, value: term})); // eslint-disable-line functional/immutable-data
+
+}
+
+function mergeRelatorTermFields(record, fix = false) {
+  /* eslint-disable */
+  // NV: 111/711, 751 and 752 where so rare that I did not add them here
+  let fields = record.get('(?:[17][01]0|720)'); 
+  let result = [];
+  const comparisonFieldsAsString = fields.map(f => fieldToString(createNormalizedCloneWithoutRelatorTerms(f)));
+
+  nvdebug(`mergeRelatorTermFields(): ${fields.length} cand field(s) found`);
+  for(let i=0; i < fields.length-1; i++) {
+    let currField = fields[i];
+    if (currField.deleted) {
+      continue;
+    }
+    nvdebug(`RT: Trying to pair ${comparisonFieldsAsString[i]}/${i}`);
+    for (let j=i+1; j < fields.length; j++) {
+      nvdebug(` Compare with ${comparisonFieldsAsString[j]}/${j}`);
+      let mergableField = fields[j];
+      // Skip 1/7 from 1XX/7XX for similarity check:
+      if ( comparisonFieldsAsString[i].substring(1) !== comparisonFieldsAsString[j].substring(1)) {
+        nvdebug("  NOT PAIR");
+        continue;
+      }
+      if (mergableField.deleted) {
+        nvdebug("  DELETED");
+        continue;
+      }
+      const str = `MERGE RELATOR TERM FIELD: ${fieldToString(mergableField)}`;
+      nvdebug(str);
+
+      if(!result.includes(str)) {
+        result.push(str)
+      }
+
+      if (fix) {
+        mergableField.deleted = 1;
+        copyRelatorSubfields(mergableField, currField);
+        fieldFixPunctuation(currField);
+        sortAdjacentSubfields(currField); // Put the added $e subfield to proper places.
+        sortAdjacentESubfields(currField); // Sort $e subfields with each other
+        fieldFixPunctuation(currField);
+
+      }
+    }
+  }
+
+  if(!fix) {
+    fields.forEach(f => delete f.deleted);
+  }
+
+  record.fields = record.fields.filter(f => !f.deleted);
+  /* eslint-enable */
+  return result;
+}

package/src/mergeRelatorTermFields.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './mergeRelatorTermFields';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'mergeRelatorTermFields'],
+  useMetadataFile: true,
+  recurse: true,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/mergeRelatorTermFields:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}

package/src/multiple-subfield-0.js

@@ -0,0 +1,129 @@
+// import createDebugLogger from 'debug';
+// import clone from 'clone';
+// const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:multiple-subfield-0');
+
+import {fieldHasSubfield, fieldToString} from './utils';
+
+const asteriNamePrefixes = ['(FI-ASTERI-N)', '(FIN11)', 'http://urn.fi/URN:NBN:fi:au:finaf:', 'https://urn.fi/URN:NBN:fi:au:finaf:'];
+
+export default function () {
+
+  return {
+    description: 'If Asteri subfield $0 is found, remove non-Asteri $0 subfields',
+    validate, fix
+  };
+
+  function fix(record) {
+    function fixField(field) {
+      const removableSubfields = fieldGetDeletableSubfields(field);
+      removableSubfields.forEach(sf => record.removeSubfield(sf, field));
+    }
+
+    const res = {message: [], fix: [], valid: true};
+
+    const relevantFields = getRelevantFields(record);
+
+    relevantFields.forEach(field => fixField(field));
+
+    // message.valid = !(message.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+
+  function validate(record) {
+    function validateField(field) {
+      const relevantSubfields = fieldGetDeletableSubfields(field);
+      if (relevantSubfields.length === 0) {
+        return 'TROUBLE';
+      }
+      return `Field '${fieldToString(field)}' contains deletable $0 subfield(s): ${relevantSubfields.map(sf => sf.value).join(', ')}`;
+    }
+    const relevantFields = getRelevantFields(record);
+    const messages = relevantFields.map(field => validateField(field));
+    const res = {message: messages};
+    res.valid = !(res.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+
+  function fieldGetSubfields(field, code) {
+    return field.subfields.filter(sf => sf.code === code);
+  }
+
+  function isDeletableNamePartID(value) {
+    // List here $0s that always refer to name part, and to never to title part
+    if (value.match(/(?:isni|orcid)/ui)) {
+      return true;
+    }
+    return false;
+  }
+
+  function isAsteriNameId(value) { // This is true if have a valid Asteri entry (nine digits etc)
+    const nineDigitTail = value.slice(-9);
+    if (!(/^[0-9]{9}$/u).test(nineDigitTail)) {
+      return false;
+    }
+    // Normalize prefix:
+    const currPrefix = value.slice(0, -9);
+
+    if (asteriNamePrefixes.includes(currPrefix)) {
+      return true;
+    }
+    return false;
+  }
+
+  function neverDropThisID(value) {
+    if (isAsteriNameId(value)) {
+      return true;
+    }
+
+    const prefixes = ['(FIN', '(FI-'];
+    if (prefixes.some(prefix => value.startsWith(prefix))) {
+      return true;
+    }
+
+    return false;
+  }
+
+
+  function fieldHasTitlePart(field) {
+    if (['600', '610', '700', '710', '800', '810'].includes(field.tag)) {
+      if (fieldHasSubfield(field, 't')) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  function fieldGetDeletableSubfields(field) {
+    const subfield0s = fieldGetSubfields(field, '0');
+
+    if (subfield0s.length < 2) {
+      return []; // We have nothing to delete
+    }
+
+    // Field must contain non-Asteri subfields and Asteri subfiels.
+    const nonAsteriNameSubfields = subfield0s.filter(sf => !isAsteriNameId(sf.value));
+    if (nonAsteriNameSubfields.length === 0 || nonAsteriNameSubfields.length === subfield0s.length) {
+      return [];
+    }
+
+    const suspiciousSubfields = nonAsteriNameSubfields.filter(sf => !neverDropThisID(sf.value));
+
+    // Field has deletable name part $0s:
+    const otherKnownNamePartIdentifiers = suspiciousSubfields.filter(sf => isDeletableNamePartID(sf.value));
+
+    if (fieldHasTitlePart(field)) {
+      return otherKnownNamePartIdentifiers;
+    }
+
+    return suspiciousSubfields;
+  }
+
+  function fieldIsRelevant(field) {
+    const subfields = fieldGetDeletableSubfields(field);
+    return subfields.length > 0;
+  }
+
+  function getRelevantFields(record) {
+    return record.fields.filter(field => field.subfields && fieldIsRelevant(field));
+  }
+}

package/src/multiple-subfield-0.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './multiple-subfield-0';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'subfield0'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/multiple-subfield-0:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}

package/src/non-breaking-space.js

@@ -0,0 +1,49 @@
+/**
+ * Provides interface to validate and fix record fields, which include subfields that contain no-breaking space character
+ * @returns {Object} Object containing interfaces required by marc-record-validators-melinda package
+ */
+export default function () {
+  return {
+    description: 'Handles subfields that contains non-breaking space character',
+    validate,
+    fix
+  };
+
+  function validate(record) {
+    const nonValidFields = record.fields.filter((field) => collectNonValidFields(field));
+
+    const valid = nonValidFields.length === 0;
+    const messages = nonValidFields.flatMap(({tag, subfields}) => subfields.map(sf => `Field ${tag} subfield $${sf.code} contains non-breaking space character(s)`));
+
+    return valid ? {valid, messages: []} : {valid, messages};
+
+    function collectNonValidFields(field) {
+      if (field.value) {
+        return (/\u00A0/u).test(field.value);
+      }
+
+      return field.subfields.filter(valueContainsNonBreakingSpace).length > 0;
+    }
+  }
+
+  /* eslint-disable functional/immutable-data,functional/no-conditional-statements */
+  function fix(record) {
+    record.fields.forEach((field) => {
+      if (field.value) {
+        field.value = field.value.replaceAll(/\u00A0/gu, ' ');
+        return;
+      }
+
+      field.subfields.forEach(subfield => {
+        if (valueContainsNonBreakingSpace(subfield)) {
+          subfield.value = subfield.value.replaceAll(/\u00A0/gu, ' ');
+        }
+      });
+    });
+  }
+  /* eslint-enable functional/immutable-data,functional/no-conditional-statements */
+
+  function valueContainsNonBreakingSpace({value}) {
+    return (/\u00A0/u).test(value);
+  }
+}

package/src/non-breaking-space.spec.js

@@ -0,0 +1,44 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './non-breaking-space';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'non-breaking-space'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, fix = false}) {
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}