@natlibfi/marc-record-validators-melinda 3.2.0 → 3.3.14-alpha.2

package/src/sortRelatorTerms.js

@@ -0,0 +1,142 @@
+// Validator/fixer for sorting $e relator term subfields
+//
+// Author(s): Nicholas Volk
+
+import clone from 'clone';
+//import createDebugLogger from 'debug';
+import {fieldToString} from './utils';
+import {fieldFixPunctuation} from './punctuation2';
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:sortRelatorTerms');
+//const debugData = debug.extend('data');
+
+const WORST_WORK = 98;
+
+const relatorTermValues = { // The higher, the better
+  // More abstract, the earlier it appears.
+  // Note that terms with same abstraction level might also have order preferences
+  // work/teos > expression/ekspressio > manifestation/manifestaatio
+  'säveltäjä': 100,
+  'kirjoittaja': 99, // Viola wants composer/säveltäjä on top (highly unlikely to ever appear together, but...)
+  'taiteilija': 98,
+  'sanoittaja': 90,
+  // ekspressio
+  'sovittaja': 80,
+  'toimittaja': 80,
+  'kuvittaja': 75,
+  'editointi': 71,
+  'kääntäjä': 70,
+  'lukija': 61,
+  // manifestaatio
+  'esittäjä': 60,
+  'johtaja': 50,
+  'kustantaja': 41,
+  'julkaisija': 40
+
+};
+
+function scoreRelatorTerm(term) {
+  const normalizedTerm = normalizeValue(term);
+  if (normalizedTerm in relatorTermValues) {
+    return relatorTermValues[normalizedTerm];
+  }
+  return 0;
+}
+
+export default function () {
+
+  return {
+    description: 'Sort adjacent $e subfields in field [1678][01]0',
+    validate, fix
+  };
+
+  function fix(record) {
+    const res = {message: [], fix: [], valid: true};
+
+    record.fields.forEach(field => {
+      sortAdjacentESubfields(field);
+    });
+
+    return res;
+  }
+
+  function validate(record) {
+    const res = {message: []};
+
+    record.fields.forEach(field => {
+      const clonedField = clone(field);
+      sortAdjacentESubfields(clonedField);
+      const clonedFieldAsString = fieldToString(clonedField);
+      const fieldAsString = fieldToString(field);
+      if (fieldAsString !== clonedFieldAsString) { // eslint-disable-line functional/no-conditional-statements
+        res.message.push(`${fieldAsString} => ${clonedFieldAsString}`); // eslint-disable-line functional/immutable-data
+      }
+    });
+
+    res.valid = !(res.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+}
+
+
+function normalizeValue(value) {
+  // Removing last punc char is good enough for our purposes.
+  // We don't handle abbreviations here etc.
+  // Brackets should not happen either, should they?
+  return value.replace(/[.,]$/u, '');
+}
+
+
+function swapESubfields(field) {
+  if (!field.subfields) {
+    return;
+  }
+
+  const loopAgain = field.subfields.some((sf, index) => {
+    if (index === 0 || sf.code !== 'e') {
+      return false;
+    }
+    const currScore = scoreRelatorTerm(sf.value);
+
+    const prevSubfield = field.subfields[index - 1];
+    if (currScore === 0 || prevSubfield.code !== 'e') {
+      return false;
+    }
+    const prevScore = scoreRelatorTerm(prevSubfield.value);
+
+
+    // If this subfield maps to a Work, then subfields can be swapped, even if we don't have a score for the prev subfield!
+    if (prevScore === 0 && currScore < WORST_WORK) {
+      return false;
+    }
+
+    if (currScore > prevScore) {
+      // Swap:
+      const tmp = field.subfields[index - 1];
+      field.subfields[index - 1] = sf; // eslint-disable-line functional/immutable-data
+      field.subfields[index] = tmp; // eslint-disable-line functional/immutable-data
+      fieldFixPunctuation(field);
+      return true;
+    }
+
+    return false;
+
+  });
+
+  if (loopAgain) {
+    swapESubfields(field); // uh, evil recursion...
+    return;
+  }
+
+  return;
+
+}
+
+export function sortAdjacentESubfields(field) {
+  if (!field.subfields) {
+    return field;
+  }
+  swapESubfields(field);
+
+  return field;
+}
+

package/src/sortRelatorTerms.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './sortRelatorTerms';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'sort-relator-terms'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sortRelatorTerms:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}

package/src/sortSubfields.js

@@ -0,0 +1,176 @@
+import clone from 'clone';
+import createDebugLogger from 'debug';
+import {fieldToString, nvdebug} from './utils';
+
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:sortSubfields');
+//const debugData = debug.extend('data');
+const debugDev = debug.extend('dev');
+
+const defaultSortOrderString = '8673abcdefghijklmnopqrstuvwxyz420159'; // NB! We Finns like $2 before $0 in 6XX...
+const defaultSortOrder = defaultSortOrderString.split('');
+
+
+// Author(s): Nicholas Volk
+export default function () {
+
+  return {
+    description: 'Swap adjacent subfields',
+    validate, fix
+  };
+
+  function fix(record) {
+    const res = {message: [], fix: [], valid: true};
+
+    record.fields.forEach(field => {
+      sortAdjacentSubfields(field);
+    });
+
+    return res;
+  }
+
+  function validate(record) {
+    const res = {message: []};
+
+    record.fields.forEach(field => {
+      const clonedField = clone(field);
+      sortAdjacentSubfields(clonedField);
+      const clonedFieldAsString = fieldToString(clonedField);
+      const fieldAsString = fieldToString(field);
+      if (fieldAsString !== clonedFieldAsString) { // eslint-disable-line functional/no-conditional-statements
+        res.message.push(clonedFieldAsString); // eslint-disable-line functional/immutable-data
+      }
+    });
+
+    res.valid = !(res.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+}
+
+
+// X00, X10, X11 and X130 could also for their own sets...
+const sortOrderForX10 = ['6', 'a', 'b', 't', 'n', 'e', 'v', 'w', '0', '5', '9']; // somewhat iffy
+const sortOrderFor7XX = ['8', '6', '7', 'i', 'a', 's', 't', 'b', 'c', 'd', 'm', 'h', 'k', 'o', 'x', 'z', 'g', 'q', 'w'];
+
+// List *only* exceptional order here. Otherwise default order is used.
+const subfieldSortOrder = [
+  {'tag': '017', 'sortOrder': ['i', 'a', 'b', 'd']},
+  {'tag': '028', 'sortOrder': ['b', 'a', 'q']}, // National convention
+  //{'tag': '031', 'sortOrder': ['a', 'b', 'c', 'm', 'e', 'd']}, // utter guesswork
+  {'tag': '040', 'sortOrder': ['8', '6', 'a', 'b', 'e', 'c', 'd', 'x']},
+  {'tag': '041', 'sortOrder': ['8', '6', 'a', 'd', 'j', 'p', 'h', 'e', 'g', 'm']}, // guesswork
+  {'tag': '048', 'sortOrder': ['8', '6', 'b', 'a']},
+  {'tag': '100', 'sortOrder': ['6', 'a', 'b', 'c', 'q', 'd', 'e', 'j', 't', 'u', 'l', 'f', '0', '5', '9']}, // don't do $g
+  {'tag': '110', 'sortOrder': sortOrderForX10},
+  {'tag': '111', 'sortOrder': ['a', 'n', 'd', 'c', 'e', 'g', 'j']},
+  {'tag': '130', 'sortOrder': ['a', 'n', 'p', 'k', 'l']},
+  {'tag': '240', 'sortOrder': ['a', 'm', 'n', 'p', 's', 'l', '2', '0', '1', '5', '9']},
+  {'tag': '245', 'sortOrder': ['6', 'a', 'b', 'n', 'p', 'k', 'f', 'c']},
+  {'tag': '246', 'sortOrder': ['i', 'a', 'n', 'p']},
+  {'tag': '382', 'sortOrder': ['a']},
+  {'tag': '385', 'sortOrder': ['8', 'm', 'n', 'a', '2', '0']},
+  {'tag': '386', 'sortOrder': ['8', 'm', 'n', 'a']},
+  {'tag': '490', 'sortOrder': ['a', 'x', 'y', 'v', 'l']},
+  {'tag': '505', 'sortOrder': ['a']},
+  {'tag': '526', 'sortOrder': ['i', 'a', 'b', 'x', 'z']},
+  {'tag': '540', 'sortOrder': ['a', 'b', 'c', 'd', 'f', '2', 'u']},
+  {'tag': '600', 'sortOrder': ['6', 'a', 'b', 'c', 'q', 'd', 'e', '0', '5', '9']},
+  {'tag': '610', 'sortOrder': sortOrderForX10},
+  {'tag': '611', 'sortOrder': ['a', 'n', 'd', 'c', 'e', 'g', 'j']},
+  {'tag': '700', 'sortOrder': ['6', 'i', 'a', 'b', 'c', 'q', 'd', 'e', 't', 'u', 'l', 'f', '0', '5', '9']},
+  {'tag': '710', 'sortOrder': sortOrderForX10},
+  {'tag': '711', 'sortOrder': ['a', 'n', 'd', 'c', 'e', 'g', 'j']},
+  {'tag': '760', 'sortOrder': sortOrderFor7XX},
+  {'tag': '762', 'sortOrder': sortOrderFor7XX},
+  {'tag': '765', 'sortOrder': sortOrderFor7XX},
+  {'tag': '767', 'sortOrder': sortOrderFor7XX},
+  {'tag': '770', 'sortOrder': sortOrderFor7XX},
+  {'tag': '772', 'sortOrder': sortOrderFor7XX},
+  {'tag': '773', 'sortOrder': sortOrderFor7XX},
+  {'tag': '774', 'sortOrder': sortOrderFor7XX},
+  {'tag': '775', 'sortOrder': sortOrderFor7XX},
+  {'tag': '776', 'sortOrder': sortOrderFor7XX},
+  {'tag': '777', 'sortOrder': sortOrderFor7XX},
+  {'tag': '780', 'sortOrder': sortOrderFor7XX},
+  {'tag': '785', 'sortOrder': sortOrderFor7XX},
+  {'tag': '786', 'sortOrder': sortOrderFor7XX},
+  {'tag': '787', 'sortOrder': sortOrderFor7XX},
+  {'tag': '788', 'sortOrder': sortOrderFor7XX},
+  {'tag': '800', 'sortOrder': ['i', 'a', 'b', 'c', 'q', 'd', 'e', 't', 'u', 'v', 'l', 'f', '0', '5', '9']},
+  {'tag': '810', 'sortOrder': sortOrderForX10},
+  {'tag': '811', 'sortOrder': ['a', 'n', 'd', 'c', 'e', 'g', 'j']},
+  {'tag': '830', 'sortOrder': ['a', 'n', 'x', 'v']}, // INCOMPLETE, SAME AS 490? APPARENTLY NOT...
+  {'tag': '880', 'sortOrder': ['6', 'a']},
+  {'tag': 'LOW', 'sortOrder': ['a', 'b', 'c', 'l', 'h']},
+  {'tag': 'SID', 'sortOrder': ['c', 'b']} // Hack, so that default order is not used
+];
+
+function getSubfieldSortOrder(field) {
+  const entry = subfieldSortOrder.filter(currEntry => field.tag === currEntry.tag);
+  if (entry.length > 0 && 'sortOrder' in entry[0]) {
+    debugDev(`sort order for ${field.tag}: ${entry[0].sortOrder}`);
+    return entry[0].sortOrder;
+  }
+  nvdebug(`NO DROPPABLE SUBFIELDS FOUND FOR ${field.tag}.`);
+  return [];
+}
+
+
+function swapSubfields(field, sortOrder) {
+  if (!field.subfields) {
+    return;
+  }
+
+  const loopAgain = field.subfields.some((sf, index) => {
+    if (index === 0) {
+      return false;
+    }
+    const currPos = getPosition(sf, sortOrder);
+    const prevPos = getPosition(field.subfields[index - 1], sortOrder);
+    if (currPos === -1 || prevPos === -1 || currPos >= prevPos) {
+      return false;
+    }
+    // Swap:
+    const tmp = field.subfields[index - 1];
+    field.subfields[index - 1] = sf; // eslint-disable-line functional/immutable-data
+    field.subfields[index] = tmp; // eslint-disable-line functional/immutable-data
+    return true;
+  });
+
+  if (loopAgain) {
+    return swapSubfields(field, sortOrder);
+  }
+
+  return;
+
+  function getPosition(subfield, sortOrder) {
+    // Magic exception that *always* comes first, used by Aleph in linking overlong fields
+    if (sortOrder.indexOf('9') > -1 && subfield.code === '9' && ['^', '^^'].includes(subfield.value)) {
+      return -0.5; // normal "best value" is 0, and "worst value" is N
+    }
+    return sortOrder.indexOf(subfield.code);
+  }
+}
+
+export function sortAdjacentSubfields(field, externalSortOrder = []) {
+  if (!field.subfields) {
+    return field;
+  }
+  // Features:
+  // - Swap only sort adjacent pairs.
+  // - No sorting over unlisted subfield codes. Thus a given subfield can not shift to wrong side of 700$t...
+
+  // Implement: 880 field should use values from $6...
+
+  // Should we support multiple sort orders per field?
+  const sortOrderForField = externalSortOrder.length > 0 ? externalSortOrder : getSubfieldSortOrder(field);
+  nvdebug(`INTERMEDIATE SUBFIELD ORDER FOR ${field.tag}: ${sortOrderForField.join(', ')}`);
+  const subfieldOrder = sortOrderForField.length > 0 ? sortOrderForField : defaultSortOrder;
+  nvdebug(`FINAL SUBFIELD ORDER FOR ${field.tag}: ${subfieldOrder.join(', ')}`);
+  //if (sortOrder === null) { return field; } //// Currently always sort..
+
+  swapSubfields(field, ['8', '6', '7', '3', 'a', '4', '2', '0', '1', '5', '9']); // <= Handle control subfield order (it never changes)
+  swapSubfields(field, subfieldOrder);
+
+  return field;
+}
+

package/src/sortSubfields.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './sortSubfields';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'sort-subfields'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sortSubfields:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}

package/src/stripPunctuation.js

@@ -0,0 +1,42 @@
+/*
+* stripPunctuation.js -- try and remove a marc field punctuation (based on and reverse of punctuation2.js)
+*
+* Author(s): Nicholas Volk <nicholas.volk@helsinki.fi>
+*
+*/
+
+import {fieldGetFixedString, fieldNeedsModification, fieldStripPunctuation} from './punctuation2';
+// import createDebugLogger from 'debug';
+import {fieldToString, nvdebug} from './utils';
+
+// const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/punctuation2');
+
+export default function () {
+  return {
+    description: 'Strip punctuation to data fields',
+    validate, fix
+  };
+
+  function fix(record) {
+    nvdebug('Strip punctuation to data fields: fixer');
+    const res = {message: [], fix: [], valid: true};
+    record.fields.forEach(f => fieldStripPunctuation(f));
+    return res;
+  }
+
+  function validate(record) {
+    nvdebug('Strip punctuation to data fields: validate');
+
+    const fieldsNeedingModification = record.fields.filter(f => fieldNeedsModification(f, false));
+
+    const values = fieldsNeedingModification.map(f => fieldToString(f));
+    const newValues = fieldsNeedingModification.map(f => fieldGetFixedString(f, false));
+
+    const messages = values.map((val, i) => `'${val}' => '${newValues[i]}'`);
+
+    const res = {message: messages};
+
+    res.valid = res.message.length < 1; // eslint-disable-line functional/immutable-data
+    return res;
+  }
+}

package/src/stripPunctuation.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './stripPunctuation';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'strip-punctuation'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/stripPunctuation:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}