@natlibfi/marc-record-validators-melinda 3.2.0 → 3.3.14-alpha.3

package/src/normalize-identifiers.js

@@ -0,0 +1,197 @@
+// Relocated from melinda-marc-record-merge-reducers (and renamed)
+//import createDebugLogger from 'debug';
+import clone from 'clone';
+import {fieldToString} from './utils';
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:normalizeIdentifiers');
+
+/*
+function nvdebug(message, func) {
+  if (func) { // eslint-disable-line functional/no-conditional-statements
+    func(message);
+  }
+  console.info(message); // eslint-disable-line no-console
+}
+*/
+
+export default function () {
+
+  // NB! We should and could handle ISNIs here as well.
+  return {
+    description: 'Normalizes identifiers such as subfield $0 values',
+    validate, fix
+  };
+
+  function fix(record) {
+    const res = {message: [], fix: [], valid: true};
+    //message.fix = []; // eslint-disable-line functional/immutable-data
+
+    // Actual parsing of all fields
+    /*
+      if (!record.fields) {
+        return false;
+      }
+      */
+
+    //nvdebug(`NORMALIZE CONTROL NUMBER FIX`, debug);
+    record.fields.forEach(field => {
+      //nvdebug(` NORMALIZE CONTROL NUMBER FIX ${fieldToString(field)}`, debug);
+
+      fieldNormalizeControlNumbers(field);
+      //validateField(field, true, message);
+    });
+
+    // message.valid = !(message.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+
+  function validate(record) {
+    const res = {message: []};
+    //nvdebug(`NORMALIZE CONTROL NUMBER VALIDATE`, debug);
+    // Actual parsing of all fields
+    /*
+      if (!record.fields) {
+        return false;
+      }
+      */
+
+    record.fields.forEach(field => {
+      //nvdebug(` NORMALIZE CONTROL NUMBER VALIDATE ${fieldToString(field)}`, debug);
+      validateField(field, res);
+    });
+
+    res.valid = !(res.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+
+  function validateField(field, res) {
+    if (!field.subfields) {
+      return;
+    }
+
+    const normalizedField = clone(field);
+    fieldNormalizeControlNumbers(normalizedField);
+
+    const orig = fieldToString(field);
+    const mod = fieldToString(normalizedField);
+    if (orig !== mod) { // Fail as the input is "broken"/"crap"/sumthing
+      res.message.push(`'${orig}' could do with control number identifier normalization`); // eslint-disable-line functional/immutable-data
+      return;
+    }
+
+    return;
+  }
+}
+
+const alephInternal01 = '(FIN01)';
+const alephInternal10 = '(FIN10)';
+const alephInternal11 = '(FIN11)';
+const alephInternal12 = '(FIN12)';
+const alephInternal13 = '(FIN13)';
+
+const standard01 = '(FI-MELINDA)';
+const standard10 = '(FI-ASTERI-S)';
+const standard11 = '(FI-ASTERI-N)';
+const standard12 = '(FI-ASTERI-A)';
+const standard13 = '(FI-ASTERI-W)';
+
+const both01 = {ALEPH_INTERNAL: alephInternal01, STANDARD: standard01};
+const both10 = {ALEPH_INTERNAL: alephInternal10, STANDARD: standard10};
+const both11 = {ALEPH_INTERNAL: alephInternal11, STANDARD: standard11};
+const both12 = {ALEPH_INTERNAL: alephInternal12, STANDARD: standard12};
+const both13 = {ALEPH_INTERNAL: alephInternal13, STANDARD: standard13};
+
+const mappings = {
+  'FCC': both01,
+  '(FI-ASTERI-A)': both12,
+  '(FI-ASTERI-N)': both11,
+  '(FI-ASTERI-S)': both10,
+  '(FI-ASTERI-W)': both13,
+  '(FI-MELINDA)': both01,
+  '(FIN01)': both01,
+  '(FIN10)': both10,
+  '(FIN11)': both11,
+  '(FIN12)': both12,
+  '(FIN13)': both13,
+  'http://urn.fi/URN:NBN:fi:au:finaf:': both11,
+  'https://urn.fi/URN:NBN:fi:au:finaf:': both11
+};
+
+function normalizeNineDigitIDs(value, targetFormat = 'ALEPH_INTERNAL') {
+  // $value should be prefix + nine-digits. Do nothing if nine-digit tail condition is not met:
+  const nineDigitTail = value.slice(-9);
+  if (!(/^[0-9]{9}$/u).test(nineDigitTail)) {
+    return value;
+  }
+  // Normalize prefix:
+  const currPrefix = value.slice(0, -9);
+
+  if (currPrefix in mappings) {
+    //nvdebug(`${currPrefix}, TF:${targetFormat}...`);
+    //nvdebug(`${JSON.stringify(mappings[currPrefix])}`);
+    return `${mappings[currPrefix][targetFormat]}${nineDigitTail}`;
+  }
+  return value;
+}
+
+export function isIsni(value) {
+  if ((/^\(isni\)(?: ?[0-9]{4}){4}$/u).test(value)) {
+    return true;
+  }
+  if ((/^https:\/\/isni.org\/isni\/[0-9]{16}$/u).test(value)) {
+    return true;
+  }
+  return false;
+}
+
+export function normalizeIsni(value) {
+  if (isIsni(value)) {
+    return `https://isni.org/isni/${value.replace(/[^0-9]/gu, '')}`;
+  }
+  return value;
+}
+
+
+export function normalizeControlSubfieldValue(value = '', targetFormat = 'ALEPH_INTERNAL') {
+  if (isIsni(value)) {
+    return normalizeIsni(value);
+  }
+
+  const normalizedValue = normalizeNineDigitIDs(value, targetFormat);
+  if (normalizedValue !== value) {
+    return normalizedValue;
+  }
+
+  // Something for isni IDs?
+  return value;
+}
+
+//export function normalizableSubfieldPrefix(tag, sf) {
+export function normalizeAs(tag, subfieldCode) {
+  //nvdebug(`nAs ${tag}, ${subfieldCode}`);
+  if (subfieldCode === '0' || subfieldCode === '1' || subfieldCode === 'w') {
+    return 'ALEPH_INTERNAL';
+  }
+
+  if (tag === '035' && ['a', 'z'].includes(subfieldCode)) {
+    return 'STANDARD';
+  }
+  return undefined;
+}
+
+export function fieldNormalizeControlNumbers(field) {
+  // Rename "Prefixes" as "ControlNumberIdentifiers"?
+  // No, since isni etc...  however, just "ControlNumber" would do...
+  // "identifiers" ?
+  if (!field.subfields) {
+    return;
+  }
+
+  field.subfields.forEach(sf => {
+    const targetFormat = normalizeAs(field.tag, sf.code);
+    if (targetFormat !== undefined) {
+      //nvdebug(`NORMALIZE SUBFIELD $${sf.code} IN FIELD: '${fieldToString(field)}' TO ${targetFormat}`);
+      sf.value = normalizeControlSubfieldValue(sf.value, targetFormat); // eslint-disable-line functional/immutable-data
+      return;
+    }
+  });
+}

package/src/normalize-identifiers.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './normalize-identifiers';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'normalize-identifiers'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-identifiers:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}

package/src/normalize-utf8-diacritics.js

@@ -0,0 +1,141 @@
+//import createDebugLogger from 'debug';
+import clone from 'clone';
+import {convert as nongenericNormalization} from './unicode-decomposition';
+import {fieldToString} from './utils';
+
+// Note that https://github.com/NatLibFi/marc-record-validators-melinda/blob/master/src/unicode-decomposition.js contains
+// similar functionalities. It's less generic and lacks diacritic removal but has it advantages as well.
+
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-utf-diacritics');
+
+// See also https://github.com/NatLibFi/marc-record-validators-melinda/blob/master/src/unicode-decomposition.js .
+// It uses a list of convertable characters whilst this uses a generic stuff as well.
+// It handles various '.' and '©' type normalizations as well.
+// NB! This version has minor bug/feature issue regarding fixComposition()
+
+// Author(s): Nicholas Volk
+export default function () {
+
+  return {
+    description: 'Generic normalization of latin UTF-8 diacritics. Precompose Finnish å, ä and ö. Decompose others.',
+    validate, fix
+  };
+
+  function fix(record) {
+    const res = {message: [], fix: [], valid: true};
+    //message.fix = []; // eslint-disable-line functional/immutable-data
+
+    // Actual parsing of all fields
+    /*
+    if (!record.fields) {
+      return false;
+    }
+    */
+
+    record.fields.forEach(field => {
+      fieldFixComposition(field);
+      //validateField(field, true, message);
+    });
+
+    // message.valid = !(message.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+
+  function validate(record) {
+    const res = {message: []};
+
+    // Actual parsing of all fields
+    /*
+    if (!record.fields) {
+      return false;
+    }
+    */
+
+    record.fields.forEach(field => {
+      validateField(field, res);
+    });
+
+    res.valid = !(res.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+
+  function validateField(field, res) {
+    if (!field.subfields) {
+      return;
+    }
+    const orig = fieldToString(field);
+
+    const normalizedField = fieldFixComposition(clone(field));
+    const mod = fieldToString(normalizedField);
+    if (orig !== mod) { // Fail as the input is "broken"/"crap"/sumthing
+      res.message.push(`'${orig}' requires normalization`); // eslint-disable-line functional/immutable-data
+      return;
+    }
+    return;
+  }
+}
+
+
+// Traditionally these six are precomposed and all the rest decomposed
+function precomposeFinnishLetters(value = '') {
+  return value.
+    replace(/å/gu, 'å').
+    replace(/ä/gu, 'ä').
+    replace(/ö/gu, 'ö').
+    replace(/Å/gu, 'Å').
+    replace(/Ä/gu, 'Ä').
+    replace(/Ö/gu, 'Ö');
+}
+
+function fixComposition(value = '') {
+  // Target: Diacritics use Melinda internal notation.
+  // General solution: Decompose everything and then compose 'å', 'ä', 'ö', 'Å', 'Ä' and 'Ö'.
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/normalize
+  // Bug/Feature: the generic normalize() function also normalizes non-latin encodings as well, is this ok?
+  // Exception: Input contains non-Latin script letters: don't decompose (see field 880 tests):
+  if (value.match(/[^\p{Script=Latin}\p{Script=Common}\p{Script=Inherited}]/u)) {
+    // Problem with this approach: mixed language content (eg. cyrillic + latin) won't get normalized.
+    // Hack/Damage control: we might add decomposition rules for most common diacritis here (eg. ü, é...).
+    // OR we could split input to words and handle them separately?
+    // NB! Hack not implemented yet. The main source of problematic case would probably be greek characters
+    // within texts, that are written with latin alphabet.
+    //return precomposeFinnishLetters(value);
+    return nongenericNormalization(value);
+  }
+  return precomposeFinnishLetters(String(value).normalize('NFD'));
+}
+
+
+export function fieldFixComposition(field) {
+  if (!field.subfields) {
+    return field;
+  }
+  //const originalValue = fieldToString(field);
+  //nvdebug(`fFC: '${originalValue}'`, debug);
+  field.subfields.forEach((subfield, index) => {
+    field.subfields[index].value = fixComposition(subfield.value); // eslint-disable-line functional/immutable-data
+  });
+  //const newValue = fieldToString(field);
+  //if (originalValue !== newValue) { // eslint-disable-line functional/no-conditional-statements
+  //  debug(`FIXCOMP: '${originalValue}' => '${newValue}'`);
+  //}
+  return field;
+}
+
+/*
+export function fieldRemoveDecomposedDiacritics(field) {
+  // Raison d'être/motivation: "Sirén" and diacriticless "Siren" might refer to a same surname, so this normalization
+  // allows us to compare authors and avoid duplicate fields.
+  field.subfields.forEach((sf) => {
+    sf.value = removeDecomposedDiacritics(sf.value); // eslint-disable-line functional/immutable-data
+  });
+
+  function removeDecomposedDiacritics(value = '') {
+    // NB #1: Does nothing to precomposed letters. String.normalize('NFD') can handle them.
+    // NB #2: Finnish letters 'å', 'ä', 'ö', 'Å', Ä', and 'Ö' should be handled (=precomposed) before calling this.
+    // NB #3: Calling our very own fixComposition() before this function handles both #1 and #2.
+    return String(value).replace(/\p{Diacritic}/gu, '');
+  }
+}
+*/
+

package/src/normalize-utf8-diacritics.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './normalize-utf8-diacritics';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'normalize-utf8-diacritics'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/normalize-utf8-diacritics:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}