@natlibfi/marc-record-validators-melinda 3.2.0 → 3.3.14-alpha.3

package/src/resolvable-ext-references-melinda.spec.js

@@ -1,39 +1,8 @@
-/**
- *
- * @licstart  The following is the entire license notice for the JavaScript code in this file.
- *
- * MARC record validators used in Melinda
- *
- * Copyright (C) 2014-2018 University Of Helsinki (The National Library Of Finland)
- *
- * This file is part of marc-record-validators-melinda
- *
- * marc-record-validators-melinda program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * marc-record-validators-melinda is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * @licend  The above is the entire license notice
- * for the JavaScript code in this file.
- *
- */
-
-/* eslint-disable no-undef, max-nested-callbacks, no-unused-expressions */
-
-'use strict';
 import chai from 'chai';
 import chaiAsPromised from 'chai-as-promised';
 import {MarcRecord} from '@natlibfi/marc-record';
 import fetchMock from 'fetch-mock';
-import * as testContext from '../src/resolvable-ext-references-melinda'; /* eslint-disable-line import/named */
+import * as testContext from '../src/resolvable-ext-references-melinda';
 import {fixture5000, fixture9550, fixture1000} from '../test-fixtures/resolvable-ext-references-melinda';
 
 const {expect} = chai;
@@ -41,174 +10,175 @@ chai.use(chaiAsPromised);
 
 const endpoint = 'http://melinda.kansalliskirjasto.fi:210/fin01';
 const queryParam = '?operation=searchRetrieve&maximumRecords=2&version=1&query=rec.id=';
-const prefixPattern = /^\(FOOBAR\)/;
+const prefixPattern = /^\(FOOBAR\)/u;
 const fields = {
-	773: ['w'],
-	833: ['w', 'p']
+  773: ['w'],
+  833: ['w', 'p']
 };
 
 describe('resolvable-ext-references-melinda', () => {
-	afterEach(() => {
-		testContext.default.__ResetDependency__('fetch');
-	});
-
-	it('Creates a validator', async () => {
-		const validator = await testContext.default({endpoint, prefixPattern, fields});
-
-		expect(validator)
-			.to.be.an('object')
-			.that.has.any.keys('description', 'validate');
-
-		expect(validator.description).to.be.a('string');
-		expect(validator.validate).to.be.a('function');
-	});
-
-	it('Throws an error when prefixPattern not provided', async () => {
-		const validator = await testContext.default({endpoint, prefixPattern, fields});
-		await expect(validator.validate()).to.be.rejectedWith(Error, 'Cannot read property \'fields\' of undefined');
-	});
-
-	describe('#validate', () => {
-		it('Finds prefixPattern on record and removes it', async () => {
-			const mock = fetchMock.sandbox();
-			mock.get(`${endpoint}${queryParam}5000`, fixture5000);
-			mock.get(`${endpoint}${queryParam}9550`, fixture9550);
-
-			testContext.default.__Rewire__('fetch', mock);
-			const validator = await testContext.default({endpoint, prefixPattern, fields});
-
-			const record = new MarcRecord({
-				fields: [
-					{
-						tag: '001',
-						value: '123456'
-					},
-					{
-						tag: '035',
-						subfields: [
-							{
-								code: 'a',
-								value: '(FI-MELINDA)123456'
-							}
-						]
-					},
-					{
-						tag: '773',
-						subfields: [
-							{
-								code: 'w',
-								value: '(FOOBAR)5000'
-							}
-						]
-					},
-					{
-						tag: '833',
-						subfields: [
-							{
-								code: 'p',
-								value: '(FOOBAR)9550'
-							},
-							{
-								code: 'c',
-								value: '(FI-MELINDA)8850'
-							}
-						]
-					}
-				]
-			});
-			const result = await validator.validate(record);
-
-			expect(result).to.eql({valid: true, messages: []});
-		});
-
-		it('Finds no matching prefixPattern on record', async () => {
-			const mock = fetchMock.sandbox();
-
-			mock.get(`${endpoint}5000`, fixture5000);
-
-			testContext.default.__Rewire__('fetch', mock);
-			const validator = await testContext.default({endpoint, prefixPattern, fields});
-
-			const record = new MarcRecord({
-				fields: [
-					{
-						tag: '001',
-						value: '123456'
-					},
-					{
-						tag: '035',
-						subfields: [
-							{
-								code: 'a',
-								value: '(FI-MELINDA)123456'
-							}
-						]
-					},
-					{
-						tag: '773',
-						subfields: [
-							{
-								code: 'w',
-								value: '(FI-MELINDA)123456'
-							}
-						]
-					},
-					{
-						tag: '833',
-						subfields: [
-							{
-								code: 'p',
-								value: '(FI-MELINDA)2620'
-							},
-							{
-								code: 'w',
-								value: '(FI-MELINDA)8850'
-							}
-						]
-					}
-				]
-			});
-			const result = await validator.validate(record);
-
-			expect(result).to.eql({valid: true, messages: []});
-		});
-
-		it('Finds prefixPattern on record but values not resolvable', async () => {
-			const mock = fetchMock.sandbox();
-			mock.get(`${endpoint}${queryParam}1000`, fixture1000);
-
-			testContext.default.__Rewire__('fetch', mock);
-			const validator = await testContext.default({endpoint, prefixPattern, fields});
-
-			const record = new MarcRecord({
-				fields: [
-					{
-						tag: '001',
-						value: '123456'
-					},
-					{
-						tag: '035',
-						subfields: [
-							{
-								code: 'a',
-								value: '(FI-MELINDA)123456'
-							}
-						]
-					},
-					{
-						tag: '773',
-						subfields: [
-							{
-								code: 'w',
-								value: '(FOOBAR)1000'
-							}
-						]
-					}
-				]
-			});
-			const result = await validator.validate(record);
-
-			expect(result).to.eql({valid: false, messages: ['Field 773$w with value 1000 is not resolvable']});
-		});
-	});
+  afterEach(() => {
+    testContext.default.__ResetDependency__('fetch');
+  });
+
+  it('Creates a validator', async () => {
+    const validator = await testContext.default({endpoint, prefixPattern, fields});
+
+    expect(validator)
+      .to.be.an('object')
+      .that.has.any.keys('description', 'validate');
+
+    expect(validator.description).to.be.a('string');
+    expect(validator.validate).to.be.a('function');
+  });
+
+  it('Throws an error when prefixPattern not provided', async () => {
+    const validator = await testContext.default({endpoint, prefixPattern, fields});
+    // Cannot read property 'fields' of undefined or Cannot read properties of undefined (reading 'fields')'
+    await expect(validator.validate()).to.be.rejectedWith(Error, /^Cannot read propert/u);
+  });
+
+  describe('#validate', () => {
+    it('Finds prefixPattern on record and removes it', async () => {
+      const mock = fetchMock.sandbox();
+      mock.get(`${endpoint}${queryParam}5000`, fixture5000);
+      mock.get(`${endpoint}${queryParam}9550`, fixture9550);
+
+      testContext.default.__Rewire__('fetch', mock);
+      const validator = await testContext.default({endpoint, prefixPattern, fields});
+
+      const record = new MarcRecord({
+        fields: [
+          {
+            tag: '001',
+            value: '123456'
+          },
+          {
+            tag: '035',
+            subfields: [
+              {
+                code: 'a',
+                value: '(FI-MELINDA)123456'
+              }
+            ]
+          },
+          {
+            tag: '773',
+            subfields: [
+              {
+                code: 'w',
+                value: '(FOOBAR)5000'
+              }
+            ]
+          },
+          {
+            tag: '833',
+            subfields: [
+              {
+                code: 'p',
+                value: '(FOOBAR)9550'
+              },
+              {
+                code: 'c',
+                value: '(FI-MELINDA)8850'
+              }
+            ]
+          }
+        ]
+      });
+      const result = await validator.validate(record);
+
+      expect(result).to.eql({valid: true, messages: []});
+    });
+
+    it('Finds no matching prefixPattern on record', async () => {
+      const mock = fetchMock.sandbox();
+
+      mock.get(`${endpoint}5000`, fixture5000);
+
+      testContext.default.__Rewire__('fetch', mock);
+      const validator = await testContext.default({endpoint, prefixPattern, fields});
+
+      const record = new MarcRecord({
+        fields: [
+          {
+            tag: '001',
+            value: '123456'
+          },
+          {
+            tag: '035',
+            subfields: [
+              {
+                code: 'a',
+                value: '(FI-MELINDA)123456'
+              }
+            ]
+          },
+          {
+            tag: '773',
+            subfields: [
+              {
+                code: 'w',
+                value: '(FI-MELINDA)123456'
+              }
+            ]
+          },
+          {
+            tag: '833',
+            subfields: [
+              {
+                code: 'p',
+                value: '(FI-MELINDA)2620'
+              },
+              {
+                code: 'w',
+                value: '(FI-MELINDA)8850'
+              }
+            ]
+          }
+        ]
+      });
+      const result = await validator.validate(record);
+
+      expect(result).to.eql({valid: true, messages: []});
+    });
+
+    it('Finds prefixPattern on record but values not resolvable', async () => {
+      const mock = fetchMock.sandbox();
+      mock.get(`${endpoint}${queryParam}1000`, fixture1000);
+
+      testContext.default.__Rewire__('fetch', mock);
+      const validator = await testContext.default({endpoint, prefixPattern, fields});
+
+      const record = new MarcRecord({
+        fields: [
+          {
+            tag: '001',
+            value: '123456'
+          },
+          {
+            tag: '035',
+            subfields: [
+              {
+                code: 'a',
+                value: '(FI-MELINDA)123456'
+              }
+            ]
+          },
+          {
+            tag: '773',
+            subfields: [
+              {
+                code: 'w',
+                value: '(FOOBAR)1000'
+              }
+            ]
+          }
+        ]
+      });
+      const result = await validator.validate(record);
+
+      expect(result).to.eql({valid: false, messages: ['Field 773$w with value 1000 is not resolvable']});
+    });
+  });
 });

package/src/resolveOrphanedSubfield6s.js

@@ -0,0 +1,115 @@
+import createDebugLogger from 'debug';
+import {fieldHasSubfield, fieldToString, nvdebug, subfieldToString} from './utils';
+import {fieldHasWantedTagAndOccurrenceNumber, isValidSubfield6, subfield6GetOccurrenceNumber, subfield6ResetOccurrenceNumber} from './subfield6Utils';
+
+// Relocated from melinda-marc-record-merge-reducers (and renamed)
+
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:resolveOrphanedSubfield6s');
+
+export default function () {
+  return {
+    description: 'Remove occurrence-number-orphaned $6 subfields. In field 880, occurrence number becomes 00',
+    validate, fix
+  };
+
+  function fix(record) {
+    nvdebug('Fix SF6 orphaned occurrence numbers');
+    const res = {message: [], fix: [], valid: true};
+    //message.fix = []; // eslint-disable-line functional/immutable-data
+
+    // This can not really fail...
+    recordFixSubfield6OccurrenceNumbers(record);
+
+    //message.valid = !(message.message.length >= 1); // eslint-disable-line functional/immutable-data
+    return res;
+  }
+
+  function validate(record) {
+    // Check max, and check number of different indexes
+    nvdebug('Validate SF6 orphaned occurrence numbers', debug);
+    const fieldsContainingSubfield6 = record.fields.filter(field => fieldHasSubfield(field, '6'));
+
+    const orphanedFields = getOrphanedFields(fieldsContainingSubfield6);
+
+    const res = {message: []};
+
+    if (orphanedFields.length > 0) { // eslint-disable-line functional/no-conditional-statements
+      res.message = [`${orphanedFields.length} orphaned occurrence number field(s) detected`]; // eslint-disable-line functional/immutable-data
+    }
+    res.valid = res.message.length < 1; // eslint-disable-line functional/immutable-data
+    return res;
+  }
+}
+
+export function recordFixSubfield6OccurrenceNumbers(record) {
+  const fieldsContainingSubfield6 = record.fields.filter(field => fieldHasSubfield(field, '6'));
+  const orphanedFields = getOrphanedFields(fieldsContainingSubfield6);
+
+  orphanedFields.forEach(field => fieldFixOrphanedSubfields(field));
+
+  function fieldFixOrphanedSubfields(field) {
+    // Field 880: orphaned $6 subfields: occurrence number is changed to '00':
+    if (field.tag === '880') {
+      field.subfields.forEach(sf => field880FixOrphanedSubfield(sf));
+      return;
+    }
+    // Non-880 fields get their orphaned $6s removed:
+    const remainingSubfields = field.subfields.filter(sf => !isOrphanedSubfield(sf, field.tag, fieldsContainingSubfield6));
+    if (remainingSubfields.length === 0) {
+      record.removeField(field);
+      return;
+    }
+    field.subfields = remainingSubfields; // eslint-disable-line functional/immutable-data
+  }
+
+  function field880FixOrphanedSubfield(subfield) {
+    if (!isOrphanedSubfield(subfield, '880', fieldsContainingSubfield6)) {
+      return;
+    }
+    // convert occurrence number to 00
+    subfield6ResetOccurrenceNumber(subfield, '00');
+  }
+}
+
+
+function findPairForSubfield6OccurrenceNumber(subfield6, myTag, candPairFields) {
+  // We keep the crap!
+  if (!isValidSubfield6(subfield6)) {
+    return undefined;
+  }
+  nvdebug(`LOOKING FOR PAIR: ${myTag} ${subfieldToString(subfield6)}`);
+  candPairFields.forEach(field => fieldToString(field));
+
+  // Only valid $6 value that fails to map to another field is iffy...
+  const referredTag = subfield6.value.substring(0, 3);
+
+  const occurrenceNumber = subfield6GetOccurrenceNumber(subfield6);
+  if (occurrenceNumber === '00') {
+    return undefined;
+  }
+  const tagAndOccurrenceNumber = `${myTag}-${occurrenceNumber}`;
+  nvdebug(`Try to find occurrence number ${tagAndOccurrenceNumber} in field ${referredTag}...`);
+  //const relevantFields = fields.filter(field => field.tag === referredTag && field.subfields.some(sf => subfield6GetOccurrenceNumber(sf) === occurrenceNumber));
+  const relevantFields = candPairFields.filter(field => field.tag === referredTag && fieldHasWantedTagAndOccurrenceNumber(field, tagAndOccurrenceNumber));
+  if (relevantFields.length === 0) {
+    return undefined;
+  }
+  // This should always return just one (not sanity checking this for now):
+  return relevantFields[0];
+}
+
+function isOrphanedSubfield(subfield, tag, pairCandidateFields) {
+  if (!isValidSubfield6(subfield) || subfield6GetOccurrenceNumber(subfield) === '00') {
+    return false;
+  }
+  return !findPairForSubfield6OccurrenceNumber(subfield, tag, pairCandidateFields);
+}
+
+
+function isOrphanedField(field, candidatePairFields) {
+  return field.subfields.some(sf => isOrphanedSubfield(sf, field.tag, candidatePairFields));
+}
+
+function getOrphanedFields(relevantFields) {
+  return relevantFields.filter(field => isOrphanedField(field, relevantFields));
+}

package/src/resolveOrphanedSubfield6s.spec.js

@@ -0,0 +1,52 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from './resolveOrphanedSubfield6s';
+import {READERS} from '@natlibfi/fixura';
+import generateTests from '@natlibfi/fixugen';
+import createDebugLogger from 'debug';
+
+generateTests({
+  callback,
+  path: [__dirname, '..', 'test-fixtures', 'remove-orphanded-sf6s'],
+  useMetadataFile: true,
+  recurse: false,
+  fixura: {
+    reader: READERS.JSON
+  },
+  mocha: {
+    before: () => testValidatorFactory()
+  }
+});
+const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/resolveOrphanedSubfield6s:test');
+
+async function testValidatorFactory() {
+  const validator = await validatorFactory();
+
+  expect(validator)
+    .to.be.an('object')
+    .that.has.any.keys('description', 'validate');
+
+  expect(validator.description).to.be.a('string');
+  expect(validator.validate).to.be.a('function');
+}
+
+async function callback({getFixture, enabled = true, fix = false}) {
+  if (enabled === false) {
+    debug('TEST SKIPPED!');
+    return;
+  }
+
+  const validator = await validatorFactory();
+  const record = new MarcRecord(getFixture('record.json'));
+  const expectedResult = getFixture('expectedResult.json');
+  // console.log(expectedResult); // eslint-disable-line
+
+  if (!fix) {
+    const result = await validator.validate(record);
+    expect(result).to.eql(expectedResult);
+    return;
+  }
+
+  await validator.fix(record);
+  expect(record).to.eql(expectedResult);
+}

package/src/sanitize-vocabulary-source-codes.js

@@ -0,0 +1,99 @@
+//import createDebugLogger from 'debug';
+import {fieldToString} from './utils';
+
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/sanitize-vocabulary-source-codes);
+
+
+// Author(s): Nicholas Volk, Joni Ollila
+export default function () {
+
+  return {
+    description: 'Validator for sanitizing vocabulary source codes in subfield $2 (MRA-532)',
+    validate, fix
+  };
+
+  function fix(record) {
+    const fixedFields = getFieldsWithCrappySubfieldCode(record, true);
+    const remainingBadFields = getFieldsWithCrappySubfieldCode(record, false);
+    const remainingBadFieldsAsStrings = remainingBadFields.map(f => fieldToString(f));
+
+    // We are content
+    const fixedFieldsAsStrings = fixedFields.map(f => fieldToString(f));
+
+    return {message: remainingBadFieldsAsStrings, fix: fixedFieldsAsStrings, valid: true};
+  }
+
+  function validate(record) {
+    const badFields = getFieldsWithCrappySubfieldCode(record, false);
+
+    if (badFields.length === 0) {
+      return {'message': [], 'valid': true};
+    }
+
+    const messages = badFields.map(f => fieldToString(f));
+
+    return {'message': messages, 'valid': false};
+  }
+}
+
+// 'mts' is here as per specs. However, I think it should be 'mts/fin' or 'mts/swe'
+const legalSubfieldCode = ['allars', 'mts', 'mts/fin', 'mts/swe', 'slm/fin', 'slm/swe', 'ysa', 'yso/fin', 'yso/swe'];
+
+function stringFixVocabularySourceCode(value) {
+  // Try to remove spaces, change '//' to '/' and remove final '.' and '/':
+  const tmp = value.replace(/ /ug, '')
+    .replace(/\/+/ug, '/')
+    .replace(/^(?:slm|yso)\/$/u, 'local')
+    .replace(/[./]$/gu, '')
+    .replace(/^yso-(?:aika|paikat)\//u, 'yso/'); // IMP-HELMET crap. Also, they still have a '.' at the end of $a...
+
+  if (legalSubfieldCode.includes(tmp)) {
+    return tmp;
+  }
+
+  return value;
+}
+
+function isCrappySubfield2(subfield, fix) {
+  if (subfield.code !== '2' || legalSubfieldCode.includes(subfield.value)) {
+    return false;
+  }
+
+  // If fixer modifies string, it's crap:
+  const fixedVersion = stringFixVocabularySourceCode(subfield.value);
+
+  if (fixedVersion !== subfield.value) {
+    if (fix) {
+      subfield.value = fixedVersion; // eslint-disable-line functional/immutable-data
+      return true;
+    }
+
+    return true;
+  }
+  // As we can't fix this here, apply this yso-rule only when validating!
+  if (!fix && subfield.value.indexOf('yso/') === 0) {
+    return !['yso/eng', 'yso/fin', 'yso/swe'].includes(subfield.value);
+  }
+
+  if (!fix && subfield.value.indexOf('slm/') === 0) {
+    return !['slm/fin', 'slm/swe'].includes(subfield.value);
+  }
+
+  if (!fix && subfield.value.indexOf('mts/') === 0) {
+    return !['mts/fin', 'mts/swe'].includes(subfield.value);
+  }
+
+  return false;
+}
+
+function fieldHasCrappySubfield2(field, fix) {
+  if (!field.tag.match(/^(?:6..|257|370|38.)$/u)) {
+    return false;
+  }
+
+  return field.subfields.some(sf => isCrappySubfield2(sf, fix));
+}
+
+function getFieldsWithCrappySubfieldCode(record, fix) {
+  return record.get(/^6..$/u).filter(f => fieldHasCrappySubfield2(f, fix));
+}