@natlibfi/marc-record-validators-melinda 3.2.0 → 3.3.14-alpha.3

package/src/identical-fields.js

@@ -1,57 +1,38 @@
-/**
- *
- * @licstart  The following is the entire license notice for the JavaScript code in this file.
- *
- * Melinda-related validators for @natlibfi/marc-record-validate
- *
- * Copyright (c) 2014-2017 University Of Helsinki (The National Library Of Finland)
- *
- * This file is part of marc-record-validators-melinda
- *
- * marc-record-validators-melinda is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * @licend  The above is the entire license notice
- * for the JavaScript code in this file.
- *
- **/
-
 /**
  * This validator de-duplicates identical fields in a record.
  */
 
-/* eslint-disable require-await */
-import {isEqual, uniqWith} from 'lodash';
+import {MarcRecord} from '@natlibfi/marc-record';
+
+export default function () {
+  return {
+    description: 'Handles identical duplicate in record fields',
+    validate,
+    fix
+  };
+
+  function validate(record) {
+    const uniq = uniqWith(record.fields);
+    const valid = uniq.length === record.fields.length;
+    const messages = record.fields.filter(tag => !uniq.includes(tag))
+      .map(obj => `Field ${obj.tag} has duplicates`);
 
-export default async function () {
-	return {
-		description: 'Handles identical duplicate in record fields',
-		validate,
-		fix
-	};
+    return valid ? {valid, messages: []} : {valid, messages};
+  }
 
-	async function validate(record) {
-		const uniq = uniqWith(record.fields, isEqual);
-		const valid = uniq.length === record.fields.length;
-		const messages = record.fields.filter(tag => !uniq.includes(tag))
-			.map(obj => `Field ${obj.tag} has duplicates`);
+  function fix(record) {
+    record.fields
+      .filter(tag => !uniqWith(record.fields).includes(tag))
+      .forEach(tag => record.removeField(tag));
+  }
 
-		return valid ? {valid, messages: []} : {valid, messages};
-	}
+  function uniqWith(fields) {
+    return fields.reduce((uniq, field) => {
+      if (!uniq.some(f => MarcRecord.isEqual(f, field))) { // eslint-disable-line functional/no-conditional-statements
+        uniq.push(field); // eslint-disable-line functional/immutable-data
+      }
 
-	async function fix(record) {
-		record.fields
-			.filter(tag => !uniqWith(record.fields, isEqual).includes(tag))
-			.forEach(tag => record.removeField(tag));
-	}
+      return uniq;
+    }, []);
+  }
 }

package/src/identical-fields.spec.js

@@ -1,161 +1,129 @@
-/**
- *
- * @licstart  The following is the entire license notice for the JavaScript code in this file.
- *
- * MARC record validators used in Melinda
- *
- * Copyright (C) 2014-2018 University Of Helsinki (The National Library Of Finland)
- *
- * This file is part of marc-record-validators-melinda
- *
- * marc-record-validators-melinda program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * marc-record-validators-melinda is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * @licend  The above is the entire license notice
- * for the JavaScript code in this file.
- *
- */
-
-/* eslint-disable no-undef, max-nested-callbacks, no-unused-expressions */
-
-'use strict';
-
 import {expect} from 'chai';
 import {MarcRecord} from '@natlibfi/marc-record';
 import validatorFactory from '../src/identical-fields';
 
 describe('identical-fields', () => {
-	it('Creates a validator', async () => {
-		const validator = await validatorFactory();
+  it('Creates a validator', async () => {
+    const validator = await validatorFactory();
 
-		expect(validator)
-			.to.be.an('object')
-			.that.has.any.keys('description', 'validate');
+    expect(validator)
+      .to.be.an('object')
+      .that.has.any.keys('description', 'validate');
 
-		expect(validator.description).to.be.a('string');
-		expect(validator.validate).to.be.a('function');
-	});
+    expect(validator.description).to.be.a('string');
+    expect(validator.validate).to.be.a('function');
+  });
 
-	describe('#validate', () => {
-		it('Finds the record valid', async () => {
-			const validator = await validatorFactory();
-			const record = new MarcRecord({
-				fields: [
-					{
-						tag: '700',
-						subfields: [
-							{
-								code: 'e',
-								value: 'foo'
-							}
-						]
-					}
-				]
-			});
-			const result = await validator.validate(record);
+  describe('#validate', () => {
+    it('Finds the record valid', async () => {
+      const validator = await validatorFactory();
+      const record = new MarcRecord({
+        fields: [
+          {
+            tag: '700',
+            subfields: [
+              {
+                code: 'e',
+                value: 'foo'
+              }
+            ]
+          }
+        ]
+      });
+      const result = await validator.validate(record);
 
-			expect(result).to.eql({valid: true, messages: []});
-		});
-		it('Finds the record invalid', async () => {
-			const validator = await validatorFactory();
-			const record = new MarcRecord({
+      expect(result).to.eql({valid: true, messages: []});
+    });
+    it('Finds the record invalid', async () => {
+      const validator = await validatorFactory();
+      const record = new MarcRecord({
 
-				fields: [
-					{
-						tag: '700',
-						subfields: [
-							{
-								code: 'e',
-								value: 'foo'
-							}
-						]
-					},
-					{
-						tag: '800',
-						subfields: [
-							{
-								code: 'e',
-								value: 'foo'
-							}
-						]
-					},
-					{
-						tag: '800',
-						subfields: [
-							{
-								code: 'e',
-								value: 'foo'
-							}
-						]
-					},
-					{
-						tag: '700',
-						subfields: [
-							{
-								code: 'e',
-								value: 'foo'
-							}
-						]
-					}
-				]
-			});
+        fields: [
+          {
+            tag: '700',
+            subfields: [
+              {
+                code: 'e',
+                value: 'foo'
+              }
+            ]
+          },
+          {
+            tag: '800',
+            subfields: [
+              {
+                code: 'e',
+                value: 'foo'
+              }
+            ]
+          },
+          {
+            tag: '800',
+            subfields: [
+              {
+                code: 'e',
+                value: 'foo'
+              }
+            ]
+          },
+          {
+            tag: '700',
+            subfields: [
+              {
+                code: 'e',
+                value: 'foo'
+              }
+            ]
+          }
+        ]
+      });
 
-			const result = await validator.validate(record);
+      const result = await validator.validate(record);
 
-			expect(result).to.eql({valid: false, messages: ['Field 800 has duplicates', 'Field 700 has duplicates']});
-		});
-	});
+      expect(result).to.eql({valid: false, messages: ['Field 800 has duplicates', 'Field 700 has duplicates']});
+    });
+  });
 
-	describe('#fix', () => {
-		it('Fixes the record', async () => {
-			const validator = await validatorFactory();
-			const record = new MarcRecord({
-				fields: [
-					{
-						tag: '700',
-						subfields: [
-							{
-								code: 'e',
-								value: 'dest'
-							}
-						]
-					},
-					{
-						tag: '700',
-						subfields: [
-							{
-								code: 'e',
-								value: 'dest'
-							}
-						]
-					}
-				]
-			});
-			await validator.fix(record);
+  describe('#fix', () => {
+    it('Fixes the record', async () => {
+      const validator = await validatorFactory();
+      const record = new MarcRecord({
+        fields: [
+          {
+            tag: '700',
+            subfields: [
+              {
+                code: 'e',
+                value: 'dest'
+              }
+            ]
+          },
+          {
+            tag: '700',
+            subfields: [
+              {
+                code: 'e',
+                value: 'dest'
+              }
+            ]
+          }
+        ]
+      });
+      await validator.fix(record);
 
-			expect(record.fields).to.eql([
-				{
-					tag: '700',
-					ind1: ' ',
-					ind2: ' ',
-					subfields: [
-						{
-							code: 'e',
-							value: 'dest'
-						}
-					]
-				}
-			]);
-		});
-	});
+      expect(record.fields).to.eql([
+        {
+          tag: '700',
+          ind1: ' ',
+          ind2: ' ',
+          subfields: [
+            {
+              code: 'e',
+              value: 'dest'
+            }
+          ]
+        }
+      ]);
+    });
+  });
 });

package/src/index.js

@@ -1,39 +1,10 @@
-/**
-*
-* @licstart  The following is the entire license notice for the JavaScript code in this file.
-*
-* MARC record validators used in Melinda
-*
-* Copyright (C) 2014-2018 University Of Helsinki (The National Library Of Finland)
-*
-* This file is part of marc-record-validators-melinda
-*
-* marc-record-validators-melinda program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Affero General Public License as
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* marc-record-validators-melinda is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Affero General Public License for more details.
-*
-* You should have received a copy of the GNU Affero General Public License
-* along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*
-* @licend  The above is the entire license notice
-* for the JavaScript code in this file.
-*
-*/
-
-/* eslint-disable no-unused-vars */
-
-'use strict';
-
+import AccessRights from './access-rights';
 import DoubleCommas from './double-commas';
 import DuplicatesInd1 from './duplicates-ind1';
 import EmptyFields from './empty-fields';
 import EndingPunctuation from './ending-punctuation';
+import EndingWhitespace from './ending-whitespace';
+import Field521Fix from './field-521-fix';
 import FieldsPresent from './fields-present';
 import FieldStructure from './field-structure';
 import FixedFields from './fixed-fields';
@@ -41,24 +12,48 @@ import FieldExclusion from './field-exclusion';
 import IdenticalFields from './identical-fields';
 import IsbnIssn from './isbn-issn';
 import ItemLanguage from './item-language';
+import NonBreakingSpace from './non-breaking-space';
+import NormalizeUTF8Diacritics from './normalize-utf8-diacritics';
+import Punctuation from './punctuation/';
+import ResolveOrphanedSubfield6s from './resolveOrphanedSubfield6s'; // Do this before reindexing!
+import ReindexSubfield6OccurenceNumbers from './reindexSubfield6OccurenceNumbers';
 import ResolvableExtReferences from './resolvable-ext-references-melinda';
+import SanitizeVocabularySourceCodes from './sanitize-vocabulary-source-codes';
+import UpdateField540 from './update-field-540';
+import SortSubfields from './sortSubfields';
 import SortTags from './sort-tags';
 import SubfieldExclusion from './subfield-exclusion';
+import TypeOfDateF008 from './typeOfDate-008';
 import UnicodeDecomposition from './unicode-decomposition';
+import Urn from './urn';
 
 export {
-	DoubleCommas,
-	DuplicatesInd1,
-	EmptyFields,
-	EndingPunctuation,
-	FieldExclusion,
-	FieldsPresent,
-	FieldStructure,
-	FixedFields,
-	IdenticalFields,
-	IsbnIssn,
-	ItemLanguage,
-	ResolvableExtReferences,
-	SortTags,
-	UnicodeDecomposition
+  AccessRights,
+  DoubleCommas,
+  DuplicatesInd1,
+  EmptyFields,
+  EndingPunctuation,
+  EndingWhitespace,
+  Field521Fix,
+  FieldExclusion,
+  FieldsPresent,
+  FieldStructure,
+  FixedFields,
+  IdenticalFields,
+  IsbnIssn,
+  ItemLanguage,
+  NonBreakingSpace,
+  NormalizeUTF8Diacritics,
+  Punctuation,
+  ResolveOrphanedSubfield6s,
+  ReindexSubfield6OccurenceNumbers,
+  ResolvableExtReferences,
+  SanitizeVocabularySourceCodes,
+  SortSubfields,
+  SortTags,
+  SubfieldExclusion,
+  TypeOfDateF008,
+  UnicodeDecomposition,
+  UpdateField540,
+  Urn
 };

package/src/indicator-fixes.js

@@ -0,0 +1,211 @@
+// Relocated from melinda-marc-record-merge-reducers (and renamed)
+//import createDebugLogger from 'debug';
+//const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:normalizeIdentifiers');
+
+import {fieldToString} from './utils';
+
+
+export default function () {
+
+  return {
+    description: 'Normalizes indicator values',
+    validate, fix
+  };
+
+  function fix(record) {
+    const res = {message: [], fix: [], valid: true};
+
+    recordNormalizeIndicators(record);
+
+    return res;
+  }
+
+  function validate(record) {
+    const res = {message: []};
+
+    validateRecord(record, res);
+
+    res.valid = res.message.length < 1; // eslint-disable-line functional/immutable-data
+    return res;
+  }
+
+
+  function validateRecord(record, res) {
+    //nvdebug(record);
+    const clonedFields = JSON.parse(JSON.stringify(record.fields));
+    recordNormalizeIndicators(record);
+
+    record.fields.forEach((field, index) => compareFields(field, index));
+
+    function compareFields(field, index) {
+      const origFieldAsString = fieldToString(clonedFields[index]);
+      //const clonedFieldAsString = fieldToString(field);
+      if (clonedFields[index].ind1 !== field.ind1) { // eslint-disable-line functional/no-conditional-statements
+        //nvdebug(`FIX IND1: '${clonedFields[index].ind1}' => '${field.ind1}': ${clonedFieldAsString}`);
+        res.message.push(`Expected IND1 for '${origFieldAsString}' is '${field.ind1}'`); // eslint-disable-line functional/immutable-data
+      }
+      if (clonedFields[index].ind2 !== field.ind2) { // eslint-disable-line functional/no-conditional-statements
+        //nvdebug(`FIX IND2: '${clonedFields[index].ind2}' => '${field.ind2}': ${clonedFieldAsString}`);
+        res.message.push(`Expected IND2 for '${origFieldAsString}' is '${field.ind2}'`); // eslint-disable-line functional/immutable-data
+      }
+    }
+    // Validator should not change the original record:
+    record.fields = clonedFields; // eslint-disable-line functional/immutable-data
+    return;
+  }
+}
+
+
+const ind1NonFilingChars = ['130', '630', '730', '740'];
+const ind2NonFilingChars = ['222', '240', '242', '243', '245', '830'];
+
+function hasNonFilingIndicator1(field) {
+  return ind1NonFilingChars.includes(field.tag);
+}
+
+function modifiableIndicatorValue(value) {
+  // If field contains a legit-looking value, don't try to modify it here...
+  return !['9', '8', '7', '6', '5', '4', '3', '2', '1'].includes(value);
+}
+
+function hasNonFilingIndicator2(field) {
+  return ind2NonFilingChars.includes(field.tag);
+}
+
+function valueBeginsWithDeterminer(value, cands) {
+  return cands.find(cand => value.substring(0, cand.length) === cand);
+}
+
+function determineNonFilingIndicatorValue(field, languages = undefined) {
+  const subfieldA = field.subfields.find(sf => sf.code === 'a');
+  if (!subfieldA) {
+    // nvdebug(' Subfield $a miss!');
+    return;
+  }
+
+  const name = subfieldA.value.toLowerCase();
+
+  if (languages.includes('eng')) {
+    const match = valueBeginsWithDeterminer(name, ['a ', 'an ', 'the ']);
+    if (match) {
+      return `${match.length}`;
+    }
+  }
+
+  if (languages.includes('fre')) {
+    const match = valueBeginsWithDeterminer(name, ['l\'', 'le ']);
+    if (match) {
+      return `${match.length}`;
+    }
+  }
+
+  if (languages.includes('ger')) {
+    const match = valueBeginsWithDeterminer(name, ['das ', 'der ', 'die ']);
+    if (match) {
+      return `${match.length}`;
+    }
+  }
+
+  if (languages.includes('swe')) {
+    const match = valueBeginsWithDeterminer(name, ['en ', 'ett ']);
+    if (match) {
+      return `${match.length}`;
+    }
+  }
+
+  // Fallback-ish: try to guess even without languages:
+  const match = valueBeginsWithDeterminer(name, ['the ']);
+  if (match) {
+    return `${match.length}`;
+  }
+
+  return '0';
+}
+
+function normalizeNonFilingIndicator1(field, languages = []) {
+  if (!hasNonFilingIndicator1(field) || !modifiableIndicatorValue(field.ind1)) {
+    return;
+  }
+
+  field.ind1 = determineNonFilingIndicatorValue(field, languages); // eslint-disable-line functional/immutable-data
+}
+
+function normalizeNonFilingIndicator2(field, languages = []) {
+  if (!hasNonFilingIndicator2(field) || !modifiableIndicatorValue(field.ind2)) {
+    return;
+  }
+
+  field.ind2 = determineNonFilingIndicatorValue(field, languages); // eslint-disable-line functional/immutable-data
+}
+
+
+function normalize245Indicator1(field, record) {
+  if (field.tag !== '245') {
+    return;
+  }
+  const field1XX = record.get('^1..$');
+  field.ind1 = field1XX.length === 0 ? '0' : '1'; // eslint-disable-line functional/immutable-data
+}
+
+
+function recordNormalize490(record) {
+  const fields490 = record.get('^490$');
+  const fields8XX = record.get('^(?:800|810|811|830)$');
+
+  if (fields490.length === 0) {
+    return;
+  }
+  if (fields490.length <= fields8XX.length) {
+    // Trace found for each field 490:
+    fields490.forEach(f => {
+      f.ind1 = '1'; // eslint-disable-line functional/immutable-data
+    });
+    return;
+  }
+  if (fields8XX.length === 0) { // Fields 490 are always untraced (no traces found)
+    fields490.forEach(f => {
+      f.ind1 = '0'; // eslint-disable-line functional/immutable-data
+    });
+    return;
+  }
+  // For other combinations we just can't be sure, so leave them as they are.
+}
+
+
+function getLanguages(record) {
+  const langFields = record.get('^041$');
+
+  if (langFields.length === 0) {
+    return [];
+  }
+
+  return langFields[0].subfields.filter(sf => isRelevantSubfield(sf)).map(subfield => subfield.value);
+
+  function isRelevantSubfield(subfield) {
+    if (subfield.code !== 'a' && subfield.code !== 'd') {
+      return false;
+    }
+    if (subfield.value.length !== 3) {
+      return false;
+    }
+    // We could require /^[a-z][a-z][a-z]$/ etc as well, but it's not really that relevant.
+    return true;
+  }
+
+}
+
+export function recordNormalizeIndicators(record) {
+  recordNormalize490(record);
+
+  // Language is used to handle non-filing indicators
+  const languages = getLanguages(record);
+
+  record.fields.forEach(field => fieldNormalizeIndicators(field, record, languages));
+
+}
+
+function fieldNormalizeIndicators(field, record, languages) {
+  normalize245Indicator1(field, record);
+  normalizeNonFilingIndicator1(field, languages);
+  normalizeNonFilingIndicator2(field, languages);
+}