@natlibfi/marc-record-validators-melinda 3.2.0 → 3.3.14-alpha.3

package/src/urn.js

@@ -0,0 +1,164 @@
+import fetch from 'node-fetch';
+import {isElectronicMaterial} from './utils';
+import createDebugLogger from 'debug';
+
+const URN_GENERATOR_URL = 'http://generator.urn.fi/cgi-bin/urn_generator.cgi?type=nbn';
+
+export default function (isLegalDeposit = false, useMelindaTemp = true) {
+  const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:urn');
+  const debugData = debug.extend('data');
+
+  // We should check that the f856 with URN has second indicator '0' (Resource), ' ' (No information provided) or '8' (No display constant generated)
+  // - if second indicator is '1' (Version of resource) or '2' (Related resource) the URN in f856 is not correct for the resource described in the record
+
+  // This checks only the existence of URNs from the Finnish urn.fi -resolver
+
+  const hasURN = f => f.tag === '856' && f.subfields.some(({code, value}) => code === 'u' && (/urn.fi/u).test(value));
+
+  return {
+    description: 'Adds URN for record, to 856-field (if not existing). If isLegalDeposit is active, adds legal deposit subfields to the f856s with URN.',
+    validate,
+    fix
+  };
+
+  async function fix(record) {
+    const f856sUrn = record.fields.filter(hasURN);
+    debugData(`f856sUrn: ${JSON.stringify(f856sUrn)}`);
+
+    const ldSubfields = isLegalDeposit ? createLDSubfields() : [];
+    debugData(`IsLegalDeposit: ${isLegalDeposit}, LegalDepositSubfields: ${JSON.stringify(ldSubfields)}`);
+
+    // We add the URN even if we're not getting the legalDeposit - where does this URN resolve?
+    // We probably should not do these additions
+
+    if (f856sUrn.length === 0) { // eslint-disable-line functional/no-conditional-statements
+      const {code, value, generated} = await createURNSubfield(record);
+
+      if (generated && useMelindaTemp) {
+        const tempSubField = {code: '9', value: 'MELINDA<TEMP>'};
+
+        record.insertField({
+          tag: '856',
+          ind1: '4',
+          ind2: '0',
+          subfields: [{code, value}, ...ldSubfields, tempSubField]
+        });
+
+        return true;
+      }
+
+      record.insertField({
+        tag: '856',
+        ind1: '4',
+        ind2: '0',
+        subfields: [{code, value}, ...ldSubfields]
+      });
+
+      return true;
+    } else if (isLegalDeposit) { // eslint-disable-line functional/no-conditional-statements
+
+      // We add here legal deposit information to all URN-f856s - we probably should not do this
+      // We should add extra f856 URN / URNs for legal deposits that already have a open (non-legal-deposit) URN
+      // How do we decide which URN to use as a template if there are several URNs
+      // We should check for existence of a legal deposit URN anyways
+
+      f856sUrn.forEach(f => {
+        ldSubfields.forEach(ldsf => {
+          if (!f.subfields.some(sf => sf.code === ldsf.code && sf.value === ldsf.value)) { // eslint-disable-line functional/no-conditional-statements
+            f.subfields.push(ldsf); // eslint-disable-line functional/immutable-data
+          }
+        });
+      });
+    }
+
+    return true;
+
+    // We should check existence of URN in f024 i1: '7' $2 urn/URN for this too
+
+    async function createURNSubfield(rec) {
+      // isbn is picked from the last 020 $a in the record
+      // what should we do in case of several 020 $a:s
+      const isbn = rec.fields.reduce((acc, f) => {
+        if (f.tag === '020') {
+          const a = f.subfields.find(sf => sf.code === 'a');
+          return a ? a.value : undefined;
+        }
+
+        return acc;
+      }, undefined);
+
+      debugData(`isbns: ${isbn}`);
+
+      const {generated, value} = await createURN(isbn);
+      return {code: 'u', value, generated};
+
+      async function createURN(isbn = false) {
+        if (isbn) {
+          return {generated: false, value: `http://urn.fi/URN:ISBN:${isbn}`};
+        }
+
+        const response = await fetch(URN_GENERATOR_URL);
+        const body = await response.text();
+
+        // If we generated URN we could also add it to the 024
+        // generated 024 should also have $9 MELINDA<TEMP>
+        return {generated: true, value: `http://urn.fi/${body}`};
+      }
+    }
+
+  }
+
+  // Later when the new subfields that have f506/f540 -type contents, we should add also them here
+  function createLDSubfields() {
+    return [
+      {
+        code: 'z',
+        value: 'Käytettävissä vapaakappalekirjastoissa'
+      },
+      {
+        code: '5',
+        value: 'FI-Vapaa'
+      }
+    ];
+  }
+
+  function fieldHasLDSubfields(field, ldSubfields) {
+    if (ldSubfields.every(ldsf => field.subfields.some(sf => sf.code === ldsf.code && sf.value === ldsf.value))) {
+      return true;
+    }
+  }
+
+  function validateLD(f856sUrn) {
+    debug(`Validating the existence of legal deposit subfields`);
+    const ldSubfields = createLDSubfields();
+    const f856sUrnWithLdSubfields = f856sUrn.filter(field => fieldHasLDSubfields(field, ldSubfields));
+    if (f856sUrnWithLdSubfields.length > 0) {
+      debug(`Record has ${f856sUrnWithLdSubfields.length} URN fields with all necessary legal deposit subfields`);
+      debugData(`f856sUrnWithLdSubfields: ${JSON.stringify(f856sUrnWithLdSubfields)}`);
+      return true;
+    }
+    return false;
+  }
+
+  function validate(record) {
+    // if not electronic skip this validator
+    if (!isElectronicMaterial(record)) {
+      debug(`Record is not electronic - no need to validate legal deposit URNs`);
+      return {valid: true};
+    }
+
+    const f856sUrn = record.fields.filter(hasURN);
+
+    if (f856sUrn.length > 0) {
+      debug(`Record has ${f856sUrn.length} URN fields`);
+      debugData(`f856sUrn: ${JSON.stringify(f856sUrn)}`);
+
+      if (!isLegalDeposit || validateLD(f856sUrn)) {
+        debug(`Record is valid`);
+        return {valid: true};
+      }
+    }
+    debug(`No (valid) URN fields - Record is not valid`);
+    return {valid: false};
+  }
+}

package/src/urn.spec.js

@@ -0,0 +1,231 @@
+import {expect} from 'chai';
+import {MarcRecord} from '@natlibfi/marc-record';
+import validatorFactory from '../src/urn';
+
+describe('urn', async () => {
+  // Fields
+  const f337 = {
+    tag: '337',
+    ind1: ' ',
+    ind2: ' ',
+    subfields: [
+      {code: 'b', value: 'c'},
+      {code: '2', value: 'rdamedia'}
+    ]
+  };
+
+  const f337nonElectronic = {
+    tag: '337',
+    ind1: ' ',
+    ind2: ' ',
+    subfields: [
+      {code: 'b', value: 'n'},
+      {code: '2', value: 'rdamedia'}
+    ]
+  };
+
+  const ldf856 = {
+    tag: '856',
+    ind1: '4',
+    ind2: '0',
+    subfields: [
+      {code: 'u', value: 'http://urn.fi/URN:ISBN:978-951-9155-47-0'},
+      {code: 'z', value: 'Käytettävissä vapaakappalekirjastoissa'},
+      {code: '5', value: 'FI-Vapaa'}
+    ]
+  };
+
+  const ldf856partial = {
+    tag: '856',
+    ind1: '4',
+    ind2: '0',
+    subfields: [
+      {code: 'u', value: 'http://urn.fi/URN:ISBN:978-951-9155-47-0'},
+      {code: '5', value: 'FI-Vapaa'}
+    ]
+  };
+
+  const f856URN = {
+    tag: '856',
+    ind1: '4',
+    ind2: '0',
+    subfields: [{code: 'u', value: 'http://urn.fi/URN:ISBN:978-951-9155-47-0'}]
+  };
+
+  const f856URNnotResource = {
+    tag: '856',
+    ind1: '4',
+    ind2: '1',
+    subfields: [{code: 'u', value: 'http://urn.fi/URN:ISBN:978-951-9155-47-0'}]
+  };
+
+
+  const f856URL = {
+    tag: '856',
+    ind1: '4',
+    ind2: '0',
+    subfields: [{code: 'u', value: 'http://foo.bar/'}]
+  };
+
+  const f020 = {
+    tag: '020',
+    ind1: ' ',
+    ind2: ' ',
+    subfields: [{code: 'a', value: '978-951-9155-47-0'}]
+  };
+
+  const f020second = {
+    tag: '020',
+    ind1: ' ',
+    ind2: ' ',
+    subfields: [{code: 'a', value: '9789519155470'}]
+  };
+
+
+  it('Creates a validator', async () => {
+    const validator = await validatorFactory();
+
+    expect(validator)
+      .to.be.an('object')
+      .that.has.any.keys('description', 'validate');
+
+    expect(validator.description).to.be.a('string');
+    expect(validator.validate).to.be.a('function');
+  });
+
+  // Tests
+  const test = async isLegalDeposit => {
+    const validator = await validatorFactory(isLegalDeposit);
+    return {
+      validate: async (valid, ...recfields) => {
+        const result = await validator.validate(new MarcRecord({fields: recfields}));
+        expect(result).to.eql({valid});
+      },
+
+      fix: async (recfields, resfields) => {
+        const record = new MarcRecord({fields: recfields});
+        await validator.fix(record);
+        expect(record.fields).to.eql(resfields);
+      }
+    };
+  };
+
+  /// Non-legal and legal deposit
+  const nonld = await test(false);
+  const ld = await test(true);
+
+  describe('#validate', () => {
+    // Validate non-electoronic
+    it('Finds the record valid; non-electronic record', async () => {
+      await nonld.validate(true, f337nonElectronic);
+    });
+
+    // Validate non-legal deposit
+    it('Finds the record valid; 856 with urn, and is non-legal deposit', async () => {
+      await nonld.validate(true, f337, f856URN);
+    });
+
+
+    // we should recognize that 856 with second indicator 1 is not describing the resource itself
+    it.skip('Finds the record invalid; 856 ind2: 1 with urn, and is non-legal deposit', async () => {
+      await nonld.validate(false, f337, f856URNnotResource);
+    });
+
+    // should we require urn if we're not handling a legal deposit
+    it('Finds the record invalid; 856 without urn, and is non-legal deposit', async () => {
+      await nonld.validate(false, f337, f856URL);
+    });
+
+
+    // should we require urn if we're not handling a legal deposit
+    it('Finds the record invalid; Missing 856, and is non-legal deposit', async () => {
+      await nonld.validate(false, f337, f020);
+    });
+
+    // Validate legal deposit
+    it('Finds the record invalid; 856 with urn, and is legal deposit', async () => {
+      await ld.validate(false, f020, f337, f856URN);
+    });
+
+    it('Finds the record invalid; 856 without urn, and is legal deposit', async () => {
+      await ld.validate(false, f020, f337, f856URL);
+    });
+
+    it('Finds the record invalid; Missing 856, and is legal deposit', async () => {
+      await ld.validate(false, f337, f020);
+    });
+
+    it('Finds the record valid; 856 with URN and legal deposit subfields, and is legal deposit', async () => {
+      await ld.validate(true, f337, f020, ldf856);
+    });
+
+    it('Finds the record valid; 856 with URN and legal deposit subfields and other f856s, and is legal deposit', async () => {
+      await ld.validate(true, f337, f020, ldf856partial, ldf856, f856URL);
+    });
+
+    it('Finds the record invalid; 856 with URN and partial legal deposit subfields, and is legal deposit', async () => {
+      await ld.validate(false, f337, f020, ldf856partial);
+    });
+
+
+  });
+
+  describe('#fix', () => {
+    // Fix non-legal deposit
+    it('856 with urn, and is non-legal deposit; Nothing to add', async () => {
+      await nonld.fix([f020, f856URL, f856URN], [f020, f856URL, f856URN]);
+    });
+
+    // should we actually add non-resolvable urns?
+    it('856 without urn, and is non-legal deposit; Adds 856 with urn', async () => {
+      await nonld.fix([f020, f856URL], [f020, f856URL, f856URN]);
+    });
+
+    // should we actually add non-resolvable urns?
+    it('Missing 856, and is non-legal deposit; Adds 856 with urn', async () => {
+      await nonld.fix([f020], [f020, f856URN]);
+    });
+
+    // should we actually add non-resolvable urns?
+    it('Missing 856, and is non-legal deposit, two 020 fields; Adds 856 with urn from second 020', async () => {
+      await nonld.fix([f020second, f020], [f020second, f020, f856URN]);
+    });
+
+    // should we actually add non resovable urns?
+    // we should think about how to choose the isbn to use in case of several ISBNs
+    it.skip('Missing 856, and is non-legal deposit, two 020 fields; Adds 856 with urn from first 020', async () => {
+      await nonld.fix([f020, f020second], [f020, f020second, f856URN]);
+    });
+
+
+    // Fix legal deposit
+    it('856 with urn and legal deposit fields, and is legal deposit; Nothing to add', async () => {
+      await ld.fix([f020, f856URL, ldf856], [f020, f856URL, ldf856]);
+    });
+
+    it('856 without urn, and is legal deposit; Adds 856 with urn and legal deposit fields', async () => {
+      await ld.fix([f020, f856URL], [f020, f856URL, ldf856]);
+    });
+
+    // we should test generating the URN in case of no ISBN
+
+    // we should test creating Melinda-temp field
+
+    // We shouldn't lock the open URN for legal deposit use
+    it('Missing 856, and is legal deposit; Adds 856 with urn and legal deposit fields', async () => {
+      await ld.fix([f020], [f020, ldf856]);
+    });
+
+    // We should actually do this instead of locking the original non-legal deposit URN for legal deposit use
+    it.skip('856 with urn, and is legal deposit; Adds another f856 with URN and legal deposit fields', async () => {
+      await ld.fix([f020, f856URL, f856URN], [f020, f856URL, f856URN, ldf856]);
+    });
+
+    // We should actually add a new urn in case of a non-resource URN
+    it.skip('856 with non-resource-urn, and is legal deposit; Adds another f856 with URN and legal deposit fields', async () => {
+      await ld.fix([f020, f856URL, f856URNnotResource], [f020, f856URL, f856URNnotResource, ldf856]);
+    });
+
+
+  });
+});

package/src/utils.js

@@ -0,0 +1,52 @@
+export function isElectronicMaterial(record) {
+  const f337s = record.get('337');
+
+  return f337s.some(field => {
+    const mediaTypeIsC = field.subfields.some(sub => sub.code === 'b' && sub.value === 'c');
+    const sourceIsRdamedia = field.subfields.some(sub => sub.code === '2' && sub.value === 'rdamedia');
+    return mediaTypeIsC && sourceIsRdamedia;
+  });
+}
+
+export function nvdebug(message, func = undefined) {
+  if (func) { // eslint-disable-line functional/no-conditional-statements
+    func(message);
+  }
+  //console.info(message); // eslint-disable-line no-console
+}
+
+export function fieldHasSubfield(field, subfieldCode, subfieldValue = null) {
+  if (!field.subfields) {
+    return false;
+  }
+  if (subfieldValue === null) {
+    return field.subfields.some(sf => sf.code === subfieldCode);
+  }
+  return field.subfields.some(sf => sf.code === subfieldCode && subfieldValue === sf.value);
+}
+
+export function subfieldToString(sf) {
+  return `‡${sf.code} ${sf.value}`;
+}
+
+function normalizeIndicatorValue(val) {
+  if (val === ' ') {
+    return '#';
+  }
+  return val;
+}
+
+export function fieldToString(f) {
+  if ('subfields' in f) {
+    return `${f.tag} ${normalizeIndicatorValue(f.ind1)}${normalizeIndicatorValue(f.ind2)}${formatSubfields(f)}`;
+  }
+  return `${f.tag}    ${f.value}`;
+
+  function formatSubfields(field) {
+    return field.subfields.map(sf => ` ${subfieldToString(sf)}`).join('');
+  }
+}
+
+export function fieldsToString(fields) {
+  return fields.map(f => fieldToString(f)).join('\t__SEPARATOR__\t');
+}

package/test-fixtures/duplicates-ind1/01/expectedResult.json

@@ -0,0 +1,4 @@
+{
+  "valid": true,
+  "messages": []
+}

package/test-fixtures/duplicates-ind1/01/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "Finds the record valid",
+  "fix": false
+}

package/test-fixtures/duplicates-ind1/01/record.json

@@ -0,0 +1,16 @@
+{
+  "fields": [
+    {
+      "tag": "500",
+      "ind1": " ",
+      "ind2": "0",
+      "subfields": [{"code": "a", "value": "foo"}]
+    },
+    {
+      "tag": "500",
+      "ind1": " ",
+      "ind2": "0",
+      "subfields": [{"code": "a", "value": "foo"}]
+    }
+  ]
+}

package/test-fixtures/duplicates-ind1/02/expectedResult.json

@@ -0,0 +1,6 @@
+{
+  "valid": false,
+  "messages": [
+    "Multiple 500 fields which only differ in the first indicator"
+  ]
+}

package/test-fixtures/duplicates-ind1/02/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "Finds the record invalid",
+  "fix": false
+}

package/test-fixtures/duplicates-ind1/02/record.json

@@ -0,0 +1,16 @@
+{
+  "fields": [
+    {
+      "tag": "500",
+      "ind1": " ",
+      "ind2": "0",
+      "subfields": [{"code": "a", "value": "foo"}]
+    },
+    {
+      "tag": "500",
+      "ind1": "1",
+      "ind2": "0",
+      "subfields": [{"code": "a", "value": "foo"}]
+    }
+  ]
+}

package/test-fixtures/duplicates-ind1/03/expectedResult.json

@@ -0,0 +1,13 @@
+[
+  {
+    "tag": "500",
+    "ind1": " ",
+    "ind2": "0",
+    "subfields": [
+      {
+        "code": "a",
+        "value": "foo"
+      }
+    ]
+  }
+]

package/test-fixtures/duplicates-ind1/03/metadata.json

@@ -0,0 +1,6 @@
+{
+  "only": false,
+  "description": "Removes duplicate values",
+  "tagPattern": "^500$",
+  "fix": true
+}

package/test-fixtures/duplicates-ind1/03/record.json

@@ -0,0 +1,16 @@
+{
+  "fields": [
+    {
+      "tag": "500",
+      "ind1": " ",
+      "ind2": "0",
+      "subfields": [{"code": "a", "value": "foo"}]
+    },
+    {
+      "tag": "500",
+      "ind1": "1",
+      "ind2": "0",
+      "subfields": [{"code": "a", "value": "foo"}]
+    }
+  ]
+}

package/test-fixtures/ending-whitespace/01/expectedResult.json

@@ -0,0 +1,4 @@
+{
+  "messages": [],
+  "valid": true
+}

package/test-fixtures/ending-whitespace/01/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "Finds the record valid when there are no subfields that end with whitespace",
+  "skip": false,
+  "only": false,
+  "fix": false
+}

package/test-fixtures/ending-whitespace/01/record.json

@@ -0,0 +1,15 @@
+{
+  "fields": [
+    {
+      "tag": "500",
+      "ind1": " ",
+      "ind2": " ",
+      "subfields": [
+        {
+          "code": "a",
+          "value": "Foo"
+        }
+      ]
+    }
+  ]
+}

package/test-fixtures/ending-whitespace/02/expectedResult.json

@@ -0,0 +1,4 @@
+{
+  "messages":["Field 500 subfield $a ends with whitespace"],
+  "valid": false
+}

package/test-fixtures/ending-whitespace/02/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "Finds the record not valid when there is a subfield that ends with whitespace",
+  "skip": false,
+  "only": false,
+  "fix": false
+}

package/test-fixtures/ending-whitespace/02/record.json

@@ -0,0 +1,15 @@
+{
+  "fields": [
+    {
+      "tag": "500",
+      "ind1": " ",
+      "ind2": " ",
+      "subfields": [
+        {
+          "code": "a",
+          "value": "Foo "
+        }
+      ]
+    }
+  ]
+}

package/test-fixtures/ending-whitespace/03/expectedResult.json

@@ -0,0 +1,17 @@
+{
+  "_validationOptions": {},
+  "fields": [
+      {
+        "tag": "500",
+        "ind1": " ",
+        "ind2": " ",
+        "subfields": [
+          {
+            "code": "a",
+            "value": "Foo"
+          }
+        ]
+      }
+  ],
+  "leader": ""
+}

package/test-fixtures/ending-whitespace/03/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "Fixes the record subfield that ends with whitespace",
+  "skip": false,
+  "only": false,
+  "fix": true
+}

package/test-fixtures/ending-whitespace/03/record.json

@@ -0,0 +1,15 @@
+{
+  "fields": [
+    {
+      "tag": "500",
+      "ind1": " ",
+      "ind2": " ",
+      "subfields": [
+        {
+          "code": "a",
+          "value": "Foo "
+        }
+      ]
+    }
+  ]
+}

package/test-fixtures/ending-whitespace/04/expectedResult.json

@@ -0,0 +1,10 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    {
+      "tag": "008",
+      "value": "Foo "
+    }
+  ],
+  "leader": ""
+}

package/test-fixtures/ending-whitespace/04/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "Does not handle control fields",
+  "skip": false,
+  "only": false,
+  "fix": true
+}

package/test-fixtures/ending-whitespace/04/record.json

@@ -0,0 +1,8 @@
+{
+  "fields": [
+    {
+      "tag": "008",
+      "value": "Foo "
+    }
+  ]
+}