@natlibfi/marc-record-validators-melinda 11.6.7 → 12.0.0-alpha.12

package/src/indicator-fixes.js

@@ -2,7 +2,7 @@
 //import createDebugLogger from 'debug';
 //const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:normalizeIdentifiers');
 
-import {fieldToString} from './utils';
+import {fieldToString} from './utils.js';
 
 
 export default function () {
@@ -35,7 +35,7 @@ export default function () {
     const clonedFields = JSON.parse(JSON.stringify(record.fields));
     recordNormalizeIndicators(record);
 
-    record.fields.forEach((field, index) => compareFields(field, index)); // eslint-disable-line array-callback-return
+    record.fields.forEach((field, index) => compareFields(field, index));
 
     function compareFields(field, index) {
       const origFieldAsString = fieldToString(clonedFields[index]);
@@ -171,6 +171,18 @@ function normalize245Indicator1(field, record) {
   field.ind1 = field1XX.length === 0 ? '0' : '1';
 }
 
+function noDisplayConstantGenerated520Indicator1(field) {
+  if (field.tag !== '520') {
+    return;
+  }
+  const as = field.subfields.filter(sf => sf.code === 'a');
+  // Set ind1=8 "no display constant generated" fro certain values (part of MELKEHITYS-2579):
+  if (as.length === 1 && ['Abstract.', 'Abstrakt.', 'Abstrakti.', 'Abstract.', 'English Summary.', 'Sammandrag.', 'Tiivistelmä.'].includes(field.subfields[0].value)) {
+    field.ind1 = '8';
+  }
+
+}
+
 function normalize776Indicator2(field) {
   if (field.tag !== '776') {
     return;
@@ -229,19 +241,20 @@ function getLanguages(record) {
 
 }
 
-export function recordNormalizeIndicators(record) {
+function recordNormalizeIndicators(record) {
   recordNormalize490(record);
 
   // Language is used to handle non-filing indicators
   const languages = getLanguages(record);
 
-  record.fields.forEach(field => fieldNormalizeIndicators(field, record, languages)); // eslint-disable-line array-callback-return
+  record.fields.forEach(field => fieldNormalizeIndicators(field, record, languages));
 
 }
 
 function fieldNormalizeIndicators(field, record, languages) {
   normalize084Indicator1(field);
   normalize245Indicator1(field, record);
+  noDisplayConstantGenerated520Indicator1(field);
   normalizeNonFilingIndicator1(field, languages);
   normalizeNonFilingIndicator2(field, languages);
   normalize776Indicator2(field);

package/src/{indicator-fixes.spec.js → indicator-fixes.test.js}

@@ -1,19 +1,19 @@
-import {expect} from 'chai';
+import assert from 'node:assert';
 import {MarcRecord} from '@natlibfi/marc-record';
-import validatorFactory from './indicator-fixes';
+import validatorFactory from './indicator-fixes.js';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
 import createDebugLogger from 'debug';
 
 generateTests({
   callback,
-  path: [__dirname, '..', 'test-fixtures', 'indicator-fixes'],
+  path: [import.meta.dirname, '..', 'test-fixtures', 'indicator-fixes'],
   useMetadataFile: true,
   recurse: false,
   fixura: {
     reader: READERS.JSON
   },
-  mocha: {
+  hooks: {
     before: () => testValidatorFactory()
   }
 });
@@ -22,12 +22,9 @@ const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/indica
 async function testValidatorFactory() {
   const validator = await validatorFactory();
 
-  expect(validator)
-    .to.be.an('object')
-    .that.has.any.keys('description', 'validate');
-
-  expect(validator.description).to.be.a('string');
-  expect(validator.validate).to.be.a('function');
+  assert.equal(typeof validator, 'object');
+  assert.equal(typeof validator.description, 'string');
+  assert.equal(typeof validator.validate, 'function');
 }
 
 async function callback({getFixture, enabled = true, fix = false}) {
@@ -43,10 +40,10 @@ async function callback({getFixture, enabled = true, fix = false}) {
 
   if (!fix) {
     const result = await validator.validate(record);
-    expect(result).to.eql(expectedResult);
+    assert.deepEqual(result, expectedResult);
     return;
   }
 
   await validator.fix(record);
-  expect(record).to.eql(expectedResult);
+  assert.deepEqual(record, expectedResult);
 }

package/src/isbn-issn.js

@@ -1,5 +1,5 @@
 import ISBN from 'isbn3';
-import validateISSN from '@natlibfi/issn-verify';
+import {issn as validateISSN} from '@natlibfi/issn-verify';
 
 // handleInvalid: move invalid 020$a to 020$z, and invalid 022$a to 022$y
 export default ({hyphenateISBN = false, handleInvalid = false} = {}) => {
@@ -29,8 +29,14 @@ export default ({hyphenateISBN = false, handleInvalid = false} = {}) => {
 
   function invalidISBN(isbn) {
     const isbnOnly = getFirstWord(isbn);
-    const auditedIsbn = ISBN.audit(isbnOnly);
-    return !auditedIsbn.validIsbn;
+    try {
+      const auditedIsbn = ISBN.audit(isbnOnly);
+      return !auditedIsbn.validIsbn;
+    }
+    catch {
+      return true;
+    }
+
   }
 
   function invalidSubfield(subfield) {
@@ -173,7 +179,7 @@ export default ({hyphenateISBN = false, handleInvalid = false} = {}) => {
   function fix(record) {
     getRelevantFields(record).forEach(field => {
       if (field.tag === '020') {
-        field.subfields.forEach(subfield => fixField020Subfield(field, subfield)); // eslint-disable-line array-callback-return
+        field.subfields.forEach(subfield => fixField020Subfield(field, subfield));
         return;
       }
       // 022 ISSN:
@@ -238,9 +244,8 @@ export default ({hyphenateISBN = false, handleInvalid = false} = {}) => {
       function normalizeIsbnValue(value) {
         const trimmedValue = getFirstWord(value);
         //const trimmedValue = trimISBN(value); // NB! This might lose information that should be stored in $q...
-        const auditResult = ISBN.audit(trimmedValue);
-        if (!auditResult.validIsbn) {
-          return undefined;
+        if (invalidISBN(trimmedValue)) {
+          return undefined; // should this return value (= nothing normalized), not undefined?
         }
         const numbersOnly = trimmedValue.replace(/[^0-9Xx]+/ug, '');
         const parsedIsbn = ISBN.parse(trimmedValue);

package/src/{isbn-issn.spec.js → isbn-issn.test.js}

@@ -1,17 +1,15 @@
-import {expect} from 'chai';
+import assert from 'node:assert';
 import {MarcRecord} from '@natlibfi/marc-record';
-import validatorFactory from '../src/isbn-issn';
+import validatorFactory from '../src/isbn-issn.js';
+import {describe, it} from 'node:test';
 
 describe('isbn-issn', () => {
   it('Creates a validator', async () => {
     const validator = await validatorFactory();
 
-    expect(validator)
-      .to.be.an('object')
-      .that.has.any.keys('description', 'validate');
-
-    expect(validator.description).to.be.a('string');
-    expect(validator.validate).to.be.a('function');
+    assert.equal(typeof validator, 'object');
+    assert.equal(typeof validator.description, 'string');
+    assert.equal(typeof validator.validate, 'function');
   });
 
   describe('#validate', () => {
@@ -42,7 +40,7 @@ describe('isbn-issn', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({valid: true});
+      assert.deepEqual(result, {valid: true});
     });
 
     it('Finds the record invalid', async () => {
@@ -71,7 +69,7 @@ describe('isbn-issn', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({
+      assert.deepEqual(result, {
         valid: false, messages: [
           'ISBN (foo) is not valid',
           'ISSN (bar) is not valid'
@@ -93,7 +91,7 @@ describe('isbn-issn', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({
+      assert.deepEqual(result, {
         valid: true
       });
     });
@@ -112,7 +110,7 @@ describe('isbn-issn', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({
+      assert.deepEqual(result, {
         valid: false, messages: ['ISSN (undefined) is not valid']
       });
     });
@@ -131,7 +129,7 @@ describe('isbn-issn', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({valid: false, messages: ['ISBN (978-600-377-017-1 (nid.)) is not valid']});
+      assert.deepEqual(result, {valid: false, messages: ['ISBN (978-600-377-017-1 (nid.)) is not valid']});
     });
 
     it('Finds the record invalid (ISSN in \'l\'-subfield)', async () => {
@@ -154,7 +152,7 @@ describe('isbn-issn', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({
+      assert.deepEqual(result, {
         valid: false, messages: [
           'ISBN (foo) is not valid',
           'ISSN (bar) is not valid'
@@ -188,7 +186,7 @@ describe('isbn-issn', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({valid: false, messages: [
+      assert.deepEqual(result, {valid: false, messages: [
         'ISBN (9789519155470) is not valid',
         'ISBN (9068-31-372-X) is not valid',
         'ISBN (386006004X) is not valid'
@@ -220,7 +218,7 @@ describe('isbn-issn', () => {
 
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         {tag: '020', ind1: ' ', ind2: ' ', subfields: [{code: 'z', value: 'foo'}]},
         {tag: '020', ind1: ' ', ind2: ' ', subfields: [{code: 'z', value: 'crappy val'}]},
         {tag: '020', ind1: ' ', ind2: ' ', subfields: [{code: 'z', value: '97895234216609'}]},
@@ -241,7 +239,7 @@ describe('isbn-issn', () => {
 
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         {
           tag: '022', ind1: ' ', ind2: ' ', subfields: [{code: 'y', value: 'foo'}]
         }
@@ -261,7 +259,7 @@ describe('isbn-issn', () => {
 
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         {
           tag: '022', ind1: ' ', ind2: ' ', subfields: [{code: 'y', value: 'foo'}]
         }
@@ -288,7 +286,7 @@ describe('isbn-issn', () => {
       });
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         // NB! Initial space does not need to be removed. It's crap, but not this fixer's crap.
         {tag: '020', ind1: ' ', ind2: ' ', subfields: [{code: 'a', value: ' 9786003770171'}]},
         {tag: '020', ind1: ' ', ind2: ' ', subfields: [{code: 'a', value: '9786003770171'}, {code: 'q', value: '(nidottu)'}]}
@@ -313,7 +311,7 @@ describe('isbn-issn', () => {
       });
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         {
           tag: '020',
           ind1: ' ',
@@ -353,7 +351,7 @@ describe('isbn-issn', () => {
       });
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         {tag: '005', value: 'whatever'},
         {tag: '020', ind1: ' ', ind2: ' ', subfields: [{code: 'q', value: 'sidottu'}]},
         {tag: '024', ind1: ' ', ind2: ' ', subfields: [{code: 'a', value: ' 9786003770171'}]}
@@ -392,7 +390,7 @@ describe('isbn-issn', () => {
 
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         {tag: '020', ind1: ' ', ind2: ' ', subfields: [{code: 'a', value: '978-9916-605-32-5'}]},
         {tag: '020', ind1: ' ', ind2: ' ', subfields: [{code: 'a', value: '91-7153-086-X'}]},
         {tag: '020', ind1: ' ', ind2: ' ', subfields: [{code: 'a', value: '90-6831-372-X'}]}, // corrected hyphens

package/src/{item-language.spec.js → item-language.test.js}

@@ -1,25 +1,24 @@
-import chai from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import assert from 'node:assert';
 import {MarcRecord} from '@natlibfi/marc-record';
-import validatorFactory from '../src/item-language';
-
-const {expect} = chai;
-chai.use(chaiAsPromised);
+import validatorFactory from '../src/item-language.js';
+import {describe, it} from 'node:test';
 
 describe('item-language', () => {
   it('Creates a validator', async () => {
     const validator = await validatorFactory(/^520$/u);
 
-    expect(validator)
-      .to.be.an('object')
-      .that.has.any.keys('description', 'validate');
-
-    expect(validator.description).to.be.a('string');
-    expect(validator.validate).to.be.a('function');
+    assert.equal(typeof validator, 'object');
+    assert.equal(typeof validator.description, 'string');
+    assert.equal(typeof validator.validate, 'function');
   });
 
   it('Throws an error when tagPattern is not provided', async () => {
-    await expect(validatorFactory()).to.be.rejectedWith(Error, 'No tagPattern provided');
+    await assert.rejects(validatorFactory(), (err) => {
+      assert.equal(err instanceof Error, true);
+      assert.equal(err.message, 'No tagPattern provided');
+      return true;
+     });
+    //await assert(validatorFactory()).to.be.rejectedWith(Error, 'No tagPattern provided');
   });
 
   describe('#validate', () => {
@@ -48,7 +47,7 @@ describe('item-language', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({valid: true});
+      assert.deepEqual(result, {valid: true});
     });
 
     it('Finds the record invalid (Language code is missing and detection failed', async () => {
@@ -70,7 +69,7 @@ describe('item-language', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({valid: false, messages: ['Language detection failed']});
+      assert.deepEqual(result, {valid: false, messages: ['Language detection failed']});
     });
 
     it('Finds the record invalid (Detected language differs)', async () => {
@@ -102,7 +101,7 @@ describe('item-language', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({valid: false, messages: ['Item language code is invalid. Correct language code: eng']});
+      assert.deepEqual(result, {valid: false, messages: ['Item language code is invalid. Correct language code: eng']});
     });
 
     it('Finds the record invalid (Probability doesn\'t meet treshold)', async () => {
@@ -134,7 +133,7 @@ describe('item-language', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({valid: true, messages: ['Item language code is invalid. Current code: fin, suggestions: eng']});
+      assert.deepEqual(result, {valid: true, messages: ['Item language code is invalid. Current code: fin, suggestions: eng']});
     });
 
     it('Finds the record invalid (No detectable text)', async () => {
@@ -155,7 +154,7 @@ describe('item-language', () => {
       });
       const result = await validator.validate(record);
 
-      expect(result).to.eql({valid: true, messages: ['Language detection failed']});
+      assert.deepEqual(result, {valid: true, messages: ['Language detection failed']});
     });
   });
 
@@ -189,7 +188,7 @@ describe('item-language', () => {
       });
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         {
           tag: '008',
           value: '151118t20162016fi^a|||^^^^^^^|0|^0|eng|^'
@@ -237,7 +236,7 @@ describe('item-language', () => {
       });
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         {
           tag: '008',
           value: '151118t20162016fi^a|||^^^^^^^|0|^0|eng|^'
@@ -291,7 +290,7 @@ describe('item-language', () => {
       });
       await validator.fix(record);
 
-      expect(record.fields).to.eql([
+      assert.deepEqual(record.fields, [
         {
           tag: '008',
           value: '151118t20162016fi^a|||^^^^^^^|0|^0|eng|^'
@@ -337,7 +336,7 @@ describe('item-language', () => {
       try {
         await validator.fix(record);
       } catch (err) {
-        expect(err.message).to.equal('Language code is missing and detection failed');
+        assert.equal(err.message, 'Language code is missing and detection failed');
       }
     });
   });

package/src/melindaCustomMergeFields.js

@@ -1,4 +1,4 @@
-export const melindaCustomMergeFields = {'fields':
+export const melindaFieldSpecs = {'fields':
   [
     {
       'tag': 'leader',

package/src/merge-fields/controlSubfields.js

@@ -3,7 +3,7 @@ import createDebugLogger from 'debug';
 import {fieldHasSubfield, fieldToString, nvdebug, nvdebugSubfieldArray, subfieldIsRepeatable, subfieldsAreIdentical} from '../utils.js';
 
 //import {normalizeControlSubfieldValue} from './normalizeIdentifier';
-import {normalizeControlSubfieldValue} from '../normalize-identifiers';
+import {normalizeControlSubfieldValue} from '../normalize-identifiers.js';
 
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:merge-fields:controlSubfields');
 //const debugData = debug.extend('data');

package/src/merge-fields/counterpartField.js

@@ -1,16 +1,17 @@
 // For each incoming field that
 
 import createDebugLogger from 'debug';
-import {fieldHasSubfield, fieldHasNSubfields, fieldHasMultipleSubfields, fieldToString, nvdebug, removeCopyright} from '../utils';
-import {cloneAndNormalizeFieldForComparison, cloneAndRemovePunctuation} from '../normalizeFieldForComparison';
+import {fieldHasSubfield, fieldHasNSubfields, fieldHasMultipleSubfields, fieldToString, nvdebug, removeCopyright} from '../utils.js';
+import {cloneAndNormalizeFieldForComparison, cloneAndRemovePunctuation} from '../normalizeFieldForComparison.js';
 // This should be done via our own normalizer:
-import {normalizeControlSubfieldValue} from '../normalize-identifiers';
+import {normalizeControlSubfieldValue} from '../normalize-identifiers.js';
 
-import {getMergeConstraintsForTag} from './mergeConstraints';
-import {controlSubfieldsPermitMerge} from './controlSubfields';
-import {mergableIndicator1, mergableIndicator2} from './mergableIndicator';
-import {partsAgree} from '../normalizeSubfieldValueForComparison';
-import {normalizeForSamenessCheck, valueCarriesMeaning} from './worldKnowledge';
+import {getMergeConstraintsForTag} from './mergeConstraints.js';
+import {controlSubfieldsPermitMerge} from './controlSubfields.js';
+import {mergableIndicator1, mergableIndicator2} from './mergableIndicator.js';
+import {partsAgree} from '../normalizeSubfieldValueForComparison.js';
+import {normalizeForSamenessCheck, valueCarriesMeaning} from './worldKnowledge.js';
+import { provenanceSubfieldsPermitMerge } from './dataProvenance.js';
 
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:mergeField:counterpart');
 //const debugData = debug.extend('data');
@@ -201,7 +202,6 @@ function counterpartExtraNormalize(tag, subfieldCode, value) {
   value = value.replace(/http:\/\//ug, 'https://'); // MET-501: http vs https
   value = normalizeForSamenessCheck(tag, subfieldCode, value);
 
-  /* eslint-enable */
   return value;
 }
 
@@ -376,6 +376,11 @@ function syntacticallyMergablePair(baseField, sourceField, config) {
     return false;
   }
 
+  if (!provenanceSubfieldsPermitMerge(baseField, sourceField)) {
+    nvdebug('non-mergable (reason: data provenance subfield)', debugDev);
+    return false;
+  }
+
   // NB! field1.tag and field2.tag might differ (1XX vs 7XX). Therefore required subfields might theoretically differ as well.
   // Note: Theoretically 260 $efg vs 264 with IND2=3 has already been handled by the preprocessor.
   // Thus check both:

package/src/merge-fields/dataProvenance.js

@@ -0,0 +1,41 @@
+// See https://www.loc.gov/marc/bibliographic/bdapndxj.html for details
+
+import {subfieldArraysContainSameData} from "../utils.js";
+
+export function tagToDataProvenanceSubfieldCode(tag) {
+    if ( ['533', '800', '810', '811', '830'].includes(tag)) {
+        return 'y';
+    }
+    if ( tag === '856' || tag === '857' ) {
+        return 'e';
+    }
+
+    if ( tag.match(/^7[678]/u) ) {
+        return 'l'
+    }
+
+    if ( tag.match(/^00/u)) {
+        return undefined;
+    }
+    return '7';
+}
+
+
+export function provenanceSubfieldsPermitMerge(baseField, sourceField) {
+    const provinanceSubfieldCode = tagToDataProvenanceSubfieldCode(baseField.tag);
+    if (!baseField.subfields) {
+        return true;
+    }
+    if (provinanceSubfieldCode === undefined) {
+        return false;
+    }
+
+    const baseProvinanceSubfields = baseField.subfields.filter(sf => sf.code === provinanceSubfieldCode);
+    const sourceProvinanceSubfields = sourceField.subfields.filter(sf => sf.code === provinanceSubfieldCode);
+
+    // Currently we just compare two arrays. Later on we might do something more sophisticated with specific $7 data provenance category/relationship codes,
+    // or actual values.
+
+    return subfieldArraysContainSameData(baseProvinanceSubfields, sourceProvinanceSubfields);
+
+}

package/src/merge-fields/index.js

@@ -8,7 +8,7 @@
 //import createDebugLogger from 'debug';
 //import fs from 'fs';
 //import path from 'path';
-import {mergeField} from './mergeField';
+import {mergeField} from './mergeField.js';
 import {MarcRecord} from '@natlibfi/marc-record';
 import {postprocessRecords} from './mergeOrAddPostprocess.js';
 
@@ -18,16 +18,25 @@ import {mergeConfig as defaultConfig} from './mergeConfig.js';
 
 //const defaultConfig = JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'merge-fields', 'config.json'), 'utf8'));
 
-export default function () {
+export default function (defaultTagPattern = undefined) {
 
   return {
     description, validate, fix
   };
 
+  function getTagPattern(config) {
+    if (config && config.tagPattern) {
+      return config.tagPattern;
+    }
+    if (defaultTagPattern) { // Used by tests
+      return defaultTagPattern;
+    }
+    return '^[1678](?:00|10|11|30)$';
+  }
 
   function mergeFieldsWithinRecord(record, config) {
     //const candFields = record.fields.toReversed(); // Node 20+ only! Filter via config?
-    const fields = config && config.tagPattern ? record.get(config.tagPattern) : record.get(/^[1678](?:00|10|11|30)$/u);
+    const fields = record.get(getTagPattern(config)); // config && config.tagPattern ? record.get(config.tagPattern) : record.get(/^[1678](?:00|10|11|30)$/u);
 
     fields.reverse();
     const mergedField = fields.find(f => mergeField(record, record, f, config));

package/src/merge-fields/mergableIndicator.js

@@ -1,5 +1,5 @@
 import createDebugLogger from 'debug';
-import {marc21GetTagsLegalInd1Value, marc21GetTagsLegalInd2Value, nvdebug} from '../utils';
+import {marc21GetTagsLegalInd1Value, marc21GetTagsLegalInd2Value, nvdebug} from '../utils.js';
 
 // Specs: https://workgroups.helsinki.fi/x/K1ohCw (though we occasionally differ from them)...
 

package/src/merge-fields/mergeField.js

@@ -1,12 +1,12 @@
 //import {MarcRecord} from '@natlibfi/marc-record';
 import createDebugLogger from 'debug';
-import {fieldToString, fieldsToString, fieldsAreIdentical, nvdebug, hasCopyright, removeCopyright, subfieldToString} from '../utils';
+import {fieldToString, fieldsToString, fieldsAreIdentical, nvdebug, hasCopyright, removeCopyright, subfieldToString} from '../utils.js';
 import {fieldGetOccurrenceNumberPairs} from '../subfield6Utils.js';
-import {cloneAndNormalizeFieldForComparison, cloneAndRemovePunctuation, isEnnakkotietoSubfieldG} from '../normalizeFieldForComparison';
-import {mergeOrAddSubfield} from './mergeOrAddSubfield';
-import {mergeIndicators} from './mergeIndicator';
-import {mergableTag} from './mergableTag';
-import {getCounterpart} from './counterpartField';
+import {cloneAndNormalizeFieldForComparison, cloneAndRemovePunctuation, isEnnakkotietoSubfieldG} from '../normalizeFieldForComparison.js';
+import {mergeOrAddSubfield} from './mergeOrAddSubfield.js';
+import {mergeIndicators} from './mergeIndicator.js';
+import {mergableTag} from './mergableTag.js';
+import {getCounterpart} from './counterpartField.js';
 //import {default as normalizeEncoding} from '@natlibfi/marc-record-validators-melinda/dist/normalize-utf8-diacritics';
 //import {postprocessRecords} from './mergeOrAddPostprocess.js';
 //import {preprocessBeforeAdd} from './processFilter.js';
@@ -106,8 +106,8 @@ function skipMergeField(baseRecord, sourceField, config) {
     return true;
   }
 
-  // Skip duplicate field:
-  if (baseRecord.fields.some(baseField => !baseField.mergeCandidate && fieldsAreIdentical(sourceField, baseField))) {
+  // Skip duplicate field when merging two records (NB! internal merge merges/removes the duplicate field):
+  if (!baseRecord.internalMerge && baseRecord.fields.some(baseField => !baseField.mergeCandidate && fieldsAreIdentical(sourceField, baseField))) {
     nvdebug(`skipMergeField(): field '${fieldToString(sourceField)}' already exists! No merge required!`, debugDev);
     sourceField.deleted = 1;
     return true;

package/src/merge-fields/mergeIndicator.js

@@ -1,5 +1,5 @@
 import createDebugLogger from 'debug';
-import {fieldToString, marc21GetTagsLegalInd1Value, marc21GetTagsLegalInd2Value, nvdebug} from '../utils';
+import {fieldToString, marc21GetTagsLegalInd1Value, marc21GetTagsLegalInd2Value, nvdebug} from '../utils.js';
 
 // Specs: https://workgroups.helsinki.fi/x/K1ohCw (though we occasionally differ from them)...
 

package/src/merge-fields/mergeOrAddPostprocess.js

@@ -1,10 +1,10 @@
 // This field should be renamed, as it is called also from outside megre.
 
 //import {MarcRecord} from '@natlibfi/marc-record';
-import {fieldFixPunctuation} from '../punctuation2';
-import {fieldRemoveDuplicateSubfields} from './removeDuplicateSubfields';
-import {sortAdjacentSubfields} from '../sortSubfields';
-import {sortAdjacentRelatorTerms} from '../sortRelatorTerms';
+import {fieldFixPunctuation} from '../punctuation2.js';
+import {fieldRemoveDuplicateSubfields} from './removeDuplicateSubfields.js';
+import {sortAdjacentSubfields} from '../sortSubfields.js';
+import {sortAdjacentRelatorTerms} from '../sortRelatorTerms.js';
 
 function postprocessBaseRecord(base) {
 

package/src/merge-fields/mergeOrAddSubfield.js

@@ -1,9 +1,9 @@
 import createDebugLogger from 'debug';
 import {cloneAndNormalizeFieldForComparison, isEnnakkotietoSubfieldG} from '../normalizeFieldForComparison.js';
-import {normalizeAs, normalizeControlSubfieldValue} from '../normalize-identifiers';
+import {normalizeAs, normalizeControlSubfieldValue} from '../normalize-identifiers.js';
 import {fieldHasSubfield, fieldToString, isControlSubfieldCode, nvdebug, subfieldIsRepeatable, subfieldToString} from '../utils.js';
 import {mergeSubfield} from './mergeSubfield.js';
-import {sortAdjacentSubfields} from '../sortSubfields'; //'./sortSubfields.js';
+import {sortAdjacentSubfields} from '../sortSubfields.js';
 
 import {valueCarriesMeaning} from './worldKnowledge.js';
 import {resetSubfield6Tag} from '../subfield6Utils.js';

package/src/merge-fields/mergeSubfield.js

@@ -1,9 +1,9 @@
 import createDebugLogger from 'debug';
-import {partsAgree, subfieldContainsPartData} from '../normalizeSubfieldValueForComparison';
-import {valueCarriesMeaning} from './worldKnowledge';
-import {nvdebug} from '../utils';
+import {partsAgree, subfieldContainsPartData} from '../normalizeSubfieldValueForComparison.js';
+import {valueCarriesMeaning} from './worldKnowledge.js';
+import {nvdebug} from '../utils.js';
 import {tagAndSubfieldCodeReferToIsbn} from '../normalizeFieldForComparison.js';
-import {canContainOptionalQualifier, splitToNameAndQualifier} from './counterpartField';
+import {canContainOptionalQualifier, splitToNameAndQualifier} from './counterpartField.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:mergeSubfield');
 //const debugData = debug.extend('data');