@natlibfi/marc-record-merge 6.0.0-beta.1 → 6.0.0-beta.10

package/src/reducers/copy.js

@@ -1,88 +1,251 @@
-/**
-*
-* @licstart  The following is the entire license notice for the JavaScript code in this file.
-*
-* Merge MARC records
-*
-* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
-*
-* This file is part of marc-record-merge-js
-
-* marc-record-merge-js program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* marc-record-merge-js is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public License
-* along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*
-* @licend  The above is the entire license notice
-* for the JavaScript code in this file.
-*
-*/
-// Import createDebugLogger from 'debug';
-
-export default (pattern) => (base, source) => {
-//  Const debug = createDebugLogger('@natlibfi/marc-record-merge');
-  const baseFields = base.get(pattern);
-  const sourceFields = source.get(pattern);
-  return copyFields();
-
-  function copyFields() {
-    // Test 01: If base does not contain the field at all, it is copied from source to base
-    if (baseFields.length === 0) {
-      sourceFields.forEach(f => base.insertField(f));
-      return base;
+/* eslint-disable max-statements */
+/* eslint-disable no-unused-vars */
+
+import {MarcRecord} from '@natlibfi/marc-record';
+import createDebugLogger from 'debug';
+
+export default ({
+  tagPattern,
+  compareTagsOnly = false,
+  compareWithoutTag = false,
+  compareWithoutIndicators = false,
+  compareWithoutIndicator1 = false,
+  compareWithoutIndicator2 = false,
+  subfieldsMustBeIdentical = true,
+  excludeSubfields = [],
+  dropSubfields = [],
+  copyUnless = [],
+  baseValidators = {subfieldValues: false},
+  sourceValidators = {subfieldValues: false},
+  swapTag = [],
+  swapSubfieldCode = [],
+  doNotCopyIfFieldPresent = false
+}) => (base, source) => {
+  const baseRecord = new MarcRecord(base, baseValidators);
+  const sourceRecord = new MarcRecord(source, sourceValidators);
+
+  const debug = createDebugLogger('@natlibfi/marc-record-merge');
+  const debugOptions = createDebugLogger('@natlibfi/marc-record-merge:compare-options');
+  const debugCompare = createDebugLogger('@natlibfi/marc-record-merge:compare');
+
+  const ignoreInd1 = compareWithoutIndicators || compareWithoutIndicator1;
+  const ignoreInd2 = compareWithoutIndicators || compareWithoutIndicator2;
+
+  debugOptions(`Tag Pattern: ${tagPattern}`);
+  debugOptions(`Compare tags only: ${compareTagsOnly}`);
+  debugOptions(`Omit indicator 1 from comparison: ${ignoreInd1}`);
+  debugOptions(`Omit indicator 2 from comparison: ${ignoreInd2}`);
+  debugOptions(`Copy if identical: ${subfieldsMustBeIdentical}`);
+  debugOptions(`Exclude subfields: [${excludeSubfields}]`);
+  debugOptions(`Drop subfields [${dropSubfields}]`);
+  debugOptions(`Copy unless contains subfields: ${JSON.stringify(copyUnless)}`);
+
+  const baseFields = baseRecord.get(tagPattern);
+  const sourceFields = sourceRecord.get(tagPattern);
+  const doNotCopy = doNotCopyIfFieldPresent ? baseRecord.get(doNotCopyIfFieldPresent).length > 0 : false;
+
+  if (doNotCopy) {
+    return baseRecord.toObject();
+  }
+  debug(`FFS: ${compareWithoutIndicator1}, ${compareWithoutIndicators}, ${ignoreInd1}`);
+  debug(`Base fields: `, baseFields);
+  debug(`Source fields: `, sourceFields);
+
+  // Logic steps
+  const baseCompareFields = baseFields.map(baseField => createCompareField(baseField));
+  const compareResultFields = compareFields(sourceFields, baseCompareFields);
+  const droppedUnwantedSubfield = checkDropSubfields(compareResultFields);
+  const droppedUnwantedFields = checkCopyUnlessFields(droppedUnwantedSubfield);
+  const swappedSubfields = checkSwapSubfieldCodes(droppedUnwantedFields);
+  const swappedTags = checkSwapTag(swappedSubfields);
+  const uniqueFields = [...new Set(swappedTags.map(field => JSON.stringify(field)))].map(field => JSON.parse(field));
+  debug('Fields to be copied');
+  debug(JSON.stringify(uniqueFields));
+
+  // Add fields to base;
+  uniqueFields.forEach(field => baseRecord.insertField(field));
+  return baseRecord.toObject();
+
+  function compareFields(sourceFields, baseCompareFields, uniqFields = []) {
+    const [sourceField, ...rest] = sourceFields;
+    if (sourceField === undefined) {
+      return uniqFields;
     }
-    const filterMissing = function(sourceField) {
-      // Test 02: Identical control fields are not copied
-      if ('value' in sourceField) {
-        return baseFields.some(isIdenticalControlField) === false;
-      }
-      // Test 04: Identical data fields in base and source, not copied
-      // Test 05: Different data fields are copied from source to base
-      if ('subfields' in sourceField) {
-        return baseFields.some(isIdenticalDataField) === false;
-      }
 
-      function normalizeControlField(field) {
-        return field.value.toLowerCase().replace(/\s+/u, '');
-      }
+    if (baseCompareFields.length === 0) {
+      return compareFields(rest, baseCompareFields, [...uniqFields, sourceField]);
+    }
 
-      function isIdenticalControlField(baseField) {
-        const normalizedBaseField = normalizeControlField(baseField);
-        const normalizedSourceField = normalizeControlField(sourceField);
-        return normalizedSourceField === normalizedBaseField;
-      }
-      function isIdenticalDataField(baseField) {
-        if (sourceField.tag === baseField.tag &&
-          sourceField.ind1 === baseField.ind1 &&
-          sourceField.ind2 === baseField.ind2 &&
-          sourceField.subfields.length === baseField.subfields.length) {
-          return baseField.subfields.every(isIdenticalSubfield);
+    // Source and base are also compared for identicalness
+    // Non-identical fields are copied from source to base as duplicates
+    const sourceCompareField = createCompareField(sourceField);
+    const unique = checkCompareFields(baseCompareFields, sourceCompareField);
+
+    debugCompare(`${JSON.stringify(sourceField)} ${unique ? 'is UNIQUE' : 'not UNIQUE'}`);
+
+    if (unique) {
+      return compareFields(rest, baseCompareFields, [...uniqFields, sourceField]);
+    }
+
+    return compareFields(rest, baseCompareFields, uniqFields);
+
+    function checkCompareFields(baseCompareFields, sourceCompareField) {
+      let unique = true; // eslint-disable-line functional/no-let
+
+      baseCompareFields.forEach(baseCompareField => {
+        debugCompare(`Comparing ${JSON.stringify(sourceCompareField)} to ${JSON.stringify(baseCompareField)}}`);
+
+        if (sourceCompareField.value !== baseCompareField.value) {
+          debugCompare(`Value is different ${sourceCompareField.value} !== ${baseCompareField.value}`);
+          return;
+        }
+
+        if (sourceCompareField.ind1 !== baseCompareField.ind1) {
+          debugCompare(`Ind1 is different ${sourceCompareField.ind1} !== ${baseCompareField.ind1}`);
+          return;
         }
-        function normalizeSubfield(subfield) {
-          return subfield.value.toLowerCase().replace(/\s+/u, '');
+
+        if (sourceCompareField.ind2 !== baseCompareField.ind2) {
+          debugCompare(`Ind2 is different ${sourceCompareField.ind2} !== ${baseCompareField.ind2}`);
+          return;
         }
-        function isIdenticalSubfield(baseSub) {
-          const normBaseSub = normalizeSubfield(baseSub);
-          return sourceField.subfields.some(sourceSub => {
-            const normSourceSub = normalizeSubfield(sourceSub);
-            return normSourceSub === normBaseSub;
-          });
+
+        if ('subfields' in sourceCompareField) {
+          const allFound = checkSubfields(sourceCompareField.subfields, baseCompareField.subfields);
+          debugCompare(`Subfields are different ${!allFound}`);
+          if (!allFound) {
+            return;
+          }
+
+          unique = false;
+          return;
         }
+
+        unique = false;
+        return;
+      });
+
+      return unique;
+    }
+
+    function checkSubfields(sourceSubfields, baseSubfields) {
+      const foundSubs = sourceSubfields.filter(sSub => baseSubfields.some(bSub => sSub.code === bSub.code && sSub.value === bSub.value));
+
+      if (subfieldsMustBeIdentical) {
+        return foundSubs.length === sourceSubfields.length && foundSubs.length === baseSubfields.length;
       }
-    };
-    // Search for fields missing from base
-    const missingFields = sourceFields.filter(filterMissing);
-    // Test 03: Add missing control field to base
-    // Test 05: Add missing data field to base
-    missingFields.forEach(f => base.insertField(f));
-    return base; // This is returned by copyFields
+
+      return foundSubs.length === sourceSubfields.length;
+    }
+  }
+
+  // compare objects have only fields that matter in comparison
+  function createCompareField(field) {
+    if (compareTagsOnly) {
+      return {tag: field.tag};
+    }
+
+    if ('value' in field) {
+      return {tag: field.tag, value: field.value};
+    }
+
+    const [filteredField] = checkDropSubfields([field]);
+    const [foundRule] = swapTag.filter(rule => new RegExp(rule.from, 'u').test(field.tag));
+    const replacementTag = foundRule ? foundRule.to : undefined;
+
+    const params = [
+      {name: 'tag', value: compareWithoutTag ? replacementTag : field.tag},
+      {name: 'ind1', value: ignoreInd1 ? undefined : field.ind1},
+      {name: 'ind2', value: ignoreInd2 ? undefined : field.ind2},
+      {name: 'subfields', value: createCompareSubfields(filteredField.subfields)}
+    ].map(param => [param.name, param.value]);
+
+    return Object.fromEntries(params);
+
+    function createCompareSubfields(subfields) {
+      const nonExcludedSubfields = subfields.filter(sub => !excludeSubfields.some(code => code === sub.code));
+      const normalizedSubfields = nonExcludedSubfields.map(sub => ({code: sub.code, value: normalizeSubfieldValue(sub.value)}));
+
+      return normalizedSubfields;
+
+      function normalizeSubfieldValue(value) {
+        return value.toLowerCase().replace(/\s+/ug, '');
+      }
+    }
+  }
+
+  function checkSwapTag(fields) {
+    if (swapTag.length > 0) {
+      return fields.map(field => ({...field, tag: swapTagsFunc(field.tag)}));
+    }
+
+    return fields;
+
+    function swapTagsFunc(tag) {
+      const [foundRule] = swapTag.filter(rule => new RegExp(rule.from, 'u').test(tag));
+
+      if (foundRule === undefined) {
+        return tag;
+      }
+
+      return foundRule.to;
+    }
+  }
+
+  function checkSwapSubfieldCodes(fields) {
+    if (swapSubfieldCode.length > 0) {
+      return fields.map(field => ({...field, subfields: swapSubfieldCodesFunc(field.subfields)}));
+    }
+
+    return fields;
+
+    function swapSubfieldCodesFunc(subfields) {
+      return subfields.map(sub => {
+        const [foundRule] = swapSubfieldCode.filter(rule => rule.from === sub.code);
+
+        if (foundRule === undefined) {
+          return sub;
+        }
+
+        return {code: foundRule.to, value: sub.value};
+      });
+    }
+  }
+
+  function checkDropSubfields(fields) {
+    if (dropSubfields.length > 0) {
+      return fields.map(field => ({...field, subfields: dropSubfieldsFunc(field.subfields)}))
+        .filter(field => field.subfields.length > 0);
+    }
+
+    return fields;
+
+    function dropSubfieldsFunc(subfields) {
+      return subfields.filter(sub => { // eslint-disable-line
+        return !dropSubfields.some(({code, value = false, condition = false}) => {
+          if (code !== sub.code) {
+            return false;
+          }
+
+          if (!condition && value) {
+            return value === sub.value;
+          }
+
+          if (condition === 'unless' && value) {
+            return !new RegExp(value, 'u').test(sub.value);
+          }
+
+          return true;
+        });
+      });
+    }
+  }
+
+  function checkCopyUnlessFields(fields) {
+    if (copyUnless.length > 0) {
+      return fields.filter(({subfields}) => copyUnless.some(filter => !subfields.some(sub => sub.code === filter.code && new RegExp(filter.value, 'u').test(sub.value))));
+    }
+
+    return fields;
   }
 };

package/src/reducers/copy.spec.js

@@ -1,52 +1,54 @@
-/**
-*
-* @licstart  The following is the entire license notice for the JavaScript code in this file.
-*
-* Merge MARC records
-*
-* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
-*
-* This file is part of marc-record-merge-js
-
-* marc-record-merge-js program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* marc-record-merge-js is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public License
-* along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*
-* @licend  The above is the entire license notice
-* for the JavaScript code in this file.
-*
-*/
-import chai from 'chai';
-import fs from 'fs';
-import path from 'path';
-import {MarcRecord} from '@natlibfi/marc-record';
+import {expect} from 'chai';
+import {READERS} from '@natlibfi/fixura';
 import createReducer from './copy';
-import fixturesFactory, {READERS} from '@natlibfi/fixura';
-
-MarcRecord.setValidationOptions({subfieldValues: false});
+import generateTests from '@natlibfi/fixugen';
 
-describe('reducers/copy', () => {
-  const {expect} = chai;
-  const fixturesPath = path.join(__dirname, '..', '..', 'test-fixtures', 'reducers', 'copy');
+//import createDebugLogger from 'debug'; // <---
+//const debug = createDebugLogger('@natlibfi/marc-record-merge/copy.spec.js'); // <---
 
-  fs.readdirSync(fixturesPath).forEach(subDir => {
-    const {getFixture} = fixturesFactory({root: [fixturesPath, subDir], reader: READERS.JSON});
-    it(subDir, () => {
-      const baseTest = new MarcRecord(getFixture('base.json'));
-      const sourceTest = new MarcRecord(getFixture('source.json'));
-      const patternTest = new RegExp(getFixture({components: ['pattern.txt'], reader: READERS.TEXT}), 'u');
-      const expectedRecord = getFixture('merged.json');
-      const mergedRecord = createReducer(patternTest)(baseTest, sourceTest);
-      expect(mergedRecord.toObject()).to.eql(expectedRecord);
-    });
-  });
+generateTests({
+  callback,
+  path: [__dirname, '..', '..', 'test-fixtures', 'reducers', 'copy'],
+  useMetadataFile: true,
+  recurse: true,
+  fixura: {
+    reader: READERS.JSON,
+    failWhenNotFound: false
+  }
 });
+
+function callback({
+  getFixture,
+  tagPatternRegExp,
+  compareTagsOnly = false,
+  compareWithoutTag = false,
+  compareWithoutIndicators = false,
+  compareWithoutIndicator1 = false,
+  compareWithoutIndicator2 = false,
+  subfieldsMustBeIdentical = false,
+  copyUnless = undefined,
+  excludeSubfields = undefined,
+  dropSubfields = undefined,
+  swapSubfieldCode = [],
+  swapTag = [],
+  doNotCopyIfFieldPresent = false
+}) {
+  const base = getFixture('base.json');
+  const source = getFixture('source.json');
+  const tagPattern = new RegExp(tagPatternRegExp, 'u');
+  const expectedRecord = getFixture('merged.json');
+
+  const merged = createReducer({
+    tagPattern, compareTagsOnly, compareWithoutTag, compareWithoutIndicators, compareWithoutIndicator1, compareWithoutIndicator2,
+    copyUnless, subfieldsMustBeIdentical, excludeSubfields,
+    dropSubfields, swapSubfieldCode, swapTag,
+    doNotCopyIfFieldPresent
+  })(base, source);
+  //debug(`***     mergedRecord: `, mergedRecord); //<--
+  //debug(`***     mergedRecord,Strfy: `, JSON.stringify(mergedRecord)); //<--
+  //debug(`***     expectedRecord: `, expectedRecord); //<--
+  //debug(`***     expectedRecord,Strfy: `, JSON.stringify(expectedRecord)); //<--
+  expect(merged.constructor.name).not.to.eql('MarcRecord');
+  expect(merged.constructor.name).to.eql('Object');
+  expect(merged).to.eql(expectedRecord);
+}

package/src/reducers/index.js

@@ -1,31 +1,4 @@
-/**
-*
-* @licstart  The following is the entire license notice for the JavaScript code in this file.
-*
-* Merge MARC records
-*
-* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
-*
-* This file is part of marc-record-merge-js
-
-* marc-record-merge-js program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* marc-record-merge-js is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public License
-* along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*
-* @licend  The above is the entire license notice
-* for the JavaScript code in this file.
-*
-*/
-
-export * from './copy';
-export * from './select';
+import copy from './copy';
+import select, {strictEquality, subsetEquality} from './select';
 
+export default {copy, select, strictEquality, subsetEquality};

package/src/reducers/select.js

@@ -1,56 +1,40 @@
-/**
-*
-* @licstart  The following is the entire license notice for the JavaScript code in this file.
-*
-* Merge MARC records
-*
-* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
-*
-* This file is part of marc-record-merge-js
-
-* marc-record-merge-js program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* marc-record-merge-js is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public License
-* along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*
-* @licend  The above is the entire license notice
-* for the JavaScript code in this file.
-*
-*/
 import {normalizeSync} from 'normalize-diacritics';
+import createDebugLogger from 'debug';
+import {MarcRecord} from '@natlibfi/marc-record';
 
 export function strictEquality(subfieldA, subfieldB) {
   return subfieldA.code === subfieldB.code &&
-  subfieldA.value === subfieldB.value;
+    subfieldA.value === subfieldB.value;
 }
 
 export function subsetEquality(subfieldA, subfieldB) {
   return subfieldA.code === subfieldB.code &&
-  (subfieldA.value.indexOf(subfieldB.value) !== -1 || subfieldB.value.indexOf(subfieldA.value) !== -1);
+    (subfieldA.value.indexOf(subfieldB.value) !== -1 || subfieldB.value.indexOf(subfieldA.value) !== -1);
 }
-
-export default ({pattern, equalityFunction = strictEquality}) => (base, source) => {
-  const baseFields = base.get(pattern);
-  const sourceFields = source.get(pattern);
+// EqualityFunction can be either strictEquality or subsetEquality
+export default ({tagPattern, equalityFunction = strictEquality}) => (base, source) => {
+  const debug = createDebugLogger('@natlibfi/marc-record-merge:select');
+  const baseRecord = new MarcRecord(base, {subfieldValues: false});
+  const sourceRecord = new MarcRecord(source, {subfieldValues: false});
+  const baseFields = baseRecord.get(tagPattern);
+  const sourceFields = sourceRecord.get(tagPattern);
+  const fieldTag = sourceFields.map(field => field.tag);
+  debug(`Comparing field ${fieldTag}`);
 
   checkFieldType(baseFields);
   checkFieldType(sourceFields);
 
   if (baseFields.length > 1 || sourceFields.length > 1) {
+    debug(`Multiple fields in base or source`);
+    debug(`No changes to base`);
     return base;
   }
   const [baseField] = baseFields;
   const [sourceField] = sourceFields;
 
   if (baseField.tag === sourceField.tag === false) {
+    debug(`Base tag ${baseField.tag} is not equal to source tag ${sourceField.tag}`);
+    debug(`No changes to base`);
     return base;
   }
   const baseSubs = baseField.subfields;
@@ -62,19 +46,26 @@ export default ({pattern, equalityFunction = strictEquality}) => (base, source)
   const sourceSubsNormalized = sourceSubs
     .map(({code, value}) => ({code, value: normalizeSubfieldValue(value)}));
 
+  // Returns the base subfields for which a matching source subfield is found
   const equalSubfieldsBase = baseSubsNormalized
     .filter(baseSubfield => sourceSubsNormalized
       .some(sourceSubfield => equalityFunction(baseSubfield, sourceSubfield)));
+  debug(`equalSubfieldsBase: ${JSON.stringify(equalSubfieldsBase, undefined, 2)}`);
 
+  // Returns the source subfields for which a matching base subfield is found
   const equalSubfieldsSource = sourceSubsNormalized
     .filter(sourceSubfield => baseSubsNormalized
       .some(baseSubfield => equalityFunction(sourceSubfield, baseSubfield)));
+  debug(`equalSubfieldsSource: ${JSON.stringify(equalSubfieldsSource, undefined, 2)}`);
 
   if (baseSubs.length === sourceSubs.length && equalSubfieldsBase.length < baseSubs.length) {
+    debug(`Base and source subfields are not equal`);
+    debug(`No changes to base`);
     return base;
   }
 
   if (baseSubs.length === sourceSubs.length && equalSubfieldsBase.length === equalSubfieldsSource.length) {
+    debug(`Checking subfield equality`);
     const totalSubfieldLengthBase = baseSubsNormalized
       .map(({value}) => value.length)
       .reduce((acc, value) => acc + value);
@@ -91,11 +82,13 @@ export default ({pattern, equalityFunction = strictEquality}) => (base, source)
     return replaceBasefieldWithSourcefield(base);
   }
 
+  debug(`No changes to base`);
   return base;
 
   function replaceBasefieldWithSourcefield(base) {
     const index = base.fields.findIndex(field => field === baseField);
     base.fields.splice(index, 1, sourceField); // eslint-disable-line functional/immutable-data
+    debug(`Source field is longer, replacing base with source`);
     return base;
   }
 
@@ -111,7 +104,7 @@ export default ({pattern, equalityFunction = strictEquality}) => (base, source)
 
   function normalizeSubfieldValue(value) {
     // Regexp options: g: global search, u: unicode
-    const punctuation = /[.,\-/#!$%^&*;:{}=_`~()[\]]/gu;
+    const punctuation = /[.,\-/#!?$%^&*;:{}=_`~()[\]]/gu;
     return normalizeSync(value).toLowerCase().replace(punctuation, '', 'u').replace(/\s+/gu, ' ').trim();
   }
 };