@natlibfi/marc-record-merge 6.0.0-beta.2 → 6.0.0-beta.6

package/package.json

@@ -13,44 +13,50 @@
 		"type": "git",
 		"url": "git@github.com:natlibfi/marc-record-merge-js.git"
 	},
-	"license": "LGPL-3.0+",
-	"version": "6.0.0-beta.2",
+	"license": "MIT",
+	"version": "6.0.0-beta.6",
 	"main": "./dist/index.js",
 	"engines": {
-		"node": ">=12"
+		"node": ">=14"
 	},
+	"type": "commonjs",
 	"scripts": {
+		"prepare": "npm run build",
 		"prepublishOnly": "npm run build:transpile",
 		"lint": "eslint ./src",
 		"lint:dev": "eslint ./src --fix",
-		"test:base": "cross-env NODE_ENV=test nyc mocha --require @babel/register",
-		"test": "npm run lint && npm run test:base -- src/**/*.spec.js",
-		"test:dev": "npm run lint:dev && npm run test:base -- src/**/*.spec.js && npm run coverage",
-		"coverage": "nyc check-coverage --per-file",
+		"test:base": "cross-env NODE_ENV=test mocha --require @babel/register --reporter-option maxDiffSize=15000 src/reducers/*.spec.js",
+		"test": "npm run lint && npm run test:base",
+		"test:dev": "npm run lint:dev && npm run coverage",
+		"coverage": "npm run coverage:unit && npm run coverage:report",
+		"coverage:unit": "nyc --silent npm run test:base",
+		"coverage:report": "nyc report",
 		"build:transpile": "babel src --copy-files --source-maps --delete-dir-on-start --out-dir=dist",
 		"build": "npm run build:transpile",
-		"watch:test": "cross-env DEBUG=1 NODE_ENV=test nodemon -w src -w test-fixtures --exec 'npm run test:dev'"
+		"dev:test": "cross-env NODE_ENV=test nodemon -w src -w test-fixtures --exec 'clear && npm run test:dev'",
+		"dev:test:debug": "cross-env DEBUG=@natlibfi/* NODE_ENV=test nodemon -w src -w test-fixtures --exec 'clear && npm run test:dev'"
 	},
 	"dependencies": {
-		"@natlibfi/fixura": "^2.1.3",
-		"@natlibfi/marc-record": "^6.1.1",
-		"debug": "^4.3.1",
-		"normalize-diacritics": "^2.13.2"
+		"@natlibfi/fixugen": "1.1.0",
+		"@natlibfi/fixura": "^2.2.1",
+		"@natlibfi/marc-record": "^7.1.0-alpha.2",
+		"debug": "^4.3.4",
+		"normalize-diacritics": "2.14.0"
 	},
 	"devDependencies": {
-		"@babel/cli": "^7.14.5",
-		"@babel/core": "^7.14.6",
-		"@babel/eslint-parser": "^7.14.5",
-		"@babel/preset-env": "^7.14.5",
-		"@babel/register": "^7.14.5",
-		"@natlibfi/eslint-config-melinda-backend": "^1.0.6",
-		"babel-plugin-istanbul": "^6.0.0",
+		"@babel/cli": "^7.17.10",
+		"@babel/core": "^7.18.0",
+		"@babel/eslint-parser": "^7.17.0",
+		"@babel/preset-env": "^7.18.0",
+		"@babel/register": "^7.17.7",
+		"@natlibfi/eslint-config-melinda-backend": "^2.0.0",
+		"babel-plugin-istanbul": "^6.1.1",
 		"babel-plugin-rewire": "^1.2.0",
 		"chai": "^4.3.4",
 		"cross-env": "^7.0.3",
-		"eslint": "^7.28.0",
-		"mocha": "^9.0.0",
-		"nodemon": "^2.0.7",
+		"eslint": "^8.16.0",
+		"mocha": "^10.0.0",
+		"nodemon": "^2.0.16",
 		"nyc": "^15.1.0"
 	},
 	"eslintConfig": {
@@ -63,9 +69,7 @@
 			[
 				"@babel/preset-env",
 				{
-					"targets": {
-						"node": "12"
-					}
+					"targets": "maintained node versions"
 				}
 			]
 		],
@@ -84,8 +88,7 @@
 			"*/**/*.spec.js"
 		],
 		"reporter": [
-			"lcov",
-			"html"
+			"text"
 		],
 		"require": [
 			"@babel/register"

package/src/index.js

@@ -1,36 +1,4 @@
-/**
-*
-* @licstart  The following is the entire license notice for the JavaScript code in this file.
-*
-* Merge MARC records
-*
-* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
-*
-* This file is part of marc-record-merge-js
-
-* marc-record-merge-js program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* marc-record-merge-js is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public License
-* along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*
-* @licend  The above is the entire license notice
-* for the JavaScript code in this file.
-*
-*/
-
-import {MarcRecord} from '@natlibfi/marc-record';
 import Reducers from './reducers';
 
 export {Reducers};
-export default ({base, source, reducers, baseValidators = {}, sourceValidators = {}}) => {
-  const sourceRecord = MarcRecord.clone(source, sourceValidators);
-  return reducers.reduce((baseRecord, reducer) => reducer(baseRecord, sourceRecord), MarcRecord.clone(base, baseValidators));
-};
+export default ({base, source, reducers}) => reducers.reduce((base, reducer) => reducer(base, source), base);

package/src/reducers/copy.js

@@ -1,140 +1,331 @@
-/**
-*
-* @licstart  The following is the entire license notice for the JavaScript code in this file.
-*
-* Merge MARC records
-*
-* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
-*
-* This file is part of marc-record-merge-js
-
-* marc-record-merge-js program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as
-* published by the Free Software Foundation, either version 3 of the
-* License, or (at your option) any later version.
-*
-* marc-record-merge-js is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public License
-* along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*
-* @licend  The above is the entire license notice
-* for the JavaScript code in this file.
-*
-*/
-
-/**
- * Test 01: If base does not contain the field at all, it is copied from source to base
- * Test 02: Identical control fields are not copied
- * Test 03: Add missing control field to base
- * Test 04: Identical data fields in base and source, not copied
- * Test 05: Different data fields are copied from source to base (multiple fields)
- * Test 06: compareTagsOnly: Field is copied from source only if it is missing in base, a different instance is not copied
- * Test 07: excludeSubfields: Ignore excluded subfields in comparing identicalness
- * Test 08: dropSubfields: Drop subfields from source before copying
- * Test 09: compareTagsOnly for repeatable fields, 2x each 260/264
- * */
+/* eslint-disable max-statements */
+/* eslint-disable no-unused-vars */
+
+import {MarcRecord} from '@natlibfi/marc-record';
 import createDebugLogger from 'debug';
 
-export default ({tagPattern, compareTagsOnly = false, excludeSubfields = [], dropSubfields = []}) => (base, source) => {
+export default ({
+  tagPattern,
+  compareTagsOnly = false,
+  compareWithoutIndicators = false,
+  subfieldsMustBeIdentical = true,
+  excludeSubfields = [],
+  dropSubfields = [],
+  copyUnless = [],
+  baseValidators = {subfieldValues: false},
+  sourceValidators = {subfieldValues: false},
+  swapTag = [],
+  swapSubfieldCode = [],
+  doNotCopyIfFieldPresent = false
+}) => (base, source) => {
+  const baseRecord = new MarcRecord(base, baseValidators);
+  const sourceRecord = new MarcRecord(source, sourceValidators);
+
   const debug = createDebugLogger('@natlibfi/marc-record-merge');
-  const baseFields = base.get(tagPattern);
-  // Check whether there are subfields to drop from source before copying
-  const sourceFields = checkDropSubfields(source.get(tagPattern));
-  return copyFields();
-
-  function copyFields() {
-    const sourceTags = sourceFields.map(field => field.tag);
-    sourceTags.forEach(tag => debug(`Comparing field ${tag}`));
-
-    // If compareTagsOnly = true, only this part is run
-    // The field is copied from source only if it is missing completely from base
-    if (baseFields.length === 0) {
-      sourceTags.forEach(tag => debug(`Missing field ${tag} copied from source to base`));
-      sourceFields.forEach(f => base.insertField(f));
-      return base;
+  const debugOptions = createDebugLogger('@natlibfi/marc-record-merge:compare-options');
+  const debugCompare = createDebugLogger('@natlibfi/marc-record-merge:compare');
+  debugOptions(`Tag Pattern: ${tagPattern}`);
+  debugOptions(`Compare tags only: ${compareTagsOnly}`);
+  debugOptions(`Compare without indicators ${compareWithoutIndicators}`);
+  debugOptions(`Copy if identical: ${subfieldsMustBeIdentical}`);
+  debugOptions(`Exclude subfields: [${excludeSubfields}]`);
+  debugOptions(`Drop subfields [${dropSubfields}]`);
+  debugOptions(`Copy unless contains subfields: ${JSON.stringify(copyUnless)}`);
+
+  const baseFields = baseRecord.get(tagPattern);
+  const sourceFields = sourceRecord.get(tagPattern);
+  const doNotCopy = doNotCopyIfFieldPresent ? baseRecord.get(doNotCopyIfFieldPresent).length > 0 : false;
+
+  if (doNotCopy) {
+    return baseRecord.toObject();
+  }
+
+  debug(`Base fields: `, baseFields);
+  debug(`Source fields: `, sourceFields);
+
+  const baseCompareFields = baseFields.map(baseField => createCompareField(baseField));
+  const compareResultFields = compareFields(sourceFields, baseCompareFields);
+  const droppedUnwantedSubfield = checkDropSubfields(compareResultFields);
+  const droppedUnwantedFields = checkCopyUnlessFields(droppedUnwantedSubfield);
+  const swappedSubfields = checkSwapSubfieldCodes(droppedUnwantedFields);
+  const swappedTags = checkSwapTag(swappedSubfields);
+  debug('Fields to be copied');
+  debug(JSON.stringify(swappedTags));
+
+  // Add fields to base;
+  swappedTags.forEach(field => baseRecord.insertField(field));
+  return baseRecord.toObject();
+  //return copyFields(baseFields, sourceFields);
+
+  function compareFields(sourceFields, baseCompareFields, uniqFields = []) {
+    const [sourceField, ...rest] = sourceFields;
+    if (sourceField === undefined) {
+      return uniqFields;
+    }
+
+    if (baseCompareFields.length === 0) {
+      return compareFields(rest, baseCompareFields, [...uniqFields, sourceField]);
     }
 
-    // If compareTagsOnly = false (default)
     // Source and base are also compared for identicalness
     // Non-identical fields are copied from source to base as duplicates
-    if (!compareTagsOnly) {
-      const filterMissing = function(sourceField) {
-        if ('value' in sourceField) {
-          debug(`Checking control field ${sourceField.tag} for identicalness`);
-          return baseFields.some(isIdenticalControlField) === false;
-        }
-        if ('subfields' in sourceField) {
-          debug(`Checking data field ${sourceField.tag} for identicalness`);
-          return baseFields.some(isIdenticalDataField) === false;
+    const sourceCompareField = createCompareField(sourceField);
+    const unique = checkCompareFields(baseCompareFields, sourceCompareField);
+
+    debugCompare(`${JSON.stringify(sourceField)} ${unique ? 'is UNIQUE' : 'not UNIQUE'}`);
+
+    if (unique) {
+      return compareFields(rest, baseCompareFields, [...uniqFields, sourceField]);
+    }
+
+    return compareFields(rest, baseCompareFields, uniqFields);
+
+    function checkCompareFields(baseCompareFields, sourceCompareField) {
+      let unique = true; // eslint-disable-line functional/no-let
+
+      baseCompareFields.forEach(baseCompareField => {
+        debugCompare(`Comparing ${JSON.stringify(sourceCompareField)} to ${JSON.stringify(baseCompareField)}}`);
+
+        if (sourceCompareField.value !== baseCompareField.value) {
+          debugCompare(`Value is different ${sourceCompareField.value} !== ${baseCompareField.value}`);
+          return;
         }
 
-        function normalizeControlField(field) {
-          return field.value.toLowerCase().replace(/\s+/u, '');
+        if (sourceCompareField.ind1 !== baseCompareField.ind1) {
+          debugCompare(`Ind1 is different ${sourceCompareField.ind1} !== ${baseCompareField.ind1}`);
+          return;
         }
 
-        function isIdenticalControlField(baseField) {
-          const normalizedBaseField = normalizeControlField(baseField);
-          const normalizedSourceField = normalizeControlField(sourceField);
-          return normalizedSourceField === normalizedBaseField;
+        if (sourceCompareField.ind2 !== baseCompareField.ind2) {
+          debugCompare(`Ind2 is different ${sourceCompareField.ind2} !== ${baseCompareField.ind2}`);
+          return;
         }
-        function isIdenticalDataField(baseField) {
-          // If excluded subfields have been defined for this field, they must be ignored first
-          // (i.e. source and base fields are considered identical if all non-excluded subfields are identical)
-          if (excludeSubfields.length > 0 &&
-            sourceField.tag === baseField.tag &&
-            sourceField.ind1 === baseField.ind1 &&
-            sourceField.ind2 === baseField.ind2) {
-            excludeSubfields.forEach(sub => debug(`Subfield ${sub} excluded from identicalness comparison`));
-            // Compare only those subfields that are not excluded
-            const baseSubsToCompare = baseField.subfields.filter(subfield => excludeSubfields.indexOf(subfield.code) === -1);
-            return baseSubsToCompare.every(isIdenticalSubfield);
-          }
-          // If there are no excluded subfields (default case)
-          if (sourceField.tag === baseField.tag &&
-              sourceField.ind1 === baseField.ind1 &&
-              sourceField.ind2 === baseField.ind2 &&
-              sourceField.subfields.length === baseField.subfields.length) {
-            return baseField.subfields.every(isIdenticalSubfield);
-          }
-          function normalizeSubfield(subfield) {
-            return subfield.value.toLowerCase().replace(/\s+/u, '');
-          }
-          function isIdenticalSubfield(baseSub) {
-            const normBaseSub = normalizeSubfield(baseSub);
-            return sourceField.subfields.some(sourceSub => {
-              const normSourceSub = normalizeSubfield(sourceSub);
-              return normSourceSub === normBaseSub;
-            });
+
+        if ('subfields' in sourceCompareField) {
+          const allFound = checkSubfields(sourceCompareField.subfields, baseCompareField.subfields);
+          debugCompare(`Subfields are different ${!allFound}`);
+          if (!allFound) {
+            return;
           }
+
+          unique = false;
+          return;
         }
-      };
-      // Search for fields missing from base
-      const missingFields = sourceFields.filter(filterMissing);
-      missingFields.forEach(f => base.insertField(f));
-      if (missingFields.length > 0) {
-        const missingTags = missingFields.map(field => field.tag);
-        missingTags.forEach(tag => debug(`Field ${tag} copied from source to base`));
-        return base;
+
+        unique = false;
+        return;
+      });
+
+      return unique;
+    }
+
+    function checkSubfields(sourceSubfields, baseSubfields) {
+      const foundSubs = sourceSubfields.filter(sSub => baseSubfields.some(bSub => sSub.code === bSub.code && sSub.value === bSub.value));
+
+      if (subfieldsMustBeIdentical) {
+        return foundSubs.length === sourceSubfields.length && foundSubs.length === baseSubfields.length;
       }
-      if (missingFields.length === 0) {
-        debug(`No missing fields found`);
-        return base;
+
+      return foundSubs.length === sourceSubfields.length;
+    }
+  }
+
+  function createCompareField(field) {
+    if (compareTagsOnly) {
+      return {tag: field.tag};
+    }
+
+    if ('value' in field) {
+      return {tag: field.tag, value: field.value};
+    }
+
+    const [filteredField] = checkDropSubfields([field]);
+
+    const params = [
+      {name: 'tag', value: field.tag},
+      {name: 'ind1', value: compareWithoutIndicators ? undefined : field.ind1},
+      {name: 'ind2', value: compareWithoutIndicators ? undefined : field.ind2},
+      {name: 'subfields', value: createCompareSubfields(filteredField.subfields)}
+    ].map(param => [param.name, param.value]);
+
+    return Object.fromEntries(params);
+
+    function createCompareSubfields(subfields) {
+      const nonExcludedSubfields = subfields.filter(sub => !excludeSubfields.some(code => code === sub.code));
+      const normalizedSubfields = nonExcludedSubfields.map(sub => ({code: sub.code, value: normalizeSubfieldValue(sub.value)}));
+
+      return normalizedSubfields;
+
+      function normalizeSubfieldValue(value) {
+        return value.toLowerCase().replace(/\s+/ug, '');
       }
     }
-    debug(`No missing fields found`);
-    return base;
+  }
+
+  function checkSwapTag(fields) {
+    if (swapTag.length > 0) {
+      return fields.map(field => ({...field, tag: swapTagsFunc(field.tag)}));
+    }
+
+    return fields;
+
+    function swapTagsFunc(tag) {
+      const [foundRule] = swapTag.filter(rule => new RegExp(rule.from, 'u').test(tag));
+
+      if (foundRule === undefined) {
+        return tag;
+      }
+
+      return foundRule.to;
+    }
+  }
+
+  function checkSwapSubfieldCodes(fields) {
+    if (swapSubfieldCode.length > 0) {
+      return fields.map(field => ({...field, subfields: swapSubfieldCodesFunc(field.subfields)}));
+    }
+
+    return fields;
+
+    function swapSubfieldCodesFunc(subfields) {
+      return subfields.map(sub => {
+        const [foundRule] = swapSubfieldCode.filter(rule => rule.from === sub.code);
+
+        if (foundRule === undefined) {
+          return sub;
+        }
+
+        return {code: foundRule.to, value: sub.value};
+      });
+    }
   }
 
   function checkDropSubfields(fields) {
     if (dropSubfields.length > 0) {
-      dropSubfields.forEach(sub => debug(`Subfield ${sub} dropped from source field before copying`));
-      return fields.map((field) => ({...field, subfields: field.subfields.filter((subfield) => dropSubfields.indexOf(subfield.code) === -1)}));
+      return fields.map(field => ({...field, subfields: dropSubfieldsFunc(field.subfields)}))
+        .filter(field => field.subfields.length > 0);
     }
-    debug(`No subfields to drop`);
+
+    return fields;
+
+    function dropSubfieldsFunc(subfields) {
+      return subfields.filter(sub => { // eslint-disable-line
+        return !dropSubfields.some(({code, value = false, condition = false}) => {
+          if (code !== sub.code) {
+            return false;
+          }
+
+          if (!condition && value) {
+            return value === sub.value;
+          }
+
+          if (condition === 'unless' && value) {
+            return !new RegExp(value, 'u').test(sub.value);
+          }
+
+          return true;
+        });
+      });
+    }
+  }
+
+  function checkCopyUnlessFields(fields) {
+    if (copyUnless.length > 0) {
+      return fields.filter(({subfields}) => copyUnless.some(filter => !subfields.some(sub => sub.code === filter.code && new RegExp(filter.value, 'u').test(sub.value))));
+    }
+
     return fields;
   }
 };
+
+// function copyFields() { //eslint-disable-line no-unused-vars
+//   const sourceTags = sourceFields.map(field => field.tag);
+//   sourceTags.forEach(tag => debug(`Comparing field ${tag}`));
+
+//   /*
+//   if (combine.length > 0) {
+//     debug(`*** NOW Copy options: ${tagPattern}, ${compareTagsOnly}, ${compareWithoutIndicators}, ${subfieldsMustBeIdentical}, [${combine}], [${excludeSubfields}], [${dropSubfields}]`);
+//     combine.forEach(row => debug(` ### combine ${row} <- `));
+//     return [];
+//   }
+//   */
+
+//   // If compareTagsOnly = true, only this part is run
+//   // The field is copied from source only if it is missing completely from base
+//   if (compareTagsOnly && baseFields.length === 0) {
+//     sourceTags.forEach(tag => debug(`Missing field ${tag} copied from source to base`));
+//     sourceFields.forEach(f => base.insertField(f));
+//     return true;
+//   }
+
+//   // If compareTagsOnly = false (default)
+//   // Source and base are also compared for identicalness
+//   // Non-identical fields are copied from source to base as duplicates
+//   if (!compareTagsOnly) {
+//     const filterMissing = function (sourceField) {
+//       if ('value' in sourceField) {
+//         debug(`Checking control field ${sourceField.tag} for identicalness`);
+//         return baseFields.some(isIdenticalControlField) === false;
+//       }
+//       if ('subfields' in sourceField) {
+//         debug(`Checking data field ${sourceField.tag} for identicalness`);
+//         return baseFields.some(isIdenticalDataField) === false;
+//       }
+
+//       function normalizeControlField(field) {
+//         return field.value.toLowerCase().replace(/\s+/u, '');
+//       }
+
+//       function isIdenticalControlField(baseField) {
+//         const normalizedBaseField = normalizeControlField(baseField);
+//         const normalizedSourceField = normalizeControlField(sourceField);
+//         return normalizedSourceField === normalizedBaseField;
+//       }
+
+//       function isIdenticalDataField(baseField) {
+//         // If excluded subfields have been defined for this field, they must be ignored first
+//         // (i.e. source and base fields are considered identical if all non-excluded subfields are identical)
+//         if (excludeSubfields.length > 0 &&
+//           sourceField.tag === baseField.tag &&
+//           sourceField.ind1 === baseField.ind1 &&
+//           sourceField.ind2 === baseField.ind2) {
+//           excludeSubfields.forEach(sub => debug(`Subfield ${sub} excluded from identicalness comparison`));
+//           // Compare only those subfields that are not excluded
+//           const baseSubsToCompare = baseField.subfields.filter(subfield => excludeSubfields.indexOf(subfield.code) === -1);
+//           return baseSubsToCompare.every(isIdenticalSubfield);
+//         }
+//         // If there are no excluded subfields (default case)
+//         if (sourceField.tag === baseField.tag &&
+//           sourceField.ind1 === baseField.ind1 &&
+//           sourceField.ind2 === baseField.ind2 &&
+//           sourceField.subfields.length === baseField.subfields.length) {
+//           return baseField.subfields.every(isIdenticalSubfield);
+//         }
+//         function normalizeSubfield(subfield) {
+//           return subfield.value.toLowerCase().replace(/\s+/u, '');
+//         }
+//         function isIdenticalSubfield(baseSub) {
+//           const normBaseSub = normalizeSubfield(baseSub);
+//           return sourceField.subfields.some(sourceSub => {
+//             const normSourceSub = normalizeSubfield(sourceSub);
+//             return normSourceSub === normBaseSub;
+//           });
+//         }
+//       }
+//     };
+//     // Search for fields missing from base
+//     const missingFields = sourceFields.filter(filterMissing);
+//     missingFields.forEach(f => base.insertField(f));
+//     if (missingFields.length > 0) {
+//       const missingTags = missingFields.map(field => field.tag);
+//       missingTags.forEach(tag => debug(`Field ${tag} copied from source to base`));
+//       return base;
+//     }
+//     if (missingFields.length === 0) {
+//       debug(`No missing fields found`);
+//       return base;
+//     }
+//   }
+//   debug(`No missing fields found`);
+//   return base;
+// }