@natlibfi/marc-record-merge 6.0.0-beta.2 → 6.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CODEOWNERS +9 -0
- package/.github/dependabot.yml +41 -0
- package/.github/workflows/melinda-node-tests.yml +60 -0
- package/LICENSE +21 -0
- package/README.md +92 -2
- package/dist/index.js +2 -37
- package/dist/index.js.map +1 -1
- package/dist/reducers/copy.js +246 -111
- package/dist/reducers/copy.js.map +1 -1
- package/dist/reducers/copy.spec.js +53 -99
- package/dist/reducers/copy.spec.js.map +1 -1
- package/dist/reducers/index.js +0 -27
- package/dist/reducers/index.js.map +1 -1
- package/dist/reducers/select.js +2 -48
- package/dist/reducers/select.js.map +1 -1
- package/dist/reducers/select.spec.js +49 -83
- package/dist/reducers/select.spec.js.map +1 -1
- package/package.json +27 -22
- package/src/index.js +1 -33
- package/src/reducers/copy.js +253 -119
- package/src/reducers/copy.spec.js +41 -65
- package/src/reducers/index.js +0 -27
- package/src/reducers/select.js +1 -47
- package/src/reducers/select.spec.js +37 -58
- package/test-fixtures/reducers/copy/{01/base.json → basic copy/01/base.json } +1 -1
- package/test-fixtures/reducers/copy/{01 → basic copy/01}/merged.json +0 -0
- package/test-fixtures/reducers/copy/basic copy/01/metadata.json +5 -0
- package/test-fixtures/reducers/copy/{01 → basic copy/01}/source.json +0 -0
- package/test-fixtures/reducers/copy/{02/source.json → basic copy/02/base.json} +0 -0
- package/test-fixtures/reducers/copy/{02 → basic copy/02}/merged.json +0 -0
- package/test-fixtures/reducers/copy/basic copy/02/metadata.json +5 -0
- package/test-fixtures/reducers/copy/{02/base.json → basic copy/02/source.json } +1 -1
- package/test-fixtures/reducers/copy/{03/base.json → basic copy/03/base.json } +1 -1
- package/test-fixtures/reducers/copy/{03 → basic copy/03}/merged.json +0 -0
- package/test-fixtures/reducers/copy/basic copy/03/metadata.json +5 -0
- package/test-fixtures/reducers/copy/{03 → basic copy/03}/source.json +0 -0
- package/test-fixtures/reducers/copy/{05/base.json → basic copy/04/base.json } +1 -1
- package/test-fixtures/reducers/copy/{04/merged.json → basic copy/04/merged.json } +1 -1
- package/test-fixtures/reducers/copy/basic copy/04/metadata.json +5 -0
- package/test-fixtures/reducers/copy/{04/source.json → basic copy/04/source.json } +1 -1
- package/test-fixtures/reducers/copy/{04/base.json → basic copy/05/base.json } +1 -1
- package/test-fixtures/reducers/copy/{05 → basic copy/05}/merged.json +0 -0
- package/test-fixtures/reducers/copy/basic copy/05/metadata.json +5 -0
- package/test-fixtures/reducers/copy/{05 → basic copy/05}/source.json +0 -0
- package/test-fixtures/reducers/copy/basic copy/06/base.json +24 -0
- package/test-fixtures/reducers/copy/basic copy/06/merged.json +39 -0
- package/test-fixtures/reducers/copy/basic copy/06/metadata.json +5 -0
- package/test-fixtures/reducers/copy/basic copy/06/source.json +24 -0
- package/test-fixtures/reducers/copy/{06 → compareTagsOnly/01}/base.json +0 -0
- package/test-fixtures/reducers/copy/{06 → compareTagsOnly/01}/merged.json +0 -0
- package/test-fixtures/reducers/copy/compareTagsOnly/01/metadata.json +6 -0
- package/test-fixtures/reducers/copy/{06 → compareTagsOnly/01}/source.json +0 -0
- package/test-fixtures/reducers/copy/{09 → compareTagsOnly/02}/base.json +0 -0
- package/test-fixtures/reducers/copy/{09 → compareTagsOnly/02}/merged.json +0 -0
- package/test-fixtures/reducers/copy/compareTagsOnly/02/metadata.json +6 -0
- package/test-fixtures/reducers/copy/{09 → compareTagsOnly/02}/source.json +0 -0
- package/test-fixtures/reducers/copy/compareWithoutIndicators/01/base.json +24 -0
- package/test-fixtures/reducers/copy/compareWithoutIndicators/01/merged.json +24 -0
- package/test-fixtures/reducers/copy/compareWithoutIndicators/01/metadata.json +6 -0
- package/test-fixtures/reducers/copy/compareWithoutIndicators/01/source.json +24 -0
- package/test-fixtures/reducers/copy/compareWithoutIndicators/02/base.json +24 -0
- package/test-fixtures/reducers/copy/compareWithoutIndicators/02/merged.json +39 -0
- package/test-fixtures/reducers/copy/compareWithoutIndicators/02/metadata.json +6 -0
- package/test-fixtures/reducers/copy/compareWithoutIndicators/02/source.json +24 -0
- package/test-fixtures/reducers/copy/copyUnless/01/base.json +9 -0
- package/test-fixtures/reducers/copy/copyUnless/01/merged.json +24 -0
- package/test-fixtures/reducers/copy/copyUnless/01/metadata.json +6 -0
- package/test-fixtures/reducers/copy/copyUnless/01/source.json +47 -0
- package/test-fixtures/reducers/copy/{08 → dropSubfields/01}/base.json +0 -0
- package/test-fixtures/reducers/copy/{08 → dropSubfields/01}/merged.json +8 -0
- package/test-fixtures/reducers/copy/dropSubfields/01/metadata.json +6 -0
- package/test-fixtures/reducers/copy/{08 → dropSubfields/01}/source.json +0 -0
- package/test-fixtures/reducers/copy/dropSubfields/02/base.json +43 -0
- package/test-fixtures/reducers/copy/dropSubfields/02/merged.json +58 -0
- package/test-fixtures/reducers/copy/dropSubfields/02/metadata.json +6 -0
- package/test-fixtures/reducers/copy/dropSubfields/02/source.json +32 -0
- package/test-fixtures/reducers/copy/dropSubfields/03/base.json +43 -0
- package/test-fixtures/reducers/copy/dropSubfields/03/merged.json +58 -0
- package/test-fixtures/reducers/copy/dropSubfields/03/metadata.json +6 -0
- package/test-fixtures/reducers/copy/dropSubfields/03/source.json +32 -0
- package/test-fixtures/reducers/copy/dropSubfields/04/base.json +43 -0
- package/test-fixtures/reducers/copy/dropSubfields/04/merged.json +43 -0
- package/test-fixtures/reducers/copy/dropSubfields/04/metadata.json +6 -0
- package/test-fixtures/reducers/copy/dropSubfields/04/source.json +32 -0
- package/test-fixtures/reducers/copy/{07 → excludeSubfields/01}/base.json +0 -0
- package/test-fixtures/reducers/copy/{07 → excludeSubfields/01}/merged.json +0 -0
- package/test-fixtures/reducers/copy/excludeSubfields/01/metadata.json +9 -0
- package/test-fixtures/reducers/copy/{07 → excludeSubfields/01}/source.json +0 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/01/base.json +24 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/01/merged.json +39 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/01/metadata.json +6 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/01/source.json +24 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/02/base.json +24 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/02/merged.json +24 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/02/metadata.json +6 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/02/source.json +24 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/03/base.json +28 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/03/merged.json +43 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/03/metadata.json +6 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/03/source.json +24 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/04/base.json +28 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/04/merged.json +28 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/04/metadata.json +6 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/04/source.json +24 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/05/base.json +20 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/05/merged.json +35 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/05/metadata.json +6 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/05/source.json +24 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/06/base.json +20 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/06/merged.json +35 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/06/metadata.json +6 -0
- package/test-fixtures/reducers/copy/subfieldsMustBeIdentical/06/source.json +24 -0
- package/test-fixtures/reducers/metadata.json +5 -0
- package/test-fixtures/reducers/select/01/metadata.json +5 -0
- package/test-fixtures/reducers/select/02/metadata.json +4 -0
- package/test-fixtures/reducers/select/03/metadata.json +4 -0
- package/test-fixtures/reducers/select/04/metadata.json +5 -0
- package/test-fixtures/reducers/select/05/metadata.json +4 -0
- package/test-fixtures/reducers/select/06/metadata.json +4 -0
- package/test-fixtures/reducers/select/07/metadata.json +5 -0
- package/test-fixtures/reducers/select/08/metadata.json +4 -0
- package/test-fixtures/reducers/select/09/metadata.json +4 -0
- package/test-fixtures/reducers/select/10/metadata.json +5 -0
- package/test-fixtures/reducers/select/11/metadata.json +4 -0
- package/test-fixtures/reducers/select/12/metadata.json +5 -0
- package/test-fixtures/reducers/select/13/metadata.json +4 -0
- package/test-fixtures/reducers/select/14/metadata.json +4 -0
- package/.nyc_output/72717fff-b4ac-4aef-a145-37cae88e07f8.json +0 -1
- package/.nyc_output/processinfo/72717fff-b4ac-4aef-a145-37cae88e07f8.json +0 -1
- package/.nyc_output/processinfo/index.json +0 -1
- package/LICENSE.txt +0 -165
- package/coverage/base.css +0 -224
- package/coverage/block-navigation.js +0 -79
- package/coverage/copy.js.html +0 -500
- package/coverage/favicon.png +0 -0
- package/coverage/index.html +0 -126
- package/coverage/lcov-report/base.css +0 -224
- package/coverage/lcov-report/block-navigation.js +0 -79
- package/coverage/lcov-report/copy.js.html +0 -500
- package/coverage/lcov-report/favicon.png +0 -0
- package/coverage/lcov-report/index.html +0 -126
- package/coverage/lcov-report/prettify.css +0 -1
- package/coverage/lcov-report/prettify.js +0 -2
- package/coverage/lcov-report/select.js.html +0 -539
- package/coverage/lcov-report/sort-arrow-sprite.png +0 -0
- package/coverage/lcov-report/sorter.js +0 -170
- package/coverage/lcov.info +0 -274
- package/coverage/prettify.css +0 -1
- package/coverage/prettify.js +0 -2
- package/coverage/select.js.html +0 -539
- package/coverage/sort-arrow-sprite.png +0 -0
- package/coverage/sorter.js +0 -170
- package/test-fixtures/reducers/copy/01/tagPattern.txt +0 -1
- package/test-fixtures/reducers/copy/02/tagPattern.txt +0 -1
- package/test-fixtures/reducers/copy/03/tagPattern.txt +0 -1
- package/test-fixtures/reducers/copy/04/tagPattern.txt +0 -1
- package/test-fixtures/reducers/copy/05/tagPattern.txt +0 -1
- package/test-fixtures/reducers/copy/06/compareTagsOnly.txt +0 -1
- package/test-fixtures/reducers/copy/06/tagPattern.txt +0 -1
- package/test-fixtures/reducers/copy/07/excludeSubfields.json +0 -1
- package/test-fixtures/reducers/copy/07/tagPattern.txt +0 -1
- package/test-fixtures/reducers/copy/08/dropSubfields.json +0 -1
- package/test-fixtures/reducers/copy/08/tagPattern.txt +0 -1
- package/test-fixtures/reducers/copy/09/compareTagsOnly.txt +0 -1
- package/test-fixtures/reducers/copy/09/tagPattern.txt +0 -1
- package/test-fixtures/reducers/select/01/expected-error.txt +0 -1
- package/test-fixtures/reducers/select/01/pattern.txt +0 -1
- package/test-fixtures/reducers/select/02/pattern.txt +0 -1
- package/test-fixtures/reducers/select/03/pattern.txt +0 -1
- package/test-fixtures/reducers/select/04/pattern.txt +0 -1
- package/test-fixtures/reducers/select/05/pattern.txt +0 -1
- package/test-fixtures/reducers/select/06/pattern.txt +0 -1
- package/test-fixtures/reducers/select/07/equalityFunction.txt +0 -1
- package/test-fixtures/reducers/select/07/pattern.txt +0 -1
- package/test-fixtures/reducers/select/08/pattern.txt +0 -1
- package/test-fixtures/reducers/select/09/pattern.txt +0 -1
- package/test-fixtures/reducers/select/10/equalityFunction.txt +0 -1
- package/test-fixtures/reducers/select/10/pattern.txt +0 -1
- package/test-fixtures/reducers/select/11/pattern.txt +0 -1
- package/test-fixtures/reducers/select/12/equalityFunction.txt +0 -1
- package/test-fixtures/reducers/select/12/pattern.txt +0 -1
- package/test-fixtures/reducers/select/13/pattern.txt +0 -1
- package/test-fixtures/reducers/select/14/pattern.txt +0 -1
package/src/reducers/copy.js
CHANGED
|
@@ -1,140 +1,274 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
*
|
|
5
|
-
* Merge MARC records
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of marc-record-merge-js
|
|
10
|
-
|
|
11
|
-
* marc-record-merge-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* marc-record-merge-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Lesser General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Test 01: If base does not contain the field at all, it is copied from source to base
|
|
31
|
-
* Test 02: Identical control fields are not copied
|
|
32
|
-
* Test 03: Add missing control field to base
|
|
33
|
-
* Test 04: Identical data fields in base and source, not copied
|
|
34
|
-
* Test 05: Different data fields are copied from source to base (multiple fields)
|
|
35
|
-
* Test 06: compareTagsOnly: Field is copied from source only if it is missing in base, a different instance is not copied
|
|
36
|
-
* Test 07: excludeSubfields: Ignore excluded subfields in comparing identicalness
|
|
37
|
-
* Test 08: dropSubfields: Drop subfields from source before copying
|
|
38
|
-
* Test 09: compareTagsOnly for repeatable fields, 2x each 260/264
|
|
39
|
-
* */
|
|
1
|
+
/* eslint-disable no-unused-vars */
|
|
2
|
+
|
|
3
|
+
import {MarcRecord} from '@natlibfi/marc-record';
|
|
40
4
|
import createDebugLogger from 'debug';
|
|
41
5
|
|
|
42
|
-
export default ({
|
|
6
|
+
export default ({
|
|
7
|
+
tagPattern,
|
|
8
|
+
compareTagsOnly = false,
|
|
9
|
+
compareWithoutIndicators = false,
|
|
10
|
+
subfieldsMustBeIdentical = true,
|
|
11
|
+
excludeSubfields = [],
|
|
12
|
+
dropSubfields = [],
|
|
13
|
+
copyUnless = [],
|
|
14
|
+
baseValidators = {subfieldValues: false},
|
|
15
|
+
sourceValidators = {subfieldValues: false}
|
|
16
|
+
}) => (base, source) => {
|
|
17
|
+
const baseRecord = new MarcRecord(base, baseValidators);
|
|
18
|
+
const sourceRecord = new MarcRecord(source, sourceValidators);
|
|
19
|
+
|
|
43
20
|
const debug = createDebugLogger('@natlibfi/marc-record-merge');
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
21
|
+
const debugOptions = createDebugLogger('@natlibfi/marc-record-merge:compare-options');
|
|
22
|
+
const debugCompare = createDebugLogger('@natlibfi/marc-record-merge:compare');
|
|
23
|
+
debugOptions(`Tag Pattern: ${tagPattern}`);
|
|
24
|
+
debugOptions(`Compare tags only: ${compareTagsOnly}`);
|
|
25
|
+
debugOptions(`Compare without indicators ${compareWithoutIndicators}`);
|
|
26
|
+
debugOptions(`Copy if identical: ${subfieldsMustBeIdentical}`);
|
|
27
|
+
debugOptions(`Exclude subfields: [${excludeSubfields}]`);
|
|
28
|
+
debugOptions(`Drop subfields [${dropSubfields}]`);
|
|
29
|
+
debugOptions(`Copy unless contains subfields: ${JSON.stringify(copyUnless)}`);
|
|
30
|
+
|
|
31
|
+
const baseFields = baseRecord.get(tagPattern);
|
|
32
|
+
const sourceFields = sourceRecord.get(tagPattern);
|
|
33
|
+
|
|
34
|
+
debug(`Base fields: `, baseFields);
|
|
35
|
+
debug(`Source fields: `, sourceFields);
|
|
48
36
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
37
|
+
const compareResultFields = compareFields(sourceFields, baseFields);
|
|
38
|
+
const droppedUnwantedSubfield = checkDropSubfields(compareResultFields);
|
|
39
|
+
const droppedUnwantedFields = checkCopyUnlessFields(droppedUnwantedSubfield);
|
|
40
|
+
debug('Fields to be copied');
|
|
41
|
+
debug(JSON.stringify(droppedUnwantedFields));
|
|
42
|
+
|
|
43
|
+
// Add fields to base;
|
|
44
|
+
droppedUnwantedFields.forEach(field => baseRecord.insertField(field));
|
|
45
|
+
return baseRecord.toObject();
|
|
46
|
+
//return copyFields(baseFields, sourceFields);
|
|
47
|
+
|
|
48
|
+
function compareFields(sourceFields, baseFields, uniqFields = []) {
|
|
49
|
+
const [sourceField, ...rest] = sourceFields;
|
|
50
|
+
if (sourceField === undefined) {
|
|
51
|
+
return uniqFields;
|
|
52
|
+
}
|
|
52
53
|
|
|
53
|
-
// If compareTagsOnly = true, only this part is run
|
|
54
|
-
// The field is copied from source only if it is missing completely from base
|
|
55
54
|
if (baseFields.length === 0) {
|
|
56
|
-
|
|
57
|
-
sourceFields.forEach(f => base.insertField(f));
|
|
58
|
-
return base;
|
|
55
|
+
return compareFields(rest, baseFields, [...uniqFields, sourceField]);
|
|
59
56
|
}
|
|
60
57
|
|
|
61
|
-
// If compareTagsOnly = false (default)
|
|
62
58
|
// Source and base are also compared for identicalness
|
|
63
59
|
// Non-identical fields are copied from source to base as duplicates
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
};
|
|
115
|
-
// Search for fields missing from base
|
|
116
|
-
const missingFields = sourceFields.filter(filterMissing);
|
|
117
|
-
missingFields.forEach(f => base.insertField(f));
|
|
118
|
-
if (missingFields.length > 0) {
|
|
119
|
-
const missingTags = missingFields.map(field => field.tag);
|
|
120
|
-
missingTags.forEach(tag => debug(`Field ${tag} copied from source to base`));
|
|
121
|
-
return base;
|
|
60
|
+
const sourceComapareField = createCompareField(sourceField);
|
|
61
|
+
const baseCompareFields = baseFields.map(baseField => createCompareField(baseField));
|
|
62
|
+
|
|
63
|
+
const unique = checkCompareFields(baseCompareFields, sourceComapareField);
|
|
64
|
+
|
|
65
|
+
debugCompare(`${JSON.stringify(sourceField)} ${unique ? 'is UNIQUE' : 'not UNIQUE'}`);
|
|
66
|
+
|
|
67
|
+
if (unique) {
|
|
68
|
+
return compareFields(rest, baseFields, [...uniqFields, sourceField]);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return compareFields(rest, baseFields, uniqFields);
|
|
72
|
+
|
|
73
|
+
function checkCompareFields(baseCompareFields, sourceComapareField) {
|
|
74
|
+
const [baseCompareField, ...rest] = baseCompareFields;
|
|
75
|
+
if (baseCompareField === undefined) {
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (sourceComapareField.value !== baseCompareField.value) {
|
|
80
|
+
debugCompare(`Value is different ${sourceComapareField.value} !== ${baseCompareField.value}`);
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (sourceComapareField.ind1 !== baseCompareField.ind1) {
|
|
85
|
+
debugCompare(`Ind1 is different ${sourceComapareField.ind1} !== ${baseCompareField.ind1}`);
|
|
86
|
+
return true;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (sourceComapareField.ind2 !== baseCompareField.ind2) {
|
|
90
|
+
debugCompare(`Ind2 is different ${sourceComapareField.ind2} !== ${baseCompareField.ind2}`);
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if ('subfields' in sourceComapareField) {
|
|
95
|
+
const allFound = checkSubfields(sourceComapareField.subfields, baseCompareField.subfields);
|
|
96
|
+
debugCompare(`Subfields are different ${!allFound}`);
|
|
97
|
+
return allFound ? false : checkCompareFields(rest, sourceComapareField);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function checkSubfields(sourceSubfields, baseSubfields) {
|
|
104
|
+
const foundSubs = sourceSubfields.filter(sSub => baseSubfields.some(bSub => sSub.code === bSub.code && sSub.value === bSub.value));
|
|
105
|
+
|
|
106
|
+
if (subfieldsMustBeIdentical) {
|
|
107
|
+
return foundSubs.length === sourceSubfields.length && foundSubs.length === baseSubfields.length;
|
|
122
108
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
109
|
+
|
|
110
|
+
return foundSubs.length === sourceSubfields.length;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function createCompareField(field) {
|
|
115
|
+
if (compareTagsOnly) {
|
|
116
|
+
return {tag: field.tag};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if ('value' in field) {
|
|
120
|
+
return {tag: field.tag, value: field.value};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const [filteredField] = checkDropSubfields([field]);
|
|
124
|
+
|
|
125
|
+
const params = [
|
|
126
|
+
{name: 'tag', value: field.tag},
|
|
127
|
+
{name: 'ind1', value: compareWithoutIndicators ? undefined : field.ind1},
|
|
128
|
+
{name: 'ind2', value: compareWithoutIndicators ? undefined : field.ind2},
|
|
129
|
+
{name: 'subfields', value: createCompareSubfields(filteredField.subfields)}
|
|
130
|
+
].map(param => [param.name, param.value]);
|
|
131
|
+
|
|
132
|
+
return Object.fromEntries(params);
|
|
133
|
+
|
|
134
|
+
function createCompareSubfields(subfields) {
|
|
135
|
+
const nonExcludedSubfields = subfields.filter(sub => !excludeSubfields.some(code => code === sub.code));
|
|
136
|
+
const normalizedSubfields = nonExcludedSubfields.map(sub => ({code: sub.code, value: normalizeSubfieldValue(sub.value)}));
|
|
137
|
+
|
|
138
|
+
return normalizedSubfields;
|
|
139
|
+
|
|
140
|
+
function normalizeSubfieldValue(value) {
|
|
141
|
+
return value.toLowerCase().replace(/\s+/ug, '');
|
|
126
142
|
}
|
|
127
143
|
}
|
|
128
|
-
debug(`No missing fields found`);
|
|
129
|
-
return base;
|
|
130
144
|
}
|
|
131
145
|
|
|
132
146
|
function checkDropSubfields(fields) {
|
|
133
147
|
if (dropSubfields.length > 0) {
|
|
134
|
-
|
|
135
|
-
|
|
148
|
+
return fields.map(field => ({...field, subfields: dropSubfieldsFunc(field.subfields)}))
|
|
149
|
+
.filter(field => field.subfields.length > 0);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return fields;
|
|
153
|
+
|
|
154
|
+
function dropSubfieldsFunc(subfields) {
|
|
155
|
+
return subfields.filter(sub => { // eslint-disable-line
|
|
156
|
+
return !dropSubfields.some(({code, value = false, condition = false}) => {
|
|
157
|
+
if (code !== sub.code) {
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (!condition && value) {
|
|
162
|
+
return value === sub.value;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (condition === 'unless' && value) {
|
|
166
|
+
return !new RegExp(value, 'u').test(sub.value);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
return true;
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function checkCopyUnlessFields(fields) {
|
|
176
|
+
if (copyUnless.length > 0) {
|
|
177
|
+
return fields.filter(({subfields}) => copyUnless.some(filter => !subfields.some(sub => sub.code === filter.code && new RegExp(filter.value, 'u').test(sub.value))));
|
|
136
178
|
}
|
|
137
|
-
|
|
179
|
+
|
|
138
180
|
return fields;
|
|
139
181
|
}
|
|
140
182
|
};
|
|
183
|
+
|
|
184
|
+
// function copyFields() { //eslint-disable-line no-unused-vars
|
|
185
|
+
// const sourceTags = sourceFields.map(field => field.tag);
|
|
186
|
+
// sourceTags.forEach(tag => debug(`Comparing field ${tag}`));
|
|
187
|
+
|
|
188
|
+
// /*
|
|
189
|
+
// if (combine.length > 0) {
|
|
190
|
+
// debug(`*** NOW Copy options: ${tagPattern}, ${compareTagsOnly}, ${compareWithoutIndicators}, ${subfieldsMustBeIdentical}, [${combine}], [${excludeSubfields}], [${dropSubfields}]`);
|
|
191
|
+
// combine.forEach(row => debug(` ### combine ${row} <- `));
|
|
192
|
+
// return [];
|
|
193
|
+
// }
|
|
194
|
+
// */
|
|
195
|
+
|
|
196
|
+
// // If compareTagsOnly = true, only this part is run
|
|
197
|
+
// // The field is copied from source only if it is missing completely from base
|
|
198
|
+
// if (compareTagsOnly && baseFields.length === 0) {
|
|
199
|
+
// sourceTags.forEach(tag => debug(`Missing field ${tag} copied from source to base`));
|
|
200
|
+
// sourceFields.forEach(f => base.insertField(f));
|
|
201
|
+
// return true;
|
|
202
|
+
// }
|
|
203
|
+
|
|
204
|
+
// // If compareTagsOnly = false (default)
|
|
205
|
+
// // Source and base are also compared for identicalness
|
|
206
|
+
// // Non-identical fields are copied from source to base as duplicates
|
|
207
|
+
// if (!compareTagsOnly) {
|
|
208
|
+
// const filterMissing = function (sourceField) {
|
|
209
|
+
// if ('value' in sourceField) {
|
|
210
|
+
// debug(`Checking control field ${sourceField.tag} for identicalness`);
|
|
211
|
+
// return baseFields.some(isIdenticalControlField) === false;
|
|
212
|
+
// }
|
|
213
|
+
// if ('subfields' in sourceField) {
|
|
214
|
+
// debug(`Checking data field ${sourceField.tag} for identicalness`);
|
|
215
|
+
// return baseFields.some(isIdenticalDataField) === false;
|
|
216
|
+
// }
|
|
217
|
+
|
|
218
|
+
// function normalizeControlField(field) {
|
|
219
|
+
// return field.value.toLowerCase().replace(/\s+/u, '');
|
|
220
|
+
// }
|
|
221
|
+
|
|
222
|
+
// function isIdenticalControlField(baseField) {
|
|
223
|
+
// const normalizedBaseField = normalizeControlField(baseField);
|
|
224
|
+
// const normalizedSourceField = normalizeControlField(sourceField);
|
|
225
|
+
// return normalizedSourceField === normalizedBaseField;
|
|
226
|
+
// }
|
|
227
|
+
|
|
228
|
+
// function isIdenticalDataField(baseField) {
|
|
229
|
+
// // If excluded subfields have been defined for this field, they must be ignored first
|
|
230
|
+
// // (i.e. source and base fields are considered identical if all non-excluded subfields are identical)
|
|
231
|
+
// if (excludeSubfields.length > 0 &&
|
|
232
|
+
// sourceField.tag === baseField.tag &&
|
|
233
|
+
// sourceField.ind1 === baseField.ind1 &&
|
|
234
|
+
// sourceField.ind2 === baseField.ind2) {
|
|
235
|
+
// excludeSubfields.forEach(sub => debug(`Subfield ${sub} excluded from identicalness comparison`));
|
|
236
|
+
// // Compare only those subfields that are not excluded
|
|
237
|
+
// const baseSubsToCompare = baseField.subfields.filter(subfield => excludeSubfields.indexOf(subfield.code) === -1);
|
|
238
|
+
// return baseSubsToCompare.every(isIdenticalSubfield);
|
|
239
|
+
// }
|
|
240
|
+
// // If there are no excluded subfields (default case)
|
|
241
|
+
// if (sourceField.tag === baseField.tag &&
|
|
242
|
+
// sourceField.ind1 === baseField.ind1 &&
|
|
243
|
+
// sourceField.ind2 === baseField.ind2 &&
|
|
244
|
+
// sourceField.subfields.length === baseField.subfields.length) {
|
|
245
|
+
// return baseField.subfields.every(isIdenticalSubfield);
|
|
246
|
+
// }
|
|
247
|
+
// function normalizeSubfield(subfield) {
|
|
248
|
+
// return subfield.value.toLowerCase().replace(/\s+/u, '');
|
|
249
|
+
// }
|
|
250
|
+
// function isIdenticalSubfield(baseSub) {
|
|
251
|
+
// const normBaseSub = normalizeSubfield(baseSub);
|
|
252
|
+
// return sourceField.subfields.some(sourceSub => {
|
|
253
|
+
// const normSourceSub = normalizeSubfield(sourceSub);
|
|
254
|
+
// return normSourceSub === normBaseSub;
|
|
255
|
+
// });
|
|
256
|
+
// }
|
|
257
|
+
// }
|
|
258
|
+
// };
|
|
259
|
+
// // Search for fields missing from base
|
|
260
|
+
// const missingFields = sourceFields.filter(filterMissing);
|
|
261
|
+
// missingFields.forEach(f => base.insertField(f));
|
|
262
|
+
// if (missingFields.length > 0) {
|
|
263
|
+
// const missingTags = missingFields.map(field => field.tag);
|
|
264
|
+
// missingTags.forEach(tag => debug(`Field ${tag} copied from source to base`));
|
|
265
|
+
// return base;
|
|
266
|
+
// }
|
|
267
|
+
// if (missingFields.length === 0) {
|
|
268
|
+
// debug(`No missing fields found`);
|
|
269
|
+
// return base;
|
|
270
|
+
// }
|
|
271
|
+
// }
|
|
272
|
+
// debug(`No missing fields found`);
|
|
273
|
+
// return base;
|
|
274
|
+
// }
|
|
@@ -1,72 +1,48 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Merge MARC records
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of marc-record-merge-js
|
|
10
|
-
|
|
11
|
-
* marc-record-merge-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* marc-record-merge-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Lesser General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
|
-
import chai from 'chai';
|
|
29
|
-
import fs from 'fs';
|
|
30
|
-
import path from 'path';
|
|
1
|
+
import {expect} from 'chai';
|
|
2
|
+
import {READERS} from '@natlibfi/fixura';
|
|
31
3
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
32
4
|
import createReducer from './copy';
|
|
33
|
-
import
|
|
5
|
+
import generateTests from '@natlibfi/fixugen';
|
|
34
6
|
|
|
35
|
-
|
|
7
|
+
// import createDebugLogger from 'debug'; // <---
|
|
8
|
+
// const debug = createDebugLogger('@natlibfi/marc-record-merge/copy.spec.js'); // <---
|
|
36
9
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
10
|
+
generateTests({
|
|
11
|
+
callback,
|
|
12
|
+
path: [__dirname, '..', '..', 'test-fixtures', 'reducers', 'copy'],
|
|
13
|
+
useMetadataFile: true,
|
|
14
|
+
recurse: true,
|
|
15
|
+
fixura: {
|
|
16
|
+
reader: READERS.JSON,
|
|
17
|
+
failWhenNotFound: false
|
|
18
|
+
}
|
|
19
|
+
});
|
|
40
20
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
21
|
+
function callback({
|
|
22
|
+
getFixture,
|
|
23
|
+
tagPatternRegExp,
|
|
24
|
+
compareTagsOnly = false,
|
|
25
|
+
compareWithoutIndicators = false,
|
|
26
|
+
subfieldsMustBeIdentical = false,
|
|
27
|
+
copyUnless = undefined,
|
|
28
|
+
excludeSubfields = undefined,
|
|
29
|
+
dropSubfields = undefined,
|
|
30
|
+
disabled = false
|
|
31
|
+
}) {
|
|
32
|
+
if (disabled) {
|
|
33
|
+
console.log('TEST DISABLED!'); // eslint-disable-line no-console
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
53
36
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
}
|
|
59
|
-
// Check whether excludeSubfields.json exists and if it does, return its contents. If not, do nothing.
|
|
60
|
-
function getExcludeSubfields() {
|
|
61
|
-
const subfieldsToExclude = getFixture({components: ['excludeSubfields.json'], reader: READERS.JSON});
|
|
62
|
-
return subfieldsToExclude ? subfieldsToExclude : undefined;
|
|
63
|
-
}
|
|
37
|
+
const base = new MarcRecord(getFixture('base.json'), {subfieldValues: false});
|
|
38
|
+
const source = new MarcRecord(getFixture('source.json'), {subfieldValues: false});
|
|
39
|
+
const tagPattern = new RegExp(tagPatternRegExp, 'u');
|
|
40
|
+
const expectedRecord = getFixture('merged.json');
|
|
64
41
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
});
|
|
42
|
+
const mergedRecord = createReducer({tagPattern, compareTagsOnly, compareWithoutIndicators, copyUnless, subfieldsMustBeIdentical, excludeSubfields, dropSubfields})(base, source);
|
|
43
|
+
//debug(`*** mergedRecord: `, mergedRecord); //<--
|
|
44
|
+
//debug(`*** mergedRecord,Strfy: `, JSON.stringify(mergedRecord.toObject())); //<--
|
|
45
|
+
//debug(`*** expectedRecord: `, expectedRecord); //<--
|
|
46
|
+
//debug(`*** expectedRecord,Strfy: `, JSON.stringify(expectedRecord)); //<--
|
|
47
|
+
expect(mergedRecord).to.eql(expectedRecord);
|
|
48
|
+
}
|
package/src/reducers/index.js
CHANGED
|
@@ -1,30 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Merge MARC records
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of marc-record-merge-js
|
|
10
|
-
|
|
11
|
-
* marc-record-merge-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* marc-record-merge-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Lesser General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
1
|
import copy from './copy';
|
|
29
2
|
import select, {strictEquality, subsetEquality} from './select';
|
|
30
3
|
|
package/src/reducers/select.js
CHANGED
|
@@ -1,49 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Merge MARC records
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2015-2019 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of marc-record-merge-js
|
|
10
|
-
|
|
11
|
-
* marc-record-merge-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* marc-record-merge-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Lesser General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
|
-
/**
|
|
29
|
-
* Test 01: The field is a control field (contains 'value') --> error message
|
|
30
|
-
* Test 02: If there are multiple fields, return base
|
|
31
|
-
* Test 03: Base and source tags are equal for one field --> return base
|
|
32
|
-
* Test 04: Base and source tags are not equal --> return base
|
|
33
|
-
* Note: to test for this, tagPattern has to allow two tags,
|
|
34
|
-
* otherwise the unequal tag does not even pass source.get(tagPattern)
|
|
35
|
-
* Test 05: Normalize subfield values (base and source are equal) --> return base
|
|
36
|
-
* Test 06: Two subfields, both equal --> return base
|
|
37
|
-
* Test 07: Two subfields, same codes, values of source are subsets of values of base --> replaceBasefieldWithSourcefield
|
|
38
|
-
* Test 08: Two subfields, one is a subset, one has a different code --> return base
|
|
39
|
-
* Test 09: Two subfields, same codes, values are not subsets --> return base
|
|
40
|
-
* Test 10: sourceField is a proper superset of baseField (subfields a and b are equal, c is new) --> replaceBasefieldWithSourcefield
|
|
41
|
-
* Test 11: sourceField is not a proper superset of baseField (different values in a and b, also new subfield c) --> return base
|
|
42
|
-
* Test 12: sourceField is a proper superset of baseField (base values a and b are subsets of source values a and b, c is new in source) --> replaceBasefieldWithSourcefield
|
|
43
|
-
* Test 13: Opposite of test 12, baseField is a proper superset of sourceField --> return base
|
|
44
|
-
* Test 14: Normalization test with Cyrillic characters
|
|
45
|
-
*/
|
|
46
|
-
|
|
47
1
|
import {normalizeSync} from 'normalize-diacritics';
|
|
48
2
|
import createDebugLogger from 'debug';
|
|
49
3
|
|
|
@@ -58,7 +12,7 @@ export function subsetEquality(subfieldA, subfieldB) {
|
|
|
58
12
|
}
|
|
59
13
|
// EqualityFunction can be either strictEquality or subsetEquality
|
|
60
14
|
export default ({tagPattern, equalityFunction = strictEquality}) => (base, source) => {
|
|
61
|
-
const debug = createDebugLogger('@natlibfi/marc-record-merge');
|
|
15
|
+
const debug = createDebugLogger('@natlibfi/marc-record-merge:select');
|
|
62
16
|
const baseFields = base.get(tagPattern);
|
|
63
17
|
const sourceFields = source.get(tagPattern);
|
|
64
18
|
const fieldTag = sourceFields.map(field => field.tag);
|