@natlibfi/marc-record-validators-melinda 10.2.3 → 10.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/melinda-node-tests.yml +2 -2
- package/dist/access-rights.js.map +1 -1
- package/dist/access-rights.spec.js.map +1 -1
- package/dist/double-commas.js.map +1 -1
- package/dist/double-commas.spec.js.map +1 -1
- package/dist/duplicates-ind1.js.map +1 -1
- package/dist/duplicates-ind1.spec.js.map +1 -1
- package/dist/empty-fields.js.map +1 -1
- package/dist/empty-fields.spec.js.map +1 -1
- package/dist/ending-punctuation-conf.js.map +1 -1
- package/dist/ending-punctuation.js.map +1 -1
- package/dist/ending-punctuation.spec.js.map +1 -1
- package/dist/ending-whitespace.js.map +1 -1
- package/dist/ending-whitespace.spec.js.map +1 -1
- package/dist/field-exclusion.js.map +1 -1
- package/dist/field-exclusion.spec.js.map +1 -1
- package/dist/field-structure.js.map +1 -1
- package/dist/field-structure.spec.js.map +1 -1
- package/dist/fields-present.js.map +1 -1
- package/dist/fields-present.spec.js.map +1 -1
- package/dist/fixed-fields.js.map +1 -1
- package/dist/fixed-fields.spec.js.map +1 -1
- package/dist/identical-fields.js.map +1 -1
- package/dist/identical-fields.spec.js.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/indicator-fixes.js.map +1 -1
- package/dist/indicator-fixes.spec.js.map +1 -1
- package/dist/isbn-issn.js.map +1 -1
- package/dist/isbn-issn.spec.js.map +1 -1
- package/dist/item-language.js.map +1 -1
- package/dist/item-language.spec.js.map +1 -1
- package/dist/mergeField500Lisapainokset.js.map +1 -1
- package/dist/mergeField500Lisapainokset.spec.js.map +1 -1
- package/dist/multiple-subfield-0.js.map +1 -1
- package/dist/multiple-subfield-0.spec.js.map +1 -1
- package/dist/non-breaking-space.js.map +1 -1
- package/dist/non-breaking-space.spec.js.map +1 -1
- package/dist/normalize-identifiers.js.map +1 -1
- package/dist/normalize-identifiers.spec.js.map +1 -1
- package/dist/normalize-utf8-diacritics.js.map +1 -1
- package/dist/normalize-utf8-diacritics.spec.js.map +1 -1
- package/dist/punctuation/index.js.map +1 -1
- package/dist/punctuation/rules/aut.js.map +1 -1
- package/dist/punctuation/rules/bib.js.map +1 -1
- package/dist/punctuation/rules/index.js.map +1 -1
- package/dist/punctuation.spec.js.map +1 -1
- package/dist/punctuation2.js +677 -0
- package/dist/punctuation2.js.map +1 -0
- package/dist/punctuation2.spec.js +51 -0
- package/dist/punctuation2.spec.js.map +1 -0
- package/dist/reindexSubfield6OccurenceNumbers.js.map +1 -1
- package/dist/reindexSubfield6OccurenceNumbers.spec.js.map +1 -1
- package/dist/removeDuplicateDataFields.js +117 -30
- package/dist/removeDuplicateDataFields.js.map +1 -1
- package/dist/removeDuplicateDataFields.spec.js.map +1 -1
- package/dist/removeInferiorDataFields.js +98 -0
- package/dist/removeInferiorDataFields.js.map +1 -0
- package/dist/removeInferiorDataFields.spec.js +51 -0
- package/dist/removeInferiorDataFields.spec.js.map +1 -0
- package/dist/resolvable-ext-references-melinda.js.map +1 -1
- package/dist/resolvable-ext-references-melinda.spec.js.map +1 -1
- package/dist/resolveOrphanedSubfield6s.js.map +1 -1
- package/dist/resolveOrphanedSubfield6s.spec.js.map +1 -1
- package/dist/sort-tags.js.map +1 -1
- package/dist/sort-tags.spec.js.map +1 -1
- package/dist/subfield-exclusion.js.map +1 -1
- package/dist/subfield-exclusion.spec.js.map +1 -1
- package/dist/subfield6Utils.js.map +1 -1
- package/dist/subfield8Utils.js +17 -10
- package/dist/subfield8Utils.js.map +1 -1
- package/dist/unicode-decomposition.js.map +1 -1
- package/dist/unicode-decomposition.spec.js.map +1 -1
- package/dist/urn.js.map +1 -1
- package/dist/urn.spec.js.map +1 -1
- package/dist/utils.js.map +1 -1
- package/package.json +2 -2
- package/src/punctuation2.js +398 -0
- package/src/punctuation2.spec.js +52 -0
- package/src/removeDuplicateDataFields.js +141 -31
- package/src/removeInferiorDataFields.js +97 -0
- package/src/removeInferiorDataFields.spec.js +52 -0
- package/src/subfield8Utils.js +16 -11
- package/test-fixtures/punctuation2/01/expectedResult.json +12 -0
- package/test-fixtures/punctuation2/01/metadata.json +6 -0
- package/test-fixtures/punctuation2/01/record.json +37 -0
- package/test-fixtures/punctuation2/02/expectedResult.json +4 -0
- package/test-fixtures/punctuation2/02/metadata.json +6 -0
- package/test-fixtures/punctuation2/02/record.json +14 -0
- package/test-fixtures/punctuation2/04/expectedResult.json +7 -0
- package/test-fixtures/punctuation2/04/metadata.json +6 -0
- package/test-fixtures/punctuation2/04/record.json +22 -0
- package/test-fixtures/punctuation2/05/expectedResult.json +6 -0
- package/test-fixtures/punctuation2/05/metadata.json +6 -0
- package/test-fixtures/punctuation2/05/record.json +12 -0
- package/test-fixtures/punctuation2/98/expectedResult.json +39 -0
- package/test-fixtures/punctuation2/98/metadata.json +6 -0
- package/test-fixtures/punctuation2/98/record.json +37 -0
- package/test-fixtures/punctuation2/99/expectedResult.json +15 -0
- package/test-fixtures/punctuation2/99/metadata.json +6 -0
- package/test-fixtures/punctuation2/99/record.json +14 -0
- package/test-fixtures/remove-duplicate-datafields/f03/expectedResult.json +20 -0
- package/test-fixtures/remove-duplicate-datafields/f03/metadata.json +6 -0
- package/test-fixtures/remove-duplicate-datafields/f03/record.json +33 -0
- package/test-fixtures/remove-duplicate-datafields/f03b/expectedResult.json +20 -0
- package/test-fixtures/remove-duplicate-datafields/f03b/metadata.json +6 -0
- package/test-fixtures/remove-duplicate-datafields/f03b/record.json +35 -0
- package/test-fixtures/remove-duplicate-datafields/f03c/expectedResult.json +25 -0
- package/test-fixtures/remove-duplicate-datafields/f03c/metadata.json +6 -0
- package/test-fixtures/remove-duplicate-datafields/f03c/record.json +43 -0
- package/test-fixtures/remove-duplicate-datafields/f06/expectedResult.json +0 -18
- package/test-fixtures/remove-duplicate-datafields/v02/expectedResult.json +1 -3
- package/test-fixtures/remove-duplicate-datafields/v03/expectedResult.json +1 -2
- package/test-fixtures/remove-inferior-datafields/f01/expectedResult.json +21 -0
- package/test-fixtures/remove-inferior-datafields/f01/metadata.json +6 -0
- package/test-fixtures/remove-inferior-datafields/f01/record.json +31 -0
- package/test-fixtures/remove-inferior-datafields/v01/expectedResult.json +6 -0
- package/test-fixtures/remove-inferior-datafields/v01/metadata.json +6 -0
- package/test-fixtures/remove-inferior-datafields/v01/record.json +31 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import createDebugLogger from 'debug';
|
|
2
2
|
import {fieldHasSubfield, fieldsToString, fieldToString, nvdebug} from './utils';
|
|
3
3
|
import {fieldHasOccurrenceNumber, fieldsToNormalizedString, isValidSubfield6, subfield6GetOccurrenceNumber} from './subfield6Utils';
|
|
4
|
-
import {getSubfield8LinkingNumber, recordGetAllSubfield8LinkingNumbers, recordGetFieldsWithSubfield8LinkingNumber} from './subfield8Utils';
|
|
4
|
+
import {fieldHasLinkingNumber, fieldsGetAllSubfield8LinkingNumbers, getSubfield8LinkingNumber, recordGetAllSubfield8LinkingNumbers, recordGetFieldsWithSubfield8LinkingNumber} from './subfield8Utils';
|
|
5
5
|
|
|
6
6
|
// Relocated from melinda-marc-record-merge-reducers (and renamed)
|
|
7
7
|
|
|
@@ -42,19 +42,6 @@ export default function () {
|
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
function add6s(field, record) {
|
|
45
|
-
|
|
46
|
-
/*
|
|
47
|
-
// Can't rely on nice pairs...
|
|
48
|
-
if (fieldHasSubfield(field, '6')) {
|
|
49
|
-
|
|
50
|
-
const pairs = fieldGetOccurrenceNumberPairs(field, record.fields);
|
|
51
|
-
if (pairs) {
|
|
52
|
-
return [field].concat(pairs);
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
}
|
|
56
|
-
*/
|
|
57
|
-
|
|
58
45
|
// Get all fields with given occurence number
|
|
59
46
|
const sixes = field.subfields.filter(sf => isValidSubfield6(sf));
|
|
60
47
|
|
|
@@ -73,16 +60,36 @@ function add6s(field, record) {
|
|
|
73
60
|
|
|
74
61
|
function add8s(fields, record) {
|
|
75
62
|
// Not implemented yet:
|
|
76
|
-
|
|
77
|
-
|
|
63
|
+
const linkingNumbers = fieldsGetAllSubfield8LinkingNumbers(fields);
|
|
64
|
+
if (linkingNumbers.length === 0) {
|
|
65
|
+
return fields;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
nvdebug(`Linking number(s): ${linkingNumbers.join(', ')}`);
|
|
69
|
+
linkingNumbers.forEach(number => collectLinkingNumberFields(number));
|
|
70
|
+
|
|
71
|
+
fields.forEach(f => nvdebug(`AFTER ADDING 8s: '${fieldToString(f)}'`));
|
|
72
|
+
|
|
73
|
+
return fields;
|
|
74
|
+
|
|
75
|
+
function collectLinkingNumberFields(linkingNumber) {
|
|
76
|
+
// Remove existing hits (to avoid field repetition):
|
|
77
|
+
fields = fields.filter(f => !fieldHasLinkingNumber(f, linkingNumber)); // eslint-disable-line functional/immutable-data, no-param-reassign
|
|
78
|
+
// Add them and their "sisters" back:
|
|
79
|
+
const addableFields = record.fields.filter(f => fieldHasLinkingNumber(f, linkingNumber));
|
|
80
|
+
addableFields.forEach(f => nvdebug(`(RE-?)ADD ${fieldToString(f)}`));
|
|
81
|
+
fields = fields.concat(addableFields); // eslint-disable-line functional/immutable-data, no-param-reassign
|
|
82
|
+
|
|
78
83
|
}
|
|
79
|
-
return record ? fields : fields;
|
|
80
84
|
}
|
|
81
85
|
|
|
86
|
+
/*
|
|
82
87
|
function numberOfLinkageSubfields(field) {
|
|
88
|
+
nvdebug(`N of Linkage Subs(${fieldToString(field)})`);
|
|
83
89
|
const subfields = field.subfields.filter(sf => sf.code === '6' || sf.code === '8');
|
|
84
90
|
return subfields.length;
|
|
85
91
|
}
|
|
92
|
+
*/
|
|
86
93
|
|
|
87
94
|
|
|
88
95
|
function getAllLinkedSubfield6Fields(field, record) {
|
|
@@ -90,12 +97,48 @@ function getAllLinkedSubfield6Fields(field, record) {
|
|
|
90
97
|
const moreFields = add8s(fields, record);
|
|
91
98
|
|
|
92
99
|
// Currently we don't handle fields with more than one $6 and/or $8 subfield.
|
|
93
|
-
if (moreFields.length
|
|
100
|
+
if (moreFields.length > fields.length) {
|
|
94
101
|
return []; // Don't fix!
|
|
95
102
|
}
|
|
96
103
|
return moreFields;
|
|
97
104
|
}
|
|
98
105
|
|
|
106
|
+
function getAllLinkedSubfield68Fields(field, record) {
|
|
107
|
+
const fields = add6s(field, record);
|
|
108
|
+
return add8s(fields, record);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function isRelevantSubfield6Chain(fields) {
|
|
112
|
+
if (fields.length < 2) { // 1 non-880-field and 1+ 880 fields
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
const non880 = fields.filter(f => f.tag !== '880');
|
|
116
|
+
if (non880.length !== 1) {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const linkingNumbers = fieldsGetAllSubfield8LinkingNumbers(fields);
|
|
121
|
+
if (linkingNumbers.length !== 0) {
|
|
122
|
+
return false;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return fields.every(f => fieldHasSubfield(f, '6'));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function isRelevantSubfield68Chain(fields) {
|
|
129
|
+
if (fields.length < 3) {
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const linkingNumbers = fieldsGetAllSubfield8LinkingNumbers(fields);
|
|
134
|
+
if (linkingNumbers.length !== 1) {
|
|
135
|
+
nvdebug(`Expected one linking number, got ${linkingNumbers.length}: ${linkingNumbers.join(', ')}`);
|
|
136
|
+
return false;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return fields.some(f => fieldHasSubfield(f, '6'));
|
|
140
|
+
}
|
|
141
|
+
|
|
99
142
|
function getFirstField(record, fields) {
|
|
100
143
|
const fieldsAsStrings = fields.map(field => fieldToString(field));
|
|
101
144
|
record.fields.forEach((field, i) => nvdebug(`${i}:\t${fieldToString(field)}`));
|
|
@@ -109,25 +152,40 @@ function getFirstField(record, fields) {
|
|
|
109
152
|
return undefined;
|
|
110
153
|
}
|
|
111
154
|
|
|
155
|
+
function fieldIsFirstFieldInChain(field, chain, record) {
|
|
156
|
+
// Interpretation of first: position of field in record (however, we might have a duplicate field. See tests...)
|
|
157
|
+
const firstField = getFirstField(record, chain);
|
|
158
|
+
if (firstField) {
|
|
159
|
+
return fieldToString(field) === fieldToString(firstField);
|
|
160
|
+
}
|
|
161
|
+
return false;
|
|
162
|
+
|
|
163
|
+
}
|
|
112
164
|
|
|
113
165
|
function isFirstLinkedSubfield6Field(field, record) {
|
|
114
166
|
if (!field.subfields) { // Is not a datafield
|
|
115
167
|
return false;
|
|
116
168
|
}
|
|
117
169
|
const chain = getAllLinkedSubfield6Fields(field, record);
|
|
118
|
-
if (chain
|
|
170
|
+
if (!isRelevantSubfield6Chain(chain)) {
|
|
171
|
+
nvdebug(`Rejected 6: ${fieldsToString(chain)}`);
|
|
119
172
|
return false;
|
|
120
173
|
}
|
|
121
174
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
175
|
+
return fieldIsFirstFieldInChain(field, chain, record);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function isFirstLinkedSubfield68Field(field, record) {
|
|
179
|
+
if (!field.subfields) { // Is not a datafield
|
|
180
|
+
return false;
|
|
181
|
+
}
|
|
182
|
+
const chain = getAllLinkedSubfield68Fields(field, record);
|
|
183
|
+
if (!isRelevantSubfield68Chain(chain)) {
|
|
184
|
+
//nvdebug(`Rejected 68: ${fieldsToString(chain)}`);
|
|
185
|
+
return false;
|
|
126
186
|
}
|
|
127
|
-
return false;
|
|
128
187
|
|
|
129
|
-
|
|
130
|
-
//return fieldToString(field) === fieldToString(chain[0]);
|
|
188
|
+
return fieldIsFirstFieldInChain(field, chain, record);
|
|
131
189
|
}
|
|
132
190
|
|
|
133
191
|
export function removeIndividualDuplicateDatafields(record, fix = true) { // No $6 nor $8 in field
|
|
@@ -253,25 +311,73 @@ export function removeDuplicateSubfield8Chains(record, fix = true) {
|
|
|
253
311
|
return removables;
|
|
254
312
|
}
|
|
255
313
|
|
|
314
|
+
export function removeDuplicateSubfield68Chains(record, fix = true) {
|
|
315
|
+
/* eslint-disable */
|
|
316
|
+
let seen = {};
|
|
317
|
+
|
|
318
|
+
let removables = []; // for validation
|
|
319
|
+
|
|
320
|
+
const fields = record.fields.filter(field => isFirstLinkedSubfield68Field(field, record));
|
|
321
|
+
|
|
322
|
+
fields.forEach(field => removeDuplicateDatafield68(field));
|
|
323
|
+
|
|
324
|
+
function removeDuplicateDatafield68(field) {
|
|
325
|
+
nvdebug(`removeDuplicateDatafield? $6-$8 ${fieldToString(field)} (and friends)`);
|
|
326
|
+
const fields = getAllLinkedSubfield68Fields(field, record);
|
|
327
|
+
if (!isRelevantSubfield68Chain(fields)) {
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
const fieldsAsString = fieldsToNormalizedString(fields);
|
|
332
|
+
|
|
333
|
+
const altFieldsAsString = fieldsAsString.substring(0, 1) === '7' ? `1${fieldsAsString.substring(1)}` : fieldsAsString;
|
|
334
|
+
nvdebug(` step 2 ${fieldsAsString}`);
|
|
335
|
+
if (fieldsAsString in seen || altFieldsAsString in seen) {
|
|
336
|
+
nvdebug(` step 3 ${fieldsAsString}`);
|
|
337
|
+
|
|
338
|
+
removables.push(fieldsAsString);
|
|
339
|
+
|
|
340
|
+
if (fix) {
|
|
341
|
+
nvdebug(`$68 DOUBLE REMOVAL: REMOVE ${fieldsAsString}`, debug);
|
|
342
|
+
fields.forEach(currField => record.removeField(currField));
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
nvdebug(`$68 VALIDATION: DUPLICATE DETECTED ${fieldsAsString}`, debug);
|
|
347
|
+
|
|
348
|
+
}
|
|
349
|
+
nvdebug(`$68 DOUBLE REMOVAL OR VALIDATION: ADD2SEEN ${fieldsAsString}`, debug);
|
|
350
|
+
seen[fieldsAsString] = 1;
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
/* eslint-enable */
|
|
354
|
+
return removables;
|
|
355
|
+
}
|
|
356
|
+
|
|
256
357
|
export function removeDuplicateSubfield6Chains(record, fix = true) {
|
|
257
358
|
/* eslint-disable */
|
|
258
359
|
let seen = {};
|
|
259
360
|
|
|
260
361
|
let removables = []; // for validation
|
|
261
362
|
|
|
262
|
-
record.fields.forEach(field => nvdebug(
|
|
363
|
+
record.fields.forEach(field => nvdebug(`$6-DUPL-CHECK CHAIN ${fieldToString(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));
|
|
263
364
|
|
|
264
365
|
const fields = record.fields.filter(field => isFirstLinkedSubfield6Field(field, record)); // Well a
|
|
366
|
+
//fields.forEach(field => nvdebug(`$6-DUPL-CHECK CHAIN HEAD ${fieldToString(field)}, mode=${fix ? 'FIX' : 'VALIDATE'}`));
|
|
265
367
|
|
|
266
|
-
fields.forEach(field => removeDuplicateDatafield(field));
|
|
267
368
|
|
|
268
|
-
|
|
369
|
+
fields.forEach(field => removeDuplicateDatafield6(field));
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
function removeDuplicateDatafield6(field) {
|
|
269
374
|
nvdebug(`removeDuplicateDatafield? $6 ${fieldToString(field)} (and friends)`);
|
|
270
375
|
const fields = getAllLinkedSubfield6Fields(field, record);
|
|
271
|
-
if(fields
|
|
376
|
+
if (!isRelevantSubfield6Chain(fields)) {
|
|
272
377
|
return;
|
|
273
378
|
}
|
|
274
379
|
|
|
380
|
+
|
|
275
381
|
const fieldsAsString = fieldsToNormalizedString(fields);
|
|
276
382
|
// Frequencly list for $6 subfields in 1XX/7XX fields:
|
|
277
383
|
// 231115 100
|
|
@@ -310,11 +416,15 @@ export function removeDuplicateSubfield6Chains(record, fix = true) {
|
|
|
310
416
|
|
|
311
417
|
export function removeDuplicateDatafields(record, fix = true) {
|
|
312
418
|
const removables = removeIndividualDuplicateDatafields(record, fix); // Lone fields
|
|
419
|
+
// NB! Do $6+$8 before mere $8!
|
|
420
|
+
const removables68 = removeDuplicateSubfield68Chains(record, fix); // Single $6 + $8
|
|
421
|
+
|
|
313
422
|
const removables8 = removeDuplicateSubfield8Chains(record, fix); // Lone subfield $8 chains
|
|
314
423
|
const removables6 = removeDuplicateSubfield6Chains(record, fix); // Lone subfield $6 chains
|
|
424
|
+
|
|
315
425
|
// HOW TO HANDLE $6+$8 combos?
|
|
316
426
|
|
|
317
|
-
const removablesAll = removables.concat(
|
|
427
|
+
const removablesAll = removables.concat(removables68).concat(removables6).concat(removables8);
|
|
318
428
|
|
|
319
429
|
return removablesAll;
|
|
320
430
|
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import createDebugLogger from 'debug';
|
|
2
|
+
import {fieldToString, nvdebug} from './utils';
|
|
3
|
+
|
|
4
|
+
// Relocated from melinda-marc-record-merge-reducers (and renamed)
|
|
5
|
+
|
|
6
|
+
const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:removeSubsetDataFields');
|
|
7
|
+
|
|
8
|
+
export default function () {
|
|
9
|
+
return {
|
|
10
|
+
description: 'Remove subset data fields. Certain exceptions apply, mainly too complited chained fields',
|
|
11
|
+
validate, fix
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
function fix(record) {
|
|
15
|
+
nvdebug('Fix record: remove subset data fields', debug);
|
|
16
|
+
const res = {message: [], fix: [], valid: true};
|
|
17
|
+
removeInferiorDatafields(record, true);
|
|
18
|
+
// This can not really fail...
|
|
19
|
+
return res;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function validate(record) {
|
|
23
|
+
// Check max, and check number of different indexes
|
|
24
|
+
nvdebug('Validate record: remove subset data fields', debug);
|
|
25
|
+
|
|
26
|
+
const duplicates = removeInferiorDatafields(record, false);
|
|
27
|
+
|
|
28
|
+
const res = {message: duplicates};
|
|
29
|
+
|
|
30
|
+
res.valid = res.message.length < 1; // eslint-disable-line functional/immutable-data
|
|
31
|
+
return res;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function deriveIndividualDeletables(record) {
|
|
36
|
+
/* eslint-disable */
|
|
37
|
+
let deletableStringsArray = [];
|
|
38
|
+
|
|
39
|
+
record.fields.forEach(field => fieldDeriveIndividualDeletables(field));
|
|
40
|
+
|
|
41
|
+
function fieldDeriveIndividualDeletables(field) {
|
|
42
|
+
const fieldAsString = fieldToString(field);
|
|
43
|
+
|
|
44
|
+
// Proof-of-concept rule:
|
|
45
|
+
let tmp = fieldAsString;
|
|
46
|
+
if (field.tag.match(/^[1678]00$/u)) {
|
|
47
|
+
while (tmp.match(/, ‡e [^‡]+\.$/)) {
|
|
48
|
+
tmp = tmp.replace(/, ‡e [^‡]+\.$/, '.');
|
|
49
|
+
deletableStringsArray.push(tmp);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
// Remove keepless versions:
|
|
55
|
+
tmp = fieldAsString;
|
|
56
|
+
while (tmp.match(/ ‡9 [A-Z]+<KEEP>/)) {
|
|
57
|
+
tmp = tmp.replace(/ ‡9 [A-Z]+<KEEP>/, '');
|
|
58
|
+
deletableStringsArray.push(tmp);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/* eslint-enable */
|
|
62
|
+
return deletableStringsArray; // we should do uniq!
|
|
63
|
+
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function removeIndividualInferiorDatafields(record, fix = true) { // No $6 nor $8 in field
|
|
67
|
+
const deletableFieldsAsStrings = deriveIndividualDeletables(record);
|
|
68
|
+
const hits = record.fields.filter(field => isDeletableField(field));
|
|
69
|
+
|
|
70
|
+
const deletedFieldsAsStrings = hits.map(f => fieldToString(f));
|
|
71
|
+
|
|
72
|
+
if (fix) { // eslint-disable-line functional/no-conditional-statement
|
|
73
|
+
hits.forEach(field => {
|
|
74
|
+
nvdebug(`Remove inferior field: ${fieldToString(field)}`);
|
|
75
|
+
record.removeField(field);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return deletedFieldsAsStrings;
|
|
80
|
+
|
|
81
|
+
function isDeletableField(field) {
|
|
82
|
+
const fieldAsString = fieldToString(field);
|
|
83
|
+
return deletableFieldsAsStrings.includes(fieldAsString);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
export function removeInferiorDatafields(record, fix = true) {
|
|
89
|
+
const removables = removeIndividualInferiorDatafields(record, fix); // Lone fields
|
|
90
|
+
//const removables8 = removeDuplicateSubfield8Chains(record, fix); // Lone subfield $8 chains
|
|
91
|
+
//const removables6 = removeDuplicateSubfield6Chains(record, fix); // Lone subfield $6 chains
|
|
92
|
+
// HOW TO HANDLE $6+$8 combos?
|
|
93
|
+
|
|
94
|
+
const removablesAll = removables; //.concat(removables8).concat(removables6);
|
|
95
|
+
|
|
96
|
+
return removablesAll;
|
|
97
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import {expect} from 'chai';
|
|
2
|
+
import {MarcRecord} from '@natlibfi/marc-record';
|
|
3
|
+
import validatorFactory from './removeInferiorDataFields';
|
|
4
|
+
import {READERS} from '@natlibfi/fixura';
|
|
5
|
+
import generateTests from '@natlibfi/fixugen';
|
|
6
|
+
import createDebugLogger from 'debug';
|
|
7
|
+
|
|
8
|
+
generateTests({
|
|
9
|
+
callback,
|
|
10
|
+
path: [__dirname, '..', 'test-fixtures', 'remove-inferior-datafields'],
|
|
11
|
+
useMetadataFile: true,
|
|
12
|
+
recurse: false,
|
|
13
|
+
fixura: {
|
|
14
|
+
reader: READERS.JSON
|
|
15
|
+
},
|
|
16
|
+
mocha: {
|
|
17
|
+
before: () => testValidatorFactory()
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/removeInferiorDataFields:test');
|
|
21
|
+
|
|
22
|
+
async function testValidatorFactory() {
|
|
23
|
+
const validator = await validatorFactory();
|
|
24
|
+
|
|
25
|
+
expect(validator)
|
|
26
|
+
.to.be.an('object')
|
|
27
|
+
.that.has.any.keys('description', 'validate');
|
|
28
|
+
|
|
29
|
+
expect(validator.description).to.be.a('string');
|
|
30
|
+
expect(validator.validate).to.be.a('function');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function callback({getFixture, enabled = true, fix = false}) {
|
|
34
|
+
if (enabled === false) {
|
|
35
|
+
debug('TEST SKIPPED!');
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const validator = await validatorFactory();
|
|
40
|
+
const record = new MarcRecord(getFixture('record.json'));
|
|
41
|
+
const expectedResult = getFixture('expectedResult.json');
|
|
42
|
+
// console.log(expectedResult); // eslint-disable-line
|
|
43
|
+
|
|
44
|
+
if (!fix) {
|
|
45
|
+
const result = await validator.validate(record);
|
|
46
|
+
expect(result).to.eql(expectedResult);
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
await validator.fix(record);
|
|
51
|
+
expect(record).to.eql(expectedResult);
|
|
52
|
+
}
|
package/src/subfield8Utils.js
CHANGED
|
@@ -10,7 +10,7 @@ export function isValidSubfield8(subfield) {
|
|
|
10
10
|
return false;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
nvdebug(` IS VALID $8? '${subfieldToString(subfield)}'`);
|
|
13
|
+
//nvdebug(` IS VALID $8? '${subfieldToString(subfield)}'`);
|
|
14
14
|
const match = subfield.value.match(sf8Regexp);
|
|
15
15
|
//nvdebug(` IS VALID $8? '${subfieldToString(subfield)}' vs ${match.length}}`);
|
|
16
16
|
return match && match.length > 0;
|
|
@@ -32,25 +32,25 @@ export function getSubfield8LinkingNumber(subfield) {
|
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
|
|
35
|
+
export function fieldHasLinkingNumber(field, linkingNumber) {
|
|
36
|
+
if (!field.subfields) {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
return field.subfields.some(sf => getSubfield8LinkingNumber(sf) === linkingNumber);
|
|
40
|
+
}
|
|
41
|
+
|
|
35
42
|
export function recordGetFieldsWithSubfield8LinkingNumber(record, linkingNumber) {
|
|
36
43
|
if (linkingNumber < 1) {
|
|
37
44
|
return;
|
|
38
45
|
}
|
|
39
|
-
return record.fields.filter(field =>
|
|
40
|
-
|
|
41
|
-
function relevant4GFWS8I(field) {
|
|
42
|
-
if (!field.subfields) {
|
|
43
|
-
return false;
|
|
44
|
-
}
|
|
45
|
-
return field.subfields.some(sf => getSubfield8LinkingNumber(sf) === linkingNumber);
|
|
46
|
-
}
|
|
46
|
+
return record.fields.filter(field => fieldHasLinkingNumber(field, linkingNumber));
|
|
47
47
|
}
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
export function
|
|
50
|
+
export function fieldsGetAllSubfield8LinkingNumbers(fields) {
|
|
51
51
|
/* eslint-disable */
|
|
52
52
|
let subfield8LinkingNumbers = [];
|
|
53
|
-
|
|
53
|
+
fields.forEach(field => {
|
|
54
54
|
if (!field.subfields) {
|
|
55
55
|
return;
|
|
56
56
|
}
|
|
@@ -66,4 +66,9 @@ export function recordGetAllSubfield8LinkingNumbers(record) {
|
|
|
66
66
|
|
|
67
67
|
return subfield8LinkingNumbers;
|
|
68
68
|
/* eslint-enable */
|
|
69
|
+
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function recordGetAllSubfield8LinkingNumbers(record) {
|
|
73
|
+
return fieldsGetAllSubfield8LinkingNumbers(record.fields);
|
|
69
74
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"message": [
|
|
3
|
+
"'100 1 ‡a Tuisku, Sara ‡e turkulainen ‡e testaaja' => '100 1 ‡a Tuisku, Sara, ‡e turkulainen, ‡e testaaja.'",
|
|
4
|
+
"'700 1 ‡a Reipas, R. ‡d 2000- ‡e esittäjä' => '700 1 ‡a Reipas, R., ‡d 2000- ‡e esittäjä.'",
|
|
5
|
+
"'700 1 ‡a Reippaahko, R. ‡d 2000-' => '700 1 ‡a Reippaahko, R., ‡d 2000-'",
|
|
6
|
+
"'700 1 ‡a Reippaampi, R. ‡d 2000-2050 ‡e esittäjä' => '700 1 ‡a Reippaampi, R., ‡d 2000-2050, ‡e esittäjä.'",
|
|
7
|
+
"'700 1 ‡a Reippain, R. ‡d 2000-2050' => '700 1 ‡a Reippain, R., ‡d 2000-2050.'",
|
|
8
|
+
"'700 1 ‡a Nalle, P. ‡d 1926- ‡0 (FIN11)000000000' => '700 1 ‡a Nalle, P., ‡d 1926- ‡0 (FIN11)000000000'"
|
|
9
|
+
],
|
|
10
|
+
"valid": false
|
|
11
|
+
|
|
12
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"leader": "01331cam a22003494i 4500",
|
|
3
|
+
"fields": [
|
|
4
|
+
{ "tag": "001", "value": "000000001" },
|
|
5
|
+
{ "tag": "041", "ind1": " ", "ind2": " ", "subfields": [ { "code": "a", "value": "fin" } ] },
|
|
6
|
+
|
|
7
|
+
{ "tag": "100", "ind1": "1", "ind2": " ", "subfields": [
|
|
8
|
+
{ "code": "a", "value": "Tuisku, Sara" },
|
|
9
|
+
{ "code": "e", "value": "turkulainen" },
|
|
10
|
+
{ "code": "e", "value": "testaaja" }
|
|
11
|
+
]
|
|
12
|
+
},
|
|
13
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
14
|
+
{ "code": "a", "value": "Reipas, R." },
|
|
15
|
+
{ "code": "d", "value": "2000-" },
|
|
16
|
+
{ "code": "e", "value": "esittäjä" }
|
|
17
|
+
]},
|
|
18
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
19
|
+
{ "code": "a", "value": "Reippaahko, R." },
|
|
20
|
+
{ "code": "d", "value": "2000-" }
|
|
21
|
+
]},
|
|
22
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
23
|
+
{ "code": "a", "value": "Reippaampi, R." },
|
|
24
|
+
{ "code": "d", "value": "2000-2050" },
|
|
25
|
+
{ "code": "e", "value": "esittäjä" }
|
|
26
|
+
]},
|
|
27
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
28
|
+
{ "code": "a", "value": "Reippain, R." },
|
|
29
|
+
{ "code": "d", "value": "2000-2050" }
|
|
30
|
+
]},
|
|
31
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
32
|
+
{ "code": "a", "value": "Nalle, P." },
|
|
33
|
+
{ "code": "d", "value": "1926-" },
|
|
34
|
+
{ "code": "0", "value": "(FIN11)000000000"}
|
|
35
|
+
]}
|
|
36
|
+
]
|
|
37
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"leader": "01331cam a22003494i 4500",
|
|
3
|
+
"fields": [
|
|
4
|
+
{ "tag": "100", "ind1": "1", "ind2": "0", "subfields": [
|
|
5
|
+
{ "code": "a", "value": "Sukunimi, Etunimi," },
|
|
6
|
+
{ "code": "e", "value": "säveltäjä." }
|
|
7
|
+
]
|
|
8
|
+
},
|
|
9
|
+
{ "tag": "245", "ind1": "1", "ind2": "0", "subfields": [
|
|
10
|
+
{ "code": "a", "value": "Only valid fields in this test /" },
|
|
11
|
+
{ "code": "c", "value": "Tekijä." }
|
|
12
|
+
]}
|
|
13
|
+
]
|
|
14
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"message": [
|
|
3
|
+
"'245 10 ‡a Manun illallinen ‡c Tellervo Koivisto' => '245 10 ‡a Manun illallinen / ‡c Tellervo Koivisto'",
|
|
4
|
+
"'245 10 ‡a Manun illallinen ‡b lentopallosta hiustöyhtöön ‡c Tellervo Koivisto' => '245 10 ‡a Manun illallinen : ‡b lentopallosta hiustöyhtöön / ‡c Tellervo Koivisto'"
|
|
5
|
+
],
|
|
6
|
+
"valid": false
|
|
7
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"leader": "01331cam a22003494i 4500",
|
|
3
|
+
"fields": [
|
|
4
|
+
{ "tag": "245", "ind1": "1", "ind2": "0", "subfields": [
|
|
5
|
+
{ "code": "a", "value": "Manun illallinen" },
|
|
6
|
+
{ "code": "c", "value": "Tellervo Koivisto" }
|
|
7
|
+
]
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"tag": "245", "ind1": "1", "ind2": "0",
|
|
11
|
+
"subfields": [
|
|
12
|
+
{ "code": "a", "value": "Manun illallinen" },
|
|
13
|
+
{ "code": "b", "value": "lentopallosta hiustöyhtöön" },
|
|
14
|
+
{ "code": "c", "value": "Tellervo Koivisto" }
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
{ "tag": "245", "ind1": "1", "ind2": "0", "subfields": [
|
|
18
|
+
{ "code": "a", "value": "OK-otsikko /" },
|
|
19
|
+
{ "code": "c", "value": "Tekijä." }
|
|
20
|
+
]}
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"leader": "01331cam a22003494i 4500",
|
|
3
|
+
"fields": [
|
|
4
|
+
{ "tag": "300", "ind1": "1", "ind2": " ", "subfields": [
|
|
5
|
+
{ "code": "a", "value": "740 sivua" },
|
|
6
|
+
{ "code": "b", "value": "kuvitettu" },
|
|
7
|
+
{ "code": "c", "value": "25 cm" },
|
|
8
|
+
{ "code": "e", "value": "1 CD-levy" }
|
|
9
|
+
]}
|
|
10
|
+
|
|
11
|
+
]
|
|
12
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_validationOptions": {},
|
|
3
|
+
"leader": "01331cam a22003494i 4500",
|
|
4
|
+
"fields": [
|
|
5
|
+
{ "tag": "001", "value": "000000001" },
|
|
6
|
+
{ "tag": "041", "ind1": " ", "ind2": " ", "subfields": [ { "code": "a", "value": "fin" } ] },
|
|
7
|
+
|
|
8
|
+
{ "tag": "100", "ind1": "1", "ind2": " ", "subfields": [
|
|
9
|
+
{ "code": "a", "value": "Tuisku, Sara," },
|
|
10
|
+
{ "code": "e", "value": "turkulainen," },
|
|
11
|
+
{ "code": "e", "value": "testaaja." }
|
|
12
|
+
]
|
|
13
|
+
},
|
|
14
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
15
|
+
{ "code": "a", "value": "Reipas, R.," },
|
|
16
|
+
{ "code": "d", "value": "2000-" },
|
|
17
|
+
{ "code": "e", "value": "esittäjä." }
|
|
18
|
+
]},
|
|
19
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
20
|
+
{ "code": "a", "value": "Reippaahko, R.," },
|
|
21
|
+
{ "code": "d", "value": "2000-" }
|
|
22
|
+
]},
|
|
23
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
24
|
+
{ "code": "a", "value": "Reippaampi, R.," },
|
|
25
|
+
{ "code": "d", "value": "2000-2050," },
|
|
26
|
+
{ "code": "e", "value": "esittäjä." }
|
|
27
|
+
]},
|
|
28
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
29
|
+
{ "code": "a", "value": "Reippain, R.," },
|
|
30
|
+
{ "code": "d", "value": "2000-2050." }
|
|
31
|
+
]},
|
|
32
|
+
{ "tag": "700", "ind1": "1", "ind2": " ", "subfields": [
|
|
33
|
+
{ "code": "a", "value": "Nalle, P.," },
|
|
34
|
+
{ "code": "d", "value": "1926-" },
|
|
35
|
+
{ "code": "0", "value": "(FIN11)000000000"}
|
|
36
|
+
]}
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
}
|