@natlibfi/marc-record-validators-melinda 12.0.0-alpha.1 → 12.0.0-alpha.12

package/src/translate-terms.test.js

@@ -1,9 +1,23 @@
 import assert from 'node:assert';
-import {MarcRecord} from '@natlibfi/marc-record';
+import createDebugLogger from 'debug';
+import fetchMock from 'fetch-mock';
+
 import validatorFactory from './translate-terms.js';
+
+import {MarcRecord} from '@natlibfi/marc-record';
 import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen';
-import createDebugLogger from 'debug';
+import {fakeData} from '../test-fixtures/translate-terms-data.js';
+
+const uris = [
+  'http://www.yso.fi/onto/yso/p13299',
+  'http://www.yso.fi/onto/yso/p111739',
+  'http://www.yso.fi/onto/yso/p6197061979',
+  'http://www.yso.fi/onto/yso/p6196061969',
+  'http://urn.fi/URN:NBN:fi:au:slm:s161'
+];
+
+
 
 generateTests({
   callback,
@@ -15,6 +29,15 @@ generateTests({
   },
   hooks: {
     before: async () => {
+
+      fetchMock.mockGlobal()
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[0]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData})
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[1]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData})
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[2]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData})
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[3]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData})
+      .get(`https://api.finto.fi/rest/v1/data?uri=${uris[4]}&format=application%2Fjson`, {status: 200, headers: {}, body: fakeData});
+
+
       testValidatorFactory();
     }
   }

package/src/utils.js

@@ -7,7 +7,7 @@ const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:ut
 //const debugData = debug.extend('data');
 const debugDev = debug.extend('dev');
 
-import {melindaCustomMergeFields as melindaFields} from './melindaCustomMergeFields.js';
+import {melindaFieldSpecs} from './melindaCustomMergeFields.js';
 
 //JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'src', 'melindaCustomMergeFields.json'), 'utf8'));
 
@@ -103,6 +103,12 @@ export function nvdebugFieldArray(fields, prefix = '  ', func = undefined) {
 }
 
 export function isControlSubfieldCode(subfieldCode) {
+  // NB! Only $w, $0, $1, $5, $6 and $8 are really control subfields. In Finland $9 is oft a control subfield
+  // $3 material (part of the whole thing)
+  // $4 means 'relationship' (similar to relator terms at least in X00 and similar)
+  // $7 is usually provinance subfield. However, it can be stored in other subfields as well. See merge-fields/dataProvenance.js for details
+  // However, change this only if needed. Maybe all provinance subfields should return true?
+  // This may become relevant when AI starts to create stuff...
   if (['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'w'].includes(subfieldCode)) {
     return true;
   }
@@ -176,7 +182,7 @@ export function subfieldIsRepeatable(tag, subfieldCode) {
     return true;
   }
 
-  const fieldSpecs = melindaFields.fields.filter(field => field.tag === tag);
+  const fieldSpecs = melindaFieldSpecs.fields.filter(field => field.tag === tag);
   if (fieldSpecs.length !== 1) {
     nvdebug(` WARNING! Getting field ${tag} data failed! ${fieldSpecs.length} hits. Default value true is used for'${subfieldCode}' .`, debugDev);
     return true;
@@ -193,7 +199,7 @@ export function subfieldIsRepeatable(tag, subfieldCode) {
 }
 
 function marc21GetTagsLegalIndicators(tag) {
-  const fieldSpecs = melindaFields.fields.filter(field => field.tag === tag);
+  const fieldSpecs = melindaFieldSpecs.fields.filter(field => field.tag === tag);
   if (fieldSpecs.length === 0) {
     return undefined;
   }
@@ -232,3 +238,13 @@ export function hasCopyright(value) {
   const modValue = removeCopyright(value);
   return value !== modValue;
 }
+
+
+
+export function subfieldArraysContainSameData(arr1, arr2) {
+  if ( !arr1.every(sf => arr2.some(sf2 => subfieldsAreIdentical(sf, sf2))) ) {
+    return false;
+  }
+
+  return arr2.every(sf2 => arr1.some(sf => subfieldsAreIdentical(sf, sf2)));
+}

package/test-fixtures/indicator-fixes/10/expectedResult.json

@@ -0,0 +1,11 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "005", "value": "20220202020202.0" },
+    { "tag": "520", "ind1": "8", "ind2": " ", "subfields": [ {"code": "a", "value": "Abstrakti."}, {"code": "9", "value": "TESTI<KEEP>"} ]},
+    { "tag": "520", "ind1": " ", "ind2": " ", "subfields": [ {"code": "a", "value": "Whatever."}]},
+    { "tag": "520", "ind1": "8", "ind2": " ", "subfields": [ {"code": "a", "value": "Tiivistelmä."}]}
+
+  ],
+  "leader": ""
+}

package/test-fixtures/indicator-fixes/10/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "10: certain value have ind1=8 as per MELKEHITYS-2579",
+  "fix": true
+}

package/test-fixtures/indicator-fixes/10/record.json

@@ -0,0 +1,11 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "005", "value": "20220202020202.0" },
+    { "tag": "520", "ind1": " ", "ind2": " ", "subfields": [ {"code": "a", "value": "Abstrakti."}, {"code": "9", "value": "TESTI<KEEP>"} ]},
+    { "tag": "520", "ind1": " ", "ind2": " ", "subfields": [ {"code": "a", "value": "Whatever."}]},
+    { "tag": "520", "ind1": "3", "ind2": " ", "subfields": [ {"code": "a", "value": "Tiivistelmä."}]}
+
+  ],
+  "leader": ""
+}

package/test-fixtures/merge-fields/f05/expectedResult.json

@@ -0,0 +1,24 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "001", "value": "f05" },
+
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "$7 does not merge"}, {"code": "2", "value": "yso/fin"}]},
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "$7 does not merge"}, {"code": "2", "value": "yso/fin"}, {"code": "7", "value": "(dpenmw)AI"}]},
+
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "non-$7 merges"}, {"code": "2", "value": "yso/fin"}, {"code": "0", "value": "something else"}]},
+
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "Identical $7s can merge"}, {"code": "2", "value": "yso/fin"}, {"code": "7", "value": "(dpenmw)AI"}]},
+
+
+    { "tag": "773", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "7", "value": "abcd"},
+      { "code": "t", "value": "Merges since $7 is not a data provenance subfield."},
+      { "code": "o", "value": "ABC-123"},
+      { "code": "x", "value": "1234567890"},
+      { "code": "w", "value": "(FI-MELINDA)020000000"}
+    ]}
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}

package/test-fixtures/merge-fields/f05/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description": "MELKEHITYS-3277: don't merge AI field",
+  "fix": true,
+  "tagPattern": "^(650|773)$",
+  "only": true
+}

package/test-fixtures/merge-fields/f05/record.json

@@ -0,0 +1,30 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "001", "value": "f05" },
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "$7 does not merge"}, {"code": "2", "value": "yso/fin"}]},
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "$7 does not merge"}, {"code": "2", "value": "yso/fin"}, {"code": "7", "value": "(dpenmw)AI"}]},
+
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "non-$7 merges"}, {"code": "2", "value": "yso/fin"}]},
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "non-$7 merges"}, {"code": "2", "value": "yso/fin"}, {"code": "0", "value": "something else"}]},
+
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "Identical $7s can merge"}, {"code": "2", "value": "yso/fin"}, {"code": "7", "value": "(dpenmw)AI"}]},
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "Identical $7s can merge"}, {"code": "2", "value": "yso/fin"}, {"code": "7", "value": "(dpenmw)AI"}]},
+
+
+    { "tag": "773", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "7", "value": "abcd"},
+      { "code": "t", "value": "Merges since $7 is not a data provenance subfield."},
+      { "code": "x", "value": "1234567890"},
+      { "code": "w", "value": "(FI-MELINDA)020000000"}
+    ]},
+    { "tag": "773", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "7", "value": "abcd"},
+      { "code": "t", "value": "Merges since $7 is not a data provenance subfield."},
+      { "code": "o", "value": "ABC-123"},
+      { "code": "w", "value": "(FI-MELINDA)020000000"}
+    ]}
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}

package/test-fixtures/remove-inferior-datafields/f16/expectedResult.json

@@ -0,0 +1,12 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "001", "value": "f16" },
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "termi1"}, {"code": "2", "value": "yso/fin"}]},
+
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "termi2"}, {"code": "2", "value": "yso/fin"}]},
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "termi2"}, {"code": "2", "value": "yso/fin"}, {"code": "7", "value": "something else"}]}
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}

package/test-fixtures/remove-inferior-datafields/f16/metadata.json

@@ -0,0 +1,5 @@
+{
+  "description": "MELKEHITYS-3277-ish: delete AI field, if there's a corresponding non-AI field",
+  "fix": true,
+  "only": false
+}

package/test-fixtures/remove-inferior-datafields/f16/record.json

@@ -0,0 +1,14 @@
+{
+  "_validationOptions": {},
+  "fields": [
+    { "tag": "001", "value": "f16" },
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "termi1"}, {"code": "2", "value": "yso/fin"}]},
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "termi1"}, {"code": "2", "value": "yso/fin"}, {"code": "7", "value": "(dpenmw)AI"}]},
+
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "termi2"}, {"code": "2", "value": "yso/fin"}]},
+    { "tag": "650", "ind1": " ", "ind2": "7", "subfields": [ { "code": "a", "value": "termi2"}, {"code": "2", "value": "yso/fin"}, {"code": "7", "value": "something else"}]}
+
+  ],
+  "leader": "01331cam a22003498i 4500"
+
+}

package/test-fixtures/translate-terms-data.js

@@ -0,0 +1,42 @@
+const fakeTerms = {
+    graph: [
+        {
+            uri: 'http://www.yso.fi/onto/yso/p13299',
+            prefLabel: [
+                { lang: 'fi', value: 'laiturit' },
+                { lang: 'sv', value: 'bryggor'}
+            ]
+        },
+        {
+            uri: 'http://www.yso.fi/onto/yso/p111739',
+            prefLabel: [
+                { lang: 'fi', value: 'Ivalo (Inari)' },
+                { lang: 'sv', value: 'Ivalo (Enare)'}
+            ]
+        },
+        {
+            uri: 'http://www.yso.fi/onto/yso/p6197061979',
+            prefLabel: [
+                { lang: 'fi', value: '1970-luku' },
+                { lang: 'sv', value: '1970-talet' }
+            ]
+        },
+        {
+            uri: 'http://www.yso.fi/onto/yso/p6196061969',
+            prefLabel: [
+                { lang: 'fi', value: '1960-luku' },
+                { lang: 'sv', value: '1960-talet' }
+            ]
+        },
+        {
+            uri: 'http://urn.fi/URN:NBN:fi:au:slm:s161',
+            prefLabel: [
+                { lang: 'fi', value: 'naistenlehdet' },
+                { lang: 'sv', value: 'damtidningar' }
+            ]
+        }
+    ]
+
+};
+
+export const fakeData = JSON.stringify(fakeTerms);