@natlibfi/marc-record-validators-melinda 12.1.0-alpha.1 → 13.0.0

package/src/item-language.js

@@ -1,10 +1,12 @@
-import {loadModule as loadCLD} from 'cld3-asm';
-import LanguageCodes from 'langs';
+import {francAll} from 'franc';
+
 import createDebugLogger from 'debug';
+import {nvdebug} from './utils.js';
+
+// NB! Should we support 041$d? (Audio books use it, don't they?)
 
-export default async function (tagPattern, treshold = 0.9) {
+export default async function (tagPattern, threshold = 0.9) {
   const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda/item-language');
-  const cldFactory = await loadCLD();
 
   if (tagPattern instanceof RegExp) {
     return {
@@ -17,8 +19,8 @@ export default async function (tagPattern, treshold = 0.9) {
 
   throw new Error('No tagPattern provided');
 
-  async function validate(record) {
-    const results = await checkLanguage(record);
+  function validate(record) {
+    const results = checkLanguage(record);
 
     if (results.failed) {
       return {valid: Boolean(results.currentCode), messages: ['Language detection failed']};
@@ -37,8 +39,8 @@ export default async function (tagPattern, treshold = 0.9) {
     }
   }
 
-  async function fix(record) {
-    const results = await checkLanguage(record);
+  function fix(record) {
+    const results = checkLanguage(record);
 
     if (results.suggested && results.currentCode) {
       return;
@@ -89,48 +91,45 @@ export default async function (tagPattern, treshold = 0.9) {
     }
   }
 
-  async function checkLanguage(record) {
+  function checkLanguage(record) {
     const text = getText(record);
     const langCode = getLanguageCode(record);
 
-    const Identifier = cldFactory.create();
+    // We don't want to change these no matter what the abstract says.
+    if (['mul', 'sgn', 'und', 'zxx'].includes(langCode)) {
+      return {failed: true, currentCode: langCode};
+    }
 
     if (text.length === 0) {
-      Identifier.dispose();
       return {failed: true, currentCode: langCode};
     }
 
-    try {
-      const results = await Identifier.findLanguage(text);
-      Identifier.dispose();
+    const results = francAll(text);
 
-      if (results.is_reliable) {
-        if (results.probability >= treshold) {
-          return {
-            detected: get2TLangCode(results.language),
-            currentCode: langCode
-          };
-        }
+    const [francLang, probability] = results[0];
 
-        return {
-          currentCode: langCode,
-          suggested: [get2TLangCode(results.language)]
-        };
-      }
+    nvdebug(`FRANC ${langCode}/${threshold} vs ${francLang}/${probability}`, debug);
 
+    if (francLang === 'und') { // franc returns ['und', 1.0] for failure!
       return {failed: true, currentCode: langCode};
-    } catch (err) {
-      /* istanbul ignore next: How to cause errors? */
-      try {
-        Identifier.dispose();
-      } catch (err2) {
-        debug(`Got error disposing identifier: ${err2 instanceof Error ? err2.stack : err2}`);
-      }
+    }
+
+    if (probability >= threshold) {
+      return {
+        detected: francLang,
+        currentCode: langCode
+      };
+    }
 
-      /* istanbul ignore next: How to cause errors? */
-      throw err instanceof Error ? err : new Error(err.message);
+    if (probability > 0.0) {
+      return {
+        currentCode: langCode,
+        suggested: [francLang]
+      };
     }
 
+    return {failed: true, currentCode: langCode};
+
     function getText(record) {
       return record.get(tagPattern).reduce((acc, field) => {
         const fieldText = field.subfields.find(sf => sf.code === 'a').value;
@@ -138,12 +137,20 @@ export default async function (tagPattern, treshold = 0.9) {
       }, '');
     }
 
+
+    function isValidLanguageCode(code) {
+      if (!(/^[a-z][a-z][a-z]$/u).test(code)) {
+        return false;
+      }
+      return true;
+    }
+
     function getLanguageCode(record) {
       const [f008] = record.get(/^008$/u);
 
       if (f008) {
         const code = f008.value.slice(35, 38);
-        if ((/^[a-z][a-z][a-z]$/u).test(code) && code !== 'zxx') {
+        if(isValidLanguageCode(code)) {
           return code;
         }
       }
@@ -151,13 +158,13 @@ export default async function (tagPattern, treshold = 0.9) {
       const [f041] = record.get(/^041$/u);
 
       if (f041) {
-        const code = f041.subfields.find(sf => sf.code === 'a').value;
-        return code;
+        const subfield = f041.subfields.find(sf => ['a'].includes(sf.code) && isValidLanguageCode(sf.value)); // sami languages are not supported by franc, so don't worry about smi/sme logic.
+        if (!subfield) {
+          return undefined;
+        }
+        return subfield.value;
       }
-    }
-
-    function get2TLangCode(code) {
-      return LanguageCodes.where('1', code)['2T'];
+      return undefined;
     }
   }
 }

package/src/item-language.test.js

@@ -104,8 +104,8 @@ describe('item-language', () => {
       assert.deepEqual(result, {valid: false, messages: ['Item language code is invalid. Correct language code: eng']});
     });
 
-    it('Finds the record invalid (Probability doesn\'t meet treshold)', async () => {
-      const validator = await validatorFactory(/^520$/u, 1);
+    it('Finds the record invalid (Probability doesn\'t meet threshold)', async () => {
+      const validator = await validatorFactory(/^520$/u, 1.1); // franc (unlike cld3) returns 1.0 for this, so I use threshold 1.1 to make test fail...
       const record = new MarcRecord({
         fields: [
           {
@@ -152,7 +152,7 @@ describe('item-language', () => {
           }
         ]
       });
-      const result = await validator.validate(record);
+      const result = validator.validate(record);
 
       assert.deepEqual(result, {valid: true, messages: ['Language detection failed']});
     });

package/src/melindaCustomMergeFields.js

@@ -4336,6 +4336,37 @@ export const melindaFieldSpecs = {'fields':
         {'code': '7', 'repeatable': false},
         {'code': '8', 'repeatable': true}
       ]
+    },
+        {
+      'tag': '857',
+      'repeatable': true,
+      'fieldType': 'dataField',
+      'indicators': {'ind1': [' ', '1', '4', '7'], 'ind2': [' ', '0', '1', '2', '3', '4', '8']},
+      'subfields': [
+        {'code': 'b', 'repeatable': false},
+        {'code': 'c', 'repeatable': false},
+        {'code': 'd', 'repeatable': false},
+        {'code': 'e', 'repeatable': true},
+        {'code': 'f', 'repeatable': false},
+        {'code': 'g', 'repeatable': true},
+        {'code': 'h', 'repeatable': true},
+        {'code': 'l', 'repeatable': true},
+        {'code': 'm', 'repeatable': true},
+        {'code': 'n', 'repeatable': true},
+        {'code': 'q', 'repeatable': true},
+        {'code': 'r', 'repeatable': true},
+        {'code': 's', 'repeatable': true},
+        {'code': 't', 'repeatable': true},
+        {'code': 'u', 'repeatable': true},
+        {'code': 'x', 'repeatable': true},
+        {'code': 'y', 'repeatable': true},
+        {'code': 'z', 'repeatable': true},
+        {'code': '2', 'repeatable': false},
+        {'code': '3', 'repeatable': false},
+        {'code': '6', 'repeatable': false},
+        {'code': '7', 'repeatable': false},
+        {'code': '8', 'repeatable': true}
+      ]
     },
     {
       'tag': '863',

package/src/merge-fields/controlSubfields.js

@@ -2,7 +2,6 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import createDebugLogger from 'debug';
 import {fieldHasSubfield, fieldToString, nvdebug, nvdebugSubfieldArray, subfieldIsRepeatable, subfieldsAreIdentical} from '../utils.js';
 
-//import {normalizeControlSubfieldValue} from './normalizeIdentifier';
 import {normalizeControlSubfieldValue} from '../normalize-identifiers.js';
 
 const debug = createDebugLogger('@natlibfi/marc-record-validators-melinda:merge-fields:controlSubfields');