@natlibfi/melinda-record-matching 2.2.1-alpha.1 → 2.2.1-alpha.3

package/package.json

@@ -14,7 +14,7 @@
 		"url": "git@github.com:natlibfi/melinda-record-matching-js.git"
 	},
 	"license": "LGPL-3.0+",
-	"version": "2.2.1-alpha.1",
+	"version": "2.2.1-alpha.3",
 	"main": "./dist/index.js",
 	"engines": {
 		"node": ">=14"
@@ -23,6 +23,7 @@
 		"access": "public"
 	},
 	"scripts": {
+		"prepare": "npm run build",
 		"start": "node dist/index.js",
 		"cli": "node dist/cli.js",
 		"lint": "eslint src",
@@ -35,35 +36,35 @@
 		"watch:test": "cross-env DEBUG=1 NODE_ENV=test nodemon -w src -w test-fixtures --exec 'npm run test:dev'"
 	},
 	"dependencies": {
-		"@natlibfi/marc-record": "^7.0.0",
-		"@natlibfi/marc-record-serializers": "^8.1.0",
-		"@natlibfi/melinda-commons": "^12.0.3",
-		"@natlibfi/sru-client": "^5.0.1",
+		"@natlibfi/marc-record": "^7.2.1",
+		"@natlibfi/marc-record-serializers": "^9.0.0",
+		"@natlibfi/melinda-commons": "^12.0.4",
+		"@natlibfi/sru-client": "^5.0.3",
 		"debug": "^4.3.4",
 		"moment": "^2.29.4",
 		"natural": "^5.2.3",
-		"uuid": "^8.3.2",
-		"winston": "^3.8.1"
+		"uuid": "^9.0.0",
+		"winston": "^3.8.2"
 	},
 	"devDependencies": {
-		"@babel/cli": "^7.18.10",
-		"@babel/core": "^7.18.13",
-		"@babel/eslint-parser": "^7.18.9",
-		"@babel/node": "^7.18.10",
-		"@babel/preset-env": "^7.18.10",
+		"@babel/cli": "^7.19.3",
+		"@babel/core": "^7.19.3",
+		"@babel/eslint-parser": "^7.19.1",
+		"@babel/node": "^7.19.1",
+		"@babel/preset-env": "^7.19.4",
 		"@babel/register": "^7.18.9",
-		"@natlibfi/eslint-config-melinda-backend": "^2.0.0",
+		"@natlibfi/eslint-config-melinda-backend": "^2.0.1",
 		"@natlibfi/fixugen": "^1.1.0",
 		"@natlibfi/fixugen-http-client": "^2.0.1",
-		"@natlibfi/fixura": "^2.2.1",
+		"@natlibfi/fixura": "^2.2.5",
 		"babel-plugin-istanbul": "^6.1.1",
 		"babel-plugin-rewire": "^1.2.0",
 		"chai": "^4.3.6",
 		"chai-as-promised": "^7.1.1",
 		"cross-env": "^7.0.3",
-		"eslint": "^8.23.0",
-		"mocha": "^9.2.2",
-		"nodemon": "^2.0.19",
+		"eslint": "^8.25.0",
+		"mocha": "^10.0.0",
+		"nodemon": "^2.0.20",
 		"nyc": "^15.1.0"
 	},
 	"eslintConfig": {

package/src/candidate-search/query-list/bib.js

@@ -5,7 +5,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -28,7 +28,7 @@
 */
 import createDebugLogger from 'debug';
 import {toQueries} from '../candidate-search-utils';
-import {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues} from '../../matching-utils';
+import {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues, testStringOrNumber} from '../../matching-utils';
 
 
 export function bibSourceIds(record) {
@@ -108,7 +108,7 @@ export function bibSourceIds(record) {
 
         function removeSourcePrefix(subfieldValue) {
           const sourcePrefixRegex = (/^(?<sourcePrefix>\([A-Za-z0-9-]+\))(?<id>.+)$/u);
-          const normalizedValue = subfieldValue ? subfieldValue.replace(sourcePrefixRegex, '$<id>') : '';
+          const normalizedValue = testStringOrNumber(subfieldValue) ? String(subfieldValue).replace(sourcePrefixRegex, '$<id>') : '';
           debugData(`Normalized ${subfieldValue} to ${normalizedValue}`);
           return normalizedValue;
         }
@@ -116,7 +116,7 @@ export function bibSourceIds(record) {
         function normalizeSidSubfieldValue(subfieldValue) {
           debugData(`Normalizing ${subfieldValue}`);
           const normalizeAwayRegex = (/[- ]/u);
-          return subfieldValue ? subfieldValue.replace(normalizeAwayRegex, '') : '';
+          return testStringOrNumber(subfieldValue) ? String(subfieldValue).replace(normalizeAwayRegex, '') : '';
         }
 
       }
@@ -151,7 +151,7 @@ export function bibMelindaIds(record) {
 // bibHostComponents should search all 773 $ws for possible host id, but what should it do in case of multiple host ids?
 export function bibHostComponents(record) {
   const id = getHostId();
-  return id ? [`melinda.partsofhost=${id}`] : [];
+  return testStringOrNumber(id) ? [`melinda.partsofhost=${id}`] : [];
 
   function getHostId() {
     const [field] = record.get(/^773$/u);
@@ -159,12 +159,12 @@ export function bibHostComponents(record) {
     if (field) {
       const {value} = field.subfields.find(({code}) => code === 'w') || {};
 
-      if (value && (/^\(FI-MELINDA\)/u).test(value)) {
-        return value.replace(/^\(FI-MELINDA\)/u, '');
+      if (testStringOrNumber(value) && (/^\(FI-MELINDA\)/u).test(String(value))) {
+        return String(value).replace(/^\(FI-MELINDA\)/u, '');
       }
 
-      if (value && (/^\(FIN01\)/u).test(value)) {
-        return value.replace(/^\(FIN01\)/u, '');
+      if (testStringOrNumber(value) && (/^\(FIN01\)/u).test(String(value))) {
+        return String(value).replace(/^\(FIN01\)/u, '');
       }
 
       return false;
@@ -185,8 +185,8 @@ export function bibTitle(record) {
   const title = getTitle();
   const booleanStartWords = ['and', 'or', 'nor', 'not'];
 
-  if (title) {
-    const formatted = title
+  if (testStringOrNumber(title)) {
+    const formatted = String(title)
       .replace(/[^\w\s\p{Alphabetic}]/gu, '')
       // Clean up concurrent spaces from fe. subfield changes
       .replace(/ +/gu, ' ')
@@ -208,7 +208,8 @@ export function bibTitle(record) {
     if (field) {
       return field.subfields
         .filter(({code}) => ['a', 'b'].includes(code))
-        .map(({value}) => value)
+        .map(({value}) => testStringOrNumber(value) ? String(value) : '')
+        .filter(value => value)
         // In Melinda's index subfield separators are indexed as ' '
         .join(' ');
     }
@@ -244,18 +245,18 @@ export function bibStandardIdentifiers(record) {
 
     if (tag === '022') {
       return subfields
-        .filter(sub => ['a', 'z', 'y'].includes(sub.code) && issnIsbnReqExp.test(sub.value) && sub.value !== undefined)
-        .map(({value}) => value);
+        .filter(sub => ['a', 'z', 'y'].includes(sub.code) && testStringOrNumber(sub.value) && issnIsbnReqExp.test(String(sub.value)))
+        .map(({value}) => String(value));
     }
 
     if (tag === '020') {
       return subfields
-        .filter(sub => ['a', 'z'].includes(sub.code) && issnIsbnReqExp.test(sub.value) && sub.value !== undefined)
-        .map(({value}) => value);
+        .filter(sub => ['a', 'z'].includes(sub.code) && testStringOrNumber(sub.value) && issnIsbnReqExp.test(String(sub.value)))
+        .map(({value}) => String(value));
     }
 
     return subfields
-      .filter(sub => ['a', 'z'].includes(sub.code) && otherIdReqExp.test(sub.value) && sub.value !== undefined)
-      .map(({value}) => value);
+      .filter(sub => ['a', 'z'].includes(sub.code) && testStringOrNumber(sub.value) && otherIdReqExp.test(String(sub.value)))
+      .map(({value}) => String(value));
   }
 }

package/src/candidate-search/query-list/index.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *

package/src/match-detection/features/bib/authors.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -27,6 +27,9 @@
 */
 
 import {LevenshteinDistance as leven} from 'natural';
+import {testStringOrNumber} from '../../../matching-utils';
+
+// We should extract also organisational authors (110/710)
 
 export default ({nameTreshold = 10} = {}) => ({
   name: 'Authors',
@@ -40,7 +43,7 @@ export default ({nameTreshold = 10} = {}) => ({
 
       function toObj({code, value}) {
         if (code === 'a') {
-          return {name: value ? value.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase() : ''};
+          return {name: testStringOrNumber(value) ? String(value).replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase() : ''};
         }
 
         return {id: value};

package/src/match-detection/features/bib/language.js

@@ -5,7 +5,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -86,7 +86,7 @@ export default () => ({
       return 0;
     }
 
-    debugData(`There area languages to compare`);
+    debugData(`There are languages to compare`);
 
     if (a[0] === b[0]) {
       return 0.1;

package/src/match-detection/features/bib/publication-time.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -26,11 +26,16 @@
 *
 */
 
+import {testStringOrNumber} from '../../../matching-utils';
+
+// We should also get copyright time and copyright/publication times from 26x
+// We could also enable matching of records which have consequent publication years
+
 export default () => ({
   name: 'Publication time',
   extract: record => {
-    const value = record.get(/^008$/u)?.[0]?.value || '';
-    return value ? [value.slice(7, 11)] : [];
+    const value = record.get(/^008$/u)?.[0]?.value || undefined;
+    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];
   },
   compare: (a, b) => a[0] === b[0] ? 0.1 : -1.0
 });

package/src/match-detection/features/bib/record-type.js

@@ -26,6 +26,8 @@
 *
 */
 
+// we could handle the case of books/notes
+
 export default () => ({
   name: 'Record type',
   extract: r => r.leader[6] ? [r.leader[6]] : [],

package/src/match-detection/features/bib/standard-identifier-factory.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -27,6 +27,7 @@
 */
 
 import createDebugLogger from 'debug';
+import {testStringOrNumber} from '../../../matching-utils';
 
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers');
 const debugData = debug.extend('data');
@@ -40,8 +41,8 @@ export default ({pattern, subfieldCodes}) => {
     if (field) {
       return field.subfields
         .filter(({code}) => subfieldCodes.includes(code))
-      //  .map(({code, value}) => ({code, value: value.replace(/-/ug, '')}));
-        .map(({code, value}) => ({code, value: value ? value.replace(/-/ug, '') : ''}));
+        .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value).replace(/-/ug, '') : ''}))
+        .filter(value => value);
     }
 
     return [];

package/src/match-detection/features/bib/title.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -28,14 +28,15 @@
 
 import createDebugLogger from 'debug';
 import {LevenshteinDistance as leven} from 'natural';
+import {testStringOrNumber} from '../../../matching-utils';
 
 export default ({treshold = 10} = {}) => ({
   name: 'Title',
   extract: record => {
     const title = getTitle();
 
-    if (title) {
-      return [title.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
+    if (testStringOrNumber(title)) {
+      return [String(title).replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
     }
 
     return [];
@@ -46,9 +47,10 @@ export default ({treshold = 10} = {}) => ({
       if (field) {
         return field.subfields
           .filter(({code}) => ['a', 'b'].includes(code))
-          .map(({value}) => value)
+          .map(({value}) => testStringOrNumber(value) ? String(value) : '')
           .join('');
       }
+      return false;
     }
   },
   compare: (a, b) => {

package/src/matching-utils.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -58,8 +58,8 @@ export function getMelindaIdsF035(record) {
 
     return subfields
       .filter(sub => ['a', 'z'].includes(sub.code))
-      .filter(sub => melindaIdRegExp.test(sub.value))
-      .map(({value}) => value ? value.replace(melindaIdRegExp, '$<id>') : '');
+      .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))
+      .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');
 
   }
 }
@@ -81,6 +81,15 @@ function countSubfields(field, subfieldCode) {
 
 export function getSubfieldValues(field, subfieldCode) {
   debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);
-  return field.subfields.filter(({code}) => code === subfieldCode).map(({value}) => value);
+  return field.subfields
+    .filter(({code}) => code === subfieldCode)
+    .map(({value}) => testStringOrNumber(value) ? String(value) : '')
+    .filter(value => value);
 }
 
+export function testStringOrNumber(value) {
+  if (typeof value === 'string' || typeof value === 'number') {
+    return true;
+  }
+  return false;
+}