@natlibfi/melinda-record-matching 2.0.1 → 2.0.7-alpha.1

package/dist/match-detection/features/bib/publication-time-allow-cons-years.js

@@ -0,0 +1,71 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _matchingUtils = require("../../../matching-utils");
+var _debug = _interopRequireDefault(require("debug"));
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020-2023 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+// We should also get copyright time and copyright/publication times from 26x
+var _default = () => ({
+  name: 'Publication time, allow consequent years',
+  extract: ({
+    record
+  }) => {
+    const value = record.get(/^008$/u)?.[0]?.value || undefined;
+    return (0, _matchingUtils.testStringOrNumber)(value) ? [String(value).slice(7, 11)] : [];
+  },
+  compare: (a, b) => {
+    const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years');
+    debug(`Comparing ${a[0]} to ${b[0]}`);
+    const [firstA] = a;
+    const [firstB] = b;
+    if (firstA === firstB) {
+      return 0.1;
+    }
+
+    // If either of years is a non string/number, values are not comparable
+    if (!(0, _matchingUtils.testStringOrNumber)(firstA) || !(0, _matchingUtils.testStringOrNumber)(firstB)) {
+      return 0;
+    }
+    const firstANumber = parseInt(firstA, 10);
+    const firstBNumber = parseInt(firstB, 10);
+    if (isNaN(firstANumber) || isNaN(firstBNumber)) {
+      return -1;
+    }
+
+    // Handle consequent years as a match
+    // see publication-time for a version that does not handle consequent years as a match
+    return firstANumber + 1 === firstBNumber || firstANumber - 1 === firstBNumber ? 0.1 : -1;
+  }
+});
+exports.default = _default;
+//# sourceMappingURL=publication-time-allow-cons-years.js.map

package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"publication-time-allow-cons-years.js","names":["_matchingUtils","require","_debug","_interopRequireDefault","obj","__esModule","default","_default","name","extract","record","value","get","undefined","testStringOrNumber","String","slice","compare","a","b","debug","createDebugLogger","firstA","firstB","firstANumber","parseInt","firstBNumber","isNaN","exports"],"sources":["../../../../src/match-detection/features/bib/publication-time-allow-cons-years.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2023 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {testStringOrNumber} from '../../../matching-utils';\nimport createDebugLogger from 'debug';\n\n// We should also get copyright time and copyright/publication times from 26x\n\nexport default () => ({\n  name: 'Publication time, allow consequent years',\n  extract: ({record}) => {\n    const value = record.get(/^008$/u)?.[0]?.value || undefined;\n    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years');\n    debug(`Comparing ${a[0]} to ${b[0]}`);\n\n    const [firstA] = a;\n    const [firstB] = b;\n\n    if (firstA === firstB) {\n      return 0.1;\n    }\n\n    // If either of years is a non string/number, values are not comparable\n    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {\n      return 0;\n    }\n\n    const firstANumber = parseInt(firstA, 10);\n    const firstBNumber = parseInt(firstB, 10);\n\n    if (isNaN(firstANumber) || isNaN(firstBNumber)) {\n      return -1;\n    }\n\n    // Handle consequent years as a match\n    // see publication-time for a version that does not handle consequent years as a match\n    return firstANumber + 1 === firstBNumber || firstANumber - 1 === firstBNumber ? 0.1 : -1;\n  }\n});\n"],"mappings":";;;;;;AA4BA,IAAAA,cAAA,GAAAC,OAAA;AACA,IAAAC,MAAA,GAAAC,sBAAA,CAAAF,OAAA;AAAsC,SAAAE,uBAAAC,GAAA,WAAAA,GAAA,IAAAA,GAAA,CAAAC,UAAA,GAAAD,GAAA,KAAAE,OAAA,EAAAF,GAAA;AA7BtC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAKA;AAAA,IAAAG,QAAA,GAEeA,CAAA,MAAO;EACpBC,IAAI,EAAE,0CAA0C;EAChDC,OAAO,EAAEA,CAAC;IAACC;EAAM,CAAC,KAAK;IACrB,MAAMC,KAAK,GAAGD,MAAM,CAACE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAED,KAAK,IAAIE,SAAS;IAC3D,OAAO,IAAAC,iCAAkB,EAACH,KAAK,CAAC,GAAG,CAACI,MAAM,CAACJ,KAAK,CAAC,CAACK,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE;EACtE,CAAC;EACDC,OAAO,EAAEA,CAACC,CAAC,EAAEC,CAAC,KAAK;IACjB,MAAMC,KAAK,GAAG,IAAAC,cAAiB,EAAC,kGAAkG,CAAC;IACnID,KAAK,CAAE,aAAYF,CAAC,CAAC,CAAC,CAAE,OAAMC,CAAC,CAAC,CAAC,CAAE,EAAC,CAAC;IAErC,MAAM,CAACG,MAAM,CAAC,GAAGJ,CAAC;IAClB,MAAM,CAACK,MAAM,CAAC,GAAGJ,CAAC;IAElB,IAAIG,MAAM,KAAKC,MAAM,EAAE;MACrB,OAAO,GAAG;IACZ;;IAEA;IACA,IAAI,CAAC,IAAAT,iCAAkB,EAACQ,MAAM,CAAC,IAAI,CAAC,IAAAR,iCAAkB,EAACS,MAAM,CAAC,EAAE;MAC9D,OAAO,CAAC;IACV;IAEA,MAAMC,YAAY,GAAGC,QAAQ,CAACH,MAAM,EAAE,EAAE,CAAC;IACzC,MAAMI,YAAY,GAAGD,QAAQ,CAACF,MAAM,EAAE,EAAE,CAAC;IAEzC,IAAII,KAAK,CAACH,YAAY,CAAC,IAAIG,KAAK,CAACD,YAAY,CAAC,EAAE;MAC9C,OAAO,CAAC,CAAC;IACX;;IAEA;IACA;IACA,OAAOF,YAAY,GAAG,CAAC,KAAKE,YAAY,IAAIF,YAAY,GAAG,CAAC,KAAKE,YAAY,GAAG,GAAG,GAAG,CAAC,CAAC;EAC1F;AACF,CAAC,CAAC;AAAAE,OAAA,CAAAtB,OAAA,GAAAC,QAAA"}

package/dist/match-detection/features/bib/publication-time.js

@@ -4,14 +4,14 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = void 0;
-
+var _matchingUtils = require("../../../matching-utils");
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -32,16 +32,17 @@ exports.default = void 0;
 * for the JavaScript code in this file.
 *
 */
+// We should also get copyright time and copyright/publication times from 26x
+// see publication-time-allow-cons-years for a version allowing consequent years to match
 var _default = () => ({
   name: 'Publication time',
-  extract: record => {
-    var _record$get, _record$get$;
-
-    const value = ((_record$get = record.get(/^008$/u)) === null || _record$get === void 0 ? void 0 : (_record$get$ = _record$get[0]) === null || _record$get$ === void 0 ? void 0 : _record$get$.value) || '';
-    return value ? [value.slice(7, 11)] : [];
+  extract: ({
+    record
+  }) => {
+    const value = record.get(/^008$/u)?.[0]?.value || undefined;
+    return (0, _matchingUtils.testStringOrNumber)(value) ? [String(value).slice(7, 11)] : [];
   },
   compare: (a, b) => a[0] === b[0] ? 0.1 : -1.0
 });
-
 exports.default = _default;
 //# sourceMappingURL=publication-time.js.map

package/dist/match-detection/features/bib/publication-time.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/match-detection/features/bib/publication-time.js"],"names":["name","extract","record","value","get","slice","compare","a","b"],"mappings":";;;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;eAEe,OAAO;AACpBA,EAAAA,IAAI,EAAE,kBADc;AAEpBC,EAAAA,OAAO,EAAEC,MAAM,IAAI;AAAA;;AACjB,UAAMC,KAAK,GAAG,gBAAAD,MAAM,CAACE,GAAP,CAAW,QAAX,6EAAuB,CAAvB,+DAA2BD,KAA3B,KAAoC,EAAlD;AACA,WAAOA,KAAK,GAAG,CAACA,KAAK,CAACE,KAAN,CAAY,CAAZ,EAAe,EAAf,CAAD,CAAH,GAA0B,EAAtC;AACD,GALmB;AAMpBC,EAAAA,OAAO,EAAE,CAACC,CAAD,EAAIC,CAAJ,KAAUD,CAAC,CAAC,CAAD,CAAD,KAASC,CAAC,CAAC,CAAD,CAAV,GAAgB,GAAhB,GAAsB,CAAC;AANtB,CAAP,C","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nexport default () => ({\n  name: 'Publication time',\n  extract: record => {\n    const value = record.get(/^008$/u)?.[0]?.value || '';\n    return value ? [value.slice(7, 11)] : [];\n  },\n  compare: (a, b) => a[0] === b[0] ? 0.1 : -1.0\n});\n"],"file":"publication-time.js"}
+{"version":3,"file":"publication-time.js","names":["_matchingUtils","require","_default","name","extract","record","value","get","undefined","testStringOrNumber","String","slice","compare","a","b","exports","default"],"sources":["../../../../src/match-detection/features/bib/publication-time.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {testStringOrNumber} from '../../../matching-utils';\n\n// We should also get copyright time and copyright/publication times from 26x\n// see publication-time-allow-cons-years for a version allowing consequent years to match\n\nexport default () => ({\n  name: 'Publication time',\n  extract: ({record}) => {\n    const value = record.get(/^008$/u)?.[0]?.value || undefined;\n    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];\n  },\n  compare: (a, b) => a[0] === b[0] ? 0.1 : -1.0\n});\n"],"mappings":";;;;;;AA4BA,IAAAA,cAAA,GAAAC,OAAA;AA5BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAIA;AACA;AAAA,IAAAC,QAAA,GAEeA,CAAA,MAAO;EACpBC,IAAI,EAAE,kBAAkB;EACxBC,OAAO,EAAEA,CAAC;IAACC;EAAM,CAAC,KAAK;IACrB,MAAMC,KAAK,GAAGD,MAAM,CAACE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAED,KAAK,IAAIE,SAAS;IAC3D,OAAO,IAAAC,iCAAkB,EAACH,KAAK,CAAC,GAAG,CAACI,MAAM,CAACJ,KAAK,CAAC,CAACK,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE;EACtE,CAAC;EACDC,OAAO,EAAEA,CAACC,CAAC,EAAEC,CAAC,KAAKD,CAAC,CAAC,CAAC,CAAC,KAAKC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC;AAC5C,CAAC,CAAC;AAAAC,OAAA,CAAAC,OAAA,GAAAd,QAAA"}

package/dist/match-detection/features/bib/record-type.js

@@ -4,7 +4,6 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = void 0;
-
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -32,11 +31,15 @@ exports.default = void 0;
 * for the JavaScript code in this file.
 *
 */
+// we could handle the case of books/notes
+// Recordtype: LDR/06 - Type of Record
+// Note: currently matchValidator fails all mismatching recordTypes, so this won't actually do much
 var _default = () => ({
   name: 'Record type',
-  extract: r => r.leader[6] ? [r.leader[6]] : [],
+  extract: ({
+    record
+  }) => record.leader[6] ? [record.leader[6]] : [],
   compare: (a, b) => a[0] === b[0] ? 0.1 : -0.5
 });
-
 exports.default = _default;
 //# sourceMappingURL=record-type.js.map

package/dist/match-detection/features/bib/record-type.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/match-detection/features/bib/record-type.js"],"names":["name","extract","r","leader","compare","a","b"],"mappings":";;;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;eAEe,OAAO;AACpBA,EAAAA,IAAI,EAAE,aADc;AAEpBC,EAAAA,OAAO,EAAEC,CAAC,IAAIA,CAAC,CAACC,MAAF,CAAS,CAAT,IAAc,CAACD,CAAC,CAACC,MAAF,CAAS,CAAT,CAAD,CAAd,GAA8B,EAFxB;AAGpBC,EAAAA,OAAO,EAAE,CAACC,CAAD,EAAIC,CAAJ,KAAUD,CAAC,CAAC,CAAD,CAAD,KAASC,CAAC,CAAC,CAAD,CAAV,GAAgB,GAAhB,GAAsB,CAAC;AAHtB,CAAP,C","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nexport default () => ({\n  name: 'Record type',\n  extract: r => r.leader[6] ? [r.leader[6]] : [],\n  compare: (a, b) => a[0] === b[0] ? 0.1 : -0.5\n});\n"],"file":"record-type.js"}
+{"version":3,"file":"record-type.js","names":["_default","name","extract","record","leader","compare","a","b","exports","default"],"sources":["../../../../src/match-detection/features/bib/record-type.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\n// we could handle the case of books/notes\n// Recordtype: LDR/06 - Type of Record\n// Note: currently matchValidator fails all mismatching recordTypes, so this won't actually do much\n\nexport default () => ({\n  name: 'Record type',\n  extract: ({record}) => record.leader[6] ? [record.leader[6]] : [],\n  compare: (a, b) => a[0] === b[0] ? 0.1 : -0.5\n});\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;AACA;AACA;AAAA,IAAAA,QAAA,GAEeA,CAAA,MAAO;EACpBC,IAAI,EAAE,aAAa;EACnBC,OAAO,EAAEA,CAAC;IAACC;EAAM,CAAC,KAAKA,MAAM,CAACC,MAAM,CAAC,CAAC,CAAC,GAAG,CAACD,MAAM,CAACC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE;EACjEC,OAAO,EAAEA,CAACC,CAAC,EAAEC,CAAC,KAAKD,CAAC,CAAC,CAAC,CAAC,KAAKC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC;AAC5C,CAAC,CAAC;AAAAC,OAAA,CAAAC,OAAA,GAAAT,QAAA"}

package/dist/match-detection/features/bib/standard-identifier-factory.js

@@ -4,18 +4,17 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = void 0;
-
 var _debug = _interopRequireDefault(require("debug"));
-
+var _matchingUtils = require("../../../matching-utils");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
+/* eslint-disable max-statements */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -36,83 +35,175 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 * for the JavaScript code in this file.
 *
 */
-const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers');
-const debugData = debug.extend('data');
-
+// Note about validity of standardIdentifiers:
+// We have three types of invalid standardIdentifiers:
+// 1. Formally invalid standardIdentifiers (ie. typos either in the resource or the record)
+// 2. Formally valid standardIdentifiers that are used in a wrong resource
+// 3. Canceled standardIdentifiers
+// Matcher could and should check that a standardIdentifier found in a subfield for a valid identifier is formally valid, and if it's not formally valid, handle it as an invalid standardIdentifier
+// Formally valid standardIdentifiers found in subfield for invalid identifier cannot be handled as valid standardIdentifiers, because they can be a case of type 2) or 3) invalid standardIdentifiers
+// We could also do a separate handling for formally valid an formally invalid standardIdentifiers
 var _default = ({
   pattern,
-  subfieldCodes
+  subfieldCodes,
+  identifier,
+  validIdentifierSubfieldCodes = ['a'],
+  invalidIdentifierSubfieldCodes = ['z'],
+  validatorAndNormalizer = undefined
 }) => {
+  const debug = (0, _debug.default)(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);
+  const debugData = debug.extend('data');
   return {
     extract,
     compare
   };
-
-  function extract(record) {
-    const [field] = record.get(pattern);
-
-    if (field) {
-      return field.subfields.filter(({
-        code
-      }) => subfieldCodes.includes(code)).map(({
-        code,
+  function extract({
+    record,
+    recordExternal
+  }) {
+    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
+    const fields = record.get(pattern);
+    debugData(`${label}: ${fields.length} ${identifier}-fields `);
+
+    // extractIdentifierSubfield normalizes hyphens away from the subfield values
+    const identifiersFromFields = fields.map(field => (0, _matchingUtils.extractSubfieldsFromField)(field, subfieldCodes));
+    debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);
+    const allIdentifiers = identifiersFromFields.flat();
+    debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);
+    const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({
+      identifierSubs: allIdentifiers,
+      validatorAndNormalizer,
+      validIdentifierSubfieldCodes,
+      invalidIdentifierSubfieldCodes
+    });
+    const identifiers = (0, _matchingUtils.uniqueSubfields)(validatedAndNormalizedIdentifiers);
+    debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);
+    return identifiers;
+    function validateAndNormalizeIdentifiers({
+      identifierSubs,
+      validatorAndNormalizer,
+      validIdentifierSubfieldCodes,
+      invalidIdentifierSubfieldCodes
+    }) {
+      if (validatorAndNormalizer) {
+        return identifierSubs.map(idSub => validateAndNormalizeIdentifier({
+          idSub,
+          validatorAndNormalizer,
+          validIdentifierSubfieldCodes,
+          invalidIdentifierSubfieldCodes
+        }));
+      }
+      return identifierSubs.map(idSub => normalizeHyphens(idSub));
+    }
+    function validateAndNormalizeIdentifier({
+      idSub,
+      validatorAndNormalizer,
+      validIdentifierSubfieldCodes,
+      invalidIdentifierSubfieldCodes
+    }) {
+      const {
+        valid,
+        value
+      } = validatorAndNormalizer(idSub.value);
+      if (validIdentifierSubfieldCodes.includes(idSub.code) && valid === false) {
+        const [code] = invalidIdentifierSubfieldCodes;
+        return {
+          code,
+          value
+        };
+      }
+      return {
+        code: idSub.code,
         value
-      }) => ({
-        code,
-        value: value.replace(/-/ug, '')
-      }));
+      };
+    }
+    function normalizeHyphens(idSub) {
+      return {
+        code: idSub.code,
+        value: idSub.value.replace(/-/ug, '')
+      };
     }
-
-    return [];
   }
-
   function compare(a, b) {
+    debug(`Comparing A and B`);
     if (a.length === 0 || b.length === 0) {
-      debugData(`No standardidentifiers to compare`);
+      debugData(`No standardidentifiers (${identifier}) to compare`);
       return 0;
     }
-
+    debugData(`A: ${JSON.stringify(a)}`);
+    debugData(`B: ${JSON.stringify(b)}`);
     if (bothHaveValidIdentifiers()) {
+      // Compare only valid identifiers, if both have valid idenfiers
       const {
         maxValues,
+        possibleMatchValues,
         matchingValues
       } = getValueCount(true);
-      return matchingValues / maxValues * 0.75;
+      if (matchingValues < 1) {
+        debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);
+        return -0.75;
+      }
+      debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues}/${possibleMatchValues} valid identifiers match.`);
+      // ignore non-matches if there is mismatching amount of values
+      debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);
+      //we give some kind of penalty for mismatching amount of values instead of simple divide?
+      const penaltyForMissing = 0.1 * (maxValues - possibleMatchValues);
+      const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues);
+      debug(`\t points: penaltyForMissing: ${penaltyForMissing}`);
+      debug(`\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);
+      return 0.75 - penaltyForMisMatch - penaltyForMissing;
+      //return matchingValues / possibleMatchValues * 0.75;
     }
-
+    // If both do not have valid identifiers, compare all identifiers
     const {
       maxValues,
       matchingValues
     } = getValueCount();
+    debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);
     return matchingValues / maxValues * 0.2;
-
     function bothHaveValidIdentifiers() {
       const aValues = a.filter(({
         code
-      }) => code === 'a');
+      }) => validIdentifierSubfieldCodes.includes(code));
       const bValues = a.filter(({
         code
-      }) => code === 'a');
+      }) => validIdentifierSubfieldCodes.includes(code));
+      debug(`A: ${aValues.length} valid ${identifier} identifiers`);
+      debug(`B: ${bValues.length} valid ${identifier} identifiers`);
       return aValues.length > 0 && bValues.length > 0;
     }
-
     function getValueCount(validOnly = false) {
-      const aValues = getIdentifiers(a);
-      const bValues = getIdentifiers(b);
+      const aValues = getIdentifiers(a, validOnly);
+      const bValues = getIdentifiers(b, validOnly);
+      const matchingValues = getMatchingValuesAmount(aValues, bValues);
       return {
         maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,
-        matchingValues: aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length
+        // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)
+        possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,
+        matchingValues
       };
+      function getMatchingValuesAmount(aValues, bValues) {
+        if (bValues.length > aValues.length) {
+          return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
+        }
+        if (aValues.length > bValues.length) {
+          return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
+        }
 
-      function getIdentifiers(values) {
+        // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases
+        // there would be duplicate values
+        const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
+        const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
+        return aToB < bToA ? aToB : bToA;
+      }
+      function getIdentifiers(values, validOnly) {
         if (validOnly) {
           return values.filter(({
             code
-          }) => code === 'a').map(({
+          }) => validIdentifierSubfieldCodes.includes(code)).map(({
             value
           }) => value);
         }
-
         return values.map(({
           value
         }) => value);
@@ -120,6 +211,5 @@ var _default = ({
     }
   }
 };
-
 exports.default = _default;
 //# sourceMappingURL=standard-identifier-factory.js.map

package/dist/match-detection/features/bib/standard-identifier-factory.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/match-detection/features/bib/standard-identifier-factory.js"],"names":["debug","debugData","extend","pattern","subfieldCodes","extract","compare","record","field","get","subfields","filter","code","includes","map","value","replace","a","b","length","bothHaveValidIdentifiers","maxValues","matchingValues","getValueCount","aValues","bValues","validOnly","getIdentifiers","aValue","some","bValue","values"],"mappings":";;;;;;;AA4BA;;;;AA5BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAIA,MAAMA,KAAK,GAAG,oBAAkB,iFAAlB,CAAd;AACA,MAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;;eAEe,CAAC;AAACC,EAAAA,OAAD;AAAUC,EAAAA;AAAV,CAAD,KAA8B;AAC3C,SAAO;AAACC,IAAAA,OAAD;AAAUC,IAAAA;AAAV,GAAP;;AAEA,WAASD,OAAT,CAAiBE,MAAjB,EAAyB;AACvB,UAAM,CAACC,KAAD,IAAUD,MAAM,CAACE,GAAP,CAAWN,OAAX,CAAhB;;AAEA,QAAIK,KAAJ,EAAW;AACT,aAAOA,KAAK,CAACE,SAAN,CACJC,MADI,CACG,CAAC;AAACC,QAAAA;AAAD,OAAD,KAAYR,aAAa,CAACS,QAAd,CAAuBD,IAAvB,CADf,EAEJE,GAFI,CAEA,CAAC;AAACF,QAAAA,IAAD;AAAOG,QAAAA;AAAP,OAAD,MAAoB;AAACH,QAAAA,IAAD;AAAOG,QAAAA,KAAK,EAAEA,KAAK,CAACC,OAAN,CAAc,KAAd,EAAqB,EAArB;AAAd,OAApB,CAFA,CAAP;AAGD;;AAED,WAAO,EAAP;AACD;;AAED,WAASV,OAAT,CAAiBW,CAAjB,EAAoBC,CAApB,EAAuB;AACrB,QAAID,CAAC,CAACE,MAAF,KAAa,CAAb,IAAkBD,CAAC,CAACC,MAAF,KAAa,CAAnC,EAAsC;AACpClB,MAAAA,SAAS,CAAE,mCAAF,CAAT;AACA,aAAO,CAAP;AACD;;AAED,QAAImB,wBAAwB,EAA5B,EAAgC;AAC9B,YAAM;AAACC,QAAAA,SAAD;AAAYC,QAAAA;AAAZ,UAA8BC,aAAa,CAAC,IAAD,CAAjD;AACA,aAAOD,cAAc,GAAGD,SAAjB,GAA6B,IAApC;AACD;;AAED,UAAM;AAACA,MAAAA,SAAD;AAAYC,MAAAA;AAAZ,QAA8BC,aAAa,EAAjD;AACA,WAAOD,cAAc,GAAGD,SAAjB,GAA6B,GAApC;;AAEA,aAASD,wBAAT,GAAoC;AAClC,YAAMI,OAAO,GAAGP,CAAC,CAACN,MAAF,CAAS,CAAC;AAACC,QAAAA;AAAD,OAAD,KAAYA,IAAI,KAAK,GAA9B,CAAhB;AACA,YAAMa,OAAO,GAAGR,CAAC,CAACN,MAAF,CAAS,CAAC;AAACC,QAAAA;AAAD,OAAD,KAAYA,IAAI,KAAK,GAA9B,CAAhB;AACA,aAAOY,OAAO,CAACL,MAAR,GAAiB,CAAjB,IAAsBM,OAAO,CAACN,MAAR,GAAiB,CAA9C;AACD;;AAED,aAASI,aAAT,CAAuBG,SAAS,GAAG,KAAnC,EAA0C;AACxC,YAAMF,OAAO,GAAGG,cAAc,CAACV,CAAD,CAA9B;AACA,YAAMQ,OAAO,GAAGE,cAAc,CAACT,CAAD,CAA9B;AAEA,aAAO;AACLG,QAAAA,SAAS,EAAEG,OAAO,CAACL,MAAR,GAAiBM,OAAO,CAACN,MAAzB,GAAkCK,OAAO,CAACL,MAA1C,GAAmDM,OAAO,CAACN,MADjE;AAELG,QAAAA,cAAc,EAAEE,OAAO,CAACb,MAAR,CAAeiB,MAAM,IAAIH,OAAO,CAACI,IAAR,CAAaC,MAAM,IAAIF,MAAM,KAAKE,MAAlC,CAAzB,EAAoEX;AAF/E,OAAP;;AAKA,eAASQ,cAAT,CAAwBI,MAAxB,EAAgC;AAC9B,YAAIL,SAAJ,EAAe;AACb,iBAAOK,MAAM,CACVpB,MADI,CACG,CAAC;AAACC,YAAAA;AAAD,WAAD,KAAYA,IAAI,KAAK,GADxB,EAEJE,GAFI,CAEA,CAAC;AAACC,YAAAA;AAAD,WAAD,KAAaA,KAFb,CAAP;AAGD;;AAED,eAAOgB,MAAM,CAACjB,GAAP,CAAW,CAAC;AAACC,UAAAA;AAAD,SAAD,KAAaA,KAAxB,CAAP;AACD;AACF;AACF;AACF,C","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers');\nconst debugData = debug.extend('data');\n\nexport default ({pattern, subfieldCodes}) => {\n  return {extract, compare};\n\n  function extract(record) {\n    const [field] = record.get(pattern);\n\n    if (field) {\n      return field.subfields\n        .filter(({code}) => subfieldCodes.includes(code))\n        .map(({code, value}) => ({code, value: value.replace(/-/ug, '')}));\n    }\n\n    return [];\n  }\n\n  function compare(a, b) {\n    if (a.length === 0 || b.length === 0) {\n      debugData(`No standardidentifiers to compare`);\n      return 0;\n    }\n\n    if (bothHaveValidIdentifiers()) {\n      const {maxValues, matchingValues} = getValueCount(true);\n      return matchingValues / maxValues * 0.75;\n    }\n\n    const {maxValues, matchingValues} = getValueCount();\n    return matchingValues / maxValues * 0.2;\n\n    function bothHaveValidIdentifiers() {\n      const aValues = a.filter(({code}) => code === 'a');\n      const bValues = a.filter(({code}) => code === 'a');\n      return aValues.length > 0 && bValues.length > 0;\n    }\n\n    function getValueCount(validOnly = false) {\n      const aValues = getIdentifiers(a);\n      const bValues = getIdentifiers(b);\n\n      return {\n        maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n        matchingValues: aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length\n      };\n\n      function getIdentifiers(values) {\n        if (validOnly) {\n          return values\n            .filter(({code}) => code === 'a')\n            .map(({value}) => value);\n        }\n\n        return values.map(({value}) => value);\n      }\n    }\n  }\n};\n"],"file":"standard-identifier-factory.js"}
+{"version":3,"file":"standard-identifier-factory.js","names":["_debug","_interopRequireDefault","require","_matchingUtils","obj","__esModule","default","_default","pattern","subfieldCodes","identifier","validIdentifierSubfieldCodes","invalidIdentifierSubfieldCodes","validatorAndNormalizer","undefined","debug","createDebugLogger","debugData","extend","extract","compare","record","recordExternal","label","fields","get","length","identifiersFromFields","map","field","extractSubfieldsFromField","JSON","stringify","allIdentifiers","flat","validatedAndNormalizedIdentifiers","validateAndNormalizeIdentifiers","identifierSubs","identifiers","uniqueSubfields","idSub","validateAndNormalizeIdentifier","normalizeHyphens","valid","value","includes","code","replace","a","b","bothHaveValidIdentifiers","maxValues","possibleMatchValues","matchingValues","getValueCount","penaltyForMissing","penaltyForMisMatch","aValues","filter","bValues","validOnly","getIdentifiers","getMatchingValuesAmount","aValue","some","bValue","aToB","bToA","values","exports"],"sources":["../../../../src/match-detection/features/bib/standard-identifier-factory.js"],"sourcesContent":["/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport {extractSubfieldsFromField, uniqueSubfields} from '../../../matching-utils';\n\n// Note about validity of standardIdentifiers:\n// We have three types of invalid standardIdentifiers:\n// 1. Formally invalid standardIdentifiers (ie. typos either in the resource or the record)\n// 2. Formally valid standardIdentifiers that are used in a wrong resource\n// 3. Canceled standardIdentifiers\n\n// Matcher could and should check that a standardIdentifier found in a subfield for a valid identifier is formally valid, and if it's not formally valid, handle it as an invalid standardIdentifier\n// Formally valid standardIdentifiers found in subfield for invalid identifier cannot be handled as valid standardIdentifiers, because they can be a case of type 2) or 3) invalid standardIdentifiers\n// We could also do a separate handling for formally valid an formally invalid standardIdentifiers\n\nexport default ({pattern, subfieldCodes, identifier, validIdentifierSubfieldCodes = ['a'], invalidIdentifierSubfieldCodes = ['z'], validatorAndNormalizer = undefined}) => {\n  const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);\n  const debugData = debug.extend('data');\n\n  return {extract, compare};\n\n  function extract({record, recordExternal}) {\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n    const fields = record.get(pattern);\n    debugData(`${label}: ${fields.length} ${identifier}-fields `);\n\n    // extractIdentifierSubfield normalizes hyphens away from the subfield values\n    const identifiersFromFields = fields.map(field => extractSubfieldsFromField(field, subfieldCodes));\n    debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);\n    const allIdentifiers = identifiersFromFields.flat();\n    debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);\n\n    const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({identifierSubs: allIdentifiers, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes});\n\n    const identifiers = uniqueSubfields(validatedAndNormalizedIdentifiers);\n\n    debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);\n    return identifiers;\n\n    function validateAndNormalizeIdentifiers({identifierSubs, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {\n      if (validatorAndNormalizer) {\n        return identifierSubs.map((idSub) => validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}));\n      }\n      return identifierSubs.map((idSub) => normalizeHyphens(idSub));\n    }\n\n    function validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {\n      const {valid, value} = validatorAndNormalizer(idSub.value);\n      if (validIdentifierSubfieldCodes.includes(idSub.code) && valid === false) {\n        const [code] = invalidIdentifierSubfieldCodes;\n        return {code, value};\n      }\n      return {code: idSub.code, value};\n    }\n\n    function normalizeHyphens(idSub) {\n      return {code: idSub.code, value: idSub.value.replace(/-/ug, '')};\n    }\n\n\n  }\n\n  function compare(a, b) {\n    debug(`Comparing A and B`);\n    if (a.length === 0 || b.length === 0) {\n      debugData(`No standardidentifiers (${identifier}) to compare`);\n      return 0;\n    }\n\n    debugData(`A: ${JSON.stringify(a)}`);\n    debugData(`B: ${JSON.stringify(b)}`);\n\n\n    if (bothHaveValidIdentifiers()) {\n      // Compare only valid identifiers, if both have valid idenfiers\n      const {maxValues, possibleMatchValues, matchingValues} = getValueCount(true);\n      if (matchingValues < 1) {\n        debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);\n        return -0.75;\n      }\n      debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues}/${possibleMatchValues} valid identifiers match.`);\n      // ignore non-matches if there is mismatching amount of values\n      debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);\n      //we give some kind of penalty for mismatching amount of values instead of simple divide?\n      const penaltyForMissing = 0.1 * (maxValues - possibleMatchValues);\n      const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues);\n      debug(`\\t points: penaltyForMissing: ${penaltyForMissing}`);\n      debug(`\\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);\n\n      return 0.75 - penaltyForMisMatch - penaltyForMissing;\n      //return matchingValues / possibleMatchValues * 0.75;\n    }\n    // If both do not have valid identifiers, compare all identifiers\n    const {maxValues, matchingValues} = getValueCount();\n    debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);\n\n    return matchingValues / maxValues * 0.2;\n\n    function bothHaveValidIdentifiers() {\n      const aValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));\n      const bValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));\n      debug(`A: ${aValues.length} valid ${identifier} identifiers`);\n      debug(`B: ${bValues.length} valid ${identifier} identifiers`);\n      return aValues.length > 0 && bValues.length > 0;\n    }\n\n    function getValueCount(validOnly = false) {\n      const aValues = getIdentifiers(a, validOnly);\n      const bValues = getIdentifiers(b, validOnly);\n\n      const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n      return {\n        maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n        // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n        possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n        matchingValues\n      };\n\n      function getMatchingValuesAmount(aValues, bValues) {\n        if (bValues.length > aValues.length) {\n          return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n        }\n        if (aValues.length > bValues.length) {\n          return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n        }\n\n        // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n        // there would be duplicate values\n        const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n        const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n        return aToB < bToA ? aToB : bToA;\n      }\n\n      function getIdentifiers(values, validOnly) {\n        if (validOnly) {\n          return values\n            .filter(({code}) => validIdentifierSubfieldCodes.includes(code))\n            .map(({value}) => value);\n        }\n\n        return values.map(({value}) => value);\n      }\n    }\n  }\n};\n"],"mappings":";;;;;;AA6BA,IAAAA,MAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,cAAA,GAAAD,OAAA;AAAmF,SAAAD,uBAAAG,GAAA,WAAAA,GAAA,IAAAA,GAAA,CAAAC,UAAA,GAAAD,GAAA,KAAAE,OAAA,EAAAF,GAAA;AA9BnF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAKA;AACA;AACA;AACA;AACA;AAEA;AACA;AACA;AAAA,IAAAG,QAAA,GAEeA,CAAC;EAACC,OAAO;EAAEC,aAAa;EAAEC,UAAU;EAAEC,4BAA4B,GAAG,CAAC,GAAG,CAAC;EAAEC,8BAA8B,GAAG,CAAC,GAAG,CAAC;EAAEC,sBAAsB,GAAGC;AAAS,CAAC,KAAK;EACzK,MAAMC,KAAK,GAAG,IAAAC,cAAiB,EAAE,mFAAkFN,UAAW,EAAC,CAAC;EAChI,MAAMO,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtC,OAAO;IAACC,OAAO;IAAEC;EAAO,CAAC;EAEzB,SAASD,OAAOA,CAAC;IAACE,MAAM;IAAEC;EAAc,CAAC,EAAE;IACzC,MAAMC,KAAK,GAAGD,cAAc,IAAIA,cAAc,CAACC,KAAK,GAAGD,cAAc,CAACC,KAAK,GAAG,QAAQ;IACtF,MAAMC,MAAM,GAAGH,MAAM,CAACI,GAAG,CAACjB,OAAO,CAAC;IAClCS,SAAS,CAAE,GAAEM,KAAM,KAAIC,MAAM,CAACE,MAAO,IAAGhB,UAAW,UAAS,CAAC;;IAE7D;IACA,MAAMiB,qBAAqB,GAAGH,MAAM,CAACI,GAAG,CAACC,KAAK,IAAI,IAAAC,wCAAyB,EAACD,KAAK,EAAEpB,aAAa,CAAC,CAAC;IAClGQ,SAAS,CAAE,GAAEM,KAAM,sBAAqBI,qBAAqB,CAACD,MAAO,MAAKK,IAAI,CAACC,SAAS,CAACL,qBAAqB,CAAE,EAAC,CAAC;IAClH,MAAMM,cAAc,GAAGN,qBAAqB,CAACO,IAAI,CAAC,CAAC;IACnDjB,SAAS,CAAE,GAAEM,KAAM,2BAA0BU,cAAc,CAACP,MAAO,MAAKK,IAAI,CAACC,SAAS,CAACC,cAAc,CAAE,EAAC,CAAC;IAEzG,MAAME,iCAAiC,GAAGC,+BAA+B,CAAC;MAACC,cAAc,EAAEJ,cAAc;MAAEpB,sBAAsB;MAAEF,4BAA4B;MAAEC;IAA8B,CAAC,CAAC;IAEjM,MAAM0B,WAAW,GAAG,IAAAC,8BAAe,EAACJ,iCAAiC,CAAC;IAEtElB,SAAS,CAAE,GAAEM,KAAM,6BAA4Be,WAAW,CAACZ,MAAO,MAAKK,IAAI,CAACC,SAAS,CAACM,WAAW,CAAE,EAAC,CAAC;IACrG,OAAOA,WAAW;IAElB,SAASF,+BAA+BA,CAAC;MAACC,cAAc;MAAExB,sBAAsB;MAAEF,4BAA4B;MAAEC;IAA8B,CAAC,EAAE;MAC/I,IAAIC,sBAAsB,EAAE;QAC1B,OAAOwB,cAAc,CAACT,GAAG,CAAEY,KAAK,IAAKC,8BAA8B,CAAC;UAACD,KAAK;UAAE3B,sBAAsB;UAAEF,4BAA4B;UAAEC;QAA8B,CAAC,CAAC,CAAC;MACrK;MACA,OAAOyB,cAAc,CAACT,GAAG,CAAEY,KAAK,IAAKE,gBAAgB,CAACF,KAAK,CAAC,CAAC;IAC/D;IAEA,SAASC,8BAA8BA,CAAC;MAACD,KAAK;MAAE3B,sBAAsB;MAAEF,4BAA4B;MAAEC;IAA8B,CAAC,EAAE;MACrI,MAAM;QAAC+B,KAAK;QAAEC;MAAK,CAAC,GAAG/B,sBAAsB,CAAC2B,KAAK,CAACI,KAAK,CAAC;MAC1D,IAAIjC,4BAA4B,CAACkC,QAAQ,CAACL,KAAK,CAACM,IAAI,CAAC,IAAIH,KAAK,KAAK,KAAK,EAAE;QACxE,MAAM,CAACG,IAAI,CAAC,GAAGlC,8BAA8B;QAC7C,OAAO;UAACkC,IAAI;UAAEF;QAAK,CAAC;MACtB;MACA,OAAO;QAACE,IAAI,EAAEN,KAAK,CAACM,IAAI;QAAEF;MAAK,CAAC;IAClC;IAEA,SAASF,gBAAgBA,CAACF,KAAK,EAAE;MAC/B,OAAO;QAACM,IAAI,EAAEN,KAAK,CAACM,IAAI;QAAEF,KAAK,EAAEJ,KAAK,CAACI,KAAK,CAACG,OAAO,CAAC,KAAK,EAAE,EAAE;MAAC,CAAC;IAClE;EAGF;EAEA,SAAS3B,OAAOA,CAAC4B,CAAC,EAAEC,CAAC,EAAE;IACrBlC,KAAK,CAAE,mBAAkB,CAAC;IAC1B,IAAIiC,CAAC,CAACtB,MAAM,KAAK,CAAC,IAAIuB,CAAC,CAACvB,MAAM,KAAK,CAAC,EAAE;MACpCT,SAAS,CAAE,2BAA0BP,UAAW,cAAa,CAAC;MAC9D,OAAO,CAAC;IACV;IAEAO,SAAS,CAAE,MAAKc,IAAI,CAACC,SAAS,CAACgB,CAAC,CAAE,EAAC,CAAC;IACpC/B,SAAS,CAAE,MAAKc,IAAI,CAACC,SAAS,CAACiB,CAAC,CAAE,EAAC,CAAC;IAGpC,IAAIC,wBAAwB,CAAC,CAAC,EAAE;MAC9B;MACA,MAAM;QAACC,SAAS;QAAEC,mBAAmB;QAAEC;MAAc,CAAC,GAAGC,aAAa,CAAC,IAAI,CAAC;MAC5E,IAAID,cAAc,GAAG,CAAC,EAAE;QACtBtC,KAAK,CAAE,wCAAuCL,UAAW,6BAA4B,CAAC;QACtF,OAAO,CAAC,IAAI;MACd;MACAK,KAAK,CAAE,wCAAuCL,UAAW,MAAK2C,cAAe,IAAGD,mBAAoB,2BAA0B,CAAC;MAC/H;MACArC,KAAK,CAAE,qBAAoBqC,mBAAoB,IAAGD,SAAU,EAAC,CAAC;MAC9D;MACA,MAAMI,iBAAiB,GAAG,GAAG,IAAIJ,SAAS,GAAGC,mBAAmB,CAAC;MACjE,MAAMI,kBAAkB,GAAG,GAAG,IAAIJ,mBAAmB,GAAGC,cAAc,CAAC;MACvEtC,KAAK,CAAE,iCAAgCwC,iBAAkB,EAAC,CAAC;MAC3DxC,KAAK,CAAE,kCAAiCyC,kBAAmB,EAAC,CAAC;MAE7D,OAAO,IAAI,GAAGA,kBAAkB,GAAGD,iBAAiB;MACpD;IACF;IACA;IACA,MAAM;MAACJ,SAAS;MAAEE;IAAc,CAAC,GAAGC,aAAa,CAAC,CAAC;IACnDvC,KAAK,CAAE,+CAA8CL,UAAW,MAAK2C,cAAe,IAAGF,SAAU,mCAAkC,CAAC;IAEpI,OAAOE,cAAc,GAAGF,SAAS,GAAG,GAAG;IAEvC,SAASD,wBAAwBA,CAAA,EAAG;MAClC,MAAMO,OAAO,GAAGT,CAAC,CAACU,MAAM,CAAC,CAAC;QAACZ;MAAI,CAAC,KAAKnC,4BAA4B,CAACkC,QAAQ,CAACC,IAAI,CAAC,CAAC;MACjF,MAAMa,OAAO,GAAGX,CAAC,CAACU,MAAM,CAAC,CAAC;QAACZ;MAAI,CAAC,KAAKnC,4BAA4B,CAACkC,QAAQ,CAACC,IAAI,CAAC,CAAC;MACjF/B,KAAK,CAAE,MAAK0C,OAAO,CAAC/B,MAAO,UAAShB,UAAW,cAAa,CAAC;MAC7DK,KAAK,CAAE,MAAK4C,OAAO,CAACjC,MAAO,UAAShB,UAAW,cAAa,CAAC;MAC7D,OAAO+C,OAAO,CAAC/B,MAAM,GAAG,CAAC,IAAIiC,OAAO,CAACjC,MAAM,GAAG,CAAC;IACjD;IAEA,SAAS4B,aAAaA,CAACM,SAAS,GAAG,KAAK,EAAE;MACxC,MAAMH,OAAO,GAAGI,cAAc,CAACb,CAAC,EAAEY,SAAS,CAAC;MAC5C,MAAMD,OAAO,GAAGE,cAAc,CAACZ,CAAC,EAAEW,SAAS,CAAC;MAE5C,MAAMP,cAAc,GAAGS,uBAAuB,CAACL,OAAO,EAAEE,OAAO,CAAC;MAEhE,OAAO;QACLR,SAAS,EAAEM,OAAO,CAAC/B,MAAM,GAAGiC,OAAO,CAACjC,MAAM,GAAG+B,OAAO,CAAC/B,MAAM,GAAGiC,OAAO,CAACjC,MAAM;QAC5E;QACA0B,mBAAmB,EAAEK,OAAO,CAAC/B,MAAM,GAAGiC,OAAO,CAACjC,MAAM,GAAGiC,OAAO,CAACjC,MAAM,GAAG+B,OAAO,CAAC/B,MAAM;QACtF2B;MACF,CAAC;MAED,SAASS,uBAAuBA,CAACL,OAAO,EAAEE,OAAO,EAAE;QACjD,IAAIA,OAAO,CAACjC,MAAM,GAAG+B,OAAO,CAAC/B,MAAM,EAAE;UACnC,OAAO+B,OAAO,CAACC,MAAM,CAACK,MAAM,IAAIJ,OAAO,CAACK,IAAI,CAACC,MAAM,IAAIF,MAAM,KAAKE,MAAM,CAAC,CAAC,CAACvC,MAAM;QACnF;QACA,IAAI+B,OAAO,CAAC/B,MAAM,GAAGiC,OAAO,CAACjC,MAAM,EAAE;UACnC,OAAOiC,OAAO,CAACD,MAAM,CAACO,MAAM,IAAIR,OAAO,CAACO,IAAI,CAACD,MAAM,IAAIE,MAAM,KAAKF,MAAM,CAAC,CAAC,CAACrC,MAAM;QACnF;;QAEA;QACA;QACA,MAAMwC,IAAI,GAAGT,OAAO,CAACC,MAAM,CAACK,MAAM,IAAIJ,OAAO,CAACK,IAAI,CAACC,MAAM,IAAIF,MAAM,KAAKE,MAAM,CAAC,CAAC,CAACvC,MAAM;QACvF,MAAMyC,IAAI,GAAGR,OAAO,CAACD,MAAM,CAACO,MAAM,IAAIR,OAAO,CAACO,IAAI,CAACD,MAAM,IAAIE,MAAM,KAAKF,MAAM,CAAC,CAAC,CAACrC,MAAM;QAEvF,OAAOwC,IAAI,GAAGC,IAAI,GAAGD,IAAI,GAAGC,IAAI;MAClC;MAEA,SAASN,cAAcA,CAACO,MAAM,EAAER,SAAS,EAAE;QACzC,IAAIA,SAAS,EAAE;UACb,OAAOQ,MAAM,CACVV,MAAM,CAAC,CAAC;YAACZ;UAAI,CAAC,KAAKnC,4BAA4B,CAACkC,QAAQ,CAACC,IAAI,CAAC,CAAC,CAC/DlB,GAAG,CAAC,CAAC;YAACgB;UAAK,CAAC,KAAKA,KAAK,CAAC;QAC5B;QAEA,OAAOwB,MAAM,CAACxC,GAAG,CAAC,CAAC;UAACgB;QAAK,CAAC,KAAKA,KAAK,CAAC;MACvC;IACF;EACF;AACF,CAAC;AAAAyB,OAAA,CAAA/D,OAAA,GAAAC,QAAA"}

package/dist/match-detection/features/bib/title-version-original.js

@@ -0,0 +1,79 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _debug = _interopRequireDefault(require("debug"));
+var _natural = require("natural");
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+var _default = ({
+  treshold = 10
+} = {}) => ({
+  name: 'titleVersionOriginal',
+  extract: ({
+    record
+  }) => {
+    const title = getTitle();
+    if (title) {
+      return [title.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
+    }
+    return [];
+    function getTitle() {
+      const [field] = record.get(/^245$/u);
+      if (field) {
+        return field.subfields.filter(({
+          code
+        }) => ['a', 'b'].includes(code)).map(({
+          value
+        }) => value).join('');
+      }
+    }
+  },
+  compare: (a, b) => {
+    const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');
+    const distance = (0, _natural.LevenshteinDistance)(a[0], b[0]);
+    if (distance === 0) {
+      return 0.5;
+    }
+    const maxLength = getMaxLength();
+    const percentage = distance / maxLength * 100;
+    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
+    if (percentage <= treshold) {
+      return 0.3;
+    }
+    return -0.5;
+    function getMaxLength() {
+      return a[0].length > b[0].length ? a[0].length : b[0].length;
+    }
+  }
+});
+exports.default = _default;
+//# sourceMappingURL=title-version-original.js.map

package/dist/match-detection/features/bib/title-version-original.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"title-version-original.js","names":["_debug","_interopRequireDefault","require","_natural","obj","__esModule","default","_default","treshold","name","extract","record","title","getTitle","replace","toLowerCase","field","get","subfields","filter","code","includes","map","value","join","compare","a","b","debug","createDebugLogger","distance","leven","maxLength","getMaxLength","percentage","length","exports"],"sources":["../../../../src/match-detection/features/bib/title-version-original.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport {LevenshteinDistance as leven} from 'natural';\n\nexport default ({treshold = 10} = {}) => ({\n  name: 'titleVersionOriginal',\n  extract: ({record}) => {\n    const title = getTitle();\n\n    if (title) {\n      return [title.replace(/[^\\p{Letter}\\p{Number}]/gu, '').toLowerCase()];\n    }\n\n    return [];\n\n    function getTitle() {\n      const [field] = record.get(/^245$/u);\n\n      if (field) {\n        return field.subfields\n          .filter(({code}) => ['a', 'b'].includes(code))\n          .map(({value}) => value)\n          .join('');\n      }\n    }\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');\n    const distance = leven(a[0], b[0]);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    const maxLength = getMaxLength();\n    const percentage = distance / maxLength * 100;\n\n    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);\n\n    if (percentage <= treshold) {\n      return 0.3;\n    }\n\n    return -0.5;\n\n    function getMaxLength() {\n      return a[0].length > b[0].length ? a[0].length : b[0].length;\n    }\n\n  }\n});\n"],"mappings":";;;;;;AA4BA,IAAAA,MAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,QAAA,GAAAD,OAAA;AAAqD,SAAAD,uBAAAG,GAAA,WAAAA,GAAA,IAAAA,GAAA,CAAAC,UAAA,GAAAD,GAAA,KAAAE,OAAA,EAAAF,GAAA;AA7BrD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AA1BA,IAAAG,QAAA,GA+BeA,CAAC;EAACC,QAAQ,GAAG;AAAE,CAAC,GAAG,CAAC,CAAC,MAAM;EACxCC,IAAI,EAAE,sBAAsB;EAC5BC,OAAO,EAAEA,CAAC;IAACC;EAAM,CAAC,KAAK;IACrB,MAAMC,KAAK,GAAGC,QAAQ,CAAC,CAAC;IAExB,IAAID,KAAK,EAAE;MACT,OAAO,CAACA,KAAK,CAACE,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAACC,WAAW,CAAC,CAAC,CAAC;IACvE;IAEA,OAAO,EAAE;IAET,SAASF,QAAQA,CAAA,EAAG;MAClB,MAAM,CAACG,KAAK,CAAC,GAAGL,MAAM,CAACM,GAAG,CAAC,QAAQ,CAAC;MAEpC,IAAID,KAAK,EAAE;QACT,OAAOA,KAAK,CAACE,SAAS,CACnBC,MAAM,CAAC,CAAC;UAACC;QAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAACC,QAAQ,CAACD,IAAI,CAAC,CAAC,CAC7CE,GAAG,CAAC,CAAC;UAACC;QAAK,CAAC,KAAKA,KAAK,CAAC,CACvBC,IAAI,CAAC,EAAE,CAAC;MACb;IACF;EACF,CAAC;EACDC,OAAO,EAAEA,CAACC,CAAC,EAAEC,CAAC,KAAK;IACjB,MAAMC,KAAK,GAAG,IAAAC,cAAiB,EAAC,uFAAuF,CAAC;IACxH,MAAMC,QAAQ,GAAG,IAAAC,4BAAK,EAACL,CAAC,CAAC,CAAC,CAAC,EAAEC,CAAC,CAAC,CAAC,CAAC,CAAC;IAElC,IAAIG,QAAQ,KAAK,CAAC,EAAE;MAClB,OAAO,GAAG;IACZ;IAEA,MAAME,SAAS,GAAGC,YAAY,CAAC,CAAC;IAChC,MAAMC,UAAU,GAAGJ,QAAQ,GAAGE,SAAS,GAAG,GAAG;IAE7CJ,KAAK,CAAE,IAAGF,CAAE,SAAQC,CAAE,mBAAkBK,SAAU,gBAAeF,QAAS,kBAAiBI,UAAW,EAAC,CAAC;IAExG,IAAIA,UAAU,IAAI1B,QAAQ,EAAE;MAC1B,OAAO,GAAG;IACZ;IAEA,OAAO,CAAC,GAAG;IAEX,SAASyB,YAAYA,CAAA,EAAG;MACtB,OAAOP,CAAC,CAAC,CAAC,CAAC,CAACS,MAAM,GAAGR,CAAC,CAAC,CAAC,CAAC,CAACQ,MAAM,GAAGT,CAAC,CAAC,CAAC,CAAC,CAACS,MAAM,GAAGR,CAAC,CAAC,CAAC,CAAC,CAACQ,MAAM;IAC9D;EAEF;AACF,CAAC,CAAC;AAAAC,OAAA,CAAA9B,OAAA,GAAAC,QAAA"}