@natlibfi/melinda-record-matching 2.0.1-alpha.1 → 2.1.0

package/dist/index.spec.js

@@ -10,6 +10,8 @@ var _marcRecord = require("@natlibfi/marc-record");
 
 var _ = _interopRequireWildcard(require("."));
 
+var _debug = _interopRequireDefault(require("debug"));
+
 function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
 
 function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
@@ -22,7 +24,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -43,6 +45,8 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 * for the JavaScript code in this file.
 *
 */
+const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:index:test');
+const debugData = debug.extend('data');
 describe('INDEX', () => {
   (0, _fixugenHttpClient.default)({
     callback,
@@ -56,9 +60,12 @@ describe('INDEX', () => {
   async function callback({
     getFixture,
     options,
-    enabled = true
+    enabled = true,
+    expectedMatchStatus,
+    expectedStopReason
   }) {
     if (!enabled) {
+      debug(`Disabled test!`);
       return;
     }
 
@@ -66,35 +73,54 @@ describe('INDEX', () => {
       subfieldValues: false
     });
     const expectedMatches = getFixture('expectedMatches.json');
+    const expectedNonMatches = getFixture('expectedNonMatches.json') || [];
     const match = (0, _.default)(formatOptions());
-    const matches = await match(record);
-    (0, _chai.expect)(formatResults()).to.eql(expectedMatches);
+    const {
+      matches,
+      matchStatus,
+      nonMatches
+    } = await match(record);
+    debugData(`${matches.length}, ${matchStatus.status}/${matchStatus.stopReason}, ${nonMatches ? nonMatches.length : nonMatches}`);
+    const formattedMatchResult = formatRecordResults(matches);
+    (0, _chai.expect)(formattedMatchResult).to.eql(expectedMatches);
+    const formattedNonMatchResult = formatRecordResults(nonMatches);
+    (0, _chai.expect)(formattedNonMatchResult).to.eql(expectedNonMatches);
+    (0, _chai.expect)(matchStatus.status).to.eql(expectedMatchStatus);
+    (0, _chai.expect)(matchStatus.stopReason).to.eql(expectedStopReason);
 
     function formatOptions() {
       const contextFeatures = _.matchDetection.features[options.detection.strategy.type];
       return { ...options,
-        search: { ...options.search,
-          maxRecordsPerRequest: 1
+        search: { ...options.search
         },
         detection: { ...options.detect,
           strategy: options.detection.strategy.features.map(v => contextFeatures[v]())
-        },
-        maxMatches: 2,
-        maxCandidates: 1
+        }
       };
     }
 
-    function formatResults() {
-      return matches.map(({
-        candidate,
-        probability
-      }) => ({
-        probability,
-        candidate: {
-          id: candidate.id,
-          record: candidate.record.toObject()
-        }
-      }));
+    function formatRecordResults(matches) {
+      if (matches) {
+        debugData(JSON.stringify(matches));
+        return matches.map(match => ({ ...match,
+          candidate: formatCandidate(match.candidate)
+        }));
+      }
+
+      return [];
+    } // Format candidate to remove validationOptions from record
+
+
+    function formatCandidate({
+      id,
+      record
+    }) {
+      const newId = id;
+      const newRecord = record;
+      return {
+        id: newId,
+        record: newRecord.toObject()
+      };
     }
   }
 });

package/dist/index.spec.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.spec.js"],"names":["describe","callback","path","__dirname","recurse","fixura","reader","READERS","JSON","getFixture","options","enabled","record","MarcRecord","subfieldValues","expectedMatches","match","formatOptions","matches","formatResults","to","eql","contextFeatures","matchDetection","features","detection","strategy","type","search","maxRecordsPerRequest","detect","map","v","maxMatches","maxCandidates","candidate","probability","id","toObject"],"mappings":";;AA4BA;;AACA;;AACA;;AACA;;AACA;;;;;;;;AAhCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAQAA,QAAQ,CAAC,OAAD,EAAU,MAAM;AACtB,kCAAc;AACZC,IAAAA,QADY;AAEZC,IAAAA,IAAI,EAAE,CAACC,SAAD,EAAY,IAAZ,EAAkB,eAAlB,EAAmC,OAAnC,CAFM;AAGZC,IAAAA,OAAO,EAAE,KAHG;AAIZC,IAAAA,MAAM,EAAE;AACNC,MAAAA,MAAM,EAAEC,gBAAQC;AADV;AAJI,GAAd;;AASA,iBAAeP,QAAf,CAAwB;AAACQ,IAAAA,UAAD;AAAaC,IAAAA,OAAb;AAAsBC,IAAAA,OAAO,GAAG;AAAhC,GAAxB,EAA+D;AAE7D,QAAI,CAACA,OAAL,EAAc;AACZ;AACD;;AAED,UAAMC,MAAM,GAAG,IAAIC,sBAAJ,CAAeJ,UAAU,CAAC,kBAAD,CAAzB,EAA+C;AAACK,MAAAA,cAAc,EAAE;AAAjB,KAA/C,CAAf;AACA,UAAMC,eAAe,GAAGN,UAAU,CAAC,sBAAD,CAAlC;AAEA,UAAMO,KAAK,GAAG,eAAqBC,aAAa,EAAlC,CAAd;AACA,UAAMC,OAAO,GAAG,MAAMF,KAAK,CAACJ,MAAD,CAA3B;AAEA,sBAAOO,aAAa,EAApB,EAAwBC,EAAxB,CAA2BC,GAA3B,CAA+BN,eAA/B;;AAEA,aAASE,aAAT,GAAyB;AACvB,YAAMK,eAAe,GAAGC,iBAAeC,QAAf,CAAwBd,OAAO,CAACe,SAAR,CAAkBC,QAAlB,CAA2BC,IAAnD,CAAxB;AAEA,aAAO,EACL,GAAGjB,OADE;AAELkB,QAAAA,MAAM,EAAE,EACN,GAAGlB,OAAO,CAACkB,MADL;AAENC,UAAAA,oBAAoB,EAAE;AAFhB,SAFH;AAMLJ,QAAAA,SAAS,EAAE,EACT,GAAGf,OAAO,CAACoB,MADF;AAETJ,UAAAA,QAAQ,EAAEhB,OAAO,CAACe,SAAR,CAAkBC,QAAlB,CAA2BF,QAA3B,CAAoCO,GAApC,CAAwCC,CAAC,IAAIV,eAAe,CAACU,CAAD,CAAf,EAA7C;AAFD,SANN;AAULC,QAAAA,UAAU,EAAE,CAVP;AAWLC,QAAAA,aAAa,EAAE;AAXV,OAAP;AAaD;;AAED,aAASf,aAAT,GAAyB;AACvB,aAAOD,OAAO,CAACa,GAAR,CAAY,CAAC;AAACI,QAAAA,SAAD;AAAYC,QAAAA;AAAZ,OAAD,MAA+B;AAChDA,QAAAA,WADgD;AAEhDD,QAAAA,SAAS,EAAE;AACTE,UAAAA,EAAE,EAAEF,SAAS,CAACE,EADL;AAETzB,UAAAA,MAAM,EAAEuB,SAAS,CAACvB,MAAV,CAAiB0B,QAAjB;AAFC;AAFqC,OAA/B,CAAZ,CAAP;AAOD;AACF;AACF,CApDO,CAAR","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {expect} from 'chai';\nimport {READERS} from '@natlibfi/fixura';\nimport generateTests from '@natlibfi/fixugen-http-client';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport createMatchInterface, {matchDetection} from '.';\n\ndescribe('INDEX', () => {\n  generateTests({\n    callback,\n    path: [__dirname, '..', 'test-fixtures', 'index'],\n    recurse: false,\n    fixura: {\n      reader: READERS.JSON\n    }\n  });\n\n  async function callback({getFixture, options, enabled = true}) {\n\n    if (!enabled) {\n      return;\n    }\n\n    const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});\n    const expectedMatches = getFixture('expectedMatches.json');\n\n    const match = createMatchInterface(formatOptions());\n    const matches = await match(record);\n\n    expect(formatResults()).to.eql(expectedMatches);\n\n    function formatOptions() {\n      const contextFeatures = matchDetection.features[options.detection.strategy.type];\n\n      return {\n        ...options,\n        search: {\n          ...options.search,\n          maxRecordsPerRequest: 1\n        },\n        detection: {\n          ...options.detect,\n          strategy: options.detection.strategy.features.map(v => contextFeatures[v]())\n        },\n        maxMatches: 2,\n        maxCandidates: 1\n      };\n    }\n\n    function formatResults() {\n      return matches.map(({candidate, probability}) => ({\n        probability,\n        candidate: {\n          id: candidate.id,\n          record: candidate.record.toObject()\n        }\n      }));\n    }\n  }\n});\n"],"file":"index.spec.js"}
+{"version":3,"sources":["../src/index.spec.js"],"names":["debug","debugData","extend","describe","callback","path","__dirname","recurse","fixura","reader","READERS","JSON","getFixture","options","enabled","expectedMatchStatus","expectedStopReason","record","MarcRecord","subfieldValues","expectedMatches","expectedNonMatches","match","formatOptions","matches","matchStatus","nonMatches","length","status","stopReason","formattedMatchResult","formatRecordResults","to","eql","formattedNonMatchResult","contextFeatures","matchDetection","features","detection","strategy","type","search","detect","map","v","stringify","candidate","formatCandidate","id","newId","newRecord","toObject"],"mappings":";;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;;;;;AAjCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASA,MAAMA,KAAK,GAAG,oBAAkB,8CAAlB,CAAd;AACA,MAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AAEAC,QAAQ,CAAC,OAAD,EAAU,MAAM;AACtB,kCAAc;AACZC,IAAAA,QADY;AAEZC,IAAAA,IAAI,EAAE,CAACC,SAAD,EAAY,IAAZ,EAAkB,eAAlB,EAAmC,OAAnC,CAFM;AAGZC,IAAAA,OAAO,EAAE,KAHG;AAIZC,IAAAA,MAAM,EAAE;AACNC,MAAAA,MAAM,EAAEC,gBAAQC;AADV;AAJI,GAAd;;AASA,iBAAeP,QAAf,CAAwB;AAACQ,IAAAA,UAAD;AAAaC,IAAAA,OAAb;AAAsBC,IAAAA,OAAO,GAAG,IAAhC;AAAsCC,IAAAA,mBAAtC;AAA2DC,IAAAA;AAA3D,GAAxB,EAAwG;AAEtG,QAAI,CAACF,OAAL,EAAc;AACZd,MAAAA,KAAK,CAAE,gBAAF,CAAL;AACA;AACD;;AAED,UAAMiB,MAAM,GAAG,IAAIC,sBAAJ,CAAeN,UAAU,CAAC,kBAAD,CAAzB,EAA+C;AAACO,MAAAA,cAAc,EAAE;AAAjB,KAA/C,CAAf;AACA,UAAMC,eAAe,GAAGR,UAAU,CAAC,sBAAD,CAAlC;AACA,UAAMS,kBAAkB,GAAGT,UAAU,CAAC,yBAAD,CAAV,IAAyC,EAApE;AAGA,UAAMU,KAAK,GAAG,eAAqBC,aAAa,EAAlC,CAAd;AACA,UAAM;AAACC,MAAAA,OAAD;AAAUC,MAAAA,WAAV;AAAuBC,MAAAA;AAAvB,QAAqC,MAAMJ,KAAK,CAACL,MAAD,CAAtD;AACAhB,IAAAA,SAAS,CAAE,GAAEuB,OAAO,CAACG,MAAO,KAAIF,WAAW,CAACG,MAAO,IAAGH,WAAW,CAACI,UAAW,KAAIH,UAAU,GAAGA,UAAU,CAACC,MAAd,GAAuBD,UAAW,EAApH,CAAT;AAEA,UAAMI,oBAAoB,GAAGC,mBAAmB,CAACP,OAAD,CAAhD;AACA,sBAAOM,oBAAP,EAA6BE,EAA7B,CAAgCC,GAAhC,CAAoCb,eAApC;AAEA,UAAMc,uBAAuB,GAAGH,mBAAmB,CAACL,UAAD,CAAnD;AACA,sBAAOQ,uBAAP,EAAgCF,EAAhC,CAAmCC,GAAnC,CAAuCZ,kBAAvC;AAEA,sBAAOI,WAAW,CAACG,MAAnB,EAA2BI,EAA3B,CAA8BC,GAA9B,CAAkClB,mBAAlC;AACA,sBAAOU,WAAW,CAACI,UAAnB,EAA+BG,EAA/B,CAAkCC,GAAlC,CAAsCjB,kBAAtC;;AAGA,aAASO,aAAT,GAAyB;AACvB,YAAMY,eAAe,GAAGC,iBAAeC,QAAf,CAAwBxB,OAAO,CAACyB,SAAR,CAAkBC,QAAlB,CAA2BC,IAAnD,CAAxB;AAEA,aAAO,EACL,GAAG3B,OADE;AAEL4B,QAAAA,MAAM,EAAE,EACN,GAAG5B,OAAO,CAAC4B;AADL,SAFH;AAKLH,QAAAA,SAAS,EAAE,EACT,GAAGzB,OAAO,CAAC6B,MADF;AAETH,UAAAA,QAAQ,EAAE1B,OAAO,CAACyB,SAAR,CAAkBC,QAAlB,CAA2BF,QAA3B,CAAoCM,GAApC,CAAwCC,CAAC,IAAIT,eAAe,CAACS,CAAD,CAAf,EAA7C;AAFD;AALN,OAAP;AAUD;;AAED,aAASb,mBAAT,CAA6BP,OAA7B,EAAsC;AACpC,UAAIA,OAAJ,EAAa;AACXvB,QAAAA,SAAS,CAACU,IAAI,CAACkC,SAAL,CAAerB,OAAf,CAAD,CAAT;AACA,eAAOA,OAAO,CAACmB,GAAR,CAAarB,KAAD,KAAY,EAC7B,GAAGA,KAD0B;AAE7BwB,UAAAA,SAAS,EAAEC,eAAe,CAACzB,KAAK,CAACwB,SAAP;AAFG,SAAZ,CAAZ,CAAP;AAID;;AACD,aAAO,EAAP;AACD,KAlDqG,CAoDtG;;;AACA,aAASC,eAAT,CAAyB;AAACC,MAAAA,EAAD;AAAK/B,MAAAA;AAAL,KAAzB,EAAuC;AACrC,YAAMgC,KAAK,GAAGD,EAAd;AACA,YAAME,SAAS,GAAGjC,MAAlB;AACA,aAAO;AACL+B,QAAAA,EAAE,EAAEC,KADC;AAELhC,QAAAA,MAAM,EAAEiC,SAAS,CAACC,QAAV;AAFH,OAAP;AAID;AAGF;AAEF,CA3EO,CAAR","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {expect} from 'chai';\nimport {READERS} from '@natlibfi/fixura';\nimport generateTests from '@natlibfi/fixugen-http-client';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport createMatchInterface, {matchDetection} from '.';\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:index:test');\nconst debugData = debug.extend('data');\n\ndescribe('INDEX', () => {\n  generateTests({\n    callback,\n    path: [__dirname, '..', 'test-fixtures', 'index'],\n    recurse: false,\n    fixura: {\n      reader: READERS.JSON\n    }\n  });\n\n  async function callback({getFixture, options, enabled = true, expectedMatchStatus, expectedStopReason}) {\n\n    if (!enabled) {\n      debug(`Disabled test!`);\n      return;\n    }\n\n    const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});\n    const expectedMatches = getFixture('expectedMatches.json');\n    const expectedNonMatches = getFixture('expectedNonMatches.json') || [];\n\n\n    const match = createMatchInterface(formatOptions());\n    const {matches, matchStatus, nonMatches} = await match(record);\n    debugData(`${matches.length}, ${matchStatus.status}/${matchStatus.stopReason}, ${nonMatches ? nonMatches.length : nonMatches}`);\n\n    const formattedMatchResult = formatRecordResults(matches);\n    expect(formattedMatchResult).to.eql(expectedMatches);\n\n    const formattedNonMatchResult = formatRecordResults(nonMatches);\n    expect(formattedNonMatchResult).to.eql(expectedNonMatches);\n\n    expect(matchStatus.status).to.eql(expectedMatchStatus);\n    expect(matchStatus.stopReason).to.eql(expectedStopReason);\n\n\n    function formatOptions() {\n      const contextFeatures = matchDetection.features[options.detection.strategy.type];\n\n      return {\n        ...options,\n        search: {\n          ...options.search\n        },\n        detection: {\n          ...options.detect,\n          strategy: options.detection.strategy.features.map(v => contextFeatures[v]())\n        }\n      };\n    }\n\n    function formatRecordResults(matches) {\n      if (matches) {\n        debugData(JSON.stringify(matches));\n        return matches.map((match) => ({\n          ...match,\n          candidate: formatCandidate(match.candidate)\n        }));\n      }\n      return [];\n    }\n\n    // Format candidate to remove validationOptions from record\n    function formatCandidate({id, record}) {\n      const newId = id;\n      const newRecord = record;\n      return {\n        id: newId,\n        record: newRecord.toObject()\n      };\n    }\n\n\n  }\n\n});\n"],"file":"index.spec.js"}

package/dist/match-detection/index.js

@@ -50,10 +50,13 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 var _default = ({
   strategy,
   treshold = 0.9
-}) => (recordA, recordB) => {
+}, returnStrategy = false) => (recordA, recordB) => {
   const minProbabilityQuantifier = 0.5;
   const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:match-detection');
   const debugData = debug.extend('data');
+  debugData(`Strategy: ${JSON.stringify(strategy)}`);
+  debugData(`Treshold: ${JSON.stringify(treshold)}`);
+  debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
   const featuresA = extractFeatures(recordA);
   const featuresB = extractFeatures(recordB);
   debugData(`Features (a): ${JSON.stringify(featuresA)}`);
@@ -64,17 +67,38 @@ var _default = ({
   if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
     const probability = calculateprobability();
     debug(`probability: ${probability} (Treshold: ${treshold})`);
-    return {
+    return returnResult({
       match: probability >= treshold,
       probability
-    };
+    });
   }
 
   debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
-  return {
+  return returnResult({
     match: false,
     probability: 0.0
-  };
+  });
+
+  function returnResult(result) {
+    if (returnStrategy) {
+      debug(`Returning detection strategy with the result`);
+      const resultWithStrategy = {
+        match: result.match,
+        probability: result.probability,
+        strategy: formatStrategy(strategy),
+        treshold
+      };
+      debugData(`${JSON.stringify(resultWithStrategy)}`);
+      return resultWithStrategy;
+    }
+
+    return result;
+  }
+
+  function formatStrategy(strategy) {
+    const strategyNames = strategy.map(element => element.name);
+    return strategyNames || [];
+  }
 
   function calculateprobability() {
     const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);

package/dist/match-detection/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/match-detection/index.js"],"names":["strategy","treshold","recordA","recordB","minProbabilityQuantifier","debug","debugData","extend","featuresA","extractFeatures","featuresB","JSON","stringify","featurePairs","generateFeaturePairs","similarityVector","generateSimilarityVector","some","v","probability","calculateprobability","match","reduce","acc","record","name","extract","concat","value","compared","map","a","b","compare","find","featureName","points","pairs","generatePairs","missingFeatures","findMissing","index","filter","length"],"mappings":";;;;;;;AA6BA;;AACA;;;;;;;;;;AA9BA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAOA;eACe,CAAC;AAACA,EAAAA,QAAD;AAAWC,EAAAA,QAAQ,GAAG;AAAtB,CAAD,KAAgC,CAACC,OAAD,EAAUC,OAAV,KAAsB;AACnE,QAAMC,wBAAwB,GAAG,GAAjC;AAEA,QAAMC,KAAK,GAAG,oBAAkB,mDAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AACA,QAAMC,SAAS,GAAGC,eAAe,CAACP,OAAD,CAAjC;AACA,QAAMQ,SAAS,GAAGD,eAAe,CAACN,OAAD,CAAjC;AAEAG,EAAAA,SAAS,CAAE,iBAAgBK,IAAI,CAACC,SAAL,CAAeJ,SAAf,CAA0B,EAA5C,CAAT;AACAF,EAAAA,SAAS,CAAE,iBAAgBK,IAAI,CAACC,SAAL,CAAeF,SAAf,CAA0B,EAA5C,CAAT;AAEA,QAAMG,YAAY,GAAGC,oBAAoB,EAAzC;AACA,QAAMC,gBAAgB,GAAGC,wBAAwB,EAAjD;;AAEA,MAAID,gBAAgB,CAACE,IAAjB,CAAsBC,CAAC,IAAIA,CAAC,IAAId,wBAAhC,CAAJ,EAA+D;AAC7D,UAAMe,WAAW,GAAGC,oBAAoB,EAAxC;AACAf,IAAAA,KAAK,CAAE,gBAAec,WAAY,eAAclB,QAAS,GAApD,CAAL;AACA,WAAO;AAACoB,MAAAA,KAAK,EAAEF,WAAW,IAAIlB,QAAvB;AAAiCkB,MAAAA;AAAjC,KAAP;AACD;;AAEDb,EAAAA,SAAS,CAAE,4DAA2DF,wBAAyB,GAAtF,CAAT;AACA,SAAO;AAACiB,IAAAA,KAAK,EAAE,KAAR;AAAeF,IAAAA,WAAW,EAAE;AAA5B,GAAP;;AAEA,WAASC,oBAAT,GAAgC;AAC9B,UAAMD,WAAW,GAAGJ,gBAAgB,CAACO,MAAjB,CAAwB,CAACC,GAAD,EAAML,CAAN,KAAYK,GAAG,GAAGL,CAA1C,EAA6C,GAA7C,CAApB;AACA,WAAOC,WAAW,GAAG,GAAd,GAAoB,GAApB,GAA0BA,WAAjC;AACD;;AAED,WAASV,eAAT,CAAyBe,MAAzB,EAAiC;AAC/B,WAAOxB,QAAQ,CAACsB,MAAT,CAAgB,CAACC,GAAD,EAAM;AAACE,MAAAA,IAAD;AAAOC,MAAAA;AAAP,KAAN,KAA0BH,GAAG,CAACI,MAAJ,CAAW;AAACF,MAAAA,IAAD;AAAOG,MAAAA,KAAK,EAAEF,OAAO,CAACF,MAAD;AAArB,KAAX,CAA1C,EAAsF,EAAtF,CAAP;AACD;;AAED,WAASR,wBAAT,GAAoC;AAClC,UAAMa,QAAQ,GAAGhB,YAAY,CAACiB,GAAb,CAAiB,CAAC;AAACL,MAAAA,IAAD;AAAOM,MAAAA,CAAP;AAAUC,MAAAA;AAAV,KAAD,KAAkB;AAClD,YAAM;AAACC,QAAAA;AAAD,UAAYjC,QAAQ,CAACkC,IAAT,CAAc,CAAC;AAACT,QAAAA,IAAI,EAAEU;AAAP,OAAD,KAAyBV,IAAI,KAAKU,WAAhD,CAAlB;AACA,YAAMC,MAAM,GAAGH,OAAO,CAACF,CAAD,EAAIC,CAAJ,CAAtB;AACA,aAAO;AAACP,QAAAA,IAAD;AAAOW,QAAAA;AAAP,OAAP;AACD,KAJgB,CAAjB;AAMA9B,IAAAA,SAAS,CAAE,WAAUK,IAAI,CAACC,SAAL,CAAeiB,QAAf,CAAyB,EAArC,CAAT;AACA,WAAOA,QAAQ,CAACC,GAAT,CAAa,CAAC;AAACM,MAAAA;AAAD,KAAD,KAAcA,MAA3B,CAAP;AACD;;AAED,WAAStB,oBAAT,GAAgC;AAC9B,UAAMuB,KAAK,GAAGC,aAAa,EAA3B;AACA,UAAMC,eAAe,GAAGC,WAAW,EAAnC;AAEAnC,IAAAA,KAAK,CAAE,2FAA0FM,IAAI,CAACC,SAAL,CAAe2B,eAAf,CAAgC,EAA5H,CAAL;AACA,WAAOF,KAAP;;AAEA,aAASC,aAAT,GAAyB;AACvB,aAAO9B,SAAS,CACbc,MADI,CACG,CAACC,GAAD,EAAM;AAACE,QAAAA,IAAD;AAAOG,QAAAA;AAAP,OAAN,EAAqBa,KAArB,KAA+BlB,GAAG,CAACI,MAAJ,CAAW;AAChDF,QAAAA,IADgD;AAEhDM,QAAAA,CAAC,EAAEH,KAF6C;AAGhDI,QAAAA,CAAC,EAAEtB,SAAS,CAAC+B,KAAD,CAAT,CAAiBb;AAH4B,OAAX,CADlC,EAKD,EALC,EAMJc,MANI,CAMG,CAAC;AAACX,QAAAA,CAAD;AAAIC,QAAAA;AAAJ,OAAD,KAAY;AAClB,YAAID,CAAC,CAACY,MAAF,KAAa,CAAb,IAAkBX,CAAC,CAACW,MAAF,KAAa,CAAnC,EAAsC;AACpC,iBAAO,KAAP;AACD;;AAED,eAAO,IAAP;AACD,OAZI,CAAP;AAaD;;AAED,aAASH,WAAT,GAAuB;AACrB,aAAOhC,SAAS,CACbsB,GADI,CACA,CAAC;AAACL,QAAAA;AAAD,OAAD,KAAYA,IADZ,EAEJiB,MAFI,CAEGxB,CAAC,IAAImB,KAAK,CAACpB,IAAN,CAAW,CAAC;AAACQ,QAAAA;AAAD,OAAD,KAAYA,IAAI,KAAKP,CAAhC,MAAuC,KAF/C,CAAP;AAGD;AACF;AACF,C","sourcesContent":["/* eslint-disable no-console */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport * as features from './features';\n\nexport {features};\n\n// eslint-disable-next-line max-statements\nexport default ({strategy, treshold = 0.9}) => (recordA, recordB) => {\n  const minProbabilityQuantifier = 0.5;\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n  const debugData = debug.extend('data');\n  const featuresA = extractFeatures(recordA);\n  const featuresB = extractFeatures(recordB);\n\n  debugData(`Features (a): ${JSON.stringify(featuresA)}`);\n  debugData(`Features (b): ${JSON.stringify(featuresB)}`);\n\n  const featurePairs = generateFeaturePairs();\n  const similarityVector = generateSimilarityVector();\n\n  if (similarityVector.some(v => v >= minProbabilityQuantifier)) {\n    const probability = calculateprobability();\n    debug(`probability: ${probability} (Treshold: ${treshold})`);\n    return {match: probability >= treshold, probability};\n  }\n\n  debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);\n  return {match: false, probability: 0.0};\n\n  function calculateprobability() {\n    const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n    return probability > 1.0 ? 1.0 : probability;\n  }\n\n  function extractFeatures(record) {\n    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract(record)}), []);\n  }\n\n  function generateSimilarityVector() {\n    const compared = featurePairs.map(({name, a, b}) => {\n      const {compare} = strategy.find(({name: featureName}) => name === featureName);\n      const points = compare(a, b);\n      return {name, points};\n    });\n\n    debugData(`Points: ${JSON.stringify(compared)}`);\n    return compared.map(({points}) => points);\n  }\n\n  function generateFeaturePairs() {\n    const pairs = generatePairs();\n    const missingFeatures = findMissing();\n\n    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n    return pairs;\n\n    function generatePairs() {\n      return featuresA\n        .reduce((acc, {name, value}, index) => acc.concat({\n          name,\n          a: value,\n          b: featuresB[index].value\n        }), [])\n        .filter(({a, b}) => {\n          if (a.length === 0 || b.length === 0) {\n            return false;\n          }\n\n          return true;\n        });\n    }\n\n    function findMissing() {\n      return featuresA\n        .map(({name}) => name)\n        .filter(v => pairs.some(({name}) => name === v) === false);\n    }\n  }\n};\n"],"file":"index.js"}
+{"version":3,"sources":["../../src/match-detection/index.js"],"names":["strategy","treshold","returnStrategy","recordA","recordB","minProbabilityQuantifier","debug","debugData","extend","JSON","stringify","featuresA","extractFeatures","featuresB","featurePairs","generateFeaturePairs","similarityVector","generateSimilarityVector","some","v","probability","calculateprobability","returnResult","match","result","resultWithStrategy","formatStrategy","strategyNames","map","element","name","reduce","acc","record","extract","concat","value","compared","a","b","compare","find","featureName","points","pairs","generatePairs","missingFeatures","findMissing","index","filter","length"],"mappings":";;;;;;;AA6BA;;AACA;;;;;;;;;;AA9BA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAOA;eACe,CAAC;AAACA,EAAAA,QAAD;AAAWC,EAAAA,QAAQ,GAAG;AAAtB,CAAD,EAA6BC,cAAc,GAAG,KAA9C,KAAwD,CAACC,OAAD,EAAUC,OAAV,KAAsB;AAC3F,QAAMC,wBAAwB,GAAG,GAAjC;AAEA,QAAMC,KAAK,GAAG,oBAAkB,mDAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AAEAD,EAAAA,SAAS,CAAE,aAAYE,IAAI,CAACC,SAAL,CAAeV,QAAf,CAAyB,EAAvC,CAAT;AACAO,EAAAA,SAAS,CAAE,aAAYE,IAAI,CAACC,SAAL,CAAeT,QAAf,CAAyB,EAAvC,CAAT;AACAM,EAAAA,SAAS,CAAE,mBAAkBE,IAAI,CAACC,SAAL,CAAeR,cAAf,CAA+B,EAAnD,CAAT;AAEA,QAAMS,SAAS,GAAGC,eAAe,CAACT,OAAD,CAAjC;AACA,QAAMU,SAAS,GAAGD,eAAe,CAACR,OAAD,CAAjC;AAEAG,EAAAA,SAAS,CAAE,iBAAgBE,IAAI,CAACC,SAAL,CAAeC,SAAf,CAA0B,EAA5C,CAAT;AACAJ,EAAAA,SAAS,CAAE,iBAAgBE,IAAI,CAACC,SAAL,CAAeG,SAAf,CAA0B,EAA5C,CAAT;AAEA,QAAMC,YAAY,GAAGC,oBAAoB,EAAzC;AACA,QAAMC,gBAAgB,GAAGC,wBAAwB,EAAjD;;AAEA,MAAID,gBAAgB,CAACE,IAAjB,CAAsBC,CAAC,IAAIA,CAAC,IAAId,wBAAhC,CAAJ,EAA+D;AAC7D,UAAMe,WAAW,GAAGC,oBAAoB,EAAxC;AACAf,IAAAA,KAAK,CAAE,gBAAec,WAAY,eAAcnB,QAAS,GAApD,CAAL;AACA,WAAOqB,YAAY,CAAC;AAACC,MAAAA,KAAK,EAAEH,WAAW,IAAInB,QAAvB;AAAiCmB,MAAAA;AAAjC,KAAD,CAAnB;AACD;;AAEDb,EAAAA,SAAS,CAAE,4DAA2DF,wBAAyB,GAAtF,CAAT;AACA,SAAOiB,YAAY,CAAC;AAACC,IAAAA,KAAK,EAAE,KAAR;AAAeH,IAAAA,WAAW,EAAE;AAA5B,GAAD,CAAnB;;AAEA,WAASE,YAAT,CAAsBE,MAAtB,EAA8B;AAC5B,QAAItB,cAAJ,EAAoB;AAClBI,MAAAA,KAAK,CAAE,8CAAF,CAAL;AACA,YAAMmB,kBAAkB,GAAG;AAACF,QAAAA,KAAK,EAAEC,MAAM,CAACD,KAAf;AAAsBH,QAAAA,WAAW,EAAEI,MAAM,CAACJ,WAA1C;AAAuDpB,QAAAA,QAAQ,EAAE0B,cAAc,CAAC1B,QAAD,CAA/E;AAA2FC,QAAAA;AAA3F,OAA3B;AACAM,MAAAA,SAAS,CAAE,GAAEE,IAAI,CAACC,SAAL,CAAee,kBAAf,CAAmC,EAAvC,CAAT;AACA,aAAOA,kBAAP;AACD;;AACD,WAAOD,MAAP;AACD;;AAED,WAASE,cAAT,CAAwB1B,QAAxB,EAAkC;AAChC,UAAM2B,aAAa,GAAG3B,QAAQ,CAAC4B,GAAT,CAAaC,OAAO,IAAIA,OAAO,CAACC,IAAhC,CAAtB;AACA,WAAOH,aAAa,IAAI,EAAxB;AACD;;AAED,WAASN,oBAAT,GAAgC;AAC9B,UAAMD,WAAW,GAAGJ,gBAAgB,CAACe,MAAjB,CAAwB,CAACC,GAAD,EAAMb,CAAN,KAAYa,GAAG,GAAGb,CAA1C,EAA6C,GAA7C,CAApB;AACA,WAAOC,WAAW,GAAG,GAAd,GAAoB,GAApB,GAA0BA,WAAjC;AACD;;AAED,WAASR,eAAT,CAAyBqB,MAAzB,EAAiC;AAC/B,WAAOjC,QAAQ,CAAC+B,MAAT,CAAgB,CAACC,GAAD,EAAM;AAACF,MAAAA,IAAD;AAAOI,MAAAA;AAAP,KAAN,KAA0BF,GAAG,CAACG,MAAJ,CAAW;AAACL,MAAAA,IAAD;AAAOM,MAAAA,KAAK,EAAEF,OAAO,CAACD,MAAD;AAArB,KAAX,CAA1C,EAAsF,EAAtF,CAAP;AACD;;AAED,WAAShB,wBAAT,GAAoC;AAClC,UAAMoB,QAAQ,GAAGvB,YAAY,CAACc,GAAb,CAAiB,CAAC;AAACE,MAAAA,IAAD;AAAOQ,MAAAA,CAAP;AAAUC,MAAAA;AAAV,KAAD,KAAkB;AAClD,YAAM;AAACC,QAAAA;AAAD,UAAYxC,QAAQ,CAACyC,IAAT,CAAc,CAAC;AAACX,QAAAA,IAAI,EAAEY;AAAP,OAAD,KAAyBZ,IAAI,KAAKY,WAAhD,CAAlB;AACA,YAAMC,MAAM,GAAGH,OAAO,CAACF,CAAD,EAAIC,CAAJ,CAAtB;AACA,aAAO;AAACT,QAAAA,IAAD;AAAOa,QAAAA;AAAP,OAAP;AACD,KAJgB,CAAjB;AAMApC,IAAAA,SAAS,CAAE,WAAUE,IAAI,CAACC,SAAL,CAAe2B,QAAf,CAAyB,EAArC,CAAT;AACA,WAAOA,QAAQ,CAACT,GAAT,CAAa,CAAC;AAACe,MAAAA;AAAD,KAAD,KAAcA,MAA3B,CAAP;AACD;;AAED,WAAS5B,oBAAT,GAAgC;AAC9B,UAAM6B,KAAK,GAAGC,aAAa,EAA3B;AACA,UAAMC,eAAe,GAAGC,WAAW,EAAnC;AAEAzC,IAAAA,KAAK,CAAE,2FAA0FG,IAAI,CAACC,SAAL,CAAeoC,eAAf,CAAgC,EAA5H,CAAL;AACA,WAAOF,KAAP;;AAEA,aAASC,aAAT,GAAyB;AACvB,aAAOlC,SAAS,CACboB,MADI,CACG,CAACC,GAAD,EAAM;AAACF,QAAAA,IAAD;AAAOM,QAAAA;AAAP,OAAN,EAAqBY,KAArB,KAA+BhB,GAAG,CAACG,MAAJ,CAAW;AAChDL,QAAAA,IADgD;AAEhDQ,QAAAA,CAAC,EAAEF,KAF6C;AAGhDG,QAAAA,CAAC,EAAE1B,SAAS,CAACmC,KAAD,CAAT,CAAiBZ;AAH4B,OAAX,CADlC,EAKD,EALC,EAMJa,MANI,CAMG,CAAC;AAACX,QAAAA,CAAD;AAAIC,QAAAA;AAAJ,OAAD,KAAY;AAClB,YAAID,CAAC,CAACY,MAAF,KAAa,CAAb,IAAkBX,CAAC,CAACW,MAAF,KAAa,CAAnC,EAAsC;AACpC,iBAAO,KAAP;AACD;;AAED,eAAO,IAAP;AACD,OAZI,CAAP;AAaD;;AAED,aAASH,WAAT,GAAuB;AACrB,aAAOpC,SAAS,CACbiB,GADI,CACA,CAAC;AAACE,QAAAA;AAAD,OAAD,KAAYA,IADZ,EAEJmB,MAFI,CAEG9B,CAAC,IAAIyB,KAAK,CAAC1B,IAAN,CAAW,CAAC;AAACY,QAAAA;AAAD,OAAD,KAAYA,IAAI,KAAKX,CAAhC,MAAuC,KAF/C,CAAP;AAGD;AACF;AACF,C","sourcesContent":["/* eslint-disable no-console */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport * as features from './features';\n\nexport {features};\n\n// eslint-disable-next-line max-statements\nexport default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA, recordB) => {\n  const minProbabilityQuantifier = 0.5;\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n  const debugData = debug.extend('data');\n\n  debugData(`Strategy: ${JSON.stringify(strategy)}`);\n  debugData(`Treshold: ${JSON.stringify(treshold)}`);\n  debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n\n  const featuresA = extractFeatures(recordA);\n  const featuresB = extractFeatures(recordB);\n\n  debugData(`Features (a): ${JSON.stringify(featuresA)}`);\n  debugData(`Features (b): ${JSON.stringify(featuresB)}`);\n\n  const featurePairs = generateFeaturePairs();\n  const similarityVector = generateSimilarityVector();\n\n  if (similarityVector.some(v => v >= minProbabilityQuantifier)) {\n    const probability = calculateprobability();\n    debug(`probability: ${probability} (Treshold: ${treshold})`);\n    return returnResult({match: probability >= treshold, probability});\n  }\n\n  debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);\n  return returnResult({match: false, probability: 0.0});\n\n  function returnResult(result) {\n    if (returnStrategy) {\n      debug(`Returning detection strategy with the result`);\n      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold};\n      debugData(`${JSON.stringify(resultWithStrategy)}`);\n      return resultWithStrategy;\n    }\n    return result;\n  }\n\n  function formatStrategy(strategy) {\n    const strategyNames = strategy.map(element => element.name);\n    return strategyNames || [];\n  }\n\n  function calculateprobability() {\n    const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n    return probability > 1.0 ? 1.0 : probability;\n  }\n\n  function extractFeatures(record) {\n    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract(record)}), []);\n  }\n\n  function generateSimilarityVector() {\n    const compared = featurePairs.map(({name, a, b}) => {\n      const {compare} = strategy.find(({name: featureName}) => name === featureName);\n      const points = compare(a, b);\n      return {name, points};\n    });\n\n    debugData(`Points: ${JSON.stringify(compared)}`);\n    return compared.map(({points}) => points);\n  }\n\n  function generateFeaturePairs() {\n    const pairs = generatePairs();\n    const missingFeatures = findMissing();\n\n    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n    return pairs;\n\n    function generatePairs() {\n      return featuresA\n        .reduce((acc, {name, value}, index) => acc.concat({\n          name,\n          a: value,\n          b: featuresB[index].value\n        }), [])\n        .filter(({a, b}) => {\n          if (a.length === 0 || b.length === 0) {\n            return false;\n          }\n\n          return true;\n        });\n    }\n\n    function findMissing() {\n      return featuresA\n        .map(({name}) => name)\n        .filter(v => pairs.some(({name}) => name === v) === false);\n    }\n  }\n};\n"],"file":"index.js"}

package/package.json

@@ -14,7 +14,7 @@
 		"url": "git@github.com:natlibfi/melinda-record-matching-js.git"
 	},
 	"license": "LGPL-3.0+",
-	"version": "2.0.1-alpha.1",
+	"version": "2.1.0",
 	"main": "./dist/index.js",
 	"engines": {
 		"node": ">=14"
@@ -36,22 +36,22 @@
 	},
 	"dependencies": {
 		"@natlibfi/marc-record": "^7.0.0",
-		"@natlibfi/marc-record-serializers": "^8.0.4",
-		"@natlibfi/sru-client": "^5.0.0-alpha.1",
+		"@natlibfi/marc-record-serializers": "^8.1.0",
+		"@natlibfi/sru-client": "^5.0.0",
 		"debug": "^4.3.3",
 		"moment": "^2.29.1",
-		"natural": "^5.1.11",
+		"natural": "^5.1.13",
 		"uuid": "^8.3.2",
-		"winston": "^3.3.3"
+		"winston": "^3.3.4"
 	},
 	"devDependencies": {
-		"@babel/cli": "^7.16.0",
-		"@babel/core": "^7.16.0",
-		"@babel/node": "^7.16.0",
-		"@babel/preset-env": "^7.16.4",
-		"@babel/register": "^7.16.0",
-		"@babel/eslint-parser": "^7.16.3",
-		"@natlibfi/eslint-config-melinda-backend": "^1.1.1",
+		"@babel/cli": "^7.16.7",
+		"@babel/core": "^7.16.7",
+		"@babel/eslint-parser": "^7.16.5",
+		"@babel/node": "^7.16.7",
+		"@babel/preset-env": "^7.16.7",
+		"@babel/register": "^7.16.7",
+		"@natlibfi/eslint-config-melinda-backend": "^2.0.0",
 		"@natlibfi/fixugen": "^1.0.2",
 		"@natlibfi/fixugen-http-client": "^1.1.3",
 		"@natlibfi/fixura": "^2.2.1",
@@ -60,7 +60,7 @@
 		"chai": "^4.3.4",
 		"chai-as-promised": "^7.1.1",
 		"cross-env": "^7.0.3",
-		"eslint": "^7.32.0",
+		"eslint": "^8.6.0",
 		"mocha": "^9.1.3",
 		"nodemon": "^2.0.15",
 		"nyc": "^15.1.0"

package/src/candidate-search/index.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -36,14 +36,29 @@ export {searchTypes} from './query-list';
 
 export class CandidateSearchError extends Error {}
 
-export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
+// serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000
+
+// eslint-disable-next-line max-statements
+export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {
   MarcRecord.setValidationOptions({subfieldValues: false});
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search');
+  const debugData = debug.extend('data');
+
+  debugData(`SearchSpec: ${JSON.stringify(searchSpec)}`);
+  debugData(`Url: ${url}`);
+  debugData(`MaxRecordsPerRequest ${maxRecordsPerRequest}`);
+  debugData(`ServerMaxResult: ${serverMaxResult}`);
+  debugData(`MaxCandidates: ${maxCandidates}`);
+
+  // Do not retrieve more candidates than defined in maxCandidates
+  const adjustedMaxRecordsPerRequest = maxRecordsPerRequest >= maxCandidates ? maxCandidates : maxRecordsPerRequest;
+
   const inputRecordId = getRecordId(record);
   const queryList = generateQueryList(record, searchSpec);
   const client = createClient({
-    url, maxRecordsPerRequest,
+    url,
+    maxRecordsPerRequest: adjustedMaxRecordsPerRequest,
     version: '2.0',
     retrieveAll: false
   });
@@ -54,27 +69,49 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
     throw new CandidateSearchError(`Generated query list contains no queries`);
   }
 
+  // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
+  // state.query : current query (undefined if there was no queries left)
+  // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
+  // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)
+  // state.queriesLeft : amount of queries left
+  // state.queryCounter : sequence for current query
+  // state.maxedQueries : queries that resulted in more than serverMaxResults hits
+
+
   // eslint-disable-next-line max-statements
-  return async ({queryOffset = 0, resultSetOffset = 1}) => {
+  return async ({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) => {
     const query = queryList[queryOffset];
 
     if (query) {
-      const {records, nextOffset} = await retrieveRecords();
+      const {records, nextOffset, total} = await retrieveRecords();
+
+      // If resultSetOffset === 1 this is the first search for the current query
+      debugData(`ResultSetOffset: ${resultSetOffset}`);
+      const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;
+      const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;
+      const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;
+      const newQueryCandidateCounter = resultSetOffset === 1 ? records.length : queryCandidateCounter + records.length;
+
+      const maxedQuery = resultSetOffset === 1 ? checkMaxedQuery(query, total, serverMaxResult) : undefined;
+      const newMaxedQueries = maxedQuery ? maxedQueries.concat(maxedQuery) : maxedQueries;
 
       if (typeof nextOffset === 'number') {
-        debug(`Running next search for query ${queryOffset} ${query}`);
-        return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1)};
+        debug(`Next search will be for query ${queryOffset} ${query}, starting from record ${nextOffset}`);
+        return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
       }
-      debug(`Query ${queryOffset} ${query} done, moving to next query. (${queryList.length - (queryOffset + 1)} queries left)`);
-      return {records, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1)};
+      debug(`Query ${queryOffset} ${query} done.`);
+      debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);
+      return {records, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
     }
 
     debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);
-    return {records: []};
+    return {records: [], queriesLeft: 0, queryCounter, maxedQueries};
 
     function retrieveRecords() {
       return new Promise((resolve, reject) => {
         const promises = [];
+        // eslint-disable-next-line functional/no-let
+        let totalRecords = 0;
 
         debug(`Searching for candidates with query: ${query} (Offset ${resultSetOffset})`);
 
@@ -88,6 +125,10 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
             debug(`SRU error for query: ${query}: ${err}`);
             reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));
           })
+          .on('total', total => {
+            debug(`Got total: ${total}`);
+            totalRecords += total;
+          })
           .on('end', async nextOffset => {
             try {
               const records = await Promise.all(promises);
@@ -95,7 +136,7 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
 
               debug(`Found ${filtered.length} candidates`);
 
-              resolve({nextOffset, records: filtered});
+              resolve({nextOffset, records: filtered, total: totalRecords});
             } catch (err) {
               reject(err);
             }
@@ -108,14 +149,6 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
                 const foundRecordMarc = await MARCXML.from(foundRecord, {subfieldValues: false});
                 const foundRecordId = getRecordId(foundRecordMarc);
 
-                // This does not work and might cause problems:
-                // Record *should* match itself AND in REST the input record is given id 000000001 always
-                debug(`Checking ${inputRecordId} vs ${foundRecordId}`);
-                if (inputRecordId === foundRecordId) {
-                  debug(`Input and candidate are the same record per 001. Discarding candidate`);
-                  return;
-                }
-
                 return {record: foundRecordMarc, id: foundRecordId};
               } catch (err) {
                 throw new Error(`Failed converting record: ${err}, record: ${foundRecord}`);
@@ -126,6 +159,15 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
     }
   };
 
+  function checkMaxedQuery(query, total, serverMaxResult) {
+    // eslint-disable-next-line functional/no-conditional-statement
+    if (total >= serverMaxResult) {
+      debug(`WARNING: Query ${query} resulted in ${total} hits which meets the serverMaxResult (${serverMaxResult}) `);
+      return query;
+    }
+  }
+
+
   function getRecordId(record) {
     const [field] = record.get(/^001$/u);
     return field ? field.value : '';

package/src/candidate-search/index.spec.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -31,6 +31,9 @@ import {READERS} from '@natlibfi/fixura';
 import generateTests from '@natlibfi/fixugen-http-client';
 import {MarcRecord} from '@natlibfi/marc-record';
 import createSearchInterface, {CandidateSearchError} from '.';
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:test');
 
 describe('candidate-search', () => {
   generateTests({
@@ -43,7 +46,7 @@ describe('candidate-search', () => {
   });
 
   // eslint-disable-next-line max-statements
-  async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError, expectedSearchError, enabled = true}) {
+  async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError = false, expectedSearchError = false, enabled = true}) {
     const url = 'http://foo.bar';
 
     if (!enabled) {
@@ -61,18 +64,20 @@ describe('candidate-search', () => {
     }
 
     const search = createSearchInterface({...formatFactoryOptions(), url});
-    await iterate({searchOptions});
+    await iterate({searchOptions, expectedSearchError});
 
     function formatFactoryOptions() {
+      debug(`Using factoryOptions: ${JSON.stringify(factoryOptions)}`);
       return {
         ...factoryOptions,
-        maxRecordsPerRequest: 1,
+        maxRecordsPerRequest: factoryOptions.maxRecordsPerRequest || 1,
+        maxServerResults: factoryOptions.maxServerResults || undefined,
         record: new MarcRecord(factoryOptions.record, {subfieldValues: false})
       };
     }
 
     // eslint-disable-next-line max-statements
-    async function iterate({searchOptions, count = 1}) {
+    async function iterate({searchOptions, expectedSearchError, count = 1}) {
       const expectedResults = getFixture(`expectedResults${count}.json`);
 
       if (expectedSearchError) { // eslint-disable-line functional/no-conditional-statement
@@ -86,15 +91,10 @@ describe('candidate-search', () => {
         }
       }
 
-      const results = await search(searchOptions);
-
-      expect(formatResults(results)).to.eql(expectedResults);
-
-      if (results.records.length > 0) {
-        return iterate({
-          searchOptions: resultsToOptions(results),
-          count: count + 1
-        });
+      // eslint-disable-next-line functional/no-conditional-statement
+      if (!expectedSearchError) {
+        const results = await search(searchOptions);
+        expect(formatResults(results)).to.eql(expectedResults);
       }
 
       function formatResults(results) {
@@ -105,11 +105,6 @@ describe('candidate-search', () => {
         };
       }
 
-      function resultsToOptions(results) {
-        return Object.entries(results)
-          .filter(([k]) => k === 'records' === false) // If key is 'records' return false
-          .reduce((acc, [k, v]) => ({...acc, [k]: v}), {});
-      }
     }
   }
 });

package/src/candidate-search/query-list/bib.js

@@ -175,12 +175,15 @@ export function bibHostComponents(record) {
 
 // SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to
 // (probably) to *headings* index TIT
+// - Aleph cannot currently handle headings searches starting with a boolean - in these cases use word search
+
 // Headings index TIT drops articles etc. from the start of the title according to the filing indicator
 // Currently filing indicator is not implemented - if the title starts with an article and the Melinda
 // record is correctly catalogued using a filing indicator -> dc.title search won't match
 
 export function bibTitle(record) {
   const title = getTitle();
+  const booleanStartWords = ['and', 'or', 'nor', 'not'];
 
   if (title) {
     const formatted = title
@@ -191,8 +194,10 @@ export function bibTitle(record) {
       .slice(0, 30)
       .trim();
 
+    // use word search for titles starting with a boolean
+    const useWordSearch = booleanStartWords.some(word => formatted.toLowerCase().startsWith(word));
     // Prevent too many matches by having a minimum length requirement
-    return formatted.length >= 5 ? [`dc.title="^${formatted}*"`] : [];
+    return formatted.length >= 5 ? [`dc.title="${useWordSearch ? '' : '^'}${formatted}*"`] : [];
   }
 
   return [];