@natlibfi/melinda-record-matching 2.0.0-alpha.1 → 2.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/candidate-search/index.js +85 -16
- package/dist/candidate-search/index.js.map +1 -1
- package/dist/candidate-search/index.spec.js +7 -2
- package/dist/candidate-search/index.spec.js.map +1 -1
- package/dist/candidate-search/query-list/bib.js +10 -3
- package/dist/candidate-search/query-list/bib.js.map +1 -1
- package/dist/index.js +357 -39
- package/dist/index.js.map +1 -1
- package/dist/index.spec.js +46 -20
- package/dist/index.spec.js.map +1 -1
- package/dist/match-detection/index.js +29 -5
- package/dist/match-detection/index.js.map +1 -1
- package/package.json +15 -15
- package/src/candidate-search/index.js +65 -11
- package/src/candidate-search/index.spec.js +7 -2
- package/src/candidate-search/query-list/bib.js +10 -1
- package/src/index.js +239 -37
- package/src/index.spec.js +44 -17
- package/src/match-detection/index.js +23 -3
package/dist/index.spec.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.spec.js"],"names":["describe","callback","path","__dirname","recurse","fixura","reader","READERS","JSON","getFixture","options","enabled","record","MarcRecord","subfieldValues","expectedMatches","match","formatOptions","matches","
|
|
1
|
+
{"version":3,"sources":["../src/index.spec.js"],"names":["debug","debugData","extend","describe","callback","path","__dirname","recurse","fixura","reader","READERS","JSON","getFixture","options","enabled","expectedMatchStatus","expectedStopReason","record","MarcRecord","subfieldValues","expectedMatches","expectedNonMatches","match","formatOptions","matches","matchStatus","nonMatches","length","status","stopReason","formattedMatchResult","formatRecordResults","to","eql","formattedNonMatchResult","contextFeatures","matchDetection","features","detection","strategy","type","search","detect","map","v","stringify","candidate","formatCandidate","id","newId","newRecord","toObject"],"mappings":";;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;;;;;AAjCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASA,MAAMA,KAAK,GAAG,oBAAkB,8CAAlB,CAAd;AACA,MAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AAEAC,QAAQ,CAAC,OAAD,EAAU,MAAM;AACtB,kCAAc;AACZC,IAAAA,QADY;AAEZC,IAAAA,IAAI,EAAE,CAACC,SAAD,EAAY,IAAZ,EAAkB,eAAlB,EAAmC,OAAnC,CAFM;AAGZC,IAAAA,OAAO,EAAE,KAHG;AAIZC,IAAAA,MAAM,EAAE;AACNC,MAAAA,MAAM,EAAEC,gBAAQC;AADV;AAJI,GAAd;;AASA,iBAAeP,QAAf,CAAwB;AAACQ,IAAAA,UAAD;AAAaC,IAAAA,OAAb;AAAsBC,IAAAA,OAAO,GAAG,IAAhC;AAAsCC,IAAAA,mBAAtC;AAA2DC,IAAAA;AAA3D,GAAxB,EAAwG;AAEtG,QAAI,CAACF,OAAL,EAAc;AACZd,MAAAA,KAAK,CAAE,gBAAF,CAAL;AACA;AACD;;AAED,UAAMiB,MAAM,GAAG,IAAIC,sBAAJ,CAAeN,UAAU,CAAC,kBAAD,CAAzB,EAA+C;AAACO,MAAAA,cAAc,EAAE;AAAjB,KAA/C,CAAf;AACA,UAAMC,eAAe,GAAGR,UAAU,CAAC,sBAAD,CAAlC;AACA,UAAMS,kBAAkB,GAAGT,UAAU,CAAC,yBAAD,CAAV,IAAyC,EAApE;AAGA,UAAMU,KAAK,GAAG,eAAqBC,aAAa,EAAlC,CAAd;AACA,UAAM;AAACC,MAAAA,OAAD;AAAUC,MAAAA,WAAV;AAAuBC,MAAAA;AAAvB,QAAqC,MAAMJ,KAAK,CAACL,MAAD,CAAtD;AACAhB,IAAAA,SAAS,CAAE,GAAEuB,OAAO,CAACG,MAAO,KAAIF,WAAW,CAACG,MAAO,IAAGH,WAAW,CAACI,UAAW,KAAIH,UAAU,GAAGA,UAAU,CAACC,MAAd,GAAuBD,UAAW,EAApH,CAAT;AAEA,UAAMI,oBAAoB,GAAGC,mBAAmB,CAACP,OAAD,CAAhD;AACA,sBAAOM,oBAAP,EAA6BE,EAA7B,CAAgCC,GAAhC,CAAoCb,eAApC;AAEA,UAAMc,uBAAuB,GAAGH,mBAAmB,CAACL,UAAD,CAAnD;AACA,sBAAOQ,uBAAP,EAAgCF,EAAhC,CAAmCC,GAAnC,CAAuCZ,kBAAvC;AAEA,sBAAOI,WAAW,CAACG,MAAnB,EAA2BI,EAA3B,CAA8BC,GAA9B,CAAkClB,mBAAlC;AACA,sBAAOU,WAAW,CAACI,UAAnB,EAA+BG,EAA/B,CAAkCC,GAAlC,CAAsCjB,kBAAtC;;AAGA,aAASO,aAAT,GAAyB;AACvB,YAAMY,eAAe,GAAGC,iBAAeC,QAAf,CAAwBxB,OAAO,CAACyB,SAAR,CAAkBC,QAAlB,CAA2BC,IAAnD,CAAxB;AAEA,aAAO,EACL,GAAG3B,OADE;AAEL4B,QAAAA,MAAM,EAAE,EACN,GAAG5B,OAAO,CAAC4B;AADL,SAFH;AAKLH,QAAAA,SAAS,EAAE,EACT,GAAGzB,OAAO,CAAC6B,MADF;AAETH,UAAAA,QAAQ,EAAE1B,OAAO,CAACyB,SAAR,CAAkBC,QAAlB,CAA2BF,QAA3B,CAAoCM,GAApC,CAAwCC,CAAC,IAAIT,eAAe,CAACS,CAAD,CAAf,EAA7C;AAFD;AALN,OAAP;AAUD;;AAED,aAASb,mBAAT,CAA6BP,OAA7B,EAAsC;AACpC,UAAIA,OAAJ,EAAa;AACXvB,QAAAA,SAAS,CAACU,IAAI,CAACkC,SAAL,CAAerB,OAAf,CAAD,CAAT;AACA,eAAOA,OAAO,CAACmB,GAAR,CAAarB,KAAD,KAAY,EAC7B,GAAGA,KAD0B;AAE7BwB,UAAAA,SAAS,EAAEC,eAAe,CAACzB,KAAK,CAACwB,SAAP;AAFG,SAAZ,CAAZ,CAAP;AAID;;AACD,aAAO,EAAP;AACD,KAlDqG,CAoDtG;;;AACA,aAASC,eAAT,CAAyB;AAACC,MAAAA,EAAD;AAAK/B,MAAAA;AAAL,KAAzB,EAAuC;AACrC,YAAMgC,KAAK,GAAGD,EAAd;AACA,YAAME,SAAS,GAAGjC,MAAlB;AACA,aAAO;AACL+B,QAAAA,EAAE,EAAEC,KADC;AAELhC,QAAAA,MAAM,EAAEiC,SAAS,CAACC,QAAV;AAFH,OAAP;AAID;AAGF;AAEF,CA3EO,CAAR","sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {expect} from 'chai';\nimport {READERS} from '@natlibfi/fixura';\nimport generateTests from '@natlibfi/fixugen-http-client';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport createMatchInterface, {matchDetection} from '.';\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:index:test');\nconst debugData = debug.extend('data');\n\ndescribe('INDEX', () => {\n generateTests({\n callback,\n path: [__dirname, '..', 'test-fixtures', 'index'],\n recurse: false,\n fixura: {\n reader: READERS.JSON\n }\n });\n\n async function callback({getFixture, options, enabled = true, expectedMatchStatus, expectedStopReason}) {\n\n if (!enabled) {\n debug(`Disabled test!`);\n return;\n }\n\n const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});\n const expectedMatches = getFixture('expectedMatches.json');\n const expectedNonMatches = getFixture('expectedNonMatches.json') || [];\n\n\n const match = createMatchInterface(formatOptions());\n const {matches, matchStatus, nonMatches} = await match(record);\n debugData(`${matches.length}, ${matchStatus.status}/${matchStatus.stopReason}, ${nonMatches ? nonMatches.length : nonMatches}`);\n\n const formattedMatchResult = formatRecordResults(matches);\n expect(formattedMatchResult).to.eql(expectedMatches);\n\n const formattedNonMatchResult = formatRecordResults(nonMatches);\n expect(formattedNonMatchResult).to.eql(expectedNonMatches);\n\n expect(matchStatus.status).to.eql(expectedMatchStatus);\n expect(matchStatus.stopReason).to.eql(expectedStopReason);\n\n\n function formatOptions() {\n const contextFeatures = matchDetection.features[options.detection.strategy.type];\n\n return {\n ...options,\n search: {\n ...options.search\n },\n detection: {\n ...options.detect,\n strategy: options.detection.strategy.features.map(v => contextFeatures[v]())\n }\n };\n }\n\n function formatRecordResults(matches) {\n if (matches) {\n debugData(JSON.stringify(matches));\n return matches.map((match) => ({\n ...match,\n candidate: formatCandidate(match.candidate)\n }));\n }\n return [];\n }\n\n // Format candidate to remove validationOptions from record\n function formatCandidate({id, record}) {\n const newId = id;\n const newRecord = record;\n return {\n id: newId,\n record: newRecord.toObject()\n };\n }\n\n\n }\n\n});\n"],"file":"index.spec.js"}
|
|
@@ -50,10 +50,13 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
|
|
|
50
50
|
var _default = ({
|
|
51
51
|
strategy,
|
|
52
52
|
treshold = 0.9
|
|
53
|
-
}) => (recordA, recordB) => {
|
|
53
|
+
}, returnStrategy = false) => (recordA, recordB) => {
|
|
54
54
|
const minProbabilityQuantifier = 0.5;
|
|
55
55
|
const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:match-detection');
|
|
56
56
|
const debugData = debug.extend('data');
|
|
57
|
+
debugData(`Strategy: ${JSON.stringify(strategy)}`);
|
|
58
|
+
debugData(`Treshold: ${JSON.stringify(treshold)}`);
|
|
59
|
+
debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
|
|
57
60
|
const featuresA = extractFeatures(recordA);
|
|
58
61
|
const featuresB = extractFeatures(recordB);
|
|
59
62
|
debugData(`Features (a): ${JSON.stringify(featuresA)}`);
|
|
@@ -64,17 +67,38 @@ var _default = ({
|
|
|
64
67
|
if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
|
|
65
68
|
const probability = calculateprobability();
|
|
66
69
|
debug(`probability: ${probability} (Treshold: ${treshold})`);
|
|
67
|
-
return {
|
|
70
|
+
return returnResult({
|
|
68
71
|
match: probability >= treshold,
|
|
69
72
|
probability
|
|
70
|
-
};
|
|
73
|
+
});
|
|
71
74
|
}
|
|
72
75
|
|
|
73
76
|
debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
|
|
74
|
-
return {
|
|
77
|
+
return returnResult({
|
|
75
78
|
match: false,
|
|
76
79
|
probability: 0.0
|
|
77
|
-
};
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
function returnResult(result) {
|
|
83
|
+
if (returnStrategy) {
|
|
84
|
+
debug(`Returning detection strategy with the result`);
|
|
85
|
+
const resultWithStrategy = {
|
|
86
|
+
match: result.match,
|
|
87
|
+
probability: result.probability,
|
|
88
|
+
strategy: formatStrategy(strategy),
|
|
89
|
+
treshold
|
|
90
|
+
};
|
|
91
|
+
debugData(`${JSON.stringify(resultWithStrategy)}`);
|
|
92
|
+
return resultWithStrategy;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function formatStrategy(strategy) {
|
|
99
|
+
const strategyNames = strategy.map(element => element.name);
|
|
100
|
+
return strategyNames || [];
|
|
101
|
+
}
|
|
78
102
|
|
|
79
103
|
function calculateprobability() {
|
|
80
104
|
const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/match-detection/index.js"],"names":["strategy","treshold","recordA","recordB","minProbabilityQuantifier","debug","debugData","extend","
|
|
1
|
+
{"version":3,"sources":["../../src/match-detection/index.js"],"names":["strategy","treshold","returnStrategy","recordA","recordB","minProbabilityQuantifier","debug","debugData","extend","JSON","stringify","featuresA","extractFeatures","featuresB","featurePairs","generateFeaturePairs","similarityVector","generateSimilarityVector","some","v","probability","calculateprobability","returnResult","match","result","resultWithStrategy","formatStrategy","strategyNames","map","element","name","reduce","acc","record","extract","concat","value","compared","a","b","compare","find","featureName","points","pairs","generatePairs","missingFeatures","findMissing","index","filter","length"],"mappings":";;;;;;;AA6BA;;AACA;;;;;;;;;;AA9BA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAOA;eACe,CAAC;AAACA,EAAAA,QAAD;AAAWC,EAAAA,QAAQ,GAAG;AAAtB,CAAD,EAA6BC,cAAc,GAAG,KAA9C,KAAwD,CAACC,OAAD,EAAUC,OAAV,KAAsB;AAC3F,QAAMC,wBAAwB,GAAG,GAAjC;AAEA,QAAMC,KAAK,GAAG,oBAAkB,mDAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AAEAD,EAAAA,SAAS,CAAE,aAAYE,IAAI,CAACC,SAAL,CAAeV,QAAf,CAAyB,EAAvC,CAAT;AACAO,EAAAA,SAAS,CAAE,aAAYE,IAAI,CAACC,SAAL,CAAeT,QAAf,CAAyB,EAAvC,CAAT;AACAM,EAAAA,SAAS,CAAE,mBAAkBE,IAAI,CAACC,SAAL,CAAeR,cAAf,CAA+B,EAAnD,CAAT;AAEA,QAAMS,SAAS,GAAGC,eAAe,CAACT,OAAD,CAAjC;AACA,QAAMU,SAAS,GAAGD,eAAe,CAACR,OAAD,CAAjC;AAEAG,EAAAA,SAAS,CAAE,iBAAgBE,IAAI,CAACC,SAAL,CAAeC,SAAf,CAA0B,EAA5C,CAAT;AACAJ,EAAAA,SAAS,CAAE,iBAAgBE,IAAI,CAACC,SAAL,CAAeG,SAAf,CAA0B,EAA5C,CAAT;AAEA,QAAMC,YAAY,GAAGC,oBAAoB,EAAzC;AACA,QAAMC,gBAAgB,GAAGC,wBAAwB,EAAjD;;AAEA,MAAID,gBAAgB,CAACE,IAAjB,CAAsBC,CAAC,IAAIA,CAAC,IAAId,wBAAhC,CAAJ,EAA+D;AAC7D,UAAMe,WAAW,GAAGC,oBAAoB,EAAxC;AACAf,IAAAA,KAAK,CAAE,gBAAec,WAAY,eAAcnB,QAAS,GAApD,CAAL;AACA,WAAOqB,YAAY,CAAC;AAACC,MAAAA,KAAK,EAAEH,WAAW,IAAInB,QAAvB;AAAiCmB,MAAAA;AAAjC,KAAD,CAAnB;AACD;;AAEDb,EAAAA,SAAS,CAAE,4DAA2DF,wBAAyB,GAAtF,CAAT;AACA,SAAOiB,YAAY,CAAC;AAACC,IAAAA,KAAK,EAAE,KAAR;AAAeH,IAAAA,WAAW,EAAE;AAA5B,GAAD,CAAnB;;AAEA,WAASE,YAAT,CAAsBE,MAAtB,EAA8B;AAC5B,QAAItB,cAAJ,EAAoB;AAClBI,MAAAA,KAAK,CAAE,8CAAF,CAAL;AACA,YAAMmB,kBAAkB,GAAG;AAACF,QAAAA,KAAK,EAAEC,MAAM,CAACD,KAAf;AAAsBH,QAAAA,WAAW,EAAEI,MAAM,CAACJ,WAA1C;AAAuDpB,QAAAA,QAAQ,EAAE0B,cAAc,CAAC1B,QAAD,CAA/E;AAA2FC,QAAAA;AAA3F,OAA3B;AACAM,MAAAA,SAAS,CAAE,GAAEE,IAAI,CAACC,SAAL,CAAee,kBAAf,CAAmC,EAAvC,CAAT;AACA,aAAOA,kBAAP;AACD;;AACD,WAAOD,MAAP;AACD;;AAED,WAASE,cAAT,CAAwB1B,QAAxB,EAAkC;AAChC,UAAM2B,aAAa,GAAG3B,QAAQ,CAAC4B,GAAT,CAAaC,OAAO,IAAIA,OAAO,CAACC,IAAhC,CAAtB;AACA,WAAOH,aAAa,IAAI,EAAxB;AACD;;AAED,WAASN,oBAAT,GAAgC;AAC9B,UAAMD,WAAW,GAAGJ,gBAAgB,CAACe,MAAjB,CAAwB,CAACC,GAAD,EAAMb,CAAN,KAAYa,GAAG,GAAGb,CAA1C,EAA6C,GAA7C,CAApB;AACA,WAAOC,WAAW,GAAG,GAAd,GAAoB,GAApB,GAA0BA,WAAjC;AACD;;AAED,WAASR,eAAT,CAAyBqB,MAAzB,EAAiC;AAC/B,WAAOjC,QAAQ,CAAC+B,MAAT,CAAgB,CAACC,GAAD,EAAM;AAACF,MAAAA,IAAD;AAAOI,MAAAA;AAAP,KAAN,KAA0BF,GAAG,CAACG,MAAJ,CAAW;AAACL,MAAAA,IAAD;AAAOM,MAAAA,KAAK,EAAEF,OAAO,CAACD,MAAD;AAArB,KAAX,CAA1C,EAAsF,EAAtF,CAAP;AACD;;AAED,WAAShB,wBAAT,GAAoC;AAClC,UAAMoB,QAAQ,GAAGvB,YAAY,CAACc,GAAb,CAAiB,CAAC;AAACE,MAAAA,IAAD;AAAOQ,MAAAA,CAAP;AAAUC,MAAAA;AAAV,KAAD,KAAkB;AAClD,YAAM;AAACC,QAAAA;AAAD,UAAYxC,QAAQ,CAACyC,IAAT,CAAc,CAAC;AAACX,QAAAA,IAAI,EAAEY;AAAP,OAAD,KAAyBZ,IAAI,KAAKY,WAAhD,CAAlB;AACA,YAAMC,MAAM,GAAGH,OAAO,CAACF,CAAD,EAAIC,CAAJ,CAAtB;AACA,aAAO;AAACT,QAAAA,IAAD;AAAOa,QAAAA;AAAP,OAAP;AACD,KAJgB,CAAjB;AAMApC,IAAAA,SAAS,CAAE,WAAUE,IAAI,CAACC,SAAL,CAAe2B,QAAf,CAAyB,EAArC,CAAT;AACA,WAAOA,QAAQ,CAACT,GAAT,CAAa,CAAC;AAACe,MAAAA;AAAD,KAAD,KAAcA,MAA3B,CAAP;AACD;;AAED,WAAS5B,oBAAT,GAAgC;AAC9B,UAAM6B,KAAK,GAAGC,aAAa,EAA3B;AACA,UAAMC,eAAe,GAAGC,WAAW,EAAnC;AAEAzC,IAAAA,KAAK,CAAE,2FAA0FG,IAAI,CAACC,SAAL,CAAeoC,eAAf,CAAgC,EAA5H,CAAL;AACA,WAAOF,KAAP;;AAEA,aAASC,aAAT,GAAyB;AACvB,aAAOlC,SAAS,CACboB,MADI,CACG,CAACC,GAAD,EAAM;AAACF,QAAAA,IAAD;AAAOM,QAAAA;AAAP,OAAN,EAAqBY,KAArB,KAA+BhB,GAAG,CAACG,MAAJ,CAAW;AAChDL,QAAAA,IADgD;AAEhDQ,QAAAA,CAAC,EAAEF,KAF6C;AAGhDG,QAAAA,CAAC,EAAE1B,SAAS,CAACmC,KAAD,CAAT,CAAiBZ;AAH4B,OAAX,CADlC,EAKD,EALC,EAMJa,MANI,CAMG,CAAC;AAACX,QAAAA,CAAD;AAAIC,QAAAA;AAAJ,OAAD,KAAY;AAClB,YAAID,CAAC,CAACY,MAAF,KAAa,CAAb,IAAkBX,CAAC,CAACW,MAAF,KAAa,CAAnC,EAAsC;AACpC,iBAAO,KAAP;AACD;;AAED,eAAO,IAAP;AACD,OAZI,CAAP;AAaD;;AAED,aAASH,WAAT,GAAuB;AACrB,aAAOpC,SAAS,CACbiB,GADI,CACA,CAAC;AAACE,QAAAA;AAAD,OAAD,KAAYA,IADZ,EAEJmB,MAFI,CAEG9B,CAAC,IAAIyB,KAAK,CAAC1B,IAAN,CAAW,CAAC;AAACY,QAAAA;AAAD,OAAD,KAAYA,IAAI,KAAKX,CAAhC,MAAuC,KAF/C,CAAP;AAGD;AACF;AACF,C","sourcesContent":["/* eslint-disable no-console */\n/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport * as features from './features';\n\nexport {features};\n\n// eslint-disable-next-line max-statements\nexport default ({strategy, treshold = 0.9}, returnStrategy = false) => (recordA, recordB) => {\n const minProbabilityQuantifier = 0.5;\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n const debugData = debug.extend('data');\n\n debugData(`Strategy: ${JSON.stringify(strategy)}`);\n debugData(`Treshold: ${JSON.stringify(treshold)}`);\n debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n\n const featuresA = extractFeatures(recordA);\n const featuresB = extractFeatures(recordB);\n\n debugData(`Features (a): ${JSON.stringify(featuresA)}`);\n debugData(`Features (b): ${JSON.stringify(featuresB)}`);\n\n const featurePairs = generateFeaturePairs();\n const similarityVector = generateSimilarityVector();\n\n if (similarityVector.some(v => v >= minProbabilityQuantifier)) {\n const probability = calculateprobability();\n debug(`probability: ${probability} (Treshold: ${treshold})`);\n return returnResult({match: probability >= treshold, probability});\n }\n\n debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);\n return returnResult({match: false, probability: 0.0});\n\n function returnResult(result) {\n if (returnStrategy) {\n debug(`Returning detection strategy with the result`);\n const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold};\n debugData(`${JSON.stringify(resultWithStrategy)}`);\n return resultWithStrategy;\n }\n return result;\n }\n\n function formatStrategy(strategy) {\n const strategyNames = strategy.map(element => element.name);\n return strategyNames || [];\n }\n\n function calculateprobability() {\n const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n return probability > 1.0 ? 1.0 : probability;\n }\n\n function extractFeatures(record) {\n return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract(record)}), []);\n }\n\n function generateSimilarityVector() {\n const compared = featurePairs.map(({name, a, b}) => {\n const {compare} = strategy.find(({name: featureName}) => name === featureName);\n const points = compare(a, b);\n return {name, points};\n });\n\n debugData(`Points: ${JSON.stringify(compared)}`);\n return compared.map(({points}) => points);\n }\n\n function generateFeaturePairs() {\n const pairs = generatePairs();\n const missingFeatures = findMissing();\n\n debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n return pairs;\n\n function generatePairs() {\n return featuresA\n .reduce((acc, {name, value}, index) => acc.concat({\n name,\n a: value,\n b: featuresB[index].value\n }), [])\n .filter(({a, b}) => {\n if (a.length === 0 || b.length === 0) {\n return false;\n }\n\n return true;\n });\n }\n\n function findMissing() {\n return featuresA\n .map(({name}) => name)\n .filter(v => pairs.some(({name}) => name === v) === false);\n }\n }\n};\n"],"file":"index.js"}
|
package/package.json
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"url": "git@github.com:natlibfi/melinda-record-matching-js.git"
|
|
15
15
|
},
|
|
16
16
|
"license": "LGPL-3.0+",
|
|
17
|
-
"version": "2.
|
|
17
|
+
"version": "2.1.0-alpha.1",
|
|
18
18
|
"main": "./dist/index.js",
|
|
19
19
|
"engines": {
|
|
20
20
|
"node": ">=14"
|
|
@@ -35,32 +35,32 @@
|
|
|
35
35
|
"watch:test": "cross-env DEBUG=1 NODE_ENV=test nodemon -w src -w test-fixtures --exec 'npm run test:dev'"
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@natlibfi/marc-record": "^7.0.0
|
|
39
|
-
"@natlibfi/marc-record-serializers": "^8.0
|
|
40
|
-
"@natlibfi/sru-client": "^
|
|
41
|
-
"debug": "^4.3.
|
|
38
|
+
"@natlibfi/marc-record": "^7.0.0",
|
|
39
|
+
"@natlibfi/marc-record-serializers": "^8.1.0",
|
|
40
|
+
"@natlibfi/sru-client": "^5.0.0",
|
|
41
|
+
"debug": "^4.3.3",
|
|
42
42
|
"moment": "^2.29.1",
|
|
43
|
-
"natural": "^5.1.
|
|
43
|
+
"natural": "^5.1.13",
|
|
44
44
|
"uuid": "^8.3.2",
|
|
45
|
-
"winston": "^3.3.
|
|
45
|
+
"winston": "^3.3.4"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
|
-
"@babel/cli": "^7.16.
|
|
49
|
-
"@babel/core": "^7.16.
|
|
50
|
-
"@babel/
|
|
51
|
-
"@babel/
|
|
52
|
-
"@babel/
|
|
53
|
-
"@
|
|
48
|
+
"@babel/cli": "^7.16.7",
|
|
49
|
+
"@babel/core": "^7.16.7",
|
|
50
|
+
"@babel/eslint-parser": "^7.16.5",
|
|
51
|
+
"@babel/node": "^7.16.7",
|
|
52
|
+
"@babel/preset-env": "^7.16.7",
|
|
53
|
+
"@babel/register": "^7.16.7",
|
|
54
|
+
"@natlibfi/eslint-config-melinda-backend": "^2.0.0",
|
|
54
55
|
"@natlibfi/fixugen": "^1.0.2",
|
|
55
56
|
"@natlibfi/fixugen-http-client": "^1.1.3",
|
|
56
57
|
"@natlibfi/fixura": "^2.2.1",
|
|
57
|
-
"@babel/eslint-parser": "^7.16.3",
|
|
58
58
|
"babel-plugin-istanbul": "^6.1.1",
|
|
59
59
|
"babel-plugin-rewire": "^1.2.0",
|
|
60
60
|
"chai": "^4.3.4",
|
|
61
61
|
"chai-as-promised": "^7.1.1",
|
|
62
62
|
"cross-env": "^7.0.3",
|
|
63
|
-
"eslint": "^
|
|
63
|
+
"eslint": "^8.6.0",
|
|
64
64
|
"mocha": "^9.1.3",
|
|
65
65
|
"nodemon": "^2.0.15",
|
|
66
66
|
"nyc": "^15.1.0"
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Melinda record matching modules for Javascript
|
|
6
6
|
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
7
|
+
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
8
8
|
*
|
|
9
9
|
* This file is part of melinda-record-matching-js
|
|
10
10
|
*
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
*/
|
|
28
28
|
|
|
29
29
|
import createDebugLogger from 'debug';
|
|
30
|
-
import createClient from '@natlibfi/sru-client';
|
|
30
|
+
import createClient, {SruSearchError} from '@natlibfi/sru-client';
|
|
31
31
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
32
32
|
import {MARCXML} from '@natlibfi/marc-record-serializers';
|
|
33
33
|
import generateQueryList from './query-list';
|
|
@@ -36,10 +36,20 @@ export {searchTypes} from './query-list';
|
|
|
36
36
|
|
|
37
37
|
export class CandidateSearchError extends Error {}
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
// serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000
|
|
40
|
+
|
|
41
|
+
// eslint-disable-next-line max-statements
|
|
42
|
+
export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {
|
|
40
43
|
MarcRecord.setValidationOptions({subfieldValues: false});
|
|
41
44
|
|
|
42
45
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search');
|
|
46
|
+
const debugData = debug.extend('data');
|
|
47
|
+
|
|
48
|
+
debugData(`SearchSpec: ${JSON.stringify(searchSpec)}`);
|
|
49
|
+
debugData(`Url: ${url}`);
|
|
50
|
+
debugData(`MaxRecordsPerRequest ${maxRecordsPerRequest}`);
|
|
51
|
+
debugData(`ServerMaxResult: ${serverMaxResult}`);
|
|
52
|
+
|
|
43
53
|
const inputRecordId = getRecordId(record);
|
|
44
54
|
const queryList = generateQueryList(record, searchSpec);
|
|
45
55
|
const client = createClient({
|
|
@@ -48,39 +58,71 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
|
|
|
48
58
|
retrieveAll: false
|
|
49
59
|
});
|
|
50
60
|
|
|
61
|
+
debug(`Searching matches for ${inputRecordId}`);
|
|
51
62
|
debug(`Generated queryList ${JSON.stringify(queryList)}`);
|
|
52
63
|
if (queryList.length === 0) { // eslint-disable-line functional/no-conditional-statement
|
|
53
64
|
throw new CandidateSearchError(`Generated query list contains no queries`);
|
|
54
65
|
}
|
|
55
66
|
|
|
67
|
+
// state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
|
|
68
|
+
// state.query : current query (undefined if there was no queries left)
|
|
69
|
+
// state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
|
|
70
|
+
// state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)
|
|
71
|
+
// state.queriesLeft : amount of queries left
|
|
72
|
+
// state.queryCounter : sequence for current query
|
|
73
|
+
// state.maxedQueries : queries that resulted in more than serverMaxResults hits
|
|
74
|
+
|
|
75
|
+
|
|
56
76
|
// eslint-disable-next-line max-statements
|
|
57
|
-
return async ({queryOffset = 0, resultSetOffset = 1}) => {
|
|
77
|
+
return async ({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) => {
|
|
58
78
|
const query = queryList[queryOffset];
|
|
59
79
|
|
|
60
80
|
if (query) {
|
|
61
|
-
const {records, nextOffset} = await retrieveRecords();
|
|
81
|
+
const {records, nextOffset, total} = await retrieveRecords();
|
|
82
|
+
|
|
83
|
+
// If resultSetOffset === 1 this is the first search for the current query
|
|
84
|
+
const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;
|
|
85
|
+
const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;
|
|
86
|
+
const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;
|
|
87
|
+
const newQueryCandidateCounter = resultSetOffset === 1 ? records.length : queryCandidateCounter + records.length;
|
|
88
|
+
|
|
89
|
+
const maxedQuery = resultSetOffset === 1 ? checkMaxedQuery(query, total, serverMaxResult) : undefined;
|
|
90
|
+
const newMaxedQueries = maxedQuery ? maxedQueries.concat(maxedQuery) : maxedQueries;
|
|
62
91
|
|
|
63
92
|
if (typeof nextOffset === 'number') {
|
|
64
|
-
debug(`
|
|
65
|
-
return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1)};
|
|
93
|
+
debug(`Next search will be for query ${queryOffset} ${query}, starting from record ${nextOffset}`);
|
|
94
|
+
return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCandidateCounter, maxedQueries: newMaxedQueries};
|
|
66
95
|
}
|
|
67
|
-
debug(`Query ${queryOffset} ${query} done
|
|
68
|
-
|
|
96
|
+
debug(`Query ${queryOffset} ${query} done.`);
|
|
97
|
+
debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);
|
|
98
|
+
return {records, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
|
|
69
99
|
}
|
|
70
100
|
|
|
71
101
|
debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);
|
|
72
|
-
return {records: []};
|
|
102
|
+
return {records: [], queriesLeft: 0, queryCounter, maxedQueries};
|
|
73
103
|
|
|
74
104
|
function retrieveRecords() {
|
|
75
105
|
return new Promise((resolve, reject) => {
|
|
76
106
|
const promises = [];
|
|
107
|
+
// eslint-disable-next-line functional/no-let
|
|
108
|
+
let totalRecords = 0;
|
|
77
109
|
|
|
78
110
|
debug(`Searching for candidates with query: ${query} (Offset ${resultSetOffset})`);
|
|
79
111
|
|
|
80
112
|
client.searchRetrieve(query, {startRecord: resultSetOffset})
|
|
81
113
|
.on('error', err => {
|
|
114
|
+
// eslint-disable-next-line functional/no-conditional-statement
|
|
115
|
+
if (err instanceof SruSearchError) {
|
|
116
|
+
debug(`SRU SruSearchError for query: ${query}: ${err}`);
|
|
117
|
+
reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));
|
|
118
|
+
}
|
|
119
|
+
debug(`SRU error for query: ${query}: ${err}`);
|
|
82
120
|
reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));
|
|
83
121
|
})
|
|
122
|
+
.on('total', total => {
|
|
123
|
+
debug(`Got total: ${total}`);
|
|
124
|
+
totalRecords += total;
|
|
125
|
+
})
|
|
84
126
|
.on('end', async nextOffset => {
|
|
85
127
|
try {
|
|
86
128
|
const records = await Promise.all(promises);
|
|
@@ -88,7 +130,7 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
|
|
|
88
130
|
|
|
89
131
|
debug(`Found ${filtered.length} candidates`);
|
|
90
132
|
|
|
91
|
-
resolve({nextOffset, records: filtered});
|
|
133
|
+
resolve({nextOffset, records: filtered, total: totalRecords});
|
|
92
134
|
} catch (err) {
|
|
93
135
|
reject(err);
|
|
94
136
|
}
|
|
@@ -101,6 +143,9 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
|
|
|
101
143
|
const foundRecordMarc = await MARCXML.from(foundRecord, {subfieldValues: false});
|
|
102
144
|
const foundRecordId = getRecordId(foundRecordMarc);
|
|
103
145
|
|
|
146
|
+
// This does not work and might cause problems:
|
|
147
|
+
// Record *should* match itself AND in REST the input record is given id 000000001 always
|
|
148
|
+
debug(`Checking record id's - this does not work ${inputRecordId} vs ${foundRecordId}`);
|
|
104
149
|
if (inputRecordId === foundRecordId) {
|
|
105
150
|
debug(`Input and candidate are the same record per 001. Discarding candidate`);
|
|
106
151
|
return;
|
|
@@ -116,6 +161,15 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {
|
|
|
116
161
|
}
|
|
117
162
|
};
|
|
118
163
|
|
|
164
|
+
function checkMaxedQuery(query, total, serverMaxResult) {
|
|
165
|
+
// eslint-disable-next-line functional/no-conditional-statement
|
|
166
|
+
if (total >= serverMaxResult) {
|
|
167
|
+
debug(`WARNING: Query ${query} resulted in ${total} hits which meets the serverMaxResult (${serverMaxResult}) `);
|
|
168
|
+
return query;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
|
|
119
173
|
function getRecordId(record) {
|
|
120
174
|
const [field] = record.get(/^001$/u);
|
|
121
175
|
return field ? field.value : '';
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Melinda record matching modules for Javascript
|
|
6
6
|
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
7
|
+
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
8
8
|
*
|
|
9
9
|
* This file is part of melinda-record-matching-js
|
|
10
10
|
*
|
|
@@ -31,6 +31,9 @@ import {READERS} from '@natlibfi/fixura';
|
|
|
31
31
|
import generateTests from '@natlibfi/fixugen-http-client';
|
|
32
32
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
33
33
|
import createSearchInterface, {CandidateSearchError} from '.';
|
|
34
|
+
import createDebugLogger from 'debug';
|
|
35
|
+
|
|
36
|
+
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:test');
|
|
34
37
|
|
|
35
38
|
describe('candidate-search', () => {
|
|
36
39
|
generateTests({
|
|
@@ -64,9 +67,11 @@ describe('candidate-search', () => {
|
|
|
64
67
|
await iterate({searchOptions});
|
|
65
68
|
|
|
66
69
|
function formatFactoryOptions() {
|
|
70
|
+
debug(`Using factoryOptions: ${JSON.stringify(factoryOptions)}`);
|
|
67
71
|
return {
|
|
68
72
|
...factoryOptions,
|
|
69
|
-
maxRecordsPerRequest: 1,
|
|
73
|
+
maxRecordsPerRequest: factoryOptions.maxRecordsPerRequest || 1,
|
|
74
|
+
maxServerResults: factoryOptions.maxServerResults || undefined,
|
|
70
75
|
record: new MarcRecord(factoryOptions.record, {subfieldValues: false})
|
|
71
76
|
};
|
|
72
77
|
}
|
|
@@ -173,12 +173,20 @@ export function bibHostComponents(record) {
|
|
|
173
173
|
}
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
+
// SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to
|
|
177
|
+
// (probably) to *headings* index TIT
|
|
178
|
+
// Headings index TIT drops articles etc. from the start of the title according to the filing indicator
|
|
179
|
+
// Currently filing indicator is not implemented - if the title starts with an article and the Melinda
|
|
180
|
+
// record is correctly catalogued using a filing indicator -> dc.title search won't match
|
|
181
|
+
|
|
176
182
|
export function bibTitle(record) {
|
|
177
183
|
const title = getTitle();
|
|
178
184
|
|
|
179
185
|
if (title) {
|
|
180
186
|
const formatted = title
|
|
181
187
|
.replace(/[^\w\s\p{Alphabetic}]/gu, '')
|
|
188
|
+
// Clean up concurrent spaces from fe. subfield changes
|
|
189
|
+
.replace(/ +/gu, ' ')
|
|
182
190
|
.trim()
|
|
183
191
|
.slice(0, 30)
|
|
184
192
|
.trim();
|
|
@@ -196,7 +204,8 @@ export function bibTitle(record) {
|
|
|
196
204
|
return field.subfields
|
|
197
205
|
.filter(({code}) => ['a', 'b'].includes(code))
|
|
198
206
|
.map(({value}) => value)
|
|
199
|
-
|
|
207
|
+
// In Melinda's index subfield separators are indexed as ' '
|
|
208
|
+
.join(' ');
|
|
200
209
|
}
|
|
201
210
|
return false;
|
|
202
211
|
}
|