@natlibfi/melinda-record-matching 2.1.0-alpha.1 → 2.1.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/candidate-search/index.js +8 -12
- package/dist/candidate-search/index.js.map +1 -1
- package/dist/candidate-search/index.spec.js +9 -18
- package/dist/candidate-search/index.spec.js.map +1 -1
- package/dist/candidate-search/query-list/bib.js +6 -2
- package/dist/candidate-search/query-list/bib.js.map +1 -1
- package/dist/index.js +52 -21
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/candidate-search/index.js +9 -11
- package/src/candidate-search/index.spec.js +7 -17
- package/src/candidate-search/query-list/bib.js +6 -1
- package/src/index.js +30 -20
|
@@ -64,6 +64,7 @@ var _default = ({
|
|
|
64
64
|
record,
|
|
65
65
|
searchSpec,
|
|
66
66
|
url,
|
|
67
|
+
maxCandidates,
|
|
67
68
|
maxRecordsPerRequest = 50,
|
|
68
69
|
serverMaxResult = 20000
|
|
69
70
|
}) => {
|
|
@@ -77,11 +78,14 @@ var _default = ({
|
|
|
77
78
|
debugData(`Url: ${url}`);
|
|
78
79
|
debugData(`MaxRecordsPerRequest ${maxRecordsPerRequest}`);
|
|
79
80
|
debugData(`ServerMaxResult: ${serverMaxResult}`);
|
|
81
|
+
debugData(`MaxCandidates: ${maxCandidates}`); // Do not retrieve more candidates than defined in maxCandidates
|
|
82
|
+
|
|
83
|
+
const adjustedMaxRecordsPerRequest = maxRecordsPerRequest >= maxCandidates ? maxCandidates : maxRecordsPerRequest;
|
|
80
84
|
const inputRecordId = getRecordId(record);
|
|
81
85
|
const queryList = (0, _queryList.default)(record, searchSpec);
|
|
82
86
|
const client = (0, _sruClient.default)({
|
|
83
87
|
url,
|
|
84
|
-
maxRecordsPerRequest,
|
|
88
|
+
maxRecordsPerRequest: adjustedMaxRecordsPerRequest,
|
|
85
89
|
version: '2.0',
|
|
86
90
|
retrieveAll: false
|
|
87
91
|
});
|
|
@@ -119,6 +123,7 @@ var _default = ({
|
|
|
119
123
|
total
|
|
120
124
|
} = await retrieveRecords(); // If resultSetOffset === 1 this is the first search for the current query
|
|
121
125
|
|
|
126
|
+
debugData(`ResultSetOffset: ${resultSetOffset}`);
|
|
122
127
|
const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;
|
|
123
128
|
const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;
|
|
124
129
|
const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;
|
|
@@ -137,7 +142,7 @@ var _default = ({
|
|
|
137
142
|
query,
|
|
138
143
|
searchCounter: newSearchCounter,
|
|
139
144
|
queryCandidateCounter: newQueryCandidateCounter,
|
|
140
|
-
queryCounter:
|
|
145
|
+
queryCounter: newQueryCounter,
|
|
141
146
|
maxedQueries: newMaxedQueries
|
|
142
147
|
};
|
|
143
148
|
}
|
|
@@ -206,16 +211,7 @@ var _default = ({
|
|
|
206
211
|
const foundRecordMarc = await _marcRecordSerializers.MARCXML.from(foundRecord, {
|
|
207
212
|
subfieldValues: false
|
|
208
213
|
});
|
|
209
|
-
const foundRecordId = getRecordId(foundRecordMarc);
|
|
210
|
-
// Record *should* match itself AND in REST the input record is given id 000000001 always
|
|
211
|
-
|
|
212
|
-
debug(`Checking record id's - this does not work ${inputRecordId} vs ${foundRecordId}`);
|
|
213
|
-
|
|
214
|
-
if (inputRecordId === foundRecordId) {
|
|
215
|
-
debug(`Input and candidate are the same record per 001. Discarding candidate`);
|
|
216
|
-
return;
|
|
217
|
-
}
|
|
218
|
-
|
|
214
|
+
const foundRecordId = getRecordId(foundRecordMarc);
|
|
219
215
|
return {
|
|
220
216
|
record: foundRecordMarc,
|
|
221
217
|
id: foundRecordId
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/candidate-search/index.js"],"names":["CandidateSearchError","Error","record","searchSpec","url","maxRecordsPerRequest","serverMaxResult","MarcRecord","setValidationOptions","subfieldValues","debug","debugData","extend","JSON","stringify","inputRecordId","getRecordId","queryList","client","version","retrieveAll","length","queryOffset","resultSetOffset","totalRecords","searchCounter","queryCandidateCounter","queryCounter","maxedQueries","query","records","nextOffset","total","retrieveRecords","newTotalRecords","newQueryCounter","newSearchCounter","newQueryCandidateCounter","maxedQuery","checkMaxedQuery","undefined","newMaxedQueries","concat","queriesLeft","Promise","resolve","reject","promises","searchRetrieve","startRecord","on","err","SruSearchError","all","filtered","filter","r","foundRecord","push","handleRecord","foundRecordMarc","MARCXML","from","foundRecordId","id","field","get","value"],"mappings":";;;;;;;;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;;;;;;;AAhCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAUO,MAAMA,oBAAN,SAAmCC,KAAnC,CAAyC,E,CAEhD;AAEA;;;;;eACe,CAAC;AAACC,EAAAA,MAAD;AAASC,EAAAA,UAAT;AAAqBC,EAAAA,GAArB;AAA0BC,EAAAA,oBAAoB,GAAG,EAAjD;AAAqDC,EAAAA,eAAe,GAAG;AAAvE,CAAD,KAAmF;AAChGC,yBAAWC,oBAAX,CAAgC;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAhC;;AAEA,QAAMC,KAAK,GAAG,oBAAkB,oDAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AAEAD,EAAAA,SAAS,CAAE,eAAcE,IAAI,CAACC,SAAL,CAAeX,UAAf,CAA2B,EAA3C,CAAT;AACAQ,EAAAA,SAAS,CAAE,QAAOP,GAAI,EAAb,CAAT;AACAO,EAAAA,SAAS,CAAE,wBAAuBN,oBAAqB,EAA9C,CAAT;AACAM,EAAAA,SAAS,CAAE,oBAAmBL,eAAgB,EAArC,CAAT;AAEA,QAAMS,aAAa,GAAGC,WAAW,CAACd,MAAD,CAAjC;AACA,QAAMe,SAAS,GAAG,wBAAkBf,MAAlB,EAA0BC,UAA1B,CAAlB;AACA,QAAMe,MAAM,GAAG,wBAAa;AAC1Bd,IAAAA,GAD0B;AACrBC,IAAAA,oBADqB;AAE1Bc,IAAAA,OAAO,EAAE,KAFiB;AAG1BC,IAAAA,WAAW,EAAE;AAHa,GAAb,CAAf;AAMAV,EAAAA,KAAK,CAAE,yBAAwBK,aAAc,EAAxC,CAAL;AACAL,EAAAA,KAAK,CAAE,uBAAsBG,IAAI,CAACC,SAAL,CAAeG,SAAf,CAA0B,EAAlD,CAAL;;AACA,MAAIA,SAAS,CAACI,MAAV,KAAqB,CAAzB,EAA4B;AAAE;AAC5B,UAAM,IAAIrB,oBAAJ,CAA0B,0CAA1B,CAAN;AACD,GAvB+F,CAyBhG;AACA;AACA;AACA;AACA;AACA;AACA;AAGA;;;AACA,SAAO,OAAO;AAACsB,IAAAA,WAAW,GAAG,CAAf;AAAkBC,IAAAA,eAAe,GAAG,CAApC;AAAuCC,IAAAA,YAAY,GAAG,CAAtD;AAAyDC,IAAAA,aAAa,GAAG,CAAzE;AAA4EC,IAAAA,qBAAqB,GAAG,CAApG;AAAuGC,IAAAA,YAAY,GAAG,CAAtH;AAAyHC,IAAAA,YAAY,GAAG;AAAxI,GAAP,KAAuJ;AAC5J,UAAMC,KAAK,GAAGZ,SAAS,CAACK,WAAD,CAAvB;;AAEA,QAAIO,KAAJ,EAAW;AACT,YAAM;AAACC,QAAAA,OAAD;AAAUC,QAAAA,UAAV;AAAsBC,QAAAA;AAAtB,UAA+B,MAAMC,eAAe,EAA1D,CADS,CAGT;;AACA,YAAMC,eAAe,GAAGX,eAAe,KAAK,CAApB,GAAwBS,KAAxB,GAAgCR,YAAxD;AACA,YAAMW,eAAe,GAAGZ,eAAe,KAAK,CAApB,GAAwBI,YAAY,GAAG,CAAvC,GAA2CA,YAAnE;AACA,YAAMS,gBAAgB,GAAGb,eAAe,KAAK,CAApB,GAAwB,CAAxB,GAA4BE,aAAa,GAAG,CAArE;AACA,YAAMY,wBAAwB,GAAGd,eAAe,KAAK,CAApB,GAAwBO,OAAO,CAACT,MAAhC,GAAyCK,qBAAqB,GAAGI,OAAO,CAACT,MAA1G;AAEA,YAAMiB,UAAU,GAAGf,eAAe,KAAK,CAApB,GAAwBgB,eAAe,CAACV,KAAD,EAAQG,KAAR,EAAe1B,eAAf,CAAvC,GAAyEkC,SAA5F;AACA,YAAMC,eAAe,GAAGH,UAAU,GAAGV,YAAY,CAACc,MAAb,CAAoBJ,UAApB,CAAH,GAAqCV,YAAvE;;AAEA,UAAI,OAAOG,UAAP,KAAsB,QAA1B,EAAoC;AAClCrB,QAAAA,KAAK,CAAE,iCAAgCY,WAAY,IAAGO,KAAM,0BAAyBE,UAAW,EAA3F,CAAL;AACA,eAAO;AAACD,UAAAA,OAAD;AAAUR,UAAAA,WAAV;AAAuBC,UAAAA,eAAe,EAAEQ,UAAxC;AAAoDY,UAAAA,WAAW,EAAE1B,SAAS,CAACI,MAAV,IAAoBC,WAAW,GAAG,CAAlC,CAAjE;AAAuGE,UAAAA,YAAY,EAAEU,eAArH;AAAsIL,UAAAA,KAAtI;AAA6IJ,UAAAA,aAAa,EAAEW,gBAA5J;AAA8KV,UAAAA,qBAAqB,EAAEW,wBAArM;AAA+NV,UAAAA,YAAY,EAAEU,wBAA7O;AAAuQT,UAAAA,YAAY,EAAEa;AAArR,SAAP;AACD;;AACD/B,MAAAA,KAAK,CAAE,SAAQY,WAAY,IAAGO,KAAM,QAA/B,CAAL;AACAnB,MAAAA,KAAK,CAAE,cAAaO,SAAS,CAACI,MAAV,IAAoBC,WAAW,GAAG,CAAlC,CAAqC,gBAApD,CAAL;AACA,aAAO;AAACQ,QAAAA,OAAD;AAAUR,QAAAA,WAAW,EAAEA,WAAW,GAAG,CAArC;AAAwCqB,QAAAA,WAAW,EAAE1B,SAAS,CAACI,MAAV,IAAoBC,WAAW,GAAG,CAAlC,CAArD;AAA2FE,QAAAA,YAAY,EAAEU,eAAzG;AAA0HL,QAAAA,KAA1H;AAAiIJ,QAAAA,aAAa,EAAEW,gBAAhJ;AAAkKV,QAAAA,qBAAqB,EAAEW,wBAAzL;AAAmNV,QAAAA,YAAY,EAAEQ,eAAjO;AAAkPP,QAAAA,YAAY,EAAEa;AAAhQ,OAAP;AACD;;AAED/B,IAAAA,KAAK,CAAE,OAAMO,SAAS,CAACI,MAAO,uCAAsCC,WAAY,EAA3E,CAAL;AACA,WAAO;AAACQ,MAAAA,OAAO,EAAE,EAAV;AAAca,MAAAA,WAAW,EAAE,CAA3B;AAA8BhB,MAAAA,YAA9B;AAA4CC,MAAAA;AAA5C,KAAP;;AAEA,aAASK,eAAT,GAA2B;AACzB,aAAO,IAAIW,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,cAAMC,QAAQ,GAAG,EAAjB,CADsC,CAEtC;;AACA,YAAIvB,YAAY,GAAG,CAAnB;AAEAd,QAAAA,KAAK,CAAE,wCAAuCmB,KAAM,YAAWN,eAAgB,GAA1E,CAAL;AAEAL,QAAAA,MAAM,CAAC8B,cAAP,CAAsBnB,KAAtB,EAA6B;AAACoB,UAAAA,WAAW,EAAE1B;AAAd,SAA7B,EACG2B,EADH,CACM,OADN,EACeC,GAAG,IAAI;AAClB;AACA,cAAIA,GAAG,YAAYC,yBAAnB,EAAmC;AACjC1C,YAAAA,KAAK,CAAE,iCAAgCmB,KAAM,KAAIsB,GAAI,EAAhD,CAAL;AACAL,YAAAA,MAAM,CAAC,IAAI9C,oBAAJ,CAA0B,iCAAgC6B,KAAM,KAAIsB,GAAI,EAAxE,CAAD,CAAN;AACD;;AACDzC,UAAAA,KAAK,CAAE,wBAAuBmB,KAAM,KAAIsB,GAAI,EAAvC,CAAL;AACAL,UAAAA,MAAM,CAAC,IAAI9C,oBAAJ,CAA0B,wBAAuB6B,KAAM,KAAIsB,GAAI,EAA/D,CAAD,CAAN;AACD,SATH,EAUGD,EAVH,CAUM,OAVN,EAUelB,KAAK,IAAI;AACpBtB,UAAAA,KAAK,CAAE,cAAasB,KAAM,EAArB,CAAL;AACAR,UAAAA,YAAY,IAAIQ,KAAhB;AACD,SAbH,EAcGkB,EAdH,CAcM,KAdN,EAca,MAAMnB,UAAN,IAAoB;AAC7B,cAAI;AACF,kBAAMD,OAAO,GAAG,MAAMc,OAAO,CAACS,GAAR,CAAYN,QAAZ,CAAtB;AACA,kBAAMO,QAAQ,GAAGxB,OAAO,CAACyB,MAAR,CAAeC,CAAC,IAAIA,CAApB,CAAjB;AAEA9C,YAAAA,KAAK,CAAE,SAAQ4C,QAAQ,CAACjC,MAAO,aAA1B,CAAL;AAEAwB,YAAAA,OAAO,CAAC;AAACd,cAAAA,UAAD;AAAaD,cAAAA,OAAO,EAAEwB,QAAtB;AAAgCtB,cAAAA,KAAK,EAAER;AAAvC,aAAD,CAAP;AACD,WAPD,CAOE,OAAO2B,GAAP,EAAY;AACZL,YAAAA,MAAM,CAACK,GAAD,CAAN;AACD;AACF,SAzBH,EA0BGD,EA1BH,CA0BM,QA1BN,EA0BgBO,WAAW,IAAI;AAC3BV,UAAAA,QAAQ,CAACW,IAAT,CAAcC,YAAY,EAA1B,EAD2B,CACI;;AAE/B,yBAAeA,YAAf,GAA8B;AAC5B,gBAAI;AACF,oBAAMC,eAAe,GAAG,MAAMC,+BAAQC,IAAR,CAAaL,WAAb,EAA0B;AAAChD,gBAAAA,cAAc,EAAE;AAAjB,eAA1B,CAA9B;AACA,oBAAMsD,aAAa,GAAG/C,WAAW,CAAC4C,eAAD,CAAjC,CAFE,CAIF;AACA;;AACAlD,cAAAA,KAAK,CAAE,6CAA4CK,aAAc,OAAMgD,aAAc,EAAhF,CAAL;;AACA,kBAAIhD,aAAa,KAAKgD,aAAtB,EAAqC;AACnCrD,gBAAAA,KAAK,CAAE,uEAAF,CAAL;AACA;AACD;;AAED,qBAAO;AAACR,gBAAAA,MAAM,EAAE0D,eAAT;AAA0BI,gBAAAA,EAAE,EAAED;AAA9B,eAAP;AACD,aAbD,CAaE,OAAOZ,GAAP,EAAY;AACZ,oBAAM,IAAIlD,KAAJ,CAAW,6BAA4BkD,GAAI,aAAYM,WAAY,EAAnE,CAAN;AACD;AACF;AACF,SA/CH;AAgDD,OAvDM,CAAP;AAwDD;AACF,GArFD;;AAuFA,WAASlB,eAAT,CAAyBV,KAAzB,EAAgCG,KAAhC,EAAuC1B,eAAvC,EAAwD;AACtD;AACA,QAAI0B,KAAK,IAAI1B,eAAb,EAA8B;AAC5BI,MAAAA,KAAK,CAAE,kBAAiBmB,KAAM,gBAAeG,KAAM,0CAAyC1B,eAAgB,IAAvG,CAAL;AACA,aAAOuB,KAAP;AACD;AACF;;AAGD,WAASb,WAAT,CAAqBd,MAArB,EAA6B;AAC3B,UAAM,CAAC+D,KAAD,IAAU/D,MAAM,CAACgE,GAAP,CAAW,QAAX,CAAhB;AACA,WAAOD,KAAK,GAAGA,KAAK,CAACE,KAAT,GAAiB,EAA7B;AACD;AACF,C","sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport createClient, {SruSearchError} from '@natlibfi/sru-client';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {MARCXML} from '@natlibfi/marc-record-serializers';\nimport generateQueryList from './query-list';\n\nexport {searchTypes} from './query-list';\n\nexport class CandidateSearchError extends Error {}\n\n// serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000\n\n// eslint-disable-next-line max-statements\nexport default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {\n MarcRecord.setValidationOptions({subfieldValues: false});\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search');\n const debugData = debug.extend('data');\n\n debugData(`SearchSpec: ${JSON.stringify(searchSpec)}`);\n debugData(`Url: ${url}`);\n debugData(`MaxRecordsPerRequest ${maxRecordsPerRequest}`);\n debugData(`ServerMaxResult: ${serverMaxResult}`);\n\n const inputRecordId = getRecordId(record);\n const queryList = generateQueryList(record, searchSpec);\n const client = createClient({\n url, maxRecordsPerRequest,\n version: '2.0',\n retrieveAll: false\n });\n\n debug(`Searching matches for ${inputRecordId}`);\n debug(`Generated queryList ${JSON.stringify(queryList)}`);\n if (queryList.length === 0) { // eslint-disable-line functional/no-conditional-statement\n throw new CandidateSearchError(`Generated query list contains no queries`);\n }\n\n // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)\n // state.query : current query (undefined if there was no queries left)\n // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)\n // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)\n // state.queriesLeft : amount of queries left\n // state.queryCounter : sequence for current query\n // state.maxedQueries : queries that resulted in more than serverMaxResults hits\n\n\n // eslint-disable-next-line max-statements\n return async ({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) => {\n const query = queryList[queryOffset];\n\n if (query) {\n const {records, nextOffset, total} = await retrieveRecords();\n\n // If resultSetOffset === 1 this is the first search for the current query\n const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;\n const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;\n const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;\n const newQueryCandidateCounter = resultSetOffset === 1 ? records.length : queryCandidateCounter + records.length;\n\n const maxedQuery = resultSetOffset === 1 ? checkMaxedQuery(query, total, serverMaxResult) : undefined;\n const newMaxedQueries = maxedQuery ? maxedQueries.concat(maxedQuery) : maxedQueries;\n\n if (typeof nextOffset === 'number') {\n debug(`Next search will be for query ${queryOffset} ${query}, starting from record ${nextOffset}`);\n return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCandidateCounter, maxedQueries: newMaxedQueries};\n }\n debug(`Query ${queryOffset} ${query} done.`);\n debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);\n return {records, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};\n }\n\n debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);\n return {records: [], queriesLeft: 0, queryCounter, maxedQueries};\n\n function retrieveRecords() {\n return new Promise((resolve, reject) => {\n const promises = [];\n // eslint-disable-next-line functional/no-let\n let totalRecords = 0;\n\n debug(`Searching for candidates with query: ${query} (Offset ${resultSetOffset})`);\n\n client.searchRetrieve(query, {startRecord: resultSetOffset})\n .on('error', err => {\n // eslint-disable-next-line functional/no-conditional-statement\n if (err instanceof SruSearchError) {\n debug(`SRU SruSearchError for query: ${query}: ${err}`);\n reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));\n }\n debug(`SRU error for query: ${query}: ${err}`);\n reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));\n })\n .on('total', total => {\n debug(`Got total: ${total}`);\n totalRecords += total;\n })\n .on('end', async nextOffset => {\n try {\n const records = await Promise.all(promises);\n const filtered = records.filter(r => r);\n\n debug(`Found ${filtered.length} candidates`);\n\n resolve({nextOffset, records: filtered, total: totalRecords});\n } catch (err) {\n reject(err);\n }\n })\n .on('record', foundRecord => {\n promises.push(handleRecord()); // eslint-disable-line functional/immutable-data\n\n async function handleRecord() {\n try {\n const foundRecordMarc = await MARCXML.from(foundRecord, {subfieldValues: false});\n const foundRecordId = getRecordId(foundRecordMarc);\n\n // This does not work and might cause problems:\n // Record *should* match itself AND in REST the input record is given id 000000001 always\n debug(`Checking record id's - this does not work ${inputRecordId} vs ${foundRecordId}`);\n if (inputRecordId === foundRecordId) {\n debug(`Input and candidate are the same record per 001. Discarding candidate`);\n return;\n }\n\n return {record: foundRecordMarc, id: foundRecordId};\n } catch (err) {\n throw new Error(`Failed converting record: ${err}, record: ${foundRecord}`);\n }\n }\n });\n });\n }\n };\n\n function checkMaxedQuery(query, total, serverMaxResult) {\n // eslint-disable-next-line functional/no-conditional-statement\n if (total >= serverMaxResult) {\n debug(`WARNING: Query ${query} resulted in ${total} hits which meets the serverMaxResult (${serverMaxResult}) `);\n return query;\n }\n }\n\n\n function getRecordId(record) {\n const [field] = record.get(/^001$/u);\n return field ? field.value : '';\n }\n};\n"],"file":"index.js"}
|
|
1
|
+
{"version":3,"sources":["../../src/candidate-search/index.js"],"names":["CandidateSearchError","Error","record","searchSpec","url","maxCandidates","maxRecordsPerRequest","serverMaxResult","MarcRecord","setValidationOptions","subfieldValues","debug","debugData","extend","JSON","stringify","adjustedMaxRecordsPerRequest","inputRecordId","getRecordId","queryList","client","version","retrieveAll","length","queryOffset","resultSetOffset","totalRecords","searchCounter","queryCandidateCounter","queryCounter","maxedQueries","query","records","nextOffset","total","retrieveRecords","newTotalRecords","newQueryCounter","newSearchCounter","newQueryCandidateCounter","maxedQuery","checkMaxedQuery","undefined","newMaxedQueries","concat","queriesLeft","Promise","resolve","reject","promises","searchRetrieve","startRecord","on","err","SruSearchError","all","filtered","filter","r","foundRecord","push","handleRecord","foundRecordMarc","MARCXML","from","foundRecordId","id","field","get","value"],"mappings":";;;;;;;;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;;;;;;;AAhCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAUO,MAAMA,oBAAN,SAAmCC,KAAnC,CAAyC,E,CAEhD;AAEA;;;;;eACe,CAAC;AAACC,EAAAA,MAAD;AAASC,EAAAA,UAAT;AAAqBC,EAAAA,GAArB;AAA0BC,EAAAA,aAA1B;AAAyCC,EAAAA,oBAAoB,GAAG,EAAhE;AAAoEC,EAAAA,eAAe,GAAG;AAAtF,CAAD,KAAkG;AAC/GC,yBAAWC,oBAAX,CAAgC;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAhC;;AAEA,QAAMC,KAAK,GAAG,oBAAkB,oDAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AAEAD,EAAAA,SAAS,CAAE,eAAcE,IAAI,CAACC,SAAL,CAAeZ,UAAf,CAA2B,EAA3C,CAAT;AACAS,EAAAA,SAAS,CAAE,QAAOR,GAAI,EAAb,CAAT;AACAQ,EAAAA,SAAS,CAAE,wBAAuBN,oBAAqB,EAA9C,CAAT;AACAM,EAAAA,SAAS,CAAE,oBAAmBL,eAAgB,EAArC,CAAT;AACAK,EAAAA,SAAS,CAAE,kBAAiBP,aAAc,EAAjC,CAAT,CAV+G,CAY/G;;AACA,QAAMW,4BAA4B,GAAGV,oBAAoB,IAAID,aAAxB,GAAwCA,aAAxC,GAAwDC,oBAA7F;AAEA,QAAMW,aAAa,GAAGC,WAAW,CAAChB,MAAD,CAAjC;AACA,QAAMiB,SAAS,GAAG,wBAAkBjB,MAAlB,EAA0BC,UAA1B,CAAlB;AACA,QAAMiB,MAAM,GAAG,wBAAa;AAC1BhB,IAAAA,GAD0B;AAE1BE,IAAAA,oBAAoB,EAAEU,4BAFI;AAG1BK,IAAAA,OAAO,EAAE,KAHiB;AAI1BC,IAAAA,WAAW,EAAE;AAJa,GAAb,CAAf;AAOAX,EAAAA,KAAK,CAAE,yBAAwBM,aAAc,EAAxC,CAAL;AACAN,EAAAA,KAAK,CAAE,uBAAsBG,IAAI,CAACC,SAAL,CAAeI,SAAf,CAA0B,EAAlD,CAAL;;AACA,MAAIA,SAAS,CAACI,MAAV,KAAqB,CAAzB,EAA4B;AAAE;AAC5B,UAAM,IAAIvB,oBAAJ,CAA0B,0CAA1B,CAAN;AACD,GA5B8G,CA8B/G;AACA;AACA;AACA;AACA;AACA;AACA;AAGA;;;AACA,SAAO,OAAO;AAACwB,IAAAA,WAAW,GAAG,CAAf;AAAkBC,IAAAA,eAAe,GAAG,CAApC;AAAuCC,IAAAA,YAAY,GAAG,CAAtD;AAAyDC,IAAAA,aAAa,GAAG,CAAzE;AAA4EC,IAAAA,qBAAqB,GAAG,CAApG;AAAuGC,IAAAA,YAAY,GAAG,CAAtH;AAAyHC,IAAAA,YAAY,GAAG;AAAxI,GAAP,KAAuJ;AAC5J,UAAMC,KAAK,GAAGZ,SAAS,CAACK,WAAD,CAAvB;;AAEA,QAAIO,KAAJ,EAAW;AACT,YAAM;AAACC,QAAAA,OAAD;AAAUC,QAAAA,UAAV;AAAsBC,QAAAA;AAAtB,UAA+B,MAAMC,eAAe,EAA1D,CADS,CAGT;;AACAvB,MAAAA,SAAS,CAAE,oBAAmBa,eAAgB,EAArC,CAAT;AACA,YAAMW,eAAe,GAAGX,eAAe,KAAK,CAApB,GAAwBS,KAAxB,GAAgCR,YAAxD;AACA,YAAMW,eAAe,GAAGZ,eAAe,KAAK,CAApB,GAAwBI,YAAY,GAAG,CAAvC,GAA2CA,YAAnE;AACA,YAAMS,gBAAgB,GAAGb,eAAe,KAAK,CAApB,GAAwB,CAAxB,GAA4BE,aAAa,GAAG,CAArE;AACA,YAAMY,wBAAwB,GAAGd,eAAe,KAAK,CAApB,GAAwBO,OAAO,CAACT,MAAhC,GAAyCK,qBAAqB,GAAGI,OAAO,CAACT,MAA1G;AAEA,YAAMiB,UAAU,GAAGf,eAAe,KAAK,CAApB,GAAwBgB,eAAe,CAACV,KAAD,EAAQG,KAAR,EAAe3B,eAAf,CAAvC,GAAyEmC,SAA5F;AACA,YAAMC,eAAe,GAAGH,UAAU,GAAGV,YAAY,CAACc,MAAb,CAAoBJ,UAApB,CAAH,GAAqCV,YAAvE;;AAEA,UAAI,OAAOG,UAAP,KAAsB,QAA1B,EAAoC;AAClCtB,QAAAA,KAAK,CAAE,iCAAgCa,WAAY,IAAGO,KAAM,0BAAyBE,UAAW,EAA3F,CAAL;AACA,eAAO;AAACD,UAAAA,OAAD;AAAUR,UAAAA,WAAV;AAAuBC,UAAAA,eAAe,EAAEQ,UAAxC;AAAoDY,UAAAA,WAAW,EAAE1B,SAAS,CAACI,MAAV,IAAoBC,WAAW,GAAG,CAAlC,CAAjE;AAAuGE,UAAAA,YAAY,EAAEU,eAArH;AAAsIL,UAAAA,KAAtI;AAA6IJ,UAAAA,aAAa,EAAEW,gBAA5J;AAA8KV,UAAAA,qBAAqB,EAAEW,wBAArM;AAA+NV,UAAAA,YAAY,EAAEQ,eAA7O;AAA8PP,UAAAA,YAAY,EAAEa;AAA5Q,SAAP;AACD;;AACDhC,MAAAA,KAAK,CAAE,SAAQa,WAAY,IAAGO,KAAM,QAA/B,CAAL;AACApB,MAAAA,KAAK,CAAE,cAAaQ,SAAS,CAACI,MAAV,IAAoBC,WAAW,GAAG,CAAlC,CAAqC,gBAApD,CAAL;AACA,aAAO;AAACQ,QAAAA,OAAD;AAAUR,QAAAA,WAAW,EAAEA,WAAW,GAAG,CAArC;AAAwCqB,QAAAA,WAAW,EAAE1B,SAAS,CAACI,MAAV,IAAoBC,WAAW,GAAG,CAAlC,CAArD;AAA2FE,QAAAA,YAAY,EAAEU,eAAzG;AAA0HL,QAAAA,KAA1H;AAAiIJ,QAAAA,aAAa,EAAEW,gBAAhJ;AAAkKV,QAAAA,qBAAqB,EAAEW,wBAAzL;AAAmNV,QAAAA,YAAY,EAAEQ,eAAjO;AAAkPP,QAAAA,YAAY,EAAEa;AAAhQ,OAAP;AACD;;AAEDhC,IAAAA,KAAK,CAAE,OAAMQ,SAAS,CAACI,MAAO,uCAAsCC,WAAY,EAA3E,CAAL;AACA,WAAO;AAACQ,MAAAA,OAAO,EAAE,EAAV;AAAca,MAAAA,WAAW,EAAE,CAA3B;AAA8BhB,MAAAA,YAA9B;AAA4CC,MAAAA;AAA5C,KAAP;;AAEA,aAASK,eAAT,GAA2B;AACzB,aAAO,IAAIW,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,cAAMC,QAAQ,GAAG,EAAjB,CADsC,CAEtC;;AACA,YAAIvB,YAAY,GAAG,CAAnB;AAEAf,QAAAA,KAAK,CAAE,wCAAuCoB,KAAM,YAAWN,eAAgB,GAA1E,CAAL;AAEAL,QAAAA,MAAM,CAAC8B,cAAP,CAAsBnB,KAAtB,EAA6B;AAACoB,UAAAA,WAAW,EAAE1B;AAAd,SAA7B,EACG2B,EADH,CACM,OADN,EACeC,GAAG,IAAI;AAClB;AACA,cAAIA,GAAG,YAAYC,yBAAnB,EAAmC;AACjC3C,YAAAA,KAAK,CAAE,iCAAgCoB,KAAM,KAAIsB,GAAI,EAAhD,CAAL;AACAL,YAAAA,MAAM,CAAC,IAAIhD,oBAAJ,CAA0B,iCAAgC+B,KAAM,KAAIsB,GAAI,EAAxE,CAAD,CAAN;AACD;;AACD1C,UAAAA,KAAK,CAAE,wBAAuBoB,KAAM,KAAIsB,GAAI,EAAvC,CAAL;AACAL,UAAAA,MAAM,CAAC,IAAIhD,oBAAJ,CAA0B,wBAAuB+B,KAAM,KAAIsB,GAAI,EAA/D,CAAD,CAAN;AACD,SATH,EAUGD,EAVH,CAUM,OAVN,EAUelB,KAAK,IAAI;AACpBvB,UAAAA,KAAK,CAAE,cAAauB,KAAM,EAArB,CAAL;AACAR,UAAAA,YAAY,IAAIQ,KAAhB;AACD,SAbH,EAcGkB,EAdH,CAcM,KAdN,EAca,MAAMnB,UAAN,IAAoB;AAC7B,cAAI;AACF,kBAAMD,OAAO,GAAG,MAAMc,OAAO,CAACS,GAAR,CAAYN,QAAZ,CAAtB;AACA,kBAAMO,QAAQ,GAAGxB,OAAO,CAACyB,MAAR,CAAeC,CAAC,IAAIA,CAApB,CAAjB;AAEA/C,YAAAA,KAAK,CAAE,SAAQ6C,QAAQ,CAACjC,MAAO,aAA1B,CAAL;AAEAwB,YAAAA,OAAO,CAAC;AAACd,cAAAA,UAAD;AAAaD,cAAAA,OAAO,EAAEwB,QAAtB;AAAgCtB,cAAAA,KAAK,EAAER;AAAvC,aAAD,CAAP;AACD,WAPD,CAOE,OAAO2B,GAAP,EAAY;AACZL,YAAAA,MAAM,CAACK,GAAD,CAAN;AACD;AACF,SAzBH,EA0BGD,EA1BH,CA0BM,QA1BN,EA0BgBO,WAAW,IAAI;AAC3BV,UAAAA,QAAQ,CAACW,IAAT,CAAcC,YAAY,EAA1B,EAD2B,CACI;;AAE/B,yBAAeA,YAAf,GAA8B;AAC5B,gBAAI;AACF,oBAAMC,eAAe,GAAG,MAAMC,+BAAQC,IAAR,CAAaL,WAAb,EAA0B;AAACjD,gBAAAA,cAAc,EAAE;AAAjB,eAA1B,CAA9B;AACA,oBAAMuD,aAAa,GAAG/C,WAAW,CAAC4C,eAAD,CAAjC;AAEA,qBAAO;AAAC5D,gBAAAA,MAAM,EAAE4D,eAAT;AAA0BI,gBAAAA,EAAE,EAAED;AAA9B,eAAP;AACD,aALD,CAKE,OAAOZ,GAAP,EAAY;AACZ,oBAAM,IAAIpD,KAAJ,CAAW,6BAA4BoD,GAAI,aAAYM,WAAY,EAAnE,CAAN;AACD;AACF;AACF,SAvCH;AAwCD,OA/CM,CAAP;AAgDD;AACF,GA9ED;;AAgFA,WAASlB,eAAT,CAAyBV,KAAzB,EAAgCG,KAAhC,EAAuC3B,eAAvC,EAAwD;AACtD;AACA,QAAI2B,KAAK,IAAI3B,eAAb,EAA8B;AAC5BI,MAAAA,KAAK,CAAE,kBAAiBoB,KAAM,gBAAeG,KAAM,0CAAyC3B,eAAgB,IAAvG,CAAL;AACA,aAAOwB,KAAP;AACD;AACF;;AAGD,WAASb,WAAT,CAAqBhB,MAArB,EAA6B;AAC3B,UAAM,CAACiE,KAAD,IAAUjE,MAAM,CAACkE,GAAP,CAAW,QAAX,CAAhB;AACA,WAAOD,KAAK,GAAGA,KAAK,CAACE,KAAT,GAAiB,EAA7B;AACD;AACF,C","sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport createClient, {SruSearchError} from '@natlibfi/sru-client';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {MARCXML} from '@natlibfi/marc-record-serializers';\nimport generateQueryList from './query-list';\n\nexport {searchTypes} from './query-list';\n\nexport class CandidateSearchError extends Error {}\n\n// serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000\n\n// eslint-disable-next-line max-statements\nexport default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {\n MarcRecord.setValidationOptions({subfieldValues: false});\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search');\n const debugData = debug.extend('data');\n\n debugData(`SearchSpec: ${JSON.stringify(searchSpec)}`);\n debugData(`Url: ${url}`);\n debugData(`MaxRecordsPerRequest ${maxRecordsPerRequest}`);\n debugData(`ServerMaxResult: ${serverMaxResult}`);\n debugData(`MaxCandidates: ${maxCandidates}`);\n\n // Do not retrieve more candidates than defined in maxCandidates\n const adjustedMaxRecordsPerRequest = maxRecordsPerRequest >= maxCandidates ? maxCandidates : maxRecordsPerRequest;\n\n const inputRecordId = getRecordId(record);\n const queryList = generateQueryList(record, searchSpec);\n const client = createClient({\n url,\n maxRecordsPerRequest: adjustedMaxRecordsPerRequest,\n version: '2.0',\n retrieveAll: false\n });\n\n debug(`Searching matches for ${inputRecordId}`);\n debug(`Generated queryList ${JSON.stringify(queryList)}`);\n if (queryList.length === 0) { // eslint-disable-line functional/no-conditional-statement\n throw new CandidateSearchError(`Generated query list contains no queries`);\n }\n\n // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)\n // state.query : current query (undefined if there was no queries left)\n // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)\n // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)\n // state.queriesLeft : amount of queries left\n // state.queryCounter : sequence for current query\n // state.maxedQueries : queries that resulted in more than serverMaxResults hits\n\n\n // eslint-disable-next-line max-statements\n return async ({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) => {\n const query = queryList[queryOffset];\n\n if (query) {\n const {records, nextOffset, total} = await retrieveRecords();\n\n // If resultSetOffset === 1 this is the first search for the current query\n debugData(`ResultSetOffset: ${resultSetOffset}`);\n const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;\n const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;\n const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;\n const newQueryCandidateCounter = resultSetOffset === 1 ? records.length : queryCandidateCounter + records.length;\n\n const maxedQuery = resultSetOffset === 1 ? checkMaxedQuery(query, total, serverMaxResult) : undefined;\n const newMaxedQueries = maxedQuery ? maxedQueries.concat(maxedQuery) : maxedQueries;\n\n if (typeof nextOffset === 'number') {\n debug(`Next search will be for query ${queryOffset} ${query}, starting from record ${nextOffset}`);\n return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};\n }\n debug(`Query ${queryOffset} ${query} done.`);\n debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);\n return {records, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};\n }\n\n debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);\n return {records: [], queriesLeft: 0, queryCounter, maxedQueries};\n\n function retrieveRecords() {\n return new Promise((resolve, reject) => {\n const promises = [];\n // eslint-disable-next-line functional/no-let\n let totalRecords = 0;\n\n debug(`Searching for candidates with query: ${query} (Offset ${resultSetOffset})`);\n\n client.searchRetrieve(query, {startRecord: resultSetOffset})\n .on('error', err => {\n // eslint-disable-next-line functional/no-conditional-statement\n if (err instanceof SruSearchError) {\n debug(`SRU SruSearchError for query: ${query}: ${err}`);\n reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));\n }\n debug(`SRU error for query: ${query}: ${err}`);\n reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));\n })\n .on('total', total => {\n debug(`Got total: ${total}`);\n totalRecords += total;\n })\n .on('end', async nextOffset => {\n try {\n const records = await Promise.all(promises);\n const filtered = records.filter(r => r);\n\n debug(`Found ${filtered.length} candidates`);\n\n resolve({nextOffset, records: filtered, total: totalRecords});\n } catch (err) {\n reject(err);\n }\n })\n .on('record', foundRecord => {\n promises.push(handleRecord()); // eslint-disable-line functional/immutable-data\n\n async function handleRecord() {\n try {\n const foundRecordMarc = await MARCXML.from(foundRecord, {subfieldValues: false});\n const foundRecordId = getRecordId(foundRecordMarc);\n\n return {record: foundRecordMarc, id: foundRecordId};\n } catch (err) {\n throw new Error(`Failed converting record: ${err}, record: ${foundRecord}`);\n }\n }\n });\n });\n }\n };\n\n function checkMaxedQuery(query, total, serverMaxResult) {\n // eslint-disable-next-line functional/no-conditional-statement\n if (total >= serverMaxResult) {\n debug(`WARNING: Query ${query} resulted in ${total} hits which meets the serverMaxResult (${serverMaxResult}) `);\n return query;\n }\n }\n\n\n function getRecordId(record) {\n const [field] = record.get(/^001$/u);\n return field ? field.value : '';\n }\n};\n"],"file":"index.js"}
|
|
@@ -60,8 +60,8 @@ describe('candidate-search', () => {
|
|
|
60
60
|
getFixture,
|
|
61
61
|
factoryOptions,
|
|
62
62
|
searchOptions,
|
|
63
|
-
expectedFactoryError,
|
|
64
|
-
expectedSearchError,
|
|
63
|
+
expectedFactoryError = false,
|
|
64
|
+
expectedSearchError = false,
|
|
65
65
|
enabled = true
|
|
66
66
|
}) {
|
|
67
67
|
const url = 'http://foo.bar';
|
|
@@ -88,7 +88,8 @@ describe('candidate-search', () => {
|
|
|
88
88
|
url
|
|
89
89
|
});
|
|
90
90
|
await iterate({
|
|
91
|
-
searchOptions
|
|
91
|
+
searchOptions,
|
|
92
|
+
expectedSearchError
|
|
92
93
|
});
|
|
93
94
|
|
|
94
95
|
function formatFactoryOptions() {
|
|
@@ -105,6 +106,7 @@ describe('candidate-search', () => {
|
|
|
105
106
|
|
|
106
107
|
async function iterate({
|
|
107
108
|
searchOptions,
|
|
109
|
+
expectedSearchError,
|
|
108
110
|
count = 1
|
|
109
111
|
}) {
|
|
110
112
|
const expectedResults = getFixture(`expectedResults${count}.json`);
|
|
@@ -119,16 +121,12 @@ describe('candidate-search', () => {
|
|
|
119
121
|
(0, _chai.expect)(err.message).to.match(new RegExp(expectedSearchError, 'u'));
|
|
120
122
|
return;
|
|
121
123
|
}
|
|
122
|
-
}
|
|
124
|
+
} // eslint-disable-next-line functional/no-conditional-statement
|
|
123
125
|
|
|
124
|
-
const results = await search(searchOptions);
|
|
125
|
-
(0, _chai.expect)(formatResults(results)).to.eql(expectedResults);
|
|
126
126
|
|
|
127
|
-
if (
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
count: count + 1
|
|
131
|
-
});
|
|
127
|
+
if (!expectedSearchError) {
|
|
128
|
+
const results = await search(searchOptions);
|
|
129
|
+
(0, _chai.expect)(formatResults(results)).to.eql(expectedResults);
|
|
132
130
|
}
|
|
133
131
|
|
|
134
132
|
function formatResults(results) {
|
|
@@ -143,13 +141,6 @@ describe('candidate-search', () => {
|
|
|
143
141
|
}))
|
|
144
142
|
};
|
|
145
143
|
}
|
|
146
|
-
|
|
147
|
-
function resultsToOptions(results) {
|
|
148
|
-
return Object.entries(results).filter(([k]) => k === 'records' === false) // If key is 'records' return false
|
|
149
|
-
.reduce((acc, [k, v]) => ({ ...acc,
|
|
150
|
-
[k]: v
|
|
151
|
-
}), {});
|
|
152
|
-
}
|
|
153
144
|
}
|
|
154
145
|
}
|
|
155
146
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/candidate-search/index.spec.js"],"names":["debug","describe","callback","path","__dirname","recurse","fixura","reader","READERS","JSON","getFixture","factoryOptions","searchOptions","expectedFactoryError","expectedSearchError","enabled","url","isCandidateSearchError","formatFactoryOptions","to","throw","CandidateSearchError","RegExp","search","iterate","stringify","maxRecordsPerRequest","maxServerResults","undefined","record","MarcRecord","subfieldValues","count","expectedResults","Error","err","be","an","message","match","results","formatResults","eql","records","
|
|
1
|
+
{"version":3,"sources":["../../src/candidate-search/index.spec.js"],"names":["debug","describe","callback","path","__dirname","recurse","fixura","reader","READERS","JSON","getFixture","factoryOptions","searchOptions","expectedFactoryError","expectedSearchError","enabled","url","isCandidateSearchError","formatFactoryOptions","to","throw","CandidateSearchError","RegExp","search","iterate","stringify","maxRecordsPerRequest","maxServerResults","undefined","record","MarcRecord","subfieldValues","count","expectedResults","Error","err","be","an","message","match","results","formatResults","eql","records","map","id","toObject"],"mappings":";;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;;;;;AAjCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASA,MAAMA,KAAK,GAAG,oBAAkB,yDAAlB,CAAd;AAEAC,QAAQ,CAAC,kBAAD,EAAqB,MAAM;AACjC,kCAAc;AACZC,IAAAA,QADY;AAEZC,IAAAA,IAAI,EAAE,CAACC,SAAD,EAAY,IAAZ,EAAkB,IAAlB,EAAwB,eAAxB,EAAyC,kBAAzC,EAA6D,OAA7D,CAFM;AAGZC,IAAAA,OAAO,EAAE,KAHG;AAIZC,IAAAA,MAAM,EAAE;AACNC,MAAAA,MAAM,EAAEC,gBAAQC;AADV;AAJI,GAAd,EADiC,CAUjC;;AACA,iBAAeP,QAAf,CAAwB;AAACQ,IAAAA,UAAD;AAAaC,IAAAA,cAAb;AAA6BC,IAAAA,aAA7B;AAA4CC,IAAAA,oBAAoB,GAAG,KAAnE;AAA0EC,IAAAA,mBAAmB,GAAG,KAAhG;AAAuGC,IAAAA,OAAO,GAAG;AAAjH,GAAxB,EAAgJ;AAC9I,UAAMC,GAAG,GAAG,gBAAZ;;AAEA,QAAI,CAACD,OAAL,EAAc;AACZ;AACD;;AAED,QAAIF,oBAAJ,EAA0B;AACxB,UAAIA,oBAAoB,CAACI,sBAAzB,EAAiD;AAC/C,0BAAO,MAAM,eAAsB,EAAC,GAAGC,oBAAoB,EAAxB;AAA4BF,UAAAA;AAA5B,SAAtB,CAAb,EAAsEG,EAAtE,CAAyEC,KAAzE,CAA+EC,sBAA/E,EAAqG,IAAIC,MAAJ,CAAWT,oBAAX,EAAiC,GAAjC,CAArG;AACA;AACD;;AAED,wBAAO,MAAM,eAAsB,EAAC,GAAGK,oBAAoB,EAAxB;AAA4BF,QAAAA;AAA5B,OAAtB,CAAb,EAAsEG,EAAtE,CAAyEC,KAAzE,CAA+E,IAAIE,MAAJ,CAAWT,oBAAX,EAAiC,GAAjC,CAA/E;AACA;AACD;;AAED,UAAMU,MAAM,GAAG,eAAsB,EAAC,GAAGL,oBAAoB,EAAxB;AAA4BF,MAAAA;AAA5B,KAAtB,CAAf;AACA,UAAMQ,OAAO,CAAC;AAACZ,MAAAA,aAAD;AAAgBE,MAAAA;AAAhB,KAAD,CAAb;;AAEA,aAASI,oBAAT,GAAgC;AAC9BlB,MAAAA,KAAK,CAAE,yBAAwBS,IAAI,CAACgB,SAAL,CAAed,cAAf,CAA+B,EAAzD,CAAL;AACA,aAAO,EACL,GAAGA,cADE;AAELe,QAAAA,oBAAoB,EAAEf,cAAc,CAACe,oBAAf,IAAuC,CAFxD;AAGLC,QAAAA,gBAAgB,EAAEhB,cAAc,CAACgB,gBAAf,IAAmCC,SAHhD;AAILC,QAAAA,MAAM,EAAE,IAAIC,sBAAJ,CAAenB,cAAc,CAACkB,MAA9B,EAAsC;AAACE,UAAAA,cAAc,EAAE;AAAjB,SAAtC;AAJH,OAAP;AAMD,KA5B6I,CA8B9I;;;AACA,mBAAeP,OAAf,CAAuB;AAACZ,MAAAA,aAAD;AAAgBE,MAAAA,mBAAhB;AAAqCkB,MAAAA,KAAK,GAAG;AAA7C,KAAvB,EAAwE;AACtE,YAAMC,eAAe,GAAGvB,UAAU,CAAE,kBAAiBsB,KAAM,OAAzB,CAAlC;;AAEA,UAAIlB,mBAAJ,EAAyB;AAAE;AACzB,YAAI;AACF,gBAAMS,MAAM,CAACX,aAAD,CAAZ;AACA,gBAAM,IAAIsB,KAAJ,CAAU,mBAAV,CAAN;AACD,SAHD,CAGE,OAAOC,GAAP,EAAY;AACZ,4BAAOA,GAAP,EAAYhB,EAAZ,CAAeiB,EAAf,CAAkBC,EAAlB,CAAqB,OAArB;AACA,4BAAOF,GAAG,CAACG,OAAX,EAAoBnB,EAApB,CAAuBoB,KAAvB,CAA6B,IAAIjB,MAAJ,CAAWR,mBAAX,EAAgC,GAAhC,CAA7B;AACA;AACD;AACF,OAZqE,CActE;;;AACA,UAAI,CAACA,mBAAL,EAA0B;AACxB,cAAM0B,OAAO,GAAG,MAAMjB,MAAM,CAACX,aAAD,CAA5B;AACA,0BAAO6B,aAAa,CAACD,OAAD,CAApB,EAA+BrB,EAA/B,CAAkCuB,GAAlC,CAAsCT,eAAtC;AACD;;AAED,eAASQ,aAAT,CAAuBD,OAAvB,EAAgC;AAC9B;AACA,eAAO,EACL,GAAGA,OADE;AAELG,UAAAA,OAAO,EAAEH,OAAO,CAACG,OAAR,CAAgBC,GAAhB,CAAoB,CAAC;AAACf,YAAAA,MAAD;AAASgB,YAAAA;AAAT,WAAD,MAAmB;AAACA,YAAAA,EAAD;AAAKhB,YAAAA,MAAM,EAAEA,MAAM,CAACiB,QAAP;AAAb,WAAnB,CAApB;AAFJ,SAAP;AAID;AAEF;AACF;AACF,CAxEO,CAAR","sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {expect} from 'chai';\nimport {READERS} from '@natlibfi/fixura';\nimport generateTests from '@natlibfi/fixugen-http-client';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport createSearchInterface, {CandidateSearchError} from '.';\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:test');\n\ndescribe('candidate-search', () => {\n generateTests({\n callback,\n path: [__dirname, '..', '..', 'test-fixtures', 'candidate-search', 'index'],\n recurse: false,\n fixura: {\n reader: READERS.JSON\n }\n });\n\n // eslint-disable-next-line max-statements\n async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError = false, expectedSearchError = false, enabled = true}) {\n const url = 'http://foo.bar';\n\n if (!enabled) {\n return;\n }\n\n if (expectedFactoryError) {\n if (expectedFactoryError.isCandidateSearchError) {\n expect(() => createSearchInterface({...formatFactoryOptions(), url})).to.throw(CandidateSearchError, new RegExp(expectedFactoryError, 'u'));\n return;\n }\n\n expect(() => createSearchInterface({...formatFactoryOptions(), url})).to.throw(new RegExp(expectedFactoryError, 'u'));\n return;\n }\n\n const search = createSearchInterface({...formatFactoryOptions(), url});\n await iterate({searchOptions, expectedSearchError});\n\n function formatFactoryOptions() {\n debug(`Using factoryOptions: ${JSON.stringify(factoryOptions)}`);\n return {\n ...factoryOptions,\n maxRecordsPerRequest: factoryOptions.maxRecordsPerRequest || 1,\n maxServerResults: factoryOptions.maxServerResults || undefined,\n record: new MarcRecord(factoryOptions.record, {subfieldValues: false})\n };\n }\n\n // eslint-disable-next-line max-statements\n async function iterate({searchOptions, expectedSearchError, count = 1}) {\n const expectedResults = getFixture(`expectedResults${count}.json`);\n\n if (expectedSearchError) { // eslint-disable-line functional/no-conditional-statement\n try {\n await search(searchOptions);\n throw new Error('Expected an error');\n } catch (err) {\n expect(err).to.be.an('error');\n expect(err.message).to.match(new RegExp(expectedSearchError, 'u'));\n return;\n }\n }\n\n // eslint-disable-next-line functional/no-conditional-statement\n if (!expectedSearchError) {\n const results = await search(searchOptions);\n expect(formatResults(results)).to.eql(expectedResults);\n }\n\n function formatResults(results) {\n // console.log(results); //eslint-disable-line\n return {\n ...results,\n records: results.records.map(({record, id}) => ({id, record: record.toObject()}))\n };\n }\n\n }\n }\n});\n"],"file":"index.spec.js"}
|
|
@@ -169,6 +169,7 @@ function bibHostComponents(record) {
|
|
|
169
169
|
}
|
|
170
170
|
} // SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to
|
|
171
171
|
// (probably) to *headings* index TIT
|
|
172
|
+
// - Aleph cannot currently handle headings searches starting with a boolean - in these cases use word search
|
|
172
173
|
// Headings index TIT drops articles etc. from the start of the title according to the filing indicator
|
|
173
174
|
// Currently filing indicator is not implemented - if the title starts with an article and the Melinda
|
|
174
175
|
// record is correctly catalogued using a filing indicator -> dc.title search won't match
|
|
@@ -176,12 +177,15 @@ function bibHostComponents(record) {
|
|
|
176
177
|
|
|
177
178
|
function bibTitle(record) {
|
|
178
179
|
const title = getTitle();
|
|
180
|
+
const booleanStartWords = ['and', 'or', 'nor', 'not'];
|
|
179
181
|
|
|
180
182
|
if (title) {
|
|
181
183
|
const formatted = title.replace(/[^\w\s\p{Alphabetic}]/gu, '') // Clean up concurrent spaces from fe. subfield changes
|
|
182
|
-
.replace(/ +/gu, ' ').trim().slice(0, 30).trim(); //
|
|
184
|
+
.replace(/ +/gu, ' ').trim().slice(0, 30).trim(); // use word search for titles starting with a boolean
|
|
183
185
|
|
|
184
|
-
|
|
186
|
+
const useWordSearch = booleanStartWords.some(word => formatted.toLowerCase().startsWith(word)); // Prevent too many matches by having a minimum length requirement
|
|
187
|
+
|
|
188
|
+
return formatted.length >= 5 ? [`dc.title="${useWordSearch ? '' : '^'}${formatted}*"`] : [];
|
|
185
189
|
}
|
|
186
190
|
|
|
187
191
|
return [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/candidate-search/query-list/bib.js"],"names":["bibSourceIds","record","debug","debugData","extend","fSids","get","length","JSON","stringify","toSidQueries","sidStrings","getSidStrings","sidQueries","map","toSidString","filter","nonEmptySid","field","createSidString","sfC","sfB","cleanedSfC","removeSourcePrefix","normalizeSidSubfieldValue","cleanedSfB","concat","subfieldValue","sourcePrefixRegex","normalizedValue","replace","normalizeAwayRegex","bibMelindaIds","melindaIds","bibHostComponents","id","getHostId","value","subfields","find","code","test","bibTitle","title","getTitle","formatted","trim","slice","includes","join","bibStandardIdentifiers","fields","identifiers","toIdentifiers","uniqueIdentifiers","Set","tag","issnIsbnReqExp","otherIdReqExp","sub","undefined"],"mappings":";;;;;;;;;;;AA4BA;;AACA;;AACA;;;;AA7BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAMO,SAASA,YAAT,CAAsBC,MAAtB,EAA8B;AAEnC;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAQE,QAAMC,KAAK,GAAG,oBAAkB,qEAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB,CA3BmC,CA4BnC;;AAEAF,EAAAA,KAAK,CAAE,iCAAF,CAAL;AAEA,QAAMG,KAAK,GAAGJ,MAAM,CAACK,GAAP,CAAW,KAAX,CAAd;AACAH,EAAAA,SAAS,CAAE,eAAcE,KAAK,CAACE,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAeJ,KAAf,CAAsB,EAAxD,CAAT;AAEA,SAAOA,KAAK,CAACE,MAAN,GAAe,CAAf,GAAmBG,YAAY,CAACL,KAAD,CAA/B,GAAyC,EAAhD;;AAEA,WAASK,YAAT,CAAsBL,KAAtB,EAA6B;AAC3BH,IAAAA,KAAK,CAAE,wCAAF,CAAL;AAEA,UAAMS,UAAU,GAAGC,aAAa,CAACP,KAAD,CAAhC;;AAEA,QAAIM,UAAU,CAACJ,MAAX,GAAoB,CAAxB,EAA2B;AACzBL,MAAAA,KAAK,CAAE,2CAAF,CAAL;AACA,aAAO,EAAP;AACD;;AAED,UAAMW,UAAU,GAAG,qCAAUF,UAAV,EAAsB,kBAAtB,CAAnB;AAEA,WAAOE,UAAP;;AAEA,aAASD,aAAT,CAAuBP,KAAvB,EAA8B;AAC5BH,MAAAA,KAAK,CAAE,qCAAF,CAAL,CAD4B,CAG5B;;AACA,YAAMS,UAAU,GAAGN,KAAK,CAACS,GAAN,CAAUC,WAAV,EAAuBC,MAAvB,CAA8BC,WAAW,IAAIA,WAA7C,CAAnB;AACA,aAAON,UAAP;;AAEA,eAASI,WAAT,CAAqBG,KAArB,EAA4B;AAC1BhB,QAAAA,KAAK,CAAE,6BAAF,CAAL;AAEA,eAAO,mDAA+BgB,KAA/B,IAAwCC,eAAe,CAACD,KAAD,CAAvD,GAAiE,EAAxE;;AAEA,iBAASC,eAAT,CAAyBD,KAAzB,EAAgC;AAC9BhB,UAAAA,KAAK,CAAE,8BAAF,CAAL;AACA,gBAAM,CAACkB,GAAD,IAAQ,sCAAkBF,KAAlB,EAAyB,GAAzB,CAAd;AACA,gBAAM,CAACG,GAAD,IAAQ,sCAAkBH,KAAlB,EAAyB,GAAzB,CAAd;AAEA,gBAAMI,UAAU,GAAGC,kBAAkB,CAACC,yBAAyB,CAACJ,GAAD,CAA1B,CAArC;AACA,gBAAMK,UAAU,GAAGD,yBAAyB,CAACH,GAAD,CAA5C;AAEAlB,UAAAA,SAAS,CAAE,GAAEK,IAAI,CAACC,SAAL,CAAeW,GAAf,CAAoB,MAAKZ,IAAI,CAACC,SAAL,CAAeY,GAAf,CAAoB,EAAjD,CAAT;AACA,iBAAOC,UAAU,CAACI,MAAX,CAAkBD,UAAlB,CAAP;AACD;;AAED,iBAASF,kBAAT,CAA4BI,aAA5B,EAA2C;AACzC,gBAAMC,iBAAiB,GAAI,gDAA3B;AACA,gBAAMC,eAAe,GAAGF,aAAa,CAACG,OAAd,CAAsBF,iBAAtB,EAAyC,OAAzC,CAAxB;AACAzB,UAAAA,SAAS,CAAE,cAAawB,aAAc,OAAME,eAAgB,EAAnD,CAAT;AACA,iBAAOA,eAAP;AACD;;AAED,iBAASL,yBAAT,CAAmCG,aAAnC,EAAkD;AAChDxB,UAAAA,SAAS,CAAE,eAAcwB,aAAc,EAA9B,CAAT;AACA,gBAAMI,kBAAkB,GAAI,OAA5B;AACA,iBAAOJ,aAAa,CAACG,OAAd,CAAsBC,kBAAtB,EAA0C,EAA1C,CAAP;AACD;AAEF;AACF;AACF;AACF;;AAEM,SAASC,aAAT,CAAuB/B,MAAvB,EAA+B;AACpC;AAEA,QAAMC,KAAK,GAAG,oBAAkB,wEAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AACAF,EAAAA,KAAK,CAAE,iCAAF,CAAL,CALoC,CAOpC;AACA;AACA;;AACA,QAAM+B,UAAU,GAAG,sCAAkBhC,MAAlB,CAAnB;AAEAE,EAAAA,SAAS,CAAE,uBAAsB8B,UAAU,CAAC1B,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAewB,UAAf,CAA2B,EAA1E,CAAT;;AAEA,MAAIA,UAAU,CAAC1B,MAAX,GAAoB,CAAxB,EAA2B;AACzBL,IAAAA,KAAK,CAAE,2CAAF,CAAL;AACA,WAAO,EAAP;AACD;;AAED,SAAO,qCAAU+B,UAAV,EAAsB,mBAAtB,CAAP;AACD,C,CAGD;AACA;;;AACO,SAASC,iBAAT,CAA2BjC,MAA3B,EAAmC;AACxC,QAAMkC,EAAE,GAAGC,SAAS,EAApB;AACA,SAAOD,EAAE,GAAG,CAAE,uBAAsBA,EAAG,EAA3B,CAAH,GAAmC,EAA5C;;AAEA,WAASC,SAAT,GAAqB;AACnB,UAAM,CAAClB,KAAD,IAAUjB,MAAM,CAACK,GAAP,CAAW,QAAX,CAAhB;;AAEA,QAAIY,KAAJ,EAAW;AACT,YAAM;AAACmB,QAAAA;AAAD,UAAUnB,KAAK,CAACoB,SAAN,CAAgBC,IAAhB,CAAqB,CAAC;AAACC,QAAAA;AAAD,OAAD,KAAYA,IAAI,KAAK,GAA1C,KAAkD,EAAlE;;AAEA,UAAIH,KAAK,IAAK,kBAAD,CAAqBI,IAArB,CAA0BJ,KAA1B,CAAb,EAA+C;AAC7C,eAAOA,KAAK,CAACP,OAAN,CAAc,kBAAd,EAAkC,EAAlC,CAAP;AACD;;AAED,UAAIO,KAAK,IAAK,aAAD,CAAgBI,IAAhB,CAAqBJ,KAArB,CAAb,EAA0C;AACxC,eAAOA,KAAK,CAACP,OAAN,CAAc,aAAd,EAA6B,EAA7B,CAAP;AACD;;AAED,aAAO,KAAP;AACD;;AACD,WAAO,KAAP;AACD;AACF,C,CAED;AACA;AACA;AACA;AACA;;;AAEO,SAASY,QAAT,CAAkBzC,MAAlB,EAA0B;AAC/B,QAAM0C,KAAK,GAAGC,QAAQ,EAAtB;;AAEA,MAAID,KAAJ,EAAW;AACT,UAAME,SAAS,GAAGF,KAAK,CACpBb,OADe,CACP,yBADO,EACoB,EADpB,EAEhB;AAFgB,KAGfA,OAHe,CAGP,MAHO,EAGC,GAHD,EAIfgB,IAJe,GAKfC,KALe,CAKT,CALS,EAKN,EALM,EAMfD,IANe,EAAlB,CADS,CAST;;AACA,WAAOD,SAAS,CAACtC,MAAV,IAAoB,CAApB,GAAwB,CAAE,cAAasC,SAAU,IAAzB,CAAxB,GAAwD,EAA/D;AACD;;AAED,SAAO,EAAP;;AAEA,WAASD,QAAT,GAAoB;AAClB,UAAM,CAAC1B,KAAD,IAAUjB,MAAM,CAACK,GAAP,CAAW,QAAX,CAAhB;;AAEA,QAAIY,KAAJ,EAAW;AACT,aAAOA,KAAK,CAACoB,SAAN,CACJtB,MADI,CACG,CAAC;AAACwB,QAAAA;AAAD,OAAD,KAAY,CAAC,GAAD,EAAM,GAAN,EAAWQ,QAAX,CAAoBR,IAApB,CADf,EAEJ1B,GAFI,CAEA,CAAC;AAACuB,QAAAA;AAAD,OAAD,KAAaA,KAFb,EAGL;AAHK,OAIJY,IAJI,CAIC,GAJD,CAAP;AAKD;;AACD,WAAO,KAAP;AACD;AACF,C,CAED;;;AACO,SAASC,sBAAT,CAAgCjD,MAAhC,EAAwC;AAE7C,QAAMC,KAAK,GAAG,oBAAkB,iFAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AACAF,EAAAA,KAAK,CAAE,2CAAF,CAAL;AAEA,QAAMiD,MAAM,GAAGlD,MAAM,CAACK,GAAP,CAAW,wBAAX,CAAf;AACA,QAAM8C,WAAW,GAAG,GAAG1B,MAAH,CAAU,GAAGyB,MAAM,CAACrC,GAAP,CAAWuC,aAAX,CAAb,CAApB;AACA,QAAMC,iBAAiB,GAAG,CAAC,GAAG,IAAIC,GAAJ,CAAQH,WAAR,CAAJ,CAA1B;AAEAjD,EAAAA,SAAS,CAAE,+BAA8BK,IAAI,CAACC,SAAL,CAAe0C,MAAf,CAAuB,EAAvD,CAAT;AACAhD,EAAAA,SAAS,CAAE,gBAAeiD,WAAW,CAAC7C,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAe2C,WAAf,CAA4B,EAArE,CAAT;AACAjD,EAAAA,SAAS,CAAE,uBAAsBmD,iBAAiB,CAAC/C,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAe6C,iBAAf,CAAkC,EAAxF,CAAT;;AAEA,MAAIA,iBAAiB,CAAC/C,MAAlB,GAA2B,CAA/B,EAAkC;AAChCL,IAAAA,KAAK,CAAE,2CAAF,CAAL;AACA,WAAO,EAAP;AACD;;AAED,SAAO,qCAAUoD,iBAAV,EAA6B,eAA7B,CAAP;;AAEA,WAASD,aAAT,CAAuB;AAACG,IAAAA,GAAD;AAAMlB,IAAAA;AAAN,GAAvB,EAAyC;AACvC,UAAMmB,cAAc,GAAI,kBAAxB;AACA,UAAMC,aAAa,GAAI,mBAAvB;;AAEA,QAAIF,GAAG,KAAK,KAAZ,EAAmB;AACjB,aAAOlB,SAAS,CACbtB,MADI,CACG2C,GAAG,IAAI,CAAC,GAAD,EAAM,GAAN,EAAW,GAAX,EAAgBX,QAAhB,CAAyBW,GAAG,CAACnB,IAA7B,KAAsCiB,cAAc,CAAChB,IAAf,CAAoBkB,GAAG,CAACtB,KAAxB,CAAtC,IAAwEsB,GAAG,CAACtB,KAAJ,KAAcuB,SADhG,EAEJ9C,GAFI,CAEA,CAAC;AAACuB,QAAAA;AAAD,OAAD,KAAaA,KAFb,CAAP;AAGD;;AAED,QAAImB,GAAG,KAAK,KAAZ,EAAmB;AACjB,aAAOlB,SAAS,CACbtB,MADI,CACG2C,GAAG,IAAI,CAAC,GAAD,EAAM,GAAN,EAAWX,QAAX,CAAoBW,GAAG,CAACnB,IAAxB,KAAiCiB,cAAc,CAAChB,IAAf,CAAoBkB,GAAG,CAACtB,KAAxB,CAAjC,IAAmEsB,GAAG,CAACtB,KAAJ,KAAcuB,SAD3F,EAEJ9C,GAFI,CAEA,CAAC;AAACuB,QAAAA;AAAD,OAAD,KAAaA,KAFb,CAAP;AAGD;;AAED,WAAOC,SAAS,CACbtB,MADI,CACG2C,GAAG,IAAI,CAAC,GAAD,EAAM,GAAN,EAAWX,QAAX,CAAoBW,GAAG,CAACnB,IAAxB,KAAiCkB,aAAa,CAACjB,IAAd,CAAmBkB,GAAG,CAACtB,KAAvB,CAAjC,IAAkEsB,GAAG,CAACtB,KAAJ,KAAcuB,SAD1F,EAEJ9C,GAFI,CAEA,CAAC;AAACuB,MAAAA;AAAD,KAAD,KAAaA,KAFb,CAAP;AAGD;AACF","sourcesContent":["\n/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\nimport createDebugLogger from 'debug';\nimport {toQueries} from '../candidate-search-utils';\nimport {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues} from '../../matching-utils';\n\n\nexport function bibSourceIds(record) {\n\n /* Melinda's SRU-index melinda.sourceid includes source IDs from SID fields in Melinda records\n SID-fields in Melinda have sf $c with local id and sf $b with a code for the local db:\n\n SID__ $c 123457 $b helka\n SID__ $c (ANDL100020)1077305 $b sata\n SID__ $c VER999999 $ FI-KV\n SID__ $c /10024/508126 $ REPO_THESEUS\n\n In melinda.sourceid -index case is kept, sourceprefixes in brackets and hyphens are normalized away:\n Note: slashes are not normalized away, but a SRU-search-string including slashes needs to be quoted\n\n 1234567helka\n 1077305sata\n VER999999FIKV\n /10024/508126REPO_THESEUS\n\n Note: All Melinda records that have a matching records in a local db do NOT have SID for that local records,\n existence of a SID field depends on how the record has been added to Melinda and how it has been handled\n afterwards. SIDs are also not reliably maintained. A record might or might not have a SID for a local db\n after the matching record is removed from the local db.\n\n */\n\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:source-ids');\n const debugData = debug.extend('data');\n //const debugInfo = debug.extend('info');\n\n debug(`Creating queries for sourceid's`);\n\n const fSids = record.get('SID');\n debugData(`SID-fields (${fSids.length}): ${JSON.stringify(fSids)}`);\n\n return fSids.length > 0 ? toSidQueries(fSids) : [];\n\n function toSidQueries(fSids) {\n debug(`Creating actual queries for sourceid's`);\n\n const sidStrings = getSidStrings(fSids);\n\n if (sidStrings.length < 1) {\n debug(`No identifiers found, no queries created.`);\n return [];\n }\n\n const sidQueries = toQueries(sidStrings, 'melinda.sourceid');\n\n return sidQueries;\n\n function getSidStrings(fSids) {\n debug(`Getting Sid strings from SID fields`);\n\n // Map SID fields to valid sidStrings, filter out empty strings\n const sidStrings = fSids.map(toSidString).filter(nonEmptySid => nonEmptySid);\n return sidStrings;\n\n function toSidString(field) {\n debug(`Getting string from a field`);\n\n return validateSidFieldSubfieldCounts(field) ? createSidString(field) : '';\n\n function createSidString(field) {\n debug(`Creating string from a field`);\n const [sfC] = getSubfieldValues(field, 'c');\n const [sfB] = getSubfieldValues(field, 'b');\n\n const cleanedSfC = removeSourcePrefix(normalizeSidSubfieldValue(sfC));\n const cleanedSfB = normalizeSidSubfieldValue(sfB);\n\n debugData(`${JSON.stringify(sfC)} + ${JSON.stringify(sfB)}`);\n return cleanedSfC.concat(cleanedSfB);\n }\n\n function removeSourcePrefix(subfieldValue) {\n const sourcePrefixRegex = (/^(?<sourcePrefix>\\([A-Za-z0-9-]+\\))(?<id>.+)$/u);\n const normalizedValue = subfieldValue.replace(sourcePrefixRegex, '$<id>');\n debugData(`Normalized ${subfieldValue} to ${normalizedValue}`);\n return normalizedValue;\n }\n\n function normalizeSidSubfieldValue(subfieldValue) {\n debugData(`Normalizing ${subfieldValue}`);\n const normalizeAwayRegex = (/[- ]/u);\n return subfieldValue.replace(normalizeAwayRegex, '');\n }\n\n }\n }\n }\n}\n\nexport function bibMelindaIds(record) {\n // Melinda's SRU-index melinda.melindaid includes f001 controlnumbers and old Melinda-IDs from f035z's for all non-deleted Melinda-records\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibMelindaIds');\n const debugData = debug.extend('data');\n debug(`Creating queries for MelindaIds`);\n\n // Note: Melinda-ID's for search queries are created just from records f035a's and f035z's\n // Both (FI-MELINDA)- and FCC-prefixed forms are found\n // f001 controlnumber is not currently included, even if record's f003 is FI-MELINDA\n const melindaIds = getMelindaIdsF035(record);\n\n debugData(`Unique identifiers (${melindaIds.length}): ${JSON.stringify(melindaIds)}`);\n\n if (melindaIds.length < 1) {\n debug(`No identifiers found, no queries created.`);\n return [];\n }\n\n return toQueries(melindaIds, 'melinda.melindaid');\n}\n\n\n// bibHostComponents returns host id from the first subfield $w of first field f773, see test-fixtures 04 and 05\n// bibHostComponents should search all 773 $ws for possible host id, but what should it do in case of multiple host ids?\nexport function bibHostComponents(record) {\n const id = getHostId();\n return id ? [`melinda.partsofhost=${id}`] : [];\n\n function getHostId() {\n const [field] = record.get(/^773$/u);\n\n if (field) {\n const {value} = field.subfields.find(({code}) => code === 'w') || {};\n\n if (value && (/^\\(FI-MELINDA\\)/u).test(value)) {\n return value.replace(/^\\(FI-MELINDA\\)/u, '');\n }\n\n if (value && (/^\\(FIN01\\)/u).test(value)) {\n return value.replace(/^\\(FIN01\\)/u, '');\n }\n\n return false;\n }\n return false;\n }\n}\n\n// SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to\n// (probably) to *headings* index TIT\n// Headings index TIT drops articles etc. from the start of the title according to the filing indicator\n// Currently filing indicator is not implemented - if the title starts with an article and the Melinda\n// record is correctly catalogued using a filing indicator -> dc.title search won't match\n\nexport function bibTitle(record) {\n const title = getTitle();\n\n if (title) {\n const formatted = title\n .replace(/[^\\w\\s\\p{Alphabetic}]/gu, '')\n // Clean up concurrent spaces from fe. subfield changes\n .replace(/ +/gu, ' ')\n .trim()\n .slice(0, 30)\n .trim();\n\n // Prevent too many matches by having a minimum length requirement\n return formatted.length >= 5 ? [`dc.title=\"^${formatted}*\"`] : [];\n }\n\n return [];\n\n function getTitle() {\n const [field] = record.get(/^245$/u);\n\n if (field) {\n return field.subfields\n .filter(({code}) => ['a', 'b'].includes(code))\n .map(({value}) => value)\n // In Melinda's index subfield separators are indexed as ' '\n .join(' ');\n }\n return false;\n }\n}\n\n// eslint-disable-next-line max-statements\nexport function bibStandardIdentifiers(record) {\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibStandardIdentifiers');\n const debugData = debug.extend('data');\n debug(`Creating queries for standard identifiers`);\n\n const fields = record.get(/^(?<def>020|022|024)$/u);\n const identifiers = [].concat(...fields.map(toIdentifiers));\n const uniqueIdentifiers = [...new Set(identifiers)];\n\n debugData(`Standard identifier fields: ${JSON.stringify(fields)}`);\n debugData(`Identifiers (${identifiers.length}): ${JSON.stringify(identifiers)}`);\n debugData(`Unique identifiers (${uniqueIdentifiers.length}): ${JSON.stringify(uniqueIdentifiers)}`);\n\n if (uniqueIdentifiers.length < 1) {\n debug(`No identifiers found, no queries created.`);\n return [];\n }\n\n return toQueries(uniqueIdentifiers, 'dc.identifier');\n\n function toIdentifiers({tag, subfields}) {\n const issnIsbnReqExp = (/^[A-Za-z0-9-]+$/u);\n const otherIdReqExp = (/^[A-Za-z0-9-:]+$/u);\n\n if (tag === '022') {\n return subfields\n .filter(sub => ['a', 'z', 'y'].includes(sub.code) && issnIsbnReqExp.test(sub.value) && sub.value !== undefined)\n .map(({value}) => value);\n }\n\n if (tag === '020') {\n return subfields\n .filter(sub => ['a', 'z'].includes(sub.code) && issnIsbnReqExp.test(sub.value) && sub.value !== undefined)\n .map(({value}) => value);\n }\n\n return subfields\n .filter(sub => ['a', 'z'].includes(sub.code) && otherIdReqExp.test(sub.value) && sub.value !== undefined)\n .map(({value}) => value);\n }\n}\n"],"file":"bib.js"}
|
|
1
|
+
{"version":3,"sources":["../../../src/candidate-search/query-list/bib.js"],"names":["bibSourceIds","record","debug","debugData","extend","fSids","get","length","JSON","stringify","toSidQueries","sidStrings","getSidStrings","sidQueries","map","toSidString","filter","nonEmptySid","field","createSidString","sfC","sfB","cleanedSfC","removeSourcePrefix","normalizeSidSubfieldValue","cleanedSfB","concat","subfieldValue","sourcePrefixRegex","normalizedValue","replace","normalizeAwayRegex","bibMelindaIds","melindaIds","bibHostComponents","id","getHostId","value","subfields","find","code","test","bibTitle","title","getTitle","booleanStartWords","formatted","trim","slice","useWordSearch","some","word","toLowerCase","startsWith","includes","join","bibStandardIdentifiers","fields","identifiers","toIdentifiers","uniqueIdentifiers","Set","tag","issnIsbnReqExp","otherIdReqExp","sub","undefined"],"mappings":";;;;;;;;;;;AA4BA;;AACA;;AACA;;;;AA7BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAMO,SAASA,YAAT,CAAsBC,MAAtB,EAA8B;AAEnC;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAQE,QAAMC,KAAK,GAAG,oBAAkB,qEAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB,CA3BmC,CA4BnC;;AAEAF,EAAAA,KAAK,CAAE,iCAAF,CAAL;AAEA,QAAMG,KAAK,GAAGJ,MAAM,CAACK,GAAP,CAAW,KAAX,CAAd;AACAH,EAAAA,SAAS,CAAE,eAAcE,KAAK,CAACE,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAeJ,KAAf,CAAsB,EAAxD,CAAT;AAEA,SAAOA,KAAK,CAACE,MAAN,GAAe,CAAf,GAAmBG,YAAY,CAACL,KAAD,CAA/B,GAAyC,EAAhD;;AAEA,WAASK,YAAT,CAAsBL,KAAtB,EAA6B;AAC3BH,IAAAA,KAAK,CAAE,wCAAF,CAAL;AAEA,UAAMS,UAAU,GAAGC,aAAa,CAACP,KAAD,CAAhC;;AAEA,QAAIM,UAAU,CAACJ,MAAX,GAAoB,CAAxB,EAA2B;AACzBL,MAAAA,KAAK,CAAE,2CAAF,CAAL;AACA,aAAO,EAAP;AACD;;AAED,UAAMW,UAAU,GAAG,qCAAUF,UAAV,EAAsB,kBAAtB,CAAnB;AAEA,WAAOE,UAAP;;AAEA,aAASD,aAAT,CAAuBP,KAAvB,EAA8B;AAC5BH,MAAAA,KAAK,CAAE,qCAAF,CAAL,CAD4B,CAG5B;;AACA,YAAMS,UAAU,GAAGN,KAAK,CAACS,GAAN,CAAUC,WAAV,EAAuBC,MAAvB,CAA8BC,WAAW,IAAIA,WAA7C,CAAnB;AACA,aAAON,UAAP;;AAEA,eAASI,WAAT,CAAqBG,KAArB,EAA4B;AAC1BhB,QAAAA,KAAK,CAAE,6BAAF,CAAL;AAEA,eAAO,mDAA+BgB,KAA/B,IAAwCC,eAAe,CAACD,KAAD,CAAvD,GAAiE,EAAxE;;AAEA,iBAASC,eAAT,CAAyBD,KAAzB,EAAgC;AAC9BhB,UAAAA,KAAK,CAAE,8BAAF,CAAL;AACA,gBAAM,CAACkB,GAAD,IAAQ,sCAAkBF,KAAlB,EAAyB,GAAzB,CAAd;AACA,gBAAM,CAACG,GAAD,IAAQ,sCAAkBH,KAAlB,EAAyB,GAAzB,CAAd;AAEA,gBAAMI,UAAU,GAAGC,kBAAkB,CAACC,yBAAyB,CAACJ,GAAD,CAA1B,CAArC;AACA,gBAAMK,UAAU,GAAGD,yBAAyB,CAACH,GAAD,CAA5C;AAEAlB,UAAAA,SAAS,CAAE,GAAEK,IAAI,CAACC,SAAL,CAAeW,GAAf,CAAoB,MAAKZ,IAAI,CAACC,SAAL,CAAeY,GAAf,CAAoB,EAAjD,CAAT;AACA,iBAAOC,UAAU,CAACI,MAAX,CAAkBD,UAAlB,CAAP;AACD;;AAED,iBAASF,kBAAT,CAA4BI,aAA5B,EAA2C;AACzC,gBAAMC,iBAAiB,GAAI,gDAA3B;AACA,gBAAMC,eAAe,GAAGF,aAAa,CAACG,OAAd,CAAsBF,iBAAtB,EAAyC,OAAzC,CAAxB;AACAzB,UAAAA,SAAS,CAAE,cAAawB,aAAc,OAAME,eAAgB,EAAnD,CAAT;AACA,iBAAOA,eAAP;AACD;;AAED,iBAASL,yBAAT,CAAmCG,aAAnC,EAAkD;AAChDxB,UAAAA,SAAS,CAAE,eAAcwB,aAAc,EAA9B,CAAT;AACA,gBAAMI,kBAAkB,GAAI,OAA5B;AACA,iBAAOJ,aAAa,CAACG,OAAd,CAAsBC,kBAAtB,EAA0C,EAA1C,CAAP;AACD;AAEF;AACF;AACF;AACF;;AAEM,SAASC,aAAT,CAAuB/B,MAAvB,EAA+B;AACpC;AAEA,QAAMC,KAAK,GAAG,oBAAkB,wEAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AACAF,EAAAA,KAAK,CAAE,iCAAF,CAAL,CALoC,CAOpC;AACA;AACA;;AACA,QAAM+B,UAAU,GAAG,sCAAkBhC,MAAlB,CAAnB;AAEAE,EAAAA,SAAS,CAAE,uBAAsB8B,UAAU,CAAC1B,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAewB,UAAf,CAA2B,EAA1E,CAAT;;AAEA,MAAIA,UAAU,CAAC1B,MAAX,GAAoB,CAAxB,EAA2B;AACzBL,IAAAA,KAAK,CAAE,2CAAF,CAAL;AACA,WAAO,EAAP;AACD;;AAED,SAAO,qCAAU+B,UAAV,EAAsB,mBAAtB,CAAP;AACD,C,CAGD;AACA;;;AACO,SAASC,iBAAT,CAA2BjC,MAA3B,EAAmC;AACxC,QAAMkC,EAAE,GAAGC,SAAS,EAApB;AACA,SAAOD,EAAE,GAAG,CAAE,uBAAsBA,EAAG,EAA3B,CAAH,GAAmC,EAA5C;;AAEA,WAASC,SAAT,GAAqB;AACnB,UAAM,CAAClB,KAAD,IAAUjB,MAAM,CAACK,GAAP,CAAW,QAAX,CAAhB;;AAEA,QAAIY,KAAJ,EAAW;AACT,YAAM;AAACmB,QAAAA;AAAD,UAAUnB,KAAK,CAACoB,SAAN,CAAgBC,IAAhB,CAAqB,CAAC;AAACC,QAAAA;AAAD,OAAD,KAAYA,IAAI,KAAK,GAA1C,KAAkD,EAAlE;;AAEA,UAAIH,KAAK,IAAK,kBAAD,CAAqBI,IAArB,CAA0BJ,KAA1B,CAAb,EAA+C;AAC7C,eAAOA,KAAK,CAACP,OAAN,CAAc,kBAAd,EAAkC,EAAlC,CAAP;AACD;;AAED,UAAIO,KAAK,IAAK,aAAD,CAAgBI,IAAhB,CAAqBJ,KAArB,CAAb,EAA0C;AACxC,eAAOA,KAAK,CAACP,OAAN,CAAc,aAAd,EAA6B,EAA7B,CAAP;AACD;;AAED,aAAO,KAAP;AACD;;AACD,WAAO,KAAP;AACD;AACF,C,CAED;AACA;AACA;AAEA;AACA;AACA;;;AAEO,SAASY,QAAT,CAAkBzC,MAAlB,EAA0B;AAC/B,QAAM0C,KAAK,GAAGC,QAAQ,EAAtB;AACA,QAAMC,iBAAiB,GAAG,CAAC,KAAD,EAAQ,IAAR,EAAc,KAAd,EAAqB,KAArB,CAA1B;;AAEA,MAAIF,KAAJ,EAAW;AACT,UAAMG,SAAS,GAAGH,KAAK,CACpBb,OADe,CACP,yBADO,EACoB,EADpB,EAEhB;AAFgB,KAGfA,OAHe,CAGP,MAHO,EAGC,GAHD,EAIfiB,IAJe,GAKfC,KALe,CAKT,CALS,EAKN,EALM,EAMfD,IANe,EAAlB,CADS,CAST;;AACA,UAAME,aAAa,GAAGJ,iBAAiB,CAACK,IAAlB,CAAuBC,IAAI,IAAIL,SAAS,CAACM,WAAV,GAAwBC,UAAxB,CAAmCF,IAAnC,CAA/B,CAAtB,CAVS,CAWT;;AACA,WAAOL,SAAS,CAACvC,MAAV,IAAoB,CAApB,GAAwB,CAAE,aAAY0C,aAAa,GAAG,EAAH,GAAQ,GAAI,GAAEH,SAAU,IAAnD,CAAxB,GAAkF,EAAzF;AACD;;AAED,SAAO,EAAP;;AAEA,WAASF,QAAT,GAAoB;AAClB,UAAM,CAAC1B,KAAD,IAAUjB,MAAM,CAACK,GAAP,CAAW,QAAX,CAAhB;;AAEA,QAAIY,KAAJ,EAAW;AACT,aAAOA,KAAK,CAACoB,SAAN,CACJtB,MADI,CACG,CAAC;AAACwB,QAAAA;AAAD,OAAD,KAAY,CAAC,GAAD,EAAM,GAAN,EAAWc,QAAX,CAAoBd,IAApB,CADf,EAEJ1B,GAFI,CAEA,CAAC;AAACuB,QAAAA;AAAD,OAAD,KAAaA,KAFb,EAGL;AAHK,OAIJkB,IAJI,CAIC,GAJD,CAAP;AAKD;;AACD,WAAO,KAAP;AACD;AACF,C,CAED;;;AACO,SAASC,sBAAT,CAAgCvD,MAAhC,EAAwC;AAE7C,QAAMC,KAAK,GAAG,oBAAkB,iFAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AACAF,EAAAA,KAAK,CAAE,2CAAF,CAAL;AAEA,QAAMuD,MAAM,GAAGxD,MAAM,CAACK,GAAP,CAAW,wBAAX,CAAf;AACA,QAAMoD,WAAW,GAAG,GAAGhC,MAAH,CAAU,GAAG+B,MAAM,CAAC3C,GAAP,CAAW6C,aAAX,CAAb,CAApB;AACA,QAAMC,iBAAiB,GAAG,CAAC,GAAG,IAAIC,GAAJ,CAAQH,WAAR,CAAJ,CAA1B;AAEAvD,EAAAA,SAAS,CAAE,+BAA8BK,IAAI,CAACC,SAAL,CAAegD,MAAf,CAAuB,EAAvD,CAAT;AACAtD,EAAAA,SAAS,CAAE,gBAAeuD,WAAW,CAACnD,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAeiD,WAAf,CAA4B,EAArE,CAAT;AACAvD,EAAAA,SAAS,CAAE,uBAAsByD,iBAAiB,CAACrD,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAemD,iBAAf,CAAkC,EAAxF,CAAT;;AAEA,MAAIA,iBAAiB,CAACrD,MAAlB,GAA2B,CAA/B,EAAkC;AAChCL,IAAAA,KAAK,CAAE,2CAAF,CAAL;AACA,WAAO,EAAP;AACD;;AAED,SAAO,qCAAU0D,iBAAV,EAA6B,eAA7B,CAAP;;AAEA,WAASD,aAAT,CAAuB;AAACG,IAAAA,GAAD;AAAMxB,IAAAA;AAAN,GAAvB,EAAyC;AACvC,UAAMyB,cAAc,GAAI,kBAAxB;AACA,UAAMC,aAAa,GAAI,mBAAvB;;AAEA,QAAIF,GAAG,KAAK,KAAZ,EAAmB;AACjB,aAAOxB,SAAS,CACbtB,MADI,CACGiD,GAAG,IAAI,CAAC,GAAD,EAAM,GAAN,EAAW,GAAX,EAAgBX,QAAhB,CAAyBW,GAAG,CAACzB,IAA7B,KAAsCuB,cAAc,CAACtB,IAAf,CAAoBwB,GAAG,CAAC5B,KAAxB,CAAtC,IAAwE4B,GAAG,CAAC5B,KAAJ,KAAc6B,SADhG,EAEJpD,GAFI,CAEA,CAAC;AAACuB,QAAAA;AAAD,OAAD,KAAaA,KAFb,CAAP;AAGD;;AAED,QAAIyB,GAAG,KAAK,KAAZ,EAAmB;AACjB,aAAOxB,SAAS,CACbtB,MADI,CACGiD,GAAG,IAAI,CAAC,GAAD,EAAM,GAAN,EAAWX,QAAX,CAAoBW,GAAG,CAACzB,IAAxB,KAAiCuB,cAAc,CAACtB,IAAf,CAAoBwB,GAAG,CAAC5B,KAAxB,CAAjC,IAAmE4B,GAAG,CAAC5B,KAAJ,KAAc6B,SAD3F,EAEJpD,GAFI,CAEA,CAAC;AAACuB,QAAAA;AAAD,OAAD,KAAaA,KAFb,CAAP;AAGD;;AAED,WAAOC,SAAS,CACbtB,MADI,CACGiD,GAAG,IAAI,CAAC,GAAD,EAAM,GAAN,EAAWX,QAAX,CAAoBW,GAAG,CAACzB,IAAxB,KAAiCwB,aAAa,CAACvB,IAAd,CAAmBwB,GAAG,CAAC5B,KAAvB,CAAjC,IAAkE4B,GAAG,CAAC5B,KAAJ,KAAc6B,SAD1F,EAEJpD,GAFI,CAEA,CAAC;AAACuB,MAAAA;AAAD,KAAD,KAAaA,KAFb,CAAP;AAGD;AACF","sourcesContent":["\n/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\nimport createDebugLogger from 'debug';\nimport {toQueries} from '../candidate-search-utils';\nimport {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues} from '../../matching-utils';\n\n\nexport function bibSourceIds(record) {\n\n /* Melinda's SRU-index melinda.sourceid includes source IDs from SID fields in Melinda records\n SID-fields in Melinda have sf $c with local id and sf $b with a code for the local db:\n\n SID__ $c 123457 $b helka\n SID__ $c (ANDL100020)1077305 $b sata\n SID__ $c VER999999 $ FI-KV\n SID__ $c /10024/508126 $ REPO_THESEUS\n\n In melinda.sourceid -index case is kept, sourceprefixes in brackets and hyphens are normalized away:\n Note: slashes are not normalized away, but a SRU-search-string including slashes needs to be quoted\n\n 1234567helka\n 1077305sata\n VER999999FIKV\n /10024/508126REPO_THESEUS\n\n Note: All Melinda records that have a matching records in a local db do NOT have SID for that local records,\n existence of a SID field depends on how the record has been added to Melinda and how it has been handled\n afterwards. SIDs are also not reliably maintained. A record might or might not have a SID for a local db\n after the matching record is removed from the local db.\n\n */\n\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:source-ids');\n const debugData = debug.extend('data');\n //const debugInfo = debug.extend('info');\n\n debug(`Creating queries for sourceid's`);\n\n const fSids = record.get('SID');\n debugData(`SID-fields (${fSids.length}): ${JSON.stringify(fSids)}`);\n\n return fSids.length > 0 ? toSidQueries(fSids) : [];\n\n function toSidQueries(fSids) {\n debug(`Creating actual queries for sourceid's`);\n\n const sidStrings = getSidStrings(fSids);\n\n if (sidStrings.length < 1) {\n debug(`No identifiers found, no queries created.`);\n return [];\n }\n\n const sidQueries = toQueries(sidStrings, 'melinda.sourceid');\n\n return sidQueries;\n\n function getSidStrings(fSids) {\n debug(`Getting Sid strings from SID fields`);\n\n // Map SID fields to valid sidStrings, filter out empty strings\n const sidStrings = fSids.map(toSidString).filter(nonEmptySid => nonEmptySid);\n return sidStrings;\n\n function toSidString(field) {\n debug(`Getting string from a field`);\n\n return validateSidFieldSubfieldCounts(field) ? createSidString(field) : '';\n\n function createSidString(field) {\n debug(`Creating string from a field`);\n const [sfC] = getSubfieldValues(field, 'c');\n const [sfB] = getSubfieldValues(field, 'b');\n\n const cleanedSfC = removeSourcePrefix(normalizeSidSubfieldValue(sfC));\n const cleanedSfB = normalizeSidSubfieldValue(sfB);\n\n debugData(`${JSON.stringify(sfC)} + ${JSON.stringify(sfB)}`);\n return cleanedSfC.concat(cleanedSfB);\n }\n\n function removeSourcePrefix(subfieldValue) {\n const sourcePrefixRegex = (/^(?<sourcePrefix>\\([A-Za-z0-9-]+\\))(?<id>.+)$/u);\n const normalizedValue = subfieldValue.replace(sourcePrefixRegex, '$<id>');\n debugData(`Normalized ${subfieldValue} to ${normalizedValue}`);\n return normalizedValue;\n }\n\n function normalizeSidSubfieldValue(subfieldValue) {\n debugData(`Normalizing ${subfieldValue}`);\n const normalizeAwayRegex = (/[- ]/u);\n return subfieldValue.replace(normalizeAwayRegex, '');\n }\n\n }\n }\n }\n}\n\nexport function bibMelindaIds(record) {\n // Melinda's SRU-index melinda.melindaid includes f001 controlnumbers and old Melinda-IDs from f035z's for all non-deleted Melinda-records\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibMelindaIds');\n const debugData = debug.extend('data');\n debug(`Creating queries for MelindaIds`);\n\n // Note: Melinda-ID's for search queries are created just from records f035a's and f035z's\n // Both (FI-MELINDA)- and FCC-prefixed forms are found\n // f001 controlnumber is not currently included, even if record's f003 is FI-MELINDA\n const melindaIds = getMelindaIdsF035(record);\n\n debugData(`Unique identifiers (${melindaIds.length}): ${JSON.stringify(melindaIds)}`);\n\n if (melindaIds.length < 1) {\n debug(`No identifiers found, no queries created.`);\n return [];\n }\n\n return toQueries(melindaIds, 'melinda.melindaid');\n}\n\n\n// bibHostComponents returns host id from the first subfield $w of first field f773, see test-fixtures 04 and 05\n// bibHostComponents should search all 773 $ws for possible host id, but what should it do in case of multiple host ids?\nexport function bibHostComponents(record) {\n const id = getHostId();\n return id ? [`melinda.partsofhost=${id}`] : [];\n\n function getHostId() {\n const [field] = record.get(/^773$/u);\n\n if (field) {\n const {value} = field.subfields.find(({code}) => code === 'w') || {};\n\n if (value && (/^\\(FI-MELINDA\\)/u).test(value)) {\n return value.replace(/^\\(FI-MELINDA\\)/u, '');\n }\n\n if (value && (/^\\(FIN01\\)/u).test(value)) {\n return value.replace(/^\\(FIN01\\)/u, '');\n }\n\n return false;\n }\n return false;\n }\n}\n\n// SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to\n// (probably) to *headings* index TIT\n// - Aleph cannot currently handle headings searches starting with a boolean - in these cases use word search\n\n// Headings index TIT drops articles etc. from the start of the title according to the filing indicator\n// Currently filing indicator is not implemented - if the title starts with an article and the Melinda\n// record is correctly catalogued using a filing indicator -> dc.title search won't match\n\nexport function bibTitle(record) {\n const title = getTitle();\n const booleanStartWords = ['and', 'or', 'nor', 'not'];\n\n if (title) {\n const formatted = title\n .replace(/[^\\w\\s\\p{Alphabetic}]/gu, '')\n // Clean up concurrent spaces from fe. subfield changes\n .replace(/ +/gu, ' ')\n .trim()\n .slice(0, 30)\n .trim();\n\n // use word search for titles starting with a boolean\n const useWordSearch = booleanStartWords.some(word => formatted.toLowerCase().startsWith(word));\n // Prevent too many matches by having a minimum length requirement\n return formatted.length >= 5 ? [`dc.title=\"${useWordSearch ? '' : '^'}${formatted}*\"`] : [];\n }\n\n return [];\n\n function getTitle() {\n const [field] = record.get(/^245$/u);\n\n if (field) {\n return field.subfields\n .filter(({code}) => ['a', 'b'].includes(code))\n .map(({value}) => value)\n // In Melinda's index subfield separators are indexed as ' '\n .join(' ');\n }\n return false;\n }\n}\n\n// eslint-disable-next-line max-statements\nexport function bibStandardIdentifiers(record) {\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibStandardIdentifiers');\n const debugData = debug.extend('data');\n debug(`Creating queries for standard identifiers`);\n\n const fields = record.get(/^(?<def>020|022|024)$/u);\n const identifiers = [].concat(...fields.map(toIdentifiers));\n const uniqueIdentifiers = [...new Set(identifiers)];\n\n debugData(`Standard identifier fields: ${JSON.stringify(fields)}`);\n debugData(`Identifiers (${identifiers.length}): ${JSON.stringify(identifiers)}`);\n debugData(`Unique identifiers (${uniqueIdentifiers.length}): ${JSON.stringify(uniqueIdentifiers)}`);\n\n if (uniqueIdentifiers.length < 1) {\n debug(`No identifiers found, no queries created.`);\n return [];\n }\n\n return toQueries(uniqueIdentifiers, 'dc.identifier');\n\n function toIdentifiers({tag, subfields}) {\n const issnIsbnReqExp = (/^[A-Za-z0-9-]+$/u);\n const otherIdReqExp = (/^[A-Za-z0-9-:]+$/u);\n\n if (tag === '022') {\n return subfields\n .filter(sub => ['a', 'z', 'y'].includes(sub.code) && issnIsbnReqExp.test(sub.value) && sub.value !== undefined)\n .map(({value}) => value);\n }\n\n if (tag === '020') {\n return subfields\n .filter(sub => ['a', 'z'].includes(sub.code) && issnIsbnReqExp.test(sub.value) && sub.value !== undefined)\n .map(({value}) => value);\n }\n\n return subfields\n .filter(sub => ['a', 'z'].includes(sub.code) && otherIdReqExp.test(sub.value) && sub.value !== undefined)\n .map(({value}) => value);\n }\n}\n"],"file":"bib.js"}
|
package/dist/index.js
CHANGED
|
@@ -69,7 +69,8 @@ var _default = ({
|
|
|
69
69
|
const detect = (0, matchDetection.default)(detectionOptions, returnStrategy);
|
|
70
70
|
return record => {
|
|
71
71
|
const search = (0, candidateSearch.default)({ ...searchOptions,
|
|
72
|
-
record
|
|
72
|
+
record,
|
|
73
|
+
maxCandidates
|
|
73
74
|
});
|
|
74
75
|
return iterate({}); // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set
|
|
75
76
|
// matches : candidates that have been detected as matches by current matcher job
|
|
@@ -88,7 +89,8 @@ var _default = ({
|
|
|
88
89
|
matches = [],
|
|
89
90
|
candidateCount = 0,
|
|
90
91
|
nonMatches = [],
|
|
91
|
-
duplicateCount = 0
|
|
92
|
+
duplicateCount = 0,
|
|
93
|
+
nonMatchCount = 0
|
|
92
94
|
}) {
|
|
93
95
|
debugData(`Starting next matcher iteration.`);
|
|
94
96
|
const {
|
|
@@ -134,6 +136,7 @@ var _default = ({
|
|
|
134
136
|
nonMatches
|
|
135
137
|
});
|
|
136
138
|
const newDuplicateCount = duplicateCount + matchResult.duplicateCount;
|
|
139
|
+
const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;
|
|
137
140
|
const {
|
|
138
141
|
newMatches,
|
|
139
142
|
newNonMatches
|
|
@@ -149,7 +152,8 @@ var _default = ({
|
|
|
149
152
|
stopReason: 'maxMatches',
|
|
150
153
|
nonMatches: newNonMatches,
|
|
151
154
|
duplicateCount: newDuplicateCount,
|
|
152
|
-
candidateCount: newCandidateCount
|
|
155
|
+
candidateCount: newCandidateCount,
|
|
156
|
+
nonMatchCount: newNonMatchCount
|
|
153
157
|
});
|
|
154
158
|
}
|
|
155
159
|
|
|
@@ -160,7 +164,8 @@ var _default = ({
|
|
|
160
164
|
stopReason: 'maxCandidates',
|
|
161
165
|
nonMatches: newNonMatches,
|
|
162
166
|
duplicateCount: newDuplicateCount,
|
|
163
|
-
candidateCount: newCandidateCount
|
|
167
|
+
candidateCount: newCandidateCount,
|
|
168
|
+
nonMatchCount: newNonMatchCount
|
|
164
169
|
});
|
|
165
170
|
}
|
|
166
171
|
|
|
@@ -169,7 +174,8 @@ var _default = ({
|
|
|
169
174
|
matches: newMatches,
|
|
170
175
|
candidateCount: newCandidateCount,
|
|
171
176
|
nonMatches: newNonMatches,
|
|
172
|
-
duplicateCount: newDuplicateCount
|
|
177
|
+
duplicateCount: newDuplicateCount,
|
|
178
|
+
nonMatchCount: newNonMatchCount
|
|
173
179
|
});
|
|
174
180
|
}
|
|
175
181
|
|
|
@@ -230,9 +236,16 @@ var _default = ({
|
|
|
230
236
|
stopReason,
|
|
231
237
|
nonMatches,
|
|
232
238
|
duplicateCount,
|
|
233
|
-
candidateCount
|
|
239
|
+
candidateCount,
|
|
240
|
+
nonMatchCount
|
|
234
241
|
}) {
|
|
235
|
-
checkCounts(
|
|
242
|
+
checkCounts({
|
|
243
|
+
matches,
|
|
244
|
+
nonMatches,
|
|
245
|
+
candidateCount,
|
|
246
|
+
duplicateCount,
|
|
247
|
+
nonMatchCount
|
|
248
|
+
});
|
|
236
249
|
const matchStatus = getMatchState(state, stopReason); // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
|
|
237
250
|
|
|
238
251
|
const result = returnNonMatches ? {
|
|
@@ -244,14 +257,20 @@ var _default = ({
|
|
|
244
257
|
matchStatus
|
|
245
258
|
};
|
|
246
259
|
debugData(`${JSON.stringify(result)}`);
|
|
247
|
-
return result; //
|
|
248
|
-
// note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
|
|
260
|
+
return result; // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
|
|
249
261
|
|
|
250
|
-
function checkCounts(
|
|
262
|
+
function checkCounts({
|
|
263
|
+
matches,
|
|
264
|
+
nonMatches,
|
|
265
|
+
candidateCount,
|
|
266
|
+
duplicateCount,
|
|
267
|
+
nonMatchCount
|
|
268
|
+
}) {
|
|
251
269
|
const matchCount = matches.length;
|
|
252
|
-
|
|
270
|
+
debugData(`Return nonMatches: ${returnNonMatches}`);
|
|
271
|
+
const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
|
|
253
272
|
const totalHandled = matchCount + nonMatchCount + duplicateCount;
|
|
254
|
-
debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${
|
|
273
|
+
debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}`);
|
|
255
274
|
debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
|
|
256
275
|
|
|
257
276
|
if (totalHandled !== candidateCount) {
|
|
@@ -263,9 +282,12 @@ var _default = ({
|
|
|
263
282
|
}
|
|
264
283
|
|
|
265
284
|
function getMatchState(state, stopReason) {
|
|
266
|
-
|
|
285
|
+
debugData(`${JSON.stringify(state)}`);
|
|
286
|
+
debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
|
|
267
287
|
debugData(`StopReason: <${stopReason}>`);
|
|
268
|
-
const
|
|
288
|
+
const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;
|
|
289
|
+
const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
|
|
290
|
+
debugData(`nonRetrieved: ${nonRetrieved}`);
|
|
269
291
|
|
|
270
292
|
if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0) {
|
|
271
293
|
const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : '';
|
|
@@ -296,12 +318,14 @@ var _default = ({
|
|
|
296
318
|
recordMatches = [],
|
|
297
319
|
recordNonMatches = [],
|
|
298
320
|
recordCount = 0,
|
|
299
|
-
recordDuplicateCount = 0
|
|
321
|
+
recordDuplicateCount = 0,
|
|
322
|
+
recordNonMatchCount = 0
|
|
300
323
|
}) {
|
|
301
324
|
// recordSetSize : total amount of records in the current record set
|
|
302
325
|
// recordCount : amount of records from the current record set that have been handled
|
|
303
326
|
// maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped
|
|
304
327
|
// recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results
|
|
328
|
+
// recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)
|
|
305
329
|
// records : non-handled records in the current record set
|
|
306
330
|
// matches : found matches in the current matcher job
|
|
307
331
|
// recordMatches : found matches in the current record set
|
|
@@ -331,15 +355,17 @@ var _default = ({
|
|
|
331
355
|
recordMatches,
|
|
332
356
|
recordCount: newRecordCount,
|
|
333
357
|
recordNonMatches,
|
|
334
|
-
recordDuplicateCount: recordDuplicateCount + 1
|
|
358
|
+
recordDuplicateCount: recordDuplicateCount + 1,
|
|
359
|
+
recordNonMatchCount
|
|
335
360
|
});
|
|
336
361
|
}
|
|
337
362
|
|
|
338
|
-
debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled${returnNonMatches ?
|
|
363
|
+
debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);
|
|
339
364
|
return {
|
|
340
365
|
matches: recordMatches,
|
|
341
366
|
nonMatches: returnNonMatches ? recordNonMatches : [],
|
|
342
|
-
duplicateCount: recordDuplicateCount
|
|
367
|
+
duplicateCount: recordDuplicateCount,
|
|
368
|
+
nonMatchCount: recordNonMatchCount
|
|
343
369
|
};
|
|
344
370
|
|
|
345
371
|
function handleDetectionResult(detectionResult, candidateId, candidateRecord) {
|
|
@@ -367,6 +393,8 @@ var _default = ({
|
|
|
367
393
|
return handleRecordMatch(detectionResult.match, newMatch);
|
|
368
394
|
}
|
|
369
395
|
|
|
396
|
+
const newRecordNonMatchCount = recordNonMatchCount + 1;
|
|
397
|
+
debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);
|
|
370
398
|
return iterateRecords({
|
|
371
399
|
records: records.slice(1),
|
|
372
400
|
recordSetSize,
|
|
@@ -375,7 +403,8 @@ var _default = ({
|
|
|
375
403
|
recordMatches,
|
|
376
404
|
recordCount: newRecordCount,
|
|
377
405
|
recordNonMatches,
|
|
378
|
-
recordDuplicateCount
|
|
406
|
+
recordDuplicateCount,
|
|
407
|
+
recordNonMatchCount: newRecordNonMatchCount
|
|
379
408
|
});
|
|
380
409
|
}
|
|
381
410
|
|
|
@@ -396,7 +425,8 @@ var _default = ({
|
|
|
396
425
|
return {
|
|
397
426
|
matches: newRecordMatches,
|
|
398
427
|
nonMatches: returnNonMatches ? newRecordNonMatches : [],
|
|
399
|
-
duplicateCount: recordDuplicateCount
|
|
428
|
+
duplicateCount: recordDuplicateCount,
|
|
429
|
+
nonMatchCount: recordNonMatchCount
|
|
400
430
|
};
|
|
401
431
|
}
|
|
402
432
|
|
|
@@ -408,7 +438,8 @@ var _default = ({
|
|
|
408
438
|
recordMatches: newRecordMatches,
|
|
409
439
|
recordCount: newRecordCount,
|
|
410
440
|
recordNonMatches: returnNonMatches ? newRecordNonMatches : [],
|
|
411
|
-
duplicateCount: recordDuplicateCount
|
|
441
|
+
duplicateCount: recordDuplicateCount,
|
|
442
|
+
recordNonMatchCount
|
|
412
443
|
});
|
|
413
444
|
}
|
|
414
445
|
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.js"],"names":["detection","detectionOptions","search","searchOptions","maxMatches","maxCandidates","returnStrategy","returnQuery","returnNonMatches","debug","debugData","extend","JSON","stringify","detect","record","iterate","initialState","matches","candidateCount","nonMatches","duplicateCount","records","state","length","recordSetSize","newCandidateCount","handleRecordSet","queriesLeft","searchCounter","query","returnResult","stopReason","matchResult","iterateRecords","newDuplicateCount","newMatches","newNonMatches","handleMatchResult","maxMatchesFound","maxCandidatesRetrieved","concat","addQuery","map","match","matchQuery","checkCounts","matchStatus","getMatchState","result","matchCount","nonMatchCount","totalHandled","resultSetOffset","totalRecords","queryCandidateCounter","maxedQueries","nonRetrieved","maxedQueriesStopReason","newStopReason","status","recordMatches","recordNonMatches","recordCount","recordDuplicateCount","candidate","newRecordCount","candidateNotInMatches","candidateRecord","id","candidateId","detectionResult","handleDetectionResult","slice","strategy","treshold","probability","strategyResult","newMatch","handleRecordMatch","isMatch","newRecordMatches","newRecordNonMatches","newCandidateId","find"],"mappings":";;;;;;;AA4BA;;AACA;;;;AACA;;;;;;;;;;AA9BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;eAQe,CAAC;AAACA,EAAAA,SAAS,EAAEC,gBAAZ;AAA8BC,EAAAA,MAAM,EAAEC,aAAtC;AAAqDC,EAAAA,UAAU,GAAG,CAAlE;AAAqEC,EAAAA,aAAa,GAAG,EAArF;AAAyFC,EAAAA,cAAc,GAAG,KAA1G;AAAiHC,EAAAA,WAAW,GAAG,KAA/H;AAAsIC,EAAAA,gBAAgB,GAAG;AAAzJ,CAAD,KAAqK;AAClL,QAAMC,KAAK,GAAG,oBAAkB,yCAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AAEAD,EAAAA,SAAS,CAAE,qBAAoBE,IAAI,CAACC,SAAL,CAAeZ,gBAAf,CAAiC,EAAvD,CAAT;AACAS,EAAAA,SAAS,CAAE,kBAAiBE,IAAI,CAACC,SAAL,CAAeV,aAAf,CAA8B,EAAjD,CAAT;AACAO,EAAAA,SAAS,CAAE,eAAcE,IAAI,CAACC,SAAL,CAAeT,UAAf,CAA2B,EAA3C,CAAT;AACAM,EAAAA,SAAS,CAAE,kBAAiBE,IAAI,CAACC,SAAL,CAAeR,aAAf,CAA8B,EAAjD,CAAT;AACAK,EAAAA,SAAS,CAAE,mBAAkBE,IAAI,CAACC,SAAL,CAAeP,cAAf,CAA+B,EAAnD,CAAT;AACAI,EAAAA,SAAS,CAAE,gBAAeE,IAAI,CAACC,SAAL,CAAeN,WAAf,CAA4B,EAA7C,CAAT;AACAG,EAAAA,SAAS,CAAE,qBAAoBE,IAAI,CAACC,SAAL,CAAeL,gBAAf,CAAiC,EAAvD,CAAT;AAEA,QAAMM,MAAM,GAAG,4BAAyBb,gBAAzB,EAA2CK,cAA3C,CAAf;AAEA,SAAOS,MAAM,IAAI;AACf,UAAMb,MAAM,GAAG,6BAAsB,EAAC,GAAGC,aAAJ;AAAmBY,MAAAA;AAAnB,KAAtB,CAAf;AACA,WAAOC,OAAO,CAAC,EAAD,CAAd,CAFe,CAIf;AACA;AACA;AACA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,mBAAeA,OAAf,CAAuB;AAACC,MAAAA,YAAY,GAAG,EAAhB;AAAoBC,MAAAA,OAAO,GAAG,EAA9B;AAAkCC,MAAAA,cAAc,GAAG,CAAnD;AAAsDC,MAAAA,UAAU,GAAG,EAAnE;AAAuEC,MAAAA,cAAc,GAAG;AAAxF,KAAvB,EAAmH;AACjHX,MAAAA,SAAS,CAAE,kCAAF,CAAT;AACA,YAAM;AAACY,QAAAA,OAAD;AAAU,WAAGC;AAAb,UAAsB,MAAMrB,MAAM,CAACe,YAAD,CAAxC;AAEAP,MAAAA,SAAS,CAAE,kBAAiBE,IAAI,CAACC,SAAL,CAAeU,KAAf,CAAsB,cAAaL,OAAO,CAACM,MAAO,qBAAoBL,cAAe,iBAAgBC,UAAU,CAACI,MAAO,EAA1I,CAAT;AACA,YAAMC,aAAa,GAAGH,OAAO,CAACE,MAA9B;AACA,YAAME,iBAAiB,GAAGP,cAAc,GAAGM,aAA3C;;AAEA,UAAIA,aAAa,GAAG,CAApB,EAAuB;AACrB,eAAOE,eAAe,EAAtB;AACD;;AAED,UAAIJ,KAAK,CAACK,WAAN,GAAoB,CAAxB,EAA2B;AACzBnB,QAAAA,KAAK,CAAE,oBAAmBc,KAAK,CAACM,aAAc,QAAON,KAAK,CAACO,KAAM,mBAAkBP,KAAK,CAACK,WAAY,eAAhG,CAAL;AACA,eAAOZ,OAAO,CAAC;AAACC,UAAAA,YAAY,EAAEM,KAAf;AAAsBL,UAAAA,OAAtB;AAA+BC,UAAAA,cAAc,EAAEO,iBAA/C;AAAkEN,UAAAA,UAAlE;AAA8EC,UAAAA;AAA9E,SAAD,CAAd;AACD;;AAEDZ,MAAAA,KAAK,CAAE,wEAAuES,OAAO,CAACM,MAAO,EAAxF,CAAL;AACA,aAAOO,YAAY,CAAC;AAACb,QAAAA,OAAD;AAAUK,QAAAA,KAAV;AAAiBS,QAAAA,UAAU,EAAE,EAA7B;AAAiCZ,QAAAA,UAAjC;AAA6CD,QAAAA,cAAc,EAAEO,iBAA7D;AAAgFL,QAAAA;AAAhF,OAAD,CAAnB;;AAEA,eAASM,eAAT,GAA2B;AACzBlB,QAAAA,KAAK,CAAE,0BAAyBgB,aAAc,qDAAoDF,KAAK,CAACM,aAAc,eAAcN,KAAK,CAACO,KAAM,EAA3I,CAAL;AAEA,cAAMG,WAAW,GAAGC,cAAc,CAAC;AAACZ,UAAAA,OAAD;AAAUG,UAAAA,aAAV;AAAyBrB,UAAAA,UAAzB;AAAqCc,UAAAA,OAArC;AAA8CE,UAAAA;AAA9C,SAAD,CAAlC;AACA,cAAMe,iBAAiB,GAAGd,cAAc,GAAGY,WAAW,CAACZ,cAAvD;AACA,cAAM;AAACe,UAAAA,UAAD;AAAaC,UAAAA;AAAb,YAA8BC,iBAAiB,CAACL,WAAD,EAAcf,OAAd,EAAuBE,UAAvB,CAArD;;AAEA,YAAImB,eAAe,CAAC;AAACrB,UAAAA,OAAO,EAAEkB,UAAV;AAAsBhC,UAAAA;AAAtB,SAAD,CAAnB,EAAwD;AACtD,iBAAO2B,YAAY,CAAC;AAACb,YAAAA,OAAO,EAAEkB,UAAV;AAAsBb,YAAAA,KAAtB;AAA6BS,YAAAA,UAAU,EAAE,YAAzC;AAAuDZ,YAAAA,UAAU,EAAEiB,aAAnE;AAAkFhB,YAAAA,cAAc,EAAEc,iBAAlG;AAAqHhB,YAAAA,cAAc,EAAEO;AAArI,WAAD,CAAnB;AACD;;AAED,YAAIc,sBAAsB,CAACd,iBAAD,EAAoBrB,aAApB,CAA1B,EAA8D;AAC5D,iBAAO0B,YAAY,CAAC;AAACb,YAAAA,OAAO,EAAEkB,UAAV;AAAsBb,YAAAA,KAAtB;AAA6BS,YAAAA,UAAU,EAAE,eAAzC;AAA0DZ,YAAAA,UAAU,EAAEiB,aAAtE;AAAqFhB,YAAAA,cAAc,EAAEc,iBAArG;AAAwHhB,YAAAA,cAAc,EAAEO;AAAxI,WAAD,CAAnB;AACD;;AAED,eAAOV,OAAO,CAAC;AAACC,UAAAA,YAAY,EAAEM,KAAf;AAAsBL,UAAAA,OAAO,EAAEkB,UAA/B;AAA2CjB,UAAAA,cAAc,EAAEO,iBAA3D;AAA8EN,UAAAA,UAAU,EAAEiB,aAA1F;AAAyGhB,UAAAA,cAAc,EAAEc;AAAzH,SAAD,CAAd;AACD;;AAED,eAASG,iBAAT,CAA2BL,WAA3B,EAAwCf,OAAxC,EAAiDE,UAAjD,EAA6D;AAC3DV,QAAAA,SAAS,CAAE,4CAA2CuB,WAAW,CAACf,OAAZ,CAAoBM,MAAO,EAAxE,CAAT,CAD2D,CAE3D;;AACA,YAAIhB,gBAAJ,EAAsB;AACpBE,UAAAA,SAAS,CAAE,+CAA8CuB,WAAW,CAACb,UAAZ,CAAuBI,MAAO,EAA9E,CAAT;AACD;;AAED,cAAMY,UAAU,GAAGlB,OAAO,CAACuB,MAAR,CAAelC,WAAW,GAAGmC,QAAQ,CAACT,WAAW,CAACf,OAAb,CAAX,GAAmCe,WAAW,CAACf,OAAzE,CAAnB;AACA,cAAMmB,aAAa,GAAG7B,gBAAgB,GAAGY,UAAU,CAACqB,MAAX,CAAkBlC,WAAW,GAAGmC,QAAQ,CAACT,WAAW,CAACb,UAAb,CAAX,GAAsCa,WAAW,CAACb,UAA/E,CAAH,GAAgG,EAAtI;AAEAV,QAAAA,SAAS,CAAE,8BAA6B0B,UAAU,CAACZ,MAAO,EAAjD,CAAT,CAV2D,CAW3D;;AACA,YAAIhB,gBAAJ,EAAsB;AACpBE,UAAAA,SAAS,CAAE,iCAAgC2B,aAAa,CAACb,MAAO,EAAvD,CAAT;AACD;;AAED,eAAO;AAACY,UAAAA,UAAD;AAAaC,UAAAA;AAAb,SAAP;AACD;;AAED,eAASK,QAAT,CAAkBxB,OAAlB,EAA2B;AACzBR,QAAAA,SAAS,CAAE,gBAAea,KAAK,CAACO,KAAM,aAA7B,CAAT;AACA,eAAOZ,OAAO,CAACyB,GAAR,CAAaC,KAAD,KAAY,EAAC,GAAGA,KAAJ;AAAWC,UAAAA,UAAU,EAAEtB,KAAK,CAACO;AAA7B,SAAZ,CAAZ,CAAP;AACD;;AAED,eAASU,sBAAT,CAAgCrB,cAAhC,EAAgDd,aAAhD,EAA+D;AAC7DK,QAAAA,SAAS,CAAE,gDAA+CgB,iBAAkB,UAASrB,aAAc,GAA1F,CAAT;;AACA,YAAIA,aAAa,IAAIc,cAAc,IAAId,aAAvC,EAAsD;AACpDI,UAAAA,KAAK,CAAE,gEAA+DU,cAAe,MAAKd,aAAc,sBAAnG,CAAL;AACA,iBAAO,IAAP;AACD;AACF;AACF,KAtFc,CAwFf;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;AAEA;AACA;AACA;AACA;;;AAEA,aAAS0B,YAAT,CAAsB;AAACb,MAAAA,OAAD;AAAUK,MAAAA,KAAV;AAAiBS,MAAAA,UAAjB;AAA6BZ,MAAAA,UAA7B;AAAyCC,MAAAA,cAAzC;AAAyDF,MAAAA;AAAzD,KAAtB,EAAgG;AAC9F2B,MAAAA,WAAW,CAAC5B,OAAD,EAAUE,UAAV,EAAsBD,cAAtB,EAAsCE,cAAtC,CAAX;AACA,YAAM0B,WAAW,GAAGC,aAAa,CAACzB,KAAD,EAAQS,UAAR,CAAjC,CAF8F,CAG9F;;AACA,YAAMiB,MAAM,GAAGzC,gBAAgB,GAAG;AAACU,QAAAA,OAAD;AAAU6B,QAAAA,WAAV;AAAuB3B,QAAAA;AAAvB,OAAH,GAAwC;AAACF,QAAAA,OAAD;AAAU6B,QAAAA;AAAV,OAAvE;AACArC,MAAAA,SAAS,CAAE,GAAEE,IAAI,CAACC,SAAL,CAAeoC,MAAf,CAAuB,EAA3B,CAAT;AACA,aAAOA,MAAP,CAN8F,CAQ9F;AACA;;AAEA,eAASH,WAAT,CAAqB5B,OAArB,EAA8BE,UAA9B,EAA0CD,cAA1C,EAA0DE,cAA1D,EAA0E;AACxE,cAAM6B,UAAU,GAAGhC,OAAO,CAACM,MAA3B;AACA,cAAM2B,aAAa,GAAG/B,UAAU,CAACI,MAAjC;AACA,cAAM4B,YAAY,GAAGF,UAAU,GAAGC,aAAb,GAA6B9B,cAAlD;AACAZ,QAAAA,KAAK,CAAE,mBAAkBU,cAAe,cAAa+B,UAAW,iBAAgBC,aAAc,qBAAoB9B,cAAe,EAA5H,CAAL;AACAZ,QAAAA,KAAK,CAAE,qBAAoB2C,YAAa,MAAKjC,cAAe,uBAAvD,CAAL;;AACA,YAAIiC,YAAY,KAAKjC,cAArB,EAAqC;AACnCV,UAAAA,KAAK,CAAE,gCAA+BU,cAAc,GAAGiC,YAAa,aAA/D,CAAL;AACA;AACD;;AACD;AACD;;AAED,eAASJ,aAAT,CAAuBzB,KAAvB,EAA8BS,UAA9B,EAA0C;AACxCvB,QAAAA,KAAK,CAAE,gBAAec,KAAK,CAACK,WAAY,sCAAqCL,KAAK,CAAC8B,eAAN,IAAyB9B,KAAK,CAAC+B,YAAa,4BAA2B/B,KAAK,CAAC+B,YAAN,GAAqB/B,KAAK,CAACgC,qBAAsB,mBAAkBhC,KAAK,CAACiC,YAAN,CAAmBhC,MAAO,MAAKD,KAAK,CAACiC,YAAa,EAApQ,CAAL;AACA9C,QAAAA,SAAS,CAAE,gBAAesB,UAAW,GAA5B,CAAT;AAEA,cAAMyB,YAAY,GAAGlC,KAAK,CAAC8B,eAAN,IAAyB9B,KAAK,CAAC+B,YAA/B,GAA8C/B,KAAK,CAAC+B,YAAN,GAAqB/B,KAAK,CAACgC,qBAAzE,GAAiG,CAAtH;;AAEA,YAAIhC,KAAK,CAACK,WAAN,GAAoB,CAApB,IAAyB6B,YAAY,GAAG,CAAxC,IAA6ClC,KAAK,CAACiC,YAAN,CAAmBhC,MAAnB,GAA4B,CAA7E,EAAgF;AAC9E,gBAAMkC,sBAAsB,GAAGnC,KAAK,CAACiC,YAAN,CAAmBhC,MAAnB,GAA4B,CAA5B,GAAgC,cAAhC,GAAiD,EAAhF;AACA,gBAAMmC,aAAa,GAAG3B,UAAU,KAAK,EAAf,GAAoB0B,sBAApB,GAA6C1B,UAAnE;AACAtB,UAAAA,SAAS,CAAE,4BAA2BgD,sBAAuB,GAApD,CAAT;AACAhD,UAAAA,SAAS,CAAE,mBAAkBiD,aAAc,GAAlC,CAAT;AACAlD,UAAAA,KAAK,CAAE,qBAAF,CAAL;AACA,iBAAO;AAACmD,YAAAA,MAAM,EAAE,KAAT;AAAgB5B,YAAAA,UAAU,EAAE2B;AAA5B,WAAP;AACD;;AAEDlD,QAAAA,KAAK,CAAE,oBAAF,CAAL;AACA,eAAO;AAACmD,UAAAA,MAAM,EAAE,IAAT;AAAe5B,UAAAA;AAAf,SAAP;AACD;AACF;;AAED,aAASE,cAAT,CAAwB;AAACZ,MAAAA,OAAD;AAAUG,MAAAA,aAAV;AAAyBrB,MAAAA,UAAzB;AAAqCc,MAAAA,OAAO,GAAG,EAA/C;AAAmDE,MAAAA,UAAU,GAAG,EAAhE;AAAoEyC,MAAAA,aAAa,GAAG,EAApF;AAAwFC,MAAAA,gBAAgB,GAAG,EAA3G;AAA+GC,MAAAA,WAAW,GAAG,CAA7H;AAAgIC,MAAAA,oBAAoB,GAAG;AAAvJ,KAAxB,EAAmL;AAEjL;AACA;AACA;AACA;AAEA;AACA;AACA;AACA;AAEA,YAAM,CAACC,SAAD,IAAc3C,OAApB;AACA,YAAM4C,cAAc,GAAGD,SAAS,GAAGF,WAAW,GAAG,CAAjB,GAAqBA,WAArD,CAbiL,CAejL;AACA;AACA;AACA;;AAEA,UAAIE,SAAJ,EAAe;AAEb,YAAIE,qBAAqB,CAACjD,OAAO,CAACuB,MAAR,CAAerB,UAAf,CAAD,EAA6B6C,SAA7B,CAAzB,EAAkE;AAChE,gBAAM;AAAClD,YAAAA,MAAM,EAAEqD,eAAT;AAA0BC,YAAAA,EAAE,EAAEC;AAA9B,cAA6CL,SAAnD;AACAxD,UAAAA,KAAK,CAAE,qCAAoC6D,WAAY,KAAIJ,cAAe,IAAGzC,aAAc,GAAtF,CAAL;AACA,gBAAM8C,eAAe,GAAGzD,MAAM,CAACC,MAAD,EAASqD,eAAT,CAA9B;AACA,iBAAOI,qBAAqB,CAACD,eAAD,EAAkBD,WAAlB,EAA+BF,eAA/B,CAA5B;AACD;;AACD,eAAOlC,cAAc,CAAC;AAACZ,UAAAA,OAAO,EAAEA,OAAO,CAACmD,KAAR,CAAc,CAAd,CAAV;AAA4BhD,UAAAA,aAA5B;AAA2CrB,UAAAA,UAA3C;AAAuDc,UAAAA,OAAvD;AAAgE2C,UAAAA,aAAhE;AAA+EE,UAAAA,WAAW,EAAEG,cAA5F;AAA4GJ,UAAAA,gBAA5G;AAA8HE,UAAAA,oBAAoB,EAAEA,oBAAoB,GAAG;AAA3K,SAAD,CAArB;AACD;;AAEDvD,MAAAA,KAAK,CAAE,mCAAkCsD,WAAY,IAAGtC,aAAc,WAAUoC,aAAa,CAACrC,MAAO,mBAAkBwC,oBAAqB,8BAA6BxD,gBAAgB,GAAI,KAAIsD,gBAAgB,CAACtC,MAAO,oBAAhC,GAAsD,GAAI,EAA9O,CAAL;AACA,aAAO;AAACN,QAAAA,OAAO,EAAE2C,aAAV;AAAyBzC,QAAAA,UAAU,EAAEZ,gBAAgB,GAAGsD,gBAAH,GAAsB,EAA3E;AAA+EzC,QAAAA,cAAc,EAAE2C;AAA/F,OAAP;;AAEA,eAASQ,qBAAT,CAA+BD,eAA/B,EAAgDD,WAAhD,EAA6DF,eAA7D,EAA8E;AAC5E1D,QAAAA,SAAS,CAAE,8BAA6B4D,WAAY,KAAIJ,cAAe,IAAGzC,aAAc,MAAKb,IAAI,CAACC,SAAL,CAAe0D,eAAf,CAAgC,EAApH,CAAT;;AAEA,YAAIA,eAAe,CAAC3B,KAAhB,IAAyBpC,gBAA7B,EAA+C;AAC7CC,UAAAA,KAAK,CAAE,GAAE8D,eAAe,CAAC3B,KAAhB,GAAyB,UAAS0B,WAAY,KAAIJ,cAAe,IAAGzC,aAAc,eAAlF,GAAoG,UAAS6C,WAAY,KAAIJ,cAAe,IAAGzC,aAAc,mBAAmB,EAApL,CAAL;AACAf,UAAAA,SAAS,CAAE,aAAYE,IAAI,CAACC,SAAL,CAAe0D,eAAe,CAACG,QAA/B,CAAyC,eAAc9D,IAAI,CAACC,SAAL,CAAe0D,eAAe,CAACI,QAA/B,CAAyC,EAA9G,CAAT;AAEA,gBAAM1C,WAAW,GAAG;AAClB2C,YAAAA,WAAW,EAAEL,eAAe,CAACK,WADX;AAElBX,YAAAA,SAAS,EAAE;AACTI,cAAAA,EAAE,EAAEC,WADK;AAETvD,cAAAA,MAAM,EAAEqD;AAFC;AAFO,WAApB;AAOA,gBAAMS,cAAc,GAAG;AACrBH,YAAAA,QAAQ,EAAEH,eAAe,CAACG,QADL;AAErBC,YAAAA,QAAQ,EAAEJ,eAAe,CAACI;AAFL,WAAvB;AAIA,gBAAMG,QAAQ,GAAGxE,cAAc,GAAG,EAAC,GAAG2B,WAAJ;AAAiB,eAAG4C;AAApB,WAAH,GAAyC,EAAC,GAAG5C;AAAJ,WAAxE;AAEAvB,UAAAA,SAAS,CAAE,GAAEE,IAAI,CAACC,SAAL,CAAeiE,QAAf,CAAyB,EAA7B,CAAT;AAEA,iBAAOC,iBAAiB,CAACR,eAAe,CAAC3B,KAAjB,EAAwBkC,QAAxB,CAAxB;AACD;;AACD,eAAO5C,cAAc,CAAC;AAACZ,UAAAA,OAAO,EAAEA,OAAO,CAACmD,KAAR,CAAc,CAAd,CAAV;AAA4BhD,UAAAA,aAA5B;AAA2CrB,UAAAA,UAA3C;AAAuDc,UAAAA,OAAvD;AAAgE2C,UAAAA,aAAhE;AAA+EE,UAAAA,WAAW,EAAEG,cAA5F;AAA4GJ,UAAAA,gBAA5G;AAA8HE,UAAAA;AAA9H,SAAD,CAArB;AACD;;AAED,eAASe,iBAAT,CAA2BC,OAA3B,EAAoCF,QAApC,EAA8C;AAC5C,cAAMG,gBAAgB,GAAGD,OAAO,GAAGnB,aAAa,CAACpB,MAAd,CAAqBqC,QAArB,CAAH,GAAoCjB,aAApE;AACA,cAAMqB,mBAAmB,GAAGF,OAAO,GAAGlB,gBAAH,GAAsBA,gBAAgB,CAACrB,MAAjB,CAAwBqC,QAAxB,CAAzD;AAEApE,QAAAA,SAAS,CAAE,yCAAwCQ,OAAO,CAACuB,MAAR,CAAewC,gBAAf,EAAiCzD,MAAO,UAASpB,UAAW,GAAtG,CAAT,CAJ4C,CAM5C;;AACA,YAAII,gBAAJ,EAAsB;AACpBE,UAAAA,SAAS,CAAE,4CAA2CU,UAAU,CAACqB,MAAX,CAAkByC,mBAAlB,EAAuC1D,MAAO,EAA3F,CAAT;AACD;;AAED,YAAIe,eAAe,CAAC;AAACrB,UAAAA,OAAO,EAAEA,OAAO,CAACuB,MAAR,CAAewC,gBAAf,CAAV;AAA4C7E,UAAAA;AAA5C,SAAD,CAAnB,EAA8E;AAC5EK,UAAAA,KAAK,CAAE,eAAcL,UAAW,gDAA+C8D,cAAe,yCAAwCzC,aAAa,GAAGyC,cAAe,EAAhK,CAAL;AACA,iBAAO;AAAChD,YAAAA,OAAO,EAAE+D,gBAAV;AAA4B7D,YAAAA,UAAU,EAAEZ,gBAAgB,GAAG0E,mBAAH,GAAyB,EAAjF;AAAqF7D,YAAAA,cAAc,EAAE2C;AAArG,WAAP;AACD;;AAED,eAAO9B,cAAc,CAAC;AAACZ,UAAAA,OAAO,EAAEA,OAAO,CAACmD,KAAR,CAAc,CAAd,CAAV;AAA4BhD,UAAAA,aAA5B;AAA2CrB,UAAAA,UAA3C;AAAuDc,UAAAA,OAAvD;AAAgE2C,UAAAA,aAAa,EAAEoB,gBAA/E;AAAiGlB,UAAAA,WAAW,EAAEG,cAA9G;AAA8HJ,UAAAA,gBAAgB,EAAEtD,gBAAgB,GAAG0E,mBAAH,GAAyB,EAAzL;AAA6L7D,UAAAA,cAAc,EAAE2C;AAA7M,SAAD,CAArB;AACD;;AAED,eAASG,qBAAT,CAA+BjD,OAA/B,EAAwC+C,SAAxC,EAAmD;AACjDxD,QAAAA,KAAK,CAAE,wBAAuBwD,SAAS,CAACI,EAAG,+BAA8BnD,OAAO,CAACM,MAAO,qBAAnF,CAAL;AACA,cAAM2D,cAAc,GAAGlB,SAAS,CAACI,EAAjC;AACA3D,QAAAA,SAAS,CAAE,mBAAkByE,cAAe,EAAnC,CAAT;AACA,cAAMlC,MAAM,GAAG/B,OAAO,CAACkE,IAAR,CAAa,CAAC;AAACnB,UAAAA;AAAD,SAAD,KAAiBA,SAAS,CAACI,EAAV,KAAiBc,cAA/C,CAAf;AACAzE,QAAAA,SAAS,CAAE,WAAUuC,MAAO,EAAnB,CAAT;;AACA,YAAIA,MAAJ,EAAY;AACVxC,UAAAA,KAAK,CAAE,GAAEwD,SAAS,CAACI,EAAG,uBAAjB,CAAL;AACA,iBAAO,KAAP;AACD;;AACD5D,QAAAA,KAAK,CAAE,GAAEwD,SAAS,CAACI,EAAG,kCAAjB,CAAL;AACA,eAAO,IAAP;AACD;AACF;;AAED,aAAS9B,eAAT,CAAyB;AAACrB,MAAAA,OAAD;AAAUd,MAAAA;AAAV,KAAzB,EAAgD;AAC9C,UAAIA,UAAU,IAAIc,OAAO,CAACM,MAAR,IAAkBpB,UAApC,EAAgD;AAC9CK,QAAAA,KAAK,CAAE,6CAA4CL,UAAW,0BAAzD,CAAL;AACA,eAAO,IAAP;AACD;AACF;AACF,GAzPD;AA0PD,C","sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport createSearchInterface, * as candidateSearch from './candidate-search';\nimport createDetectionInterface, * as matchDetection from './match-detection';\n\nexport {candidateSearch, matchDetection};\n\nexport default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false}) => {\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:index');\n const debugData = debug.extend('data');\n\n debugData(`DetectionOptions: ${JSON.stringify(detectionOptions)}`);\n debugData(`SearchOptions: ${JSON.stringify(searchOptions)}`);\n debugData(`MaxMatches: ${JSON.stringify(maxMatches)}`);\n debugData(`MaxCandidates: ${JSON.stringify(maxCandidates)}`);\n debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);\n debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);\n\n const detect = createDetectionInterface(detectionOptions, returnStrategy);\n\n return record => {\n const search = createSearchInterface({...searchOptions, record});\n return iterate({});\n\n // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set\n // matches : candidates that have been detected as matches by current matcher job\n // nonMatches : candidates that have been detected as non-matches by current matcher job (only if returnNonMatches is 'true')\n // duplicateCount : amount of candidate records that were retrieved from the SRU but not handled further because they were already found in the matches/nonMatches\n\n // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)\n // state.query : current query (undefined if there was no queries left)\n // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)\n // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)\n // state.queriesLeft : amount of queries left\n // state.queryCounter : sequence for current query\n // state.maxedQueries : queries that resulted in more than serverMaxResults hits\n\n async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0}) {\n debugData(`Starting next matcher iteration.`);\n const {records, ...state} = await search(initialState);\n\n debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}`);\n const recordSetSize = records.length;\n const newCandidateCount = candidateCount + recordSetSize;\n\n if (recordSetSize > 0) {\n return handleRecordSet();\n }\n\n if (state.queriesLeft > 0) {\n debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);\n return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, duplicateCount});\n }\n\n debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);\n return returnResult({matches, state, stopReason: '', nonMatches, candidateCount: newCandidateCount, duplicateCount});\n\n function handleRecordSet() {\n debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);\n\n const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches});\n const newDuplicateCount = duplicateCount + matchResult.duplicateCount;\n const {newMatches, newNonMatches} = handleMatchResult(matchResult, matches, nonMatches);\n\n if (maxMatchesFound({matches: newMatches, maxMatches})) {\n return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount});\n }\n\n if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {\n return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount});\n }\n\n return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount});\n }\n\n function handleMatchResult(matchResult, matches, nonMatches) {\n debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);\n // eslint-disable-next-line functional/no-conditional-statement\n if (returnNonMatches) {\n debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);\n }\n\n const newMatches = matches.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);\n const newNonMatches = returnNonMatches ? nonMatches.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];\n\n debugData(`- Total amount of matches: ${newMatches.length}`);\n // eslint-disable-next-line functional/no-conditional-statement\n if (returnNonMatches) {\n debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);\n }\n\n return {newMatches, newNonMatches};\n }\n\n function addQuery(matches) {\n debugData(`Adding query ${state.query} to matches`);\n return matches.map((match) => ({...match, matchQuery: state.query}));\n }\n\n function maxCandidatesRetrieved(candidateCount, maxCandidates) {\n debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${maxCandidates})`);\n if (maxCandidates && candidateCount >= maxCandidates) {\n debug(`Stopped matching because maximum number of candidate records ${candidateCount} / ${maxCandidates} have been retrieved`);\n return true;\n }\n }\n }\n\n // matches : array of matching candidate records\n // nonMatches : array of nonMatching candidate records (if returnNonMatches option is true, otherwise empty array)\n // - candidate.id\n // - candidate.record\n // - probability\n // - strategy (if returnStrategy option is true)\n // - treshold (if returnStrategy option is true)\n // - matchQuery (if returnQuery option is true)\n\n // we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data\n\n // matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not\n // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries',empty string/undefined), reason for stopping retrieving or handling the candidate records\n // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)\n // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record\n\n function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount}) {\n checkCounts(matches, nonMatches, candidateCount, duplicateCount);\n const matchStatus = getMatchState(state, stopReason);\n // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered\n const result = returnNonMatches ? {matches, matchStatus, nonMatches} : {matches, matchStatus};\n debugData(`${JSON.stringify(result)}`);\n return result;\n\n // we could have a nonMatchCount even if nonMatches won't be returned so that checkCounts would make sense\n // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match\n\n function checkCounts(matches, nonMatches, candidateCount, duplicateCount) {\n const matchCount = matches.length;\n const nonMatchCount = nonMatches.length;\n const totalHandled = matchCount + nonMatchCount + duplicateCount;\n debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${nonMatchCount}, duplicateCount: ${duplicateCount}`);\n debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);\n if (totalHandled !== candidateCount) {\n debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);\n return;\n }\n return;\n }\n\n function getMatchState(state, stopReason) {\n debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset >= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);\n debugData(`StopReason: <${stopReason}>`);\n\n const nonRetrieved = state.resultSetOffset >= state.totalRecords ? state.totalRecords - state.queryCandidateCounter : 0;\n\n if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0) {\n const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : '';\n const newStopReason = stopReason === '' ? maxedQueriesStopReason : stopReason;\n debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);\n debugData(`NewStopReason: <${newStopReason}>`);\n debug(`Match status: false`);\n return {status: false, stopReason: newStopReason};\n }\n\n debug(`Match status: true`);\n return {status: true, stopReason};\n }\n }\n\n function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0}) {\n\n // recordSetSize : total amount of records in the current record set\n // recordCount : amount of records from the current record set that have been handled\n // maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped\n // recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results\n\n // records : non-handled records in the current record set\n // matches : found matches in the current matcher job\n // recordMatches : found matches in the current record set\n // recordNonMatches : found nonMatches in the current record set (only if returnNonMatches setting is true)\n\n const [candidate] = records;\n const newRecordCount = candidate ? recordCount + 1 : recordCount;\n\n // The matcher uses same matchDetection strategy for candidates from all candidate-searches -> matchDetection result for the same candidate is always same\n // Exceptions would happen if the candidate would have been updated in the database between candidate searches\n // Note that if returnNonMatches is false, matcher won't remember candidates that didn't match, so they will be matched again everytime they are retrieved by\n // different candidate search queries. Same candidate search query won't have duplicate records.\n\n if (candidate) {\n\n if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {\n const {record: candidateRecord, id: candidateId} = candidate;\n debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);\n const detectionResult = detect(record, candidateRecord);\n return handleDetectionResult(detectionResult, candidateId, candidateRecord);\n }\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1});\n }\n\n debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled${returnNonMatches ? `, ${recordNonMatches.length} nonMatches found.` : '.'}`);\n return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount};\n\n function handleDetectionResult(detectionResult, candidateId, candidateRecord) {\n debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);\n\n if (detectionResult.match || returnNonMatches) {\n debug(`${detectionResult.match ? `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is a match!` : `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is NOT a match!`}`);\n debugData(`Strategy: ${JSON.stringify(detectionResult.strategy)}, Treshold: ${JSON.stringify(detectionResult.treshold)}`);\n\n const matchResult = {\n probability: detectionResult.probability,\n candidate: {\n id: candidateId,\n record: candidateRecord\n }\n };\n const strategyResult = {\n strategy: detectionResult.strategy,\n treshold: detectionResult.treshold\n };\n const newMatch = returnStrategy ? {...matchResult, ...strategyResult} : {...matchResult};\n\n debugData(`${JSON.stringify(newMatch)}`);\n\n return handleRecordMatch(detectionResult.match, newMatch);\n }\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount});\n }\n\n function handleRecordMatch(isMatch, newMatch) {\n const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;\n const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);\n\n debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);\n\n // eslint-disable-next-line functional/no-conditional-statement\n if (returnNonMatches) {\n debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);\n }\n\n if (maxMatchesFound({matches: matches.concat(newRecordMatches), maxMatches})) {\n debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);\n return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount};\n }\n\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount});\n }\n\n function candidateNotInMatches(matches, candidate) {\n debug(`Checking that record ${candidate.id} is not already included in ${matches.length} matches/nonMatches`);\n const newCandidateId = candidate.id;\n debugData(`newCandidateId: ${newCandidateId}`);\n const result = matches.find(({candidate}) => candidate.id === newCandidateId);\n debugData(`Result: ${result}`);\n if (result) {\n debug(`${candidate.id} was already handled.`);\n return false;\n }\n debug(`${candidate.id} not found in matches/nonMatches`);\n return true;\n }\n }\n\n function maxMatchesFound({matches, maxMatches}) {\n if (maxMatches && matches.length >= maxMatches) {\n debug(`Stopping recordSet iteration: maxMatches (${maxMatches}) for matcher job found.`);\n return true;\n }\n }\n };\n};\n"],"file":"index.js"}
|
|
1
|
+
{"version":3,"sources":["../src/index.js"],"names":["detection","detectionOptions","search","searchOptions","maxMatches","maxCandidates","returnStrategy","returnQuery","returnNonMatches","debug","debugData","extend","JSON","stringify","detect","record","iterate","initialState","matches","candidateCount","nonMatches","duplicateCount","nonMatchCount","records","state","length","recordSetSize","newCandidateCount","handleRecordSet","queriesLeft","searchCounter","query","returnResult","stopReason","matchResult","iterateRecords","newDuplicateCount","newNonMatchCount","newMatches","newNonMatches","handleMatchResult","maxMatchesFound","maxCandidatesRetrieved","concat","addQuery","map","match","matchQuery","checkCounts","matchStatus","getMatchState","result","matchCount","chosenNonMatchCount","totalHandled","resultSetOffset","totalRecords","queryCandidateCounter","maxedQueries","searchesLeft","nonRetrieved","maxedQueriesStopReason","newStopReason","status","recordMatches","recordNonMatches","recordCount","recordDuplicateCount","recordNonMatchCount","candidate","newRecordCount","candidateNotInMatches","candidateRecord","id","candidateId","detectionResult","handleDetectionResult","slice","strategy","treshold","probability","strategyResult","newMatch","handleRecordMatch","newRecordNonMatchCount","isMatch","newRecordMatches","newRecordNonMatches","newCandidateId","find"],"mappings":";;;;;;;AA4BA;;AACA;;;;AACA;;;;;;;;;;AA9BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;eAQe,CAAC;AAACA,EAAAA,SAAS,EAAEC,gBAAZ;AAA8BC,EAAAA,MAAM,EAAEC,aAAtC;AAAqDC,EAAAA,UAAU,GAAG,CAAlE;AAAqEC,EAAAA,aAAa,GAAG,EAArF;AAAyFC,EAAAA,cAAc,GAAG,KAA1G;AAAiHC,EAAAA,WAAW,GAAG,KAA/H;AAAsIC,EAAAA,gBAAgB,GAAG;AAAzJ,CAAD,KAAqK;AAClL,QAAMC,KAAK,GAAG,oBAAkB,yCAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AAEAD,EAAAA,SAAS,CAAE,qBAAoBE,IAAI,CAACC,SAAL,CAAeZ,gBAAf,CAAiC,EAAvD,CAAT;AACAS,EAAAA,SAAS,CAAE,kBAAiBE,IAAI,CAACC,SAAL,CAAeV,aAAf,CAA8B,EAAjD,CAAT;AACAO,EAAAA,SAAS,CAAE,eAAcE,IAAI,CAACC,SAAL,CAAeT,UAAf,CAA2B,EAA3C,CAAT;AACAM,EAAAA,SAAS,CAAE,kBAAiBE,IAAI,CAACC,SAAL,CAAeR,aAAf,CAA8B,EAAjD,CAAT;AACAK,EAAAA,SAAS,CAAE,mBAAkBE,IAAI,CAACC,SAAL,CAAeP,cAAf,CAA+B,EAAnD,CAAT;AACAI,EAAAA,SAAS,CAAE,gBAAeE,IAAI,CAACC,SAAL,CAAeN,WAAf,CAA4B,EAA7C,CAAT;AACAG,EAAAA,SAAS,CAAE,qBAAoBE,IAAI,CAACC,SAAL,CAAeL,gBAAf,CAAiC,EAAvD,CAAT;AAEA,QAAMM,MAAM,GAAG,4BAAyBb,gBAAzB,EAA2CK,cAA3C,CAAf;AAEA,SAAOS,MAAM,IAAI;AACf,UAAMb,MAAM,GAAG,6BAAsB,EAAC,GAAGC,aAAJ;AAAmBY,MAAAA,MAAnB;AAA2BV,MAAAA;AAA3B,KAAtB,CAAf;AACA,WAAOW,OAAO,CAAC,EAAD,CAAd,CAFe,CAIf;AACA;AACA;AACA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,mBAAeA,OAAf,CAAuB;AAACC,MAAAA,YAAY,GAAG,EAAhB;AAAoBC,MAAAA,OAAO,GAAG,EAA9B;AAAkCC,MAAAA,cAAc,GAAG,CAAnD;AAAsDC,MAAAA,UAAU,GAAG,EAAnE;AAAuEC,MAAAA,cAAc,GAAG,CAAxF;AAA2FC,MAAAA,aAAa,GAAG;AAA3G,KAAvB,EAAsI;AACpIZ,MAAAA,SAAS,CAAE,kCAAF,CAAT;AACA,YAAM;AAACa,QAAAA,OAAD;AAAU,WAAGC;AAAb,UAAsB,MAAMtB,MAAM,CAACe,YAAD,CAAxC;AAEAP,MAAAA,SAAS,CAAE,kBAAiBE,IAAI,CAACC,SAAL,CAAeW,KAAf,CAAsB,cAAaN,OAAO,CAACO,MAAO,qBAAoBN,cAAe,iBAAgBC,UAAU,CAACK,MAAO,EAA1I,CAAT;AACA,YAAMC,aAAa,GAAGH,OAAO,CAACE,MAA9B;AACA,YAAME,iBAAiB,GAAGR,cAAc,GAAGO,aAA3C;;AAEA,UAAIA,aAAa,GAAG,CAApB,EAAuB;AACrB,eAAOE,eAAe,EAAtB;AACD;;AAED,UAAIJ,KAAK,CAACK,WAAN,GAAoB,CAAxB,EAA2B;AACzBpB,QAAAA,KAAK,CAAE,oBAAmBe,KAAK,CAACM,aAAc,QAAON,KAAK,CAACO,KAAM,mBAAkBP,KAAK,CAACK,WAAY,eAAhG,CAAL;AACA,eAAOb,OAAO,CAAC;AAACC,UAAAA,YAAY,EAAEO,KAAf;AAAsBN,UAAAA,OAAtB;AAA+BC,UAAAA,cAAc,EAAEQ,iBAA/C;AAAkEP,UAAAA,UAAlE;AAA8EC,UAAAA;AAA9E,SAAD,CAAd;AACD;;AAEDZ,MAAAA,KAAK,CAAE,wEAAuES,OAAO,CAACO,MAAO,EAAxF,CAAL;AACA,aAAOO,YAAY,CAAC;AAACd,QAAAA,OAAD;AAAUM,QAAAA,KAAV;AAAiBS,QAAAA,UAAU,EAAE,EAA7B;AAAiCb,QAAAA,UAAjC;AAA6CD,QAAAA,cAAc,EAAEQ,iBAA7D;AAAgFN,QAAAA;AAAhF,OAAD,CAAnB;;AAEA,eAASO,eAAT,GAA2B;AACzBnB,QAAAA,KAAK,CAAE,0BAAyBiB,aAAc,qDAAoDF,KAAK,CAACM,aAAc,eAAcN,KAAK,CAACO,KAAM,EAA3I,CAAL;AAEA,cAAMG,WAAW,GAAGC,cAAc,CAAC;AAACZ,UAAAA,OAAD;AAAUG,UAAAA,aAAV;AAAyBtB,UAAAA,UAAzB;AAAqCc,UAAAA,OAArC;AAA8CE,UAAAA;AAA9C,SAAD,CAAlC;AACA,cAAMgB,iBAAiB,GAAGf,cAAc,GAAGa,WAAW,CAACb,cAAvD;AACA,cAAMgB,gBAAgB,GAAGf,aAAa,GAAGY,WAAW,CAACZ,aAArD;AACA,cAAM;AAACgB,UAAAA,UAAD;AAAaC,UAAAA;AAAb,YAA8BC,iBAAiB,CAACN,WAAD,EAAchB,OAAd,EAAuBE,UAAvB,CAArD;;AAEA,YAAIqB,eAAe,CAAC;AAACvB,UAAAA,OAAO,EAAEoB,UAAV;AAAsBlC,UAAAA;AAAtB,SAAD,CAAnB,EAAwD;AACtD,iBAAO4B,YAAY,CAAC;AAACd,YAAAA,OAAO,EAAEoB,UAAV;AAAsBd,YAAAA,KAAtB;AAA6BS,YAAAA,UAAU,EAAE,YAAzC;AAAuDb,YAAAA,UAAU,EAAEmB,aAAnE;AAAkFlB,YAAAA,cAAc,EAAEe,iBAAlG;AAAqHjB,YAAAA,cAAc,EAAEQ,iBAArI;AAAwJL,YAAAA,aAAa,EAAEe;AAAvK,WAAD,CAAnB;AACD;;AAED,YAAIK,sBAAsB,CAACf,iBAAD,EAAoBtB,aAApB,CAA1B,EAA8D;AAC5D,iBAAO2B,YAAY,CAAC;AAACd,YAAAA,OAAO,EAAEoB,UAAV;AAAsBd,YAAAA,KAAtB;AAA6BS,YAAAA,UAAU,EAAE,eAAzC;AAA0Db,YAAAA,UAAU,EAAEmB,aAAtE;AAAqFlB,YAAAA,cAAc,EAAEe,iBAArG;AAAwHjB,YAAAA,cAAc,EAAEQ,iBAAxI;AAA2JL,YAAAA,aAAa,EAAEe;AAA1K,WAAD,CAAnB;AACD;;AAED,eAAOrB,OAAO,CAAC;AAACC,UAAAA,YAAY,EAAEO,KAAf;AAAsBN,UAAAA,OAAO,EAAEoB,UAA/B;AAA2CnB,UAAAA,cAAc,EAAEQ,iBAA3D;AAA8EP,UAAAA,UAAU,EAAEmB,aAA1F;AAAyGlB,UAAAA,cAAc,EAAEe,iBAAzH;AAA4Id,UAAAA,aAAa,EAAEe;AAA3J,SAAD,CAAd;AACD;;AAED,eAASG,iBAAT,CAA2BN,WAA3B,EAAwChB,OAAxC,EAAiDE,UAAjD,EAA6D;AAC3DV,QAAAA,SAAS,CAAE,4CAA2CwB,WAAW,CAAChB,OAAZ,CAAoBO,MAAO,EAAxE,CAAT,CAD2D,CAE3D;;AACA,YAAIjB,gBAAJ,EAAsB;AACpBE,UAAAA,SAAS,CAAE,+CAA8CwB,WAAW,CAACd,UAAZ,CAAuBK,MAAO,EAA9E,CAAT;AACD;;AAED,cAAMa,UAAU,GAAGpB,OAAO,CAACyB,MAAR,CAAepC,WAAW,GAAGqC,QAAQ,CAACV,WAAW,CAAChB,OAAb,CAAX,GAAmCgB,WAAW,CAAChB,OAAzE,CAAnB;AACA,cAAMqB,aAAa,GAAG/B,gBAAgB,GAAGY,UAAU,CAACuB,MAAX,CAAkBpC,WAAW,GAAGqC,QAAQ,CAACV,WAAW,CAACd,UAAb,CAAX,GAAsCc,WAAW,CAACd,UAA/E,CAAH,GAAgG,EAAtI;AAEAV,QAAAA,SAAS,CAAE,8BAA6B4B,UAAU,CAACb,MAAO,EAAjD,CAAT,CAV2D,CAW3D;;AACA,YAAIjB,gBAAJ,EAAsB;AACpBE,UAAAA,SAAS,CAAE,iCAAgC6B,aAAa,CAACd,MAAO,EAAvD,CAAT;AACD;;AAED,eAAO;AAACa,UAAAA,UAAD;AAAaC,UAAAA;AAAb,SAAP;AACD;;AAED,eAASK,QAAT,CAAkB1B,OAAlB,EAA2B;AACzBR,QAAAA,SAAS,CAAE,gBAAec,KAAK,CAACO,KAAM,aAA7B,CAAT;AACA,eAAOb,OAAO,CAAC2B,GAAR,CAAaC,KAAD,KAAY,EAAC,GAAGA,KAAJ;AAAWC,UAAAA,UAAU,EAAEvB,KAAK,CAACO;AAA7B,SAAZ,CAAZ,CAAP;AACD;;AAED,eAASW,sBAAT,CAAgCvB,cAAhC,EAAgDd,aAAhD,EAA+D;AAC7DK,QAAAA,SAAS,CAAE,gDAA+CiB,iBAAkB,UAAStB,aAAc,GAA1F,CAAT;;AACA,YAAIA,aAAa,IAAIc,cAAc,IAAId,aAAvC,EAAsD;AACpDI,UAAAA,KAAK,CAAE,gEAA+DU,cAAe,MAAKd,aAAc,sBAAnG,CAAL;AACA,iBAAO,IAAP;AACD;AACF;AACF,KAvFc,CAyFf;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;AAEA;AACA;AACA;AACA;;;AAEA,aAAS2B,YAAT,CAAsB;AAACd,MAAAA,OAAD;AAAUM,MAAAA,KAAV;AAAiBS,MAAAA,UAAjB;AAA6Bb,MAAAA,UAA7B;AAAyCC,MAAAA,cAAzC;AAAyDF,MAAAA,cAAzD;AAAyEG,MAAAA;AAAzE,KAAtB,EAA+G;AAC7G0B,MAAAA,WAAW,CAAC;AAAC9B,QAAAA,OAAD;AAAUE,QAAAA,UAAV;AAAsBD,QAAAA,cAAtB;AAAsCE,QAAAA,cAAtC;AAAsDC,QAAAA;AAAtD,OAAD,CAAX;AACA,YAAM2B,WAAW,GAAGC,aAAa,CAAC1B,KAAD,EAAQS,UAAR,CAAjC,CAF6G,CAG7G;;AACA,YAAMkB,MAAM,GAAG3C,gBAAgB,GAAG;AAACU,QAAAA,OAAD;AAAU+B,QAAAA,WAAV;AAAuB7B,QAAAA;AAAvB,OAAH,GAAwC;AAACF,QAAAA,OAAD;AAAU+B,QAAAA;AAAV,OAAvE;AACAvC,MAAAA,SAAS,CAAE,GAAEE,IAAI,CAACC,SAAL,CAAesC,MAAf,CAAuB,EAA3B,CAAT;AACA,aAAOA,MAAP,CAN6G,CAQ7G;;AAEA,eAASH,WAAT,CAAqB;AAAC9B,QAAAA,OAAD;AAAUE,QAAAA,UAAV;AAAsBD,QAAAA,cAAtB;AAAsCE,QAAAA,cAAtC;AAAsDC,QAAAA;AAAtD,OAArB,EAA2F;AACzF,cAAM8B,UAAU,GAAGlC,OAAO,CAACO,MAA3B;AACAf,QAAAA,SAAS,CAAE,sBAAqBF,gBAAiB,EAAxC,CAAT;AACA,cAAM6C,mBAAmB,GAAG7C,gBAAgB,GAAGY,UAAU,CAACK,MAAd,GAAuBH,aAAnE;AACA,cAAMgC,YAAY,GAAGF,UAAU,GAAG9B,aAAb,GAA6BD,cAAlD;AACAZ,QAAAA,KAAK,CAAE,mBAAkBU,cAAe,cAAaiC,UAAW,iBAAgBC,mBAAoB,qBAAoBhC,cAAe,EAAlI,CAAL;AACAZ,QAAAA,KAAK,CAAE,qBAAoB6C,YAAa,MAAKnC,cAAe,uBAAvD,CAAL;;AACA,YAAImC,YAAY,KAAKnC,cAArB,EAAqC;AACnCV,UAAAA,KAAK,CAAE,gCAA+BU,cAAc,GAAGmC,YAAa,aAA/D,CAAL;AACA;AACD;;AACD;AACD;;AAED,eAASJ,aAAT,CAAuB1B,KAAvB,EAA8BS,UAA9B,EAA0C;AACxCvB,QAAAA,SAAS,CAAE,GAAEE,IAAI,CAACC,SAAL,CAAeW,KAAf,CAAsB,EAA1B,CAAT;AACAf,QAAAA,KAAK,CAAE,gBAAee,KAAK,CAACK,WAAY,sCAAqCL,KAAK,CAAC+B,eAAN,IAAyB/B,KAAK,CAAC+B,eAAN,IAAyB/B,KAAK,CAACgC,YAAa,4BAA2BhC,KAAK,CAACgC,YAAN,GAAqBhC,KAAK,CAACiC,qBAAsB,mBAAkBjC,KAAK,CAACkC,YAAN,CAAmBjC,MAAO,MAAKD,KAAK,CAACkC,YAAa,EAA7R,CAAL;AAEAhD,QAAAA,SAAS,CAAE,gBAAeuB,UAAW,GAA5B,CAAT;AAEA,cAAM0B,YAAY,GAAGnC,KAAK,CAAC+B,eAAN,IAAyB/B,KAAK,CAAC+B,eAAN,IAAyB/B,KAAK,CAACgC,YAA7E;AACA,cAAMI,YAAY,GAAGD,YAAY,GAAGnC,KAAK,CAACgC,YAAN,GAAqBhC,KAAK,CAACiC,qBAA9B,GAAsD,CAAvF;AACA/C,QAAAA,SAAS,CAAE,iBAAgBkD,YAAa,EAA/B,CAAT;;AAEA,YAAIpC,KAAK,CAACK,WAAN,GAAoB,CAApB,IAAyB+B,YAAY,GAAG,CAAxC,IAA6CpC,KAAK,CAACkC,YAAN,CAAmBjC,MAAnB,GAA4B,CAA7E,EAAgF;AAC9E,gBAAMoC,sBAAsB,GAAGrC,KAAK,CAACkC,YAAN,CAAmBjC,MAAnB,GAA4B,CAA5B,GAAgC,cAAhC,GAAiD,EAAhF;AACA,gBAAMqC,aAAa,GAAG7B,UAAU,KAAK,EAAf,GAAoB4B,sBAApB,GAA6C5B,UAAnE;AACAvB,UAAAA,SAAS,CAAE,4BAA2BmD,sBAAuB,GAApD,CAAT;AACAnD,UAAAA,SAAS,CAAE,mBAAkBoD,aAAc,GAAlC,CAAT;AACArD,UAAAA,KAAK,CAAE,qBAAF,CAAL;AACA,iBAAO;AAACsD,YAAAA,MAAM,EAAE,KAAT;AAAgB9B,YAAAA,UAAU,EAAE6B;AAA5B,WAAP;AACD;;AAEDrD,QAAAA,KAAK,CAAE,oBAAF,CAAL;AACA,eAAO;AAACsD,UAAAA,MAAM,EAAE,IAAT;AAAe9B,UAAAA;AAAf,SAAP;AACD;AACF;;AAED,aAASE,cAAT,CAAwB;AAACZ,MAAAA,OAAD;AAAUG,MAAAA,aAAV;AAAyBtB,MAAAA,UAAzB;AAAqCc,MAAAA,OAAO,GAAG,EAA/C;AAAmDE,MAAAA,UAAU,GAAG,EAAhE;AAAoE4C,MAAAA,aAAa,GAAG,EAApF;AAAwFC,MAAAA,gBAAgB,GAAG,EAA3G;AAA+GC,MAAAA,WAAW,GAAG,CAA7H;AAAgIC,MAAAA,oBAAoB,GAAG,CAAvJ;AAA0JC,MAAAA,mBAAmB,GAAG;AAAhL,KAAxB,EAA4M;AAE1M;AACA;AACA;AACA;AACA;AAEA;AACA;AACA;AACA;AAEA,YAAM,CAACC,SAAD,IAAc9C,OAApB;AACA,YAAM+C,cAAc,GAAGD,SAAS,GAAGH,WAAW,GAAG,CAAjB,GAAqBA,WAArD,CAd0M,CAgB1M;AACA;AACA;AACA;;AAEA,UAAIG,SAAJ,EAAe;AAEb,YAAIE,qBAAqB,CAACrD,OAAO,CAACyB,MAAR,CAAevB,UAAf,CAAD,EAA6BiD,SAA7B,CAAzB,EAAkE;AAChE,gBAAM;AAACtD,YAAAA,MAAM,EAAEyD,eAAT;AAA0BC,YAAAA,EAAE,EAAEC;AAA9B,cAA6CL,SAAnD;AACA5D,UAAAA,KAAK,CAAE,qCAAoCiE,WAAY,KAAIJ,cAAe,IAAG5C,aAAc,GAAtF,CAAL;AACA,gBAAMiD,eAAe,GAAG7D,MAAM,CAACC,MAAD,EAASyD,eAAT,CAA9B;AACA,iBAAOI,qBAAqB,CAACD,eAAD,EAAkBD,WAAlB,EAA+BF,eAA/B,CAA5B;AACD;;AACD,eAAOrC,cAAc,CAAC;AAACZ,UAAAA,OAAO,EAAEA,OAAO,CAACsD,KAAR,CAAc,CAAd,CAAV;AAA4BnD,UAAAA,aAA5B;AAA2CtB,UAAAA,UAA3C;AAAuDc,UAAAA,OAAvD;AAAgE8C,UAAAA,aAAhE;AAA+EE,UAAAA,WAAW,EAAEI,cAA5F;AAA4GL,UAAAA,gBAA5G;AAA8HE,UAAAA,oBAAoB,EAAEA,oBAAoB,GAAG,CAA3K;AAA8KC,UAAAA;AAA9K,SAAD,CAArB;AACD;;AAED3D,MAAAA,KAAK,CAAE,mCAAkCyD,WAAY,IAAGxC,aAAc,WAAUsC,aAAa,CAACvC,MAAO,mBAAkB0C,oBAAqB,gCAA+B3D,gBAAgB,GAAI,GAAEyD,gBAAgB,CAACxC,MAAO,EAA9B,GAAmC,GAAE2C,mBAAoB,EAAE,oBAAjP,CAAL;AACA,aAAO;AAAClD,QAAAA,OAAO,EAAE8C,aAAV;AAAyB5C,QAAAA,UAAU,EAAEZ,gBAAgB,GAAGyD,gBAAH,GAAsB,EAA3E;AAA+E5C,QAAAA,cAAc,EAAE8C,oBAA/F;AAAqH7C,QAAAA,aAAa,EAAE8C;AAApI,OAAP;;AAEA,eAASQ,qBAAT,CAA+BD,eAA/B,EAAgDD,WAAhD,EAA6DF,eAA7D,EAA8E;AAC5E9D,QAAAA,SAAS,CAAE,8BAA6BgE,WAAY,KAAIJ,cAAe,IAAG5C,aAAc,MAAKd,IAAI,CAACC,SAAL,CAAe8D,eAAf,CAAgC,EAApH,CAAT;;AAEA,YAAIA,eAAe,CAAC7B,KAAhB,IAAyBtC,gBAA7B,EAA+C;AAC7CC,UAAAA,KAAK,CAAE,GAAEkE,eAAe,CAAC7B,KAAhB,GAAyB,UAAS4B,WAAY,KAAIJ,cAAe,IAAG5C,aAAc,eAAlF,GAAoG,UAASgD,WAAY,KAAIJ,cAAe,IAAG5C,aAAc,mBAAmB,EAApL,CAAL;AACAhB,UAAAA,SAAS,CAAE,aAAYE,IAAI,CAACC,SAAL,CAAe8D,eAAe,CAACG,QAA/B,CAAyC,eAAclE,IAAI,CAACC,SAAL,CAAe8D,eAAe,CAACI,QAA/B,CAAyC,EAA9G,CAAT;AAEA,gBAAM7C,WAAW,GAAG;AAClB8C,YAAAA,WAAW,EAAEL,eAAe,CAACK,WADX;AAElBX,YAAAA,SAAS,EAAE;AACTI,cAAAA,EAAE,EAAEC,WADK;AAET3D,cAAAA,MAAM,EAAEyD;AAFC;AAFO,WAApB;AAOA,gBAAMS,cAAc,GAAG;AACrBH,YAAAA,QAAQ,EAAEH,eAAe,CAACG,QADL;AAErBC,YAAAA,QAAQ,EAAEJ,eAAe,CAACI;AAFL,WAAvB;AAIA,gBAAMG,QAAQ,GAAG5E,cAAc,GAAG,EAAC,GAAG4B,WAAJ;AAAiB,eAAG+C;AAApB,WAAH,GAAyC,EAAC,GAAG/C;AAAJ,WAAxE;AAEAxB,UAAAA,SAAS,CAAE,GAAEE,IAAI,CAACC,SAAL,CAAeqE,QAAf,CAAyB,EAA7B,CAAT;AAEA,iBAAOC,iBAAiB,CAACR,eAAe,CAAC7B,KAAjB,EAAwBoC,QAAxB,CAAxB;AACD;;AAED,cAAME,sBAAsB,GAAGhB,mBAAmB,GAAG,CAArD;AACA1D,QAAAA,SAAS,CAAE,4CAA2C0E,sBAAuB,EAApE,CAAT;AAEA,eAAOjD,cAAc,CAAC;AAACZ,UAAAA,OAAO,EAAEA,OAAO,CAACsD,KAAR,CAAc,CAAd,CAAV;AAA4BnD,UAAAA,aAA5B;AAA2CtB,UAAAA,UAA3C;AAAuDc,UAAAA,OAAvD;AAAgE8C,UAAAA,aAAhE;AAA+EE,UAAAA,WAAW,EAAEI,cAA5F;AAA4GL,UAAAA,gBAA5G;AAA8HE,UAAAA,oBAA9H;AAAoJC,UAAAA,mBAAmB,EAAEgB;AAAzK,SAAD,CAArB;AACD;;AAED,eAASD,iBAAT,CAA2BE,OAA3B,EAAoCH,QAApC,EAA8C;AAC5C,cAAMI,gBAAgB,GAAGD,OAAO,GAAGrB,aAAa,CAACrB,MAAd,CAAqBuC,QAArB,CAAH,GAAoClB,aAApE;AACA,cAAMuB,mBAAmB,GAAGF,OAAO,GAAGpB,gBAAH,GAAsBA,gBAAgB,CAACtB,MAAjB,CAAwBuC,QAAxB,CAAzD;AAEAxE,QAAAA,SAAS,CAAE,yCAAwCQ,OAAO,CAACyB,MAAR,CAAe2C,gBAAf,EAAiC7D,MAAO,UAASrB,UAAW,GAAtG,CAAT,CAJ4C,CAM5C;;AACA,YAAII,gBAAJ,EAAsB;AACpBE,UAAAA,SAAS,CAAE,4CAA2CU,UAAU,CAACuB,MAAX,CAAkB4C,mBAAlB,EAAuC9D,MAAO,EAA3F,CAAT;AACD;;AAED,YAAIgB,eAAe,CAAC;AAACvB,UAAAA,OAAO,EAAEA,OAAO,CAACyB,MAAR,CAAe2C,gBAAf,CAAV;AAA4ClF,UAAAA;AAA5C,SAAD,CAAnB,EAA8E;AAC5EK,UAAAA,KAAK,CAAE,eAAcL,UAAW,gDAA+CkE,cAAe,yCAAwC5C,aAAa,GAAG4C,cAAe,EAAhK,CAAL;AACA,iBAAO;AAACpD,YAAAA,OAAO,EAAEoE,gBAAV;AAA4BlE,YAAAA,UAAU,EAAEZ,gBAAgB,GAAG+E,mBAAH,GAAyB,EAAjF;AAAqFlE,YAAAA,cAAc,EAAE8C,oBAArG;AAA2H7C,YAAAA,aAAa,EAAE8C;AAA1I,WAAP;AACD;;AAED,eAAOjC,cAAc,CAAC;AAACZ,UAAAA,OAAO,EAAEA,OAAO,CAACsD,KAAR,CAAc,CAAd,CAAV;AAA4BnD,UAAAA,aAA5B;AAA2CtB,UAAAA,UAA3C;AAAuDc,UAAAA,OAAvD;AAAgE8C,UAAAA,aAAa,EAAEsB,gBAA/E;AAAiGpB,UAAAA,WAAW,EAAEI,cAA9G;AAA8HL,UAAAA,gBAAgB,EAAEzD,gBAAgB,GAAG+E,mBAAH,GAAyB,EAAzL;AAA6LlE,UAAAA,cAAc,EAAE8C,oBAA7M;AAAmOC,UAAAA;AAAnO,SAAD,CAArB;AACD;;AAED,eAASG,qBAAT,CAA+BrD,OAA/B,EAAwCmD,SAAxC,EAAmD;AACjD5D,QAAAA,KAAK,CAAE,wBAAuB4D,SAAS,CAACI,EAAG,+BAA8BvD,OAAO,CAACO,MAAO,qBAAnF,CAAL;AACA,cAAM+D,cAAc,GAAGnB,SAAS,CAACI,EAAjC;AACA/D,QAAAA,SAAS,CAAE,mBAAkB8E,cAAe,EAAnC,CAAT;AACA,cAAMrC,MAAM,GAAGjC,OAAO,CAACuE,IAAR,CAAa,CAAC;AAACpB,UAAAA;AAAD,SAAD,KAAiBA,SAAS,CAACI,EAAV,KAAiBe,cAA/C,CAAf;AACA9E,QAAAA,SAAS,CAAE,WAAUyC,MAAO,EAAnB,CAAT;;AACA,YAAIA,MAAJ,EAAY;AACV1C,UAAAA,KAAK,CAAE,GAAE4D,SAAS,CAACI,EAAG,uBAAjB,CAAL;AACA,iBAAO,KAAP;AACD;;AACDhE,QAAAA,KAAK,CAAE,GAAE4D,SAAS,CAACI,EAAG,kCAAjB,CAAL;AACA,eAAO,IAAP;AACD;AACF;;AAED,aAAShC,eAAT,CAAyB;AAACvB,MAAAA,OAAD;AAAUd,MAAAA;AAAV,KAAzB,EAAgD;AAC9C,UAAIA,UAAU,IAAIc,OAAO,CAACO,MAAR,IAAkBrB,UAApC,EAAgD;AAC9CK,QAAAA,KAAK,CAAE,6CAA4CL,UAAW,0BAAzD,CAAL;AACA,eAAO,IAAP;AACD;AACF;AACF,GAnQD;AAoQD,C","sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport createSearchInterface, * as candidateSearch from './candidate-search';\nimport createDetectionInterface, * as matchDetection from './match-detection';\n\nexport {candidateSearch, matchDetection};\n\nexport default ({detection: detectionOptions, search: searchOptions, maxMatches = 1, maxCandidates = 25, returnStrategy = false, returnQuery = false, returnNonMatches = false}) => {\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:index');\n const debugData = debug.extend('data');\n\n debugData(`DetectionOptions: ${JSON.stringify(detectionOptions)}`);\n debugData(`SearchOptions: ${JSON.stringify(searchOptions)}`);\n debugData(`MaxMatches: ${JSON.stringify(maxMatches)}`);\n debugData(`MaxCandidates: ${JSON.stringify(maxCandidates)}`);\n debugData(`ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n debugData(`ReturnQuery: ${JSON.stringify(returnQuery)}`);\n debugData(`ReturnNonMatches: ${JSON.stringify(returnNonMatches)}`);\n\n const detect = createDetectionInterface(detectionOptions, returnStrategy);\n\n return record => {\n const search = createSearchInterface({...searchOptions, record, maxCandidates});\n return iterate({});\n\n // candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set\n // matches : candidates that have been detected as matches by current matcher job\n // nonMatches : candidates that have been detected as non-matches by current matcher job (only if returnNonMatches is 'true')\n // duplicateCount : amount of candidate records that were retrieved from the SRU but not handled further because they were already found in the matches/nonMatches\n\n // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)\n // state.query : current query (undefined if there was no queries left)\n // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)\n // state.queryCandidateCounter: amount of candidate records retrieved from SRU for matching for current query, including the current record set (undefined if there were no queries left)\n // state.queriesLeft : amount of queries left\n // state.queryCounter : sequence for current query\n // state.maxedQueries : queries that resulted in more than serverMaxResults hits\n\n async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0}) {\n debugData(`Starting next matcher iteration.`);\n const {records, ...state} = await search(initialState);\n\n debugData(`Current state: ${JSON.stringify(state)}, matches: ${matches.length}, candidateCount: ${candidateCount}, nonMatches: ${nonMatches.length}`);\n const recordSetSize = records.length;\n const newCandidateCount = candidateCount + recordSetSize;\n\n if (recordSetSize > 0) {\n return handleRecordSet();\n }\n\n if (state.queriesLeft > 0) {\n debug(`Empty record set ${state.searchCounter} for ${state.query}, but there are ${state.queriesLeft} queries left`);\n return iterate({initialState: state, matches, candidateCount: newCandidateCount, nonMatches, duplicateCount});\n }\n\n debug(`No (more) candidate records to check, no more queries left, matches: ${matches.length}`);\n return returnResult({matches, state, stopReason: '', nonMatches, candidateCount: newCandidateCount, duplicateCount});\n\n function handleRecordSet() {\n debug(`Checking record set of ${recordSetSize} candidate records for possible matches, found by ${state.searchCounter} search for ${state.query}`);\n\n const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches});\n const newDuplicateCount = duplicateCount + matchResult.duplicateCount;\n const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;\n const {newMatches, newNonMatches} = handleMatchResult(matchResult, matches, nonMatches);\n\n if (maxMatchesFound({matches: newMatches, maxMatches})) {\n return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount});\n }\n\n if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {\n return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount});\n }\n\n return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount});\n }\n\n function handleMatchResult(matchResult, matches, nonMatches) {\n debugData(`- Amount of new matches from record set: ${matchResult.matches.length}`);\n // eslint-disable-next-line functional/no-conditional-statement\n if (returnNonMatches) {\n debugData(`- Amount of new nonMatches from record set: ${matchResult.nonMatches.length}`);\n }\n\n const newMatches = matches.concat(returnQuery ? addQuery(matchResult.matches) : matchResult.matches);\n const newNonMatches = returnNonMatches ? nonMatches.concat(returnQuery ? addQuery(matchResult.nonMatches) : matchResult.nonMatches) : [];\n\n debugData(`- Total amount of matches: ${newMatches.length}`);\n // eslint-disable-next-line functional/no-conditional-statement\n if (returnNonMatches) {\n debugData(`- Total amount of nonMatches: ${newNonMatches.length}`);\n }\n\n return {newMatches, newNonMatches};\n }\n\n function addQuery(matches) {\n debugData(`Adding query ${state.query} to matches`);\n return matches.map((match) => ({...match, matchQuery: state.query}));\n }\n\n function maxCandidatesRetrieved(candidateCount, maxCandidates) {\n debugData(`Total amount of candidate records retrieved: ${newCandidateCount} (max: ${maxCandidates})`);\n if (maxCandidates && candidateCount >= maxCandidates) {\n debug(`Stopped matching because maximum number of candidate records ${candidateCount} / ${maxCandidates} have been retrieved`);\n return true;\n }\n }\n }\n\n // matches : array of matching candidate records\n // nonMatches : array of nonMatching candidate records (if returnNonMatches option is true, otherwise empty array)\n // - candidate.id\n // - candidate.record\n // - probability\n // - strategy (if returnStrategy option is true)\n // - treshold (if returnStrategy option is true)\n // - matchQuery (if returnQuery option is true)\n\n // we could have here also returnRecords/returnMatchRecords/returnNonMatchRecord options that could be turned false for not to return actual record data\n\n // matchStatus.status: boolean, true if matcher retrieved and handled all found candidate records, false if it did not\n // matchStatus.stopReason: string ('maxMatches','maxCandidates','maxedQueries',empty string/undefined), reason for stopping retrieving or handling the candidate records\n // - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)\n // - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record\n\n function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount}) {\n checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount});\n const matchStatus = getMatchState(state, stopReason);\n // add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered\n const result = returnNonMatches ? {matches, matchStatus, nonMatches} : {matches, matchStatus};\n debugData(`${JSON.stringify(result)}`);\n return result;\n\n // note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match\n\n function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount}) {\n const matchCount = matches.length;\n debugData(`Return nonMatches: ${returnNonMatches}`);\n const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;\n const totalHandled = matchCount + nonMatchCount + duplicateCount;\n debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}`);\n debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);\n if (totalHandled !== candidateCount) {\n debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);\n return;\n }\n return;\n }\n\n function getMatchState(state, stopReason) {\n debugData(`${JSON.stringify(state)}`);\n debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);\n\n debugData(`StopReason: <${stopReason}>`);\n\n const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;\n const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;\n debugData(`nonRetrieved: ${nonRetrieved}`);\n\n if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0) {\n const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : '';\n const newStopReason = stopReason === '' ? maxedQueriesStopReason : stopReason;\n debugData(`MaxedQueriesStopReason: <${maxedQueriesStopReason}>`);\n debugData(`NewStopReason: <${newStopReason}>`);\n debug(`Match status: false`);\n return {status: false, stopReason: newStopReason};\n }\n\n debug(`Match status: true`);\n return {status: true, stopReason};\n }\n }\n\n function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0}) {\n\n // recordSetSize : total amount of records in the current record set\n // recordCount : amount of records from the current record set that have been handled\n // maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped\n // recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results\n // recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)\n\n // records : non-handled records in the current record set\n // matches : found matches in the current matcher job\n // recordMatches : found matches in the current record set\n // recordNonMatches : found nonMatches in the current record set (only if returnNonMatches setting is true)\n\n const [candidate] = records;\n const newRecordCount = candidate ? recordCount + 1 : recordCount;\n\n // The matcher uses same matchDetection strategy for candidates from all candidate-searches -> matchDetection result for the same candidate is always same\n // Exceptions would happen if the candidate would have been updated in the database between candidate searches\n // Note that if returnNonMatches is false, matcher won't remember candidates that didn't match, so they will be matched again everytime they are retrieved by\n // different candidate search queries. Same candidate search query won't have duplicate records.\n\n if (candidate) {\n\n if (candidateNotInMatches(matches.concat(nonMatches), candidate)) {\n const {record: candidateRecord, id: candidateId} = candidate;\n debug(`Running matchDetection for record ${candidateId} (${newRecordCount}/${recordSetSize})`);\n const detectionResult = detect(record, candidateRecord);\n return handleDetectionResult(detectionResult, candidateId, candidateRecord);\n }\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount});\n }\n\n debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);\n return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount};\n\n function handleDetectionResult(detectionResult, candidateId, candidateRecord) {\n debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);\n\n if (detectionResult.match || returnNonMatches) {\n debug(`${detectionResult.match ? `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is a match!` : `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is NOT a match!`}`);\n debugData(`Strategy: ${JSON.stringify(detectionResult.strategy)}, Treshold: ${JSON.stringify(detectionResult.treshold)}`);\n\n const matchResult = {\n probability: detectionResult.probability,\n candidate: {\n id: candidateId,\n record: candidateRecord\n }\n };\n const strategyResult = {\n strategy: detectionResult.strategy,\n treshold: detectionResult.treshold\n };\n const newMatch = returnStrategy ? {...matchResult, ...strategyResult} : {...matchResult};\n\n debugData(`${JSON.stringify(newMatch)}`);\n\n return handleRecordMatch(detectionResult.match, newMatch);\n }\n\n const newRecordNonMatchCount = recordNonMatchCount + 1;\n debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);\n\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount});\n }\n\n function handleRecordMatch(isMatch, newMatch) {\n const newRecordMatches = isMatch ? recordMatches.concat(newMatch) : recordMatches;\n const newRecordNonMatches = isMatch ? recordNonMatches : recordNonMatches.concat(newMatch);\n\n debugData(`- Total matches after this detection: ${matches.concat(newRecordMatches).length} (max: ${maxMatches})`);\n\n // eslint-disable-next-line functional/no-conditional-statement\n if (returnNonMatches) {\n debugData(`- Total nonMatches after this detection: ${nonMatches.concat(newRecordNonMatches).length}`);\n }\n\n if (maxMatchesFound({matches: matches.concat(newRecordMatches), maxMatches})) {\n debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);\n return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount};\n }\n\n return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount});\n }\n\n function candidateNotInMatches(matches, candidate) {\n debug(`Checking that record ${candidate.id} is not already included in ${matches.length} matches/nonMatches`);\n const newCandidateId = candidate.id;\n debugData(`newCandidateId: ${newCandidateId}`);\n const result = matches.find(({candidate}) => candidate.id === newCandidateId);\n debugData(`Result: ${result}`);\n if (result) {\n debug(`${candidate.id} was already handled.`);\n return false;\n }\n debug(`${candidate.id} not found in matches/nonMatches`);\n return true;\n }\n }\n\n function maxMatchesFound({matches, maxMatches}) {\n if (maxMatches && matches.length >= maxMatches) {\n debug(`Stopping recordSet iteration: maxMatches (${maxMatches}) for matcher job found.`);\n return true;\n }\n }\n };\n};\n"],"file":"index.js"}
|
package/package.json
CHANGED
|
@@ -39,7 +39,7 @@ export class CandidateSearchError extends Error {}
|
|
|
39
39
|
// serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000
|
|
40
40
|
|
|
41
41
|
// eslint-disable-next-line max-statements
|
|
42
|
-
export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {
|
|
42
|
+
export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {
|
|
43
43
|
MarcRecord.setValidationOptions({subfieldValues: false});
|
|
44
44
|
|
|
45
45
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search');
|
|
@@ -49,11 +49,16 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxRe
|
|
|
49
49
|
debugData(`Url: ${url}`);
|
|
50
50
|
debugData(`MaxRecordsPerRequest ${maxRecordsPerRequest}`);
|
|
51
51
|
debugData(`ServerMaxResult: ${serverMaxResult}`);
|
|
52
|
+
debugData(`MaxCandidates: ${maxCandidates}`);
|
|
53
|
+
|
|
54
|
+
// Do not retrieve more candidates than defined in maxCandidates
|
|
55
|
+
const adjustedMaxRecordsPerRequest = maxRecordsPerRequest >= maxCandidates ? maxCandidates : maxRecordsPerRequest;
|
|
52
56
|
|
|
53
57
|
const inputRecordId = getRecordId(record);
|
|
54
58
|
const queryList = generateQueryList(record, searchSpec);
|
|
55
59
|
const client = createClient({
|
|
56
|
-
url,
|
|
60
|
+
url,
|
|
61
|
+
maxRecordsPerRequest: adjustedMaxRecordsPerRequest,
|
|
57
62
|
version: '2.0',
|
|
58
63
|
retrieveAll: false
|
|
59
64
|
});
|
|
@@ -81,6 +86,7 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxRe
|
|
|
81
86
|
const {records, nextOffset, total} = await retrieveRecords();
|
|
82
87
|
|
|
83
88
|
// If resultSetOffset === 1 this is the first search for the current query
|
|
89
|
+
debugData(`ResultSetOffset: ${resultSetOffset}`);
|
|
84
90
|
const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;
|
|
85
91
|
const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;
|
|
86
92
|
const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;
|
|
@@ -91,7 +97,7 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxRe
|
|
|
91
97
|
|
|
92
98
|
if (typeof nextOffset === 'number') {
|
|
93
99
|
debug(`Next search will be for query ${queryOffset} ${query}, starting from record ${nextOffset}`);
|
|
94
|
-
return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter:
|
|
100
|
+
return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
|
|
95
101
|
}
|
|
96
102
|
debug(`Query ${queryOffset} ${query} done.`);
|
|
97
103
|
debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);
|
|
@@ -143,14 +149,6 @@ export default ({record, searchSpec, url, maxRecordsPerRequest = 50, serverMaxRe
|
|
|
143
149
|
const foundRecordMarc = await MARCXML.from(foundRecord, {subfieldValues: false});
|
|
144
150
|
const foundRecordId = getRecordId(foundRecordMarc);
|
|
145
151
|
|
|
146
|
-
// This does not work and might cause problems:
|
|
147
|
-
// Record *should* match itself AND in REST the input record is given id 000000001 always
|
|
148
|
-
debug(`Checking record id's - this does not work ${inputRecordId} vs ${foundRecordId}`);
|
|
149
|
-
if (inputRecordId === foundRecordId) {
|
|
150
|
-
debug(`Input and candidate are the same record per 001. Discarding candidate`);
|
|
151
|
-
return;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
152
|
return {record: foundRecordMarc, id: foundRecordId};
|
|
155
153
|
} catch (err) {
|
|
156
154
|
throw new Error(`Failed converting record: ${err}, record: ${foundRecord}`);
|
|
@@ -46,7 +46,7 @@ describe('candidate-search', () => {
|
|
|
46
46
|
});
|
|
47
47
|
|
|
48
48
|
// eslint-disable-next-line max-statements
|
|
49
|
-
async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError, expectedSearchError, enabled = true}) {
|
|
49
|
+
async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError = false, expectedSearchError = false, enabled = true}) {
|
|
50
50
|
const url = 'http://foo.bar';
|
|
51
51
|
|
|
52
52
|
if (!enabled) {
|
|
@@ -64,7 +64,7 @@ describe('candidate-search', () => {
|
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
const search = createSearchInterface({...formatFactoryOptions(), url});
|
|
67
|
-
await iterate({searchOptions});
|
|
67
|
+
await iterate({searchOptions, expectedSearchError});
|
|
68
68
|
|
|
69
69
|
function formatFactoryOptions() {
|
|
70
70
|
debug(`Using factoryOptions: ${JSON.stringify(factoryOptions)}`);
|
|
@@ -77,7 +77,7 @@ describe('candidate-search', () => {
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
// eslint-disable-next-line max-statements
|
|
80
|
-
async function iterate({searchOptions, count = 1}) {
|
|
80
|
+
async function iterate({searchOptions, expectedSearchError, count = 1}) {
|
|
81
81
|
const expectedResults = getFixture(`expectedResults${count}.json`);
|
|
82
82
|
|
|
83
83
|
if (expectedSearchError) { // eslint-disable-line functional/no-conditional-statement
|
|
@@ -91,15 +91,10 @@ describe('candidate-search', () => {
|
|
|
91
91
|
}
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
if (results.records.length > 0) {
|
|
99
|
-
return iterate({
|
|
100
|
-
searchOptions: resultsToOptions(results),
|
|
101
|
-
count: count + 1
|
|
102
|
-
});
|
|
94
|
+
// eslint-disable-next-line functional/no-conditional-statement
|
|
95
|
+
if (!expectedSearchError) {
|
|
96
|
+
const results = await search(searchOptions);
|
|
97
|
+
expect(formatResults(results)).to.eql(expectedResults);
|
|
103
98
|
}
|
|
104
99
|
|
|
105
100
|
function formatResults(results) {
|
|
@@ -110,11 +105,6 @@ describe('candidate-search', () => {
|
|
|
110
105
|
};
|
|
111
106
|
}
|
|
112
107
|
|
|
113
|
-
function resultsToOptions(results) {
|
|
114
|
-
return Object.entries(results)
|
|
115
|
-
.filter(([k]) => k === 'records' === false) // If key is 'records' return false
|
|
116
|
-
.reduce((acc, [k, v]) => ({...acc, [k]: v}), {});
|
|
117
|
-
}
|
|
118
108
|
}
|
|
119
109
|
}
|
|
120
110
|
});
|
|
@@ -175,12 +175,15 @@ export function bibHostComponents(record) {
|
|
|
175
175
|
|
|
176
176
|
// SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to
|
|
177
177
|
// (probably) to *headings* index TIT
|
|
178
|
+
// - Aleph cannot currently handle headings searches starting with a boolean - in these cases use word search
|
|
179
|
+
|
|
178
180
|
// Headings index TIT drops articles etc. from the start of the title according to the filing indicator
|
|
179
181
|
// Currently filing indicator is not implemented - if the title starts with an article and the Melinda
|
|
180
182
|
// record is correctly catalogued using a filing indicator -> dc.title search won't match
|
|
181
183
|
|
|
182
184
|
export function bibTitle(record) {
|
|
183
185
|
const title = getTitle();
|
|
186
|
+
const booleanStartWords = ['and', 'or', 'nor', 'not'];
|
|
184
187
|
|
|
185
188
|
if (title) {
|
|
186
189
|
const formatted = title
|
|
@@ -191,8 +194,10 @@ export function bibTitle(record) {
|
|
|
191
194
|
.slice(0, 30)
|
|
192
195
|
.trim();
|
|
193
196
|
|
|
197
|
+
// use word search for titles starting with a boolean
|
|
198
|
+
const useWordSearch = booleanStartWords.some(word => formatted.toLowerCase().startsWith(word));
|
|
194
199
|
// Prevent too many matches by having a minimum length requirement
|
|
195
|
-
return formatted.length >= 5 ? [`dc.title="
|
|
200
|
+
return formatted.length >= 5 ? [`dc.title="${useWordSearch ? '' : '^'}${formatted}*"`] : [];
|
|
196
201
|
}
|
|
197
202
|
|
|
198
203
|
return [];
|
package/src/index.js
CHANGED
|
@@ -47,7 +47,7 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
47
47
|
const detect = createDetectionInterface(detectionOptions, returnStrategy);
|
|
48
48
|
|
|
49
49
|
return record => {
|
|
50
|
-
const search = createSearchInterface({...searchOptions, record});
|
|
50
|
+
const search = createSearchInterface({...searchOptions, record, maxCandidates});
|
|
51
51
|
return iterate({});
|
|
52
52
|
|
|
53
53
|
// candidateCount : amount of candidate records retrived from SRU for matching, NOT including current record set
|
|
@@ -63,7 +63,7 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
63
63
|
// state.queryCounter : sequence for current query
|
|
64
64
|
// state.maxedQueries : queries that resulted in more than serverMaxResults hits
|
|
65
65
|
|
|
66
|
-
async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0}) {
|
|
66
|
+
async function iterate({initialState = {}, matches = [], candidateCount = 0, nonMatches = [], duplicateCount = 0, nonMatchCount = 0}) {
|
|
67
67
|
debugData(`Starting next matcher iteration.`);
|
|
68
68
|
const {records, ...state} = await search(initialState);
|
|
69
69
|
|
|
@@ -88,17 +88,18 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
88
88
|
|
|
89
89
|
const matchResult = iterateRecords({records, recordSetSize, maxMatches, matches, nonMatches});
|
|
90
90
|
const newDuplicateCount = duplicateCount + matchResult.duplicateCount;
|
|
91
|
+
const newNonMatchCount = nonMatchCount + matchResult.nonMatchCount;
|
|
91
92
|
const {newMatches, newNonMatches} = handleMatchResult(matchResult, matches, nonMatches);
|
|
92
93
|
|
|
93
94
|
if (maxMatchesFound({matches: newMatches, maxMatches})) {
|
|
94
|
-
return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount});
|
|
95
|
+
return returnResult({matches: newMatches, state, stopReason: 'maxMatches', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount});
|
|
95
96
|
}
|
|
96
97
|
|
|
97
98
|
if (maxCandidatesRetrieved(newCandidateCount, maxCandidates)) {
|
|
98
|
-
return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount});
|
|
99
|
+
return returnResult({matches: newMatches, state, stopReason: 'maxCandidates', nonMatches: newNonMatches, duplicateCount: newDuplicateCount, candidateCount: newCandidateCount, nonMatchCount: newNonMatchCount});
|
|
99
100
|
}
|
|
100
101
|
|
|
101
|
-
return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount});
|
|
102
|
+
return iterate({initialState: state, matches: newMatches, candidateCount: newCandidateCount, nonMatches: newNonMatches, duplicateCount: newDuplicateCount, nonMatchCount: newNonMatchCount});
|
|
102
103
|
}
|
|
103
104
|
|
|
104
105
|
function handleMatchResult(matchResult, matches, nonMatches) {
|
|
@@ -150,22 +151,22 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
150
151
|
// - only one stopReason is returned (if there would be several possible stopReasons, stopReason is picked in the above order)
|
|
151
152
|
// - currently stopReason can be non-empty also in cases where status is true, if matcher hit the stop reason when handling the last available candidate record
|
|
152
153
|
|
|
153
|
-
function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount}) {
|
|
154
|
-
checkCounts(matches, nonMatches, candidateCount, duplicateCount);
|
|
154
|
+
function returnResult({matches, state, stopReason, nonMatches, duplicateCount, candidateCount, nonMatchCount}) {
|
|
155
|
+
checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount});
|
|
155
156
|
const matchStatus = getMatchState(state, stopReason);
|
|
156
157
|
// add nonMatches to result only if returnNonMatches is 'true', otherwise nonMatches have not been gathered
|
|
157
158
|
const result = returnNonMatches ? {matches, matchStatus, nonMatches} : {matches, matchStatus};
|
|
158
159
|
debugData(`${JSON.stringify(result)}`);
|
|
159
160
|
return result;
|
|
160
161
|
|
|
161
|
-
// we could have a nonMatchCount even if nonMatches won't be returned so that checkCounts would make sense
|
|
162
162
|
// note that in cases where the matching has been stopped because of maxMatches checkCounts won't (in most cases) match
|
|
163
163
|
|
|
164
|
-
function checkCounts(matches, nonMatches, candidateCount, duplicateCount) {
|
|
164
|
+
function checkCounts({matches, nonMatches, candidateCount, duplicateCount, nonMatchCount}) {
|
|
165
165
|
const matchCount = matches.length;
|
|
166
|
-
|
|
166
|
+
debugData(`Return nonMatches: ${returnNonMatches}`);
|
|
167
|
+
const chosenNonMatchCount = returnNonMatches ? nonMatches.length : nonMatchCount;
|
|
167
168
|
const totalHandled = matchCount + nonMatchCount + duplicateCount;
|
|
168
|
-
debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${
|
|
169
|
+
debug(`candidateCount: ${candidateCount}, matches: ${matchCount}, nonMatches: ${chosenNonMatchCount}, duplicateCount: ${duplicateCount}`);
|
|
169
170
|
debug(`We got result for ${totalHandled} / ${candidateCount} retrieved candidates`);
|
|
170
171
|
if (totalHandled !== candidateCount) {
|
|
171
172
|
debug(`WARNING: Missing results for ${candidateCount - totalHandled} candidates`);
|
|
@@ -175,10 +176,14 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
175
176
|
}
|
|
176
177
|
|
|
177
178
|
function getMatchState(state, stopReason) {
|
|
178
|
-
|
|
179
|
+
debugData(`${JSON.stringify(state)}`);
|
|
180
|
+
debug(`Queries left ${state.queriesLeft}, Searches for current query left: ${state.resultSetOffset && state.resultSetOffset <= state.totalRecords}, non-retrieved records: ${state.totalRecords - state.queryCandidateCounter}, maxedQueries (${state.maxedQueries.length}): ${state.maxedQueries}`);
|
|
181
|
+
|
|
179
182
|
debugData(`StopReason: <${stopReason}>`);
|
|
180
183
|
|
|
181
|
-
const
|
|
184
|
+
const searchesLeft = state.resultSetOffset && state.resultSetOffset <= state.totalRecords;
|
|
185
|
+
const nonRetrieved = searchesLeft ? state.totalRecords - state.queryCandidateCounter : 0;
|
|
186
|
+
debugData(`nonRetrieved: ${nonRetrieved}`);
|
|
182
187
|
|
|
183
188
|
if (state.queriesLeft > 0 || nonRetrieved > 0 || state.maxedQueries.length > 0) {
|
|
184
189
|
const maxedQueriesStopReason = state.maxedQueries.length > 0 ? 'maxedQueries' : '';
|
|
@@ -194,12 +199,13 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
194
199
|
}
|
|
195
200
|
}
|
|
196
201
|
|
|
197
|
-
function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0}) {
|
|
202
|
+
function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0}) {
|
|
198
203
|
|
|
199
204
|
// recordSetSize : total amount of records in the current record set
|
|
200
205
|
// recordCount : amount of records from the current record set that have been handled
|
|
201
206
|
// maxMatches : setting for maximum amount found by current matcher job before the matcher job is stopped
|
|
202
207
|
// recordDuplicateCount : amount of records from the current record set that are already included in matches/nonMatches results
|
|
208
|
+
// recordNonMatchCount: amount of records from the current record set that are nonMatches (only is returnNonMatches setting is false)
|
|
203
209
|
|
|
204
210
|
// records : non-handled records in the current record set
|
|
205
211
|
// matches : found matches in the current matcher job
|
|
@@ -222,11 +228,11 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
222
228
|
const detectionResult = detect(record, candidateRecord);
|
|
223
229
|
return handleDetectionResult(detectionResult, candidateId, candidateRecord);
|
|
224
230
|
}
|
|
225
|
-
return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1});
|
|
231
|
+
return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount: recordDuplicateCount + 1, recordNonMatchCount});
|
|
226
232
|
}
|
|
227
233
|
|
|
228
|
-
debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled${returnNonMatches ?
|
|
229
|
-
return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount};
|
|
234
|
+
debug(`No more candidates, record set (${recordCount}/${recordSetSize}) done, ${recordMatches.length} matches found, ${recordDuplicateCount} candidates already handled, ${returnNonMatches ? `${recordNonMatches.length}` : `${recordNonMatchCount}`} nonMatches found.`);
|
|
235
|
+
return {matches: recordMatches, nonMatches: returnNonMatches ? recordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount};
|
|
230
236
|
|
|
231
237
|
function handleDetectionResult(detectionResult, candidateId, candidateRecord) {
|
|
232
238
|
debugData(`MatchDetection results for ${candidateId} (${newRecordCount}/${recordSetSize}): ${JSON.stringify(detectionResult)}`);
|
|
@@ -252,7 +258,11 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
252
258
|
|
|
253
259
|
return handleRecordMatch(detectionResult.match, newMatch);
|
|
254
260
|
}
|
|
255
|
-
|
|
261
|
+
|
|
262
|
+
const newRecordNonMatchCount = recordNonMatchCount + 1;
|
|
263
|
+
debugData(`- Total nonMatches after this detection: ${newRecordNonMatchCount}`);
|
|
264
|
+
|
|
265
|
+
return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches, recordCount: newRecordCount, recordNonMatches, recordDuplicateCount, recordNonMatchCount: newRecordNonMatchCount});
|
|
256
266
|
}
|
|
257
267
|
|
|
258
268
|
function handleRecordMatch(isMatch, newMatch) {
|
|
@@ -268,10 +278,10 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
268
278
|
|
|
269
279
|
if (maxMatchesFound({matches: matches.concat(newRecordMatches), maxMatches})) {
|
|
270
280
|
debug(`MaxMatches (${maxMatches}) reached, handled candidates in record set: ${newRecordCount} non-handled candidates in record set ${recordSetSize - newRecordCount}`);
|
|
271
|
-
return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount};
|
|
281
|
+
return {matches: newRecordMatches, nonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, nonMatchCount: recordNonMatchCount};
|
|
272
282
|
}
|
|
273
283
|
|
|
274
|
-
return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount});
|
|
284
|
+
return iterateRecords({records: records.slice(1), recordSetSize, maxMatches, matches, recordMatches: newRecordMatches, recordCount: newRecordCount, recordNonMatches: returnNonMatches ? newRecordNonMatches : [], duplicateCount: recordDuplicateCount, recordNonMatchCount});
|
|
275
285
|
}
|
|
276
286
|
|
|
277
287
|
function candidateNotInMatches(matches, candidate) {
|