@natlibfi/melinda-record-matching 2.0.1-alpha.1 → 2.0.7-alpha.1

package/.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# With this line @NatLibFi/melinda-js-lead owns any files in this repository
+* @NatLibFi/melinda-js-lead

package/.github/dependabot.yml

@@ -0,0 +1,40 @@
+# NatLibFi/Melinda maintenance strategy
+# https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+      time: "06:30"
+      timezone: "Europe/Helsinki"
+    target-branch: "dependencies"
+
+  # Minor updates to npm production dependencies daily
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"
+      time: "06:45"
+      timezone: "Europe/Helsinki"
+    versioning-strategy: lockfile-only
+    allow:
+      - dependency-type: "production"
+    target-branch: "dependencies"
+
+  # Major updates to npm dependencies weekly @tuesday
+  # Not possible yet https://github.com/dependabot/dependabot-core/issues/1778
+  # - package-ecosystem: "npm"
+  #   directory: "/"
+  #   schedule:
+  #     interval: "weekly"
+  #     day: "tuesday"
+  #     time: "07:00"
+  #     timezone: "Europe/Helsinki"
+  #   versioning-strategy: increase-if-necessary
+  #   labels:
+  #     - "npm major dependencies"
+  #   reviewers:
+  #     - "natlibfi/melinda-js-lead"

package/.github/workflows/melinda-node-tests.yml

@@ -0,0 +1,70 @@
+# Melinda node tests
+
+name: Melinda node tests
+
+on: push
+
+jobs:
+  build-node-versions:
+    name: Node version matrix
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [16.x, 18.x, 20.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+
+    steps:
+    - name: Checkout the code
+      uses: actions/checkout@v3
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+      env:
+        NPM_CONFIG_IGNORE_SCRIPTS: true
+    - run: npm audit --package-lock-only --production --audit-level=high
+    - run: npm ci
+    - run: npm test
+    - run: npm run build --if-present
+
+  njsscan:
+    name: Njsscan check
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout the code
+      uses: actions/checkout@v3
+    - name: nodejsscan scan
+      id: njsscan
+      uses: ajinabraham/njsscan-action@master
+      with:
+        args: '.'
+
+  license-scan:
+    name: License compliance check
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: mikaelvesavuori/license-compliance-action@v1.0.2
+        with:
+          exclude_pattern: /^@natlibfi/
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-node-versions, njsscan]
+    if: contains(github.ref, 'refs/tags/')
+
+    steps:
+      - uses: actions/checkout@v3
+      # Setup .npmrc file to publish to npm
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+          registry-url: 'https://registry.npmjs.org'
+      - run: npm ci
+      - run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.MELINDA_RECORD_MATCHING_NPM_TOKEN }}

package/README.md

@@ -2,6 +2,6 @@
 
 ## License and copyright
 
-Copyright (c) 2020 **University Of Helsinki (The National Library Of Finland)**
+Copyright (c) 2020-2023 **University Of Helsinki (The National Library Of Finland)**
 
 This project's source code is licensed under the terms of **GNU Lesser General Public License Version 3** or any later version.

package/dist/candidate-search/candidate-search-utils.js

@@ -4,11 +4,8 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.toQueries = toQueries;
-
 var _debug = _interopRequireDefault(require("debug"));
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -36,23 +33,23 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 * for the JavaScript code in this file.
 *
 */
+
 function toQueries(identifiers, queryString) {
   const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:toQueries');
   const debugData = debug.extend('data');
-  const quotedIdentifiers = identifiers.map(identifier => identifier.match(/\//u) ? `"${identifier}"` : `${identifier}`); // Aleph supports only two queries with or -operator (This is not actually true)
+  const quotedIdentifiers = identifiers.map(identifier => identifier.match(/\//u) ? `"${identifier}"` : `${identifier}`);
 
+  // Aleph supports only two queries with or -operator (This is not actually true)
   const pairs = toPairs(quotedIdentifiers);
   const queries = pairs.map(([a, b]) => b ? `${queryString}=${a} or ${queryString}=${b}` : `${queryString}=${a}`);
   debugData(`Pairs (${pairs.length}): ${JSON.stringify(pairs)}`);
   debugData(`Queries (${queries.length}): ${JSON.stringify(queries)}`);
   return queries;
 }
-
 function toPairs(array) {
   if (array.length === 0) {
     return [];
   }
-
   return [array.slice(0, 2)].concat(toPairs(array.slice(2), 2));
 }
 //# sourceMappingURL=candidate-search-utils.js.map

package/dist/candidate-search/candidate-search-utils.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/candidate-search/candidate-search-utils.js"],"names":["toQueries","identifiers","queryString","debug","debugData","extend","quotedIdentifiers","map","identifier","match","pairs","toPairs","queries","a","b","length","JSON","stringify","array","slice","concat"],"mappings":";;;;;;;AA4BA;;;;AA5BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAIO,SAASA,SAAT,CAAmBC,WAAnB,EAAgCC,WAAhC,EAA6C;AAElD,QAAMC,KAAK,GAAG,oBAAkB,6CAAlB,CAAd;AACA,QAAMC,SAAS,GAAGD,KAAK,CAACE,MAAN,CAAa,MAAb,CAAlB;AAEA,QAAMC,iBAAiB,GAAGL,WAAW,CAACM,GAAZ,CAAgBC,UAAU,IAAIA,UAAU,CAACC,KAAX,CAAiB,KAAjB,IAA2B,IAAGD,UAAW,GAAzC,GAA+C,GAAEA,UAAW,EAA1F,CAA1B,CALkD,CAOlD;;AACA,QAAME,KAAK,GAAGC,OAAO,CAACL,iBAAD,CAArB;AACA,QAAMM,OAAO,GAAGF,KAAK,CAACH,GAAN,CAAU,CAAC,CAACM,CAAD,EAAIC,CAAJ,CAAD,KAAYA,CAAC,GAAI,GAAEZ,WAAY,IAAGW,CAAE,OAAMX,WAAY,IAAGY,CAAE,EAA9C,GAAmD,GAAEZ,WAAY,IAAGW,CAAE,EAA7F,CAAhB;AAEAT,EAAAA,SAAS,CAAE,UAASM,KAAK,CAACK,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAeP,KAAf,CAAsB,EAAnD,CAAT;AACAN,EAAAA,SAAS,CAAE,YAAWQ,OAAO,CAACG,MAAO,MAAKC,IAAI,CAACC,SAAL,CAAeL,OAAf,CAAwB,EAAzD,CAAT;AAEA,SAAOA,OAAP;AACD;;AAED,SAASD,OAAT,CAAiBO,KAAjB,EAAwB;AACtB,MAAIA,KAAK,CAACH,MAAN,KAAiB,CAArB,EAAwB;AACtB,WAAO,EAAP;AACD;;AACD,SAAO,CAACG,KAAK,CAACC,KAAN,CAAY,CAAZ,EAAe,CAAf,CAAD,EAAoBC,MAApB,CAA2BT,OAAO,CAACO,KAAK,CAACC,KAAN,CAAY,CAAZ,CAAD,EAAiB,CAAjB,CAAlC,CAAP;AACD","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\n\nexport function toQueries(identifiers, queryString) {\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:toQueries');\n  const debugData = debug.extend('data');\n\n  const quotedIdentifiers = identifiers.map(identifier => identifier.match(/\\//u) ? `\"${identifier}\"` : `${identifier}`);\n\n  // Aleph supports only two queries with or -operator (This is not actually true)\n  const pairs = toPairs(quotedIdentifiers);\n  const queries = pairs.map(([a, b]) => b ? `${queryString}=${a} or ${queryString}=${b}` : `${queryString}=${a}`);\n\n  debugData(`Pairs (${pairs.length}): ${JSON.stringify(pairs)}`);\n  debugData(`Queries (${queries.length}): ${JSON.stringify(queries)}`);\n\n  return queries;\n}\n\nfunction toPairs(array) {\n  if (array.length === 0) {\n    return [];\n  }\n  return [array.slice(0, 2)].concat(toPairs(array.slice(2), 2));\n}\n"],"file":"candidate-search-utils.js"}
+{"version":3,"file":"candidate-search-utils.js","names":["_debug","_interopRequireDefault","require","obj","__esModule","default","toQueries","identifiers","queryString","debug","createDebugLogger","debugData","extend","quotedIdentifiers","map","identifier","match","pairs","toPairs","queries","a","b","length","JSON","stringify","array","slice","concat"],"sources":["../../src/candidate-search/candidate-search-utils.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\n\nexport function toQueries(identifiers, queryString) {\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:toQueries');\n  const debugData = debug.extend('data');\n\n  const quotedIdentifiers = identifiers.map(identifier => identifier.match(/\\//u) ? `\"${identifier}\"` : `${identifier}`);\n\n  // Aleph supports only two queries with or -operator (This is not actually true)\n  const pairs = toPairs(quotedIdentifiers);\n  const queries = pairs.map(([a, b]) => b ? `${queryString}=${a} or ${queryString}=${b}` : `${queryString}=${a}`);\n\n  debugData(`Pairs (${pairs.length}): ${JSON.stringify(pairs)}`);\n  debugData(`Queries (${queries.length}): ${JSON.stringify(queries)}`);\n\n  return queries;\n}\n\nfunction toPairs(array) {\n  if (array.length === 0) {\n    return [];\n  }\n  return [array.slice(0, 2)].concat(toPairs(array.slice(2), 2));\n}\n"],"mappings":";;;;;;AA4BA,IAAAA,MAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAsC,SAAAD,uBAAAE,GAAA,WAAAA,GAAA,IAAAA,GAAA,CAAAC,UAAA,GAAAD,GAAA,KAAAE,OAAA,EAAAF,GAAA;AA5BtC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAIO,SAASG,SAASA,CAACC,WAAW,EAAEC,WAAW,EAAE;EAElD,MAAMC,KAAK,GAAG,IAAAC,cAAiB,EAAC,6CAA6C,CAAC;EAC9E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtC,MAAMC,iBAAiB,GAAGN,WAAW,CAACO,GAAG,CAACC,UAAU,IAAIA,UAAU,CAACC,KAAK,CAAC,KAAK,CAAC,GAAI,IAAGD,UAAW,GAAE,GAAI,GAAEA,UAAW,EAAC,CAAC;;EAEtH;EACA,MAAME,KAAK,GAAGC,OAAO,CAACL,iBAAiB,CAAC;EACxC,MAAMM,OAAO,GAAGF,KAAK,CAACH,GAAG,CAAC,CAAC,CAACM,CAAC,EAAEC,CAAC,CAAC,KAAKA,CAAC,GAAI,GAAEb,WAAY,IAAGY,CAAE,OAAMZ,WAAY,IAAGa,CAAE,EAAC,GAAI,GAAEb,WAAY,IAAGY,CAAE,EAAC,CAAC;EAE/GT,SAAS,CAAE,UAASM,KAAK,CAACK,MAAO,MAAKC,IAAI,CAACC,SAAS,CAACP,KAAK,CAAE,EAAC,CAAC;EAC9DN,SAAS,CAAE,YAAWQ,OAAO,CAACG,MAAO,MAAKC,IAAI,CAACC,SAAS,CAACL,OAAO,CAAE,EAAC,CAAC;EAEpE,OAAOA,OAAO;AAChB;AAEA,SAASD,OAAOA,CAACO,KAAK,EAAE;EACtB,IAAIA,KAAK,CAACH,MAAM,KAAK,CAAC,EAAE;IACtB,OAAO,EAAE;EACX;EACA,OAAO,CAACG,KAAK,CAACC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAACC,MAAM,CAACT,OAAO,CAACO,KAAK,CAACC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/D"}

package/dist/candidate-search/index.js

@@ -10,30 +10,22 @@ Object.defineProperty(exports, "searchTypes", {
     return _queryList.searchTypes;
   }
 });
-
 var _debug = _interopRequireDefault(require("debug"));
-
 var _sruClient = _interopRequireWildcard(require("@natlibfi/sru-client"));
-
 var _marcRecord = require("@natlibfi/marc-record");
-
 var _marcRecordSerializers = require("@natlibfi/marc-record-serializers");
-
 var _queryList = _interopRequireWildcard(require("./query-list"));
-
+var _melindaCommons = require("@natlibfi/melinda-commons");
 function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
-
 function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -54,136 +46,209 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 * for the JavaScript code in this file.
 *
 */
+
 class CandidateSearchError extends Error {}
 
+// serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000
 exports.CandidateSearchError = CandidateSearchError;
-
 var _default = ({
   record,
   searchSpec,
   url,
-  maxRecordsPerRequest = 50
+  maxCandidates,
+  maxRecordsPerRequest = 50,
+  serverMaxResult = 20000
 }) => {
   _marcRecord.MarcRecord.setValidationOptions({
     subfieldValues: false
   });
-
   const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:candidate-search');
+  const debugData = debug.extend('data');
+  debugData(`SearchSpec: ${JSON.stringify(searchSpec)}`);
+  debugData(`Url: ${url}`);
+  debugData(`MaxRecordsPerRequest ${maxRecordsPerRequest}`);
+  debugData(`ServerMaxResult: ${serverMaxResult}`);
+  debugData(`MaxCandidates: ${maxCandidates}`);
+
+  // Do not retrieve more candidates than defined in maxCandidates
+  const adjustedMaxRecordsPerRequest = maxRecordsPerRequest >= maxCandidates ? maxCandidates : maxRecordsPerRequest;
   const inputRecordId = getRecordId(record);
   const queryList = (0, _queryList.default)(record, searchSpec);
   const client = (0, _sruClient.default)({
     url,
-    maxRecordsPerRequest,
+    maxRecordsPerRequest: adjustedMaxRecordsPerRequest,
     version: '2.0',
     retrieveAll: false
   });
   debug(`Searching matches for ${inputRecordId}`);
   debug(`Generated queryList ${JSON.stringify(queryList)}`);
 
+  // if generateQueryList errored we should throw 422
+
   if (queryList.length === 0) {
-    // eslint-disable-line functional/no-conditional-statement
     throw new CandidateSearchError(`Generated query list contains no queries`);
-  } // eslint-disable-next-line max-statements
+  }
 
+  // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
+  // state.query : current query (undefined if there was no queries left)
+  // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
+  // state.queryCandidateCounter: amount of candidates (records+failures) retrieved from SRU for matching for current query, including the current record+failure set (undefined if there were no queries left)
+  // state.queriesLeft : amount of queries left
+  // state.queryCounter : sequence for current query
+  // state.maxedQueries : queries that resulted in more than serverMaxResults hits
 
   return async ({
     queryOffset = 0,
-    resultSetOffset = 1
+    resultSetOffset = 1,
+    totalRecords = 0,
+    searchCounter = 0,
+    queryCandidateCounter = 0,
+    queryCounter = 0,
+    maxedQueries = []
   }) => {
     const query = queryList[queryOffset];
-
     if (query) {
       const {
         records,
-        nextOffset
+        failures,
+        nextOffset,
+        total
       } = await retrieveRecords();
 
+      // If resultSetOffset === 1 this is the first search for the current query
+      debugData(`ResultSetOffset: ${resultSetOffset}`);
+      const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;
+      const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;
+      const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;
+      const newQueryCandidateCounter = resultSetOffset === 1 ? records.length + failures.length : queryCandidateCounter + records.length + failures.length;
+      const maxedQuery = resultSetOffset === 1 ? checkMaxedQuery(query, total, serverMaxResult) : undefined;
+      const newMaxedQueries = maxedQuery ? maxedQueries.concat(maxedQuery) : maxedQueries;
       if (typeof nextOffset === 'number') {
-        debug(`Running next search for query ${queryOffset} ${query}`);
+        debug(`Next search will be for query ${queryOffset} ${query}, starting from record ${nextOffset}`);
         return {
           records,
+          failures,
           queryOffset,
           resultSetOffset: nextOffset,
-          queriesLeft: queryList.length - (queryOffset + 1)
+          queriesLeft: queryList.length - (queryOffset + 1),
+          totalRecords: newTotalRecords,
+          query,
+          searchCounter: newSearchCounter,
+          queryCandidateCounter: newQueryCandidateCounter,
+          queryCounter: newQueryCounter,
+          maxedQueries: newMaxedQueries
         };
       }
-
-      debug(`Query ${queryOffset} ${query} done, moving to next query. (${queryList.length - (queryOffset + 1)} queries left)`);
+      debug(`Query ${queryOffset} ${query} done.`);
+      debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);
       return {
         records,
+        failures,
         queryOffset: queryOffset + 1,
-        queriesLeft: queryList.length - (queryOffset + 1)
+        queriesLeft: queryList.length - (queryOffset + 1),
+        totalRecords: newTotalRecords,
+        query,
+        searchCounter: newSearchCounter,
+        queryCandidateCounter: newQueryCandidateCounter,
+        queryCounter: newQueryCounter,
+        maxedQueries: newMaxedQueries
       };
     }
-
     debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);
     return {
-      records: []
+      records: [],
+      failures: [],
+      queriesLeft: 0,
+      queryCounter,
+      maxedQueries
     };
-
     function retrieveRecords() {
       return new Promise((resolve, reject) => {
         const promises = [];
+        // eslint-disable-next-line functional/no-let
+        let totalRecords = 0;
         debug(`Searching for candidates with query: ${query} (Offset ${resultSetOffset})`);
         client.searchRetrieve(query, {
           startRecord: resultSetOffset
         }).on('error', err => {
-          // eslint-disable-next-line functional/no-conditional-statement
+          // eslint-disable-next-line functional/no-conditional-statements
           if (err instanceof _sruClient.SruSearchError) {
             debug(`SRU SruSearchError for query: ${query}: ${err}`);
             reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));
           }
-
           debug(`SRU error for query: ${query}: ${err}`);
           reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));
+        }).on('total', total => {
+          debug(`Got total: ${total}`);
+          totalRecords += total;
         }).on('end', async nextOffset => {
           try {
-            const records = await Promise.all(promises);
-            const filtered = records.filter(r => r);
-            debug(`Found ${filtered.length} candidates`);
+            const recordPromises = await Promise.allSettled(promises);
+            debugData(`All recordPromises: ${JSON.stringify(recordPromises)}`);
+            const filtered = recordPromises.filter(r => r.status === 'fulfilled').map(r => r.value);
+            const failures = recordPromises.filter(r => r.status === 'rejected').map(r => ({
+              status: r.reason.status,
+              payload: r.reason.payload
+            }));
+            debug(`Found ${recordPromises.length} records`);
+            debug(`Found ${filtered.length} convertable candidates`);
+            debug(`Found ${failures.length} NON-convertable candidates`);
+            debugData(`Converted: ${JSON.stringify(filtered)}.`);
+            debugData(`Not converted: ${JSON.stringify(failures)}.`);
             resolve({
               nextOffset,
-              records: filtered
+              records: filtered,
+              failures,
+              total: totalRecords
             });
           } catch (err) {
+            debug(`Error caught on END`);
             reject(err);
           }
-        }).on('record', foundRecord => {
+        }).on('record', recordXML => {
           promises.push(handleRecord()); // eslint-disable-line functional/immutable-data
 
           async function handleRecord() {
             try {
-              const foundRecordMarc = await _marcRecordSerializers.MARCXML.from(foundRecord, {
+              const recordMarc = await _marcRecordSerializers.MARCXML.from(recordXML, {
                 subfieldValues: false
               });
-              const foundRecordId = getRecordId(foundRecordMarc); // This does not work and might cause problems:
-              // Record *should* match itself AND in REST the input record is given id 000000001 always
-
-              debug(`Checking ${inputRecordId} vs ${foundRecordId}`);
-
-              if (inputRecordId === foundRecordId) {
-                debug(`Input and candidate are the same record per 001. Discarding candidate`);
-                return;
-              }
-
+              const recordId = getRecordId(recordMarc);
               return {
-                record: foundRecordMarc,
-                id: foundRecordId
+                record: recordMarc,
+                id: recordId
               };
             } catch (err) {
-              throw new Error(`Failed converting record: ${err}, record: ${foundRecord}`);
+              // What should this do?
+              const idFromXML = getRecordIdFromXML(recordXML);
+              debugData(`Failed converting record: ${err.message}, id: ${idFromXML}, data: ${recordXML}`);
+              //return {message: `Failed converting record: ${err.message}`, id: idFromXML, data: recordXML};
+              throw new _melindaCommons.Error(422, {
+                message: `Failed converting record: ${err.message}`,
+                id: idFromXML || '000000000',
+                data: recordXML
+              });
             }
           }
         });
       });
     }
   };
-
+  function checkMaxedQuery(query, total, serverMaxResult) {
+    if (total >= serverMaxResult) {
+      debug(`WARNING: Query ${query} resulted in ${total} hits which meets the serverMaxResult (${serverMaxResult}) `);
+      return query;
+    }
+  }
   function getRecordId(record) {
     const [field] = record.get(/^001$/u);
     return field ? field.value : '';
   }
+  function getRecordIdFromXML(recordXML) {
+    //<controlfield tag=\"001\">015376846</controlfield
+    debug(`Cannot yet find possible database record id from recordXML (length ${recordXML.length})`);
+    return undefined;
+  }
 };
-
 exports.default = _default;
 //# sourceMappingURL=index.js.map

package/dist/candidate-search/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/candidate-search/index.js"],"names":["CandidateSearchError","Error","record","searchSpec","url","maxRecordsPerRequest","MarcRecord","setValidationOptions","subfieldValues","debug","inputRecordId","getRecordId","queryList","client","version","retrieveAll","JSON","stringify","length","queryOffset","resultSetOffset","query","records","nextOffset","retrieveRecords","queriesLeft","Promise","resolve","reject","promises","searchRetrieve","startRecord","on","err","SruSearchError","all","filtered","filter","r","foundRecord","push","handleRecord","foundRecordMarc","MARCXML","from","foundRecordId","id","field","get","value"],"mappings":";;;;;;;;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;;;;;;;AAhCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAUO,MAAMA,oBAAN,SAAmCC,KAAnC,CAAyC;;;;eAEjC,CAAC;AAACC,EAAAA,MAAD;AAASC,EAAAA,UAAT;AAAqBC,EAAAA,GAArB;AAA0BC,EAAAA,oBAAoB,GAAG;AAAjD,CAAD,KAA0D;AACvEC,yBAAWC,oBAAX,CAAgC;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAhC;;AAEA,QAAMC,KAAK,GAAG,oBAAkB,oDAAlB,CAAd;AACA,QAAMC,aAAa,GAAGC,WAAW,CAACT,MAAD,CAAjC;AACA,QAAMU,SAAS,GAAG,wBAAkBV,MAAlB,EAA0BC,UAA1B,CAAlB;AACA,QAAMU,MAAM,GAAG,wBAAa;AAC1BT,IAAAA,GAD0B;AACrBC,IAAAA,oBADqB;AAE1BS,IAAAA,OAAO,EAAE,KAFiB;AAG1BC,IAAAA,WAAW,EAAE;AAHa,GAAb,CAAf;AAMAN,EAAAA,KAAK,CAAE,yBAAwBC,aAAc,EAAxC,CAAL;AACAD,EAAAA,KAAK,CAAE,uBAAsBO,IAAI,CAACC,SAAL,CAAeL,SAAf,CAA0B,EAAlD,CAAL;;AACA,MAAIA,SAAS,CAACM,MAAV,KAAqB,CAAzB,EAA4B;AAAE;AAC5B,UAAM,IAAIlB,oBAAJ,CAA0B,0CAA1B,CAAN;AACD,GAhBsE,CAkBvE;;;AACA,SAAO,OAAO;AAACmB,IAAAA,WAAW,GAAG,CAAf;AAAkBC,IAAAA,eAAe,GAAG;AAApC,GAAP,KAAkD;AACvD,UAAMC,KAAK,GAAGT,SAAS,CAACO,WAAD,CAAvB;;AAEA,QAAIE,KAAJ,EAAW;AACT,YAAM;AAACC,QAAAA,OAAD;AAAUC,QAAAA;AAAV,UAAwB,MAAMC,eAAe,EAAnD;;AAEA,UAAI,OAAOD,UAAP,KAAsB,QAA1B,EAAoC;AAClCd,QAAAA,KAAK,CAAE,iCAAgCU,WAAY,IAAGE,KAAM,EAAvD,CAAL;AACA,eAAO;AAACC,UAAAA,OAAD;AAAUH,UAAAA,WAAV;AAAuBC,UAAAA,eAAe,EAAEG,UAAxC;AAAoDE,UAAAA,WAAW,EAAEb,SAAS,CAACM,MAAV,IAAoBC,WAAW,GAAG,CAAlC;AAAjE,SAAP;AACD;;AACDV,MAAAA,KAAK,CAAE,SAAQU,WAAY,IAAGE,KAAM,iCAAgCT,SAAS,CAACM,MAAV,IAAoBC,WAAW,GAAG,CAAlC,CAAqC,gBAApG,CAAL;AACA,aAAO;AAACG,QAAAA,OAAD;AAAUH,QAAAA,WAAW,EAAEA,WAAW,GAAG,CAArC;AAAwCM,QAAAA,WAAW,EAAEb,SAAS,CAACM,MAAV,IAAoBC,WAAW,GAAG,CAAlC;AAArD,OAAP;AACD;;AAEDV,IAAAA,KAAK,CAAE,OAAMG,SAAS,CAACM,MAAO,uCAAsCC,WAAY,EAA3E,CAAL;AACA,WAAO;AAACG,MAAAA,OAAO,EAAE;AAAV,KAAP;;AAEA,aAASE,eAAT,GAA2B;AACzB,aAAO,IAAIE,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,cAAMC,QAAQ,GAAG,EAAjB;AAEApB,QAAAA,KAAK,CAAE,wCAAuCY,KAAM,YAAWD,eAAgB,GAA1E,CAAL;AAEAP,QAAAA,MAAM,CAACiB,cAAP,CAAsBT,KAAtB,EAA6B;AAACU,UAAAA,WAAW,EAAEX;AAAd,SAA7B,EACGY,EADH,CACM,OADN,EACeC,GAAG,IAAI;AAClB;AACA,cAAIA,GAAG,YAAYC,yBAAnB,EAAmC;AACjCzB,YAAAA,KAAK,CAAE,iCAAgCY,KAAM,KAAIY,GAAI,EAAhD,CAAL;AACAL,YAAAA,MAAM,CAAC,IAAI5B,oBAAJ,CAA0B,iCAAgCqB,KAAM,KAAIY,GAAI,EAAxE,CAAD,CAAN;AACD;;AACDxB,UAAAA,KAAK,CAAE,wBAAuBY,KAAM,KAAIY,GAAI,EAAvC,CAAL;AACAL,UAAAA,MAAM,CAAC,IAAI5B,oBAAJ,CAA0B,wBAAuBqB,KAAM,KAAIY,GAAI,EAA/D,CAAD,CAAN;AACD,SATH,EAUGD,EAVH,CAUM,KAVN,EAUa,MAAMT,UAAN,IAAoB;AAC7B,cAAI;AACF,kBAAMD,OAAO,GAAG,MAAMI,OAAO,CAACS,GAAR,CAAYN,QAAZ,CAAtB;AACA,kBAAMO,QAAQ,GAAGd,OAAO,CAACe,MAAR,CAAeC,CAAC,IAAIA,CAApB,CAAjB;AAEA7B,YAAAA,KAAK,CAAE,SAAQ2B,QAAQ,CAAClB,MAAO,aAA1B,CAAL;AAEAS,YAAAA,OAAO,CAAC;AAACJ,cAAAA,UAAD;AAAaD,cAAAA,OAAO,EAAEc;AAAtB,aAAD,CAAP;AACD,WAPD,CAOE,OAAOH,GAAP,EAAY;AACZL,YAAAA,MAAM,CAACK,GAAD,CAAN;AACD;AACF,SArBH,EAsBGD,EAtBH,CAsBM,QAtBN,EAsBgBO,WAAW,IAAI;AAC3BV,UAAAA,QAAQ,CAACW,IAAT,CAAcC,YAAY,EAA1B,EAD2B,CACI;;AAE/B,yBAAeA,YAAf,GAA8B;AAC5B,gBAAI;AACF,oBAAMC,eAAe,GAAG,MAAMC,+BAAQC,IAAR,CAAaL,WAAb,EAA0B;AAAC/B,gBAAAA,cAAc,EAAE;AAAjB,eAA1B,CAA9B;AACA,oBAAMqC,aAAa,GAAGlC,WAAW,CAAC+B,eAAD,CAAjC,CAFE,CAIF;AACA;;AACAjC,cAAAA,KAAK,CAAE,YAAWC,aAAc,OAAMmC,aAAc,EAA/C,CAAL;;AACA,kBAAInC,aAAa,KAAKmC,aAAtB,EAAqC;AACnCpC,gBAAAA,KAAK,CAAE,uEAAF,CAAL;AACA;AACD;;AAED,qBAAO;AAACP,gBAAAA,MAAM,EAAEwC,eAAT;AAA0BI,gBAAAA,EAAE,EAAED;AAA9B,eAAP;AACD,aAbD,CAaE,OAAOZ,GAAP,EAAY;AACZ,oBAAM,IAAIhC,KAAJ,CAAW,6BAA4BgC,GAAI,aAAYM,WAAY,EAAnE,CAAN;AACD;AACF;AACF,SA3CH;AA4CD,OAjDM,CAAP;AAkDD;AACF,GArED;;AAuEA,WAAS5B,WAAT,CAAqBT,MAArB,EAA6B;AAC3B,UAAM,CAAC6C,KAAD,IAAU7C,MAAM,CAAC8C,GAAP,CAAW,QAAX,CAAhB;AACA,WAAOD,KAAK,GAAGA,KAAK,CAACE,KAAT,GAAiB,EAA7B;AACD;AACF,C","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport createClient, {SruSearchError} from '@natlibfi/sru-client';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {MARCXML} from '@natlibfi/marc-record-serializers';\nimport generateQueryList from './query-list';\n\nexport {searchTypes} from './query-list';\n\nexport class CandidateSearchError extends Error {}\n\nexport default ({record, searchSpec, url, maxRecordsPerRequest = 50}) => {\n  MarcRecord.setValidationOptions({subfieldValues: false});\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search');\n  const inputRecordId = getRecordId(record);\n  const queryList = generateQueryList(record, searchSpec);\n  const client = createClient({\n    url, maxRecordsPerRequest,\n    version: '2.0',\n    retrieveAll: false\n  });\n\n  debug(`Searching matches for ${inputRecordId}`);\n  debug(`Generated queryList ${JSON.stringify(queryList)}`);\n  if (queryList.length === 0) { // eslint-disable-line functional/no-conditional-statement\n    throw new CandidateSearchError(`Generated query list contains no queries`);\n  }\n\n  // eslint-disable-next-line max-statements\n  return async ({queryOffset = 0, resultSetOffset = 1}) => {\n    const query = queryList[queryOffset];\n\n    if (query) {\n      const {records, nextOffset} = await retrieveRecords();\n\n      if (typeof nextOffset === 'number') {\n        debug(`Running next search for query ${queryOffset} ${query}`);\n        return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1)};\n      }\n      debug(`Query ${queryOffset} ${query} done, moving to next query. (${queryList.length - (queryOffset + 1)} queries left)`);\n      return {records, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1)};\n    }\n\n    debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);\n    return {records: []};\n\n    function retrieveRecords() {\n      return new Promise((resolve, reject) => {\n        const promises = [];\n\n        debug(`Searching for candidates with query: ${query} (Offset ${resultSetOffset})`);\n\n        client.searchRetrieve(query, {startRecord: resultSetOffset})\n          .on('error', err => {\n            // eslint-disable-next-line functional/no-conditional-statement\n            if (err instanceof SruSearchError) {\n              debug(`SRU SruSearchError for query: ${query}: ${err}`);\n              reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));\n            }\n            debug(`SRU error for query: ${query}: ${err}`);\n            reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));\n          })\n          .on('end', async nextOffset => {\n            try {\n              const records = await Promise.all(promises);\n              const filtered = records.filter(r => r);\n\n              debug(`Found ${filtered.length} candidates`);\n\n              resolve({nextOffset, records: filtered});\n            } catch (err) {\n              reject(err);\n            }\n          })\n          .on('record', foundRecord => {\n            promises.push(handleRecord()); // eslint-disable-line functional/immutable-data\n\n            async function handleRecord() {\n              try {\n                const foundRecordMarc = await MARCXML.from(foundRecord, {subfieldValues: false});\n                const foundRecordId = getRecordId(foundRecordMarc);\n\n                // This does not work and might cause problems:\n                // Record *should* match itself AND in REST the input record is given id 000000001 always\n                debug(`Checking ${inputRecordId} vs ${foundRecordId}`);\n                if (inputRecordId === foundRecordId) {\n                  debug(`Input and candidate are the same record per 001. Discarding candidate`);\n                  return;\n                }\n\n                return {record: foundRecordMarc, id: foundRecordId};\n              } catch (err) {\n                throw new Error(`Failed converting record: ${err}, record: ${foundRecord}`);\n              }\n            }\n          });\n      });\n    }\n  };\n\n  function getRecordId(record) {\n    const [field] = record.get(/^001$/u);\n    return field ? field.value : '';\n  }\n};\n"],"file":"index.js"}
+{"version":3,"file":"index.js","names":["_debug","_interopRequireDefault","require","_sruClient","_interopRequireWildcard","_marcRecord","_marcRecordSerializers","_queryList","_melindaCommons","_getRequireWildcardCache","nodeInterop","WeakMap","cacheBabelInterop","cacheNodeInterop","obj","__esModule","default","cache","has","get","newObj","hasPropertyDescriptor","Object","defineProperty","getOwnPropertyDescriptor","key","prototype","hasOwnProperty","call","desc","set","CandidateSearchError","Error","exports","_default","record","searchSpec","url","maxCandidates","maxRecordsPerRequest","serverMaxResult","MarcRecord","setValidationOptions","subfieldValues","debug","createDebugLogger","debugData","extend","JSON","stringify","adjustedMaxRecordsPerRequest","inputRecordId","getRecordId","queryList","generateQueryList","client","createClient","version","retrieveAll","length","queryOffset","resultSetOffset","totalRecords","searchCounter","queryCandidateCounter","queryCounter","maxedQueries","query","records","failures","nextOffset","total","retrieveRecords","newTotalRecords","newQueryCounter","newSearchCounter","newQueryCandidateCounter","maxedQuery","checkMaxedQuery","undefined","newMaxedQueries","concat","queriesLeft","Promise","resolve","reject","promises","searchRetrieve","startRecord","on","err","SruSearchError","recordPromises","allSettled","filtered","filter","r","status","map","value","reason","payload","recordXML","push","handleRecord","recordMarc","MARCXML","from","recordId","id","idFromXML","getRecordIdFromXML","message","MatchingError","data","field"],"sources":["../../src/candidate-search/index.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport createClient, {SruSearchError} from '@natlibfi/sru-client';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {MARCXML} from '@natlibfi/marc-record-serializers';\nimport generateQueryList from './query-list';\nimport {Error as MatchingError} from '@natlibfi/melinda-commons';\n\nexport {searchTypes} from './query-list';\n\nexport class CandidateSearchError extends Error {}\n\n// serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000\n\nexport default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {\n  MarcRecord.setValidationOptions({subfieldValues: false});\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search');\n  const debugData = debug.extend('data');\n\n  debugData(`SearchSpec: ${JSON.stringify(searchSpec)}`);\n  debugData(`Url: ${url}`);\n  debugData(`MaxRecordsPerRequest ${maxRecordsPerRequest}`);\n  debugData(`ServerMaxResult: ${serverMaxResult}`);\n  debugData(`MaxCandidates: ${maxCandidates}`);\n\n  // Do not retrieve more candidates than defined in maxCandidates\n  const adjustedMaxRecordsPerRequest = maxRecordsPerRequest >= maxCandidates ? maxCandidates : maxRecordsPerRequest;\n\n  const inputRecordId = getRecordId(record);\n  const queryList = generateQueryList(record, searchSpec);\n  const client = createClient({\n    url,\n    maxRecordsPerRequest: adjustedMaxRecordsPerRequest,\n    version: '2.0',\n    retrieveAll: false\n  });\n\n  debug(`Searching matches for ${inputRecordId}`);\n  debug(`Generated queryList ${JSON.stringify(queryList)}`);\n\n  // if generateQueryList errored we should throw 422\n\n  if (queryList.length === 0) {\n    throw new CandidateSearchError(`Generated query list contains no queries`);\n  }\n\n  // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)\n  // state.query : current query (undefined if there was no queries left)\n  // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)\n  // state.queryCandidateCounter: amount of candidates (records+failures) retrieved from SRU for matching for current query, including the current record+failure set (undefined if there were no queries left)\n  // state.queriesLeft : amount of queries left\n  // state.queryCounter : sequence for current query\n  // state.maxedQueries : queries that resulted in more than serverMaxResults hits\n\n\n  return async ({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) => {\n    const query = queryList[queryOffset];\n\n    if (query) {\n      const {records, failures, nextOffset, total} = await retrieveRecords();\n\n      // If resultSetOffset === 1 this is the first search for the current query\n      debugData(`ResultSetOffset: ${resultSetOffset}`);\n      const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;\n      const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;\n      const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;\n      const newQueryCandidateCounter = resultSetOffset === 1 ? records.length + failures.length : queryCandidateCounter + records.length + failures.length;\n\n      const maxedQuery = resultSetOffset === 1 ? checkMaxedQuery(query, total, serverMaxResult) : undefined;\n      const newMaxedQueries = maxedQuery ? maxedQueries.concat(maxedQuery) : maxedQueries;\n\n      if (typeof nextOffset === 'number') {\n        debug(`Next search will be for query ${queryOffset} ${query}, starting from record ${nextOffset}`);\n        return {records, failures, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};\n      }\n      debug(`Query ${queryOffset} ${query} done.`);\n      debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);\n      return {records, failures, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};\n    }\n\n    debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);\n    return {records: [], failures: [], queriesLeft: 0, queryCounter, maxedQueries};\n\n    function retrieveRecords() {\n      return new Promise((resolve, reject) => {\n        const promises = [];\n        // eslint-disable-next-line functional/no-let\n        let totalRecords = 0;\n\n        debug(`Searching for candidates with query: ${query} (Offset ${resultSetOffset})`);\n\n        client.searchRetrieve(query, {startRecord: resultSetOffset})\n          .on('error', err => {\n            // eslint-disable-next-line functional/no-conditional-statements\n            if (err instanceof SruSearchError) {\n              debug(`SRU SruSearchError for query: ${query}: ${err}`);\n              reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));\n            }\n            debug(`SRU error for query: ${query}: ${err}`);\n            reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));\n          })\n          .on('total', total => {\n            debug(`Got total: ${total}`);\n            totalRecords += total;\n          })\n          .on('end', async nextOffset => {\n            try {\n              const recordPromises = await Promise.allSettled(promises);\n              debugData(`All recordPromises: ${JSON.stringify(recordPromises)}`);\n              const filtered = recordPromises.filter(r => r.status === 'fulfilled').map(r => r.value);\n              const failures = recordPromises.filter(r => r.status === 'rejected').map(r => ({status: r.reason.status, payload: r.reason.payload}));\n\n              debug(`Found ${recordPromises.length} records`);\n              debug(`Found ${filtered.length} convertable candidates`);\n              debug(`Found ${failures.length} NON-convertable candidates`);\n              debugData(`Converted: ${JSON.stringify(filtered)}.`);\n              debugData(`Not converted: ${JSON.stringify(failures)}.`);\n\n\n              resolve({nextOffset, records: filtered, failures, total: totalRecords});\n            } catch (err) {\n              debug(`Error caught on END`);\n              reject(err);\n            }\n          })\n          .on('record', recordXML => {\n            promises.push(handleRecord()); // eslint-disable-line functional/immutable-data\n\n            async function handleRecord() {\n              try {\n                const recordMarc = await MARCXML.from(recordXML, {subfieldValues: false});\n                const recordId = getRecordId(recordMarc);\n\n                return {record: recordMarc, id: recordId};\n              } catch (err) {\n                // What should this do?\n                const idFromXML = getRecordIdFromXML(recordXML);\n                debugData(`Failed converting record: ${err.message}, id: ${idFromXML}, data: ${recordXML}`);\n                //return {message: `Failed converting record: ${err.message}`, id: idFromXML, data: recordXML};\n                throw new MatchingError(422, {message: `Failed converting record: ${err.message}`, id: idFromXML || '000000000', data: recordXML});\n              }\n            }\n          });\n      });\n    }\n  };\n\n  function checkMaxedQuery(query, total, serverMaxResult) {\n    if (total >= serverMaxResult) {\n      debug(`WARNING: Query ${query} resulted in ${total} hits which meets the serverMaxResult (${serverMaxResult}) `);\n      return query;\n    }\n  }\n\n\n  function getRecordId(record) {\n    const [field] = record.get(/^001$/u);\n    return field ? field.value : '';\n  }\n\n  function getRecordIdFromXML(recordXML) {\n    //<controlfield tag=\\\"001\\\">015376846</controlfield\n    debug(`Cannot yet find possible database record id from recordXML (length ${recordXML.length})`);\n    return undefined;\n  }\n\n};\n"],"mappings":";;;;;;;;;;;;AA4BA,IAAAA,MAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,UAAA,GAAAC,uBAAA,CAAAF,OAAA;AACA,IAAAG,WAAA,GAAAH,OAAA;AACA,IAAAI,sBAAA,GAAAJ,OAAA;AACA,IAAAK,UAAA,GAAAH,uBAAA,CAAAF,OAAA;AACA,IAAAM,eAAA,GAAAN,OAAA;AAAiE,SAAAO,yBAAAC,WAAA,eAAAC,OAAA,kCAAAC,iBAAA,OAAAD,OAAA,QAAAE,gBAAA,OAAAF,OAAA,YAAAF,wBAAA,YAAAA,CAAAC,WAAA,WAAAA,WAAA,GAAAG,gBAAA,GAAAD,iBAAA,KAAAF,WAAA;AAAA,SAAAN,wBAAAU,GAAA,EAAAJ,WAAA,SAAAA,WAAA,IAAAI,GAAA,IAAAA,GAAA,CAAAC,UAAA,WAAAD,GAAA,QAAAA,GAAA,oBAAAA,GAAA,wBAAAA,GAAA,4BAAAE,OAAA,EAAAF,GAAA,UAAAG,KAAA,GAAAR,wBAAA,CAAAC,WAAA,OAAAO,KAAA,IAAAA,KAAA,CAAAC,GAAA,CAAAJ,GAAA,YAAAG,KAAA,CAAAE,GAAA,CAAAL,GAAA,SAAAM,MAAA,WAAAC,qBAAA,GAAAC,MAAA,CAAAC,cAAA,IAAAD,MAAA,CAAAE,wBAAA,WAAAC,GAAA,IAAAX,GAAA,QAAAW,GAAA,kBAAAH,MAAA,CAAAI,SAAA,CAAAC,cAAA,CAAAC,IAAA,CAAAd,GAAA,EAAAW,GAAA,SAAAI,IAAA,GAAAR,qBAAA,GAAAC,MAAA,CAAAE,wBAAA,CAAAV,GAAA,EAAAW,GAAA,cAAAI,IAAA,KAAAA,IAAA,CAAAV,GAAA,IAAAU,IAAA,CAAAC,GAAA,KAAAR,MAAA,CAAAC,cAAA,CAAAH,MAAA,EAAAK,GAAA,EAAAI,IAAA,YAAAT,MAAA,CAAAK,GAAA,IAAAX,GAAA,CAAAW,GAAA,SAAAL,MAAA,CAAAJ,OAAA,GAAAF,GAAA,MAAAG,KAAA,IAAAA,KAAA,CAAAa,GAAA,CAAAhB,GAAA,EAAAM,MAAA,YAAAA,MAAA;AAAA,SAAAnB,uBAAAa,GAAA,WAAAA,GAAA,IAAAA,GAAA,CAAAC,UAAA,GAAAD,GAAA,KAAAE,OAAA,EAAAF,GAAA;AAjCjE;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWO,MAAMiB,oBAAoB,SAASC,KAAK,CAAC;;AAEhD;AAAAC,OAAA,CAAAF,oBAAA,GAAAA,oBAAA;AAAA,IAAAG,QAAA,GAEeA,CAAC;EAACC,MAAM;EAAEC,UAAU;EAAEC,GAAG;EAAEC,aAAa;EAAEC,oBAAoB,GAAG,EAAE;EAAEC,eAAe,GAAG;AAAK,CAAC,KAAK;EAC/GC,sBAAU,CAACC,oBAAoB,CAAC;IAACC,cAAc,EAAE;EAAK,CAAC,CAAC;EAExD,MAAMC,KAAK,GAAG,IAAAC,cAAiB,EAAC,oDAAoD,CAAC;EACrF,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtCD,SAAS,CAAE,eAAcE,IAAI,CAACC,SAAS,CAACb,UAAU,CAAE,EAAC,CAAC;EACtDU,SAAS,CAAE,QAAOT,GAAI,EAAC,CAAC;EACxBS,SAAS,CAAE,wBAAuBP,oBAAqB,EAAC,CAAC;EACzDO,SAAS,CAAE,oBAAmBN,eAAgB,EAAC,CAAC;EAChDM,SAAS,CAAE,kBAAiBR,aAAc,EAAC,CAAC;;EAE5C;EACA,MAAMY,4BAA4B,GAAGX,oBAAoB,IAAID,aAAa,GAAGA,aAAa,GAAGC,oBAAoB;EAEjH,MAAMY,aAAa,GAAGC,WAAW,CAACjB,MAAM,CAAC;EACzC,MAAMkB,SAAS,GAAG,IAAAC,kBAAiB,EAACnB,MAAM,EAAEC,UAAU,CAAC;EACvD,MAAMmB,MAAM,GAAG,IAAAC,kBAAY,EAAC;IAC1BnB,GAAG;IACHE,oBAAoB,EAAEW,4BAA4B;IAClDO,OAAO,EAAE,KAAK;IACdC,WAAW,EAAE;EACf,CAAC,CAAC;EAEFd,KAAK,CAAE,yBAAwBO,aAAc,EAAC,CAAC;EAC/CP,KAAK,CAAE,uBAAsBI,IAAI,CAACC,SAAS,CAACI,SAAS,CAAE,EAAC,CAAC;;EAEzD;;EAEA,IAAIA,SAAS,CAACM,MAAM,KAAK,CAAC,EAAE;IAC1B,MAAM,IAAI5B,oBAAoB,CAAE,0CAAyC,CAAC;EAC5E;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;;EAGA,OAAO,OAAO;IAAC6B,WAAW,GAAG,CAAC;IAAEC,eAAe,GAAG,CAAC;IAAEC,YAAY,GAAG,CAAC;IAAEC,aAAa,GAAG,CAAC;IAAEC,qBAAqB,GAAG,CAAC;IAAEC,YAAY,GAAG,CAAC;IAAEC,YAAY,GAAG;EAAE,CAAC,KAAK;IAC5J,MAAMC,KAAK,GAAGd,SAAS,CAACO,WAAW,CAAC;IAEpC,IAAIO,KAAK,EAAE;MACT,MAAM;QAACC,OAAO;QAAEC,QAAQ;QAAEC,UAAU;QAAEC;MAAK,CAAC,GAAG,MAAMC,eAAe,CAAC,CAAC;;MAEtE;MACA1B,SAAS,CAAE,oBAAmBe,eAAgB,EAAC,CAAC;MAChD,MAAMY,eAAe,GAAGZ,eAAe,KAAK,CAAC,GAAGU,KAAK,GAAGT,YAAY;MACpE,MAAMY,eAAe,GAAGb,eAAe,KAAK,CAAC,GAAGI,YAAY,GAAG,CAAC,GAAGA,YAAY;MAC/E,MAAMU,gBAAgB,GAAGd,eAAe,KAAK,CAAC,GAAG,CAAC,GAAGE,aAAa,GAAG,CAAC;MACtE,MAAMa,wBAAwB,GAAGf,eAAe,KAAK,CAAC,GAAGO,OAAO,CAACT,MAAM,GAAGU,QAAQ,CAACV,MAAM,GAAGK,qBAAqB,GAAGI,OAAO,CAACT,MAAM,GAAGU,QAAQ,CAACV,MAAM;MAEpJ,MAAMkB,UAAU,GAAGhB,eAAe,KAAK,CAAC,GAAGiB,eAAe,CAACX,KAAK,EAAEI,KAAK,EAAE/B,eAAe,CAAC,GAAGuC,SAAS;MACrG,MAAMC,eAAe,GAAGH,UAAU,GAAGX,YAAY,CAACe,MAAM,CAACJ,UAAU,CAAC,GAAGX,YAAY;MAEnF,IAAI,OAAOI,UAAU,KAAK,QAAQ,EAAE;QAClC1B,KAAK,CAAE,iCAAgCgB,WAAY,IAAGO,KAAM,0BAAyBG,UAAW,EAAC,CAAC;QAClG,OAAO;UAACF,OAAO;UAAEC,QAAQ;UAAET,WAAW;UAAEC,eAAe,EAAES,UAAU;UAAEY,WAAW,EAAE7B,SAAS,CAACM,MAAM,IAAIC,WAAW,GAAG,CAAC,CAAC;UAAEE,YAAY,EAAEW,eAAe;UAAEN,KAAK;UAAEJ,aAAa,EAAEY,gBAAgB;UAAEX,qBAAqB,EAAEY,wBAAwB;UAAEX,YAAY,EAAES,eAAe;UAAER,YAAY,EAAEc;QAAe,CAAC;MAC/S;MACApC,KAAK,CAAE,SAAQgB,WAAY,IAAGO,KAAM,QAAO,CAAC;MAC5CvB,KAAK,CAAE,cAAaS,SAAS,CAACM,MAAM,IAAIC,WAAW,GAAG,CAAC,CAAE,gBAAe,CAAC;MACzE,OAAO;QAACQ,OAAO;QAAEC,QAAQ;QAAET,WAAW,EAAEA,WAAW,GAAG,CAAC;QAAEsB,WAAW,EAAE7B,SAAS,CAACM,MAAM,IAAIC,WAAW,GAAG,CAAC,CAAC;QAAEE,YAAY,EAAEW,eAAe;QAAEN,KAAK;QAAEJ,aAAa,EAAEY,gBAAgB;QAAEX,qBAAqB,EAAEY,wBAAwB;QAAEX,YAAY,EAAES,eAAe;QAAER,YAAY,EAAEc;MAAe,CAAC;IACnS;IAEApC,KAAK,CAAE,OAAMS,SAAS,CAACM,MAAO,uCAAsCC,WAAY,EAAC,CAAC;IAClF,OAAO;MAACQ,OAAO,EAAE,EAAE;MAAEC,QAAQ,EAAE,EAAE;MAAEa,WAAW,EAAE,CAAC;MAAEjB,YAAY;MAAEC;IAAY,CAAC;IAE9E,SAASM,eAAeA,CAAA,EAAG;MACzB,OAAO,IAAIW,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtC,MAAMC,QAAQ,GAAG,EAAE;QACnB;QACA,IAAIxB,YAAY,GAAG,CAAC;QAEpBlB,KAAK,CAAE,wCAAuCuB,KAAM,YAAWN,eAAgB,GAAE,CAAC;QAElFN,MAAM,CAACgC,cAAc,CAACpB,KAAK,EAAE;UAACqB,WAAW,EAAE3B;QAAe,CAAC,CAAC,CACzD4B,EAAE,CAAC,OAAO,EAAEC,GAAG,IAAI;UAClB;UACA,IAAIA,GAAG,YAAYC,yBAAc,EAAE;YACjC/C,KAAK,CAAE,iCAAgCuB,KAAM,KAAIuB,GAAI,EAAC,CAAC;YACvDL,MAAM,CAAC,IAAItD,oBAAoB,CAAE,iCAAgCoC,KAAM,KAAIuB,GAAI,EAAC,CAAC,CAAC;UACpF;UACA9C,KAAK,CAAE,wBAAuBuB,KAAM,KAAIuB,GAAI,EAAC,CAAC;UAC9CL,MAAM,CAAC,IAAItD,oBAAoB,CAAE,wBAAuBoC,KAAM,KAAIuB,GAAI,EAAC,CAAC,CAAC;QAC3E,CAAC,CAAC,CACDD,EAAE,CAAC,OAAO,EAAElB,KAAK,IAAI;UACpB3B,KAAK,CAAE,cAAa2B,KAAM,EAAC,CAAC;UAC5BT,YAAY,IAAIS,KAAK;QACvB,CAAC,CAAC,CACDkB,EAAE,CAAC,KAAK,EAAE,MAAMnB,UAAU,IAAI;UAC7B,IAAI;YACF,MAAMsB,cAAc,GAAG,MAAMT,OAAO,CAACU,UAAU,CAACP,QAAQ,CAAC;YACzDxC,SAAS,CAAE,uBAAsBE,IAAI,CAACC,SAAS,CAAC2C,cAAc,CAAE,EAAC,CAAC;YAClE,MAAME,QAAQ,GAAGF,cAAc,CAACG,MAAM,CAACC,CAAC,IAAIA,CAAC,CAACC,MAAM,KAAK,WAAW,CAAC,CAACC,GAAG,CAACF,CAAC,IAAIA,CAAC,CAACG,KAAK,CAAC;YACvF,MAAM9B,QAAQ,GAAGuB,cAAc,CAACG,MAAM,CAACC,CAAC,IAAIA,CAAC,CAACC,MAAM,KAAK,UAAU,CAAC,CAACC,GAAG,CAACF,CAAC,KAAK;cAACC,MAAM,EAAED,CAAC,CAACI,MAAM,CAACH,MAAM;cAAEI,OAAO,EAAEL,CAAC,CAACI,MAAM,CAACC;YAAO,CAAC,CAAC,CAAC;YAErIzD,KAAK,CAAE,SAAQgD,cAAc,CAACjC,MAAO,UAAS,CAAC;YAC/Cf,KAAK,CAAE,SAAQkD,QAAQ,CAACnC,MAAO,yBAAwB,CAAC;YACxDf,KAAK,CAAE,SAAQyB,QAAQ,CAACV,MAAO,6BAA4B,CAAC;YAC5Db,SAAS,CAAE,cAAaE,IAAI,CAACC,SAAS,CAAC6C,QAAQ,CAAE,GAAE,CAAC;YACpDhD,SAAS,CAAE,kBAAiBE,IAAI,CAACC,SAAS,CAACoB,QAAQ,CAAE,GAAE,CAAC;YAGxDe,OAAO,CAAC;cAACd,UAAU;cAAEF,OAAO,EAAE0B,QAAQ;cAAEzB,QAAQ;cAAEE,KAAK,EAAET;YAAY,CAAC,CAAC;UACzE,CAAC,CAAC,OAAO4B,GAAG,EAAE;YACZ9C,KAAK,CAAE,qBAAoB,CAAC;YAC5ByC,MAAM,CAACK,GAAG,CAAC;UACb;QACF,CAAC,CAAC,CACDD,EAAE,CAAC,QAAQ,EAAEa,SAAS,IAAI;UACzBhB,QAAQ,CAACiB,IAAI,CAACC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;;UAE/B,eAAeA,YAAYA,CAAA,EAAG;YAC5B,IAAI;cACF,MAAMC,UAAU,GAAG,MAAMC,8BAAO,CAACC,IAAI,CAACL,SAAS,EAAE;gBAAC3D,cAAc,EAAE;cAAK,CAAC,CAAC;cACzE,MAAMiE,QAAQ,GAAGxD,WAAW,CAACqD,UAAU,CAAC;cAExC,OAAO;gBAACtE,MAAM,EAAEsE,UAAU;gBAAEI,EAAE,EAAED;cAAQ,CAAC;YAC3C,CAAC,CAAC,OAAOlB,GAAG,EAAE;cACZ;cACA,MAAMoB,SAAS,GAAGC,kBAAkB,CAACT,SAAS,CAAC;cAC/CxD,SAAS,CAAE,6BAA4B4C,GAAG,CAACsB,OAAQ,SAAQF,SAAU,WAAUR,SAAU,EAAC,CAAC;cAC3F;cACA,MAAM,IAAIW,qBAAa,CAAC,GAAG,EAAE;gBAACD,OAAO,EAAG,6BAA4BtB,GAAG,CAACsB,OAAQ,EAAC;gBAAEH,EAAE,EAAEC,SAAS,IAAI,WAAW;gBAAEI,IAAI,EAAEZ;cAAS,CAAC,CAAC;YACpI;UACF;QACF,CAAC,CAAC;MACN,CAAC,CAAC;IACJ;EACF,CAAC;EAED,SAASxB,eAAeA,CAACX,KAAK,EAAEI,KAAK,EAAE/B,eAAe,EAAE;IACtD,IAAI+B,KAAK,IAAI/B,eAAe,EAAE;MAC5BI,KAAK,CAAE,kBAAiBuB,KAAM,gBAAeI,KAAM,0CAAyC/B,eAAgB,IAAG,CAAC;MAChH,OAAO2B,KAAK;IACd;EACF;EAGA,SAASf,WAAWA,CAACjB,MAAM,EAAE;IAC3B,MAAM,CAACgF,KAAK,CAAC,GAAGhF,MAAM,CAAChB,GAAG,CAAC,QAAQ,CAAC;IACpC,OAAOgG,KAAK,GAAGA,KAAK,CAAChB,KAAK,GAAG,EAAE;EACjC;EAEA,SAASY,kBAAkBA,CAACT,SAAS,EAAE;IACrC;IACA1D,KAAK,CAAE,sEAAqE0D,SAAS,CAAC3C,MAAO,GAAE,CAAC;IAChG,OAAOoB,SAAS;EAClB;AAEF,CAAC;AAAA9C,OAAA,CAAAjB,OAAA,GAAAkB,QAAA"}