@natlibfi/melinda-record-matching 3.0.0-alpha.1 → 3.0.1-alpha.2

package/dist/matching-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"matching-utils.js","names":["debug","createDebugLogger","debugData","extend","getMelindaIdsF035","record","f035s","getFields","length","allF035MelindaIds","concat","map","toMelindaIds","f035MelindaIds","Set","JSON","stringify","subfields","melindaIdRegExp","filter","sub","includes","code","testStringOrNumber","value","test","String","replace","validateSidFieldSubfieldCounts","field","countC","countSubfields","countB","subfieldCode","getSubfieldValues"],"sources":["../src/matching-utils.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\n// eslint-disable-next-line max-statements\nexport function getMelindaIdsF035(record) {\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n  const debugData = debug.extend('data');\n\n  const f035s = record.getFields('035');\n\n  if (f035s.length < 1) {\n    debug(`No f035s found.`);\n    return [];\n  }\n\n  const allF035MelindaIds = [].concat(...f035s.map(toMelindaIds));\n  const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n  debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n  debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n  debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n  return f035MelindaIds;\n\n  function toMelindaIds({subfields}) {\n    const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n    return subfields\n      .filter(sub => ['a', 'z'].includes(sub.code))\n      .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n      .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n\n  }\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n  // Valid SID-fields have just one $c and one $b\n  debugData(`Validating SID field ${JSON.stringify(field)}`);\n  const countC = countSubfields(field, 'c');\n  const countB = countSubfields(field, 'b');\n  debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n  return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n  // debug(`Counting subfields ${subfieldCode}`);\n  return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n  return field.subfields\n    .filter(({code}) => code === subfieldCode)\n    .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n    .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n  if (typeof value === 'string' || typeof value === 'number') {\n    return true;\n  }\n  return false;\n}\n"],"mappings":";;;;;;;;;AA4BA;AAAsC;AA5BtC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAIA,MAAMA,KAAK,GAAG,IAAAC,cAAiB,EAAC,yCAAyC,CAAC;AAC1E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;;AAEtC;AACO,SAASC,iBAAiB,CAACC,MAAM,EAAE;EAExC,MAAML,KAAK,GAAG,IAAAC,cAAiB,EAAC,8CAA8C,CAAC;EAC/E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtC,MAAMG,KAAK,GAAGD,MAAM,CAACE,SAAS,CAAC,KAAK,CAAC;EAErC,IAAID,KAAK,CAACE,MAAM,GAAG,CAAC,EAAE;IACpBR,KAAK,CAAE,iBAAgB,CAAC;IACxB,OAAO,EAAE;EACX;EAEA,MAAMS,iBAAiB,GAAG,EAAE,CAACC,MAAM,CAAC,GAAGJ,KAAK,CAACK,GAAG,CAACC,YAAY,CAAC,CAAC;EAC/D,MAAMC,cAAc,GAAG,CAAC,GAAG,IAAIC,GAAG,CAACL,iBAAiB,CAAC,CAAC;EAEtDP,SAAS,CAAE,WAAUI,KAAK,CAACE,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACV,KAAK,CAAE,EAAC,CAAC;EAC/DJ,SAAS,CAAE,QAAOO,iBAAiB,CAACD,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACP,iBAAiB,CAAE,EAAC,CAAC;EACpFP,SAAS,CAAE,eAAcW,cAAc,CAACL,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACH,cAAc,CAAE,EAAC,CAAC;EAErF,OAAOA,cAAc;EAErB,SAASD,YAAY,CAAC;IAACK;EAAS,CAAC,EAAE;IACjC,MAAMC,eAAe,GAAG,8CAA8C;IAEtE,OAAOD,SAAS,CACbE,MAAM,CAACC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAACC,QAAQ,CAACD,GAAG,CAACE,IAAI,CAAC,CAAC,CAC5CH,MAAM,CAACC,GAAG,IAAIG,kBAAkB,CAACH,GAAG,CAACI,KAAK,CAAC,IAAIN,eAAe,CAACO,IAAI,CAACC,MAAM,CAACN,GAAG,CAACI,KAAK,CAAC,CAAC,CAAC,CACvFb,GAAG,CAAC,CAAC;MAACa;IAAK,CAAC,KAAKD,kBAAkB,CAACC,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,CAACG,OAAO,CAACT,eAAe,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;EAEvG;AACF;AAEO,SAASU,8BAA8B,CAACC,KAAK,EAAE;EACpD;EACA3B,SAAS,CAAE,wBAAuBa,IAAI,CAACC,SAAS,CAACa,KAAK,CAAE,EAAC,CAAC;EAC1D,MAAMC,MAAM,GAAGC,cAAc,CAACF,KAAK,EAAE,GAAG,CAAC;EACzC,MAAMG,MAAM,GAAGD,cAAc,CAACF,KAAK,EAAE,GAAG,CAAC;EACzC7B,KAAK,CAAE,SAAQ8B,MAAO,eAAcE,MAAO,qBAAoBF,MAAM,KAAK,CAAC,IAAIE,MAAM,KAAK,CAAE,EAAC,CAAC;EAE9F,OAAOF,MAAM,KAAK,CAAC,IAAIE,MAAM,KAAK,CAAC;AACrC;AAEA,SAASD,cAAc,CAACF,KAAK,EAAEI,YAAY,EAAE;EAC3C;EACA,OAAOJ,KAAK,CAACZ,SAAS,CAACE,MAAM,CAAC,CAAC;IAACG;EAAI,CAAC,KAAKA,IAAI,KAAKW,YAAY,CAAC,CAACzB,MAAM;AACzE;AAEO,SAAS0B,iBAAiB,CAACL,KAAK,EAAEI,YAAY,EAAE;EACrD/B,SAAS,CAAE,oBAAmB+B,YAAa,SAAQlB,IAAI,CAACC,SAAS,CAACa,KAAK,CAAE,EAAC,CAAC;EAC3E,OAAOA,KAAK,CAACZ,SAAS,CACnBE,MAAM,CAAC,CAAC;IAACG;EAAI,CAAC,KAAKA,IAAI,KAAKW,YAAY,CAAC,CACzCtB,GAAG,CAAC,CAAC;IAACa;EAAK,CAAC,KAAKD,kBAAkB,CAACC,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,GAAG,EAAE,CAAC,CAChEL,MAAM,CAACK,KAAK,IAAIA,KAAK,CAAC;AAC3B;AAEO,SAASD,kBAAkB,CAACC,KAAK,EAAE;EACxC,IAAI,OAAOA,KAAK,KAAK,QAAQ,IAAI,OAAOA,KAAK,KAAK,QAAQ,EAAE;IAC1D,OAAO,IAAI;EACb;EACA,OAAO,KAAK;AACd"}
+{"version":3,"file":"matching-utils.js","names":["debug","createDebugLogger","debugData","extend","getMelindaIdsF035","record","f035s","getFields","length","allF035MelindaIds","concat","map","toMelindaIds","f035MelindaIds","Set","JSON","stringify","subfields","melindaIdRegExp","filter","sub","includes","code","testStringOrNumber","value","test","String","replace","validateSidFieldSubfieldCounts","field","countC","countSubfields","countB","subfieldCode","getSubfieldValues"],"sources":["../src/matching-utils.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\nexport function getMelindaIdsF035(record) {\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n  const debugData = debug.extend('data');\n\n  const f035s = record.getFields('035');\n\n  if (f035s.length < 1) {\n    debug(`No f035s found.`);\n    return [];\n  }\n\n  const allF035MelindaIds = [].concat(...f035s.map(toMelindaIds));\n  const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n  debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n  debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n  debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n  return f035MelindaIds;\n\n  function toMelindaIds({subfields}) {\n    const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n    return subfields\n      .filter(sub => ['a', 'z'].includes(sub.code))\n      .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n      .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n\n  }\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n  // Valid SID-fields have just one $c and one $b\n  debugData(`Validating SID field ${JSON.stringify(field)}`);\n  const countC = countSubfields(field, 'c');\n  const countB = countSubfields(field, 'b');\n  debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n  return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n  // debug(`Counting subfields ${subfieldCode}`);\n  return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n  return field.subfields\n    .filter(({code}) => code === subfieldCode)\n    .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n    .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n  if (typeof value === 'string' || typeof value === 'number') {\n    return true;\n  }\n  return false;\n}\n"],"mappings":";;;;;;;;;AA4BA;AAAsC;AA5BtC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAIA,MAAMA,KAAK,GAAG,IAAAC,cAAiB,EAAC,yCAAyC,CAAC;AAC1E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;AAE/B,SAASC,iBAAiB,CAACC,MAAM,EAAE;EAExC,MAAML,KAAK,GAAG,IAAAC,cAAiB,EAAC,8CAA8C,CAAC;EAC/E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtC,MAAMG,KAAK,GAAGD,MAAM,CAACE,SAAS,CAAC,KAAK,CAAC;EAErC,IAAID,KAAK,CAACE,MAAM,GAAG,CAAC,EAAE;IACpBR,KAAK,CAAE,iBAAgB,CAAC;IACxB,OAAO,EAAE;EACX;EAEA,MAAMS,iBAAiB,GAAG,EAAE,CAACC,MAAM,CAAC,GAAGJ,KAAK,CAACK,GAAG,CAACC,YAAY,CAAC,CAAC;EAC/D,MAAMC,cAAc,GAAG,CAAC,GAAG,IAAIC,GAAG,CAACL,iBAAiB,CAAC,CAAC;EAEtDP,SAAS,CAAE,WAAUI,KAAK,CAACE,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACV,KAAK,CAAE,EAAC,CAAC;EAC/DJ,SAAS,CAAE,QAAOO,iBAAiB,CAACD,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACP,iBAAiB,CAAE,EAAC,CAAC;EACpFP,SAAS,CAAE,eAAcW,cAAc,CAACL,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACH,cAAc,CAAE,EAAC,CAAC;EAErF,OAAOA,cAAc;EAErB,SAASD,YAAY,CAAC;IAACK;EAAS,CAAC,EAAE;IACjC,MAAMC,eAAe,GAAG,8CAA8C;IAEtE,OAAOD,SAAS,CACbE,MAAM,CAACC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAACC,QAAQ,CAACD,GAAG,CAACE,IAAI,CAAC,CAAC,CAC5CH,MAAM,CAACC,GAAG,IAAIG,kBAAkB,CAACH,GAAG,CAACI,KAAK,CAAC,IAAIN,eAAe,CAACO,IAAI,CAACC,MAAM,CAACN,GAAG,CAACI,KAAK,CAAC,CAAC,CAAC,CACvFb,GAAG,CAAC,CAAC;MAACa;IAAK,CAAC,KAAKD,kBAAkB,CAACC,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,CAACG,OAAO,CAACT,eAAe,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;EAEvG;AACF;AAEO,SAASU,8BAA8B,CAACC,KAAK,EAAE;EACpD;EACA3B,SAAS,CAAE,wBAAuBa,IAAI,CAACC,SAAS,CAACa,KAAK,CAAE,EAAC,CAAC;EAC1D,MAAMC,MAAM,GAAGC,cAAc,CAACF,KAAK,EAAE,GAAG,CAAC;EACzC,MAAMG,MAAM,GAAGD,cAAc,CAACF,KAAK,EAAE,GAAG,CAAC;EACzC7B,KAAK,CAAE,SAAQ8B,MAAO,eAAcE,MAAO,qBAAoBF,MAAM,KAAK,CAAC,IAAIE,MAAM,KAAK,CAAE,EAAC,CAAC;EAE9F,OAAOF,MAAM,KAAK,CAAC,IAAIE,MAAM,KAAK,CAAC;AACrC;AAEA,SAASD,cAAc,CAACF,KAAK,EAAEI,YAAY,EAAE;EAC3C;EACA,OAAOJ,KAAK,CAACZ,SAAS,CAACE,MAAM,CAAC,CAAC;IAACG;EAAI,CAAC,KAAKA,IAAI,KAAKW,YAAY,CAAC,CAACzB,MAAM;AACzE;AAEO,SAAS0B,iBAAiB,CAACL,KAAK,EAAEI,YAAY,EAAE;EACrD/B,SAAS,CAAE,oBAAmB+B,YAAa,SAAQlB,IAAI,CAACC,SAAS,CAACa,KAAK,CAAE,EAAC,CAAC;EAC3E,OAAOA,KAAK,CAACZ,SAAS,CACnBE,MAAM,CAAC,CAAC;IAACG;EAAI,CAAC,KAAKA,IAAI,KAAKW,YAAY,CAAC,CACzCtB,GAAG,CAAC,CAAC;IAACa;EAAK,CAAC,KAAKD,kBAAkB,CAACC,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,GAAG,EAAE,CAAC,CAChEL,MAAM,CAACK,KAAK,IAAIA,KAAK,CAAC;AAC3B;AAEO,SAASD,kBAAkB,CAACC,KAAK,EAAE;EACxC,IAAI,OAAOA,KAAK,KAAK,QAAQ,IAAI,OAAOA,KAAK,KAAK,QAAQ,EAAE;IAC1D,OAAO,IAAI;EACb;EACA,OAAO,KAAK;AACd"}

package/package.json

@@ -14,7 +14,7 @@
 		"url": "git@github.com:natlibfi/melinda-record-matching-js.git"
 	},
 	"license": "LGPL-3.0+",
-	"version": "3.0.0-alpha.1",
+	"version": "3.0.1-alpha.2",
 	"main": "./dist/index.js",
 	"engines": {
 		"node": ">=14"
@@ -36,9 +36,9 @@
 		"watch:test": "cross-env DEBUG=1 NODE_ENV=test nodemon -w src -w test-fixtures --exec 'npm run test:dev'"
 	},
 	"dependencies": {
-		"@natlibfi/marc-record": "^7.2.1",
-		"@natlibfi/marc-record-serializers": "^9.0.0",
-		"@natlibfi/melinda-commons": "^12.0.4",
+		"@natlibfi/marc-record": "^7.2.2",
+		"@natlibfi/marc-record-serializers": "^9.0.2",
+		"@natlibfi/melinda-commons": "^12.0.6",
 		"@natlibfi/sru-client": "^5.0.3",
 		"debug": "^4.3.4",
 		"moment": "^2.29.4",
@@ -48,9 +48,9 @@
 	},
 	"devDependencies": {
 		"@babel/cli": "^7.19.3",
-		"@babel/core": "^7.19.3",
+		"@babel/core": "^7.19.6",
 		"@babel/eslint-parser": "^7.19.1",
-		"@babel/node": "^7.19.1",
+		"@babel/node": "^7.20.0",
 		"@babel/preset-env": "^7.19.4",
 		"@babel/register": "^7.18.9",
 		"@natlibfi/eslint-config-melinda-backend": "^2.0.1",
@@ -62,8 +62,8 @@
 		"chai": "^4.3.6",
 		"chai-as-promised": "^7.1.1",
 		"cross-env": "^7.0.3",
-		"eslint": "^8.25.0",
-		"mocha": "^10.0.0",
+		"eslint": "^8.26.0",
+		"mocha": "^10.1.0",
 		"nodemon": "^2.0.20",
 		"nyc": "^15.1.0"
 	},

package/src/candidate-search/index.js

@@ -39,7 +39,6 @@ export class CandidateSearchError extends Error {}
 
 // serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000
 
-// eslint-disable-next-line max-statements
 export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {
   MarcRecord.setValidationOptions({subfieldValues: false});
 
@@ -69,7 +68,7 @@ export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest =
 
   // if generateQueryList errored we should throw 422
 
-  if (queryList.length === 0) { // eslint-disable-line functional/no-conditional-statement
+  if (queryList.length === 0) {
     throw new CandidateSearchError(`Generated query list contains no queries`);
   }
 
@@ -175,7 +174,6 @@ export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest =
   };
 
   function checkMaxedQuery(query, total, serverMaxResult) {
-    // eslint-disable-next-line functional/no-conditional-statement
     if (total >= serverMaxResult) {
       debug(`WARNING: Query ${query} resulted in ${total} hits which meets the serverMaxResult (${serverMaxResult}) `);
       return query;

package/src/candidate-search/index.spec.js

@@ -46,7 +46,6 @@ describe('candidate-search', () => {
     }
   });
 
-  // eslint-disable-next-line max-statements
   async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError = false, expectedSearchError = false, enabled = true}) {
     const url = 'http://foo.bar';
 
@@ -77,7 +76,6 @@ describe('candidate-search', () => {
       };
     }
 
-    // eslint-disable-next-line max-statements
     async function iterate({searchOptions, expectedSearchError, expectedErrorStatus, count = 1}) {
       const expectedResults = getFixture(`expectedResults${count}.json`);
 
@@ -108,7 +106,7 @@ describe('candidate-search', () => {
       }
 
       function formatResults(results) {
-        debug(results); //eslint-disable-line
+        debug(results);
         return {
           ...results,
           records: results.records.map(({record, id}) => ({id, record: record.toObject()}))

package/src/candidate-search/query-list/bib.js

@@ -217,7 +217,6 @@ export function bibTitle(record) {
   }
 }
 
-// eslint-disable-next-line max-statements
 export function bibStandardIdentifiers(record) {
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibStandardIdentifiers');

package/src/index.js

@@ -235,7 +235,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // - if creating the featureSet for the incoming record fails we have an unprocessable entity
     // - if creating the featureSet for a candidate record fails we could skip that candidate - but list the case as a detectionFailure, same as conversionFailures
 
-    // eslint-disable-next-line max-statements
     function iterateRecords({records, recordSetSize, maxMatches, matches = [], nonMatches = [], recordMatches = [], recordNonMatches = [], recordCount = 0, recordDuplicateCount = 0, recordNonMatchCount = 0, recordMatchErrors = []}) {
 
       // recordSetSize : total amount of records in the current record set

package/src/match-detection/features/bib/all-source-ids.js

@@ -1,4 +1,3 @@
-/* eslint-disable max-statements */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.

package/src/match-detection/features/bib/index.js

@@ -31,6 +31,7 @@ export {default as isbn} from './isbn';
 export {default as issn} from './issn';
 export {default as otherStandardIdentifier} from './other-standard-identifier';
 export {default as title} from './title';
+export {default as titleVersionOriginal} from './title-version-original';
 export {default as authors} from './authors';
 export {default as recordType} from './record-type';
 export {default as publicationTime} from './publication-time';

package/src/match-detection/features/bib/index.spec.js

@@ -40,7 +40,7 @@ describe('match-detection/features/bib/', () => {
     fixura: {
       reader: READERS.JSON
     },
-    // eslint-disable-next-line max-statements
+
     callback: ({enabled = true, feature, options, type, ...expectations}) => {
 
       if (!enabled) {

package/src/match-detection/features/bib/language.js

@@ -1,4 +1,3 @@
-/* eslint-disable max-statements */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -42,14 +41,14 @@ export default () => ({
     debugData(`${label} 008: ${JSON.stringify(value008)}, 041: ${JSON.stringify(values041)}`);
 
     if (value008 && values041.length > 0) {
-      debugData(`There's both 008 and 041, searching for value in both`);
+      debugData(`${label} There's both 008 and 041, searching for value in both`);
       const correspondingValue = values041.find(v => v === value008);
-      debugData(`Corresponding value: ${correspondingValue}`);
+      debugData(`${label} Corresponding value: ${correspondingValue}`);
       return correspondingValue ? [correspondingValue] : [];
     }
 
     if (!value008 && values041.length < 1) {
-      debugData(`No actual values found`);
+      debugData(`{$label} No actual values found`);
       return [];
     }
 
@@ -57,18 +56,19 @@ export default () => ({
 
     function get008Value() {
       const value = record.get(/^008$/u)?.[0]?.value || undefined;
-      debugData(`008 value: ${value}`);
+      debugData(`${label} 008 value: ${value}`);
 
       if (!value) {
         return undefined;
       }
 
       const code = value.slice(35, 38);
-      debugData(`008 code: ${code}`);
+      debugData(`${label} 008 code: ${code}`);
       return code === '|||' || code === '   ' || code === '^^^' ? undefined : code;
     }
 
     // Main language for the resource: in the first f041 $a or f041 $d
+    // Should we get all $a or $d languages instead of just the first?
     // Uses only f041s that have 2nd ind ' ', which means that the codes used are MARC 21 language codes
 
     function get041Values() {

package/src/match-detection/features/bib/melinda-identifier-factory.js

@@ -62,7 +62,6 @@ export default () => {
 
   }
 
-  // eslint-disable-next-line max-statements
   function compare(a, b) {
 
     if (a.isMelindaRecord && b.isMelindaRecord &&

package/src/match-detection/features/bib/title-version-original.js

@@ -0,0 +1,78 @@
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+
+import createDebugLogger from 'debug';
+import {LevenshteinDistance as leven} from 'natural';
+
+export default ({treshold = 10} = {}) => ({
+  name: 'titleVersionOriginal',
+  extract: ({record}) => {
+    const title = getTitle();
+
+    if (title) {
+      return [title.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
+    }
+
+    return [];
+
+    function getTitle() {
+      const [field] = record.get(/^245$/u);
+
+      if (field) {
+        return field.subfields
+          .filter(({code}) => ['a', 'b'].includes(code))
+          .map(({value}) => value)
+          .join('');
+      }
+    }
+  },
+  compare: (a, b) => {
+    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');
+    const distance = leven(a[0], b[0]);
+
+    if (distance === 0) {
+      return 0.5;
+    }
+
+    const maxLength = getMaxLength();
+    const percentage = distance / maxLength * 100;
+
+    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
+
+    if (percentage <= treshold) {
+      return 0.3;
+    }
+
+    return -0.5;
+
+    function getMaxLength() {
+      return a[0].length > b[0].length ? a[0].length : b[0].length;
+    }
+
+  }
+});

package/src/match-detection/features/bib/title.js

@@ -26,27 +26,45 @@
 *
 */
 
-import createDebugLogger from 'debug';
+
 import {LevenshteinDistance as leven} from 'natural';
 import {testStringOrNumber} from '../../../matching-utils';
+import createDebugLogger from 'debug';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');
+const debugData = debug.extend('data');
+
 
 export default ({treshold = 10} = {}) => ({
   name: 'Title',
-  extract: ({record}) => {
+  extract: ({record, recordExternal}) => {
+    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
     const title = getTitle();
+    debug(`${label} title: ${title}`);
 
     if (testStringOrNumber(title)) {
-      return [String(title).replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
+      const titleAsNormalizedString = String(title)
+        // decompose unicode diacritics
+        .normalize('NFD')
+        // strip non-letters/numbers
+        // - note: combined with decomposing unicode diactics this normalizes both 'saa' and 'sää' as 'saa'
+        // - we could precompose the finnish letters back to avoid this
+        .replace(/[^\p{Letter}\p{Number}]/gu, '')
+        .toLowerCase();
+      debug(`${label} titleString: ${titleAsNormalizedString}`);
+      return [titleAsNormalizedString];
     }
 
     return [];
 
     function getTitle() {
       const [field] = record.get(/^245$/u);
+      debugData(`${label} titleField: ${JSON.stringify(field)}`);
 
       if (field) {
         return field.subfields
-          .filter(({code}) => ['a', 'b'].includes(code))
+          // get also $n:s and $p:s here
+          .filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))
           .map(({value}) => testStringOrNumber(value) ? String(value) : '')
           .join('');
       }

package/src/match-detection/index.js

@@ -1,4 +1,3 @@
-/* eslint-disable no-console */
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -32,7 +31,7 @@ import * as features from './features';
 
 export {features};
 
-export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
+export default ({strategy, treshold = 0.9}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {
   const minProbabilityQuantifier = 0.5;
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');
@@ -41,8 +40,9 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA
   debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
   debugData(`Records: A: ${recordA}\nB: ${recordB}`);
   debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);
+  // We could add here labels for records if we didn't get external labels
 
-  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal});
+  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});
 
   debug(`We got an array of records: ${Array.isArray(recordB)}`);
   const recordsB = Array.isArray(recordB) ? recordB : [recordB];
@@ -54,13 +54,13 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA
   // if we got a singular record, we return a singular result
   return Array.isArray(recordB) ? detectionResults : detectionResults[0];
 
-  function actualDetection({featuresA, record, recordExternal, index}) {
+  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {
     const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';
     const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';
 
 
     debug(`Actual detection for record ${index + 1} ${labelB}`);
-    const featuresB = extractFeatures({record, recordExternal});
+    const featuresB = extractFeatures({record, recordExternal, localSettings});
 
     debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);
     debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);
@@ -69,7 +69,7 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA
     const similarityVector = generateSimilarityVector(featurePairs);
 
     if (similarityVector.some(v => v >= minProbabilityQuantifier)) {
-      const probability = calculateprobability(similarityVector);
+      const probability = calculateProbability(similarityVector);
       debug(`probability: ${probability} (Treshold: ${treshold})`);
       return returnResult({match: probability >= treshold, probability});
     }
@@ -97,7 +97,7 @@ export default ({strategy, treshold = 0.9}, returnStrategy = false) => ({recordA
     return strategyNames || [];
   }
 
-  function calculateprobability(similarityVector) {
+  function calculateProbability(similarityVector) {
     const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);
     return probability > 1.0 ? 1.0 : probability;
   }

package/src/matching-utils.js

@@ -31,7 +31,6 @@ import createDebugLogger from 'debug';
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');
 const debugData = debug.extend('data');
 
-// eslint-disable-next-line max-statements
 export function getMelindaIdsF035(record) {
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');