@natlibfi/melinda-record-matching 5.0.1-alpha.1 → 5.0.2-alpha.1

package/dist/match-detection/features/bib/title-version-original.js

@@ -1,7 +1,7 @@
 import createDebugLogger from "debug";
 import naturalPkg from "natural";
 const { LevenshteinDistance: leven } = naturalPkg;
-export default ({ treshold = 10 } = {}) => ({
+export default ({ threshold = 10 } = {}) => ({
   name: "titleVersionOriginal",
   extract: ({ record }) => {
     const title = getTitle();
@@ -25,7 +25,7 @@ export default ({ treshold = 10 } = {}) => ({
     const maxLength = getMaxLength();
     const percentage = distance / maxLength * 100;
     debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
-    if (percentage <= treshold) {
+    if (percentage <= threshold) {
       return 0.3;
     }
     return -0.5;

package/dist/match-detection/features/bib/title-version-original.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../../src/match-detection/features/bib/title-version-original.js"],
-  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\n\nexport default ({treshold = 10} = {}) => ({\n  name: 'titleVersionOriginal',\n  extract: ({record}) => {\n    const title = getTitle();\n\n    if (title) {\n      return [title.replace(/[^\\p{Letter}\\p{Number}]/gu, '').toLowerCase()];\n    }\n\n    return [];\n\n    function getTitle() {\n      const [field] = record.get(/^245$/u);\n\n      if (field) {\n        return field.subfields\n          .filter(({code}) => ['a', 'b'].includes(code))\n          .map(({value}) => value)\n          .join('');\n      }\n    }\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');\n    const distance = leven(a[0], b[0]);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    const maxLength = getMaxLength();\n    const percentage = distance / maxLength * 100;\n\n    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);\n\n    if (percentage <= treshold) {\n      return 0.3;\n    }\n\n    return -0.5;\n\n    function getMaxLength() {\n      return a[0].length > b[0].length ? a[0].length : b[0].length;\n    }\n\n  }\n});\n"],
-  "mappings": "AACA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AAErC,eAAe,CAAC,EAAC,WAAW,GAAE,IAAI,CAAC,OAAO;AAAA,EACxC,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,UAAM,QAAQ,SAAS;AAEvB,QAAI,OAAO;AACT,aAAO,CAAC,MAAM,QAAQ,6BAA6B,EAAE,EAAE,YAAY,CAAC;AAAA,IACtE;AAEA,WAAO,CAAC;AAER,aAAS,WAAW;AAClB,YAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AAEnC,UAAI,OAAO;AACT,eAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,CAAC,KAAK,GAAG,EAAE,SAAS,IAAI,CAAC,EAC5C,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,KAAK,EAAE;AAAA,MACZ;AAAA,IACF;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,QAAQ,kBAAkB,uFAAuF;AACvH,UAAM,WAAW,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;AAEjC,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,aAAa;AAC/B,UAAM,aAAa,WAAW,YAAY;AAE1C,UAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,SAAS,gBAAgB,QAAQ,kBAAkB,UAAU,EAAE;AAEvG,QAAI,cAAc,UAAU;AAC1B,aAAO;AAAA,IACT;AAEA,WAAO;AAEP,aAAS,eAAe;AACtB,aAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;AAAA,IACxD;AAAA,EAEF;AACF;",
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\n\nexport default ({threshold = 10} = {}) => ({\n  name: 'titleVersionOriginal',\n  extract: ({record}) => {\n    const title = getTitle();\n\n    if (title) {\n      return [title.replace(/[^\\p{Letter}\\p{Number}]/gu, '').toLowerCase()];\n    }\n\n    return [];\n\n    function getTitle() {\n      const [field] = record.get(/^245$/u);\n\n      if (field) {\n        return field.subfields\n          .filter(({code}) => ['a', 'b'].includes(code))\n          .map(({value}) => value)\n          .join('');\n      }\n    }\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');\n    const distance = leven(a[0], b[0]);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    const maxLength = getMaxLength();\n    const percentage = distance / maxLength * 100;\n\n    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);\n\n    if (percentage <= threshold) {\n      return 0.3;\n    }\n\n    return -0.5;\n\n    function getMaxLength() {\n      return a[0].length > b[0].length ? a[0].length : b[0].length;\n    }\n\n  }\n});\n"],
+  "mappings": "AACA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AAErC,eAAe,CAAC,EAAC,YAAY,GAAE,IAAI,CAAC,OAAO;AAAA,EACzC,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,UAAM,QAAQ,SAAS;AAEvB,QAAI,OAAO;AACT,aAAO,CAAC,MAAM,QAAQ,6BAA6B,EAAE,EAAE,YAAY,CAAC;AAAA,IACtE;AAEA,WAAO,CAAC;AAER,aAAS,WAAW;AAClB,YAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AAEnC,UAAI,OAAO;AACT,eAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,CAAC,KAAK,GAAG,EAAE,SAAS,IAAI,CAAC,EAC5C,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,KAAK,EAAE;AAAA,MACZ;AAAA,IACF;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,QAAQ,kBAAkB,uFAAuF;AACvH,UAAM,WAAW,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;AAEjC,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,aAAa;AAC/B,UAAM,aAAa,WAAW,YAAY;AAE1C,UAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,SAAS,gBAAgB,QAAQ,kBAAkB,UAAU,EAAE;AAEvG,QAAI,cAAc,WAAW;AAC3B,aAAO;AAAA,IACT;AAEA,WAAO;AAEP,aAAS,eAAe;AACtB,aAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;AAAA,IACxD;AAAA,EAEF;AACF;",
   "names": []
 }

package/dist/match-detection/features/bib/title.js

@@ -4,7 +4,7 @@ const { LevenshteinDistance: leven } = naturalPkg;
 import { testStringOrNumber } from "../../../matching-utils.js";
 const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features:title");
 const debugData = debug.extend("data");
-export default ({ treshold = 10 } = {}) => ({
+export default ({ threshold = 10 } = {}) => ({
   name: "Title",
   extract: ({ record, recordExternal }) => {
     const label = recordExternal && recordExternal.label ? recordExternal.label : "record";
@@ -34,7 +34,7 @@ export default ({ treshold = 10 } = {}) => ({
     const maxLength = getMaxLength();
     const percentage = distance / maxLength * 100;
     debug2(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
-    if (percentage <= treshold) {
+    if (percentage <= threshold) {
       return 0.3;
     }
     return -0.5;

package/dist/match-detection/features/bib/title.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../../src/match-detection/features/bib/title.js"],
-  "sourcesContent": ["import createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');\nconst debugData = debug.extend('data');\n\n\nexport default ({treshold = 10} = {}) => ({\n  name: 'Title',\n  extract: ({record, recordExternal}) => {\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n    const title = getTitle();\n    debug(`${label}: title: ${title}`);\n\n    if (testStringOrNumber(title)) {\n      const titleAsNormalizedString = String(title)\n        // decompose unicode diacritics\n        .normalize('NFD')\n        // strip non-letters/numbers\n        // - note: combined with decomposing unicode diactics this normalizes both 'saa' and 's\u00E4\u00E4' as 'saa'\n        // - we could precompose the finnish letters back to avoid this\n        .replace(/[^\\p{Letter}\\p{Number}]/gu, '')\n        .toLowerCase();\n      debug(`${label}: titleString: ${titleAsNormalizedString}`);\n      return [titleAsNormalizedString];\n    }\n\n    return [];\n\n    function getTitle() {\n      const [field] = record.get(/^245$/u);\n      debugData(`${label}: titleField: ${JSON.stringify(field)}`);\n\n      if (field) {\n        return field.subfields\n          // get also $n:s and $p:s here\n          .filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))\n          .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n          .join('');\n      }\n      return false;\n    }\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');\n    const distance = leven(a[0], b[0]);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    const maxLength = getMaxLength();\n    const percentage = distance / maxLength * 100;\n\n    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);\n\n    if (percentage <= treshold) {\n      return 0.3;\n    }\n\n    return -0.5;\n\n    function getMaxLength() {\n      return a[0].length > b[0].length ? a[0].length : b[0].length;\n    }\n\n  }\n});\n"],
-  "mappings": "AAAA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AACrC,SAAQ,0BAAyB;AAEjC,MAAM,QAAQ,kBAAkB,kEAAkE;AAClG,MAAM,YAAY,MAAM,OAAO,MAAM;AAGrC,eAAe,CAAC,EAAC,WAAW,GAAE,IAAI,CAAC,OAAO;AAAA,EACxC,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,QAAQ,SAAS;AACvB,UAAM,GAAG,KAAK,YAAY,KAAK,EAAE;AAEjC,QAAI,mBAAmB,KAAK,GAAG;AAC7B,YAAM,0BAA0B,OAAO,KAAK,EAEzC,UAAU,KAAK,EAIf,QAAQ,6BAA6B,EAAE,EACvC,YAAY;AACf,YAAM,GAAG,KAAK,kBAAkB,uBAAuB,EAAE;AACzD,aAAO,CAAC,uBAAuB;AAAA,IACjC;AAEA,WAAO,CAAC;AAER,aAAS,WAAW;AAClB,YAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AACnC,gBAAU,GAAG,KAAK,iBAAiB,KAAK,UAAU,KAAK,CAAC,EAAE;AAE1D,UAAI,OAAO;AACT,eAAO,MAAM,UAEV,OAAO,CAAC,EAAC,KAAI,MAAM,CAAC,KAAK,KAAK,KAAK,GAAG,EAAE,SAAS,IAAI,CAAC,EACtD,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,KAAK,EAAE;AAAA,MACZ;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAMA,SAAQ,kBAAkB,sEAAsE;AACtG,UAAM,WAAW,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;AAEjC,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,aAAa;AAC/B,UAAM,aAAa,WAAW,YAAY;AAE1C,IAAAA,OAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,SAAS,gBAAgB,QAAQ,kBAAkB,UAAU,EAAE;AAEvG,QAAI,cAAc,UAAU;AAC1B,aAAO;AAAA,IACT;AAEA,WAAO;AAEP,aAAS,eAAe;AACtB,aAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;AAAA,IACxD;AAAA,EAEF;AACF;",
+  "sourcesContent": ["import createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');\nconst debugData = debug.extend('data');\n\n\nexport default ({threshold = 10} = {}) => ({\n  name: 'Title',\n  extract: ({record, recordExternal}) => {\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n    const title = getTitle();\n    debug(`${label}: title: ${title}`);\n\n    if (testStringOrNumber(title)) {\n      const titleAsNormalizedString = String(title)\n        // decompose unicode diacritics\n        .normalize('NFD')\n        // strip non-letters/numbers\n        // - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 's\u00E4\u00E4' as 'saa'\n        // - we could precompose the Finnish letters back to avoid this\n        // - see validator normalize-utf8-diacritics for details\n        .replace(/[^\\p{Letter}\\p{Number}]/gu, '')\n        .toLowerCase();\n      debug(`${label}: titleString: ${titleAsNormalizedString}`);\n      return [titleAsNormalizedString];\n    }\n\n    return [];\n\n    function getTitle() {\n      const [field] = record.get(/^245$/u);\n      debugData(`${label}: titleField: ${JSON.stringify(field)}`);\n\n      if (field) {\n        return field.subfields\n          // get also $n:s and $p:s here\n          .filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))\n          .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n          .join('');\n      }\n      return false;\n    }\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');\n    const distance = leven(a[0], b[0]);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    const maxLength = getMaxLength();\n    const percentage = distance / maxLength * 100;\n\n    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);\n\n    if (percentage <= threshold) {\n      return 0.3;\n    }\n\n    return -0.5;\n\n    function getMaxLength() {\n      return a[0].length > b[0].length ? a[0].length : b[0].length;\n    }\n\n  }\n});\n"],
+  "mappings": "AAAA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AACrC,SAAQ,0BAAyB;AAEjC,MAAM,QAAQ,kBAAkB,kEAAkE;AAClG,MAAM,YAAY,MAAM,OAAO,MAAM;AAGrC,eAAe,CAAC,EAAC,YAAY,GAAE,IAAI,CAAC,OAAO;AAAA,EACzC,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,QAAQ,SAAS;AACvB,UAAM,GAAG,KAAK,YAAY,KAAK,EAAE;AAEjC,QAAI,mBAAmB,KAAK,GAAG;AAC7B,YAAM,0BAA0B,OAAO,KAAK,EAEzC,UAAU,KAAK,EAKf,QAAQ,6BAA6B,EAAE,EACvC,YAAY;AACf,YAAM,GAAG,KAAK,kBAAkB,uBAAuB,EAAE;AACzD,aAAO,CAAC,uBAAuB;AAAA,IACjC;AAEA,WAAO,CAAC;AAER,aAAS,WAAW;AAClB,YAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AACnC,gBAAU,GAAG,KAAK,iBAAiB,KAAK,UAAU,KAAK,CAAC,EAAE;AAE1D,UAAI,OAAO;AACT,eAAO,MAAM,UAEV,OAAO,CAAC,EAAC,KAAI,MAAM,CAAC,KAAK,KAAK,KAAK,GAAG,EAAE,SAAS,IAAI,CAAC,EACtD,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,KAAK,EAAE;AAAA,MACZ;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAMA,SAAQ,kBAAkB,sEAAsE;AACtG,UAAM,WAAW,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;AAEjC,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,aAAa;AAC/B,UAAM,aAAa,WAAW,YAAY;AAE1C,IAAAA,OAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,SAAS,gBAAgB,QAAQ,kBAAkB,UAAU,EAAE;AAEvG,QAAI,cAAc,WAAW;AAC3B,aAAO;AAAA,IACT;AAEA,WAAO;AAEP,aAAS,eAAe;AACtB,aAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;AAAA,IACxD;AAAA,EAEF;AACF;",
   "names": ["debug"]
 }

package/dist/match-detection/index.js

@@ -1,11 +1,11 @@
 import createDebugLogger from "debug";
 import * as features from "./features/index.js";
 export { features };
-export default ({ strategy, treshold = 0.9 }, returnStrategy = false, localSettings = {}) => ({ recordA, recordB, recordAExternal = {}, recordBExternal = {} }) => {
+export default ({ strategy, threshold = 0.8999 }, returnStrategy = false, localSettings = {}) => ({ recordA, recordB, recordAExternal = {}, recordBExternal = {} }) => {
   const minProbabilityQuantifier = 0.5;
   const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection");
   const debugData = debug.extend("data");
-  debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
+  debugData(`Strategy: ${JSON.stringify(strategy)}, Threshold: ${JSON.stringify(threshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
   debugData(`Records: A: ${recordA}
 B: ${recordB}`);
   debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);
@@ -26,8 +26,8 @@ B: ${recordB}`);
     const similarityVector = generateSimilarityVector(featurePairs);
     if (similarityVector.some((v) => v >= minProbabilityQuantifier)) {
       const probability = calculateProbability(similarityVector);
-      debug(`probability: ${probability} (Treshold: ${treshold})`);
-      return returnResult({ match: probability >= treshold, probability });
+      debug(`probability: ${probability} (Threshold: ${threshold})`);
+      return returnResult({ match: probability >= threshold, probability });
     }
     debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
     return returnResult({ match: false, probability: 0 });
@@ -38,7 +38,7 @@ B: ${recordB}`);
   function returnResult(result) {
     if (returnStrategy) {
       debug(`Returning detection strategy with the result`);
-      const resultWithStrategy = { match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold };
+      const resultWithStrategy = { match: result.match, probability: result.probability, strategy: formatStrategy(strategy), threshold };
       debugData(`${JSON.stringify(resultWithStrategy)}`);
       return resultWithStrategy;
     }

package/dist/match-detection/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/match-detection/index.js"],
-  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport * as features from './features/index.js';\n\nexport {features};\n\nexport default ({strategy, treshold = 0.9}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {\n  const minProbabilityQuantifier = 0.5;\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n  const debugData = debug.extend('data');\n\n  debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n  debugData(`Records: A: ${recordA}\\nB: ${recordB}`);\n  debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);\n  // We could add here labels for records if we didn't get external labels\n\n  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});\n\n  debug(`We got an array of records: ${Array.isArray(recordB)}`);\n  const recordsB = Array.isArray(recordB) ? recordB : [recordB];\n  const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];\n\n  const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));\n\n  // if we got array of records, we return an array of result\n  // if we got a singular record, we return a singular result\n  return Array.isArray(recordB) ? detectionResults : detectionResults[0];\n\n  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {\n    const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';\n    const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';\n\n\n    debug(`Actual detection for record ${index + 1} ${labelB}`);\n    const featuresB = extractFeatures({record, recordExternal, localSettings});\n\n    debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);\n    debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);\n\n    const featurePairs = generateFeaturePairs(featuresA, featuresB);\n    const similarityVector = generateSimilarityVector(featurePairs);\n\n    if (similarityVector.some(v => v >= minProbabilityQuantifier)) {\n      const probability = calculateProbability(similarityVector);\n      debug(`probability: ${probability} (Treshold: ${treshold})`);\n      return returnResult({match: probability >= treshold, probability});\n    }\n\n    debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);\n    return returnResult({match: false, probability: 0.0});\n  }\n\n  function extractFeatures({record, recordExternal}) {\n    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);\n  }\n\n  function returnResult(result) {\n    if (returnStrategy) {\n      debug(`Returning detection strategy with the result`);\n      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold};\n      debugData(`${JSON.stringify(resultWithStrategy)}`);\n      return resultWithStrategy;\n    }\n    return result;\n  }\n\n  function formatStrategy(strategy) {\n    const strategyNames = strategy.map(element => element.name);\n    return strategyNames || [];\n  }\n\n  function calculateProbability(similarityVector) {\n    const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n    return probability > 1.0 ? 1.0 : probability;\n  }\n\n  function generateSimilarityVector(featurePairs) {\n    const compared = featurePairs.map(({name, a, b}) => {\n      const {compare} = strategy.find(({name: featureName}) => name === featureName);\n      const points = compare(a, b);\n      return {name, points};\n    });\n\n    debugData(`Points: ${JSON.stringify(compared)}`);\n    return compared.map(({points}) => points);\n  }\n\n  function generateFeaturePairs(featuresA, featuresB) {\n    const pairs = generatePairs();\n    const missingFeatures = findMissing();\n\n    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n    return pairs;\n\n    function generatePairs() {\n      return featuresA\n        .reduce((acc, {name, value}, index) => acc.concat({\n          name,\n          a: value,\n          b: featuresB[index].value\n        }), [])\n        .filter(({a, b}) => {\n          if (a.length === 0 || b.length === 0) {\n            return false;\n          }\n\n          return true;\n        });\n    }\n\n    function findMissing() {\n      return featuresA\n        .map(({name}) => name)\n        .filter(v => pairs.some(({name}) => name === v) === false);\n    }\n  }\n\n};\n"],
-  "mappings": "AACA,OAAO,uBAAuB;AAC9B,YAAY,cAAc;AAE1B,SAAQ;AAER,eAAe,CAAC,EAAC,UAAU,WAAW,IAAG,GAAG,iBAAiB,OAAO,gBAAgB,CAAC,MAAM,CAAC,EAAC,SAAS,SAAS,kBAAkB,CAAC,GAAG,kBAAkB,CAAC,EAAC,MAAM;AAC7J,QAAM,2BAA2B;AAEjC,QAAM,QAAQ,kBAAkB,mDAAmD;AACnF,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,YAAU,aAAa,KAAK,UAAU,QAAQ,CAAC,eAAe,KAAK,UAAU,QAAQ,CAAC,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAC3I,YAAU,eAAe,OAAO;AAAA,KAAQ,OAAO,EAAE;AACjD,QAAM,iBAAiB,KAAK,UAAU,eAAe,CAAC,QAAQ,KAAK,UAAU,eAAe,CAAC,EAAE;AAG/F,QAAM,YAAY,gBAAgB,EAAC,QAAQ,SAAS,gBAAgB,iBAAiB,cAAa,CAAC;AAEnG,QAAM,+BAA+B,MAAM,QAAQ,OAAO,CAAC,EAAE;AAC7D,QAAM,WAAW,MAAM,QAAQ,OAAO,IAAI,UAAU,CAAC,OAAO;AAC5D,QAAM,mBAAmB,MAAM,QAAQ,OAAO,IAAI,kBAAkB,CAAC,eAAe;AAEpF,QAAM,mBAAmB,SAAS,IAAI,CAAC,QAAQ,UAAU,gBAAgB,EAAC,WAAW,iBAAiB,QAAQ,gBAAgB,iBAAiB,KAAK,GAAG,MAAK,CAAC,CAAC;AAI9J,SAAO,MAAM,QAAQ,OAAO,IAAI,mBAAmB,iBAAiB,CAAC;AAErE,WAAS,gBAAgB,EAAC,WAAAA,YAAW,QAAQ,gBAAgB,OAAO,eAAAC,eAAa,GAAG;AAClF,UAAM,SAAS,mBAAmB,gBAAgB,QAAQ,gBAAgB,QAAQ;AAClF,UAAM,SAAS,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAG/E,UAAM,+BAA+B,QAAQ,CAAC,IAAI,MAAM,EAAE;AAC1D,UAAM,YAAY,gBAAgB,EAAC,QAAQ,gBAAgB,eAAAA,eAAa,CAAC;AAEzE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAUD,UAAS,CAAC,EAAE;AACjE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAU,SAAS,CAAC,EAAE;AAEjE,UAAM,eAAe,qBAAqBA,YAAW,SAAS;AAC9D,UAAM,mBAAmB,yBAAyB,YAAY;AAE9D,QAAI,iBAAiB,KAAK,OAAK,KAAK,wBAAwB,GAAG;AAC7D,YAAM,cAAc,qBAAqB,gBAAgB;AACzD,YAAM,gBAAgB,WAAW,eAAe,QAAQ,GAAG;AAC3D,aAAO,aAAa,EAAC,OAAO,eAAe,UAAU,YAAW,CAAC;AAAA,IACnE;AAEA,cAAU,4DAA4D,wBAAwB,GAAG;AACjG,WAAO,aAAa,EAAC,OAAO,OAAO,aAAa,EAAG,CAAC;AAAA,EACtD;AAEA,WAAS,gBAAgB,EAAC,QAAQ,eAAc,GAAG;AACjD,WAAO,SAAS,OAAO,CAAC,KAAK,EAAC,MAAM,QAAO,MAAM,IAAI,OAAO,EAAC,MAAM,OAAO,QAAQ,EAAC,QAAQ,eAAc,CAAC,EAAC,CAAC,GAAG,CAAC,CAAC;AAAA,EACnH;AAEA,WAAS,aAAa,QAAQ;AAC5B,QAAI,gBAAgB;AAClB,YAAM,8CAA8C;AACpD,YAAM,qBAAqB,EAAC,OAAO,OAAO,OAAO,aAAa,OAAO,aAAa,UAAU,eAAe,QAAQ,GAAG,SAAQ;AAC9H,gBAAU,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AACjD,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAEA,WAAS,eAAeE,WAAU;AAChC,UAAM,gBAAgBA,UAAS,IAAI,aAAW,QAAQ,IAAI;AAC1D,WAAO,iBAAiB,CAAC;AAAA,EAC3B;AAEA,WAAS,qBAAqB,kBAAkB;AAC9C,UAAM,cAAc,iBAAiB,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAG;AACpE,WAAO,cAAc,IAAM,IAAM;AAAA,EACnC;AAEA,WAAS,yBAAyB,cAAc;AAC9C,UAAM,WAAW,aAAa,IAAI,CAAC,EAAC,MAAM,GAAG,EAAC,MAAM;AAClD,YAAM,EAAC,QAAO,IAAI,SAAS,KAAK,CAAC,EAAC,MAAM,YAAW,MAAM,SAAS,WAAW;AAC7E,YAAM,SAAS,QAAQ,GAAG,CAAC;AAC3B,aAAO,EAAC,MAAM,OAAM;AAAA,IACtB,CAAC;AAED,cAAU,WAAW,KAAK,UAAU,QAAQ,CAAC,EAAE;AAC/C,WAAO,SAAS,IAAI,CAAC,EAAC,OAAM,MAAM,MAAM;AAAA,EAC1C;AAEA,WAAS,qBAAqBF,YAAW,WAAW;AAClD,UAAM,QAAQ,cAAc;AAC5B,UAAM,kBAAkB,YAAY;AAEpC,UAAM,2FAA2F,KAAK,UAAU,eAAe,CAAC,EAAE;AAClI,WAAO;AAEP,aAAS,gBAAgB;AACvB,aAAOA,WACJ,OAAO,CAAC,KAAK,EAAC,MAAM,MAAK,GAAG,UAAU,IAAI,OAAO;AAAA,QAChD;AAAA,QACA,GAAG;AAAA,QACH,GAAG,UAAU,KAAK,EAAE;AAAA,MACtB,CAAC,GAAG,CAAC,CAAC,EACL,OAAO,CAAC,EAAC,GAAG,EAAC,MAAM;AAClB,YAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,iBAAO;AAAA,QACT;AAEA,eAAO;AAAA,MACT,CAAC;AAAA,IACL;AAEA,aAAS,cAAc;AACrB,aAAOA,WACJ,IAAI,CAAC,EAAC,KAAI,MAAM,IAAI,EACpB,OAAO,OAAK,MAAM,KAAK,CAAC,EAAC,KAAI,MAAM,SAAS,CAAC,MAAM,KAAK;AAAA,IAC7D;AAAA,EACF;AAEF;",
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport * as features from './features/index.js';\n\nexport {features};\n\nexport default ({strategy, threshold = 0.8999}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {\n  const minProbabilityQuantifier = 0.5;\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n  const debugData = debug.extend('data');\n\n  debugData(`Strategy: ${JSON.stringify(strategy)}, Threshold: ${JSON.stringify(threshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n  debugData(`Records: A: ${recordA}\\nB: ${recordB}`);\n  debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);\n  // We could add here labels for records if we didn't get external labels\n\n  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});\n\n  debug(`We got an array of records: ${Array.isArray(recordB)}`);\n  const recordsB = Array.isArray(recordB) ? recordB : [recordB];\n  const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];\n\n  const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));\n\n  // if we got array of records, we return an array of result\n  // if we got a singular record, we return a singular result\n  return Array.isArray(recordB) ? detectionResults : detectionResults[0];\n\n  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {\n    const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';\n    const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';\n\n\n    debug(`Actual detection for record ${index + 1} ${labelB}`);\n    const featuresB = extractFeatures({record, recordExternal, localSettings});\n\n    debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);\n    debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);\n\n    const featurePairs = generateFeaturePairs(featuresA, featuresB);\n    const similarityVector = generateSimilarityVector(featurePairs);\n\n    if (similarityVector.some(v => v >= minProbabilityQuantifier)) {\n      const probability = calculateProbability(similarityVector);\n      debug(`probability: ${probability} (Threshold: ${threshold})`);\n      return returnResult({match: probability >= threshold, probability});\n    }\n\n    debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);\n    return returnResult({match: false, probability: 0.0});\n  }\n\n  function extractFeatures({record, recordExternal}) {\n    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);\n  }\n\n  function returnResult(result) {\n    if (returnStrategy) {\n      debug(`Returning detection strategy with the result`);\n      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), threshold};\n      debugData(`${JSON.stringify(resultWithStrategy)}`);\n      return resultWithStrategy;\n    }\n    return result;\n  }\n\n  function formatStrategy(strategy) {\n    const strategyNames = strategy.map(element => element.name);\n    return strategyNames || [];\n  }\n\n  function calculateProbability(similarityVector) {\n    const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n    return probability > 1.0 ? 1.0 : probability;\n  }\n\n  function generateSimilarityVector(featurePairs) {\n    const compared = featurePairs.map(({name, a, b}) => {\n      const {compare} = strategy.find(({name: featureName}) => name === featureName);\n      const points = compare(a, b);\n      return {name, points};\n    });\n\n    debugData(`Points: ${JSON.stringify(compared)}`);\n    return compared.map(({points}) => points);\n  }\n\n  function generateFeaturePairs(featuresA, featuresB) {\n    const pairs = generatePairs();\n    const missingFeatures = findMissing();\n\n    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n    return pairs;\n\n    function generatePairs() {\n      return featuresA\n        .reduce((acc, {name, value}, index) => acc.concat({\n          name,\n          a: value,\n          b: featuresB[index].value\n        }), [])\n        .filter(({a, b}) => {\n          if (a.length === 0 || b.length === 0) {\n            return false;\n          }\n\n          return true;\n        });\n    }\n\n    function findMissing() {\n      return featuresA\n        .map(({name}) => name)\n        .filter(v => pairs.some(({name}) => name === v) === false);\n    }\n  }\n\n};\n"],
+  "mappings": "AACA,OAAO,uBAAuB;AAC9B,YAAY,cAAc;AAE1B,SAAQ;AAER,eAAe,CAAC,EAAC,UAAU,YAAY,OAAM,GAAG,iBAAiB,OAAO,gBAAgB,CAAC,MAAM,CAAC,EAAC,SAAS,SAAS,kBAAkB,CAAC,GAAG,kBAAkB,CAAC,EAAC,MAAM;AACjK,QAAM,2BAA2B;AAEjC,QAAM,QAAQ,kBAAkB,mDAAmD;AACnF,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,YAAU,aAAa,KAAK,UAAU,QAAQ,CAAC,gBAAgB,KAAK,UAAU,SAAS,CAAC,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAC7I,YAAU,eAAe,OAAO;AAAA,KAAQ,OAAO,EAAE;AACjD,QAAM,iBAAiB,KAAK,UAAU,eAAe,CAAC,QAAQ,KAAK,UAAU,eAAe,CAAC,EAAE;AAG/F,QAAM,YAAY,gBAAgB,EAAC,QAAQ,SAAS,gBAAgB,iBAAiB,cAAa,CAAC;AAEnG,QAAM,+BAA+B,MAAM,QAAQ,OAAO,CAAC,EAAE;AAC7D,QAAM,WAAW,MAAM,QAAQ,OAAO,IAAI,UAAU,CAAC,OAAO;AAC5D,QAAM,mBAAmB,MAAM,QAAQ,OAAO,IAAI,kBAAkB,CAAC,eAAe;AAEpF,QAAM,mBAAmB,SAAS,IAAI,CAAC,QAAQ,UAAU,gBAAgB,EAAC,WAAW,iBAAiB,QAAQ,gBAAgB,iBAAiB,KAAK,GAAG,MAAK,CAAC,CAAC;AAI9J,SAAO,MAAM,QAAQ,OAAO,IAAI,mBAAmB,iBAAiB,CAAC;AAErE,WAAS,gBAAgB,EAAC,WAAAA,YAAW,QAAQ,gBAAgB,OAAO,eAAAC,eAAa,GAAG;AAClF,UAAM,SAAS,mBAAmB,gBAAgB,QAAQ,gBAAgB,QAAQ;AAClF,UAAM,SAAS,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAG/E,UAAM,+BAA+B,QAAQ,CAAC,IAAI,MAAM,EAAE;AAC1D,UAAM,YAAY,gBAAgB,EAAC,QAAQ,gBAAgB,eAAAA,eAAa,CAAC;AAEzE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAUD,UAAS,CAAC,EAAE;AACjE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAU,SAAS,CAAC,EAAE;AAEjE,UAAM,eAAe,qBAAqBA,YAAW,SAAS;AAC9D,UAAM,mBAAmB,yBAAyB,YAAY;AAE9D,QAAI,iBAAiB,KAAK,OAAK,KAAK,wBAAwB,GAAG;AAC7D,YAAM,cAAc,qBAAqB,gBAAgB;AACzD,YAAM,gBAAgB,WAAW,gBAAgB,SAAS,GAAG;AAC7D,aAAO,aAAa,EAAC,OAAO,eAAe,WAAW,YAAW,CAAC;AAAA,IACpE;AAEA,cAAU,4DAA4D,wBAAwB,GAAG;AACjG,WAAO,aAAa,EAAC,OAAO,OAAO,aAAa,EAAG,CAAC;AAAA,EACtD;AAEA,WAAS,gBAAgB,EAAC,QAAQ,eAAc,GAAG;AACjD,WAAO,SAAS,OAAO,CAAC,KAAK,EAAC,MAAM,QAAO,MAAM,IAAI,OAAO,EAAC,MAAM,OAAO,QAAQ,EAAC,QAAQ,eAAc,CAAC,EAAC,CAAC,GAAG,CAAC,CAAC;AAAA,EACnH;AAEA,WAAS,aAAa,QAAQ;AAC5B,QAAI,gBAAgB;AAClB,YAAM,8CAA8C;AACpD,YAAM,qBAAqB,EAAC,OAAO,OAAO,OAAO,aAAa,OAAO,aAAa,UAAU,eAAe,QAAQ,GAAG,UAAS;AAC/H,gBAAU,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AACjD,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAEA,WAAS,eAAeE,WAAU;AAChC,UAAM,gBAAgBA,UAAS,IAAI,aAAW,QAAQ,IAAI;AAC1D,WAAO,iBAAiB,CAAC;AAAA,EAC3B;AAEA,WAAS,qBAAqB,kBAAkB;AAC9C,UAAM,cAAc,iBAAiB,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAG;AACpE,WAAO,cAAc,IAAM,IAAM;AAAA,EACnC;AAEA,WAAS,yBAAyB,cAAc;AAC9C,UAAM,WAAW,aAAa,IAAI,CAAC,EAAC,MAAM,GAAG,EAAC,MAAM;AAClD,YAAM,EAAC,QAAO,IAAI,SAAS,KAAK,CAAC,EAAC,MAAM,YAAW,MAAM,SAAS,WAAW;AAC7E,YAAM,SAAS,QAAQ,GAAG,CAAC;AAC3B,aAAO,EAAC,MAAM,OAAM;AAAA,IACtB,CAAC;AAED,cAAU,WAAW,KAAK,UAAU,QAAQ,CAAC,EAAE;AAC/C,WAAO,SAAS,IAAI,CAAC,EAAC,OAAM,MAAM,MAAM;AAAA,EAC1C;AAEA,WAAS,qBAAqBF,YAAW,WAAW;AAClD,UAAM,QAAQ,cAAc;AAC5B,UAAM,kBAAkB,YAAY;AAEpC,UAAM,2FAA2F,KAAK,UAAU,eAAe,CAAC,EAAE;AAClI,WAAO;AAEP,aAAS,gBAAgB;AACvB,aAAOA,WACJ,OAAO,CAAC,KAAK,EAAC,MAAM,MAAK,GAAG,UAAU,IAAI,OAAO;AAAA,QAChD;AAAA,QACA,GAAG;AAAA,QACH,GAAG,UAAU,KAAK,EAAE;AAAA,MACtB,CAAC,GAAG,CAAC,CAAC,EACL,OAAO,CAAC,EAAC,GAAG,EAAC,MAAM;AAClB,YAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,iBAAO;AAAA,QACT;AAEA,eAAO;AAAA,MACT,CAAC;AAAA,IACL;AAEA,aAAS,cAAc;AACrB,aAAOA,WACJ,IAAI,CAAC,EAAC,KAAI,MAAM,IAAI,EACpB,OAAO,OAAK,MAAM,KAAK,CAAC,EAAC,KAAI,MAAM,SAAS,CAAC,MAAM,KAAK;AAAA,IAC7D;AAAA,EACF;AAEF;",
   "names": ["featuresA", "localSettings", "strategy"]
 }

package/package.json

@@ -13,7 +13,7 @@
 		"url": "https://github.com/NatLibFi/melinda-record-matching-js"
 	},
 	"license": "MIT",
-	"version": "5.0.1-alpha.1",
+	"version": "5.0.2-alpha.1",
 	"type": "module",
 	"main": "./dist/index.js",
 	"bin": "./dist/cli.js",
@@ -39,10 +39,11 @@
 	"dependencies": {
 		"@natlibfi/marc-record": "^10.0.1",
 		"@natlibfi/marc-record-serializers": "^11.0.1",
+		"@natlibfi/marc-record-validators-melinda": "^12.0.6",
 		"@natlibfi/melinda-commons": "^14.0.2",
 		"@natlibfi/sru-client": "^7.0.1",
 		"debug": "^4.4.3",
-		"isbn3": "^2.0.2",
+		"isbn3": "^2.0.4",
 		"moment": "^2.30.1",
 		"natural": "^8.1.0",
 		"uuid": "^13.0.0",
@@ -50,10 +51,14 @@
 	},
 	"devDependencies": {
 		"@natlibfi/fixugen": "^3.0.0",
-		"@natlibfi/fixugen-http-client": "^4.0.1",
+		"@natlibfi/fixugen-http-client": "^4.0.2",
 		"@natlibfi/fixura": "^4.0.0",
 		"cross-env": "^10.1.0",
 		"esbuild": "^0.27.2",
 		"eslint": "^9.39.2"
-	}
+	},
+	"overrides": {
+		"nanoid": "^3.3.8"
+	},
+	"nvolkComment": "cld3-asm 4.0.0 uses emscripten-wasm-loader ^3.0.3 which uses problematic, non-secure nanoid version 2.X.X."
 }

package/src/candidate-search/index.js

@@ -112,9 +112,14 @@ export default async ({record, searchSpec, url, maxCandidates, maxRecordsPerRequ
   }
 
   function getRecordId(record) {
-    const [field] = record.get(/^001$/u);
-    return field ? field.value : '';
+    const [f001] = record.get(/^001$/u);
+    const [f003] = record.get(/^003$/u);
+    if (f001 && f003) {
+      return `${f003.value}-${f001.value}`;
+    }
+    return f001 ? f001.value : '';
   }
+
 };
 
 export function retrieveRecords(client, query, resultSetOffset) {

package/src/candidate-search/query-list/bib.js

@@ -152,14 +152,9 @@ export function bibHostComponents(record) {
   }
 }
 
-// SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to
-// (probably) to *headings* index TIT
+// SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to (probably) to *headings* index TIT
 // - Aleph cannot currently handle headings searches starting with a boolean - in these cases use word search
 
-// Headings index TIT drops articles etc. from the start of the title according to the filing indicator
-// Currently filing indicator is not implemented - if the title starts with an article and the Melinda
-// record is correctly catalogued using a filing indicator -> dc.title search won't match
-
 export function bibTitle(record) {
   // We get author/publisher only when formatted title is shorter than 5 chars
   return bibTitleAuthorPublisher({record, onlyTitleLength: 5});
@@ -193,13 +188,7 @@ export function bibTitleAuthorPublisher({record, onlyTitleLength, addYear = fals
   debug(`bibTitleAuthorPublisher, onlyTitleLength: ${onlyTitleLength}, addYear: ${addYear}, alternates: ${alternates}`);
   const title = getTitle();
   if (testStringOrNumber(title)) {
-    const formatted = String(title)
-      .replace(/[^\w\s\p{Alphabetic}]/gu, '')
-      // Clean up concurrent spaces from fe. subfield changes
-      .replace(/ +/gu, ' ')
-      .trim()
-      .slice(0, 30)
-      .trim();
+    const formatted = getFormatted(title);
 
     // use word search for titles starting with a boolean
     const useWordSearch = checkUseWordSearch(formatted);
@@ -216,6 +205,19 @@ export function bibTitleAuthorPublisher({record, onlyTitleLength, addYear = fals
     const newAlternateQueries = alternates ? [...alternateQueries, query] : alternateQueries;
 
     return addAuthorsToSearch({query, queryIsOkAlone, addYear, alternates, alternateQueries: newAlternateQueries});
+
+    function getFormatted(title) {
+      const formatted = String(title)
+        .replace(/[\-+ !"(){}\[\]<>;:.?/@*%=^_`~]/gu, ' ')
+        .replace(/[^\w\s\p{Alphabetic}]/gu, '')
+        // Clean up concurrent spaces from fe. subfield changes
+        .replace(/  +/gu, ' ')
+        .trim()
+        .replace(/^(.{30}\S*) .*$/, "$1")
+        .trim();
+
+      return formatted;
+    }
   }
 
   return [];
@@ -273,6 +275,10 @@ export function bibTitleAuthorPublisher({record, onlyTitleLength, addYear = fals
         .filter(value => value)
         // In Melinda's index subfield separators are indexed as ' '
         .join(' ');
+
+      if (/^[1-9]$/u.test(field.ind2)) { // Skip non-filing characters
+        return titleString.slice(parseInt(field.ind2));
+      }
       return titleString;
     }
     return false;
@@ -316,7 +322,7 @@ export function bibAuthors(record) {
 
     if (field) {
       const authorString = field.subfields
-        .filter(({code}) => ['a'].includes(code))
+        .filter(({code}) => ['a'].includes(code)) // We might use different subfield code sets for X00, X10 and X11?
         .map(({value}) => testStringOrNumber(value) ? String(value) : '')
         .filter(value => value)
         // In Melinda's index subfield separators are indexed as ' '
@@ -393,9 +399,10 @@ export function bibYear(record) {
 }
 
 export function checkUseWordSearch(formatted) {
+  const lowercased = formatted.toLowerCase();
   // Note: add a space to startWords to catch just actual boolean words
   const booleanStartWords = ['and ', 'or ', 'nor ', 'not '];
-  return booleanStartWords.some(word => formatted.toLowerCase().startsWith(word));
+  return booleanStartWords.some(word => lowercased.startsWith(word));
 }
 
 export function bibStandardIdentifiers(record) {

package/src/cli.js

@@ -55,7 +55,7 @@ async function cli() {
   debug(JSON.stringify(args));
 
   const detection = {
-    treshold: 0.9,
+    threshold: 0.8999,
     strategy: generateStrategy(searchType)
   };
 

package/src/index.js

@@ -124,7 +124,7 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
     // - candidate.record
     // - probability
     // - strategy (if returnStrategy option is true)
-    // - treshold (if returnStrategy option is true)
+    // - threshold (if returnStrategy option is true)
     // - matchQuery (if returnQuery option is true)
     // failures: array of conversionFailures from candidate-search and matchErrors from matchDetection in error format {status, payload: {message, id}} if returnFailures is true
 
@@ -257,7 +257,7 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
 
         if (detectionResult.match || returnNonMatches) {
           debug(`${detectionResult.match ? `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is a match!` : `Record ${candidateId} (${newRecordCount}/${recordSetSize}) is NOT a match!`}`);
-          debugData(`Strategy: ${JSON.stringify(detectionResult.strategy)}, Treshold: ${JSON.stringify(detectionResult.treshold)}`);
+          debugData(`Strategy: ${JSON.stringify(detectionResult.strategy)}, Threshold: ${JSON.stringify(detectionResult.threshold)}`);
 
           const matchResult = {
             probability: detectionResult.probability,
@@ -268,7 +268,7 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
           };
           const strategyResult = {
             strategy: detectionResult.strategy,
-            treshold: detectionResult.treshold
+            threshold: detectionResult.threshold
           };
           const newMatch = returnStrategy ? {...matchResult, ...strategyResult} : {...matchResult};
 

package/src/match-detection/features/bib/authors.js

@@ -6,7 +6,7 @@ import {testStringOrNumber} from '../../../matching-utils.js';
 
 // We should extract also organisational authors (110/710)
 
-export default ({nameTreshold = 10} = {}) => ({
+export default ({nameThreshold = 10} = {}) => ({
   name: 'Authors',
   extract: ({record}) => {
     //const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
@@ -58,7 +58,7 @@ export default ({nameTreshold = 10} = {}) => ({
         const maxLength = getMaxLength();
         const percentage = distance / maxLength * 100;
 
-        return percentage <= nameTreshold;
+        return percentage <= nameThreshold;
 
         function getMaxLength() {
           return a.length > b.length ? a.length : b.length;

package/src/match-detection/features/bib/bibliographic-level.js

@@ -2,5 +2,17 @@
 export default () => ({
   name: 'Bibliographic level',
   extract: ({record}) => record.leader[7] ? [record.leader[7]] : [],
-  compare: (a, b) => a[0] === b[0] ? 0.1 : -0.2
+  compare: (a, b) => {
+    if (a[0] === b[0]) {
+      return 0.1;
+    }
+    // See MELINDA-7427: We have millions of LDR/07='b' where they should be LDR/07='a'. So don't penalize for these...
+    if (a[0] === 'a' && b[0] === 'b') {
+      return 0.0;
+    }
+    if (a[0] === 'b' && b[0] === 'a') {
+      return 0.0;
+    }
+    return -0.2;
+  }
 });

package/src/match-detection/features/bib/host-component.js

@@ -1,6 +1,6 @@
 
 export default () => ({
   name: 'Host/Component record',
-  extract: ({record}) => record.get(/^773$/u).length > 0 ? ['component'] : ['host'],
+  extract: ({record}) => record.get(/^[79]73$/u).length > 0 ? ['component'] : ['host'],
   compare: (a, b) => a[0] === b[0] ? 0.0 : -1.0
 });

package/src/match-detection/features/bib/isbn.js

@@ -1,26 +1,174 @@
 
 import createDebugLogger from 'debug';
 import {parse as isbnParse} from 'isbn3';
-import createInterface from './standard-identifier-factory.js';
+
+import {uniqArray} from './issn.js';
+import {isComponentRecord} from '@natlibfi/melinda-commons';
+
 
 const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:ISBN`);
 const debugData = debug.extend('data');
 
-export default () => {
-  const IDENTIFIER_NAME = 'ISBN';
+const MAX_SCORE = 0.75;
+
+export default () => ({
+  name: 'ISBN',
+  extract: ({record/*, recordExternal*/}) => {
+    //const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
+
+    const host = !isComponentRecord(record, false, []);
+
+    if (host) {
+      return record.get('020').filter(f => f.subfields?.some(sf => ['a', 'z'].includes(sf.code) && sf.value));
+    }
+    return record.get('773').filter(f => f.subfields?.some(sf => ['z'].includes(sf.code) && sf.value));
+  },
+  // eslint-disable-next-line max-statements
+  compare: (aa, bb) => {
+    debugData(`Comparing ISBN sets ${JSON.stringify(aa)} and ${JSON.stringify(bb)}`);
+    if (aa.length === 0 || bb.length === 0) {
+      // No data for decision
+      return 0;
+    }
+
+    const [subfieldCodeForGoodValues, subfieldCodeForBadValues] = getSubfieldCodes(aa[0].tag);
+
+    const [goodValuesA, badValuesA] = getValues(aa);
+    if (goodValuesA.length) {
+      debug(`GOOD VALUES (A): ${goodValuesA.join(', ')}`);
+    }
+    if (badValuesA.length) {
+      debug(`BAD VALUES (A): ${badValuesA.join(', ')}`);
+    }
+
+    const [goodValuesB, badValuesB] = getValues(bb);
+    if (goodValuesB.length) {
+      debug(`GOOD VALUES (B): ${goodValuesB.join(', ')}`);
+    }
+    if (badValuesB.length) {
+      debug(`BAD VALUES (B): ${badValuesB.join(', ')}`);
+    }
+
+    const [sharedGoodValues, goodValuesAOnly, goodValuesBOnly] = getUnionData(goodValuesA, goodValuesB);
+
+    //debug(`GOOD\tBOTH: ${sharedGoodValues.length}, A only: ${goodValuesAOnly.length}, B only: ${goodValuesBOnly.length}`);
+
+    const hitScore = scoreHit();
+
+    function scoreHit() {
+      if (sharedGoodValues.length > 0) {
+        return MAX_SCORE;
+      }
+      // One record consider ISBN good and the other record considered it's canceled:
+      if (goodValuesA.some(valA => badValuesB.includes(valA)) || goodValuesB.some(valB => badValuesA.includes(valB))) {
+        return MAX_SCORE;
+      }
+
+      // Value is bad, but looks isbn-ish to a human eye (not validating the isbn again, note than invalid isbns in 020$a are considered bad):
+      // Could happen for two canceled ISBNs for example. I'll give this two thirds of the full score
+      if (badValuesA.some(valA => looksGood(valA) && badValuesB.includes(valA)) || badValuesB.some(valB => looksGood(valB) && badValuesA.includes(valB))) {
+        return MAX_SCORE * 2 / 3;
+      }
+
+      return 0;
+    }
+
+    if (sharedGoodValues.length > 0) {
+      // Third argument (aka 'N times') is >= 0
+      return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);
+    }
+
+    if (hitScore === MAX_SCORE) {
+      // -1 is needed to make the third argument >= 0 (otherwise min val would be 0)
+      return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length - 1);
+    }
+
+    if (hitScore > 0) { // Canceled/invalid ISBNs match
+      // Note that this is not (currently) penalized for non-matching canceled/invalid ISBNs. Maybe it should be.
+      return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);
+    }
+
+    // No match:
+
+    if (goodValuesA.length === 0 || goodValuesB === 0) { // At least one record did not have any good ISBNs, so not penalizing here! (Invalid 020$as are counted a bad.)
+      return 0.0;
+    }
+
+    return -0.75; // Has good ISBNs on both records, but they did not match
+
+
+    function getSubfieldCodes(tag) {
+      if (tag === '773') {
+        return ['z', undefined];
+      }
+      return ['a', 'z'];
+    }
+
+    function looksGood(val) {
+      // isbn10 can end in X:
+      if (/^([0-9]-?){9}[0-9X]$/u.test(val)) {
+        return true;
+      }
+      // isbn13 can not:
+      if (/^([0-9]-?){12}[0-9]$/u.test(val)) {
+        return true;
+      }
+      return false;
+    }
+
+    function getValues(fields) {
+      const goodValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForGoodValues)).map(sf => validatorAndNormalizer(sf.value));
+      const trueGoodValues = goodValues.filter(val => val.valid).map(val => val.value);
+      const wannabeGoodValues = goodValues.filter(val => !val.valid).map(val => val.value);
+      if (!subfieldCodeForBadValues) { // 773
+        return [trueGoodValues, wannabeGoodValues];
+      }
+      const badValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForBadValues)).map(sf => sf.value);
+      return [uniqArray(trueGoodValues), uniqArray([...wannabeGoodValues, ...badValues])];
+    }
+
+    function validatorAndNormalizer(string) {
+      const string2 = string.replace(/\. -$/u, ''); // Remove punctuation (773$z)
+
+      // Hack: Historically we 020$a "1234567890 sidottu" etc. Try the LHS alone:
+      const string3 = string2.replace(/ .*$/u, '');
+      if (string2 !== string3) {
+        const altResult = validatorAndNormalizer(string3);
+        if (altResult.valid) {
+          return altResult;
+        }
+      }
+
+      const isbnParseResult = isbnParse(string2, '') || '';
+      debugData(`isbnParseResult: ${JSON.stringify(isbnParseResult)}`);
+      if (!isbnParseResult.isValid) {
+        debug(`Not parseable ISBN '${string2}', just removing hyphens`);
+        return {valid: false, value: string2.replace(/-/ug, '')};
+      }
+
+      debug(`Parseable ISBN '${string2}', normalizing to ISBN-13 '${isbnParseResult.isbn13}'`);
+      return {valid: true, value: isbnParseResult.isbn13};
+
 
-  function validatorAndNormalizer(string) {
-    const isbnParseResult = isbnParse(string);
-    debugData(`isbnParseResult: ${JSON.stringify(isbnParseResult)}`);
-    if (isbnParseResult === null) {
-      debug(`Not parseable ISBN, just removing hyphens`);
-      return {valid: false, value: string.replace(/-/ug, '')};
     }
-    debug(`Parseable ISBN, normalizing to ISBN-13`);
-    return {valid: true, value: isbnParseResult.isbn13};
   }
+});
+
+// These are outside the default function as I'll probably want to export these later on
 
-  const {extract, compare} = createInterface({identifier: IDENTIFIER_NAME, pattern: /^020$/u, subfieldCodes: ['a', 'z'], validIdentifierSubfieldCodes: ['a'], invalidIdentifierSubfieldCodes: ['z'], validatorAndNormalizer});
-  return {extract, compare, name: IDENTIFIER_NAME};
-};
+function getUnionData(set1, set2) {
+  const shared = set1.filter(val => set2.includes(val));
+  const onlyInSet1 = set1.filter(val => !shared.includes(val));
+  const onlyInSet2 = set2.filter(val => !shared.includes(val));
+  return [shared, onlyInSet1, onlyInSet2];
+}
 
+function scoreData(score, factor, n) {
+  return innerScoreData(score, n);
+  function innerScoreData(currScore, remaining) {
+    if (remaining > 0) {
+      return innerScoreData(currScore * factor, remaining-1);
+    }
+    return Math.round(currScore * 100)/100; // 0.600000000001 => 0.6
+  }
+}