@natlibfi/melinda-record-matching 5.0.0-alpha.1 → 5.0.0-alpha.3

package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/match-detection/features/bib/publication-time-allow-cons-years-multi.js"],
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\n// We should also get copyright time and copyright/publication times from 26x\n// We should also get publishing time type from f008\n// We should get reprint times from f500 $a \"Lis\u00E4painos/Lis\u00E4painokset:\"\n\nexport default () => ({\n  name: 'Publication time, allow consequent years, years from multiple sources',\n  extract: ({record, recordExternal}) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years-multi');\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n\n    const f008Values = extractF008Values(record);\n    debug(`${label}: f008: ${JSON.stringify(f008Values)}`);\n\n    const f26xValues = extractF26xValues(record);\n    debug(`${label}: f26x: ${JSON.stringify(f26xValues)}`);\n\n    const f500Values = extractF500Years(record);\n    debug(`${label}: f500: ${JSON.stringify(f500Values)}`);\n\n    // We should get copyrightYear from f008Date2 to copyrightYears when f008YearType = 'r'\n    // Is the original year (f008Date2) in f008YearType === 'r' comparable to copyrightYear?\n    // We should handle unknown years (here or comparison?)\n    // We should handle year ranges for continuing resources / collections\n\n    const normalYears = [...new Set(f26xValues.normalYears.concat(f008Values.f008Date1).filter(value => value && value !== '    ' && value !== '||||'))].sort();\n    const copyrightYears = [...new Set(f26xValues.copyrightYears)].sort();\n    const reprintYears = [...new Set(f500Values)].sort();\n\n    const combined = {normalYears, copyrightYears, reprintYears};\n\n    debug(`Combined: ${JSON.stringify(combined)}`);\n\n    return combined;\n\n    function extractF008Values(record) {\n      // Record should have only one f008 - in case of several, we handle the first one\n      const value = record.get(/^008$/u)?.[0]?.value || undefined;\n      if (value && testStringOrNumber(value)) {\n        const f008Date1 = extractF008Date1(value);\n        const f008Date2 = extractF008Date2(value);\n        const f008YearType = extractF008YearType(value);\n        return {f008Date1, f008Date2, f008YearType};\n      }\n      return {f008Date1: undefined, f008Date2: undefined, f008YearType: undefined};\n\n      function extractF008Date1(value) {\n        return String(value).slice(7, 11);\n      }\n\n      function extractF008Date2(value) {\n        return String(value).slice(11, 15);\n      }\n\n      function extractF008YearType(value) {\n        return String(value).slice(6, 7);\n      }\n    }\n\n    function extractF26xValues(record) {\n      const copyrightRegex = /^(?<copyrightPrefix>cop|cop.|c|\u00A9|p|\u2117)/u;\n\n      const pubNormalSubFieldValues = extractPubNormalSubfieldValues(record, copyrightRegex);\n      debug(`Normal years: ${JSON.stringify(pubNormalSubFieldValues)}`);\n\n      const pubF264CopySubFieldValues = extractPubF264CopySubfieldValues(record);\n      debug(`F264 copyright years: ${JSON.stringify(pubF264CopySubFieldValues)}`);\n\n      const pubF260CopySubFieldValues = extractPubF260CopySubfieldValues(record, copyrightRegex);\n      debug(`F260 copyright years: ${JSON.stringify(pubF260CopySubFieldValues)}`);\n\n      return {normalYears: pubNormalSubFieldValues, copyrightYears: [...pubF260CopySubFieldValues, ...pubF264CopySubFieldValues]};\n\n      function extractPubNormalSubfieldValues(record, copyrightRegex) {\n        return record.get(/^26[04]$/u)\n          .filter((field) => !(field.tag === '264' && field.ind2 === '4'))\n          .map(({subfields}) => subfields)\n          .flat()\n          .filter(({code}) => code && code === 'c')\n          .filter(({value}) => value && !copyrightRegex.test(value))\n          .map(({value}) => value)\n          .map((value) => removeNonAlphaNumeric(value));\n      }\n\n      function extractPubF264CopySubfieldValues(record, copyrightRegex) {\n        return record.get(/^264$/u)\n          .filter((field) => field.ind2 === '4')\n          .map(({subfields}) => subfields)\n          .flat()\n          .filter(({code}) => code && code === 'c')\n          .filter(({value}) => value)\n          .map(({value}) => value)\n          .map((value) => value.replace(copyrightRegex, ''))\n          .map((value) => removeNonAlphaNumeric(value));\n      }\n\n      function extractPubF260CopySubfieldValues(record, copyrightRegex) {\n        return record.get(/^260$/u)\n          .map(({subfields}) => subfields)\n          .flat()\n          .filter(({code}) => code && code === 'c')\n          .filter(({value}) => value && copyrightRegex.test(value))\n          .map(({value}) => value)\n          .map((value) => value.replace(copyrightRegex, ''))\n          .map((value) => removeNonAlphaNumeric(value));\n      }\n\n      function removeNonAlphaNumeric(value) {\n        debug(`Cleaning: ${JSON.stringify(value)}`);\n        const nonAlphaNumericRegex = /[^A-Za-z0-9]/ug;\n        return value ? value.replace(nonAlphaNumericRegex, '') : value;\n      }\n    }\n\n    function extractF500Years(record) {\n\n      const reprintRegex = /(?<reprint>Lis\u00E4painokset:|Lis\u00E4painos:)/u;\n      const reprintFieldContents = record.get(/^500$/u)\n        .map(({subfields}) => subfields)\n        .flat()\n        .filter(({code}) => code === 'a')\n        .map(({value}) => value);\n      //.filter(value => value.test(reprintRegex));\n\n      debug(`f500 reprint field contents: ${JSON.stringify(reprintFieldContents)}`);\n\n      const filteredF500 = reprintFieldContents.filter((content) => content && content.match(reprintRegex));\n\n      debug(`f500 reprint field contents (filtered): ${JSON.stringify(filteredF500)}`);\n\n      const reprintYears = extractReprintYears(filteredF500);\n\n      return reprintYears;\n    }\n\n    function extractReprintYears(contents) {\n      const yearRegex = /[0-9][0-9][0-9][0-9]/gu;\n      const years = contents.map(content => content.match(yearRegex))\n        .flat();\n      debug(`${JSON.stringify(years)}`);\n      const uniqYears = [...new Set(years)].sort();\n      debug(`${JSON.stringify(uniqYears)}`);\n      return uniqYears;\n    }\n\n  },\n  // eslint-disable-next-line max-statements\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years-multi');\n    debug(`Comparing ${JSON.stringify(a)} to ${JSON.stringify(b)}`);\n\n    const [firstA] = a.normalYears ? a.normalYears : a;\n    const [firstB] = b.normalYears ? b.normalYears : b;\n\n    debug(`Comparing ${JSON.stringify(firstA)} to ${JSON.stringify(firstB)}`);\n\n    if (firstA === firstB) {\n      return 0.1;\n    }\n\n    // If either of years is a non string/number, values are not comparable\n    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {\n      return 0;\n    }\n\n    const firstANumber = parseInt(firstA, 10);\n    const firstBNumber = parseInt(firstB, 10);\n\n    // Handle consequent years as a match\n    // see publication-time for a version that does not handle consequent years as a match\n    if (!(isNaN(firstANumber) || isNaN(firstBNumber)) &&\n      (firstANumber + 1 === firstBNumber || firstANumber - 1 === firstBNumber)) {\n      return 0.1;\n    }\n\n    // We should do something with copyrightYears, too\n\n    // Do not give minus points if a normal publishing year is found in normal years\n    const bNormalInANormal = a.normalYears.filter(aValue => b.normalYears.some(bValue => aValue === bValue));\n    const aNormalInBNormal = b.normalYears.filter(bValue => a.normalYears.some(aValue => bValue === aValue));\n    debug(`BNorm in ANorm: ${JSON.stringify(bNormalInANormal)}`);\n    debug(`ANorm in BNorm: ${JSON.stringify(aNormalInBNormal)}`);\n\n    if (bNormalInANormal > 0 || aNormalInBNormal > 0) {\n      return 0;\n    }\n\n    // Do not give minus points if a normal publishing year is found in reprint years\n    const bNormalInAReprint = a.reprintYears.filter(aValue => b.normalYears.some(bValue => aValue === bValue));\n    const aNormalInBReprint = b.reprintYears.filter(bValue => a.normalYears.some(aValue => bValue === aValue));\n    debug(`BNorm in AReprint: ${JSON.stringify(bNormalInAReprint)}`);\n    debug(`ANorm in BReprint: ${JSON.stringify(aNormalInBReprint)}`);\n\n    if (bNormalInAReprint > 0 || aNormalInBReprint > 0) {\n      return 0;\n    }\n\n    return -1;\n\n  }\n});\n\n// https://www.loc.gov/marc/bibliographic/bd008.html\n// field 008\n// 06 - Type of date/Publication status\n// 07-10 - Date 1\n// 11-14 - Date 2\n//\n// 06 - Type of date/Publication status\n// b - No dates given; B.C. date involved\n// c - Continuing resource currently published\n// d - Continuing resource ceased publication\n// e - Detailed date\n// i - Inclusive dates of collection\n// k - Range of years of bulk of collection\n// m - Multiple dates\n// n - Dates unknown\n// p - Date of distribution/release/issue and production/recording session when different\n// q - Questionable date\n// r - Reprint/reissue date and original date\n// s - Single known date/probable date\n// t - Publication date and copyright date\n// u - Continuing resource status unknown\n// | - No attempt to code\n//\n// 07-10 - Date 1\n// 1-9 - Date digit\n// # - Date element is not applicable\n// u - Date element is totally or partially unknown\n// |||| - No attempt to code\n//\n// 11-14 - Date 2\n// 1-9 - Date digit\n// # - Date element is not applicable\n// u - Date element is totally or partially unknown\n// |||| - No attempt to code\n//\n\n"],
+  "mappings": "AACA,OAAO,uBAAuB;AAC9B,SAAQ,0BAAyB;AAMjC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,wGAAwG;AACxI,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAE9E,UAAM,aAAa,kBAAkB,MAAM;AAC3C,UAAM,GAAG,KAAK,WAAW,KAAK,UAAU,UAAU,CAAC,EAAE;AAErD,UAAM,aAAa,kBAAkB,MAAM;AAC3C,UAAM,GAAG,KAAK,WAAW,KAAK,UAAU,UAAU,CAAC,EAAE;AAErD,UAAM,aAAa,iBAAiB,MAAM;AAC1C,UAAM,GAAG,KAAK,WAAW,KAAK,UAAU,UAAU,CAAC,EAAE;AAOrD,UAAM,cAAc,CAAC,GAAG,IAAI,IAAI,WAAW,YAAY,OAAO,WAAW,SAAS,EAAE,OAAO,WAAS,SAAS,UAAU,UAAU,UAAU,MAAM,CAAC,CAAC,EAAE,KAAK;AAC1J,UAAM,iBAAiB,CAAC,GAAG,IAAI,IAAI,WAAW,cAAc,CAAC,EAAE,KAAK;AACpE,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,KAAK;AAEnD,UAAM,WAAW,EAAC,aAAa,gBAAgB,aAAY;AAE3D,UAAM,aAAa,KAAK,UAAU,QAAQ,CAAC,EAAE;AAE7C,WAAO;AAEP,aAAS,kBAAkBA,SAAQ;AAEjC,YAAM,QAAQA,QAAO,IAAI,QAAQ,IAAI,CAAC,GAAG,SAAS;AAClD,UAAI,SAAS,mBAAmB,KAAK,GAAG;AACtC,cAAM,YAAY,iBAAiB,KAAK;AACxC,cAAM,YAAY,iBAAiB,KAAK;AACxC,cAAM,eAAe,oBAAoB,KAAK;AAC9C,eAAO,EAAC,WAAW,WAAW,aAAY;AAAA,MAC5C;AACA,aAAO,EAAC,WAAW,QAAW,WAAW,QAAW,cAAc,OAAS;AAE3E,eAAS,iBAAiBC,QAAO;AAC/B,eAAO,OAAOA,MAAK,EAAE,MAAM,GAAG,EAAE;AAAA,MAClC;AAEA,eAAS,iBAAiBA,QAAO;AAC/B,eAAO,OAAOA,MAAK,EAAE,MAAM,IAAI,EAAE;AAAA,MACnC;AAEA,eAAS,oBAAoBA,QAAO;AAClC,eAAO,OAAOA,MAAK,EAAE,MAAM,GAAG,CAAC;AAAA,MACjC;AAAA,IACF;AAEA,aAAS,kBAAkBD,SAAQ;AACjC,YAAM,iBAAiB;AAEvB,YAAM,0BAA0B,+BAA+BA,SAAQ,cAAc;AACrF,YAAM,iBAAiB,KAAK,UAAU,uBAAuB,CAAC,EAAE;AAEhE,YAAM,4BAA4B,iCAAiCA,OAAM;AACzE,YAAM,yBAAyB,KAAK,UAAU,yBAAyB,CAAC,EAAE;AAE1E,YAAM,4BAA4B,iCAAiCA,SAAQ,cAAc;AACzF,YAAM,yBAAyB,KAAK,UAAU,yBAAyB,CAAC,EAAE;AAE1E,aAAO,EAAC,aAAa,yBAAyB,gBAAgB,CAAC,GAAG,2BAA2B,GAAG,yBAAyB,EAAC;AAE1H,eAAS,+BAA+BA,SAAQE,iBAAgB;AAC9D,eAAOF,QAAO,IAAI,WAAW,EAC1B,OAAO,CAAC,UAAU,EAAE,MAAM,QAAQ,SAAS,MAAM,SAAS,IAAI,EAC9D,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,QAAQ,SAAS,GAAG,EACvC,OAAO,CAAC,EAAC,MAAK,MAAM,SAAS,CAACE,gBAAe,KAAK,KAAK,CAAC,EACxD,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,IAAI,CAAC,UAAU,sBAAsB,KAAK,CAAC;AAAA,MAChD;AAEA,eAAS,iCAAiCF,SAAQE,iBAAgB;AAChE,eAAOF,QAAO,IAAI,QAAQ,EACvB,OAAO,CAAC,UAAU,MAAM,SAAS,GAAG,EACpC,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,QAAQ,SAAS,GAAG,EACvC,OAAO,CAAC,EAAC,MAAK,MAAM,KAAK,EACzB,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,IAAI,CAAC,UAAU,MAAM,QAAQE,iBAAgB,EAAE,CAAC,EAChD,IAAI,CAAC,UAAU,sBAAsB,KAAK,CAAC;AAAA,MAChD;AAEA,eAAS,iCAAiCF,SAAQE,iBAAgB;AAChE,eAAOF,QAAO,IAAI,QAAQ,EACvB,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,QAAQ,SAAS,GAAG,EACvC,OAAO,CAAC,EAAC,MAAK,MAAM,SAASE,gBAAe,KAAK,KAAK,CAAC,EACvD,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,IAAI,CAAC,UAAU,MAAM,QAAQA,iBAAgB,EAAE,CAAC,EAChD,IAAI,CAAC,UAAU,sBAAsB,KAAK,CAAC;AAAA,MAChD;AAEA,eAAS,sBAAsB,OAAO;AACpC,cAAM,aAAa,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1C,cAAM,uBAAuB;AAC7B,eAAO,QAAQ,MAAM,QAAQ,sBAAsB,EAAE,IAAI;AAAA,MAC3D;AAAA,IACF;AAEA,aAAS,iBAAiBF,SAAQ;AAEhC,YAAM,eAAe;AACrB,YAAM,uBAAuBA,QAAO,IAAI,QAAQ,EAC7C,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,GAAG,EAC/B,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK;AAGzB,YAAM,gCAAgC,KAAK,UAAU,oBAAoB,CAAC,EAAE;AAE5E,YAAM,eAAe,qBAAqB,OAAO,CAAC,YAAY,WAAW,QAAQ,MAAM,YAAY,CAAC;AAEpG,YAAM,2CAA2C,KAAK,UAAU,YAAY,CAAC,EAAE;AAE/E,YAAMG,gBAAe,oBAAoB,YAAY;AAErD,aAAOA;AAAA,IACT;AAEA,aAAS,oBAAoB,UAAU;AACrC,YAAM,YAAY;AAClB,YAAM,QAAQ,SAAS,IAAI,aAAW,QAAQ,MAAM,SAAS,CAAC,EAC3D,KAAK;AACR,YAAM,GAAG,KAAK,UAAU,KAAK,CAAC,EAAE;AAChC,YAAM,YAAY,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,EAAE,KAAK;AAC3C,YAAM,GAAG,KAAK,UAAU,SAAS,CAAC,EAAE;AACpC,aAAO;AAAA,IACT;AAAA,EAEF;AAAA;AAAA,EAEA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,QAAQ,kBAAkB,wGAAwG;AACxI,UAAM,aAAa,KAAK,UAAU,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,EAAE;AAE9D,UAAM,CAAC,MAAM,IAAI,EAAE,cAAc,EAAE,cAAc;AACjD,UAAM,CAAC,MAAM,IAAI,EAAE,cAAc,EAAE,cAAc;AAEjD,UAAM,aAAa,KAAK,UAAU,MAAM,CAAC,OAAO,KAAK,UAAU,MAAM,CAAC,EAAE;AAExE,QAAI,WAAW,QAAQ;AACrB,aAAO;AAAA,IACT;AAGA,QAAI,CAAC,mBAAmB,MAAM,KAAK,CAAC,mBAAmB,MAAM,GAAG;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,SAAS,QAAQ,EAAE;AACxC,UAAM,eAAe,SAAS,QAAQ,EAAE;AAIxC,QAAI,EAAE,MAAM,YAAY,KAAK,MAAM,YAAY,OAC5C,eAAe,MAAM,gBAAgB,eAAe,MAAM,eAAe;AAC1E,aAAO;AAAA,IACT;AAKA,UAAM,mBAAmB,EAAE,YAAY,OAAO,YAAU,EAAE,YAAY,KAAK,YAAU,WAAW,MAAM,CAAC;AACvG,UAAM,mBAAmB,EAAE,YAAY,OAAO,YAAU,EAAE,YAAY,KAAK,YAAU,WAAW,MAAM,CAAC;AACvG,UAAM,mBAAmB,KAAK,UAAU,gBAAgB,CAAC,EAAE;AAC3D,UAAM,mBAAmB,KAAK,UAAU,gBAAgB,CAAC,EAAE;AAE3D,QAAI,mBAAmB,KAAK,mBAAmB,GAAG;AAChD,aAAO;AAAA,IACT;AAGA,UAAM,oBAAoB,EAAE,aAAa,OAAO,YAAU,EAAE,YAAY,KAAK,YAAU,WAAW,MAAM,CAAC;AACzG,UAAM,oBAAoB,EAAE,aAAa,OAAO,YAAU,EAAE,YAAY,KAAK,YAAU,WAAW,MAAM,CAAC;AACzG,UAAM,sBAAsB,KAAK,UAAU,iBAAiB,CAAC,EAAE;AAC/D,UAAM,sBAAsB,KAAK,UAAU,iBAAiB,CAAC,EAAE;AAE/D,QAAI,oBAAoB,KAAK,oBAAoB,GAAG;AAClD,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EAET;AACF;",
+  "names": ["record", "value", "copyrightRegex", "reprintYears"]
+}

package/dist/match-detection/features/bib/publication-time-allow-cons-years.js

@@ -0,0 +1,28 @@
+import createDebugLogger from "debug";
+import { testStringOrNumber } from "../../../matching-utils.js";
+export default () => ({
+  name: "Publication time, allow consequent years",
+  extract: ({ record }) => {
+    const value = record.get(/^008$/u)?.[0]?.value || void 0;
+    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];
+  },
+  compare: (a, b) => {
+    const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years");
+    debug(`Comparing ${a[0]} to ${b[0]}`);
+    const [firstA] = a;
+    const [firstB] = b;
+    if (firstA === firstB) {
+      return 0.1;
+    }
+    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {
+      return 0;
+    }
+    const firstANumber = parseInt(firstA, 10);
+    const firstBNumber = parseInt(firstB, 10);
+    if (isNaN(firstANumber) || isNaN(firstBNumber)) {
+      return -1;
+    }
+    return firstANumber + 1 === firstBNumber || firstANumber - 1 === firstBNumber ? 0.1 : -1;
+  }
+});
+//# sourceMappingURL=publication-time-allow-cons-years.js.map

package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/match-detection/features/bib/publication-time-allow-cons-years.js"],
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\n// We should also get copyright time and copyright/publication times from 26x\n\nexport default () => ({\n  name: 'Publication time, allow consequent years',\n  extract: ({record}) => {\n    const value = record.get(/^008$/u)?.[0]?.value || undefined;\n    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years');\n    debug(`Comparing ${a[0]} to ${b[0]}`);\n\n    const [firstA] = a;\n    const [firstB] = b;\n\n    if (firstA === firstB) {\n      return 0.1;\n    }\n\n    // If either of years is a non string/number, values are not comparable\n    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {\n      return 0;\n    }\n\n    const firstANumber = parseInt(firstA, 10);\n    const firstBNumber = parseInt(firstB, 10);\n\n    if (isNaN(firstANumber) || isNaN(firstBNumber)) {\n      return -1;\n    }\n\n    // Handle consequent years as a match\n    // see publication-time for a version that does not handle consequent years as a match\n    return firstANumber + 1 === firstBNumber || firstANumber - 1 === firstBNumber ? 0.1 : -1;\n  }\n});\n"],
+  "mappings": "AACA,OAAO,uBAAuB;AAC9B,SAAQ,0BAAyB;AAIjC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,UAAM,QAAQ,OAAO,IAAI,QAAQ,IAAI,CAAC,GAAG,SAAS;AAClD,WAAO,mBAAmB,KAAK,IAAI,CAAC,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC;AAAA,EACrE;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,QAAQ,kBAAkB,kGAAkG;AAClI,UAAM,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE;AAEpC,UAAM,CAAC,MAAM,IAAI;AACjB,UAAM,CAAC,MAAM,IAAI;AAEjB,QAAI,WAAW,QAAQ;AACrB,aAAO;AAAA,IACT;AAGA,QAAI,CAAC,mBAAmB,MAAM,KAAK,CAAC,mBAAmB,MAAM,GAAG;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,SAAS,QAAQ,EAAE;AACxC,UAAM,eAAe,SAAS,QAAQ,EAAE;AAExC,QAAI,MAAM,YAAY,KAAK,MAAM,YAAY,GAAG;AAC9C,aAAO;AAAA,IACT;AAIA,WAAO,eAAe,MAAM,gBAAgB,eAAe,MAAM,eAAe,MAAM;AAAA,EACxF;AACF;",
+  "names": []
+}

package/dist/match-detection/features/bib/publication-time.js

@@ -0,0 +1,10 @@
+import { testStringOrNumber } from "../../../matching-utils.js";
+export default () => ({
+  name: "Publication time",
+  extract: ({ record }) => {
+    const value = record.get(/^008$/u)?.[0]?.value || void 0;
+    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];
+  },
+  compare: (a, b) => a[0] === b[0] ? 0.1 : -1
+});
+//# sourceMappingURL=publication-time.js.map

package/dist/match-detection/features/bib/publication-time.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/match-detection/features/bib/publication-time.js"],
+  "sourcesContent": ["\n\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\n// We should also get copyright time and copyright/publication times from 26x\n// see publication-time-allow-cons-years for a version allowing consequent years to match\n\nexport default () => ({\n  name: 'Publication time',\n  extract: ({record}) => {\n    const value = record.get(/^008$/u)?.[0]?.value || undefined;\n    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];\n  },\n  compare: (a, b) => a[0] === b[0] ? 0.1 : -1.0\n});\n"],
+  "mappings": "AAEA,SAAQ,0BAAyB;AAKjC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,UAAM,QAAQ,OAAO,IAAI,QAAQ,IAAI,CAAC,GAAG,SAAS;AAClD,WAAO,mBAAmB,KAAK,IAAI,CAAC,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC;AAAA,EACrE;AAAA,EACA,SAAS,CAAC,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,MAAM;AAC3C;",
+  "names": []
+}

package/dist/match-detection/features/bib/record-type.js

@@ -0,0 +1,6 @@
+export default () => ({
+  name: "Record type",
+  extract: ({ record }) => record.leader[6] ? [record.leader[6]] : [],
+  compare: (a, b) => a[0] === b[0] ? 0.1 : -0.5
+});
+//# sourceMappingURL=record-type.js.map

package/dist/match-detection/features/bib/record-type.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/match-detection/features/bib/record-type.js"],
+  "sourcesContent": ["\n// we could handle the case of books/notes\n// Recordtype: LDR/06 - Type of Record\n// Note: currently matchValidator fails all mismatching recordTypes, so this won't actually do much\n\nexport default () => ({\n  name: 'Record type',\n  extract: ({record}) => record.leader[6] ? [record.leader[6]] : [],\n  compare: (a, b) => a[0] === b[0] ? 0.1 : -0.5\n});\n"],
+  "mappings": "AAKA,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,OAAO,CAAC,CAAC,IAAI,CAAC;AAAA,EAChE,SAAS,CAAC,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,MAAM;AAC3C;",
+  "names": []
+}

package/dist/match-detection/features/bib/standard-identifier-factory.js

@@ -0,0 +1,99 @@
+import createDebugLogger from "debug";
+import { extractSubfieldsFromField, uniqueSubfields } from "../../../matching-utils.js";
+export default ({ pattern, subfieldCodes, identifier, validIdentifierSubfieldCodes = ["a"], invalidIdentifierSubfieldCodes = ["z"], validatorAndNormalizer = void 0 }) => {
+  const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);
+  const debugData = debug.extend("data");
+  return { extract, compare };
+  function extract({ record, recordExternal }) {
+    const label = recordExternal && recordExternal.label ? recordExternal.label : "record";
+    const fields = record.get(pattern);
+    debugData(`${label}: ${fields.length} ${identifier}-fields `);
+    const identifiersFromFields = fields.map((field) => extractSubfieldsFromField(field, subfieldCodes));
+    debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);
+    const allIdentifiers = identifiersFromFields.flat();
+    debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);
+    const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({ identifierSubs: allIdentifiers, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes });
+    const identifiers = uniqueSubfields(validatedAndNormalizedIdentifiers);
+    debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);
+    return identifiers;
+    function validateAndNormalizeIdentifiers({ identifierSubs, validatorAndNormalizer: validatorAndNormalizer2, validIdentifierSubfieldCodes: validIdentifierSubfieldCodes2, invalidIdentifierSubfieldCodes: invalidIdentifierSubfieldCodes2 }) {
+      if (validatorAndNormalizer2) {
+        return identifierSubs.map((idSub) => validateAndNormalizeIdentifier({ idSub, validatorAndNormalizer: validatorAndNormalizer2, validIdentifierSubfieldCodes: validIdentifierSubfieldCodes2, invalidIdentifierSubfieldCodes: invalidIdentifierSubfieldCodes2 }));
+      }
+      return identifierSubs.map((idSub) => normalizeHyphens(idSub));
+    }
+    function validateAndNormalizeIdentifier({ idSub, validatorAndNormalizer: validatorAndNormalizer2, validIdentifierSubfieldCodes: validIdentifierSubfieldCodes2, invalidIdentifierSubfieldCodes: invalidIdentifierSubfieldCodes2 }) {
+      const { valid, value } = validatorAndNormalizer2(idSub.value);
+      if (validIdentifierSubfieldCodes2.includes(idSub.code) && valid === false) {
+        const [code] = invalidIdentifierSubfieldCodes2;
+        return { code, value };
+      }
+      return { code: idSub.code, value };
+    }
+    function normalizeHyphens(idSub) {
+      return { code: idSub.code, value: idSub.value.replace(/-/ug, "") };
+    }
+  }
+  function compare(a, b) {
+    debug(`Comparing A and B`);
+    if (a.length === 0 || b.length === 0) {
+      debugData(`No standardidentifiers (${identifier}) to compare`);
+      return 0;
+    }
+    debugData(`A: ${JSON.stringify(a)}`);
+    debugData(`B: ${JSON.stringify(b)}`);
+    if (bothHaveValidIdentifiers()) {
+      const { maxValues: maxValues2, possibleMatchValues, matchingValues: matchingValues2 } = getValueCount(true);
+      if (matchingValues2 < 1) {
+        debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);
+        return -0.75;
+      }
+      debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues2}/${possibleMatchValues} valid identifiers match.`);
+      debug(`Possible matches: ${possibleMatchValues}/${maxValues2}`);
+      const penaltyForMissing = 0.1 * (maxValues2 - possibleMatchValues);
+      const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues2);
+      debug(`	 points: penaltyForMissing: ${penaltyForMissing}`);
+      debug(`	 points: penaltyForMisMatch: ${penaltyForMisMatch}`);
+      return 0.75 - penaltyForMisMatch - penaltyForMissing;
+    }
+    const { maxValues, matchingValues } = getValueCount();
+    debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);
+    return matchingValues / maxValues * 0.2;
+    function bothHaveValidIdentifiers() {
+      const aValues = a.filter(({ code }) => validIdentifierSubfieldCodes.includes(code));
+      const bValues = a.filter(({ code }) => validIdentifierSubfieldCodes.includes(code));
+      debug(`A: ${aValues.length} valid ${identifier} identifiers`);
+      debug(`B: ${bValues.length} valid ${identifier} identifiers`);
+      return aValues.length > 0 && bValues.length > 0;
+    }
+    function getValueCount(validOnly = false) {
+      const aValues = getIdentifiers(a, validOnly);
+      const bValues = getIdentifiers(b, validOnly);
+      const matchingValues2 = getMatchingValuesAmount(aValues, bValues);
+      return {
+        maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,
+        // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)
+        possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,
+        matchingValues: matchingValues2
+      };
+      function getMatchingValuesAmount(aValues2, bValues2) {
+        if (bValues2.length > aValues2.length) {
+          return aValues2.filter((aValue) => bValues2.some((bValue) => aValue === bValue)).length;
+        }
+        if (aValues2.length > bValues2.length) {
+          return bValues2.filter((bValue) => aValues2.some((aValue) => bValue === aValue)).length;
+        }
+        const aToB = aValues2.filter((aValue) => bValues2.some((bValue) => aValue === bValue)).length;
+        const bToA = bValues2.filter((bValue) => aValues2.some((aValue) => bValue === aValue)).length;
+        return aToB < bToA ? aToB : bToA;
+      }
+      function getIdentifiers(values, validOnly2) {
+        if (validOnly2) {
+          return values.filter(({ code }) => validIdentifierSubfieldCodes.includes(code)).map(({ value }) => value);
+        }
+        return values.map(({ value }) => value);
+      }
+    }
+  }
+};
+//# sourceMappingURL=standard-identifier-factory.js.map

package/dist/match-detection/features/bib/standard-identifier-factory.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/match-detection/features/bib/standard-identifier-factory.js"],
+  "sourcesContent": ["/* eslint-disable max-statements */\n\nimport createDebugLogger from 'debug';\nimport {extractSubfieldsFromField, uniqueSubfields} from '../../../matching-utils.js';\n\n// Note about validity of standardIdentifiers:\n// We have three types of invalid standardIdentifiers:\n// 1. Formally invalid standardIdentifiers (ie. typos either in the resource or the record)\n// 2. Formally valid standardIdentifiers that are used in a wrong resource\n// 3. Canceled standardIdentifiers\n\n// Matcher could and should check that a standardIdentifier found in a subfield for a valid identifier is formally valid, and if it's not formally valid, handle it as an invalid standardIdentifier\n// Formally valid standardIdentifiers found in subfield for invalid identifier cannot be handled as valid standardIdentifiers, because they can be a case of type 2) or 3) invalid standardIdentifiers\n// We could also do a separate handling for formally valid an formally invalid standardIdentifiers\n\nexport default ({pattern, subfieldCodes, identifier, validIdentifierSubfieldCodes = ['a'], invalidIdentifierSubfieldCodes = ['z'], validatorAndNormalizer = undefined}) => {\n  const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);\n  const debugData = debug.extend('data');\n\n  return {extract, compare};\n\n  function extract({record, recordExternal}) {\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n    const fields = record.get(pattern);\n    debugData(`${label}: ${fields.length} ${identifier}-fields `);\n\n    // extractIdentifierSubfield normalizes hyphens away from the subfield values\n    const identifiersFromFields = fields.map(field => extractSubfieldsFromField(field, subfieldCodes));\n    debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);\n    const allIdentifiers = identifiersFromFields.flat();\n    debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);\n\n    const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({identifierSubs: allIdentifiers, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes});\n\n    const identifiers = uniqueSubfields(validatedAndNormalizedIdentifiers);\n\n    debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);\n    return identifiers;\n\n    function validateAndNormalizeIdentifiers({identifierSubs, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {\n      if (validatorAndNormalizer) {\n        return identifierSubs.map((idSub) => validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}));\n      }\n      return identifierSubs.map((idSub) => normalizeHyphens(idSub));\n    }\n\n    function validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {\n      const {valid, value} = validatorAndNormalizer(idSub.value);\n      if (validIdentifierSubfieldCodes.includes(idSub.code) && valid === false) {\n        const [code] = invalidIdentifierSubfieldCodes;\n        return {code, value};\n      }\n      return {code: idSub.code, value};\n    }\n\n    function normalizeHyphens(idSub) {\n      return {code: idSub.code, value: idSub.value.replace(/-/ug, '')};\n    }\n\n\n  }\n\n  function compare(a, b) {\n    debug(`Comparing A and B`);\n    if (a.length === 0 || b.length === 0) {\n      debugData(`No standardidentifiers (${identifier}) to compare`);\n      return 0;\n    }\n\n    debugData(`A: ${JSON.stringify(a)}`);\n    debugData(`B: ${JSON.stringify(b)}`);\n\n\n    if (bothHaveValidIdentifiers()) {\n      // Compare only valid identifiers, if both have valid idenfiers\n      const {maxValues, possibleMatchValues, matchingValues} = getValueCount(true);\n      if (matchingValues < 1) {\n        debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);\n        return -0.75;\n      }\n      debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues}/${possibleMatchValues} valid identifiers match.`);\n      // ignore non-matches if there is mismatching amount of values\n      debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);\n      //we give some kind of penalty for mismatching amount of values instead of simple divide?\n      const penaltyForMissing = 0.1 * (maxValues - possibleMatchValues);\n      const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues);\n      debug(`\\t points: penaltyForMissing: ${penaltyForMissing}`);\n      debug(`\\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);\n\n      return 0.75 - penaltyForMisMatch - penaltyForMissing;\n      //return matchingValues / possibleMatchValues * 0.75;\n    }\n    // If both do not have valid identifiers, compare all identifiers\n    const {maxValues, matchingValues} = getValueCount();\n    debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);\n\n    return matchingValues / maxValues * 0.2;\n\n    function bothHaveValidIdentifiers() {\n      const aValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));\n      const bValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));\n      debug(`A: ${aValues.length} valid ${identifier} identifiers`);\n      debug(`B: ${bValues.length} valid ${identifier} identifiers`);\n      return aValues.length > 0 && bValues.length > 0;\n    }\n\n    function getValueCount(validOnly = false) {\n      const aValues = getIdentifiers(a, validOnly);\n      const bValues = getIdentifiers(b, validOnly);\n\n      const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n      return {\n        maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n        // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n        possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n        matchingValues\n      };\n\n      function getMatchingValuesAmount(aValues, bValues) {\n        if (bValues.length > aValues.length) {\n          return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n        }\n        if (aValues.length > bValues.length) {\n          return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n        }\n\n        // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n        // there would be duplicate values\n        const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n        const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n        return aToB < bToA ? aToB : bToA;\n      }\n\n      function getIdentifiers(values, validOnly) {\n        if (validOnly) {\n          return values\n            .filter(({code}) => validIdentifierSubfieldCodes.includes(code))\n            .map(({value}) => value);\n        }\n\n        return values.map(({value}) => value);\n      }\n    }\n  }\n};\n"],
+  "mappings": "AAEA,OAAO,uBAAuB;AAC9B,SAAQ,2BAA2B,uBAAsB;AAYzD,eAAe,CAAC,EAAC,SAAS,eAAe,YAAY,+BAA+B,CAAC,GAAG,GAAG,iCAAiC,CAAC,GAAG,GAAG,yBAAyB,OAAS,MAAM;AACzK,QAAM,QAAQ,kBAAkB,mFAAmF,UAAU,EAAE;AAC/H,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,SAAO,EAAC,SAAS,QAAO;AAExB,WAAS,QAAQ,EAAC,QAAQ,eAAc,GAAG;AACzC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,SAAS,OAAO,IAAI,OAAO;AACjC,cAAU,GAAG,KAAK,KAAK,OAAO,MAAM,IAAI,UAAU,UAAU;AAG5D,UAAM,wBAAwB,OAAO,IAAI,WAAS,0BAA0B,OAAO,aAAa,CAAC;AACjG,cAAU,GAAG,KAAK,sBAAsB,sBAAsB,MAAM,MAAM,KAAK,UAAU,qBAAqB,CAAC,EAAE;AACjH,UAAM,iBAAiB,sBAAsB,KAAK;AAClD,cAAU,GAAG,KAAK,2BAA2B,eAAe,MAAM,MAAM,KAAK,UAAU,cAAc,CAAC,EAAE;AAExG,UAAM,oCAAoC,gCAAgC,EAAC,gBAAgB,gBAAgB,wBAAwB,8BAA8B,+BAA8B,CAAC;AAEhM,UAAM,cAAc,gBAAgB,iCAAiC;AAErE,cAAU,GAAG,KAAK,6BAA6B,YAAY,MAAM,MAAM,KAAK,UAAU,WAAW,CAAC,EAAE;AACpG,WAAO;AAEP,aAAS,gCAAgC,EAAC,gBAAgB,wBAAAA,yBAAwB,8BAAAC,+BAA8B,gCAAAC,gCAA8B,GAAG;AAC/I,UAAIF,yBAAwB;AAC1B,eAAO,eAAe,IAAI,CAAC,UAAU,+BAA+B,EAAC,OAAO,wBAAAA,yBAAwB,8BAAAC,+BAA8B,gCAAAC,gCAA8B,CAAC,CAAC;AAAA,MACpK;AACA,aAAO,eAAe,IAAI,CAAC,UAAU,iBAAiB,KAAK,CAAC;AAAA,IAC9D;AAEA,aAAS,+BAA+B,EAAC,OAAO,wBAAAF,yBAAwB,8BAAAC,+BAA8B,gCAAAC,gCAA8B,GAAG;AACrI,YAAM,EAAC,OAAO,MAAK,IAAIF,wBAAuB,MAAM,KAAK;AACzD,UAAIC,8BAA6B,SAAS,MAAM,IAAI,KAAK,UAAU,OAAO;AACxE,cAAM,CAAC,IAAI,IAAIC;AACf,eAAO,EAAC,MAAM,MAAK;AAAA,MACrB;AACA,aAAO,EAAC,MAAM,MAAM,MAAM,MAAK;AAAA,IACjC;AAEA,aAAS,iBAAiB,OAAO;AAC/B,aAAO,EAAC,MAAM,MAAM,MAAM,OAAO,MAAM,MAAM,QAAQ,OAAO,EAAE,EAAC;AAAA,IACjE;AAAA,EAGF;AAEA,WAAS,QAAQ,GAAG,GAAG;AACrB,UAAM,mBAAmB;AACzB,QAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,gBAAU,2BAA2B,UAAU,cAAc;AAC7D,aAAO;AAAA,IACT;AAEA,cAAU,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE;AACnC,cAAU,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE;AAGnC,QAAI,yBAAyB,GAAG;AAE9B,YAAM,EAAC,WAAAC,YAAW,qBAAqB,gBAAAC,gBAAc,IAAI,cAAc,IAAI;AAC3E,UAAIA,kBAAiB,GAAG;AACtB,cAAM,wCAAwC,UAAU,6BAA6B;AACrF,eAAO;AAAA,MACT;AACA,YAAM,wCAAwC,UAAU,MAAMA,eAAc,IAAI,mBAAmB,2BAA2B;AAE9H,YAAM,qBAAqB,mBAAmB,IAAID,UAAS,EAAE;AAE7D,YAAM,oBAAoB,OAAOA,aAAY;AAC7C,YAAM,qBAAqB,OAAO,sBAAsBC;AACxD,YAAM,gCAAiC,iBAAiB,EAAE;AAC1D,YAAM,iCAAkC,kBAAkB,EAAE;AAE5D,aAAO,OAAO,qBAAqB;AAAA,IAErC;AAEA,UAAM,EAAC,WAAW,eAAc,IAAI,cAAc;AAClD,UAAM,+CAA+C,UAAU,MAAM,cAAc,IAAI,SAAS,mCAAmC;AAEnI,WAAO,iBAAiB,YAAY;AAEpC,aAAS,2BAA2B;AAClC,YAAM,UAAU,EAAE,OAAO,CAAC,EAAC,KAAI,MAAM,6BAA6B,SAAS,IAAI,CAAC;AAChF,YAAM,UAAU,EAAE,OAAO,CAAC,EAAC,KAAI,MAAM,6BAA6B,SAAS,IAAI,CAAC;AAChF,YAAM,MAAM,QAAQ,MAAM,UAAU,UAAU,cAAc;AAC5D,YAAM,MAAM,QAAQ,MAAM,UAAU,UAAU,cAAc;AAC5D,aAAO,QAAQ,SAAS,KAAK,QAAQ,SAAS;AAAA,IAChD;AAEA,aAAS,cAAc,YAAY,OAAO;AACxC,YAAM,UAAU,eAAe,GAAG,SAAS;AAC3C,YAAM,UAAU,eAAe,GAAG,SAAS;AAE3C,YAAMA,kBAAiB,wBAAwB,SAAS,OAAO;AAE/D,aAAO;AAAA,QACL,WAAW,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA;AAAA,QAEtE,qBAAqB,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA,QAChF,gBAAAA;AAAA,MACF;AAEA,eAAS,wBAAwBC,UAASC,UAAS;AACjD,YAAIA,SAAQ,SAASD,SAAQ,QAAQ;AACnC,iBAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,QAC7E;AACA,YAAID,SAAQ,SAASC,SAAQ,QAAQ;AACnC,iBAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,QAC7E;AAIA,cAAM,OAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AACjF,cAAM,OAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAEjF,eAAO,OAAO,OAAO,OAAO;AAAA,MAC9B;AAEA,eAAS,eAAe,QAAQE,YAAW;AACzC,YAAIA,YAAW;AACb,iBAAO,OACJ,OAAO,CAAC,EAAC,KAAI,MAAM,6BAA6B,SAAS,IAAI,CAAC,EAC9D,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK;AAAA,QAC3B;AAEA,eAAO,OAAO,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;",
+  "names": ["validatorAndNormalizer", "validIdentifierSubfieldCodes", "invalidIdentifierSubfieldCodes", "maxValues", "matchingValues", "aValues", "bValues", "validOnly"]
+}

package/dist/match-detection/features/bib/title-version-original.js

@@ -0,0 +1,37 @@
+import createDebugLogger from "debug";
+import naturalPkg from "natural";
+const { LevenshteinDistance: leven } = naturalPkg;
+export default ({ treshold = 10 } = {}) => ({
+  name: "titleVersionOriginal",
+  extract: ({ record }) => {
+    const title = getTitle();
+    if (title) {
+      return [title.replace(/[^\p{Letter}\p{Number}]/gu, "").toLowerCase()];
+    }
+    return [];
+    function getTitle() {
+      const [field] = record.get(/^245$/u);
+      if (field) {
+        return field.subfields.filter(({ code }) => ["a", "b"].includes(code)).map(({ value }) => value).join("");
+      }
+    }
+  },
+  compare: (a, b) => {
+    const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original");
+    const distance = leven(a[0], b[0]);
+    if (distance === 0) {
+      return 0.5;
+    }
+    const maxLength = getMaxLength();
+    const percentage = distance / maxLength * 100;
+    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
+    if (percentage <= treshold) {
+      return 0.3;
+    }
+    return -0.5;
+    function getMaxLength() {
+      return a[0].length > b[0].length ? a[0].length : b[0].length;
+    }
+  }
+});
+//# sourceMappingURL=title-version-original.js.map

package/dist/match-detection/features/bib/title-version-original.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/match-detection/features/bib/title-version-original.js"],
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\n\nexport default ({treshold = 10} = {}) => ({\n  name: 'titleVersionOriginal',\n  extract: ({record}) => {\n    const title = getTitle();\n\n    if (title) {\n      return [title.replace(/[^\\p{Letter}\\p{Number}]/gu, '').toLowerCase()];\n    }\n\n    return [];\n\n    function getTitle() {\n      const [field] = record.get(/^245$/u);\n\n      if (field) {\n        return field.subfields\n          .filter(({code}) => ['a', 'b'].includes(code))\n          .map(({value}) => value)\n          .join('');\n      }\n    }\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title-version-original');\n    const distance = leven(a[0], b[0]);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    const maxLength = getMaxLength();\n    const percentage = distance / maxLength * 100;\n\n    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);\n\n    if (percentage <= treshold) {\n      return 0.3;\n    }\n\n    return -0.5;\n\n    function getMaxLength() {\n      return a[0].length > b[0].length ? a[0].length : b[0].length;\n    }\n\n  }\n});\n"],
+  "mappings": "AACA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AAErC,eAAe,CAAC,EAAC,WAAW,GAAE,IAAI,CAAC,OAAO;AAAA,EACxC,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,UAAM,QAAQ,SAAS;AAEvB,QAAI,OAAO;AACT,aAAO,CAAC,MAAM,QAAQ,6BAA6B,EAAE,EAAE,YAAY,CAAC;AAAA,IACtE;AAEA,WAAO,CAAC;AAER,aAAS,WAAW;AAClB,YAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AAEnC,UAAI,OAAO;AACT,eAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,CAAC,KAAK,GAAG,EAAE,SAAS,IAAI,CAAC,EAC5C,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,KAAK,EAAE;AAAA,MACZ;AAAA,IACF;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,QAAQ,kBAAkB,uFAAuF;AACvH,UAAM,WAAW,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;AAEjC,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,aAAa;AAC/B,UAAM,aAAa,WAAW,YAAY;AAE1C,UAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,SAAS,gBAAgB,QAAQ,kBAAkB,UAAU,EAAE;AAEvG,QAAI,cAAc,UAAU;AAC1B,aAAO;AAAA,IACT;AAEA,WAAO;AAEP,aAAS,eAAe;AACtB,aAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;AAAA,IACxD;AAAA,EAEF;AACF;",
+  "names": []
+}

package/dist/match-detection/features/bib/title.js

@@ -0,0 +1,46 @@
+import createDebugLogger from "debug";
+import naturalPkg from "natural";
+const { LevenshteinDistance: leven } = naturalPkg;
+import { testStringOrNumber } from "../../../matching-utils.js";
+const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features:title");
+const debugData = debug.extend("data");
+export default ({ treshold = 10 } = {}) => ({
+  name: "Title",
+  extract: ({ record, recordExternal }) => {
+    const label = recordExternal && recordExternal.label ? recordExternal.label : "record";
+    const title = getTitle();
+    debug(`${label}: title: ${title}`);
+    if (testStringOrNumber(title)) {
+      const titleAsNormalizedString = String(title).normalize("NFD").replace(/[^\p{Letter}\p{Number}]/gu, "").toLowerCase();
+      debug(`${label}: titleString: ${titleAsNormalizedString}`);
+      return [titleAsNormalizedString];
+    }
+    return [];
+    function getTitle() {
+      const [field] = record.get(/^245$/u);
+      debugData(`${label}: titleField: ${JSON.stringify(field)}`);
+      if (field) {
+        return field.subfields.filter(({ code }) => ["a", "b", "n", "p"].includes(code)).map(({ value }) => testStringOrNumber(value) ? String(value) : "").join("");
+      }
+      return false;
+    }
+  },
+  compare: (a, b) => {
+    const debug2 = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features/bib/title");
+    const distance = leven(a[0], b[0]);
+    if (distance === 0) {
+      return 0.5;
+    }
+    const maxLength = getMaxLength();
+    const percentage = distance / maxLength * 100;
+    debug2(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
+    if (percentage <= treshold) {
+      return 0.3;
+    }
+    return -0.5;
+    function getMaxLength() {
+      return a[0].length > b[0].length ? a[0].length : b[0].length;
+    }
+  }
+});
+//# sourceMappingURL=title.js.map

package/dist/match-detection/features/bib/title.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/match-detection/features/bib/title.js"],
+  "sourcesContent": ["import createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');\nconst debugData = debug.extend('data');\n\n\nexport default ({treshold = 10} = {}) => ({\n  name: 'Title',\n  extract: ({record, recordExternal}) => {\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n    const title = getTitle();\n    debug(`${label}: title: ${title}`);\n\n    if (testStringOrNumber(title)) {\n      const titleAsNormalizedString = String(title)\n        // decompose unicode diacritics\n        .normalize('NFD')\n        // strip non-letters/numbers\n        // - note: combined with decomposing unicode diactics this normalizes both 'saa' and 's\u00E4\u00E4' as 'saa'\n        // - we could precompose the finnish letters back to avoid this\n        .replace(/[^\\p{Letter}\\p{Number}]/gu, '')\n        .toLowerCase();\n      debug(`${label}: titleString: ${titleAsNormalizedString}`);\n      return [titleAsNormalizedString];\n    }\n\n    return [];\n\n    function getTitle() {\n      const [field] = record.get(/^245$/u);\n      debugData(`${label}: titleField: ${JSON.stringify(field)}`);\n\n      if (field) {\n        return field.subfields\n          // get also $n:s and $p:s here\n          .filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))\n          .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n          .join('');\n      }\n      return false;\n    }\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');\n    const distance = leven(a[0], b[0]);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    const maxLength = getMaxLength();\n    const percentage = distance / maxLength * 100;\n\n    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);\n\n    if (percentage <= treshold) {\n      return 0.3;\n    }\n\n    return -0.5;\n\n    function getMaxLength() {\n      return a[0].length > b[0].length ? a[0].length : b[0].length;\n    }\n\n  }\n});\n"],
+  "mappings": "AAAA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AACrC,SAAQ,0BAAyB;AAEjC,MAAM,QAAQ,kBAAkB,kEAAkE;AAClG,MAAM,YAAY,MAAM,OAAO,MAAM;AAGrC,eAAe,CAAC,EAAC,WAAW,GAAE,IAAI,CAAC,OAAO;AAAA,EACxC,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,QAAQ,SAAS;AACvB,UAAM,GAAG,KAAK,YAAY,KAAK,EAAE;AAEjC,QAAI,mBAAmB,KAAK,GAAG;AAC7B,YAAM,0BAA0B,OAAO,KAAK,EAEzC,UAAU,KAAK,EAIf,QAAQ,6BAA6B,EAAE,EACvC,YAAY;AACf,YAAM,GAAG,KAAK,kBAAkB,uBAAuB,EAAE;AACzD,aAAO,CAAC,uBAAuB;AAAA,IACjC;AAEA,WAAO,CAAC;AAER,aAAS,WAAW;AAClB,YAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AACnC,gBAAU,GAAG,KAAK,iBAAiB,KAAK,UAAU,KAAK,CAAC,EAAE;AAE1D,UAAI,OAAO;AACT,eAAO,MAAM,UAEV,OAAO,CAAC,EAAC,KAAI,MAAM,CAAC,KAAK,KAAK,KAAK,GAAG,EAAE,SAAS,IAAI,CAAC,EACtD,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,KAAK,EAAE;AAAA,MACZ;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAMA,SAAQ,kBAAkB,sEAAsE;AACtG,UAAM,WAAW,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;AAEjC,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,aAAa;AAC/B,UAAM,aAAa,WAAW,YAAY;AAE1C,IAAAA,OAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,SAAS,gBAAgB,QAAQ,kBAAkB,UAAU,EAAE;AAEvG,QAAI,cAAc,UAAU;AAC1B,aAAO;AAAA,IACT;AAEA,WAAO;AAEP,aAAS,eAAe;AACtB,aAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;AAAA,IACxD;AAAA,EAEF;AACF;",
+  "names": ["debug"]
+}

package/dist/match-detection/features/index.js

@@ -0,0 +1,3 @@
+import * as bib from "./bib/index.js";
+export { bib };
+//# sourceMappingURL=index.js.map

package/dist/match-detection/features/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/match-detection/features/index.js"],
+  "sourcesContent": ["\nimport * as bib from './bib/index.js';\nexport {bib};\n"],
+  "mappings": "AACA,YAAY,SAAS;AACrB,SAAQ;",
+  "names": []
+}

package/dist/match-detection/index.js

@@ -0,0 +1,86 @@
+import createDebugLogger from "debug";
+import * as features from "./features/index.js";
+export { features };
+export default ({ strategy, treshold = 0.9 }, returnStrategy = false, localSettings = {}) => ({ recordA, recordB, recordAExternal = {}, recordBExternal = {} }) => {
+  const minProbabilityQuantifier = 0.5;
+  const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection");
+  const debugData = debug.extend("data");
+  debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
+  debugData(`Records: A: ${recordA}
+B: ${recordB}`);
+  debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);
+  const featuresA = extractFeatures({ record: recordA, recordExternal: recordAExternal, localSettings });
+  debug(`We got an array of records: ${Array.isArray(recordB)}`);
+  const recordsB = Array.isArray(recordB) ? recordB : [recordB];
+  const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];
+  const detectionResults = recordsB.map((record, index) => actualDetection({ featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index }));
+  return Array.isArray(recordB) ? detectionResults : detectionResults[0];
+  function actualDetection({ featuresA: featuresA2, record, recordExternal, index, localSettings: localSettings2 }) {
+    const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : "a";
+    const labelB = recordExternal && recordExternal.label ? recordExternal.label : "b";
+    debug(`Actual detection for record ${index + 1} ${labelB}`);
+    const featuresB = extractFeatures({ record, recordExternal, localSettings: localSettings2 });
+    debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA2)}`);
+    debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);
+    const featurePairs = generateFeaturePairs(featuresA2, featuresB);
+    const similarityVector = generateSimilarityVector(featurePairs);
+    if (similarityVector.some((v) => v >= minProbabilityQuantifier)) {
+      const probability = calculateProbability(similarityVector);
+      debug(`probability: ${probability} (Treshold: ${treshold})`);
+      return returnResult({ match: probability >= treshold, probability });
+    }
+    debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
+    return returnResult({ match: false, probability: 0 });
+  }
+  function extractFeatures({ record, recordExternal }) {
+    return strategy.reduce((acc, { name, extract }) => acc.concat({ name, value: extract({ record, recordExternal }) }), []);
+  }
+  function returnResult(result) {
+    if (returnStrategy) {
+      debug(`Returning detection strategy with the result`);
+      const resultWithStrategy = { match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold };
+      debugData(`${JSON.stringify(resultWithStrategy)}`);
+      return resultWithStrategy;
+    }
+    return result;
+  }
+  function formatStrategy(strategy2) {
+    const strategyNames = strategy2.map((element) => element.name);
+    return strategyNames || [];
+  }
+  function calculateProbability(similarityVector) {
+    const probability = similarityVector.reduce((acc, v) => acc + v, 0);
+    return probability > 1 ? 1 : probability;
+  }
+  function generateSimilarityVector(featurePairs) {
+    const compared = featurePairs.map(({ name, a, b }) => {
+      const { compare } = strategy.find(({ name: featureName }) => name === featureName);
+      const points = compare(a, b);
+      return { name, points };
+    });
+    debugData(`Points: ${JSON.stringify(compared)}`);
+    return compared.map(({ points }) => points);
+  }
+  function generateFeaturePairs(featuresA2, featuresB) {
+    const pairs = generatePairs();
+    const missingFeatures = findMissing();
+    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);
+    return pairs;
+    function generatePairs() {
+      return featuresA2.reduce((acc, { name, value }, index) => acc.concat({
+        name,
+        a: value,
+        b: featuresB[index].value
+      }), []).filter(({ a, b }) => {
+        if (a.length === 0 || b.length === 0) {
+          return false;
+        }
+        return true;
+      });
+    }
+    function findMissing() {
+      return featuresA2.map(({ name }) => name).filter((v) => pairs.some(({ name }) => name === v) === false);
+    }
+  }
+};
+//# sourceMappingURL=index.js.map

package/dist/match-detection/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/match-detection/index.js"],
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport * as features from './features/index.js';\n\nexport {features};\n\nexport default ({strategy, treshold = 0.9}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {\n  const minProbabilityQuantifier = 0.5;\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n  const debugData = debug.extend('data');\n\n  debugData(`Strategy: ${JSON.stringify(strategy)}, Treshold: ${JSON.stringify(treshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n  debugData(`Records: A: ${recordA}\\nB: ${recordB}`);\n  debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);\n  // We could add here labels for records if we didn't get external labels\n\n  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});\n\n  debug(`We got an array of records: ${Array.isArray(recordB)}`);\n  const recordsB = Array.isArray(recordB) ? recordB : [recordB];\n  const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];\n\n  const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));\n\n  // if we got array of records, we return an array of result\n  // if we got a singular record, we return a singular result\n  return Array.isArray(recordB) ? detectionResults : detectionResults[0];\n\n  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {\n    const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';\n    const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';\n\n\n    debug(`Actual detection for record ${index + 1} ${labelB}`);\n    const featuresB = extractFeatures({record, recordExternal, localSettings});\n\n    debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);\n    debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);\n\n    const featurePairs = generateFeaturePairs(featuresA, featuresB);\n    const similarityVector = generateSimilarityVector(featurePairs);\n\n    if (similarityVector.some(v => v >= minProbabilityQuantifier)) {\n      const probability = calculateProbability(similarityVector);\n      debug(`probability: ${probability} (Treshold: ${treshold})`);\n      return returnResult({match: probability >= treshold, probability});\n    }\n\n    debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);\n    return returnResult({match: false, probability: 0.0});\n  }\n\n  function extractFeatures({record, recordExternal}) {\n    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);\n  }\n\n  function returnResult(result) {\n    if (returnStrategy) {\n      debug(`Returning detection strategy with the result`);\n      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), treshold};\n      debugData(`${JSON.stringify(resultWithStrategy)}`);\n      return resultWithStrategy;\n    }\n    return result;\n  }\n\n  function formatStrategy(strategy) {\n    const strategyNames = strategy.map(element => element.name);\n    return strategyNames || [];\n  }\n\n  function calculateProbability(similarityVector) {\n    const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n    return probability > 1.0 ? 1.0 : probability;\n  }\n\n  function generateSimilarityVector(featurePairs) {\n    const compared = featurePairs.map(({name, a, b}) => {\n      const {compare} = strategy.find(({name: featureName}) => name === featureName);\n      const points = compare(a, b);\n      return {name, points};\n    });\n\n    debugData(`Points: ${JSON.stringify(compared)}`);\n    return compared.map(({points}) => points);\n  }\n\n  function generateFeaturePairs(featuresA, featuresB) {\n    const pairs = generatePairs();\n    const missingFeatures = findMissing();\n\n    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n    return pairs;\n\n    function generatePairs() {\n      return featuresA\n        .reduce((acc, {name, value}, index) => acc.concat({\n          name,\n          a: value,\n          b: featuresB[index].value\n        }), [])\n        .filter(({a, b}) => {\n          if (a.length === 0 || b.length === 0) {\n            return false;\n          }\n\n          return true;\n        });\n    }\n\n    function findMissing() {\n      return featuresA\n        .map(({name}) => name)\n        .filter(v => pairs.some(({name}) => name === v) === false);\n    }\n  }\n\n};\n"],
+  "mappings": "AACA,OAAO,uBAAuB;AAC9B,YAAY,cAAc;AAE1B,SAAQ;AAER,eAAe,CAAC,EAAC,UAAU,WAAW,IAAG,GAAG,iBAAiB,OAAO,gBAAgB,CAAC,MAAM,CAAC,EAAC,SAAS,SAAS,kBAAkB,CAAC,GAAG,kBAAkB,CAAC,EAAC,MAAM;AAC7J,QAAM,2BAA2B;AAEjC,QAAM,QAAQ,kBAAkB,mDAAmD;AACnF,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,YAAU,aAAa,KAAK,UAAU,QAAQ,CAAC,eAAe,KAAK,UAAU,QAAQ,CAAC,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAC3I,YAAU,eAAe,OAAO;AAAA,KAAQ,OAAO,EAAE;AACjD,QAAM,iBAAiB,KAAK,UAAU,eAAe,CAAC,QAAQ,KAAK,UAAU,eAAe,CAAC,EAAE;AAG/F,QAAM,YAAY,gBAAgB,EAAC,QAAQ,SAAS,gBAAgB,iBAAiB,cAAa,CAAC;AAEnG,QAAM,+BAA+B,MAAM,QAAQ,OAAO,CAAC,EAAE;AAC7D,QAAM,WAAW,MAAM,QAAQ,OAAO,IAAI,UAAU,CAAC,OAAO;AAC5D,QAAM,mBAAmB,MAAM,QAAQ,OAAO,IAAI,kBAAkB,CAAC,eAAe;AAEpF,QAAM,mBAAmB,SAAS,IAAI,CAAC,QAAQ,UAAU,gBAAgB,EAAC,WAAW,iBAAiB,QAAQ,gBAAgB,iBAAiB,KAAK,GAAG,MAAK,CAAC,CAAC;AAI9J,SAAO,MAAM,QAAQ,OAAO,IAAI,mBAAmB,iBAAiB,CAAC;AAErE,WAAS,gBAAgB,EAAC,WAAAA,YAAW,QAAQ,gBAAgB,OAAO,eAAAC,eAAa,GAAG;AAClF,UAAM,SAAS,mBAAmB,gBAAgB,QAAQ,gBAAgB,QAAQ;AAClF,UAAM,SAAS,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAG/E,UAAM,+BAA+B,QAAQ,CAAC,IAAI,MAAM,EAAE;AAC1D,UAAM,YAAY,gBAAgB,EAAC,QAAQ,gBAAgB,eAAAA,eAAa,CAAC;AAEzE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAUD,UAAS,CAAC,EAAE;AACjE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAU,SAAS,CAAC,EAAE;AAEjE,UAAM,eAAe,qBAAqBA,YAAW,SAAS;AAC9D,UAAM,mBAAmB,yBAAyB,YAAY;AAE9D,QAAI,iBAAiB,KAAK,OAAK,KAAK,wBAAwB,GAAG;AAC7D,YAAM,cAAc,qBAAqB,gBAAgB;AACzD,YAAM,gBAAgB,WAAW,eAAe,QAAQ,GAAG;AAC3D,aAAO,aAAa,EAAC,OAAO,eAAe,UAAU,YAAW,CAAC;AAAA,IACnE;AAEA,cAAU,4DAA4D,wBAAwB,GAAG;AACjG,WAAO,aAAa,EAAC,OAAO,OAAO,aAAa,EAAG,CAAC;AAAA,EACtD;AAEA,WAAS,gBAAgB,EAAC,QAAQ,eAAc,GAAG;AACjD,WAAO,SAAS,OAAO,CAAC,KAAK,EAAC,MAAM,QAAO,MAAM,IAAI,OAAO,EAAC,MAAM,OAAO,QAAQ,EAAC,QAAQ,eAAc,CAAC,EAAC,CAAC,GAAG,CAAC,CAAC;AAAA,EACnH;AAEA,WAAS,aAAa,QAAQ;AAC5B,QAAI,gBAAgB;AAClB,YAAM,8CAA8C;AACpD,YAAM,qBAAqB,EAAC,OAAO,OAAO,OAAO,aAAa,OAAO,aAAa,UAAU,eAAe,QAAQ,GAAG,SAAQ;AAC9H,gBAAU,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AACjD,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAEA,WAAS,eAAeE,WAAU;AAChC,UAAM,gBAAgBA,UAAS,IAAI,aAAW,QAAQ,IAAI;AAC1D,WAAO,iBAAiB,CAAC;AAAA,EAC3B;AAEA,WAAS,qBAAqB,kBAAkB;AAC9C,UAAM,cAAc,iBAAiB,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAG;AACpE,WAAO,cAAc,IAAM,IAAM;AAAA,EACnC;AAEA,WAAS,yBAAyB,cAAc;AAC9C,UAAM,WAAW,aAAa,IAAI,CAAC,EAAC,MAAM,GAAG,EAAC,MAAM;AAClD,YAAM,EAAC,QAAO,IAAI,SAAS,KAAK,CAAC,EAAC,MAAM,YAAW,MAAM,SAAS,WAAW;AAC7E,YAAM,SAAS,QAAQ,GAAG,CAAC;AAC3B,aAAO,EAAC,MAAM,OAAM;AAAA,IACtB,CAAC;AAED,cAAU,WAAW,KAAK,UAAU,QAAQ,CAAC,EAAE;AAC/C,WAAO,SAAS,IAAI,CAAC,EAAC,OAAM,MAAM,MAAM;AAAA,EAC1C;AAEA,WAAS,qBAAqBF,YAAW,WAAW;AAClD,UAAM,QAAQ,cAAc;AAC5B,UAAM,kBAAkB,YAAY;AAEpC,UAAM,2FAA2F,KAAK,UAAU,eAAe,CAAC,EAAE;AAClI,WAAO;AAEP,aAAS,gBAAgB;AACvB,aAAOA,WACJ,OAAO,CAAC,KAAK,EAAC,MAAM,MAAK,GAAG,UAAU,IAAI,OAAO;AAAA,QAChD;AAAA,QACA,GAAG;AAAA,QACH,GAAG,UAAU,KAAK,EAAE;AAAA,MACtB,CAAC,GAAG,CAAC,CAAC,EACL,OAAO,CAAC,EAAC,GAAG,EAAC,MAAM;AAClB,YAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,iBAAO;AAAA,QACT;AAEA,eAAO;AAAA,MACT,CAAC;AAAA,IACL;AAEA,aAAS,cAAc;AACrB,aAAOA,WACJ,IAAI,CAAC,EAAC,KAAI,MAAM,IAAI,EACpB,OAAO,OAAK,MAAM,KAAK,CAAC,EAAC,KAAI,MAAM,SAAS,CAAC,MAAM,KAAK;AAAA,IAC7D;AAAA,EACF;AAEF;",
+  "names": ["featuresA", "localSettings", "strategy"]
+}

package/dist/match-detection/index.test.js

@@ -0,0 +1,44 @@
+import assert from "node:assert";
+import { describe } from "node:test";
+import createDebugLogger from "debug";
+import { inspect } from "util";
+import generateTests from "@natlibfi/fixugen";
+import { READERS } from "@natlibfi/fixura";
+import { MarcRecord } from "@natlibfi/marc-record";
+import createDetectionInterface from "./index.js";
+import * as features from "./features/index.js";
+const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:test");
+const debugData = debug.extend("data");
+describe("match-detection", () => {
+  generateTests({
+    path: [import.meta.dirname, "..", "..", "test-fixtures", "match-detection", "index"],
+    useMetadataFile: true,
+    recurse: false,
+    fixura: {
+      reader: READERS.JSON
+    },
+    callback: ({ getFixture, options, expectedResults, array, enabled = true }) => {
+      if (!enabled) {
+        debug(`*** DISABLED TEST! ***`);
+        return;
+      }
+      const detect = createDetectionInterface(formatOptions());
+      const recordA = new MarcRecord(getFixture("recordA.json"), { subfieldValues: false });
+      debugData(inspect(recordA));
+      debug(`Our recordB is an array of records: ${array}`);
+      const recordB = array ? getFixture("recordB.json").map((recordJson) => new MarcRecord(recordJson, { subfieldValues: false })) : new MarcRecord(getFixture("recordB.json"), { subfieldValues: false });
+      debugData(inspect(recordB));
+      const results = detect({ recordA, recordB });
+      debugData(`${JSON.stringify(results)}`);
+      assert.deepStrictEqual(results, expectedResults);
+      function formatOptions() {
+        const contextFeatures = features[options.strategy.type];
+        return {
+          ...options,
+          strategy: options.strategy.features.map((v) => contextFeatures[v]())
+        };
+      }
+    }
+  });
+});
+//# sourceMappingURL=index.test.js.map

package/dist/match-detection/index.test.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/match-detection/index.test.js"],
+  "sourcesContent": ["import assert from 'node:assert';\nimport {describe} from 'node:test';\nimport createDebugLogger from 'debug';\nimport {inspect} from 'util';\nimport generateTests from '@natlibfi/fixugen';\nimport {READERS} from '@natlibfi/fixura';\nimport {MarcRecord} from '@natlibfi/marc-record';\n\nimport createDetectionInterface from './index.js';\nimport * as features from './features/index.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:test');\nconst debugData = debug.extend('data');\n\ndescribe('match-detection', () => {\n  generateTests({\n    path: [import.meta.dirname, '..', '..', 'test-fixtures', 'match-detection', 'index'],\n    useMetadataFile: true,\n    recurse: false,\n    fixura: {\n      reader: READERS.JSON\n    },\n    callback: ({getFixture, options, expectedResults, array, enabled = true}) => {\n\n      if (!enabled) {\n        debug(`*** DISABLED TEST! ***`);\n        return;\n      }\n\n      const detect = createDetectionInterface(formatOptions());\n      const recordA = new MarcRecord(getFixture('recordA.json'), {subfieldValues: false});\n      debugData(inspect(recordA));\n\n      debug(`Our recordB is an array of records: ${array}`);\n      const recordB = array\n        ? getFixture('recordB.json').map(recordJson => new MarcRecord(recordJson, {subfieldValues: false}))\n        : new MarcRecord(getFixture('recordB.json'), {subfieldValues: false});\n      debugData(inspect(recordB));\n\n      const results = detect({recordA, recordB});\n      debugData(`${JSON.stringify(results)}`);\n\n      assert.deepStrictEqual(results, expectedResults);\n\n      function formatOptions() {\n        const contextFeatures = features[options.strategy.type];\n\n        return {\n          ...options,\n          strategy: options.strategy.features.map(v => contextFeatures[v]())\n        };\n      }\n    }\n  });\n});\n"],
+  "mappings": "AAAA,OAAO,YAAY;AACnB,SAAQ,gBAAe;AACvB,OAAO,uBAAuB;AAC9B,SAAQ,eAAc;AACtB,OAAO,mBAAmB;AAC1B,SAAQ,eAAc;AACtB,SAAQ,kBAAiB;AAEzB,OAAO,8BAA8B;AACrC,YAAY,cAAc;AAE1B,MAAM,QAAQ,kBAAkB,wDAAwD;AACxF,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,SAAS,mBAAmB,MAAM;AAChC,gBAAc;AAAA,IACZ,MAAM,CAAC,YAAY,SAAS,MAAM,MAAM,iBAAiB,mBAAmB,OAAO;AAAA,IACnF,iBAAiB;AAAA,IACjB,SAAS;AAAA,IACT,QAAQ;AAAA,MACN,QAAQ,QAAQ;AAAA,IAClB;AAAA,IACA,UAAU,CAAC,EAAC,YAAY,SAAS,iBAAiB,OAAO,UAAU,KAAI,MAAM;AAE3E,UAAI,CAAC,SAAS;AACZ,cAAM,wBAAwB;AAC9B;AAAA,MACF;AAEA,YAAM,SAAS,yBAAyB,cAAc,CAAC;AACvD,YAAM,UAAU,IAAI,WAAW,WAAW,cAAc,GAAG,EAAC,gBAAgB,MAAK,CAAC;AAClF,gBAAU,QAAQ,OAAO,CAAC;AAE1B,YAAM,uCAAuC,KAAK,EAAE;AACpD,YAAM,UAAU,QACZ,WAAW,cAAc,EAAE,IAAI,gBAAc,IAAI,WAAW,YAAY,EAAC,gBAAgB,MAAK,CAAC,CAAC,IAChG,IAAI,WAAW,WAAW,cAAc,GAAG,EAAC,gBAAgB,MAAK,CAAC;AACtE,gBAAU,QAAQ,OAAO,CAAC;AAE1B,YAAM,UAAU,OAAO,EAAC,SAAS,QAAO,CAAC;AACzC,gBAAU,GAAG,KAAK,UAAU,OAAO,CAAC,EAAE;AAEtC,aAAO,gBAAgB,SAAS,eAAe;AAE/C,eAAS,gBAAgB;AACvB,cAAM,kBAAkB,SAAS,QAAQ,SAAS,IAAI;AAEtD,eAAO;AAAA,UACL,GAAG;AAAA,UACH,UAAU,QAAQ,SAAS,SAAS,IAAI,OAAK,gBAAgB,CAAC,EAAE,CAAC;AAAA,QACnE;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AACH,CAAC;",
+  "names": []
+}