@natlibfi/melinda-record-matching 5.0.4 → 5.0.5

package/dist/match-detection/features/bib/language.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../../src/match-detection/features/bib/language.js"],
-  "sourcesContent": ["\nimport createDebugLogger from 'debug';\n\nimport {MarcRecord} from '@natlibfi/marc-record';\n\nimport {FixSami041, Remove041zxx} from '@natlibfi/marc-record-validators-melinda';\n\nimport {getMatchCounts} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:language');\nconst debugData = debug.extend('data');\n\n// NB! 2025-12-17: I changed the logic drastically. However, I tried to keep the scores as same as possible.\n// 'extract' no longer extracts anything. Instead it clones the record, and does some preprocessing (using validators) instead.\n// 'compare' had multiple changes\n// - f041 ind1 differences ('0' vs '1') now result in a small penalty\n// - Two sami languages related changes:\n// -- validator adds a 'smi' subfield before a corresponding sma/sme/...subfield, if needed (national)\n// -- 'smi' only vs 'smi'+'sma' does not cause penalty\n// - 'mul' vs 'fin'+'swe' does not cause a penalty. However, 'mul' vs 'fin' alone triggers penalty.\n// - 008/35-37 and f041$a/$d are calculated separately\n// - a threshold is applied: return value is always between -1.0 and +0.1 (as it was before)\n// - we try to handle 041 $2 ISO 639-2 and ISO 639-3 as well.\n// - 'und' vs one other language does not cause a penalty in certain contexts. In other contexts it's treated as a normal \"language code\". (Tune/alleviate penalties later on.)\n\nexport default () => ({\n  name: 'Language',\n  extract: ({record, recordExternal}) => {\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n    const clonedRecord = new MarcRecord(record, record._validationOptions); // clone(record); // NB! This loses record.get()...\n\n    FixSami041().fix(clonedRecord); // Handle 'smi' adding if needed\n    Remove041zxx().fix(clonedRecord); // Remove 'zxx' from f041s\n    // Add other language code normalizations?\n\n    // Should we return all the fields, or just the relevant fields?\n    return [{leader: clonedRecord.leader, fields: clonedRecord.fields}, label];\n  },\n  // eslint-disable-next-line max-statements\n  compare: (aa, bb) => {\n    const [a, aLabel] = aa;\n    const [b, bLabel] = bb;\n    debugData(`Comparing language ${JSON.stringify(a)} and ${JSON.stringify(b)}`);\n\n    const score008 = compare008();\n    const score041 = compare041();\n    return applyLimits(score008 + score041);\n\n    function applyLimits(score) {\n      if (score > 0.1) { // Keeps the original max, we might have 0.10 (041) + 0.05 (008/35-37) = 0.15 now\n        return 0.1;\n      }\n      if (score < -1) {\n        return -1.0;\n      }\n      return score;\n    }\n\n    function compare008() {\n      const a008 = get008Value(a, aLabel);\n      const b008 = get008Value(b, bLabel);\n      // Something seriously wrong with 008:\n      if (a008 === undefined || b008 === undefined) {\n        return 0.0; // Punish for generic badness?\n      }\n\n      if (containsNoData(a008) || containsNoData(b008)){\n        // Nothing to compare\n        return 0.0;\n      }\n\n      if (a008 === b008) {\n        return 0.05;\n      }\n\n      if (a008 === 'mul' || b008 === 'mul') {\n        return 0.0;\n      }\n\n      return -0.2;\n    }\n\n\n\n\n\n    function compare041() {\n      const a041s = getFields041(a);\n      const b041s = getFields041(b);\n      if (a041s.length === 0 || b041s.length === 0) {\n        return 0.0; // Should we punish these for badness?\n      }\n      if (a041s.length === 1 && b041s.length === 1) {\n        // The language codes are typically same between different sources. Incur a small penalty for differences though...\n        const sourcePenalty = getSourceOfLanguageCode(a041s[0]) === getSourceOfLanguageCode(b041s[0]) ? 0.0 : -0.05;\n        const mainPenalty = compareLanguageCodeLists(getFieldsLanguageCodes(a041s[0]), getFieldsLanguageCodes(b041s[0]));\n        const scoreInd1 = mainPenalty === 0.0 ? 0.0 : indicator1Penalty(a041s[0], b041s[0]);\n        return mainPenalty + sourcePenalty + scoreInd1;\n      }\n      // Things are already complicated (likely to pe penalized, so no nedd to worrty about language code sources, I daresay.\n      return compareLanguageCodeLists(getLanguageCodesFrom041Fields(a), getLanguageCodesFrom041Fields(b));\n\n      function getFields041(record) {\n        if (!record.fields) {\n          return [];\n        }\n        return record.fields.filter(f => f.tag === '041');\n      }\n\n    }\n\n    function getSourceOfLanguageCode(field) {\n      const sf2 = field.subfields.find(sf => sf.code === '2');\n      if (!sf2) {\n        return 'MARC';\n      }\n      // Normalize the two relevant language code names:\n      if (sf2.value.match(/639.2/u)) {\n        return 'ISO 639-2';\n      }\n      if (sf2.value.match(/639.3/u)) {\n        return 'ISO 639-3';\n      }\n      return sf2.value; // We don't actually have anything else in Melinda though\n    }\n\n    function indicator1Penalty(aField, bField) {\n      if (aField.ind1 === ' ' || bField.ind1 === ' '|| aField.ind1 === bField.ind1 ) {\n        return 0.0;\n      }\n      debug(`\\t Indicator penalty: '${aField.ind1}' vs '${bField.ind1}'`);\n      return -0.1;\n    }\n\n\n\n    function containsNoData(languageCode) {\n      return ['   ', '|||', 'und'].includes(languageCode);\n    }\n\n    function compareLanguageCodeLists(a, b) {\n      if (a.length === 0 || b.length === 0) {\n        debugData(`No language to compare`);\n        return 0;\n      }\n\n      // If either one of the sets has only the generic 'smi', remove the specific sami languages codes from the other as well:\n      const samiLanguageCodes = ['sma', 'sme', 'smj', 'smn', 'sms']; // NB! Don't put generic 'smi' here!\n\n      if (a.includes('smi') && b.includes('smi')) {\n        if (a.some(code => samiLanguageCodes.includes(code)) && !b.some(code => samiLanguageCodes.includes(code))) {\n          return compareLanguageCodeLists(a.filter(val => !samiLanguageCodes.includes(val)), b); // recurse\n        }\n        if (b.some(code => samiLanguageCodes.includes(code)) && !a.some(code => samiLanguageCodes.includes(code))) {\n          return compareLanguageCodeLists(a, b.filter(val => !samiLanguageCodes.includes(val))); // recurse\n        }\n      }\n\n\n\n\n      if (a.length === b.length && a.every((element, index) => element === b[index])) {\n        debugData(`All languages match`);\n        return 0.1;\n      }\n\n      // Handle 'mul'. (Should we check whether the other array contains mul as well, probably not...)\n      // (If 'mul' does not appear alone, then it is very iffy... )\n      if (a.length === 1 && a[0] === 'mul' && b.length > 1) {\n        return 0;\n      }\n      if (b.length === 1 && b[0] === 'mul' && a.length > 1) {\n        return 0;\n      }\n\n\n\n      // Damage control:\n\n      // Not using the generic solution here as eg. 'und' needs a special treatment\n      const sharedValues = getSharedValues(a, b);\n      const aOnly = a.filter(val => !sharedValues.includes(val));\n      const bOnly = b.filter(val => !sharedValues.includes(val));\n      const hasUnd = [...aOnly, ...bOnly].includes('und');\n\n      if (sharedValues.length < 1) {\n        console.info(`NV: ${sharedValues.join(\", \")}`);\n        if (aOnly.length === 1 && bOnly.length === 1 && hasUnd) {\n          debug(`Both have languages, but none of these match. However, the benefit of doubt is given: '${aOnly[0]}' and '${bOnly[0]}' might mean the same}`);\n          return 0;\n        }\n        debug(`Both have languages, but none of these match.`); // includes 'mul' vs a single language code\n        return -1.0;\n      }\n\n      const {matchingValues, possibleMatchValues, maxValues} = getMatchCounts(a, b);\n\n      debug(`Both have languages, ${matchingValues}/${possibleMatchValues} valid languages match.`);\n      // ignore non-matches if there is mismatching amount of values\n      debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);\n      // we give some kind of penalty for mismatching amount of values instead of simple divide?\n      const missingCount = maxValues - possibleMatchValues;\n      const misMatchCount = possibleMatchValues - matchingValues;\n      debug(`\\t missing: ${missingCount}`);\n      debug(`\\t mismatches: ${misMatchCount}`);\n\n      const penaltyForMissing = 0.02 * (maxValues - possibleMatchValues);\n      const penaltyForMisMatch = 0.05 * (possibleMatchValues - matchingValues);\n      debug(`\\t points: penaltyForMissing: ${penaltyForMissing}`);\n      debug(`\\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);\n\n      const points = Number(Number(0.1 - penaltyForMisMatch - penaltyForMissing).toFixed(2));\n      debug(`Total points: ${points}`);\n\n      return points;\n    }\n  }\n\n});\n\nfunction get008Value(record, label = 'record') {\n  if (!record || !record.fields) {\n    return;\n  }\n  const f008 = record.fields.find(f => f.tag === '008');\n  if (!f008) {\n    return undefined;\n  }\n  const value = f008.value || defaultValue;\n\n  if (!value) {\n    debugData(`${label}: Failed to extact 008/35-37 value from '${value}'`);\n    return defaultValue;\n  }\n\n  const code = value.slice(35, 38);\n  debugData(`${label}: 008 code: '${code}'`);\n  return code;\n}\n\n// Check if a string is a possible, validly formed language code for a single language\n// Currently accept also codes in capitals\nfunction isLangCodeForALanguage(code, label = 'record', encoding = undefined) {\n  if (!code) {\n    return false;\n  }\n\n  if ([undefined, 'ISO 639-2', 'ISO-639-3'].includes(encoding) && code.length !== 3) {\n    debugData(`${label}: Code ${code} is not correct length (3) for a language code.`);\n    return false;\n  }\n  if (!encoding) {\n    // 'mul' should be passed as 'mul' can match 'fin' + 'swe' etc.\n    // 'zxx' should be removed by a validator\n    // '^^^' is Aleph-specific corruption, not supporting it anymore\n    if (code === '|||' || code === '   ' ) { // || code === '^^^' || code === 'mul' || code === 'zxx') {\n      debugData(`${label}: Code ${code} is not code for a spesific language.`);\n      return false;\n    }\n  }\n  // Marc, ISO 639-2 and ISO 639-3 are all three-letters long. Not other language codes seen in Melinda:\n  const langCodePattern = /^[a-z][a-z][a-z]$/ui;\n  if (!langCodePattern.test(code)) {\n    debugData(`${label}: Code ${code} is not valid as a language code`);\n    return false;\n  }\n  return true;\n}\n\n\n\nfunction getLanguageCodesFrom041Fields(record) {\n  // NB! We brutally don't check $2 (language code source) as marc's language codes is practically a subset of ISO 639-2,\n  // and also ISO 639-2 and ISO 639-3 overlap to a degree. If we ever run into a trouble with some ISO 639-2 vs 639-3 mismatch, then we'll work it out.\n  // Also we could write a validator that converts ISO 639-2 to marc if applicable and maybe even partial support for ISO-639-3.\n  // Note that ISO 639-2-B values correspond with marc beteer that ISO 639-2-T. Eg. code for Chinese 'zho' should/code be normalized to 'chi'!\n  const values = record.fields.filter(f => f.tag === '041').flatMap(f => getFieldsLanguageCodes(f));\n  /*\n        // .filter(({ind2}) => ind2 === ' ')\n        .map(({subfields}) => subfields)\n        .flat()\n        .filter(({code}) => code === 'a' || code === 'd')\n        .filter(({value}) => value && isLangCodeForALanguage(value))\n        .map(({value}) => value);\n    */\n  return [...new Set(values)].sort();\n}\n\nfunction getFieldsLanguageCodes(field) {\n  const relevantSubfields = field.subfields.filter(sf => sf.code === 'a' || sf.code === 'd').filter(sf => isLangCodeForALanguage(sf.value, 'field'));\n\n  const values = relevantSubfields.map(sf => sf.value).flat();\n\n  return [...new Set(values)].sort();\n}\n\nfunction getSharedValues(list1, list2) {\n  return list1.filter(val => list2.includes(val));\n}"],
-  "mappings": "AACA,OAAO,uBAAuB;AAE9B,SAAQ,kBAAiB;AAEzB,SAAQ,YAAY,oBAAmB;AAEvC,SAAQ,sBAAqB;AAE7B,MAAM,QAAQ,kBAAkB,qEAAqE;AACrG,MAAM,YAAY,MAAM,OAAO,MAAM;AAerC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,eAAe,IAAI,WAAW,QAAQ,OAAO,kBAAkB;AAErE,eAAW,EAAE,IAAI,YAAY;AAC7B,iBAAa,EAAE,IAAI,YAAY;AAI/B,WAAO,CAAC,EAAC,QAAQ,aAAa,QAAQ,QAAQ,aAAa,OAAM,GAAG,KAAK;AAAA,EAC3E;AAAA;AAAA,EAEA,SAAS,CAAC,IAAI,OAAO;AACnB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,cAAU,sBAAsB,KAAK,UAAU,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,EAAE;AAE5E,UAAM,WAAW,WAAW;AAC5B,UAAM,WAAW,WAAW;AAC5B,WAAO,YAAY,WAAW,QAAQ;AAEtC,aAAS,YAAY,OAAO;AAC1B,UAAI,QAAQ,KAAK;AACf,eAAO;AAAA,MACT;AACA,UAAI,QAAQ,IAAI;AACd,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT;AAEA,aAAS,aAAa;AACpB,YAAM,OAAO,YAAY,GAAG,MAAM;AAClC,YAAM,OAAO,YAAY,GAAG,MAAM;AAElC,UAAI,SAAS,UAAa,SAAS,QAAW;AAC5C,eAAO;AAAA,MACT;AAEA,UAAI,eAAe,IAAI,KAAK,eAAe,IAAI,GAAE;AAE/C,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,MAAM;AACjB,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,SAAS,SAAS,OAAO;AACpC,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT;AAMA,aAAS,aAAa;AACpB,YAAM,QAAQ,aAAa,CAAC;AAC5B,YAAM,QAAQ,aAAa,CAAC;AAC5B,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAC5C,eAAO;AAAA,MACT;AACA,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAE5C,cAAM,gBAAgB,wBAAwB,MAAM,CAAC,CAAC,MAAM,wBAAwB,MAAM,CAAC,CAAC,IAAI,IAAM;AACtG,cAAM,cAAc,yBAAyB,uBAAuB,MAAM,CAAC,CAAC,GAAG,uBAAuB,MAAM,CAAC,CAAC,CAAC;AAC/G,cAAM,YAAY,gBAAgB,IAAM,IAAM,kBAAkB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC;AAClF,eAAO,cAAc,gBAAgB;AAAA,MACvC;AAEA,aAAO,yBAAyB,8BAA8B,CAAC,GAAG,8BAA8B,CAAC,CAAC;AAElG,eAAS,aAAa,QAAQ;AAC5B,YAAI,CAAC,OAAO,QAAQ;AAClB,iBAAO,CAAC;AAAA,QACV;AACA,eAAO,OAAO,OAAO,OAAO,OAAK,EAAE,QAAQ,KAAK;AAAA,MAClD;AAAA,IAEF;AAEA,aAAS,wBAAwB,OAAO;AACtC,YAAM,MAAM,MAAM,UAAU,KAAK,QAAM,GAAG,SAAS,GAAG;AACtD,UAAI,CAAC,KAAK;AACR,eAAO;AAAA,MACT;AAEA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,aAAO,IAAI;AAAA,IACb;AAEA,aAAS,kBAAkB,QAAQ,QAAQ;AACzC,UAAI,OAAO,SAAS,OAAO,OAAO,SAAS,OAAM,OAAO,SAAS,OAAO,MAAO;AAC7E,eAAO;AAAA,MACT;AACA,YAAM,yBAA0B,OAAO,IAAI,SAAS,OAAO,IAAI,GAAG;AAClE,aAAO;AAAA,IACT;AAIA,aAAS,eAAe,cAAc;AACpC,aAAO,CAAC,OAAO,OAAO,KAAK,EAAE,SAAS,YAAY;AAAA,IACpD;AAEA,aAAS,yBAAyBA,IAAGC,IAAG;AACtC,UAAID,GAAE,WAAW,KAAKC,GAAE,WAAW,GAAG;AACpC,kBAAU,wBAAwB;AAClC,eAAO;AAAA,MACT;AAGA,YAAM,oBAAoB,CAAC,OAAO,OAAO,OAAO,OAAO,KAAK;AAE5D,UAAID,GAAE,SAAS,KAAK,KAAKC,GAAE,SAAS,KAAK,GAAG;AAC1C,YAAID,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,KAAK,CAACC,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,GAAG;AACzG,iBAAO,yBAAyBD,GAAE,OAAO,SAAO,CAAC,kBAAkB,SAAS,GAAG,CAAC,GAAGC,EAAC;AAAA,QACtF;AACA,YAAIA,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,KAAK,CAACD,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,GAAG;AACzG,iBAAO,yBAAyBA,IAAGC,GAAE,OAAO,SAAO,CAAC,kBAAkB,SAAS,GAAG,CAAC,CAAC;AAAA,QACtF;AAAA,MACF;AAKA,UAAID,GAAE,WAAWC,GAAE,UAAUD,GAAE,MAAM,CAAC,SAAS,UAAU,YAAYC,GAAE,KAAK,CAAC,GAAG;AAC9E,kBAAU,qBAAqB;AAC/B,eAAO;AAAA,MACT;AAIA,UAAID,GAAE,WAAW,KAAKA,GAAE,CAAC,MAAM,SAASC,GAAE,SAAS,GAAG;AACpD,eAAO;AAAA,MACT;AACA,UAAIA,GAAE,WAAW,KAAKA,GAAE,CAAC,MAAM,SAASD,GAAE,SAAS,GAAG;AACpD,eAAO;AAAA,MACT;AAOA,YAAM,eAAe,gBAAgBA,IAAGC,EAAC;AACzC,YAAM,QAAQD,GAAE,OAAO,SAAO,CAAC,aAAa,SAAS,GAAG,CAAC;AACzD,YAAM,QAAQC,GAAE,OAAO,SAAO,CAAC,aAAa,SAAS,GAAG,CAAC;AACzD,YAAM,SAAS,CAAC,GAAG,OAAO,GAAG,KAAK,EAAE,SAAS,KAAK;AAElD,UAAI,aAAa,SAAS,GAAG;AAC3B,gBAAQ,KAAK,OAAO,aAAa,KAAK,IAAI,CAAC,EAAE;AAC7C,YAAI,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,QAAQ;AACtD,gBAAM,0FAA0F,MAAM,CAAC,CAAC,UAAU,MAAM,CAAC,CAAC,wBAAwB;AAClJ,iBAAO;AAAA,QACT;AACA,cAAM,+CAA+C;AACrD,eAAO;AAAA,MACT;AAEA,YAAM,EAAC,gBAAgB,qBAAqB,UAAS,IAAI,eAAeD,IAAGC,EAAC;AAE5E,YAAM,wBAAwB,cAAc,IAAI,mBAAmB,yBAAyB;AAE5F,YAAM,qBAAqB,mBAAmB,IAAI,SAAS,EAAE;AAE7D,YAAM,eAAe,YAAY;AACjC,YAAM,gBAAgB,sBAAsB;AAC5C,YAAM,cAAe,YAAY,EAAE;AACnC,YAAM,iBAAkB,aAAa,EAAE;AAEvC,YAAM,oBAAoB,QAAQ,YAAY;AAC9C,YAAM,qBAAqB,QAAQ,sBAAsB;AACzD,YAAM,gCAAiC,iBAAiB,EAAE;AAC1D,YAAM,iCAAkC,kBAAkB,EAAE;AAE5D,YAAM,SAAS,OAAO,OAAO,MAAM,qBAAqB,iBAAiB,EAAE,QAAQ,CAAC,CAAC;AACrF,YAAM,iBAAiB,MAAM,EAAE;AAE/B,aAAO;AAAA,IACT;AAAA,EACF;AAEF;AAEA,SAAS,YAAY,QAAQ,QAAQ,UAAU;AAC7C,MAAI,CAAC,UAAU,CAAC,OAAO,QAAQ;AAC7B;AAAA,EACF;AACA,QAAM,OAAO,OAAO,OAAO,KAAK,OAAK,EAAE,QAAQ,KAAK;AACpD,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,KAAK,SAAS;AAE5B,MAAI,CAAC,OAAO;AACV,cAAU,GAAG,KAAK,4CAA4C,KAAK,GAAG;AACtE,WAAO;AAAA,EACT;AAEA,QAAM,OAAO,MAAM,MAAM,IAAI,EAAE;AAC/B,YAAU,GAAG,KAAK,gBAAgB,IAAI,GAAG;AACzC,SAAO;AACT;AAIA,SAAS,uBAAuB,MAAM,QAAQ,UAAU,WAAW,QAAW;AAC5E,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AAEA,MAAI,CAAC,QAAW,aAAa,WAAW,EAAE,SAAS,QAAQ,KAAK,KAAK,WAAW,GAAG;AACjF,cAAU,GAAG,KAAK,UAAU,IAAI,iDAAiD;AACjF,WAAO;AAAA,EACT;AACA,MAAI,CAAC,UAAU;AAIb,QAAI,SAAS,SAAS,SAAS,OAAQ;AACrC,gBAAU,GAAG,KAAK,UAAU,IAAI,uCAAuC;AACvE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,kBAAkB;AACxB,MAAI,CAAC,gBAAgB,KAAK,IAAI,GAAG;AAC/B,cAAU,GAAG,KAAK,UAAU,IAAI,kCAAkC;AAClE,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAIA,SAAS,8BAA8B,QAAQ;AAK7C,QAAM,SAAS,OAAO,OAAO,OAAO,OAAK,EAAE,QAAQ,KAAK,EAAE,QAAQ,OAAK,uBAAuB,CAAC,CAAC;AAShG,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,uBAAuB,OAAO;AACrC,QAAM,oBAAoB,MAAM,UAAU,OAAO,QAAM,GAAG,SAAS,OAAO,GAAG,SAAS,GAAG,EAAE,OAAO,QAAM,uBAAuB,GAAG,OAAO,OAAO,CAAC;AAEjJ,QAAM,SAAS,kBAAkB,IAAI,QAAM,GAAG,KAAK,EAAE,KAAK;AAE1D,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,gBAAgB,OAAO,OAAO;AACrC,SAAO,MAAM,OAAO,SAAO,MAAM,SAAS,GAAG,CAAC;AAChD;",
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\n\nimport {MarcRecord} from '@natlibfi/marc-record';\n\nimport {FixSami041, Remove041zxx} from '@natlibfi/marc-record-validators-melinda';\n\nimport {getMatchCounts} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:language');\nconst debugData = debug.extend('data');\n\n// NB! 2025-12-17: I changed the logic drastically. However, I tried to keep the scores as same as possible.\n// 'extract' no longer extracts anything. Instead it clones the record, and does some preprocessing (using validators) instead.\n// 'compare' had multiple changes\n// - f041 ind1 differences ('0' vs '1') now result in a small penalty\n// - Two sami languages related changes:\n// -- validator adds a 'smi' subfield before a corresponding sma/sme/...subfield, if needed (national)\n// -- 'smi' only vs 'smi'+'sma' does not cause penalty\n// - 'mul' vs 'fin'+'swe' does not cause a penalty. However, 'mul' vs 'fin' alone triggers penalty.\n// - 008/35-37 and f041$a/$d are calculated separately\n// - a threshold is applied: return value is always between -1.0 and +0.1 (as it was before)\n// - we try to handle 041 $2 ISO 639-2 and ISO 639-3 as well.\n// - 'und' vs one other language does not cause a penalty in certain contexts. In other contexts it's treated as a normal \"language code\". (Tune/alleviate penalties later on.)\n\nexport default () => ({\n  name: 'Language',\n  extract: ({record, recordExternal}) => {\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n    const clonedRecord = new MarcRecord(record, record._validationOptions); // clone(record); // NB! This loses record.get()...\n\n    FixSami041().fix(clonedRecord); // Handle 'smi' adding if needed\n    Remove041zxx().fix(clonedRecord); // Remove 'zxx' from f041s\n    clonedRecord.fields = clonedRecord.fields.filter(f => ['008', '041'].includes(f.tag));\n    // Add other language code normalizations?\n\n    // Should we return all the fields, or just the relevant fields?\n    return [clonedRecord.fields, label];\n  },\n  // eslint-disable-next-line max-statements\n  compare: (aa, bb) => {\n    const [a, aLabel] = aa;\n    const [b, bLabel] = bb;\n    debugData(`Comparing language ${JSON.stringify(a)} and ${JSON.stringify(b)}`);\n\n    const score008 = compare008();\n    const score041 = compare041();\n    return applyLimits(score008 + score041);\n\n    function applyLimits(score) {\n      if (score > 0.1) { // Keeps the original max, we might have 0.10 (041) + 0.05 (008/35-37) = 0.15 now\n        return 0.1;\n      }\n      return score;\n    }\n\n    function compare008() {\n      const a008 = get008Value(a, aLabel);\n      const b008 = get008Value(b, bLabel);\n      // Something seriously wrong with 008:\n      if (a008 === undefined || b008 === undefined) {\n        return 0.0; // Punish for generic badness?\n      }\n\n      if (containsNoData(a008) || containsNoData(b008)){\n        // Nothing to compare\n        return 0.0;\n      }\n\n      if (a008 === b008) {\n        return 0.05;\n      }\n\n      if (a008 === 'mul' || b008 === 'mul') { // We'll let 041 decide\n        return 0.0;\n      }\n\n      return -0.2;\n    }\n\n\n\n\n\n    function compare041() {\n      const a041s = getFields041(a);\n      const b041s = getFields041(b);\n\n      if (a041s.length === 0 || b041s.length === 0) {\n        return 0.0; // Should we punish these for badness?\n      }\n      if (a041s.length === 1 && b041s.length === 1) {\n        // The language codes are typically same between different sources. Incur a small penalty for differences though...\n        const sourcePenalty = getSourceOfLanguageCode(a041s[0]) === getSourceOfLanguageCode(b041s[0]) ? 0.0 : -0.05;\n        const mainPenalty = compareLanguageCodeLists(getFieldsLanguageCodes(a041s[0]), getFieldsLanguageCodes(b041s[0]));\n        const scoreInd1 = mainPenalty === 0.0 ? 0.0 : indicator1Penalty(a041s[0], b041s[0]);\n        return mainPenalty + sourcePenalty + scoreInd1;\n      }\n      // Things are already complicated (likely to pe penalized, so no nedd to worry about language code sources, I daresay.\n      return compareLanguageCodeLists(getLanguageCodesFrom041Fields(a), getLanguageCodesFrom041Fields(b));\n\n      function getFields041(fields) {\n        if (!fields) {\n          return [];\n        }\n        return fields.filter(f => f.tag === '041');\n      }\n\n    }\n\n    function getSourceOfLanguageCode(field) {\n      const sf2 = field.subfields.find(sf => sf.code === '2');\n      if (!sf2) {\n        return 'MARC';\n      }\n      // Normalize the two relevant language code names:\n      if (sf2.value.match(/639.2/u)) {\n        return 'ISO 639-2';\n      }\n      if (sf2.value.match(/639.3/u)) {\n        return 'ISO 639-3';\n      }\n      return sf2.value; // We don't actually have anything else in Melinda though\n    }\n\n    function indicator1Penalty(aField, bField) {\n      if (aField.ind1 === ' ' || bField.ind1 === ' '|| aField.ind1 === bField.ind1 ) {\n        return 0.0;\n      }\n      debug(`\\t Indicator penalty: '${aField.ind1}' vs '${bField.ind1}'`);\n      return -0.1;\n    }\n\n\n\n    function containsNoData(languageCode) {\n      return ['   ', '|||', 'und'].includes(languageCode);\n    }\n\n    function compareLanguageCodeLists(a, b) {\n      if (a.length === 0 || b.length === 0 || a.every(val => containsNoData(val)) || b.every(val => containsNoData(val))) {\n        debugData(`No language to compare`);\n        return 0;\n      }\n\n      // If either one of the sets has only the generic 'smi', remove the specific sami languages codes from the other as well:\n      const samiLanguageCodes = ['sma', 'sme', 'smj', 'smn', 'sms']; // NB! Don't put generic 'smi' here!\n\n      if (a.includes('smi') && b.includes('smi')) {\n        if (a.some(code => samiLanguageCodes.includes(code)) && !b.some(code => samiLanguageCodes.includes(code))) {\n          return compareLanguageCodeLists(a.filter(val => !samiLanguageCodes.includes(val)), b); // recurse\n        }\n        if (b.some(code => samiLanguageCodes.includes(code)) && !a.some(code => samiLanguageCodes.includes(code))) {\n          return compareLanguageCodeLists(a, b.filter(val => !samiLanguageCodes.includes(val))); // recurse\n        }\n      }\n\n\n\n\n      if (a.length === b.length && a.every((element, index) => element === b[index])) {\n        debugData(`All languages match`);\n        return 0.1;\n      }\n\n      // Handle 'mul'. (Should we check whether the other array contains mul as well, probably not...)\n      // (If 'mul' does not appear alone, then it is very iffy... )\n      if (a.length === 1 && a[0] === 'mul' && b.length > 1) {\n        return 0;\n      }\n      if (b.length === 1 && b[0] === 'mul' && a.length > 1) {\n        return 0;\n      }\n\n\n\n      // Damage control:\n      const sharedValues = getSharedValues(a, b).filter(val => !containsNoData(val));\n\n      if (sharedValues.length < 1) {\n        debug(`Both have languages, but none of these match.`); // includes 'mul' vs a single language code\n        return -1.0;\n      }\n\n      const {matchingValues, possibleMatchValues, maxValues} = getMatchCounts(a, b);\n\n      debug(`Both have languages, ${matchingValues}/${possibleMatchValues} valid languages match.`);\n      // ignore non-matches if there is mismatching amount of values\n      debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);\n      // we give some kind of penalty for mismatching amount of values instead of simple divide?\n      const missingCount = maxValues - possibleMatchValues;\n      const misMatchCount = possibleMatchValues - matchingValues;\n      debug(`\\t missing: ${missingCount}`);\n      debug(`\\t mismatches: ${misMatchCount}`);\n\n      const penaltyForMissing = 0.02 * (maxValues - possibleMatchValues);\n      const penaltyForMisMatch = 0.05 * (possibleMatchValues - matchingValues);\n      debug(`\\t points: penaltyForMissing: ${penaltyForMissing}`);\n      debug(`\\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);\n\n      const points = Number(Number(0.1 - penaltyForMisMatch - penaltyForMissing).toFixed(2));\n      debug(`Total points: ${points}`);\n\n      return points;\n    }\n  }\n\n});\n\nfunction get008Value(fields, label = 'record') {\n  if (!fields) {\n    return;\n  }\n  const f008 = fields.find(f => f.tag === '008');\n  if (!f008) {\n    return undefined;\n  }\n  const value = f008.value || defaultValue;\n\n  if (!value) {\n    debugData(`${label}: Failed to extact 008/35-37 value from '${value}'`);\n    return defaultValue;\n  }\n\n  const code = value.slice(35, 38);\n  debugData(`${label}: 008 code: '${code}'`);\n  return code;\n}\n\n// Check if a string is a possible, validly formed language code for a single language\n// Currently accept also codes in capitals\nfunction isLangCodeForALanguage(code, label = 'record', encoding = undefined) {\n  if (!code) {\n    return false;\n  }\n\n  if ([undefined, 'ISO 639-2', 'ISO-639-3'].includes(encoding) && code.length !== 3) {\n    debugData(`${label}: Code ${code} is not correct length (3) for a language code.`);\n    return false;\n  }\n  if (!encoding) {\n    // 'mul' should be passed as 'mul' can match 'fin' + 'swe' etc.\n    // 'zxx' should be removed by a validator\n    // '^^^' is Aleph-specific corruption, not supporting it anymore\n    if (code === '|||' || code === '   ' ) { // || code === '^^^' || code === 'mul' || code === 'zxx') {\n      debugData(`${label}: Code ${code} is not code for a spesific language.`);\n      return false;\n    }\n  }\n  // Marc, ISO 639-2 and ISO 639-3 are all three-letters long. Not other language codes seen in Melinda:\n  const langCodePattern = /^[a-z][a-z][a-z]$/ui;\n  if (!langCodePattern.test(code)) {\n    debugData(`${label}: Code ${code} is not valid as a language code`);\n    return false;\n  }\n  return true;\n}\n\n\n\nfunction getLanguageCodesFrom041Fields(fields) {\n  // NB! We brutally don't check $2 (language code source) as marc's language codes is practically a subset of ISO 639-2,\n  // and also ISO 639-2 and ISO 639-3 overlap to a degree. If we ever run into a trouble with some ISO 639-2 vs 639-3 mismatch, then we'll work it out.\n  // Also we could write a validator that converts ISO 639-2 to marc if applicable and maybe even partial support for ISO-639-3.\n  // Note that ISO 639-2-B values correspond with marc beteer that ISO 639-2-T. Eg. code for Chinese 'zho' should/code be normalized to 'chi'!\n  const values = fields.filter(f => f.tag === '041').flatMap(f => getFieldsLanguageCodes(f));\n  /*\n        // .filter(({ind2}) => ind2 === ' ')\n        .map(({subfields}) => subfields)\n        .flat()\n        .filter(({code}) => code === 'a' || code === 'd')\n        .filter(({value}) => value && isLangCodeForALanguage(value))\n        .map(({value}) => value);\n    */\n  return [...new Set(values)].sort();\n}\n\nfunction getFieldsLanguageCodes(field) {\n  const relevantSubfields = field.subfields.filter(sf => sf.code === 'a' || sf.code === 'd').filter(sf => isLangCodeForALanguage(sf.value, 'field'));\n\n  const values = relevantSubfields.map(sf => sf.value).flat();\n\n  return [...new Set(values)].sort();\n}\n\nfunction getSharedValues(list1, list2) {\n  return list1.filter(val => list2.includes(val));\n}"],
+  "mappings": "AACA,OAAO,uBAAuB;AAE9B,SAAQ,kBAAiB;AAEzB,SAAQ,YAAY,oBAAmB;AAEvC,SAAQ,sBAAqB;AAE7B,MAAM,QAAQ,kBAAkB,qEAAqE;AACrG,MAAM,YAAY,MAAM,OAAO,MAAM;AAerC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,eAAe,IAAI,WAAW,QAAQ,OAAO,kBAAkB;AAErE,eAAW,EAAE,IAAI,YAAY;AAC7B,iBAAa,EAAE,IAAI,YAAY;AAC/B,iBAAa,SAAS,aAAa,OAAO,OAAO,OAAK,CAAC,OAAO,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC;AAIpF,WAAO,CAAC,aAAa,QAAQ,KAAK;AAAA,EACpC;AAAA;AAAA,EAEA,SAAS,CAAC,IAAI,OAAO;AACnB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,cAAU,sBAAsB,KAAK,UAAU,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,EAAE;AAE5E,UAAM,WAAW,WAAW;AAC5B,UAAM,WAAW,WAAW;AAC5B,WAAO,YAAY,WAAW,QAAQ;AAEtC,aAAS,YAAY,OAAO;AAC1B,UAAI,QAAQ,KAAK;AACf,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT;AAEA,aAAS,aAAa;AACpB,YAAM,OAAO,YAAY,GAAG,MAAM;AAClC,YAAM,OAAO,YAAY,GAAG,MAAM;AAElC,UAAI,SAAS,UAAa,SAAS,QAAW;AAC5C,eAAO;AAAA,MACT;AAEA,UAAI,eAAe,IAAI,KAAK,eAAe,IAAI,GAAE;AAE/C,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,MAAM;AACjB,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,SAAS,SAAS,OAAO;AACpC,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT;AAMA,aAAS,aAAa;AACpB,YAAM,QAAQ,aAAa,CAAC;AAC5B,YAAM,QAAQ,aAAa,CAAC;AAE5B,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAC5C,eAAO;AAAA,MACT;AACA,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAE5C,cAAM,gBAAgB,wBAAwB,MAAM,CAAC,CAAC,MAAM,wBAAwB,MAAM,CAAC,CAAC,IAAI,IAAM;AACtG,cAAM,cAAc,yBAAyB,uBAAuB,MAAM,CAAC,CAAC,GAAG,uBAAuB,MAAM,CAAC,CAAC,CAAC;AAC/G,cAAM,YAAY,gBAAgB,IAAM,IAAM,kBAAkB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC;AAClF,eAAO,cAAc,gBAAgB;AAAA,MACvC;AAEA,aAAO,yBAAyB,8BAA8B,CAAC,GAAG,8BAA8B,CAAC,CAAC;AAElG,eAAS,aAAa,QAAQ;AAC5B,YAAI,CAAC,QAAQ;AACX,iBAAO,CAAC;AAAA,QACV;AACA,eAAO,OAAO,OAAO,OAAK,EAAE,QAAQ,KAAK;AAAA,MAC3C;AAAA,IAEF;AAEA,aAAS,wBAAwB,OAAO;AACtC,YAAM,MAAM,MAAM,UAAU,KAAK,QAAM,GAAG,SAAS,GAAG;AACtD,UAAI,CAAC,KAAK;AACR,eAAO;AAAA,MACT;AAEA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,aAAO,IAAI;AAAA,IACb;AAEA,aAAS,kBAAkB,QAAQ,QAAQ;AACzC,UAAI,OAAO,SAAS,OAAO,OAAO,SAAS,OAAM,OAAO,SAAS,OAAO,MAAO;AAC7E,eAAO;AAAA,MACT;AACA,YAAM,yBAA0B,OAAO,IAAI,SAAS,OAAO,IAAI,GAAG;AAClE,aAAO;AAAA,IACT;AAIA,aAAS,eAAe,cAAc;AACpC,aAAO,CAAC,OAAO,OAAO,KAAK,EAAE,SAAS,YAAY;AAAA,IACpD;AAEA,aAAS,yBAAyBA,IAAGC,IAAG;AACtC,UAAID,GAAE,WAAW,KAAKC,GAAE,WAAW,KAAKD,GAAE,MAAM,SAAO,eAAe,GAAG,CAAC,KAAKC,GAAE,MAAM,SAAO,eAAe,GAAG,CAAC,GAAG;AAClH,kBAAU,wBAAwB;AAClC,eAAO;AAAA,MACT;AAGA,YAAM,oBAAoB,CAAC,OAAO,OAAO,OAAO,OAAO,KAAK;AAE5D,UAAID,GAAE,SAAS,KAAK,KAAKC,GAAE,SAAS,KAAK,GAAG;AAC1C,YAAID,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,KAAK,CAACC,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,GAAG;AACzG,iBAAO,yBAAyBD,GAAE,OAAO,SAAO,CAAC,kBAAkB,SAAS,GAAG,CAAC,GAAGC,EAAC;AAAA,QACtF;AACA,YAAIA,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,KAAK,CAACD,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,GAAG;AACzG,iBAAO,yBAAyBA,IAAGC,GAAE,OAAO,SAAO,CAAC,kBAAkB,SAAS,GAAG,CAAC,CAAC;AAAA,QACtF;AAAA,MACF;AAKA,UAAID,GAAE,WAAWC,GAAE,UAAUD,GAAE,MAAM,CAAC,SAAS,UAAU,YAAYC,GAAE,KAAK,CAAC,GAAG;AAC9E,kBAAU,qBAAqB;AAC/B,eAAO;AAAA,MACT;AAIA,UAAID,GAAE,WAAW,KAAKA,GAAE,CAAC,MAAM,SAASC,GAAE,SAAS,GAAG;AACpD,eAAO;AAAA,MACT;AACA,UAAIA,GAAE,WAAW,KAAKA,GAAE,CAAC,MAAM,SAASD,GAAE,SAAS,GAAG;AACpD,eAAO;AAAA,MACT;AAKA,YAAM,eAAe,gBAAgBA,IAAGC,EAAC,EAAE,OAAO,SAAO,CAAC,eAAe,GAAG,CAAC;AAE7E,UAAI,aAAa,SAAS,GAAG;AAC3B,cAAM,+CAA+C;AACrD,eAAO;AAAA,MACT;AAEA,YAAM,EAAC,gBAAgB,qBAAqB,UAAS,IAAI,eAAeD,IAAGC,EAAC;AAE5E,YAAM,wBAAwB,cAAc,IAAI,mBAAmB,yBAAyB;AAE5F,YAAM,qBAAqB,mBAAmB,IAAI,SAAS,EAAE;AAE7D,YAAM,eAAe,YAAY;AACjC,YAAM,gBAAgB,sBAAsB;AAC5C,YAAM,cAAe,YAAY,EAAE;AACnC,YAAM,iBAAkB,aAAa,EAAE;AAEvC,YAAM,oBAAoB,QAAQ,YAAY;AAC9C,YAAM,qBAAqB,QAAQ,sBAAsB;AACzD,YAAM,gCAAiC,iBAAiB,EAAE;AAC1D,YAAM,iCAAkC,kBAAkB,EAAE;AAE5D,YAAM,SAAS,OAAO,OAAO,MAAM,qBAAqB,iBAAiB,EAAE,QAAQ,CAAC,CAAC;AACrF,YAAM,iBAAiB,MAAM,EAAE;AAE/B,aAAO;AAAA,IACT;AAAA,EACF;AAEF;AAEA,SAAS,YAAY,QAAQ,QAAQ,UAAU;AAC7C,MAAI,CAAC,QAAQ;AACX;AAAA,EACF;AACA,QAAM,OAAO,OAAO,KAAK,OAAK,EAAE,QAAQ,KAAK;AAC7C,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,KAAK,SAAS;AAE5B,MAAI,CAAC,OAAO;AACV,cAAU,GAAG,KAAK,4CAA4C,KAAK,GAAG;AACtE,WAAO;AAAA,EACT;AAEA,QAAM,OAAO,MAAM,MAAM,IAAI,EAAE;AAC/B,YAAU,GAAG,KAAK,gBAAgB,IAAI,GAAG;AACzC,SAAO;AACT;AAIA,SAAS,uBAAuB,MAAM,QAAQ,UAAU,WAAW,QAAW;AAC5E,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AAEA,MAAI,CAAC,QAAW,aAAa,WAAW,EAAE,SAAS,QAAQ,KAAK,KAAK,WAAW,GAAG;AACjF,cAAU,GAAG,KAAK,UAAU,IAAI,iDAAiD;AACjF,WAAO;AAAA,EACT;AACA,MAAI,CAAC,UAAU;AAIb,QAAI,SAAS,SAAS,SAAS,OAAQ;AACrC,gBAAU,GAAG,KAAK,UAAU,IAAI,uCAAuC;AACvE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,kBAAkB;AACxB,MAAI,CAAC,gBAAgB,KAAK,IAAI,GAAG;AAC/B,cAAU,GAAG,KAAK,UAAU,IAAI,kCAAkC;AAClE,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAIA,SAAS,8BAA8B,QAAQ;AAK7C,QAAM,SAAS,OAAO,OAAO,OAAK,EAAE,QAAQ,KAAK,EAAE,QAAQ,OAAK,uBAAuB,CAAC,CAAC;AASzF,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,uBAAuB,OAAO;AACrC,QAAM,oBAAoB,MAAM,UAAU,OAAO,QAAM,GAAG,SAAS,OAAO,GAAG,SAAS,GAAG,EAAE,OAAO,QAAM,uBAAuB,GAAG,OAAO,OAAO,CAAC;AAEjJ,QAAM,SAAS,kBAAkB,IAAI,QAAM,GAAG,KAAK,EAAE,KAAK;AAE1D,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,gBAAgB,OAAO,OAAO;AACrC,SAAO,MAAM,OAAO,SAAO,MAAM,SAAS,GAAG,CAAC;AAChD;",
   "names": ["a", "b"]
 }

package/dist/match-detection/features/bib/publication-time-allow-cons-years.js

@@ -1,5 +1,6 @@
 import createDebugLogger from "debug";
 import { testStringOrNumber } from "../../../matching-utils.js";
+import { matchingYears } from "./publication-time.js";
 export default () => ({
   name: "Publication time, allow consequent years",
   extract: ({ record }) => {
@@ -11,11 +12,14 @@ export default () => ({
     debug(`Comparing ${a[0]} to ${b[0]}`);
     const [firstA] = a;
     const [firstB] = b;
+    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {
+      return 0;
+    }
     if (firstA === firstB) {
       return 0.1;
     }
-    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {
-      return 0;
+    if (matchingYears(firstA, firstB)) {
+      return 0.05;
     }
     const firstANumber = parseInt(firstA, 10);
     const firstBNumber = parseInt(firstB, 10);

package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../../src/match-detection/features/bib/publication-time-allow-cons-years.js"],
-  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\n// We should also get copyright time and copyright/publication times from 26x\n\nexport default () => ({\n  name: 'Publication time, allow consequent years',\n  extract: ({record}) => {\n    const value = record.get(/^008$/u)?.[0]?.value || undefined;\n    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years');\n    debug(`Comparing ${a[0]} to ${b[0]}`);\n\n    const [firstA] = a;\n    const [firstB] = b;\n\n    if (firstA === firstB) {\n      return 0.1;\n    }\n\n    // If either of years is a non string/number, values are not comparable\n    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {\n      return 0;\n    }\n\n    const firstANumber = parseInt(firstA, 10);\n    const firstBNumber = parseInt(firstB, 10);\n\n    if (isNaN(firstANumber) || isNaN(firstBNumber)) {\n      return -1;\n    }\n\n    // Handle consequent years as a match\n    // see publication-time for a version that does not handle consequent years as a match\n    return firstANumber + 1 === firstBNumber || firstANumber - 1 === firstBNumber ? 0.1 : -1;\n  }\n});\n"],
-  "mappings": "AACA,OAAO,uBAAuB;AAC9B,SAAQ,0BAAyB;AAIjC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,UAAM,QAAQ,OAAO,IAAI,QAAQ,IAAI,CAAC,GAAG,SAAS;AAClD,WAAO,mBAAmB,KAAK,IAAI,CAAC,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC;AAAA,EACrE;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,QAAQ,kBAAkB,kGAAkG;AAClI,UAAM,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE;AAEpC,UAAM,CAAC,MAAM,IAAI;AACjB,UAAM,CAAC,MAAM,IAAI;AAEjB,QAAI,WAAW,QAAQ;AACrB,aAAO;AAAA,IACT;AAGA,QAAI,CAAC,mBAAmB,MAAM,KAAK,CAAC,mBAAmB,MAAM,GAAG;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,SAAS,QAAQ,EAAE;AACxC,UAAM,eAAe,SAAS,QAAQ,EAAE;AAExC,QAAI,MAAM,YAAY,KAAK,MAAM,YAAY,GAAG;AAC9C,aAAO;AAAA,IACT;AAIA,WAAO,eAAe,MAAM,gBAAgB,eAAe,MAAM,eAAe,MAAM;AAAA,EACxF;AACF;",
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport {testStringOrNumber} from '../../../matching-utils.js';\nimport {matchingYears} from './publication-time.js';\n\n// We should also get copyright time and copyright/publication times from 26x\n\nexport default () => ({\n  name: 'Publication time, allow consequent years',\n  extract: ({record}) => {\n    const value = record.get(/^008$/u)?.[0]?.value || undefined;\n    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years');\n    debug(`Comparing ${a[0]} to ${b[0]}`);\n\n    const [firstA] = a;\n    const [firstB] = b;\n    // Sanity check: If either of years is a non string/number, values are not comparable\n    if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {\n      return 0.0;\n    }\n\n    if (firstA === firstB) {\n      return 0.1;\n    }\n    if (matchingYears(firstA, firstB)) { // Handle 'u' Eg. '18uu' vs '1812'\n      return 0.05;\n    }\n\n    const firstANumber = parseInt(firstA, 10);\n    const firstBNumber = parseInt(firstB, 10);\n\n    if (isNaN(firstANumber) || isNaN(firstBNumber)) {\n      return -1;\n    }\n\n    // Handle consequent years as a match\n    // see publication-time for a version that does not handle consequent years as a match\n    return firstANumber + 1 === firstBNumber || firstANumber - 1 === firstBNumber ? 0.1 : -1;\n  }\n});\n"],
+  "mappings": "AACA,OAAO,uBAAuB;AAC9B,SAAQ,0BAAyB;AACjC,SAAQ,qBAAoB;AAI5B,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,UAAM,QAAQ,OAAO,IAAI,QAAQ,IAAI,CAAC,GAAG,SAAS;AAClD,WAAO,mBAAmB,KAAK,IAAI,CAAC,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC;AAAA,EACrE;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,QAAQ,kBAAkB,kGAAkG;AAClI,UAAM,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE;AAEpC,UAAM,CAAC,MAAM,IAAI;AACjB,UAAM,CAAC,MAAM,IAAI;AAEjB,QAAI,CAAC,mBAAmB,MAAM,KAAK,CAAC,mBAAmB,MAAM,GAAG;AAC9D,aAAO;AAAA,IACT;AAEA,QAAI,WAAW,QAAQ;AACrB,aAAO;AAAA,IACT;AACA,QAAI,cAAc,QAAQ,MAAM,GAAG;AACjC,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,SAAS,QAAQ,EAAE;AACxC,UAAM,eAAe,SAAS,QAAQ,EAAE;AAExC,QAAI,MAAM,YAAY,KAAK,MAAM,YAAY,GAAG;AAC9C,aAAO;AAAA,IACT;AAIA,WAAO,eAAe,MAAM,gBAAgB,eAAe,MAAM,eAAe,MAAM;AAAA,EACxF;AACF;",
   "names": []
 }

package/dist/match-detection/features/bib/publication-time.js

@@ -1,9 +1,15 @@
-import { testStringOrNumber } from "../../../matching-utils.js";
+import { extractPublicationYearFrom773 } from "../../../candidate-search/query-list/component.js";
+const MAX = 0.1;
+const MIN = -1;
 export default () => ({
   name: "Publication time",
   extract: ({ record }) => {
-    return getDateData();
-    function getDateData() {
+    let dateData = getDataFrom008();
+    const [f773] = record.get(/^773$/u);
+    const hostYear = f773 && extractPublicationYearFrom773(f773) || null;
+    dateData.hostYear = hostYear;
+    return dateData;
+    function getDataFrom008() {
       const [f008] = record.get(/^008$/u);
       if (!f008 || f008.value.length < 16) {
         return splitDateData("|        ");
@@ -18,35 +24,75 @@ export default () => ({
     }
   },
   compare: (aa, bb) => {
+    if (aa.hostYear && bb.hostYear && aa.hostYear === bb.hostYear) {
+      return MAX;
+    }
     if (aa.typeOfDate === "b") {
       if (bb.typeOfDate === "b") {
         return 0;
       }
-      return -1;
+      return MIN;
     }
     if (aa.typeOfDate === "n" || bb.typeOfDate === "n") {
       return 0;
     }
+    if (aa.typeOfDate === "q") {
+      if (bb.typeOfDate !== "q") {
+        if (yearInBetween(aa.date1, bb.date1, aa.date2)) {
+          return MAX;
+        }
+        return MIN;
+      }
+      if (aa.date1 === bb.date1 && bb.date1 === bb.date2) {
+        return MAX;
+      }
+      return 0;
+    }
+    if (bb.typeOfDate === "q") {
+      if (yearInBetween(bb.date1, aa.date1, bb.date2)) {
+        return MAX;
+      }
+      return MIN;
+    }
     const skipList = ["    ", "||||", "uuuu"];
     if (skipList.includes(aa.date1) || skipList.includes(bb.date1)) {
       return 0;
     }
     if (matchingYears(aa.date1, bb.date1)) {
-      return 0.1;
+      return MAX;
     }
-    return -1;
-    function matchingYears(yyyy1, yyyy2) {
-      if (yyyy1.length === 0) {
-        return true;
+    if (isValidYear(aa.date1) && bb.date1) {
+      const difference = Math.abs(aa.date1 - bb.date1);
+      if (difference === 1) {
+        return MIN / 8;
       }
-      if (yyyy1[0] === "u" || yyyy2[0] === "u") {
-        return matchingYears(yyyy1.slice(1), yyyy2.slice(1));
-      }
-      if (yyyy1[0] !== yyyy2[0]) {
-        return false;
-      }
-      return matchingYears(yyyy1.slice(1), yyyy2.slice(1));
     }
+    return MIN;
   }
 });
+function yearInBetween(start, curr, end) {
+  if (!isValidYear(start) || !isValidYear(curr) || !isValidYear(end)) {
+    return false;
+  }
+  return start <= curr && curr <= end;
+}
+export function matchingYears(yyyy1, yyyy2) {
+  if (yyyy1.length === 0) {
+    return true;
+  }
+  if (yyyy1[0] === "u" || yyyy2[0] === "u") {
+    return matchingYears(yyyy1.slice(1), yyyy2.slice(1));
+  }
+  if (yyyy1[0] !== yyyy2[0]) {
+    return false;
+  }
+  if (yyyy1[0] >= "0" && yyyy1[0] <= "9") {
+    return matchingYears(yyyy1.slice(1), yyyy2.slice(1));
+  }
+  return false;
+}
+const validYearRegexp = /^(?:1[89][0-9][0-9]|20[012][0-9])$/u;
+function isValidYear(yyyy) {
+  return validYearRegexp.test(yyyy);
+}
 //# sourceMappingURL=publication-time.js.map

package/dist/match-detection/features/bib/publication-time.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../../src/match-detection/features/bib/publication-time.js"],
-  "sourcesContent": ["\n\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\n// We should also get copyright time and copyright/publication times from 26x\n// see publication-time-allow-cons-years for a version allowing consequent years to match\n\nexport default () => ({\n  name: 'Publication time',\n  extract: ({record}) => {\n    return getDateData();\n    //const value = record.get(/^008$/u)?.[0]?.value || undefined;\n    //return testStringOrNumber(value) ? [String(value).slice(6, 15)] : [];\n\n    function getDateData() {\n      const [f008] = record.get(/^008$/u);\n      if (!f008 || f008.value.length < 16) {\n        return splitDateData('|        ');\n      }\n      return splitDateData(f008.value.slice(6, 15));\n    }\n\n    function splitDateData(data) {\n      const typeOfDate = data[0]; // 008/06\n      const date1 = data.slice(1,5); // 008/07-10\n      const date2 = data.slice(5); // 008/11-14\n      return {typeOfDate, date1, date2};\n    }\n  },\n\n  compare: (aa, bb) => {\n    if (aa.typeOfDate === 'b') { // Berfore Christ. No really makes sense in our domain, though.\n      if (bb.typeOfDate === 'b') {\n        return 0;\n      }\n      return -1.0;\n    }\n    if (aa.typeOfDate === 'n' || bb.typeOfDate === 'n') { // n=unknown\n      return 0;\n    }\n\n    const skipList = ['    ', '||||', 'uuuu'];\n    if (skipList.includes(aa.date1) || skipList.includes(bb.date1)) { // 008/07-10 carries no information\n      return 0;\n    }\n\n    if (matchingYears(aa.date1, bb.date1)) { // 'u' support\n      return 0.1;\n    }\n\n    // TODO: add $q support here\n\n    return -1.0;\n\n    function matchingYears(yyyy1, yyyy2) {\n      if (yyyy1.length === 0) { // All digits have been succesfully consumed -> success\n        return true;\n      }\n      if (yyyy1[0] === 'u' || yyyy2[0] === 'u') { // Ignore 'u' (it refers to unknown millenia, century, decade or year)\n        return matchingYears(yyyy1.slice(1), yyyy2.slice(1));\n      }\n      if (yyyy1[0] !== yyyy2[0]) {\n        return false;\n      }\n      // Should we require that yyyy[0] is a digit at this point?\n      return matchingYears(yyyy1.slice(1), yyyy2.slice(1));\n    }\n\n    /*\n    function hasFourDigits(yyyy) { // Will be needed by 'q' support\n      return yyyy.match(/^[0-9]{4}$/u);\n    }\n      */\n  }\n\n});\n\n\n"],
-  "mappings": "AAEA,SAAQ,0BAAyB;AAKjC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,WAAO,YAAY;AAInB,aAAS,cAAc;AACrB,YAAM,CAAC,IAAI,IAAI,OAAO,IAAI,QAAQ;AAClC,UAAI,CAAC,QAAQ,KAAK,MAAM,SAAS,IAAI;AACnC,eAAO,cAAc,WAAW;AAAA,MAClC;AACA,aAAO,cAAc,KAAK,MAAM,MAAM,GAAG,EAAE,CAAC;AAAA,IAC9C;AAEA,aAAS,cAAc,MAAM;AAC3B,YAAM,aAAa,KAAK,CAAC;AACzB,YAAM,QAAQ,KAAK,MAAM,GAAE,CAAC;AAC5B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,aAAO,EAAC,YAAY,OAAO,MAAK;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,SAAS,CAAC,IAAI,OAAO;AACnB,QAAI,GAAG,eAAe,KAAK;AACzB,UAAI,GAAG,eAAe,KAAK;AACzB,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT;AACA,QAAI,GAAG,eAAe,OAAO,GAAG,eAAe,KAAK;AAClD,aAAO;AAAA,IACT;AAEA,UAAM,WAAW,CAAC,QAAQ,QAAQ,MAAM;AACxC,QAAI,SAAS,SAAS,GAAG,KAAK,KAAK,SAAS,SAAS,GAAG,KAAK,GAAG;AAC9D,aAAO;AAAA,IACT;AAEA,QAAI,cAAc,GAAG,OAAO,GAAG,KAAK,GAAG;AACrC,aAAO;AAAA,IACT;AAIA,WAAO;AAEP,aAAS,cAAc,OAAO,OAAO;AACnC,UAAI,MAAM,WAAW,GAAG;AACtB,eAAO;AAAA,MACT;AACA,UAAI,MAAM,CAAC,MAAM,OAAO,MAAM,CAAC,MAAM,KAAK;AACxC,eAAO,cAAc,MAAM,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,CAAC;AAAA,MACrD;AACA,UAAI,MAAM,CAAC,MAAM,MAAM,CAAC,GAAG;AACzB,eAAO;AAAA,MACT;AAEA,aAAO,cAAc,MAAM,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,CAAC;AAAA,IACrD;AAAA,EAOF;AAEF;",
+  "sourcesContent": ["// We should also get copyright time and copyright/publication times from 26x\n// see publication-time-allow-cons-years for a version allowing consequent years to match\n\nimport {extractPublicationYearFrom773} from '../../../candidate-search/query-list/component.js';\n\nconst MAX = 0.1;\nconst MIN = -1.0;\n\n\n\nexport default () => ({\n  name: 'Publication time',\n  extract: ({record}) => {\n    let dateData = getDataFrom008();\n    const [f773] = record.get(/^773$/u);\n    const hostYear = f773 && extractPublicationYearFrom773(f773) || null;\n    dateData.hostYear = hostYear;\n    return dateData;\n\n    function getDataFrom008() {\n      const [f008] = record.get(/^008$/u);\n      if (!f008 || f008.value.length < 16) {\n        return splitDateData('|        ');\n      }\n      return splitDateData(f008.value.slice(6, 15));\n    }\n\n    function splitDateData(data) {\n      const typeOfDate = data[0]; // 008/06\n      const date1 = data.slice(1,5); // 008/07-10\n      const date2 = data.slice(5); // 008/11-14\n      return {typeOfDate, date1, date2};\n    }\n  },\n\n  compare: (aa, bb) => {\n    // Be happy with a f773$g match:\n    if (aa.hostYear && bb.hostYear && aa.hostYear === bb.hostYear) {\n      return MAX;\n    }\n    // Check 008\n    if (aa.typeOfDate === 'b') { // 008/06: Before Christ. No really makes sense in our domain, though.\n      if (bb.typeOfDate === 'b') {\n        return 0;\n      }\n      return MIN;\n    }\n    if (aa.typeOfDate === 'n' || bb.typeOfDate === 'n') { // n=unknown\n      return 0;\n    }\n\n    // Try to handle questionable dates:\n    if (aa.typeOfDate === 'q') { // questionable data\n      if (bb.typeOfDate !== 'q') {\n        if (yearInBetween(aa.date1, bb.date1, aa.date2)) {\n          return MAX;\n        }\n        return MIN;\n      }\n\n      // What if there are questionable dates on both sides?\n      if (aa.date1 === bb.date1 && bb.date1 === bb.date2) {\n        return MAX;\n      }\n      // Lazily return 0. (We could check for overlap etc. but not really worth the effort)\n      return 0.0;\n    }\n    if (bb.typeOfDate === 'q') {\n      if (yearInBetween(bb.date1, aa.date1, bb.date2)) {\n        return MAX;\n      }\n      return MIN;\n    }\n\n    const skipList = ['    ', '||||', 'uuuu'];\n    if (skipList.includes(aa.date1) || skipList.includes(bb.date1)) { // 008/07-10 carries no information\n      return 0;\n    }\n\n    if (matchingYears(aa.date1, bb.date1)) { // 'u' support\n      return MAX;\n    }\n\n    if (isValidYear(aa.date1) && bb.date1) {\n      const difference = Math.abs(aa.date1 - bb.date1);\n      if (difference === 1) {\n        return MIN/8;\n      }\n    }\n\n    return MIN;\n  }\n\n});\n\nfunction yearInBetween(start, curr, end) {\n  if (!isValidYear(start) || !isValidYear(curr) || !isValidYear(end)) {\n    return false;\n  }\n  return start <= curr && curr <= end;\n}\n\nexport function matchingYears(yyyy1, yyyy2) {\n  if (yyyy1.length === 0) { // All digits have been succesfully consumed -> success\n    return true;\n  }\n  if (yyyy1[0] === 'u' || yyyy2[0] === 'u') { // Ignore 'u' (it refers to unknown millenia, century, decade or year)\n    return matchingYears(yyyy1.slice(1), yyyy2.slice(1));\n  }\n  if (yyyy1[0] !== yyyy2[0]) {\n    return false;\n  }\n  if (yyyy1[0] >= '0' && yyyy1[0] <= '9') { // Require that yyyy[0] is a digit at this point? (What if year is 983?)\n    return matchingYears(yyyy1.slice(1), yyyy2.slice(1));\n  }\n  return false;\n}\n\n\nconst validYearRegexp = /^(?:1[89][0-9][0-9]|20[012][0-9])$/u;\n\nfunction isValidYear(yyyy) {\n  return validYearRegexp.test(yyyy); // Currently supports 1800-2029\n}\n"],
+  "mappings": "AAGA,SAAQ,qCAAoC;AAE5C,MAAM,MAAM;AACZ,MAAM,MAAM;AAIZ,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,QAAI,WAAW,eAAe;AAC9B,UAAM,CAAC,IAAI,IAAI,OAAO,IAAI,QAAQ;AAClC,UAAM,WAAW,QAAQ,8BAA8B,IAAI,KAAK;AAChE,aAAS,WAAW;AACpB,WAAO;AAEP,aAAS,iBAAiB;AACxB,YAAM,CAAC,IAAI,IAAI,OAAO,IAAI,QAAQ;AAClC,UAAI,CAAC,QAAQ,KAAK,MAAM,SAAS,IAAI;AACnC,eAAO,cAAc,WAAW;AAAA,MAClC;AACA,aAAO,cAAc,KAAK,MAAM,MAAM,GAAG,EAAE,CAAC;AAAA,IAC9C;AAEA,aAAS,cAAc,MAAM;AAC3B,YAAM,aAAa,KAAK,CAAC;AACzB,YAAM,QAAQ,KAAK,MAAM,GAAE,CAAC;AAC5B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,aAAO,EAAC,YAAY,OAAO,MAAK;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,SAAS,CAAC,IAAI,OAAO;AAEnB,QAAI,GAAG,YAAY,GAAG,YAAY,GAAG,aAAa,GAAG,UAAU;AAC7D,aAAO;AAAA,IACT;AAEA,QAAI,GAAG,eAAe,KAAK;AACzB,UAAI,GAAG,eAAe,KAAK;AACzB,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT;AACA,QAAI,GAAG,eAAe,OAAO,GAAG,eAAe,KAAK;AAClD,aAAO;AAAA,IACT;AAGA,QAAI,GAAG,eAAe,KAAK;AACzB,UAAI,GAAG,eAAe,KAAK;AACzB,YAAI,cAAc,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,GAAG;AAC/C,iBAAO;AAAA,QACT;AACA,eAAO;AAAA,MACT;AAGA,UAAI,GAAG,UAAU,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO;AAClD,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT;AACA,QAAI,GAAG,eAAe,KAAK;AACzB,UAAI,cAAc,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,GAAG;AAC/C,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT;AAEA,UAAM,WAAW,CAAC,QAAQ,QAAQ,MAAM;AACxC,QAAI,SAAS,SAAS,GAAG,KAAK,KAAK,SAAS,SAAS,GAAG,KAAK,GAAG;AAC9D,aAAO;AAAA,IACT;AAEA,QAAI,cAAc,GAAG,OAAO,GAAG,KAAK,GAAG;AACrC,aAAO;AAAA,IACT;AAEA,QAAI,YAAY,GAAG,KAAK,KAAK,GAAG,OAAO;AACrC,YAAM,aAAa,KAAK,IAAI,GAAG,QAAQ,GAAG,KAAK;AAC/C,UAAI,eAAe,GAAG;AACpB,eAAO,MAAI;AAAA,MACb;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAEF;AAEA,SAAS,cAAc,OAAO,MAAM,KAAK;AACvC,MAAI,CAAC,YAAY,KAAK,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,GAAG,GAAG;AAClE,WAAO;AAAA,EACT;AACA,SAAO,SAAS,QAAQ,QAAQ;AAClC;AAEO,gBAAS,cAAc,OAAO,OAAO;AAC1C,MAAI,MAAM,WAAW,GAAG;AACtB,WAAO;AAAA,EACT;AACA,MAAI,MAAM,CAAC,MAAM,OAAO,MAAM,CAAC,MAAM,KAAK;AACxC,WAAO,cAAc,MAAM,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,CAAC;AAAA,EACrD;AACA,MAAI,MAAM,CAAC,MAAM,MAAM,CAAC,GAAG;AACzB,WAAO;AAAA,EACT;AACA,MAAI,MAAM,CAAC,KAAK,OAAO,MAAM,CAAC,KAAK,KAAK;AACtC,WAAO,cAAc,MAAM,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,CAAC;AAAA,EACrD;AACA,SAAO;AACT;AAGA,MAAM,kBAAkB;AAExB,SAAS,YAAY,MAAM;AACzB,SAAO,gBAAgB,KAAK,IAAI;AAClC;",
   "names": []
 }

package/dist/match-detection/features/bib/title.js

@@ -2,45 +2,95 @@ import createDebugLogger from "debug";
 import naturalPkg from "natural";
 const { LevenshteinDistance: leven } = naturalPkg;
 import { testStringOrNumber } from "../../../matching-utils.js";
-const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features:title");
+const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features/bib/title");
 const debugData = debug.extend("data");
-export default ({ threshold = 10 } = {}) => ({
+export default ({ threshold = 0.9 } = {}) => ({
   name: "Title",
   extract: ({ record, recordExternal }) => {
     const label = recordExternal && recordExternal.label ? recordExternal.label : "record";
-    const title = getTitle();
-    debug(`${label}: title: ${title}`);
-    if (testStringOrNumber(title)) {
-      const titleAsNormalizedString = String(title).normalize("NFD").replace(/[^\p{Letter}\p{Number}]/gu, "").toLowerCase();
-      debug(`${label}: titleString: ${titleAsNormalizedString}`);
-      return [titleAsNormalizedString];
-    }
-    return [];
-    function getTitle() {
-      const [field] = record.get(/^245$/u);
-      debugData(`${label}: titleField: ${JSON.stringify(field)}`);
-      if (field) {
-        return field.subfields.filter(({ code }) => ["a", "b", "n", "p"].includes(code)).map(({ value }) => testStringOrNumber(value) ? String(value) : "").join("");
-      }
-      return false;
+    const a = getTitle(record, ["a"]);
+    debug(`${label}: title: ${a}`);
+    if (a) {
+      const b = normalizeTitle(getTitle(record, ["b"])) || "";
+      const n = normalizeTitle(getTitle(record, ["n"])) || "";
+      const p = normalizeTitle(getTitle(record, ["p"])) || "";
+      return [normalizeTitle(a), b, n, p];
+    }
+    return ["", "", "", ""];
+    function normalizeTitle(title) {
+      return title.normalize("NFD").replace(/[^\p{Letter}\p{Number}]/gu, "").toLowerCase();
     }
   },
   compare: (a, b) => {
-    const debug2 = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features/bib/title");
-    const distance = leven(a[0], b[0]);
+    const [aa, ab, an, ap] = a;
+    const [ba, bb, bn, bp] = b;
+    if (isEmpty(aa) || isEmpty(ba)) {
+      return -1;
+    }
+    if (an.length && bn.length && an !== bn) {
+      return -1;
+    }
+    const aFull = toFullTitle(a);
+    const bFull = toFullTitle(b);
+    const [distance, maxLength, correctness] = doLevenshtein(aFull, bFull);
+    debug(`'${aFull}' vs '${bFull}': Max length = ${maxLength}, distance = ${distance}, correctness = ${correctness}`);
     if (distance === 0) {
       return 0.5;
     }
-    const maxLength = getMaxLength();
-    const percentage = distance / maxLength * 100;
-    debug2(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);
-    if (percentage <= threshold) {
-      return 0.3;
+    if (correctness >= threshold) {
+      return 0.4;
+    }
+    if (an && bn) {
+      if (ab && isEmpty(ap) && bp && isEmpty(bb)) {
+        return compare([aa, ap, an, ab], [ba, bb, bn, bp]);
+      }
+      if (ap && isEmpty(ab) && bb && isEmpty(bp)) {
+        return compare([aa, ab, an, ap], [ba, bp, bn, bb]);
+      }
+    }
+    if (localXor(ap, bp)) {
+      const result = compare([aa, ab, an, ""], [ba, bb, bn, ""]);
+      return result > 0 ? result * 0.8 : result;
+    }
+    if (isEmpty(ap) && isEmpty(bp) && localXor(ab, bb)) {
+      const result = compare([aa, "", an, ""], [ba, "", bn, ""]);
+      return result > 0 ? result * 0.8 : result;
     }
     return -0.5;
-    function getMaxLength() {
-      return a[0].length > b[0].length ? a[0].length : b[0].length;
+    function isEmpty(x) {
+      return !x || x.length === 0;
+    }
+    function localXor(x, y) {
+      if (isEmpty(x)) {
+        return !isEmpty(y);
+      }
+      return isEmpty(y);
+    }
+    function doLevenshtein(string1, string2) {
+      const distance2 = leven(string1, string2);
+      const len = getMaxLength(string1, string2);
+      const correctness2 = 1 - distance2 / len;
+      return [distance2, len, correctness2];
+    }
+    function toFullTitle(arr) {
+      const relevant = arr.filter((val) => typeof val === "string" && val.length);
+      return relevant.join(" ");
+    }
+    function getMaxLength(str1, str2) {
+      return str1.length > str2.length ? str1.length : str2.length;
     }
   }
 });
+export function getTitle(record, targetSubfieldCodes = ["a", "b", "n", "p"]) {
+  const [field] = record.get(/^245$/u);
+  debugData(`titleField: ${JSON.stringify(field)}`);
+  if (field) {
+    const title = field.subfields.filter(({ code }) => targetSubfieldCodes.includes(code)).map(({ value }) => testStringOrNumber(value) ? String(value) : "").join(" ").replace(/\[[^\]]*\]/ug, " ").replace(/ [=\/:](?:$| )/ug, " ").replace(/ +/ug, " ").replace(/^ +/u, "").replace(/ +$/u, "");
+    if (/^[1-8]$/u.test(field.ind2)) {
+      return title.slice(parseInt(field.ind2));
+    }
+    return title;
+  }
+  return false;
+}
 //# sourceMappingURL=title.js.map

package/dist/match-detection/features/bib/title.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../../src/match-detection/features/bib/title.js"],
-  "sourcesContent": ["import createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');\nconst debugData = debug.extend('data');\n\n\nexport default ({threshold = 10} = {}) => ({\n  name: 'Title',\n  extract: ({record, recordExternal}) => {\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n    const title = getTitle();\n    debug(`${label}: title: ${title}`);\n\n    if (testStringOrNumber(title)) {\n      const titleAsNormalizedString = String(title)\n        // decompose unicode diacritics\n        .normalize('NFD')\n        // strip non-letters/numbers\n        // - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 's\u00E4\u00E4' as 'saa'\n        // - we could precompose the Finnish letters back to avoid this\n        // - see validator normalize-utf8-diacritics for details\n        .replace(/[^\\p{Letter}\\p{Number}]/gu, '')\n        .toLowerCase();\n      debug(`${label}: titleString: ${titleAsNormalizedString}`);\n      return [titleAsNormalizedString];\n    }\n\n    return [];\n\n    function getTitle() {\n      const [field] = record.get(/^245$/u);\n      debugData(`${label}: titleField: ${JSON.stringify(field)}`);\n\n      if (field) {\n        return field.subfields\n          // get also $n:s and $p:s here\n          .filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))\n          .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n          .join('');\n      }\n      return false;\n    }\n  },\n  compare: (a, b) => {\n    const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');\n    const distance = leven(a[0], b[0]);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    const maxLength = getMaxLength();\n    const percentage = distance / maxLength * 100;\n\n    debug(`'${a}' vs '${b}': Max length = ${maxLength}, distance = ${distance}, percentage = ${percentage}`);\n\n    if (percentage <= threshold) {\n      return 0.3;\n    }\n\n    return -0.5;\n\n    function getMaxLength() {\n      return a[0].length > b[0].length ? a[0].length : b[0].length;\n    }\n\n  }\n});\n"],
-  "mappings": "AAAA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AACrC,SAAQ,0BAAyB;AAEjC,MAAM,QAAQ,kBAAkB,kEAAkE;AAClG,MAAM,YAAY,MAAM,OAAO,MAAM;AAGrC,eAAe,CAAC,EAAC,YAAY,GAAE,IAAI,CAAC,OAAO;AAAA,EACzC,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,QAAQ,SAAS;AACvB,UAAM,GAAG,KAAK,YAAY,KAAK,EAAE;AAEjC,QAAI,mBAAmB,KAAK,GAAG;AAC7B,YAAM,0BAA0B,OAAO,KAAK,EAEzC,UAAU,KAAK,EAKf,QAAQ,6BAA6B,EAAE,EACvC,YAAY;AACf,YAAM,GAAG,KAAK,kBAAkB,uBAAuB,EAAE;AACzD,aAAO,CAAC,uBAAuB;AAAA,IACjC;AAEA,WAAO,CAAC;AAER,aAAS,WAAW;AAClB,YAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AACnC,gBAAU,GAAG,KAAK,iBAAiB,KAAK,UAAU,KAAK,CAAC,EAAE;AAE1D,UAAI,OAAO;AACT,eAAO,MAAM,UAEV,OAAO,CAAC,EAAC,KAAI,MAAM,CAAC,KAAK,KAAK,KAAK,GAAG,EAAE,SAAS,IAAI,CAAC,EACtD,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,KAAK,EAAE;AAAA,MACZ;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAMA,SAAQ,kBAAkB,sEAAsE;AACtG,UAAM,WAAW,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;AAEjC,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,aAAa;AAC/B,UAAM,aAAa,WAAW,YAAY;AAE1C,IAAAA,OAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,SAAS,gBAAgB,QAAQ,kBAAkB,UAAU,EAAE;AAEvG,QAAI,cAAc,WAAW;AAC3B,aAAO;AAAA,IACT;AAEA,WAAO;AAEP,aAAS,eAAe;AACtB,aAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;AAAA,IACxD;AAAA,EAEF;AACF;",
-  "names": ["debug"]
+  "sourcesContent": ["import createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');\nconst debugData = debug.extend('data');\n\nexport default ({threshold = 0.9} = {}) => ({\n  name: 'Title',\n  extract: ({record, recordExternal}) => {\n    const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n    const a = getTitle(record, ['a']);\n    debug(`${label}: title: ${a}`);\n\n    if (a) {\n      const b = normalizeTitle(getTitle(record, ['b'])) || '';\n      const n = normalizeTitle(getTitle(record, ['n'])) || '';\n      const p = normalizeTitle(getTitle(record, ['p'])) || '';\n      return [normalizeTitle(a), b, n, p];\n    }\n\n    return ['', '', '', ''];\n\n    function normalizeTitle(title) {\n      return title\n        // decompose unicode diacritics\n        .normalize('NFD')\n        // strip non-letters/numbers\n        // - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 's\u00E4\u00E4' as 'saa'\n        // - we could precompose the Finnish letters back to avoid this\n        // - see validator normalize-utf8-diacritics for details\n        .replace(/[^\\p{Letter}\\p{Number}]/gu, '')\n        .toLowerCase();\n    }\n\n  },\n  compare: (a, b) => {\n    const [aa, ab, an, ap] = a;\n    const [ba, bb, bn, bp] = b;\n\n    if (isEmpty(aa) || isEmpty(ba)) {\n      return -1.0;\n    }\n    // F245$n information is critical; it can not mismatch at all:\n    if (an.length && bn.length && an !== bn) { // If these exists, they must be the same (we might convert Roman numbers to Arabic numbers though)\n      return -1.0;\n    }\n\n    const aFull = toFullTitle(a);\n    const bFull = toFullTitle(b);\n\n    const [distance, maxLength, correctness] = doLevenshtein(aFull, bFull);\n\n    debug(`'${aFull}' vs '${bFull}': Max length = ${maxLength}, distance = ${distance}, correctness = ${correctness}`);\n\n    if (distance === 0) {\n      return 0.5;\n    }\n\n    if (correctness >= threshold) {\n      return 0.4;\n    }\n\n    if (an && bn) {\n      // There seems to be some wobble between $b and $p, for example:\n      if (ab && isEmpty(ap) && bp && isEmpty(bb)) {\n        return compare([aa, ap, an, ab], [ba, bb, bn, bp]);\n      }\n      if (ap && isEmpty(ab) && bb && isEmpty(bp)) {\n        return compare([aa, ab, an, ap], [ba, bp, bn, bb]);\n      }\n    }\n\n    // Try the same without $p:\n    if (localXor(ap, bp)) {\n      const result = compare([aa, ab, an, ''], [ba, bb, bn, '']);\n      return result > 0.0 ? result * 0.8 : result;\n    }\n\n    if (isEmpty(ap) && isEmpty(bp) && localXor(ab, bb)) {\n      // Try the same without $b ($p is not here)\n      const result = compare([aa, '', an, ''], [ba, '', bn, '']);\n      return result > 0.0 ? result * 0.8 : result;\n    }\n\n    return -0.5; // Not likely\n\n    function isEmpty(x) {\n      return !x || x.length === 0;\n    }\n\n    function localXor(x, y) {\n      if (isEmpty(x)) {\n        return !isEmpty(y);\n      }\n      // 'x' exists, thus 'y' can not exist:\n      return isEmpty(y);\n    }\n\n    function doLevenshtein(string1, string2) {\n      const distance = leven(string1, string2);\n      const len = getMaxLength(string1, string2);\n      const correctness = 1.0 - (distance / len);\n      return [distance, len, correctness];\n    }\n\n    function toFullTitle(arr) {\n      const relevant = arr.filter(val => typeof val === 'string' && val.length);\n      return relevant.join(' ');\n    }\n\n    function getMaxLength(str1, str2) {\n      return str1.length > str2.length ? str1.length : str2.length;\n    }\n\n  }\n});\n\nexport function getTitle(record, targetSubfieldCodes = ['a', 'b', 'n', 'p']) {\n  const [field] = record.get(/^245$/u);\n  debugData(`titleField: ${JSON.stringify(field)}`);\n\n  if (field) {\n    const title = field.subfields\n      // get also $n:s and $p:s here\n      .filter(({code}) => targetSubfieldCodes.includes(code))\n      // Would be nice to normalize $n values...\n      .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n      .join(' ')\n      .replace(/\\[[^\\]]*\\]/ug, ' ') // Remove [whatever] stuff. ADD TEST FOR THIS\n      .replace(/ [=\\/:](?:$| )/ug, ' ') // Strip punctuation\n      // Also \u0111 vs d pairs seen:\n      //.replace(/(?:\u0111|\u0227|t\u0304|k\u030C|\u01E7|s\u0306|c\u0306)/ug, '') // Hack. Saamelaisbibliografia has often dropped this wierd characters (oft old articles, no longer used in sami either)\n      // trim:\n      .replace(/ +/ug, ' ')\n      .replace(/^ +/u, '')\n      .replace(/ +$/u, '');\n\n    // Skip non-filing indicator (note that '9' is a magic indicator value, so we don't do it):\n    if (/^[1-8]$/u.test(field.ind2)) { // Skip non-filing characters\n      return title.slice(parseInt(field.ind2));\n    }\n    return title;\n\n  }\n  return false;\n}"],
+  "mappings": "AAAA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AACrC,SAAQ,0BAAyB;AAGjC,MAAM,QAAQ,kBAAkB,sEAAsE;AACtG,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,eAAe,CAAC,EAAC,YAAY,IAAG,IAAI,CAAC,OAAO;AAAA,EAC1C,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,IAAI,SAAS,QAAQ,CAAC,GAAG,CAAC;AAChC,UAAM,GAAG,KAAK,YAAY,CAAC,EAAE;AAE7B,QAAI,GAAG;AACL,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,aAAO,CAAC,eAAe,CAAC,GAAG,GAAG,GAAG,CAAC;AAAA,IACpC;AAEA,WAAO,CAAC,IAAI,IAAI,IAAI,EAAE;AAEtB,aAAS,eAAe,OAAO;AAC7B,aAAO,MAEJ,UAAU,KAAK,EAKf,QAAQ,6BAA6B,EAAE,EACvC,YAAY;AAAA,IACjB;AAAA,EAEF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI;AACzB,UAAM,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI;AAEzB,QAAI,QAAQ,EAAE,KAAK,QAAQ,EAAE,GAAG;AAC9B,aAAO;AAAA,IACT;AAEA,QAAI,GAAG,UAAU,GAAG,UAAU,OAAO,IAAI;AACvC,aAAO;AAAA,IACT;AAEA,UAAM,QAAQ,YAAY,CAAC;AAC3B,UAAM,QAAQ,YAAY,CAAC;AAE3B,UAAM,CAAC,UAAU,WAAW,WAAW,IAAI,cAAc,OAAO,KAAK;AAErE,UAAM,IAAI,KAAK,SAAS,KAAK,mBAAmB,SAAS,gBAAgB,QAAQ,mBAAmB,WAAW,EAAE;AAEjH,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,QAAI,eAAe,WAAW;AAC5B,aAAO;AAAA,IACT;AAEA,QAAI,MAAM,IAAI;AAEZ,UAAI,MAAM,QAAQ,EAAE,KAAK,MAAM,QAAQ,EAAE,GAAG;AAC1C,eAAO,QAAQ,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACnD;AACA,UAAI,MAAM,QAAQ,EAAE,KAAK,MAAM,QAAQ,EAAE,GAAG;AAC1C,eAAO,QAAQ,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACnD;AAAA,IACF;AAGA,QAAI,SAAS,IAAI,EAAE,GAAG;AACpB,YAAM,SAAS,QAAQ,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AACzD,aAAO,SAAS,IAAM,SAAS,MAAM;AAAA,IACvC;AAEA,QAAI,QAAQ,EAAE,KAAK,QAAQ,EAAE,KAAK,SAAS,IAAI,EAAE,GAAG;AAElD,YAAM,SAAS,QAAQ,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AACzD,aAAO,SAAS,IAAM,SAAS,MAAM;AAAA,IACvC;AAEA,WAAO;AAEP,aAAS,QAAQ,GAAG;AAClB,aAAO,CAAC,KAAK,EAAE,WAAW;AAAA,IAC5B;AAEA,aAAS,SAAS,GAAG,GAAG;AACtB,UAAI,QAAQ,CAAC,GAAG;AACd,eAAO,CAAC,QAAQ,CAAC;AAAA,MACnB;AAEA,aAAO,QAAQ,CAAC;AAAA,IAClB;AAEA,aAAS,cAAc,SAAS,SAAS;AACvC,YAAMA,YAAW,MAAM,SAAS,OAAO;AACvC,YAAM,MAAM,aAAa,SAAS,OAAO;AACzC,YAAMC,eAAc,IAAOD,YAAW;AACtC,aAAO,CAACA,WAAU,KAAKC,YAAW;AAAA,IACpC;AAEA,aAAS,YAAY,KAAK;AACxB,YAAM,WAAW,IAAI,OAAO,SAAO,OAAO,QAAQ,YAAY,IAAI,MAAM;AACxE,aAAO,SAAS,KAAK,GAAG;AAAA,IAC1B;AAEA,aAAS,aAAa,MAAM,MAAM;AAChC,aAAO,KAAK,SAAS,KAAK,SAAS,KAAK,SAAS,KAAK;AAAA,IACxD;AAAA,EAEF;AACF;AAEO,gBAAS,SAAS,QAAQ,sBAAsB,CAAC,KAAK,KAAK,KAAK,GAAG,GAAG;AAC3E,QAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AACnC,YAAU,eAAe,KAAK,UAAU,KAAK,CAAC,EAAE;AAEhD,MAAI,OAAO;AACT,UAAM,QAAQ,MAAM,UAEjB,OAAO,CAAC,EAAC,KAAI,MAAM,oBAAoB,SAAS,IAAI,CAAC,EAErD,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,KAAK,GAAG,EACR,QAAQ,gBAAgB,GAAG,EAC3B,QAAQ,oBAAoB,GAAG,EAI/B,QAAQ,QAAQ,GAAG,EACnB,QAAQ,QAAQ,EAAE,EAClB,QAAQ,QAAQ,EAAE;AAGrB,QAAI,WAAW,KAAK,MAAM,IAAI,GAAG;AAC/B,aAAO,MAAM,MAAM,SAAS,MAAM,IAAI,CAAC;AAAA,IACzC;AACA,WAAO;AAAA,EAET;AACA,SAAO;AACT;",
+  "names": ["distance", "correctness"]
 }

package/dist/match-detection/index.js

@@ -2,7 +2,8 @@ import createDebugLogger from "debug";
 import * as features from "./features/index.js";
 export { features };
 export default ({ strategy, threshold = 0.8999 }, returnStrategy = false, localSettings = {}) => ({ recordA, recordB, recordAExternal = {}, recordBExternal = {} }) => {
-  const minProbabilityQuantifier = 0.5;
+  const MIN_PROPABILITY_QUALIFIER = 0.4;
+  const FORCE_FAIL = -1;
   const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection");
   const debugData = debug.extend("data");
   debugData(`Strategy: ${JSON.stringify(strategy)}, Threshold: ${JSON.stringify(threshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);
@@ -24,12 +25,16 @@ B: ${recordB}`);
     debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);
     const featurePairs = generateFeaturePairs(featuresA2, featuresB);
     const similarityVector = generateSimilarityVector(featurePairs);
-    if (similarityVector.some((v) => v >= minProbabilityQuantifier)) {
+    if (similarityVector.some((v) => v >= MIN_PROPABILITY_QUALIFIER)) {
       const probability = calculateProbability(similarityVector);
       debug(`probability: ${probability} (Threshold: ${threshold})`);
+      if (similarityVector.some((v) => v <= FORCE_FAIL)) {
+        debug(`Some feature resulted in utter failure (${FORCE_FAIL} or less)`);
+        return returnResult({ match: false, probability: 0 });
+      }
       return returnResult({ match: probability >= threshold, probability });
     }
-    debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);
+    debugData(`No feature yielded minimum probability amount of points (${MIN_PROPABILITY_QUALIFIER})`);
     return returnResult({ match: false, probability: 0 });
   }
   function extractFeatures({ record, recordExternal }) {

package/dist/match-detection/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/match-detection/index.js"],
-  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport * as features from './features/index.js';\n\nexport {features};\n\nexport default ({strategy, threshold = 0.8999}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {\n  const minProbabilityQuantifier = 0.5;\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n  const debugData = debug.extend('data');\n\n  debugData(`Strategy: ${JSON.stringify(strategy)}, Threshold: ${JSON.stringify(threshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n  debugData(`Records: A: ${recordA}\\nB: ${recordB}`);\n  debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);\n  // We could add here labels for records if we didn't get external labels\n\n  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});\n\n  debug(`We got an array of records: ${Array.isArray(recordB)}`);\n  const recordsB = Array.isArray(recordB) ? recordB : [recordB];\n  const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];\n\n  const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));\n\n  // if we got array of records, we return an array of result\n  // if we got a singular record, we return a singular result\n  return Array.isArray(recordB) ? detectionResults : detectionResults[0];\n\n  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {\n    const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';\n    const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';\n\n\n    debug(`Actual detection for record ${index + 1} ${labelB}`);\n    const featuresB = extractFeatures({record, recordExternal, localSettings});\n\n    debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);\n    debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);\n\n    const featurePairs = generateFeaturePairs(featuresA, featuresB);\n    const similarityVector = generateSimilarityVector(featurePairs);\n\n    if (similarityVector.some(v => v >= minProbabilityQuantifier)) {\n      const probability = calculateProbability(similarityVector);\n      debug(`probability: ${probability} (Threshold: ${threshold})`);\n      return returnResult({match: probability >= threshold, probability});\n    }\n\n    debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);\n    return returnResult({match: false, probability: 0.0});\n  }\n\n  function extractFeatures({record, recordExternal}) {\n    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);\n  }\n\n  function returnResult(result) {\n    if (returnStrategy) {\n      debug(`Returning detection strategy with the result`);\n      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), threshold};\n      debugData(`${JSON.stringify(resultWithStrategy)}`);\n      return resultWithStrategy;\n    }\n    return result;\n  }\n\n  function formatStrategy(strategy) {\n    const strategyNames = strategy.map(element => element.name);\n    return strategyNames || [];\n  }\n\n  function calculateProbability(similarityVector) {\n    const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n    return probability > 1.0 ? 1.0 : probability;\n  }\n\n  function generateSimilarityVector(featurePairs) {\n    const compared = featurePairs.map(({name, a, b}) => {\n      const {compare} = strategy.find(({name: featureName}) => name === featureName);\n      const points = compare(a, b);\n      return {name, points};\n    });\n\n    debugData(`Points: ${JSON.stringify(compared)}`);\n    return compared.map(({points}) => points);\n  }\n\n  function generateFeaturePairs(featuresA, featuresB) {\n    const pairs = generatePairs();\n    const missingFeatures = findMissing();\n\n    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n    return pairs;\n\n    function generatePairs() {\n      return featuresA\n        .reduce((acc, {name, value}, index) => acc.concat({\n          name,\n          a: value,\n          b: featuresB[index].value\n        }), [])\n        .filter(({a, b}) => {\n          if (a.length === 0 || b.length === 0) {\n            return false;\n          }\n\n          return true;\n        });\n    }\n\n    function findMissing() {\n      return featuresA\n        .map(({name}) => name)\n        .filter(v => pairs.some(({name}) => name === v) === false);\n    }\n  }\n\n};\n"],
-  "mappings": "AACA,OAAO,uBAAuB;AAC9B,YAAY,cAAc;AAE1B,SAAQ;AAER,eAAe,CAAC,EAAC,UAAU,YAAY,OAAM,GAAG,iBAAiB,OAAO,gBAAgB,CAAC,MAAM,CAAC,EAAC,SAAS,SAAS,kBAAkB,CAAC,GAAG,kBAAkB,CAAC,EAAC,MAAM;AACjK,QAAM,2BAA2B;AAEjC,QAAM,QAAQ,kBAAkB,mDAAmD;AACnF,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,YAAU,aAAa,KAAK,UAAU,QAAQ,CAAC,gBAAgB,KAAK,UAAU,SAAS,CAAC,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAC7I,YAAU,eAAe,OAAO;AAAA,KAAQ,OAAO,EAAE;AACjD,QAAM,iBAAiB,KAAK,UAAU,eAAe,CAAC,QAAQ,KAAK,UAAU,eAAe,CAAC,EAAE;AAG/F,QAAM,YAAY,gBAAgB,EAAC,QAAQ,SAAS,gBAAgB,iBAAiB,cAAa,CAAC;AAEnG,QAAM,+BAA+B,MAAM,QAAQ,OAAO,CAAC,EAAE;AAC7D,QAAM,WAAW,MAAM,QAAQ,OAAO,IAAI,UAAU,CAAC,OAAO;AAC5D,QAAM,mBAAmB,MAAM,QAAQ,OAAO,IAAI,kBAAkB,CAAC,eAAe;AAEpF,QAAM,mBAAmB,SAAS,IAAI,CAAC,QAAQ,UAAU,gBAAgB,EAAC,WAAW,iBAAiB,QAAQ,gBAAgB,iBAAiB,KAAK,GAAG,MAAK,CAAC,CAAC;AAI9J,SAAO,MAAM,QAAQ,OAAO,IAAI,mBAAmB,iBAAiB,CAAC;AAErE,WAAS,gBAAgB,EAAC,WAAAA,YAAW,QAAQ,gBAAgB,OAAO,eAAAC,eAAa,GAAG;AAClF,UAAM,SAAS,mBAAmB,gBAAgB,QAAQ,gBAAgB,QAAQ;AAClF,UAAM,SAAS,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAG/E,UAAM,+BAA+B,QAAQ,CAAC,IAAI,MAAM,EAAE;AAC1D,UAAM,YAAY,gBAAgB,EAAC,QAAQ,gBAAgB,eAAAA,eAAa,CAAC;AAEzE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAUD,UAAS,CAAC,EAAE;AACjE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAU,SAAS,CAAC,EAAE;AAEjE,UAAM,eAAe,qBAAqBA,YAAW,SAAS;AAC9D,UAAM,mBAAmB,yBAAyB,YAAY;AAE9D,QAAI,iBAAiB,KAAK,OAAK,KAAK,wBAAwB,GAAG;AAC7D,YAAM,cAAc,qBAAqB,gBAAgB;AACzD,YAAM,gBAAgB,WAAW,gBAAgB,SAAS,GAAG;AAC7D,aAAO,aAAa,EAAC,OAAO,eAAe,WAAW,YAAW,CAAC;AAAA,IACpE;AAEA,cAAU,4DAA4D,wBAAwB,GAAG;AACjG,WAAO,aAAa,EAAC,OAAO,OAAO,aAAa,EAAG,CAAC;AAAA,EACtD;AAEA,WAAS,gBAAgB,EAAC,QAAQ,eAAc,GAAG;AACjD,WAAO,SAAS,OAAO,CAAC,KAAK,EAAC,MAAM,QAAO,MAAM,IAAI,OAAO,EAAC,MAAM,OAAO,QAAQ,EAAC,QAAQ,eAAc,CAAC,EAAC,CAAC,GAAG,CAAC,CAAC;AAAA,EACnH;AAEA,WAAS,aAAa,QAAQ;AAC5B,QAAI,gBAAgB;AAClB,YAAM,8CAA8C;AACpD,YAAM,qBAAqB,EAAC,OAAO,OAAO,OAAO,aAAa,OAAO,aAAa,UAAU,eAAe,QAAQ,GAAG,UAAS;AAC/H,gBAAU,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AACjD,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAEA,WAAS,eAAeE,WAAU;AAChC,UAAM,gBAAgBA,UAAS,IAAI,aAAW,QAAQ,IAAI;AAC1D,WAAO,iBAAiB,CAAC;AAAA,EAC3B;AAEA,WAAS,qBAAqB,kBAAkB;AAC9C,UAAM,cAAc,iBAAiB,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAG;AACpE,WAAO,cAAc,IAAM,IAAM;AAAA,EACnC;AAEA,WAAS,yBAAyB,cAAc;AAC9C,UAAM,WAAW,aAAa,IAAI,CAAC,EAAC,MAAM,GAAG,EAAC,MAAM;AAClD,YAAM,EAAC,QAAO,IAAI,SAAS,KAAK,CAAC,EAAC,MAAM,YAAW,MAAM,SAAS,WAAW;AAC7E,YAAM,SAAS,QAAQ,GAAG,CAAC;AAC3B,aAAO,EAAC,MAAM,OAAM;AAAA,IACtB,CAAC;AAED,cAAU,WAAW,KAAK,UAAU,QAAQ,CAAC,EAAE;AAC/C,WAAO,SAAS,IAAI,CAAC,EAAC,OAAM,MAAM,MAAM;AAAA,EAC1C;AAEA,WAAS,qBAAqBF,YAAW,WAAW;AAClD,UAAM,QAAQ,cAAc;AAC5B,UAAM,kBAAkB,YAAY;AAEpC,UAAM,2FAA2F,KAAK,UAAU,eAAe,CAAC,EAAE;AAClI,WAAO;AAEP,aAAS,gBAAgB;AACvB,aAAOA,WACJ,OAAO,CAAC,KAAK,EAAC,MAAM,MAAK,GAAG,UAAU,IAAI,OAAO;AAAA,QAChD;AAAA,QACA,GAAG;AAAA,QACH,GAAG,UAAU,KAAK,EAAE;AAAA,MACtB,CAAC,GAAG,CAAC,CAAC,EACL,OAAO,CAAC,EAAC,GAAG,EAAC,MAAM;AAClB,YAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,iBAAO;AAAA,QACT;AAEA,eAAO;AAAA,MACT,CAAC;AAAA,IACL;AAEA,aAAS,cAAc;AACrB,aAAOA,WACJ,IAAI,CAAC,EAAC,KAAI,MAAM,IAAI,EACpB,OAAO,OAAK,MAAM,KAAK,CAAC,EAAC,KAAI,MAAM,SAAS,CAAC,MAAM,KAAK;AAAA,IAC7D;AAAA,EACF;AAEF;",
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport * as features from './features/index.js';\n\nexport {features};\n\nexport default ({strategy, threshold = 0.8999}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {\n  const MIN_PROPABILITY_QUALIFIER = 0.4; // Title perfect match: 0.5\n  const FORCE_FAIL = -1.0;\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n  const debugData = debug.extend('data');\n\n  debugData(`Strategy: ${JSON.stringify(strategy)}, Threshold: ${JSON.stringify(threshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n  debugData(`Records: A: ${recordA}\\nB: ${recordB}`);\n  debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);\n  // We could add here labels for records if we didn't get external labels\n\n  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});\n\n  debug(`We got an array of records: ${Array.isArray(recordB)}`);\n  const recordsB = Array.isArray(recordB) ? recordB : [recordB];\n  const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];\n\n  const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));\n\n  // if we got array of records, we return an array of result\n  // if we got a singular record, we return a singular result\n  return Array.isArray(recordB) ? detectionResults : detectionResults[0];\n\n  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {\n    const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';\n    const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';\n\n\n    debug(`Actual detection for record ${index + 1} ${labelB}`);\n    const featuresB = extractFeatures({record, recordExternal, localSettings});\n\n    debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);\n    debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);\n\n    const featurePairs = generateFeaturePairs(featuresA, featuresB);\n    const similarityVector = generateSimilarityVector(featurePairs);\n\n    if (similarityVector.some(v => v >= MIN_PROPABILITY_QUALIFIER)) {\n      const probability = calculateProbability(similarityVector);\n      debug(`probability: ${probability} (Threshold: ${threshold})`);\n      if (similarityVector.some(v => v <= FORCE_FAIL)) {\n        debug(`Some feature resulted in utter failure (${FORCE_FAIL} or less)`);\n        return returnResult({match: false, probability: 0.0});\n      }\n\n      return returnResult({match: probability >= threshold, probability});\n    }\n\n    debugData(`No feature yielded minimum probability amount of points (${MIN_PROPABILITY_QUALIFIER})`);\n    return returnResult({match: false, probability: 0.0});\n  }\n\n  function extractFeatures({record, recordExternal}) {\n    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);\n  }\n\n  function returnResult(result) {\n    if (returnStrategy) {\n      debug(`Returning detection strategy with the result`);\n      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), threshold};\n      debugData(`${JSON.stringify(resultWithStrategy)}`);\n      return resultWithStrategy;\n    }\n    return result;\n  }\n\n  function formatStrategy(strategy) {\n    const strategyNames = strategy.map(element => element.name);\n    return strategyNames || [];\n  }\n\n  function calculateProbability(similarityVector) {\n    const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n    return probability > 1.0 ? 1.0 : probability;\n  }\n\n  function generateSimilarityVector(featurePairs) {\n    const compared = featurePairs.map(({name, a, b}) => {\n      const {compare} = strategy.find(({name: featureName}) => name === featureName);\n      const points = compare(a, b);\n      return {name, points};\n    });\n\n    debugData(`Points: ${JSON.stringify(compared)}`);\n    return compared.map(({points}) => points);\n  }\n\n  function generateFeaturePairs(featuresA, featuresB) {\n    const pairs = generatePairs();\n    const missingFeatures = findMissing();\n\n    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n    return pairs;\n\n    function generatePairs() {\n      return featuresA\n        .reduce((acc, {name, value}, index) => acc.concat({\n          name,\n          a: value,\n          b: featuresB[index].value\n        }), [])\n        .filter(({a, b}) => {\n          if (a.length === 0 || b.length === 0) {\n            return false;\n          }\n\n          return true;\n        });\n    }\n\n    function findMissing() {\n      return featuresA\n        .map(({name}) => name)\n        .filter(v => pairs.some(({name}) => name === v) === false);\n    }\n  }\n\n};\n"],
+  "mappings": "AACA,OAAO,uBAAuB;AAC9B,YAAY,cAAc;AAE1B,SAAQ;AAER,eAAe,CAAC,EAAC,UAAU,YAAY,OAAM,GAAG,iBAAiB,OAAO,gBAAgB,CAAC,MAAM,CAAC,EAAC,SAAS,SAAS,kBAAkB,CAAC,GAAG,kBAAkB,CAAC,EAAC,MAAM;AACjK,QAAM,4BAA4B;AAClC,QAAM,aAAa;AAEnB,QAAM,QAAQ,kBAAkB,mDAAmD;AACnF,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,YAAU,aAAa,KAAK,UAAU,QAAQ,CAAC,gBAAgB,KAAK,UAAU,SAAS,CAAC,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAC7I,YAAU,eAAe,OAAO;AAAA,KAAQ,OAAO,EAAE;AACjD,QAAM,iBAAiB,KAAK,UAAU,eAAe,CAAC,QAAQ,KAAK,UAAU,eAAe,CAAC,EAAE;AAG/F,QAAM,YAAY,gBAAgB,EAAC,QAAQ,SAAS,gBAAgB,iBAAiB,cAAa,CAAC;AAEnG,QAAM,+BAA+B,MAAM,QAAQ,OAAO,CAAC,EAAE;AAC7D,QAAM,WAAW,MAAM,QAAQ,OAAO,IAAI,UAAU,CAAC,OAAO;AAC5D,QAAM,mBAAmB,MAAM,QAAQ,OAAO,IAAI,kBAAkB,CAAC,eAAe;AAEpF,QAAM,mBAAmB,SAAS,IAAI,CAAC,QAAQ,UAAU,gBAAgB,EAAC,WAAW,iBAAiB,QAAQ,gBAAgB,iBAAiB,KAAK,GAAG,MAAK,CAAC,CAAC;AAI9J,SAAO,MAAM,QAAQ,OAAO,IAAI,mBAAmB,iBAAiB,CAAC;AAErE,WAAS,gBAAgB,EAAC,WAAAA,YAAW,QAAQ,gBAAgB,OAAO,eAAAC,eAAa,GAAG;AAClF,UAAM,SAAS,mBAAmB,gBAAgB,QAAQ,gBAAgB,QAAQ;AAClF,UAAM,SAAS,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAG/E,UAAM,+BAA+B,QAAQ,CAAC,IAAI,MAAM,EAAE;AAC1D,UAAM,YAAY,gBAAgB,EAAC,QAAQ,gBAAgB,eAAAA,eAAa,CAAC;AAEzE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAUD,UAAS,CAAC,EAAE;AACjE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAU,SAAS,CAAC,EAAE;AAEjE,UAAM,eAAe,qBAAqBA,YAAW,SAAS;AAC9D,UAAM,mBAAmB,yBAAyB,YAAY;AAE9D,QAAI,iBAAiB,KAAK,OAAK,KAAK,yBAAyB,GAAG;AAC9D,YAAM,cAAc,qBAAqB,gBAAgB;AACzD,YAAM,gBAAgB,WAAW,gBAAgB,SAAS,GAAG;AAC7D,UAAI,iBAAiB,KAAK,OAAK,KAAK,UAAU,GAAG;AAC/C,cAAM,2CAA2C,UAAU,WAAW;AACtE,eAAO,aAAa,EAAC,OAAO,OAAO,aAAa,EAAG,CAAC;AAAA,MACtD;AAEA,aAAO,aAAa,EAAC,OAAO,eAAe,WAAW,YAAW,CAAC;AAAA,IACpE;AAEA,cAAU,4DAA4D,yBAAyB,GAAG;AAClG,WAAO,aAAa,EAAC,OAAO,OAAO,aAAa,EAAG,CAAC;AAAA,EACtD;AAEA,WAAS,gBAAgB,EAAC,QAAQ,eAAc,GAAG;AACjD,WAAO,SAAS,OAAO,CAAC,KAAK,EAAC,MAAM,QAAO,MAAM,IAAI,OAAO,EAAC,MAAM,OAAO,QAAQ,EAAC,QAAQ,eAAc,CAAC,EAAC,CAAC,GAAG,CAAC,CAAC;AAAA,EACnH;AAEA,WAAS,aAAa,QAAQ;AAC5B,QAAI,gBAAgB;AAClB,YAAM,8CAA8C;AACpD,YAAM,qBAAqB,EAAC,OAAO,OAAO,OAAO,aAAa,OAAO,aAAa,UAAU,eAAe,QAAQ,GAAG,UAAS;AAC/H,gBAAU,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AACjD,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAEA,WAAS,eAAeE,WAAU;AAChC,UAAM,gBAAgBA,UAAS,IAAI,aAAW,QAAQ,IAAI;AAC1D,WAAO,iBAAiB,CAAC;AAAA,EAC3B;AAEA,WAAS,qBAAqB,kBAAkB;AAC9C,UAAM,cAAc,iBAAiB,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAG;AACpE,WAAO,cAAc,IAAM,IAAM;AAAA,EACnC;AAEA,WAAS,yBAAyB,cAAc;AAC9C,UAAM,WAAW,aAAa,IAAI,CAAC,EAAC,MAAM,GAAG,EAAC,MAAM;AAClD,YAAM,EAAC,QAAO,IAAI,SAAS,KAAK,CAAC,EAAC,MAAM,YAAW,MAAM,SAAS,WAAW;AAC7E,YAAM,SAAS,QAAQ,GAAG,CAAC;AAC3B,aAAO,EAAC,MAAM,OAAM;AAAA,IACtB,CAAC;AAED,cAAU,WAAW,KAAK,UAAU,QAAQ,CAAC,EAAE;AAC/C,WAAO,SAAS,IAAI,CAAC,EAAC,OAAM,MAAM,MAAM;AAAA,EAC1C;AAEA,WAAS,qBAAqBF,YAAW,WAAW;AAClD,UAAM,QAAQ,cAAc;AAC5B,UAAM,kBAAkB,YAAY;AAEpC,UAAM,2FAA2F,KAAK,UAAU,eAAe,CAAC,EAAE;AAClI,WAAO;AAEP,aAAS,gBAAgB;AACvB,aAAOA,WACJ,OAAO,CAAC,KAAK,EAAC,MAAM,MAAK,GAAG,UAAU,IAAI,OAAO;AAAA,QAChD;AAAA,QACA,GAAG;AAAA,QACH,GAAG,UAAU,KAAK,EAAE;AAAA,MACtB,CAAC,GAAG,CAAC,CAAC,EACL,OAAO,CAAC,EAAC,GAAG,EAAC,MAAM;AAClB,YAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,iBAAO;AAAA,QACT;AAEA,eAAO;AAAA,MACT,CAAC;AAAA,IACL;AAEA,aAAS,cAAc;AACrB,aAAOA,WACJ,IAAI,CAAC,EAAC,KAAI,MAAM,IAAI,EACpB,OAAO,OAAK,MAAM,KAAK,CAAC,EAAC,KAAI,MAAM,SAAS,CAAC,MAAM,KAAK;AAAA,IAC7D;AAAA,EACF;AAEF;",
   "names": ["featuresA", "localSettings", "strategy"]
 }

package/dist/matching-utils.js

@@ -76,4 +76,18 @@ export function getMatchCounts(aValues, bValues) {
     return aToB < bToA ? aToB : bToA;
   }
 }
+export function stringAfter(string, substring) {
+  const pos = string.indexOf(substring);
+  if (pos === -1) {
+    return string;
+  }
+  return string.substring(pos + substring.length);
+}
+export function stringBefore(string, substring) {
+  const pos = string.indexOf(substring);
+  if (pos === -1) {
+    return string;
+  }
+  return string.substring(0, pos);
+}
 //# sourceMappingURL=matching-utils.js.map

package/dist/matching-utils.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/matching-utils.js"],
-  "sourcesContent": ["import createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\nexport function getMelindaIdsF035(record) {\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n  const debugData = debug.extend('data');\n\n  const f035s = record.getFields('035');\n\n  if (f035s.length < 1) {\n    debug(`No f035s found.`);\n    return [];\n  }\n\n  const allF035MelindaIds = [].concat(...f035s.map(field => toMelindaIds(field)));\n  const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n  debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n  debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n  debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n  return f035MelindaIds;\n}\n\nexport function toMelindaIds({subfields}, subfieldsToParse = ['a', 'z']) {\n  const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n  return subfields\n    .filter(sub => subfieldsToParse.includes(sub.code))\n    .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n    .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n  // Valid SID-fields have just one $c and one $b\n  debugData(`Validating SID field ${JSON.stringify(field)}`);\n  const countC = countSubfields(field, 'c');\n  const countB = countSubfields(field, 'b');\n  debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n  return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n  // debug(`Counting subfields ${subfieldCode}`);\n  return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n  return field.subfields\n    .filter(({code}) => code === subfieldCode)\n    .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n    .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n  if (typeof value === 'string' || typeof value === 'number') {\n    return true;\n  }\n  return false;\n}\n\nexport function extractSubfieldsFromField(field, subfieldCodes) {\n  if (field === undefined || field.subfields === undefined) {\n    return [];\n  }\n  const resultSubfields = field.subfields\n    .filter(({code}) => subfieldCodes.includes(code))\n    .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))\n    .filter(value => value);\n  return resultSubfields;\n}\n\nexport function uniqueSubfields(subfields) {\n  return subfields.reduce((arr, e) => {\n    if (!arr.find(item => item.code === e.code && item.value === e.value)) {\n      const newArr = arr.concat(e);\n      return newArr;\n    }\n    return arr;\n  }, []);\n}\n\nexport function getMatchCounts(aValues, bValues) {\n\n  const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n  return {\n    maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n    // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n    possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n    matchingValues\n  };\n\n  function getMatchingValuesAmount(aValues, bValues) {\n    if (bValues.length > aValues.length) {\n      return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n    }\n    if (aValues.length > bValues.length) {\n      return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n    }\n\n    // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n    // there would be duplicate values\n    const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n    const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n    return aToB < bToA ? aToB : bToA;\n  }\n}\n"],
-  "mappings": "AAAA,OAAO,uBAAuB;AAE9B,MAAM,QAAQ,kBAAkB,yCAAyC;AACzE,MAAM,YAAY,MAAM,OAAO,MAAM;AAE9B,gBAAS,kBAAkB,QAAQ;AAExC,QAAMA,SAAQ,kBAAkB,8CAA8C;AAC9E,QAAMC,aAAYD,OAAM,OAAO,MAAM;AAErC,QAAM,QAAQ,OAAO,UAAU,KAAK;AAEpC,MAAI,MAAM,SAAS,GAAG;AACpB,IAAAA,OAAM,iBAAiB;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,CAAC,EAAE,OAAO,GAAG,MAAM,IAAI,WAAS,aAAa,KAAK,CAAC,CAAC;AAC9E,QAAM,iBAAiB,CAAC,GAAG,IAAI,IAAI,iBAAiB,CAAC;AAErD,EAAAC,WAAU,WAAW,MAAM,MAAM,MAAM,KAAK,UAAU,KAAK,CAAC,EAAE;AAC9D,EAAAA,WAAU,QAAQ,kBAAkB,MAAM,MAAM,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACnF,EAAAA,WAAU,eAAe,eAAe,MAAM,MAAM,KAAK,UAAU,cAAc,CAAC,EAAE;AAEpF,SAAO;AACT;AAEO,gBAAS,aAAa,EAAC,UAAS,GAAG,mBAAmB,CAAC,KAAK,GAAG,GAAG;AACvE,QAAM,kBAAkB;AAExB,SAAO,UACJ,OAAO,SAAO,iBAAiB,SAAS,IAAI,IAAI,CAAC,EACjD,OAAO,SAAO,mBAAmB,IAAI,KAAK,KAAK,gBAAgB,KAAK,OAAO,IAAI,KAAK,CAAC,CAAC,EACtF,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,EAAE,QAAQ,iBAAiB,OAAO,IAAI,EAAE;AACtG;AAEO,gBAAS,+BAA+B,OAAO;AAEpD,YAAU,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AACzD,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,MAAM,eAAe,MAAM,qBAAqB,WAAW,KAAK,WAAW,CAAC,EAAE;AAE7F,SAAO,WAAW,KAAK,WAAW;AACpC;AAEA,SAAS,eAAe,OAAO,cAAc;AAE3C,SAAO,MAAM,UAAU,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EAAE;AACnE;AAEO,gBAAS,kBAAkB,OAAO,cAAc;AACrD,YAAU,oBAAoB,YAAY,SAAS,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E,SAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EACxC,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,OAAO,WAAS,KAAK;AAC1B;AAEO,gBAAS,mBAAmB,OAAO;AACxC,MAAI,OAAO,UAAU,YAAY,OAAO,UAAU,UAAU;AAC1D,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEO,gBAAS,0BAA0B,OAAO,eAAe;AAC9D,MAAI,UAAU,UAAa,MAAM,cAAc,QAAW;AACxD,WAAO,CAAC;AAAA,EACV;AACA,QAAM,kBAAkB,MAAM,UAC3B,OAAO,CAAC,EAAC,KAAI,MAAM,cAAc,SAAS,IAAI,CAAC,EAC/C,IAAI,CAAC,EAAC,MAAM,MAAK,OAAO,EAAC,MAAM,OAAO,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,GAAE,EAAE,EACtF,OAAO,WAAS,KAAK;AACxB,SAAO;AACT;AAEO,gBAAS,gBAAgB,WAAW;AACzC,SAAO,UAAU,OAAO,CAAC,KAAK,MAAM;AAClC,QAAI,CAAC,IAAI,KAAK,UAAQ,KAAK,SAAS,EAAE,QAAQ,KAAK,UAAU,EAAE,KAAK,GAAG;AACrE,YAAM,SAAS,IAAI,OAAO,CAAC;AAC3B,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,GAAG,CAAC,CAAC;AACP;AAEO,gBAAS,eAAe,SAAS,SAAS;AAE/C,QAAM,iBAAiB,wBAAwB,SAAS,OAAO;AAE/D,SAAO;AAAA,IACL,WAAW,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA;AAAA,IAEtE,qBAAqB,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA,IAChF;AAAA,EACF;AAEA,WAAS,wBAAwBC,UAASC,UAAS;AACjD,QAAIA,SAAQ,SAASD,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AACA,QAAID,SAAQ,SAASC,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AAIA,UAAM,OAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AACjF,UAAM,OAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAEjF,WAAO,OAAO,OAAO,OAAO;AAAA,EAC9B;AACF;",
+  "sourcesContent": ["import createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\nexport function getMelindaIdsF035(record) {\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n  const debugData = debug.extend('data');\n\n  const f035s = record.getFields('035');\n\n  if (f035s.length < 1) {\n    debug(`No f035s found.`);\n    return [];\n  }\n\n  const allF035MelindaIds = [].concat(...f035s.map(field => toMelindaIds(field)));\n  const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n  debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n  debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n  debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n  return f035MelindaIds;\n}\n\nexport function toMelindaIds({subfields}, subfieldsToParse = ['a', 'z']) {\n  const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n  return subfields\n    .filter(sub => subfieldsToParse.includes(sub.code))\n    .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n    .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n  // Valid SID-fields have just one $c and one $b\n  debugData(`Validating SID field ${JSON.stringify(field)}`);\n  const countC = countSubfields(field, 'c');\n  const countB = countSubfields(field, 'b');\n  debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n  return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n  // debug(`Counting subfields ${subfieldCode}`);\n  return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n  return field.subfields\n    .filter(({code}) => code === subfieldCode)\n    .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n    .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n  if (typeof value === 'string' || typeof value === 'number') {\n    return true;\n  }\n  return false;\n}\n\nexport function extractSubfieldsFromField(field, subfieldCodes) {\n  if (field === undefined || field.subfields === undefined) {\n    return [];\n  }\n  const resultSubfields = field.subfields\n    .filter(({code}) => subfieldCodes.includes(code))\n    .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))\n    .filter(value => value);\n  return resultSubfields;\n}\n\nexport function uniqueSubfields(subfields) {\n  return subfields.reduce((arr, e) => {\n    if (!arr.find(item => item.code === e.code && item.value === e.value)) {\n      const newArr = arr.concat(e);\n      return newArr;\n    }\n    return arr;\n  }, []);\n}\n\nexport function getMatchCounts(aValues, bValues) {\n\n  const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n  return {\n    maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n    // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n    possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n    matchingValues\n  };\n\n  function getMatchingValuesAmount(aValues, bValues) {\n    if (bValues.length > aValues.length) {\n      return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n    }\n    if (aValues.length > bValues.length) {\n      return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n    }\n\n    // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n    // there would be duplicate values\n    const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n    const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n    return aToB < bToA ? aToB : bToA;\n  }\n}\n\n\nexport function stringAfter(string, substring) {\n  const pos = string.indexOf(substring);\n  if (pos === -1) {\n    return string;\n  }\n  return string.substring(pos+substring.length);\n}\n\nexport function stringBefore(string, substring) {\n  const pos = string.indexOf(substring);\n  if (pos === -1) {\n    return string;\n  }\n  return string.substring(0, pos);\n}\n"],
+  "mappings": "AAAA,OAAO,uBAAuB;AAE9B,MAAM,QAAQ,kBAAkB,yCAAyC;AACzE,MAAM,YAAY,MAAM,OAAO,MAAM;AAE9B,gBAAS,kBAAkB,QAAQ;AAExC,QAAMA,SAAQ,kBAAkB,8CAA8C;AAC9E,QAAMC,aAAYD,OAAM,OAAO,MAAM;AAErC,QAAM,QAAQ,OAAO,UAAU,KAAK;AAEpC,MAAI,MAAM,SAAS,GAAG;AACpB,IAAAA,OAAM,iBAAiB;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,CAAC,EAAE,OAAO,GAAG,MAAM,IAAI,WAAS,aAAa,KAAK,CAAC,CAAC;AAC9E,QAAM,iBAAiB,CAAC,GAAG,IAAI,IAAI,iBAAiB,CAAC;AAErD,EAAAC,WAAU,WAAW,MAAM,MAAM,MAAM,KAAK,UAAU,KAAK,CAAC,EAAE;AAC9D,EAAAA,WAAU,QAAQ,kBAAkB,MAAM,MAAM,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACnF,EAAAA,WAAU,eAAe,eAAe,MAAM,MAAM,KAAK,UAAU,cAAc,CAAC,EAAE;AAEpF,SAAO;AACT;AAEO,gBAAS,aAAa,EAAC,UAAS,GAAG,mBAAmB,CAAC,KAAK,GAAG,GAAG;AACvE,QAAM,kBAAkB;AAExB,SAAO,UACJ,OAAO,SAAO,iBAAiB,SAAS,IAAI,IAAI,CAAC,EACjD,OAAO,SAAO,mBAAmB,IAAI,KAAK,KAAK,gBAAgB,KAAK,OAAO,IAAI,KAAK,CAAC,CAAC,EACtF,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,EAAE,QAAQ,iBAAiB,OAAO,IAAI,EAAE;AACtG;AAEO,gBAAS,+BAA+B,OAAO;AAEpD,YAAU,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AACzD,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,MAAM,eAAe,MAAM,qBAAqB,WAAW,KAAK,WAAW,CAAC,EAAE;AAE7F,SAAO,WAAW,KAAK,WAAW;AACpC;AAEA,SAAS,eAAe,OAAO,cAAc;AAE3C,SAAO,MAAM,UAAU,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EAAE;AACnE;AAEO,gBAAS,kBAAkB,OAAO,cAAc;AACrD,YAAU,oBAAoB,YAAY,SAAS,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E,SAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EACxC,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,OAAO,WAAS,KAAK;AAC1B;AAEO,gBAAS,mBAAmB,OAAO;AACxC,MAAI,OAAO,UAAU,YAAY,OAAO,UAAU,UAAU;AAC1D,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEO,gBAAS,0BAA0B,OAAO,eAAe;AAC9D,MAAI,UAAU,UAAa,MAAM,cAAc,QAAW;AACxD,WAAO,CAAC;AAAA,EACV;AACA,QAAM,kBAAkB,MAAM,UAC3B,OAAO,CAAC,EAAC,KAAI,MAAM,cAAc,SAAS,IAAI,CAAC,EAC/C,IAAI,CAAC,EAAC,MAAM,MAAK,OAAO,EAAC,MAAM,OAAO,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,GAAE,EAAE,EACtF,OAAO,WAAS,KAAK;AACxB,SAAO;AACT;AAEO,gBAAS,gBAAgB,WAAW;AACzC,SAAO,UAAU,OAAO,CAAC,KAAK,MAAM;AAClC,QAAI,CAAC,IAAI,KAAK,UAAQ,KAAK,SAAS,EAAE,QAAQ,KAAK,UAAU,EAAE,KAAK,GAAG;AACrE,YAAM,SAAS,IAAI,OAAO,CAAC;AAC3B,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,GAAG,CAAC,CAAC;AACP;AAEO,gBAAS,eAAe,SAAS,SAAS;AAE/C,QAAM,iBAAiB,wBAAwB,SAAS,OAAO;AAE/D,SAAO;AAAA,IACL,WAAW,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA;AAAA,IAEtE,qBAAqB,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA,IAChF;AAAA,EACF;AAEA,WAAS,wBAAwBC,UAASC,UAAS;AACjD,QAAIA,SAAQ,SAASD,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AACA,QAAID,SAAQ,SAASC,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AAIA,UAAM,OAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AACjF,UAAM,OAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAEjF,WAAO,OAAO,OAAO,OAAO;AAAA,EAC9B;AACF;AAGO,gBAAS,YAAY,QAAQ,WAAW;AAC7C,QAAM,MAAM,OAAO,QAAQ,SAAS;AACpC,MAAI,QAAQ,IAAI;AACd,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,MAAI,UAAU,MAAM;AAC9C;AAEO,gBAAS,aAAa,QAAQ,WAAW;AAC9C,QAAM,MAAM,OAAO,QAAQ,SAAS;AACpC,MAAI,QAAQ,IAAI;AACd,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,GAAG,GAAG;AAChC;",
   "names": ["debug", "debugData", "aValues", "bValues"]
 }