@natlibfi/melinda-record-matching 5.1.4 → 5.1.6-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/candidate-search/index.js +2 -0
- package/dist/candidate-search/index.js.map +2 -2
- package/dist/candidate-search/query-list/bib.js +16 -3
- package/dist/candidate-search/query-list/bib.js.map +2 -2
- package/dist/cli.js.map +2 -2
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +7 -0
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +2 -2
- package/dist/match-detection/features/bib/title.js +51 -5
- package/dist/match-detection/features/bib/title.js.map +3 -3
- package/dist/matching-utils.js +39 -0
- package/dist/matching-utils.js.map +2 -2
- package/package.json +9 -11
- package/src/candidate-search/index.js +3 -0
- package/src/candidate-search/query-list/bib.js +24 -4
- package/src/cli.js +1 -0
- package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +14 -0
- package/src/match-detection/features/bib/title.js +62 -7
- package/src/matching-utils.js +50 -0
- package/{src → test}/candidate-search/index.test.js +3 -1
- package/{src → test}/candidate-search/query-list/auth.test.js +1 -1
- package/{src → test}/candidate-search/query-list/bib.test.js +1 -1
- package/{src → test}/index.test.js +10 -5
- package/{src → test}/match-detection/features/auth/index.test.js +1 -1
- package/{src → test}/match-detection/features/bib/index.test.js +1 -1
- package/{src → test}/match-detection/index.test.js +2 -2
- package/dist/candidate-search/index.test.js +0 -89
- package/dist/candidate-search/index.test.js.map +0 -7
- package/dist/candidate-search/query-list/auth.test.js +0 -31
- package/dist/candidate-search/query-list/auth.test.js.map +0 -7
- package/dist/candidate-search/query-list/bib.test.js +0 -31
- package/dist/candidate-search/query-list/bib.test.js.map +0 -7
- package/dist/index.test.js +0 -69
- package/dist/index.test.js.map +0 -7
- package/dist/match-detection/features/auth/index.test.js +0 -37
- package/dist/match-detection/features/auth/index.test.js.map +0 -7
- package/dist/match-detection/features/bib/index.test.js +0 -37
- package/dist/match-detection/features/bib/index.test.js.map +0 -7
- package/dist/match-detection/index.test.js +0 -40
- package/dist/match-detection/index.test.js.map +0 -7
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/match-detection/features/bib/title.js"],
|
|
4
|
-
"sourcesContent": ["import createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\nimport {testStringOrNumber} from '../../../matching-utils.js';\nimport { subfieldToString } from '@natlibfi/marc-record-validators-melinda/dist/utils.js';\n\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');\nconst debugData = debug.extend('data');\n\nconst TITLE_MAX = 0.5;\nexport default ({threshold = 0.9} = {}) => ({\n name: 'Title',\n extract: ({record, recordExternal}) => {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const a = getTitle(record, ['a']);\n debug(`${label}: title: ${a}`);\n\n if (a) {\n const b = normalizeTitle(getTitle(record, ['b'])) || '';\n const n = normalizeTitle(getTitle(record, ['n'])) || '';\n const p = normalizeTitle(getTitle(record, ['p'])) || '';\n return [normalizeTitle(a), b, n, p];\n }\n\n return ['', '', '', ''];\n\n function normalizeTitle(title) {\n return title\n // decompose unicode diacritics\n .normalize('NFD')\n .replace(/\\b(?:ein|ett|I|one|uno|yksi)\\b/uig, '1')\n .replace(/\\b(?:dos|kaksi|II|tv\u00E5|two|zwei)\\b/uig, '2')\n .replace(/\\b(?:drei|III|kolme|three|tre|tres)\\b/uig, '3')\n .replace(/\\b(?:four|fyra|IV|nelj\u00E4|quat+ro|vier)\\b/uig, '4')\n .replace(/\\b(?:five|f\u00FCnf|fyra|viisi|V)\\b/uig, '5')\n .replace(/\\b(?:kuusi|sechts|sex|six|VI)\\b/uig, '6')\n .replace(/\\b(and|ja|och|und)\\b/uig, '&')\n // strip non-letters/numbers\n // - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 's\u00E4\u00E4' as 'saa'\n // - we could precompose the Finnish letters back to avoid this\n // - see validator normalize-utf8-diacritics for details\n .replace(/[^\\p{Letter}\\p{Number}&]/gu, '')\n .toLowerCase();\n }\n\n },\n compare: (origA, origB) => {\n return compare2(origA, origB);\n\n function compare2(a, b) { // Added this wrapper function as I had issues with recursion\n\n const [aa, ab, an, ap] = a;\n const [ba, bb, bn, bp] = b;\n\n const aFull = toFullTitle(a);\n const bFull = toFullTitle(b);\n\n
|
|
5
|
-
"mappings": "AAAA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AACrC,SAAQ,0BAAyB;AACjC,SAAS,wBAAwB;AAGjC,MAAM,QAAQ,kBAAkB,sEAAsE;AACtG,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,MAAM,YAAY;AAClB,eAAe,CAAC,EAAC,YAAY,IAAG,IAAI,CAAC,OAAO;AAAA,EAC1C,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,IAAI,SAAS,QAAQ,CAAC,GAAG,CAAC;AAChC,UAAM,GAAG,KAAK,YAAY,CAAC,EAAE;AAE7B,QAAI,GAAG;AACL,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,aAAO,CAAC,eAAe,CAAC,GAAG,GAAG,GAAG,CAAC;AAAA,IACpC;AAEA,WAAO,CAAC,IAAI,IAAI,IAAI,EAAE;AAEtB,aAAS,eAAe,OAAO;AAC7B,aAAO,MAEJ,UAAU,KAAK,EACf,QAAQ,qCAAqC,GAAG,EAChD,QAAQ,wCAAwC,GAAG,EACnD,QAAQ,4CAA4C,GAAG,EACvD,QAAQ,8CAA8C,GAAG,EACzD,QAAQ,qCAAqC,GAAG,EAChD,QAAQ,sCAAsC,GAAG,EACjD,QAAQ,2BAA2B,GAAG,EAKtC,QAAQ,
|
|
6
|
-
"names": []
|
|
4
|
+
"sourcesContent": ["import createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\nimport {testStringOrNumber} from '../../../matching-utils.js';\nimport { subfieldToString } from '@natlibfi/marc-record-validators-melinda/dist/utils.js';\n\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');\nconst debugData = debug.extend('data');\n\nconst TITLE_MAX = 0.5;\nexport default ({threshold = 0.9} = {}) => ({\n name: 'Title',\n extract: ({record, recordExternal}) => {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const a = getTitle(record, ['a']);\n debug(`${label}: title: ${a}`);\n\n if (a) {\n const b = normalizeTitle(getTitle(record, ['b'])) || '';\n const n = normalizeTitle(getTitle(record, ['n'])) || '';\n const p = normalizeTitle(getTitle(record, ['p'])) || '';\n return [normalizeTitle(a), b, n, p];\n }\n\n return ['', '', '', ''];\n\n function normalizeTitle(title) {\n return title\n // decompose unicode diacritics\n .normalize('NFD')\n .replace(/\\b(?:ein|ett|I|one|uno|yksi)\\b/uig, '1')\n .replace(/\\b(?:dos|kaksi|II|tv\u00E5|two|zwei)\\b/uig, '2')\n .replace(/\\b(?:drei|III|kolme|three|tre|tres)\\b/uig, '3')\n .replace(/\\b(?:four|fyra|IV|nelj\u00E4|quat+ro|vier)\\b/uig, '4')\n .replace(/\\b(?:five|f\u00FCnf|fyra|viisi|V)\\b/uig, '5')\n .replace(/\\b(?:kuusi|sechts|sex|six|VI)\\b/uig, '6')\n .replace(/\\b(and|ja|och|und)\\b/uig, '&')\n // strip non-letters/numbers\n // - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 's\u00E4\u00E4' as 'saa'\n // - we could precompose the Finnish letters back to avoid this\n // - see validator normalize-utf8-diacritics for details\n .replace(/[^\\p{Letter}\\p{Number}& ]/gu, '') // 2026-07-01: keep ' '. We need it for splitToBaseAndNumber()\n .replace(/\\s+/ug, ' ')\n .replace(/ $/u, '')\n .replace(/^ /u, '')\n .toLowerCase();\n }\n\n },\n compare: (origA, origB) => {\n return compare2(origA, origB);\n\n function compare2(a, b) { // Added this wrapper function as I had issues with recursion\n\n const [aa, ab, an, ap] = a;\n const [ba, bb, bn, bp] = b;\n debugData(`COMPARE:\\n['${aa}', '${ab}', '${an}', '${ap}'] VS\\n['${ba}', '${bb}', '${bn}', '${bp}']`);\n\n const aFull = toFullTitle(a);\n const bFull = toFullTitle(b);\n\n // debugData(`JOINED COMPARE:\\n '${aFull}' vs \\n '${bFull}'`);\n\n if (isEmpty(aa) || isEmpty(ba)) {\n return -1.0;\n }\n\n if (aFull === bFull) {\n return TITLE_MAX;\n }\n\n // MELKEHITYS-3496-ish: move part of $a to $n:\n const [altAa, altAn, altBa, altBn] = reconstructTitleAndNumber(aa, an, ba, bn);\n if (altAa) {\n return compare2([altAa, ab, altAn, ap], [altBa, bb, altBn, bp]);\n }\n\n if (ab && ab !== '' && ab === bb) { // Remove $b from equation (MELKEHITYS-3494)\n debug(`Ignore \\$b ${ab}`);\n return compare2([aa, '', an, ap], [ba, '', bn, bp]);\n }\n if (an && an !== '' && an === bn) { // Remove $n from equation\n debug(`Ignore \\$n ${an}`);\n return compare2([aa, ab, '', ap], [ba, bb, '', bp]);\n }\n if (ap && ap !== '' && ap === bp) { // Remove $p from equation\n debug(`Ignore \\$p ${ap}`);\n return compare2([aa, ab, an, ''], [ba, bb, bn, '']);\n }\n // There seems to be wobble between $b and $p:\n if (!isEmpty(ab) && ab === bp) {\n return compare2([aa, '', an, ap], [ba, bb, bn, '']);\n }\n if (!isEmpty(ap) && ap === bp) {\n return compare2([aa, ab, an, ''], [ba, '', bn, bp]);\n }\n\n if (an && bn && an.match(/[0-9]/u)) {\n debug(`NUMBER FOUND. Compare ${an} and ${bn}`);\n const atmp = an.replace(/[^0-9]/ug, '');\n const btmp = bn.replace(/[^0-9]/ug, '');\n if (atmp === btmp) {\n debug(`Ignore \\$n '${an}' vs '${bn}'`);\n return compare2([aa, ab, '', ap], [ba, bb, '', bp]);\n }\n }\n\n\n // f245$n information is critical; it can not mismatch at all (exceptions have already been handled above):\n if (an && an !== bn) {\n return -1.0;\n }\n\n const [distance, maxLength, correctness] = doLevenshtein(aFull.replace(/ /ug, ''), bFull.replace(/ /ug, ''));\n\n debug(`'${aFull}' vs '${bFull}': Max length = ${maxLength}, distance = ${distance}, correctness = ${correctness}`);\n\n\n if (correctness >= threshold) {\n return TITLE_MAX * 0.8;\n }\n\n // Subset removal (MELKEHITYS-3498-ish)\n if (ab && bb) { // At this point ab and bb are never equal...\n // If X is a subset of Y, then remove X and shorten Y (= remove the shared content)\n if (ab.indexOf(bb) === 0) {\n return compare2([aa, ab.substring(bb.length), an, ap], [ba, '', bn, bp]);\n }\n if (bb.indexOf(ab) === 0) {\n return compare2([aa, '', an, ap], [ba, bb.substring(ab.length), bn, bp]);\n }\n }\n\n // Try the same without $p:\n if (localXor(ap, bp)) {\n const result = compare2([aa, ab, an, ''], [ba, bb, bn, '']);\n return result > 0.0 ? result * 0.8 : result;\n }\n\n if (aa === ba && an === bn) {\n const candScore = TITLE_MAX/2;\n // $b is missing from one:\n if (ap === bp && localXor(ab, bb)) {\n debug(`Handle omitted \\$b using x ${candScore}`);\n return candScore;\n }\n if (ab === bb && localXor(ap, bp)) {\n debug(`Handle omitted \\$p using x ${candScore}`);\n return candScore;\n }\n }\n\n if (an === bn && ap === bp ) {\n // $b-less $a is other record's $b, see MELKEHITYS-3485 for motivation (though we skip 246 and 490 here):\n const candScore = TITLE_MAX/2;\n if (aa === bb && !ab) {\n return candScore;\n }\n if (ba === ab && !bb) {\n return candScore;\n }\n }\n\n return -0.5; // Not likely\n }\n\n function isEmpty(x) {\n return !x || x.length === 0;\n }\n\n function localXor(x, y) {\n if (isEmpty(x)) {\n return !isEmpty(y);\n }\n // 'x' exists, thus 'y' can not exist:\n return isEmpty(y);\n }\n\n function doLevenshtein(string1, string2) {\n const distance = leven(string1, string2);\n const len = getMaxLength(string1, string2);\n const correctness = 1.0 - (distance / len);\n return [distance, len, correctness];\n }\n\n function toFullTitle(arr) {\n const relevant = arr.filter(val => typeof val === 'string' && val.length);\n // No longer add space between subfields. Due to heavy normalization there are no spaces left in array elements either!\n return relevant.join('');\n }\n\n function getMaxLength(str1, str2) {\n return str1.length > str2.length ? str1.length : str2.length;\n }\n\n function reconstructTitleAndNumber(a1, n1, a2, n2) {\n if (!n1 && n2) {\n const [altA1, altN1] = splitToBaseAndNumber(a1);\n if (altN1) {\n debugData(`A: Reconstruct:\\n $a '${a1}' =>\\n $a '${altA1}' +\\n $n '${altN1}`);\n return [altA1, altN1, a2, n2];\n }\n }\n if (n1 && !n2) {\n const [altA2, altN2] = splitToBaseAndNumber(a2);\n if (altN2) {\n debugData(`B: Reconstruct $a '${a2}' as $a '${altA2}' and $n '${altN2}`);\n return [a1, n1, altA2, altN2];\n }\n\n }\n return [null, null, null, null]; // Fail\n }\n\n\n function splitToBaseAndNumber(val) {\n const words = val.split(' ');\n const len = words.length;\n if (len < 2 || !words[len-1].match(/^[1-9][0-9]*$/)) {\n return [val, null];\n }\n if (len > 2 && words[len-2].match(/^(del|osa|vol|volume)$/u)) { // NB! \"vol.\" has lost it's '.' during normalization!\n const number = `${words[len-2]} ${words[len-1]}`;\n return [words.slice(0, len-2).join(' '), number];\n }\n\n const number = words.pop();\n return [words.join(' '), number];\n }\n\n }\n});\n\nexport function getTitle(record, targetSubfieldCodes = ['a', 'b', 'n', 'p']) {\n const [field] = record.get(/^245$/u);\n debugData(`titleField: ${JSON.stringify(field)}`);\n\n if (field) {\n const title = field.subfields\n // get also $n:s and $p:s here\n .filter(({code}) => targetSubfieldCodes.includes(code))\n // Would be nice to normalize $n values...\n .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n .join(' ')\n .replace(/\\[[^\\]]*\\]/ug, ' ') // Remove [whatever] stuff. ADD TEST FOR THIS\n .replace(/ [=\\/:](?:$| )/ug, ' ') // Strip punctuation\n // Also \u0111 vs d pairs seen:\n //.replace(/(?:\u0111|\u0227|t\u0304|k\u030C|\u01E7|s\u0306|c\u0306)/ug, '') // Hack. Saamelaisbibliografia has often dropped this wierd characters (oft old articles, no longer used in sami either)\n // trim:\n .replace(/ +/ug, ' ')\n .replace(/^ +/u, '')\n .replace(/ +$/u, '');\n\n // Skip non-filing indicator (note that '9' is a magic indicator value, so we don't do it):\n if (/^[1-8]$/u.test(field.ind2)) { // Skip non-filing characters\n return title.slice(parseInt(field.ind2));\n }\n return title;\n\n }\n return false;\n}"],
|
|
5
|
+
"mappings": "AAAA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AACrC,SAAQ,0BAAyB;AACjC,SAAS,wBAAwB;AAGjC,MAAM,QAAQ,kBAAkB,sEAAsE;AACtG,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,MAAM,YAAY;AAClB,eAAe,CAAC,EAAC,YAAY,IAAG,IAAI,CAAC,OAAO;AAAA,EAC1C,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,IAAI,SAAS,QAAQ,CAAC,GAAG,CAAC;AAChC,UAAM,GAAG,KAAK,YAAY,CAAC,EAAE;AAE7B,QAAI,GAAG;AACL,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,aAAO,CAAC,eAAe,CAAC,GAAG,GAAG,GAAG,CAAC;AAAA,IACpC;AAEA,WAAO,CAAC,IAAI,IAAI,IAAI,EAAE;AAEtB,aAAS,eAAe,OAAO;AAC7B,aAAO,MAEJ,UAAU,KAAK,EACf,QAAQ,qCAAqC,GAAG,EAChD,QAAQ,wCAAwC,GAAG,EACnD,QAAQ,4CAA4C,GAAG,EACvD,QAAQ,8CAA8C,GAAG,EACzD,QAAQ,qCAAqC,GAAG,EAChD,QAAQ,sCAAsC,GAAG,EACjD,QAAQ,2BAA2B,GAAG,EAKtC,QAAQ,+BAA+B,EAAE,EACzC,QAAQ,SAAS,GAAG,EACpB,QAAQ,OAAO,EAAE,EACjB,QAAQ,OAAO,EAAE,EACjB,YAAY;AAAA,IACjB;AAAA,EAEF;AAAA,EACA,SAAS,CAAC,OAAO,UAAU;AACzB,WAAO,SAAS,OAAO,KAAK;AAE5B,aAAS,SAAS,GAAG,GAAG;AAEtB,YAAM,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI;AACzB,YAAM,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI;AACzB,gBAAU;AAAA,IAAe,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE;AAAA,IAAY,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI;AAEnG,YAAM,QAAQ,YAAY,CAAC;AAC3B,YAAM,QAAQ,YAAY,CAAC;AAI3B,UAAI,QAAQ,EAAE,KAAK,QAAQ,EAAE,GAAG;AAC9B,eAAO;AAAA,MACT;AAEA,UAAI,UAAU,OAAO;AACnB,eAAO;AAAA,MACT;AAGA,YAAM,CAAC,OAAO,OAAO,OAAO,KAAK,IAAI,0BAA0B,IAAI,IAAI,IAAI,EAAE;AAC7E,UAAI,OAAO;AACT,eAAO,SAAS,CAAC,OAAO,IAAI,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,OAAO,EAAE,CAAC;AAAA,MAChE;AAEA,UAAI,MAAM,OAAO,MAAM,OAAO,IAAI;AAChC,cAAM,aAAc,EAAE,EAAE;AACxB,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AACA,UAAI,MAAM,OAAO,MAAM,OAAO,IAAI;AAChC,cAAM,aAAc,EAAE,EAAE;AACxB,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AACA,UAAI,MAAM,OAAO,MAAM,OAAO,IAAI;AAChC,cAAM,aAAc,EAAE,EAAE;AACxB,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AAEA,UAAI,CAAC,QAAQ,EAAE,KAAK,OAAO,IAAI;AAC7B,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AACA,UAAI,CAAC,QAAQ,EAAE,KAAK,OAAO,IAAI;AAC7B,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AAEA,UAAI,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG;AAClC,cAAM,yBAAyB,EAAE,QAAQ,EAAE,EAAE;AAC7C,cAAM,OAAO,GAAG,QAAQ,YAAY,EAAE;AACtC,cAAM,OAAO,GAAG,QAAQ,YAAY,EAAE;AACtC,YAAI,SAAS,MAAM;AACjB,gBAAM,cAAe,EAAE,SAAS,EAAE,GAAG;AACrC,iBAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,QACpD;AAAA,MACF;AAIA,UAAI,MAAM,OAAO,IAAI;AACnB,eAAO;AAAA,MACT;AAEA,YAAM,CAAC,UAAU,WAAW,WAAW,IAAI,cAAc,MAAM,QAAQ,OAAO,EAAE,GAAG,MAAM,QAAQ,OAAO,EAAE,CAAC;AAE3G,YAAM,IAAI,KAAK,SAAS,KAAK,mBAAmB,SAAS,gBAAgB,QAAQ,mBAAmB,WAAW,EAAE;AAGjH,UAAI,eAAe,WAAW;AAC5B,eAAO,YAAY;AAAA,MACrB;AAGA,UAAI,MAAM,IAAI;AAEZ,YAAI,GAAG,QAAQ,EAAE,MAAM,GAAG;AACxB,iBAAO,SAAS,CAAC,IAAI,GAAG,UAAU,GAAG,MAAM,GAAG,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,QACzE;AACA,YAAI,GAAG,QAAQ,EAAE,MAAM,GAAG;AACxB,iBAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,UAAU,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;AAAA,QACzE;AAAA,MACF;AAGA,UAAI,SAAS,IAAI,EAAE,GAAG;AACpB,cAAM,SAAS,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAC1D,eAAO,SAAS,IAAM,SAAS,MAAM;AAAA,MACvC;AAEA,UAAI,OAAO,MAAM,OAAO,IAAI;AAC1B,cAAM,YAAY,YAAU;AAE5B,YAAI,OAAO,MAAM,SAAS,IAAI,EAAE,GAAG;AACjC,gBAAM,6BAA8B,SAAS,EAAE;AAC/C,iBAAO;AAAA,QACT;AACA,YAAI,OAAO,MAAM,SAAS,IAAI,EAAE,GAAG;AACjC,gBAAM,6BAA8B,SAAS,EAAE;AAC/C,iBAAO;AAAA,QACT;AAAA,MACF;AAEA,UAAI,OAAO,MAAM,OAAO,IAAK;AAE3B,cAAM,YAAY,YAAU;AAC5B,YAAI,OAAO,MAAM,CAAC,IAAI;AACpB,iBAAO;AAAA,QACT;AACA,YAAI,OAAO,MAAM,CAAC,IAAI;AACpB,iBAAO;AAAA,QACT;AAAA,MACF;AAEA,aAAO;AAAA,IACT;AAEA,aAAS,QAAQ,GAAG;AAClB,aAAO,CAAC,KAAK,EAAE,WAAW;AAAA,IAC5B;AAEA,aAAS,SAAS,GAAG,GAAG;AACtB,UAAI,QAAQ,CAAC,GAAG;AACd,eAAO,CAAC,QAAQ,CAAC;AAAA,MACnB;AAEA,aAAO,QAAQ,CAAC;AAAA,IAClB;AAEA,aAAS,cAAc,SAAS,SAAS;AACvC,YAAM,WAAW,MAAM,SAAS,OAAO;AACvC,YAAM,MAAM,aAAa,SAAS,OAAO;AACzC,YAAM,cAAc,IAAO,WAAW;AACtC,aAAO,CAAC,UAAU,KAAK,WAAW;AAAA,IACpC;AAEA,aAAS,YAAY,KAAK;AACxB,YAAM,WAAW,IAAI,OAAO,SAAO,OAAO,QAAQ,YAAY,IAAI,MAAM;AAExE,aAAO,SAAS,KAAK,EAAE;AAAA,IACzB;AAEA,aAAS,aAAa,MAAM,MAAM;AAChC,aAAO,KAAK,SAAS,KAAK,SAAS,KAAK,SAAS,KAAK;AAAA,IACxD;AAEA,aAAS,0BAA0B,IAAI,IAAI,IAAI,IAAI;AACjD,UAAI,CAAC,MAAM,IAAI;AACb,cAAM,CAAC,OAAO,KAAK,IAAI,qBAAqB,EAAE;AAC9C,YAAI,OAAO;AACT,oBAAU;AAAA,OAAyB,EAAE;AAAA,OAAc,KAAK;AAAA,OAAa,KAAK,EAAE;AAC5E,iBAAO,CAAC,OAAO,OAAO,IAAI,EAAE;AAAA,QAC9B;AAAA,MACF;AACA,UAAI,MAAM,CAAC,IAAI;AACb,cAAM,CAAC,OAAO,KAAK,IAAI,qBAAqB,EAAE;AAC9C,YAAI,OAAO;AACT,oBAAU,sBAAsB,EAAE,YAAY,KAAK,aAAa,KAAK,EAAE;AACvE,iBAAO,CAAC,IAAI,IAAI,OAAO,KAAK;AAAA,QAC9B;AAAA,MAEF;AACA,aAAO,CAAC,MAAM,MAAM,MAAM,IAAI;AAAA,IAChC;AAGA,aAAS,qBAAqB,KAAK;AACjC,YAAM,QAAQ,IAAI,MAAM,GAAG;AAC3B,YAAM,MAAM,MAAM;AAClB,UAAI,MAAM,KAAK,CAAC,MAAM,MAAI,CAAC,EAAE,MAAM,eAAe,GAAG;AACnD,eAAO,CAAC,KAAK,IAAI;AAAA,MACnB;AACA,UAAI,MAAM,KAAK,MAAM,MAAI,CAAC,EAAE,MAAM,yBAAyB,GAAG;AAC5D,cAAMA,UAAS,GAAG,MAAM,MAAI,CAAC,CAAC,IAAI,MAAM,MAAI,CAAC,CAAC;AAC9C,eAAO,CAAC,MAAM,MAAM,GAAG,MAAI,CAAC,EAAE,KAAK,GAAG,GAAGA,OAAM;AAAA,MACjD;AAEA,YAAM,SAAS,MAAM,IAAI;AACzB,aAAO,CAAC,MAAM,KAAK,GAAG,GAAG,MAAM;AAAA,IACjC;AAAA,EAEF;AACF;AAEO,gBAAS,SAAS,QAAQ,sBAAsB,CAAC,KAAK,KAAK,KAAK,GAAG,GAAG;AAC3E,QAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AACnC,YAAU,eAAe,KAAK,UAAU,KAAK,CAAC,EAAE;AAEhD,MAAI,OAAO;AACT,UAAM,QAAQ,MAAM,UAEjB,OAAO,CAAC,EAAC,KAAI,MAAM,oBAAoB,SAAS,IAAI,CAAC,EAErD,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,KAAK,GAAG,EACR,QAAQ,gBAAgB,GAAG,EAC3B,QAAQ,oBAAoB,GAAG,EAI/B,QAAQ,QAAQ,GAAG,EACnB,QAAQ,QAAQ,EAAE,EAClB,QAAQ,QAAQ,EAAE;AAGrB,QAAI,WAAW,KAAK,MAAM,IAAI,GAAG;AAC/B,aAAO,MAAM,MAAM,SAAS,MAAM,IAAI,CAAC;AAAA,IACzC;AACA,WAAO;AAAA,EAET;AACA,SAAO;AACT;",
|
|
6
|
+
"names": ["number"]
|
|
7
7
|
}
|
package/dist/matching-utils.js
CHANGED
|
@@ -90,4 +90,43 @@ export function stringBefore(string, substring) {
|
|
|
90
90
|
}
|
|
91
91
|
return string.substring(0, pos);
|
|
92
92
|
}
|
|
93
|
+
export function preferenceSortRecords(x, y) {
|
|
94
|
+
debug(`sortRecords: RECORD COMPARISON
|
|
95
|
+
${JSON.stringify(x)} vs
|
|
96
|
+
${JSON.stringify(y)}`);
|
|
97
|
+
const xFikka = hasLow(x.record, "FIKKA");
|
|
98
|
+
const yFikka = hasLow(y.record, "FIKKA");
|
|
99
|
+
if (xFikka && !yFikka) {
|
|
100
|
+
debug("sortRecords: prefer FIKKA");
|
|
101
|
+
return -1;
|
|
102
|
+
}
|
|
103
|
+
if (yFikka && !xFikka) {
|
|
104
|
+
return 1;
|
|
105
|
+
}
|
|
106
|
+
const xAllCaps = hasAllCapsTitle(x.record);
|
|
107
|
+
const yAllCaps = hasAllCapsTitle(y.record);
|
|
108
|
+
if (!xAllCaps && yAllCaps) {
|
|
109
|
+
debug("sortRecords: prefer non-all caps");
|
|
110
|
+
return -1;
|
|
111
|
+
}
|
|
112
|
+
if (!yAllCaps && xAllCaps) {
|
|
113
|
+
return 1;
|
|
114
|
+
}
|
|
115
|
+
return 0;
|
|
116
|
+
function hasLow(record, lowTag) {
|
|
117
|
+
return record.fields?.some((f) => f.tag === "LOW" && f.subfields?.some((sf) => sf.code === "a" && sf.value === lowTag));
|
|
118
|
+
}
|
|
119
|
+
function hasAllCapsTitle(record) {
|
|
120
|
+
const f245 = record.fields?.find((f) => f.tag === "245");
|
|
121
|
+
if (!f245) {
|
|
122
|
+
debugData(`WARNING: No f245 found for ALL CAPS check`);
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
const title = f245.subfields.filter((sf) => ["a", "b", "c"].includes(sf.code)).map((sf) => sf.value).join(" ");
|
|
126
|
+
if (title.match(/[A-Z]/u) && !title.match(/[a-z]/u)) {
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
return false;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
93
132
|
//# sourceMappingURL=matching-utils.js.map
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/matching-utils.js"],
|
|
4
|
-
"sourcesContent": ["import createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\nexport function getMelindaIdsF035(record) {\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n const debugData = debug.extend('data');\n\n const f035s = record.getFields('035');\n\n if (f035s.length < 1) {\n debug(`No f035s found.`);\n return [];\n }\n\n const allF035MelindaIds = [].concat(...f035s.map(field => toMelindaIds(field)));\n const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n return f035MelindaIds;\n}\n\nexport function toMelindaIds({subfields}, subfieldsToParse = ['a', 'z']) {\n const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n return subfields\n .filter(sub => subfieldsToParse.includes(sub.code))\n .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n // Valid SID-fields have just one $c and one $b\n debugData(`Validating SID field ${JSON.stringify(field)}`);\n const countC = countSubfields(field, 'c');\n const countB = countSubfields(field, 'b');\n debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n // debug(`Counting subfields ${subfieldCode}`);\n return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n return field.subfields\n .filter(({code}) => code === subfieldCode)\n .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n if (typeof value === 'string' || typeof value === 'number') {\n return true;\n }\n return false;\n}\n\nexport function extractSubfieldsFromField(field, subfieldCodes) {\n if (field === undefined || field.subfields === undefined) {\n return [];\n }\n const resultSubfields = field.subfields\n .filter(({code}) => subfieldCodes.includes(code))\n .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))\n .filter(value => value);\n return resultSubfields;\n}\n\nexport function uniqueSubfields(subfields) {\n return subfields.reduce((arr, e) => {\n if (!arr.find(item => item.code === e.code && item.value === e.value)) {\n const newArr = arr.concat(e);\n return newArr;\n }\n return arr;\n }, []);\n}\n\nexport function getMatchCounts(aValues, bValues) {\n\n const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n return {\n maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n matchingValues\n };\n\n function getMatchingValuesAmount(aValues, bValues) {\n if (bValues.length > aValues.length) {\n return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n }\n if (aValues.length > bValues.length) {\n return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n }\n\n // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n // there would be duplicate values\n const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n return aToB < bToA ? aToB : bToA;\n }\n}\n\n\nexport function stringAfter(string, substring) {\n const pos = string.indexOf(substring);\n if (pos === -1) {\n return string;\n }\n return string.substring(pos+substring.length);\n}\n\nexport function stringBefore(string, substring) {\n const pos = string.indexOf(substring);\n if (pos === -1) {\n return string;\n }\n return string.substring(0, pos);\n}\n"],
|
|
5
|
-
"mappings": "AAAA,OAAO,uBAAuB;AAE9B,MAAM,QAAQ,kBAAkB,yCAAyC;AACzE,MAAM,YAAY,MAAM,OAAO,MAAM;AAE9B,gBAAS,kBAAkB,QAAQ;AAExC,QAAMA,SAAQ,kBAAkB,8CAA8C;AAC9E,QAAMC,aAAYD,OAAM,OAAO,MAAM;AAErC,QAAM,QAAQ,OAAO,UAAU,KAAK;AAEpC,MAAI,MAAM,SAAS,GAAG;AACpB,IAAAA,OAAM,iBAAiB;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,CAAC,EAAE,OAAO,GAAG,MAAM,IAAI,WAAS,aAAa,KAAK,CAAC,CAAC;AAC9E,QAAM,iBAAiB,CAAC,GAAG,IAAI,IAAI,iBAAiB,CAAC;AAErD,EAAAC,WAAU,WAAW,MAAM,MAAM,MAAM,KAAK,UAAU,KAAK,CAAC,EAAE;AAC9D,EAAAA,WAAU,QAAQ,kBAAkB,MAAM,MAAM,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACnF,EAAAA,WAAU,eAAe,eAAe,MAAM,MAAM,KAAK,UAAU,cAAc,CAAC,EAAE;AAEpF,SAAO;AACT;AAEO,gBAAS,aAAa,EAAC,UAAS,GAAG,mBAAmB,CAAC,KAAK,GAAG,GAAG;AACvE,QAAM,kBAAkB;AAExB,SAAO,UACJ,OAAO,SAAO,iBAAiB,SAAS,IAAI,IAAI,CAAC,EACjD,OAAO,SAAO,mBAAmB,IAAI,KAAK,KAAK,gBAAgB,KAAK,OAAO,IAAI,KAAK,CAAC,CAAC,EACtF,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,EAAE,QAAQ,iBAAiB,OAAO,IAAI,EAAE;AACtG;AAEO,gBAAS,+BAA+B,OAAO;AAEpD,YAAU,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AACzD,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,MAAM,eAAe,MAAM,qBAAqB,WAAW,KAAK,WAAW,CAAC,EAAE;AAE7F,SAAO,WAAW,KAAK,WAAW;AACpC;AAEA,SAAS,eAAe,OAAO,cAAc;AAE3C,SAAO,MAAM,UAAU,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EAAE;AACnE;AAEO,gBAAS,kBAAkB,OAAO,cAAc;AACrD,YAAU,oBAAoB,YAAY,SAAS,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E,SAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EACxC,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,OAAO,WAAS,KAAK;AAC1B;AAEO,gBAAS,mBAAmB,OAAO;AACxC,MAAI,OAAO,UAAU,YAAY,OAAO,UAAU,UAAU;AAC1D,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEO,gBAAS,0BAA0B,OAAO,eAAe;AAC9D,MAAI,UAAU,UAAa,MAAM,cAAc,QAAW;AACxD,WAAO,CAAC;AAAA,EACV;AACA,QAAM,kBAAkB,MAAM,UAC3B,OAAO,CAAC,EAAC,KAAI,MAAM,cAAc,SAAS,IAAI,CAAC,EAC/C,IAAI,CAAC,EAAC,MAAM,MAAK,OAAO,EAAC,MAAM,OAAO,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,GAAE,EAAE,EACtF,OAAO,WAAS,KAAK;AACxB,SAAO;AACT;AAEO,gBAAS,gBAAgB,WAAW;AACzC,SAAO,UAAU,OAAO,CAAC,KAAK,MAAM;AAClC,QAAI,CAAC,IAAI,KAAK,UAAQ,KAAK,SAAS,EAAE,QAAQ,KAAK,UAAU,EAAE,KAAK,GAAG;AACrE,YAAM,SAAS,IAAI,OAAO,CAAC;AAC3B,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,GAAG,CAAC,CAAC;AACP;AAEO,gBAAS,eAAe,SAAS,SAAS;AAE/C,QAAM,iBAAiB,wBAAwB,SAAS,OAAO;AAE/D,SAAO;AAAA,IACL,WAAW,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA;AAAA,IAEtE,qBAAqB,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA,IAChF;AAAA,EACF;AAEA,WAAS,wBAAwBC,UAASC,UAAS;AACjD,QAAIA,SAAQ,SAASD,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AACA,QAAID,SAAQ,SAASC,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AAIA,UAAM,OAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AACjF,UAAM,OAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAEjF,WAAO,OAAO,OAAO,OAAO;AAAA,EAC9B;AACF;AAGO,gBAAS,YAAY,QAAQ,WAAW;AAC7C,QAAM,MAAM,OAAO,QAAQ,SAAS;AACpC,MAAI,QAAQ,IAAI;AACd,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,MAAI,UAAU,MAAM;AAC9C;AAEO,gBAAS,aAAa,QAAQ,WAAW;AAC9C,QAAM,MAAM,OAAO,QAAQ,SAAS;AACpC,MAAI,QAAQ,IAAI;AACd,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,GAAG,GAAG;AAChC;",
|
|
4
|
+
"sourcesContent": ["import createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\nexport function getMelindaIdsF035(record) {\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n const debugData = debug.extend('data');\n\n const f035s = record.getFields('035');\n\n if (f035s.length < 1) {\n debug(`No f035s found.`);\n return [];\n }\n\n const allF035MelindaIds = [].concat(...f035s.map(field => toMelindaIds(field)));\n const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n return f035MelindaIds;\n}\n\nexport function toMelindaIds({subfields}, subfieldsToParse = ['a', 'z']) {\n const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n return subfields\n .filter(sub => subfieldsToParse.includes(sub.code))\n .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n // Valid SID-fields have just one $c and one $b\n debugData(`Validating SID field ${JSON.stringify(field)}`);\n const countC = countSubfields(field, 'c');\n const countB = countSubfields(field, 'b');\n debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n // debug(`Counting subfields ${subfieldCode}`);\n return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n return field.subfields\n .filter(({code}) => code === subfieldCode)\n .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n if (typeof value === 'string' || typeof value === 'number') {\n return true;\n }\n return false;\n}\n\nexport function extractSubfieldsFromField(field, subfieldCodes) {\n if (field === undefined || field.subfields === undefined) {\n return [];\n }\n const resultSubfields = field.subfields\n .filter(({code}) => subfieldCodes.includes(code))\n .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))\n .filter(value => value);\n return resultSubfields;\n}\n\nexport function uniqueSubfields(subfields) {\n return subfields.reduce((arr, e) => {\n if (!arr.find(item => item.code === e.code && item.value === e.value)) {\n const newArr = arr.concat(e);\n return newArr;\n }\n return arr;\n }, []);\n}\n\nexport function getMatchCounts(aValues, bValues) {\n\n const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n return {\n maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n matchingValues\n };\n\n function getMatchingValuesAmount(aValues, bValues) {\n if (bValues.length > aValues.length) {\n return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n }\n if (aValues.length > bValues.length) {\n return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n }\n\n // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n // there would be duplicate values\n const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n return aToB < bToA ? aToB : bToA;\n }\n}\n\n\nexport function stringAfter(string, substring) {\n const pos = string.indexOf(substring);\n if (pos === -1) {\n return string;\n }\n return string.substring(pos+substring.length);\n}\n\nexport function stringBefore(string, substring) {\n const pos = string.indexOf(substring);\n if (pos === -1) {\n return string;\n }\n return string.substring(0, pos);\n}\n\n// We sort records here (put FIKKA and low-cased records first, ref MELINDA-3492)\n// NB! this only sorts the records the current set. If the best stuff would come in a later set and this set contains good enough record, we'll never reach the best...\n// Still this would alleviate the issue when maxMatches === 1...\nexport function preferenceSortRecords(x, y) {\n debug(`sortRecords: RECORD COMPARISON\\n${JSON.stringify(x)} vs\\n${JSON.stringify(y)}`);\n // Prefer the record with a FIKKA LOW tag:\n const xFikka = hasLow(x.record, 'FIKKA');\n const yFikka = hasLow(y.record, 'FIKKA');\n\n if (xFikka && !yFikka) {\n debug('sortRecords: prefer FIKKA');\n return -1;\n }\n if (yFikka && !xFikka) {\n return 1;\n }\n\n // Prefer downcased:\n const xAllCaps = hasAllCapsTitle(x.record);\n const yAllCaps = hasAllCapsTitle(y.record);\n\n if (!xAllCaps && yAllCaps) {\n debug('sortRecords: prefer non-all caps');\n return -1; // x is better\n }\n if (!yAllCaps && xAllCaps) {\n return 1; // y is better\n }\n // Default:\n return 0;\n\n function hasLow(record, lowTag) {\n return record.fields?.some(f => f.tag === 'LOW' && f.subfields?.some(sf => sf.code === 'a' && sf.value === lowTag));\n }\n\n function hasAllCapsTitle(record) {\n const f245 = record.fields?.find(f => f.tag === '245');\n if (!f245) {\n debugData(`WARNING: No f245 found for ALL CAPS check`);\n return false;\n }\n\n const title = f245.subfields.filter(sf => ['a', 'b', 'c'].includes(sf.code)).map(sf => sf.value).join(' ');\n if (title.match(/[A-Z]/u) && !title.match(/[a-z]/u)) {\n return true;\n }\n return false;\n }\n}"],
|
|
5
|
+
"mappings": "AAAA,OAAO,uBAAuB;AAE9B,MAAM,QAAQ,kBAAkB,yCAAyC;AACzE,MAAM,YAAY,MAAM,OAAO,MAAM;AAE9B,gBAAS,kBAAkB,QAAQ;AAExC,QAAMA,SAAQ,kBAAkB,8CAA8C;AAC9E,QAAMC,aAAYD,OAAM,OAAO,MAAM;AAErC,QAAM,QAAQ,OAAO,UAAU,KAAK;AAEpC,MAAI,MAAM,SAAS,GAAG;AACpB,IAAAA,OAAM,iBAAiB;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,CAAC,EAAE,OAAO,GAAG,MAAM,IAAI,WAAS,aAAa,KAAK,CAAC,CAAC;AAC9E,QAAM,iBAAiB,CAAC,GAAG,IAAI,IAAI,iBAAiB,CAAC;AAErD,EAAAC,WAAU,WAAW,MAAM,MAAM,MAAM,KAAK,UAAU,KAAK,CAAC,EAAE;AAC9D,EAAAA,WAAU,QAAQ,kBAAkB,MAAM,MAAM,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACnF,EAAAA,WAAU,eAAe,eAAe,MAAM,MAAM,KAAK,UAAU,cAAc,CAAC,EAAE;AAEpF,SAAO;AACT;AAEO,gBAAS,aAAa,EAAC,UAAS,GAAG,mBAAmB,CAAC,KAAK,GAAG,GAAG;AACvE,QAAM,kBAAkB;AAExB,SAAO,UACJ,OAAO,SAAO,iBAAiB,SAAS,IAAI,IAAI,CAAC,EACjD,OAAO,SAAO,mBAAmB,IAAI,KAAK,KAAK,gBAAgB,KAAK,OAAO,IAAI,KAAK,CAAC,CAAC,EACtF,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,EAAE,QAAQ,iBAAiB,OAAO,IAAI,EAAE;AACtG;AAEO,gBAAS,+BAA+B,OAAO;AAEpD,YAAU,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AACzD,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,MAAM,eAAe,MAAM,qBAAqB,WAAW,KAAK,WAAW,CAAC,EAAE;AAE7F,SAAO,WAAW,KAAK,WAAW;AACpC;AAEA,SAAS,eAAe,OAAO,cAAc;AAE3C,SAAO,MAAM,UAAU,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EAAE;AACnE;AAEO,gBAAS,kBAAkB,OAAO,cAAc;AACrD,YAAU,oBAAoB,YAAY,SAAS,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E,SAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EACxC,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,OAAO,WAAS,KAAK;AAC1B;AAEO,gBAAS,mBAAmB,OAAO;AACxC,MAAI,OAAO,UAAU,YAAY,OAAO,UAAU,UAAU;AAC1D,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEO,gBAAS,0BAA0B,OAAO,eAAe;AAC9D,MAAI,UAAU,UAAa,MAAM,cAAc,QAAW;AACxD,WAAO,CAAC;AAAA,EACV;AACA,QAAM,kBAAkB,MAAM,UAC3B,OAAO,CAAC,EAAC,KAAI,MAAM,cAAc,SAAS,IAAI,CAAC,EAC/C,IAAI,CAAC,EAAC,MAAM,MAAK,OAAO,EAAC,MAAM,OAAO,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,GAAE,EAAE,EACtF,OAAO,WAAS,KAAK;AACxB,SAAO;AACT;AAEO,gBAAS,gBAAgB,WAAW;AACzC,SAAO,UAAU,OAAO,CAAC,KAAK,MAAM;AAClC,QAAI,CAAC,IAAI,KAAK,UAAQ,KAAK,SAAS,EAAE,QAAQ,KAAK,UAAU,EAAE,KAAK,GAAG;AACrE,YAAM,SAAS,IAAI,OAAO,CAAC;AAC3B,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,GAAG,CAAC,CAAC;AACP;AAEO,gBAAS,eAAe,SAAS,SAAS;AAE/C,QAAM,iBAAiB,wBAAwB,SAAS,OAAO;AAE/D,SAAO;AAAA,IACL,WAAW,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA;AAAA,IAEtE,qBAAqB,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA,IAChF;AAAA,EACF;AAEA,WAAS,wBAAwBC,UAASC,UAAS;AACjD,QAAIA,SAAQ,SAASD,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AACA,QAAID,SAAQ,SAASC,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AAIA,UAAM,OAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AACjF,UAAM,OAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAEjF,WAAO,OAAO,OAAO,OAAO;AAAA,EAC9B;AACF;AAGO,gBAAS,YAAY,QAAQ,WAAW;AAC7C,QAAM,MAAM,OAAO,QAAQ,SAAS;AACpC,MAAI,QAAQ,IAAI;AACd,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,MAAI,UAAU,MAAM;AAC9C;AAEO,gBAAS,aAAa,QAAQ,WAAW;AAC9C,QAAM,MAAM,OAAO,QAAQ,SAAS;AACpC,MAAI,QAAQ,IAAI;AACd,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,GAAG,GAAG;AAChC;AAKO,gBAAS,sBAAsB,GAAG,GAAG;AAC1C,QAAM;AAAA,EAAmC,KAAK,UAAU,CAAC,CAAC;AAAA,EAAQ,KAAK,UAAU,CAAC,CAAC,EAAE;AAErF,QAAM,SAAS,OAAO,EAAE,QAAQ,OAAO;AACvC,QAAM,SAAS,OAAO,EAAE,QAAQ,OAAO;AAEvC,MAAI,UAAU,CAAC,QAAQ;AACrB,UAAM,2BAA2B;AACjC,WAAO;AAAA,EACT;AACA,MAAI,UAAU,CAAC,QAAQ;AACrB,WAAO;AAAA,EACT;AAGA,QAAM,WAAW,gBAAgB,EAAE,MAAM;AACzC,QAAM,WAAW,gBAAgB,EAAE,MAAM;AAEzC,MAAI,CAAC,YAAY,UAAU;AACzB,UAAM,kCAAkC;AACxC,WAAO;AAAA,EACT;AACA,MAAI,CAAC,YAAY,UAAU;AACzB,WAAO;AAAA,EACT;AAEA,SAAO;AAEP,WAAS,OAAO,QAAQ,QAAQ;AAC9B,WAAO,OAAO,QAAQ,KAAK,OAAK,EAAE,QAAQ,SAAS,EAAE,WAAW,KAAK,QAAM,GAAG,SAAS,OAAO,GAAG,UAAU,MAAM,CAAC;AAAA,EACpH;AAEA,WAAS,gBAAgB,QAAQ;AAC/B,UAAM,OAAO,OAAO,QAAQ,KAAK,OAAK,EAAE,QAAQ,KAAK;AACrD,QAAI,CAAC,MAAM;AACT,gBAAU,2CAA2C;AACrD,aAAO;AAAA,IACT;AAEA,UAAM,QAAQ,KAAK,UAAU,OAAO,QAAM,CAAC,KAAK,KAAK,GAAG,EAAE,SAAS,GAAG,IAAI,CAAC,EAAE,IAAI,QAAM,GAAG,KAAK,EAAE,KAAK,GAAG;AACzG,QAAI,MAAM,MAAM,QAAQ,KAAK,CAAC,MAAM,MAAM,QAAQ,GAAG;AACnD,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AACF;",
|
|
6
6
|
"names": ["debug", "debugData", "aValues", "bValues"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -13,12 +13,12 @@
|
|
|
13
13
|
"url": "https://github.com/NatLibFi/melinda-record-matching-js"
|
|
14
14
|
},
|
|
15
15
|
"license": "MIT",
|
|
16
|
-
"version": "5.1.
|
|
16
|
+
"version": "5.1.6-alpha.1",
|
|
17
17
|
"type": "module",
|
|
18
18
|
"main": "./dist/index.js",
|
|
19
19
|
"bin": "./dist/cli.js",
|
|
20
20
|
"engines": {
|
|
21
|
-
"node": ">=
|
|
21
|
+
"node": ">=24"
|
|
22
22
|
},
|
|
23
23
|
"publishConfig": {
|
|
24
24
|
"access": "public"
|
|
@@ -30,11 +30,12 @@
|
|
|
30
30
|
"cli": "node dist/cli.js",
|
|
31
31
|
"lint": "eslint ./src",
|
|
32
32
|
"lint:dev": "eslint --fix ./src",
|
|
33
|
-
"test:base": "node --test --test-force-exit --experimental-test-coverage --test-reporter=spec '
|
|
33
|
+
"test:base": "node --test --test-force-exit --experimental-test-coverage --test-reporter=spec 'test/**/*.test.js'",
|
|
34
34
|
"test": "npm run lint && npm run test:base",
|
|
35
35
|
"test:only": "npm run lint && npm run test:base:only",
|
|
36
|
-
"test:base:only": "cross-env DEBUG=@natlibfi/*
|
|
37
|
-
"
|
|
36
|
+
"test:base:only": "cross-env DEBUG=@natlibfi/* NODE_ENV=test node --test --test-only --test-force-exit --experimental-test-coverage --test-reporter=spec 'test/**/*.test.js'",
|
|
37
|
+
"test:base:only:index": "cross-env DEBUG=@natlibfi/* NODE_ENV=test node --test --test-only --test-force-exit --experimental-test-coverage --test-reporter=spec './test/index.test.js'",
|
|
38
|
+
"watch:test": "cross-env DEBUG=@natlibfi/* NODE_ENV=test node --watch --test --experimental-test-coverage --test-reporter=spec 'test/**/*.test.js'",
|
|
38
39
|
"dev": "npm run watch:test",
|
|
39
40
|
"dev:debug": "cross-env LOG_LEVEL=debug DEBUG=@natlibfi/* NODE_ENV=test npm run watch:test"
|
|
40
41
|
},
|
|
@@ -42,9 +43,9 @@
|
|
|
42
43
|
"@natlibfi/marc-record": "^10.0.2",
|
|
43
44
|
"@natlibfi/marc-record-validators-melinda": "^12.0.15",
|
|
44
45
|
"@natlibfi/melinda-commons": "^14.0.2",
|
|
45
|
-
"@natlibfi/sru-client": "^7.0.
|
|
46
|
+
"@natlibfi/sru-client": "^7.0.2",
|
|
46
47
|
"debug": "^4.4.3",
|
|
47
|
-
"isbn3": "^2.0.
|
|
48
|
+
"isbn3": "^2.0.9",
|
|
48
49
|
"natural": "^8.1.1",
|
|
49
50
|
"yargs": "^18.0.0"
|
|
50
51
|
},
|
|
@@ -54,9 +55,6 @@
|
|
|
54
55
|
"@natlibfi/fixura": "^4.0.1",
|
|
55
56
|
"cross-env": "^10.1.0",
|
|
56
57
|
"esbuild": "^0.28.1",
|
|
57
|
-
"eslint": "^10.
|
|
58
|
-
},
|
|
59
|
-
"overrides": {
|
|
60
|
-
"uuid": "^14.0.0"
|
|
58
|
+
"eslint": "^10.7.0"
|
|
61
59
|
}
|
|
62
60
|
}
|
|
@@ -4,6 +4,7 @@ import {MarcRecord} from '@natlibfi/marc-record';
|
|
|
4
4
|
|
|
5
5
|
import generateQueryList from './query-list/index.js';
|
|
6
6
|
import chooseQueries from './choose-queries.js';
|
|
7
|
+
import {preferenceSortRecords} from '../matching-utils.js';
|
|
7
8
|
|
|
8
9
|
export {searchTypes} from './query-list/index.js';
|
|
9
10
|
|
|
@@ -96,6 +97,8 @@ export default async ({record, searchSpec, url, maxCandidates, maxRecordsPerRequ
|
|
|
96
97
|
}
|
|
97
98
|
debug(`Query ${queryOffset} ${query} done.`);
|
|
98
99
|
debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);
|
|
100
|
+
|
|
101
|
+
records.sort(preferenceSortRecords);
|
|
99
102
|
return {records, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
|
|
100
103
|
}
|
|
101
104
|
|
|
@@ -435,7 +435,8 @@ export function bibStandardIdentifiers(record) {
|
|
|
435
435
|
|
|
436
436
|
// DEVELOP: should we query also f015 and f028?
|
|
437
437
|
|
|
438
|
-
const fields = record.get(/^(?<def>020|022|024)$/u);
|
|
438
|
+
// const fields = record.get(/^(?<def>020|022|024)$/u);
|
|
439
|
+
const fields = record.get(/^(?<def>015|020|022|024|028)$/u);
|
|
439
440
|
const identifiers = [].concat(...fields.map(toIdentifiers));
|
|
440
441
|
const uniqueIdentifiers = [...new Set(identifiers)];
|
|
441
442
|
|
|
@@ -454,10 +455,10 @@ export function bibStandardIdentifiers(record) {
|
|
|
454
455
|
const issnIsbnReqExp = (/^[A-Za-z0-9-]+$/u);
|
|
455
456
|
const otherIdReqExp = (/^[A-Za-z0-9-:]+$/u);
|
|
456
457
|
|
|
457
|
-
if (tag === '
|
|
458
|
+
if (tag === '015') { // TODO: test (does this use the right index etc.)
|
|
458
459
|
return subfields
|
|
459
|
-
.filter(
|
|
460
|
-
.map(
|
|
460
|
+
.filter(sf => sf.code === 'a' && sf.value.match(/^[A-Z0-9\-]+$/ui))
|
|
461
|
+
.map(sf => sf.value)
|
|
461
462
|
}
|
|
462
463
|
|
|
463
464
|
if (tag === '020') {
|
|
@@ -466,6 +467,25 @@ export function bibStandardIdentifiers(record) {
|
|
|
466
467
|
.map(({value}) => String(value));
|
|
467
468
|
}
|
|
468
469
|
|
|
470
|
+
if (tag === '022') {
|
|
471
|
+
return subfields
|
|
472
|
+
.filter(sub => ['a', 'z', 'y'].includes(sub.code) && testStringOrNumber(sub.value) && issnIsbnReqExp.test(String(sub.value)))
|
|
473
|
+
.map(({value}) => String(value));
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
if (tag === '028') { // TODO: test
|
|
477
|
+
const a = subfields.find(sf => sf.code === 'a');
|
|
478
|
+
if (a) {
|
|
479
|
+
const b = subfields.find(sf => sf.code === 'b');
|
|
480
|
+
// TODO: normalize?
|
|
481
|
+
if (b) {
|
|
482
|
+
return [a.value, `${b.value} ${a.value}`];
|
|
483
|
+
}
|
|
484
|
+
return [a.value];
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// Default seems to be 024:
|
|
469
489
|
return subfields
|
|
470
490
|
.filter(sub => ['a', 'z'].includes(sub.code) && testStringOrNumber(sub.value) && otherIdReqExp.test(String(sub.value)))
|
|
471
491
|
.map(({value}) => String(value));
|
package/src/cli.js
CHANGED
|
@@ -197,6 +197,20 @@ export default () => ({
|
|
|
197
197
|
return 0;
|
|
198
198
|
}
|
|
199
199
|
|
|
200
|
+
// Do not give minus points if a normal publishing year is found in copyright years
|
|
201
|
+
const bNormalInACopyright = a.copyrightYears.filter(aValue => b.normalYears.some(bValue => aValue === bValue));
|
|
202
|
+
const aNormalInBCopyright= b.copyrightYears.filter(bValue => a.normalYears.some(aValue => bValue === aValue));
|
|
203
|
+
debug(`BNorm in ACopyright: ${JSON.stringify(bNormalInACopyright)}`);
|
|
204
|
+
debug(`ANorm in BCopyright: ${JSON.stringify(aNormalInBCopyright)}`);
|
|
205
|
+
|
|
206
|
+
if (bNormalInACopyright > 0 || aNormalInBCopyright > 0) {
|
|
207
|
+
return 0;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
|
|
200
214
|
return -1;
|
|
201
215
|
|
|
202
216
|
}
|
|
@@ -40,7 +40,10 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
40
40
|
// - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 'sää' as 'saa'
|
|
41
41
|
// - we could precompose the Finnish letters back to avoid this
|
|
42
42
|
// - see validator normalize-utf8-diacritics for details
|
|
43
|
-
.replace(/[^\p{Letter}\p{Number}&]/gu, '')
|
|
43
|
+
.replace(/[^\p{Letter}\p{Number}& ]/gu, '') // 2026-07-01: keep ' '. We need it for splitToBaseAndNumber()
|
|
44
|
+
.replace(/\s+/ug, ' ')
|
|
45
|
+
.replace(/ $/u, '')
|
|
46
|
+
.replace(/^ /u, '')
|
|
44
47
|
.toLowerCase();
|
|
45
48
|
}
|
|
46
49
|
|
|
@@ -52,11 +55,12 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
52
55
|
|
|
53
56
|
const [aa, ab, an, ap] = a;
|
|
54
57
|
const [ba, bb, bn, bp] = b;
|
|
58
|
+
debugData(`COMPARE:\n['${aa}', '${ab}', '${an}', '${ap}'] VS\n['${ba}', '${bb}', '${bn}', '${bp}']`);
|
|
55
59
|
|
|
56
60
|
const aFull = toFullTitle(a);
|
|
57
61
|
const bFull = toFullTitle(b);
|
|
58
62
|
|
|
59
|
-
|
|
63
|
+
// debugData(`JOINED COMPARE:\n '${aFull}' vs \n '${bFull}'`);
|
|
60
64
|
|
|
61
65
|
if (isEmpty(aa) || isEmpty(ba)) {
|
|
62
66
|
return -1.0;
|
|
@@ -66,6 +70,11 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
66
70
|
return TITLE_MAX;
|
|
67
71
|
}
|
|
68
72
|
|
|
73
|
+
// MELKEHITYS-3496-ish: move part of $a to $n:
|
|
74
|
+
const [altAa, altAn, altBa, altBn] = reconstructTitleAndNumber(aa, an, ba, bn);
|
|
75
|
+
if (altAa) {
|
|
76
|
+
return compare2([altAa, ab, altAn, ap], [altBa, bb, altBn, bp]);
|
|
77
|
+
}
|
|
69
78
|
|
|
70
79
|
if (ab && ab !== '' && ab === bb) { // Remove $b from equation (MELKEHITYS-3494)
|
|
71
80
|
debug(`Ignore \$b ${ab}`);
|
|
@@ -79,7 +88,7 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
79
88
|
debug(`Ignore \$p ${ap}`);
|
|
80
89
|
return compare2([aa, ab, an, ''], [ba, bb, bn, '']);
|
|
81
90
|
}
|
|
82
|
-
// There seems to be wobble between $b and $p
|
|
91
|
+
// There seems to be wobble between $b and $p:
|
|
83
92
|
if (!isEmpty(ab) && ab === bp) {
|
|
84
93
|
return compare2([aa, '', an, ap], [ba, bb, bn, '']);
|
|
85
94
|
}
|
|
@@ -103,7 +112,7 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
103
112
|
return -1.0;
|
|
104
113
|
}
|
|
105
114
|
|
|
106
|
-
const [distance, maxLength, correctness] = doLevenshtein(aFull, bFull);
|
|
115
|
+
const [distance, maxLength, correctness] = doLevenshtein(aFull.replace(/ /ug, ''), bFull.replace(/ /ug, ''));
|
|
107
116
|
|
|
108
117
|
debug(`'${aFull}' vs '${bFull}': Max length = ${maxLength}, distance = ${distance}, correctness = ${correctness}`);
|
|
109
118
|
|
|
@@ -122,16 +131,16 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
122
131
|
return compare2([aa, '', an, ap], [ba, bb.substring(ab.length), bn, bp]);
|
|
123
132
|
}
|
|
124
133
|
}
|
|
125
|
-
|
|
134
|
+
|
|
126
135
|
// Try the same without $p:
|
|
127
136
|
if (localXor(ap, bp)) {
|
|
128
137
|
const result = compare2([aa, ab, an, ''], [ba, bb, bn, '']);
|
|
129
138
|
return result > 0.0 ? result * 0.8 : result;
|
|
130
139
|
}
|
|
131
|
-
|
|
140
|
+
|
|
132
141
|
if (aa === ba && an === bn) {
|
|
133
|
-
// No $n: Don't score $a vs $ab. Return 0, and let other match detectors decide
|
|
134
142
|
const candScore = TITLE_MAX/2;
|
|
143
|
+
// $b is missing from one:
|
|
135
144
|
if (ap === bp && localXor(ab, bb)) {
|
|
136
145
|
debug(`Handle omitted \$b using x ${candScore}`);
|
|
137
146
|
return candScore;
|
|
@@ -142,6 +151,17 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
142
151
|
}
|
|
143
152
|
}
|
|
144
153
|
|
|
154
|
+
if (an === bn && ap === bp ) {
|
|
155
|
+
// $b-less $a is other record's $b, see MELKEHITYS-3485 for motivation (though we skip 246 and 490 here):
|
|
156
|
+
const candScore = TITLE_MAX/2;
|
|
157
|
+
if (aa === bb && !ab) {
|
|
158
|
+
return candScore;
|
|
159
|
+
}
|
|
160
|
+
if (ba === ab && !bb) {
|
|
161
|
+
return candScore;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
145
165
|
return -0.5; // Not likely
|
|
146
166
|
}
|
|
147
167
|
|
|
@@ -174,6 +194,41 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
174
194
|
return str1.length > str2.length ? str1.length : str2.length;
|
|
175
195
|
}
|
|
176
196
|
|
|
197
|
+
function reconstructTitleAndNumber(a1, n1, a2, n2) {
|
|
198
|
+
if (!n1 && n2) {
|
|
199
|
+
const [altA1, altN1] = splitToBaseAndNumber(a1);
|
|
200
|
+
if (altN1) {
|
|
201
|
+
debugData(`A: Reconstruct:\n $a '${a1}' =>\n $a '${altA1}' +\n $n '${altN1}`);
|
|
202
|
+
return [altA1, altN1, a2, n2];
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
if (n1 && !n2) {
|
|
206
|
+
const [altA2, altN2] = splitToBaseAndNumber(a2);
|
|
207
|
+
if (altN2) {
|
|
208
|
+
debugData(`B: Reconstruct $a '${a2}' as $a '${altA2}' and $n '${altN2}`);
|
|
209
|
+
return [a1, n1, altA2, altN2];
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
}
|
|
213
|
+
return [null, null, null, null]; // Fail
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
function splitToBaseAndNumber(val) {
|
|
218
|
+
const words = val.split(' ');
|
|
219
|
+
const len = words.length;
|
|
220
|
+
if (len < 2 || !words[len-1].match(/^[1-9][0-9]*$/)) {
|
|
221
|
+
return [val, null];
|
|
222
|
+
}
|
|
223
|
+
if (len > 2 && words[len-2].match(/^(del|osa|vol|volume)$/u)) { // NB! "vol." has lost it's '.' during normalization!
|
|
224
|
+
const number = `${words[len-2]} ${words[len-1]}`;
|
|
225
|
+
return [words.slice(0, len-2).join(' '), number];
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const number = words.pop();
|
|
229
|
+
return [words.join(' '), number];
|
|
230
|
+
}
|
|
231
|
+
|
|
177
232
|
}
|
|
178
233
|
});
|
|
179
234
|
|
package/src/matching-utils.js
CHANGED
|
@@ -129,3 +129,53 @@ export function stringBefore(string, substring) {
|
|
|
129
129
|
}
|
|
130
130
|
return string.substring(0, pos);
|
|
131
131
|
}
|
|
132
|
+
|
|
133
|
+
// We sort records here (put FIKKA and low-cased records first, ref MELINDA-3492)
|
|
134
|
+
// NB! this only sorts the records the current set. If the best stuff would come in a later set and this set contains good enough record, we'll never reach the best...
|
|
135
|
+
// Still this would alleviate the issue when maxMatches === 1...
|
|
136
|
+
export function preferenceSortRecords(x, y) {
|
|
137
|
+
debug(`sortRecords: RECORD COMPARISON\n${JSON.stringify(x)} vs\n${JSON.stringify(y)}`);
|
|
138
|
+
// Prefer the record with a FIKKA LOW tag:
|
|
139
|
+
const xFikka = hasLow(x.record, 'FIKKA');
|
|
140
|
+
const yFikka = hasLow(y.record, 'FIKKA');
|
|
141
|
+
|
|
142
|
+
if (xFikka && !yFikka) {
|
|
143
|
+
debug('sortRecords: prefer FIKKA');
|
|
144
|
+
return -1;
|
|
145
|
+
}
|
|
146
|
+
if (yFikka && !xFikka) {
|
|
147
|
+
return 1;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Prefer downcased:
|
|
151
|
+
const xAllCaps = hasAllCapsTitle(x.record);
|
|
152
|
+
const yAllCaps = hasAllCapsTitle(y.record);
|
|
153
|
+
|
|
154
|
+
if (!xAllCaps && yAllCaps) {
|
|
155
|
+
debug('sortRecords: prefer non-all caps');
|
|
156
|
+
return -1; // x is better
|
|
157
|
+
}
|
|
158
|
+
if (!yAllCaps && xAllCaps) {
|
|
159
|
+
return 1; // y is better
|
|
160
|
+
}
|
|
161
|
+
// Default:
|
|
162
|
+
return 0;
|
|
163
|
+
|
|
164
|
+
function hasLow(record, lowTag) {
|
|
165
|
+
return record.fields?.some(f => f.tag === 'LOW' && f.subfields?.some(sf => sf.code === 'a' && sf.value === lowTag));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function hasAllCapsTitle(record) {
|
|
169
|
+
const f245 = record.fields?.find(f => f.tag === '245');
|
|
170
|
+
if (!f245) {
|
|
171
|
+
debugData(`WARNING: No f245 found for ALL CAPS check`);
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const title = f245.subfields.filter(sf => ['a', 'b', 'c'].includes(sf.code)).map(sf => sf.value).join(' ');
|
|
176
|
+
if (title.match(/[A-Z]/u) && !title.match(/[a-z]/u)) {
|
|
177
|
+
return true;
|
|
178
|
+
}
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
@@ -4,11 +4,13 @@ import {READERS} from '@natlibfi/fixura';
|
|
|
4
4
|
import generateTests from '@natlibfi/fixugen-http-client';
|
|
5
5
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
6
6
|
import {Error as MatchingError} from '@natlibfi/melinda-commons';
|
|
7
|
-
import createSearchInterface, {CandidateSearchError} from '
|
|
7
|
+
import createSearchInterface, {CandidateSearchError} from '../../src/candidate-search/index.js';
|
|
8
8
|
import createDebugLogger from 'debug';
|
|
9
9
|
|
|
10
10
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:test');
|
|
11
11
|
|
|
12
|
+
// NOTE: it seems that "only" does not work when using generateTests from @natlibfi/fixugen-http-client'
|
|
13
|
+
|
|
12
14
|
describe('candidate-search', () => {
|
|
13
15
|
generateTests({
|
|
14
16
|
callback,
|
|
@@ -4,7 +4,7 @@ import createDebugLogger from 'debug';
|
|
|
4
4
|
import generateTests from '@natlibfi/fixugen';
|
|
5
5
|
import {READERS} from '@natlibfi/fixura';
|
|
6
6
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
7
|
-
import * as generators from '
|
|
7
|
+
import * as generators from '../../../src/candidate-search/query-list/auth.js';
|
|
8
8
|
|
|
9
9
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query-list:auth-test');
|
|
10
10
|
const debugData = debug.extend('data');
|
|
@@ -4,7 +4,7 @@ import createDebugLogger from 'debug';
|
|
|
4
4
|
import generateTests from '@natlibfi/fixugen';
|
|
5
5
|
import {READERS} from '@natlibfi/fixura';
|
|
6
6
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
7
|
-
import * as generators from '
|
|
7
|
+
import * as generators from '../../../src/candidate-search/query-list/bib.js';
|
|
8
8
|
|
|
9
9
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibHostComponents:test');
|
|
10
10
|
const debugData = debug.extend('data');
|
|
@@ -4,12 +4,14 @@ import createDebugLogger from 'debug';
|
|
|
4
4
|
import {READERS} from '@natlibfi/fixura';
|
|
5
5
|
import generateTests from '@natlibfi/fixugen-http-client';
|
|
6
6
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
7
|
-
import createMatchInterface, {matchDetection} from '
|
|
7
|
+
import createMatchInterface, {matchDetection} from '../src/index.js';
|
|
8
8
|
|
|
9
9
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:index:test');
|
|
10
10
|
const debugData = debug.extend('data');
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
// NOTE: it seems that "only" does not work when using generateTests from @natlibfi/fixugen-http-client'
|
|
13
|
+
|
|
14
|
+
describe('INDEX-FOO', () => {
|
|
13
15
|
generateTests({
|
|
14
16
|
callback,
|
|
15
17
|
path: [import.meta.dirname, '..', 'test-fixtures', 'index'],
|
|
@@ -21,12 +23,15 @@ describe('INDEX', () => {
|
|
|
21
23
|
|
|
22
24
|
async function callback({getFixture, options, expectedMatchStatus, expectedStopReason, expectedFailures, expectedCandidateCount}) {
|
|
23
25
|
|
|
26
|
+
debug(`Running tests from indezx.test.js`);
|
|
27
|
+
|
|
24
28
|
const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});
|
|
25
29
|
const expectedMatches = getFixture('expectedMatches.json');
|
|
26
30
|
const expectedNonMatches = getFixture('expectedNonMatches.json') || [];
|
|
27
31
|
|
|
28
|
-
|
|
29
|
-
|
|
32
|
+
debugData(`Options: ${JSON.stringify(options)}`);
|
|
33
|
+
debugData(`Formatted options: ${JSON.stringify(formatOptions(options))}`);
|
|
34
|
+
const match = await createMatchInterface(formatOptions(options));
|
|
30
35
|
const {matches, matchStatus, nonMatches, conversionFailures, candidateCount} = await match({record});
|
|
31
36
|
debugData(`Matches: ${matches.length}, Status: ${matchStatus.status}/${matchStatus.stopReason}, NonMatches: ${nonMatches ? nonMatches.length : 'not returned'}, ConversionFailures: ${conversionFailures ? conversionFailures.length : 'not returned'}`);
|
|
32
37
|
|
|
@@ -44,7 +49,7 @@ describe('INDEX', () => {
|
|
|
44
49
|
assert.deepStrictEqual(conversionFailures, expectedFailures);
|
|
45
50
|
}
|
|
46
51
|
|
|
47
|
-
function formatOptions() {
|
|
52
|
+
function formatOptions(options) {
|
|
48
53
|
const contextFeatures = matchDetection.features[options.detection.strategy.type];
|
|
49
54
|
|
|
50
55
|
return {
|
|
@@ -5,7 +5,7 @@ import createDebugLogger from 'debug';
|
|
|
5
5
|
import generateTests from '@natlibfi/fixugen';
|
|
6
6
|
import {READERS} from '@natlibfi/fixura';
|
|
7
7
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
8
|
-
import * as features from '
|
|
8
|
+
import * as features from '../../../../src/match-detection/features/auth/index.js';
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/auth:test');
|
|
@@ -5,7 +5,7 @@ import createDebugLogger from 'debug';
|
|
|
5
5
|
import generateTests from '@natlibfi/fixugen';
|
|
6
6
|
import {READERS} from '@natlibfi/fixura';
|
|
7
7
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
8
|
-
import * as features from '
|
|
8
|
+
import * as features from '../../../../src/match-detection/features/bib/index.js';
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib:test');
|
|
@@ -6,8 +6,8 @@ import generateTests from '@natlibfi/fixugen';
|
|
|
6
6
|
import {READERS} from '@natlibfi/fixura';
|
|
7
7
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
8
8
|
|
|
9
|
-
import createDetectionInterface from '
|
|
10
|
-
import * as features from '
|
|
9
|
+
import createDetectionInterface from '../../src/match-detection/index.js';
|
|
10
|
+
import * as features from '../../src/match-detection/features/index.js';
|
|
11
11
|
|
|
12
12
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:test');
|
|
13
13
|
const debugData = debug.extend('data');
|