@natlibfi/melinda-record-matching 5.0.5 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/candidate-search/index.js.map +2 -2
- package/dist/candidate-search/index.test.js +1 -4
- package/dist/candidate-search/index.test.js.map +2 -2
- package/dist/candidate-search/query-list/auth.js +21 -0
- package/dist/candidate-search/query-list/auth.js.map +7 -0
- package/dist/candidate-search/query-list/auth.test.js +31 -0
- package/dist/candidate-search/query-list/auth.test.js.map +7 -0
- package/dist/candidate-search/query-list/bib.js +1 -1
- package/dist/candidate-search/query-list/bib.js.map +2 -2
- package/dist/candidate-search/query-list/bib.test.js +1 -4
- package/dist/candidate-search/query-list/bib.test.js.map +2 -2
- package/dist/candidate-search/query-list/component.js +1 -1
- package/dist/candidate-search/query-list/component.js.map +2 -2
- package/dist/candidate-search/query-list/index.js +8 -1
- package/dist/candidate-search/query-list/index.js.map +2 -2
- package/dist/cli.js +11 -1
- package/dist/cli.js.map +2 -2
- package/dist/index.js.map +2 -2
- package/dist/index.test.js.map +2 -2
- package/dist/match-detection/features/auth/index.js +2 -0
- package/dist/match-detection/features/auth/index.js.map +7 -0
- package/dist/match-detection/features/auth/index.test.js +37 -0
- package/dist/match-detection/features/auth/index.test.js.map +7 -0
- package/dist/match-detection/features/auth/yso.js +43 -0
- package/dist/match-detection/features/auth/yso.js.map +7 -0
- package/dist/match-detection/features/bib/index.test.js +1 -4
- package/dist/match-detection/features/bib/index.test.js.map +2 -2
- package/dist/match-detection/features/bib/isbn.js +0 -1
- package/dist/match-detection/features/bib/isbn.js.map +2 -2
- package/dist/match-detection/features/bib/issn.js +0 -1
- package/dist/match-detection/features/bib/issn.js.map +2 -2
- package/dist/match-detection/features/bib/language.js +0 -1
- package/dist/match-detection/features/bib/language.js.map +2 -2
- package/dist/match-detection/features/bib/media-type.js.map +2 -2
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +0 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +2 -2
- package/dist/match-detection/features/bib/standard-identifier-factory.js.map +2 -2
- package/dist/match-detection/features/index.js +2 -1
- package/dist/match-detection/features/index.js.map +2 -2
- package/dist/match-detection/index.test.js +1 -5
- package/dist/match-detection/index.test.js.map +2 -2
- package/package.json +6 -3
- package/src/candidate-search/index.js +0 -2
- package/src/candidate-search/index.test.js +1 -6
- package/src/candidate-search/query-list/auth.js +25 -0
- package/src/candidate-search/query-list/auth.test.js +34 -0
- package/src/candidate-search/query-list/bib.js +1 -2
- package/src/candidate-search/query-list/bib.test.js +1 -5
- package/src/candidate-search/query-list/component.js +4 -4
- package/src/candidate-search/query-list/index.js +7 -1
- package/src/cli.js +13 -1
- package/src/index.js +0 -1
- package/src/index.test.js +0 -1
- package/src/match-detection/features/auth/index.js +1 -0
- package/src/match-detection/features/auth/index.test.js +47 -0
- package/src/match-detection/features/auth/yso.js +53 -0
- package/src/match-detection/features/bib/index.test.js +1 -6
- package/src/match-detection/features/bib/isbn.js +0 -1
- package/src/match-detection/features/bib/issn.js +0 -1
- package/src/match-detection/features/bib/language.js +0 -2
- package/src/match-detection/features/bib/media-type.js +0 -3
- package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +0 -1
- package/src/match-detection/features/bib/standard-identifier-factory.js +0 -2
- package/src/match-detection/features/index.js +3 -1
- package/src/match-detection/index.test.js +1 -7
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/match-detection/features/bib/language.js"],
|
|
4
|
-
"sourcesContent": ["\nimport createDebugLogger from 'debug';\n\nimport {MarcRecord} from '@natlibfi/marc-record';\n\nimport {FixSami041, Remove041zxx} from '@natlibfi/marc-record-validators-melinda';\n\nimport {getMatchCounts} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:language');\nconst debugData = debug.extend('data');\n\n// NB! 2025-12-17: I changed the logic drastically. However, I tried to keep the scores as same as possible.\n// 'extract' no longer extracts anything. Instead it clones the record, and does some preprocessing (using validators) instead.\n// 'compare' had multiple changes\n// - f041 ind1 differences ('0' vs '1') now result in a small penalty\n// - Two sami languages related changes:\n// -- validator adds a 'smi' subfield before a corresponding sma/sme/...subfield, if needed (national)\n// -- 'smi' only vs 'smi'+'sma' does not cause penalty\n// - 'mul' vs 'fin'+'swe' does not cause a penalty. However, 'mul' vs 'fin' alone triggers penalty.\n// - 008/35-37 and f041$a/$d are calculated separately\n// - a threshold is applied: return value is always between -1.0 and +0.1 (as it was before)\n// - we try to handle 041 $2 ISO 639-2 and ISO 639-3 as well.\n// - 'und' vs one other language does not cause a penalty in certain contexts. In other contexts it's treated as a normal \"language code\". (Tune/alleviate penalties later on.)\n\nexport default () => ({\n name: 'Language',\n extract: ({record, recordExternal}) => {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const clonedRecord = new MarcRecord(record, record._validationOptions); // clone(record); // NB! This loses record.get()...\n\n FixSami041().fix(clonedRecord); // Handle 'smi' adding if needed\n Remove041zxx().fix(clonedRecord); // Remove 'zxx' from f041s\n clonedRecord.fields = clonedRecord.fields.filter(f => ['008', '041'].includes(f.tag));\n // Add other language code normalizations?\n\n // Should we return all the fields, or just the relevant fields?\n return [clonedRecord.fields, label];\n },\n // eslint-disable-next-line max-statements\n compare: (aa, bb) => {\n const [a, aLabel] = aa;\n const [b, bLabel] = bb;\n debugData(`Comparing language ${JSON.stringify(a)} and ${JSON.stringify(b)}`);\n\n const score008 = compare008();\n const score041 = compare041();\n return applyLimits(score008 + score041);\n\n function applyLimits(score) {\n if (score > 0.1) { // Keeps the original max, we might have 0.10 (041) + 0.05 (008/35-37) = 0.15 now\n return 0.1;\n }\n return score;\n }\n\n function compare008() {\n const a008 = get008Value(a, aLabel);\n const b008 = get008Value(b, bLabel);\n // Something seriously wrong with 008:\n if (a008 === undefined || b008 === undefined) {\n return 0.0; // Punish for generic badness?\n }\n\n if (containsNoData(a008) || containsNoData(b008)){\n // Nothing to compare\n return 0.0;\n }\n\n if (a008 === b008) {\n return 0.05;\n }\n\n if (a008 === 'mul' || b008 === 'mul') { // We'll let 041 decide\n return 0.0;\n }\n\n return -0.2;\n }\n\n\n\n\n\n function compare041() {\n const a041s = getFields041(a);\n const b041s = getFields041(b);\n\n if (a041s.length === 0 || b041s.length === 0) {\n return 0.0; // Should we punish these for badness?\n }\n if (a041s.length === 1 && b041s.length === 1) {\n // The language codes are typically same between different sources. Incur a small penalty for differences though...\n const sourcePenalty = getSourceOfLanguageCode(a041s[0]) === getSourceOfLanguageCode(b041s[0]) ? 0.0 : -0.05;\n const mainPenalty = compareLanguageCodeLists(getFieldsLanguageCodes(a041s[0]), getFieldsLanguageCodes(b041s[0]));\n const scoreInd1 = mainPenalty === 0.0 ? 0.0 : indicator1Penalty(a041s[0], b041s[0]);\n return mainPenalty + sourcePenalty + scoreInd1;\n }\n // Things are already complicated (likely to pe penalized, so no nedd to worry about language code sources, I daresay.\n return compareLanguageCodeLists(getLanguageCodesFrom041Fields(a), getLanguageCodesFrom041Fields(b));\n\n function getFields041(fields) {\n if (!fields) {\n return [];\n }\n return fields.filter(f => f.tag === '041');\n }\n\n }\n\n function getSourceOfLanguageCode(field) {\n const sf2 = field.subfields.find(sf => sf.code === '2');\n if (!sf2) {\n return 'MARC';\n }\n // Normalize the two relevant language code names:\n if (sf2.value.match(/639.2/u)) {\n return 'ISO 639-2';\n }\n if (sf2.value.match(/639.3/u)) {\n return 'ISO 639-3';\n }\n return sf2.value; // We don't actually have anything else in Melinda though\n }\n\n function indicator1Penalty(aField, bField) {\n if (aField.ind1 === ' ' || bField.ind1 === ' '|| aField.ind1 === bField.ind1 ) {\n return 0.0;\n }\n debug(`\\t Indicator penalty: '${aField.ind1}' vs '${bField.ind1}'`);\n return -0.1;\n }\n\n\n\n function containsNoData(languageCode) {\n return [' ', '|||', 'und'].includes(languageCode);\n }\n\n function compareLanguageCodeLists(a, b) {\n if (a.length === 0 || b.length === 0 || a.every(val => containsNoData(val)) || b.every(val => containsNoData(val))) {\n debugData(`No language to compare`);\n return 0;\n }\n\n // If either one of the sets has only the generic 'smi', remove the specific sami languages codes from the other as well:\n const samiLanguageCodes = ['sma', 'sme', 'smj', 'smn', 'sms']; // NB! Don't put generic 'smi' here!\n\n if (a.includes('smi') && b.includes('smi')) {\n if (a.some(code => samiLanguageCodes.includes(code)) && !b.some(code => samiLanguageCodes.includes(code))) {\n return compareLanguageCodeLists(a.filter(val => !samiLanguageCodes.includes(val)), b); // recurse\n }\n if (b.some(code => samiLanguageCodes.includes(code)) && !a.some(code => samiLanguageCodes.includes(code))) {\n return compareLanguageCodeLists(a, b.filter(val => !samiLanguageCodes.includes(val))); // recurse\n }\n }\n\n\n\n\n if (a.length === b.length && a.every((element, index) => element === b[index])) {\n debugData(`All languages match`);\n return 0.1;\n }\n\n // Handle 'mul'. (Should we check whether the other array contains mul as well, probably not...)\n // (If 'mul' does not appear alone, then it is very iffy... )\n if (a.length === 1 && a[0] === 'mul' && b.length > 1) {\n return 0;\n }\n if (b.length === 1 && b[0] === 'mul' && a.length > 1) {\n return 0;\n }\n\n\n\n // Damage control:\n const sharedValues = getSharedValues(a, b).filter(val => !containsNoData(val));\n\n if (sharedValues.length < 1) {\n debug(`Both have languages, but none of these match.`); // includes 'mul' vs a single language code\n return -1.0;\n }\n\n const {matchingValues, possibleMatchValues, maxValues} = getMatchCounts(a, b);\n\n debug(`Both have languages, ${matchingValues}/${possibleMatchValues} valid languages match.`);\n // ignore non-matches if there is mismatching amount of values\n debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);\n // we give some kind of penalty for mismatching amount of values instead of simple divide?\n const missingCount = maxValues - possibleMatchValues;\n const misMatchCount = possibleMatchValues - matchingValues;\n debug(`\\t missing: ${missingCount}`);\n debug(`\\t mismatches: ${misMatchCount}`);\n\n const penaltyForMissing = 0.02 * (maxValues - possibleMatchValues);\n const penaltyForMisMatch = 0.05 * (possibleMatchValues - matchingValues);\n debug(`\\t points: penaltyForMissing: ${penaltyForMissing}`);\n debug(`\\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);\n\n const points = Number(Number(0.1 - penaltyForMisMatch - penaltyForMissing).toFixed(2));\n debug(`Total points: ${points}`);\n\n return points;\n }\n }\n\n});\n\nfunction get008Value(fields, label = 'record') {\n if (!fields) {\n return;\n }\n const f008 = fields.find(f => f.tag === '008');\n if (!f008) {\n return undefined;\n }\n const value = f008.value || defaultValue;\n\n if (!value) {\n debugData(`${label}: Failed to extact 008/35-37 value from '${value}'`);\n return defaultValue;\n }\n\n const code = value.slice(35, 38);\n debugData(`${label}: 008 code: '${code}'`);\n return code;\n}\n\n// Check if a string is a possible, validly formed language code for a single language\n// Currently accept also codes in capitals\nfunction isLangCodeForALanguage(code, label = 'record', encoding = undefined) {\n if (!code) {\n return false;\n }\n\n if ([undefined, 'ISO 639-2', 'ISO-639-3'].includes(encoding) && code.length !== 3) {\n debugData(`${label}: Code ${code} is not correct length (3) for a language code.`);\n return false;\n }\n if (!encoding) {\n // 'mul' should be passed as 'mul' can match 'fin' + 'swe' etc.\n // 'zxx' should be removed by a validator\n // '^^^' is Aleph-specific corruption, not supporting it anymore\n if (code === '|||' || code === ' ' ) { // || code === '^^^' || code === 'mul' || code === 'zxx') {\n debugData(`${label}: Code ${code} is not code for a spesific language.`);\n return false;\n }\n }\n // Marc, ISO 639-2 and ISO 639-3 are all three-letters long. Not other language codes seen in Melinda:\n const langCodePattern = /^[a-z][a-z][a-z]$/ui;\n if (!langCodePattern.test(code)) {\n debugData(`${label}: Code ${code} is not valid as a language code`);\n return false;\n }\n return true;\n}\n\n\n\nfunction getLanguageCodesFrom041Fields(fields) {\n // NB! We brutally don't check $2 (language code source) as marc's language codes is practically a subset of ISO 639-2,\n // and also ISO 639-2 and ISO 639-3 overlap to a degree. If we ever run into a trouble with some ISO 639-2 vs 639-3 mismatch, then we'll work it out.\n // Also we could write a validator that converts ISO 639-2 to marc if applicable and maybe even partial support for ISO-639-3.\n // Note that ISO 639-2-B values correspond with marc beteer that ISO 639-2-T. Eg. code for Chinese 'zho' should/code be normalized to 'chi'!\n const values = fields.filter(f => f.tag === '041').flatMap(f => getFieldsLanguageCodes(f));\n /*\n // .filter(({ind2}) => ind2 === ' ')\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code === 'a' || code === 'd')\n .filter(({value}) => value && isLangCodeForALanguage(value))\n .map(({value}) => value);\n */\n return [...new Set(values)].sort();\n}\n\nfunction getFieldsLanguageCodes(field) {\n const relevantSubfields = field.subfields.filter(sf => sf.code === 'a' || sf.code === 'd').filter(sf => isLangCodeForALanguage(sf.value, 'field'));\n\n const values = relevantSubfields.map(sf => sf.value).flat();\n\n return [...new Set(values)].sort();\n}\n\nfunction getSharedValues(list1, list2) {\n return list1.filter(val => list2.includes(val));\n}"],
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["import createDebugLogger from 'debug';\n\nimport {MarcRecord} from '@natlibfi/marc-record';\n\nimport {FixSami041, Remove041zxx} from '@natlibfi/marc-record-validators-melinda';\n\nimport {getMatchCounts} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:language');\nconst debugData = debug.extend('data');\n\n// NB! 2025-12-17: I changed the logic drastically. However, I tried to keep the scores as same as possible.\n// 'extract' no longer extracts anything. Instead it clones the record, and does some preprocessing (using validators) instead.\n// 'compare' had multiple changes\n// - f041 ind1 differences ('0' vs '1') now result in a small penalty\n// - Two sami languages related changes:\n// -- validator adds a 'smi' subfield before a corresponding sma/sme/...subfield, if needed (national)\n// -- 'smi' only vs 'smi'+'sma' does not cause penalty\n// - 'mul' vs 'fin'+'swe' does not cause a penalty. However, 'mul' vs 'fin' alone triggers penalty.\n// - 008/35-37 and f041$a/$d are calculated separately\n// - a threshold is applied: return value is always between -1.0 and +0.1 (as it was before)\n// - we try to handle 041 $2 ISO 639-2 and ISO 639-3 as well.\n// - 'und' vs one other language does not cause a penalty in certain contexts. In other contexts it's treated as a normal \"language code\". (Tune/alleviate penalties later on.)\n\nexport default () => ({\n name: 'Language',\n extract: ({record, recordExternal}) => {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const clonedRecord = new MarcRecord(record, record._validationOptions); // clone(record); // NB! This loses record.get()...\n\n FixSami041().fix(clonedRecord); // Handle 'smi' adding if needed\n Remove041zxx().fix(clonedRecord); // Remove 'zxx' from f041s\n clonedRecord.fields = clonedRecord.fields.filter(f => ['008', '041'].includes(f.tag));\n // Add other language code normalizations?\n\n // Should we return all the fields, or just the relevant fields?\n return [clonedRecord.fields, label];\n },\n compare: (aa, bb) => {\n const [a, aLabel] = aa;\n const [b, bLabel] = bb;\n debugData(`Comparing language ${JSON.stringify(a)} and ${JSON.stringify(b)}`);\n\n const score008 = compare008();\n const score041 = compare041();\n return applyLimits(score008 + score041);\n\n function applyLimits(score) {\n if (score > 0.1) { // Keeps the original max, we might have 0.10 (041) + 0.05 (008/35-37) = 0.15 now\n return 0.1;\n }\n return score;\n }\n\n function compare008() {\n const a008 = get008Value(a, aLabel);\n const b008 = get008Value(b, bLabel);\n // Something seriously wrong with 008:\n if (a008 === undefined || b008 === undefined) {\n return 0.0; // Punish for generic badness?\n }\n\n if (containsNoData(a008) || containsNoData(b008)){\n // Nothing to compare\n return 0.0;\n }\n\n if (a008 === b008) {\n return 0.05;\n }\n\n if (a008 === 'mul' || b008 === 'mul') { // We'll let 041 decide\n return 0.0;\n }\n\n return -0.2;\n }\n\n\n\n\n\n function compare041() {\n const a041s = getFields041(a);\n const b041s = getFields041(b);\n\n if (a041s.length === 0 || b041s.length === 0) {\n return 0.0; // Should we punish these for badness?\n }\n if (a041s.length === 1 && b041s.length === 1) {\n // The language codes are typically same between different sources. Incur a small penalty for differences though...\n const sourcePenalty = getSourceOfLanguageCode(a041s[0]) === getSourceOfLanguageCode(b041s[0]) ? 0.0 : -0.05;\n const mainPenalty = compareLanguageCodeLists(getFieldsLanguageCodes(a041s[0]), getFieldsLanguageCodes(b041s[0]));\n const scoreInd1 = mainPenalty === 0.0 ? 0.0 : indicator1Penalty(a041s[0], b041s[0]);\n return mainPenalty + sourcePenalty + scoreInd1;\n }\n // Things are already complicated (likely to pe penalized, so no nedd to worry about language code sources, I daresay.\n return compareLanguageCodeLists(getLanguageCodesFrom041Fields(a), getLanguageCodesFrom041Fields(b));\n\n function getFields041(fields) {\n if (!fields) {\n return [];\n }\n return fields.filter(f => f.tag === '041');\n }\n\n }\n\n function getSourceOfLanguageCode(field) {\n const sf2 = field.subfields.find(sf => sf.code === '2');\n if (!sf2) {\n return 'MARC';\n }\n // Normalize the two relevant language code names:\n if (sf2.value.match(/639.2/u)) {\n return 'ISO 639-2';\n }\n if (sf2.value.match(/639.3/u)) {\n return 'ISO 639-3';\n }\n return sf2.value; // We don't actually have anything else in Melinda though\n }\n\n function indicator1Penalty(aField, bField) {\n if (aField.ind1 === ' ' || bField.ind1 === ' '|| aField.ind1 === bField.ind1 ) {\n return 0.0;\n }\n debug(`\\t Indicator penalty: '${aField.ind1}' vs '${bField.ind1}'`);\n return -0.1;\n }\n\n\n\n function containsNoData(languageCode) {\n return [' ', '|||', 'und'].includes(languageCode);\n }\n\n function compareLanguageCodeLists(a, b) {\n if (a.length === 0 || b.length === 0 || a.every(val => containsNoData(val)) || b.every(val => containsNoData(val))) {\n debugData(`No language to compare`);\n return 0;\n }\n\n // If either one of the sets has only the generic 'smi', remove the specific sami languages codes from the other as well:\n const samiLanguageCodes = ['sma', 'sme', 'smj', 'smn', 'sms']; // NB! Don't put generic 'smi' here!\n\n if (a.includes('smi') && b.includes('smi')) {\n if (a.some(code => samiLanguageCodes.includes(code)) && !b.some(code => samiLanguageCodes.includes(code))) {\n return compareLanguageCodeLists(a.filter(val => !samiLanguageCodes.includes(val)), b); // recurse\n }\n if (b.some(code => samiLanguageCodes.includes(code)) && !a.some(code => samiLanguageCodes.includes(code))) {\n return compareLanguageCodeLists(a, b.filter(val => !samiLanguageCodes.includes(val))); // recurse\n }\n }\n\n\n\n\n if (a.length === b.length && a.every((element, index) => element === b[index])) {\n debugData(`All languages match`);\n return 0.1;\n }\n\n // Handle 'mul'. (Should we check whether the other array contains mul as well, probably not...)\n // (If 'mul' does not appear alone, then it is very iffy... )\n if (a.length === 1 && a[0] === 'mul' && b.length > 1) {\n return 0;\n }\n if (b.length === 1 && b[0] === 'mul' && a.length > 1) {\n return 0;\n }\n\n\n\n // Damage control:\n const sharedValues = getSharedValues(a, b).filter(val => !containsNoData(val));\n\n if (sharedValues.length < 1) {\n debug(`Both have languages, but none of these match.`); // includes 'mul' vs a single language code\n return -1.0;\n }\n\n const {matchingValues, possibleMatchValues, maxValues} = getMatchCounts(a, b);\n\n debug(`Both have languages, ${matchingValues}/${possibleMatchValues} valid languages match.`);\n // ignore non-matches if there is mismatching amount of values\n debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);\n // we give some kind of penalty for mismatching amount of values instead of simple divide?\n const missingCount = maxValues - possibleMatchValues;\n const misMatchCount = possibleMatchValues - matchingValues;\n debug(`\\t missing: ${missingCount}`);\n debug(`\\t mismatches: ${misMatchCount}`);\n\n const penaltyForMissing = 0.02 * (maxValues - possibleMatchValues);\n const penaltyForMisMatch = 0.05 * (possibleMatchValues - matchingValues);\n debug(`\\t points: penaltyForMissing: ${penaltyForMissing}`);\n debug(`\\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);\n\n const points = Number(Number(0.1 - penaltyForMisMatch - penaltyForMissing).toFixed(2));\n debug(`Total points: ${points}`);\n\n return points;\n }\n }\n\n});\n\nfunction get008Value(fields, label = 'record') {\n if (!fields) {\n return;\n }\n const f008 = fields.find(f => f.tag === '008');\n if (!f008) {\n return undefined;\n }\n const value = f008.value || defaultValue;\n\n if (!value) {\n debugData(`${label}: Failed to extact 008/35-37 value from '${value}'`);\n return defaultValue;\n }\n\n const code = value.slice(35, 38);\n debugData(`${label}: 008 code: '${code}'`);\n return code;\n}\n\n// Check if a string is a possible, validly formed language code for a single language\n// Currently accept also codes in capitals\nfunction isLangCodeForALanguage(code, label = 'record', encoding = undefined) {\n if (!code) {\n return false;\n }\n\n if ([undefined, 'ISO 639-2', 'ISO-639-3'].includes(encoding) && code.length !== 3) {\n debugData(`${label}: Code ${code} is not correct length (3) for a language code.`);\n return false;\n }\n if (!encoding) {\n // 'mul' should be passed as 'mul' can match 'fin' + 'swe' etc.\n // 'zxx' should be removed by a validator\n // '^^^' is Aleph-specific corruption, not supporting it anymore\n if (code === '|||' || code === ' ' ) { // || code === '^^^' || code === 'mul' || code === 'zxx') {\n debugData(`${label}: Code ${code} is not code for a spesific language.`);\n return false;\n }\n }\n // Marc, ISO 639-2 and ISO 639-3 are all three-letters long. Not other language codes seen in Melinda:\n const langCodePattern = /^[a-z][a-z][a-z]$/ui;\n if (!langCodePattern.test(code)) {\n debugData(`${label}: Code ${code} is not valid as a language code`);\n return false;\n }\n return true;\n}\n\n\n\nfunction getLanguageCodesFrom041Fields(fields) {\n // NB! We brutally don't check $2 (language code source) as marc's language codes is practically a subset of ISO 639-2,\n // and also ISO 639-2 and ISO 639-3 overlap to a degree. If we ever run into a trouble with some ISO 639-2 vs 639-3 mismatch, then we'll work it out.\n // Also we could write a validator that converts ISO 639-2 to marc if applicable and maybe even partial support for ISO-639-3.\n // Note that ISO 639-2-B values correspond with marc beteer that ISO 639-2-T. Eg. code for Chinese 'zho' should/code be normalized to 'chi'!\n const values = fields.filter(f => f.tag === '041').flatMap(f => getFieldsLanguageCodes(f));\n /*\n // .filter(({ind2}) => ind2 === ' ')\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code === 'a' || code === 'd')\n .filter(({value}) => value && isLangCodeForALanguage(value))\n .map(({value}) => value);\n */\n return [...new Set(values)].sort();\n}\n\nfunction getFieldsLanguageCodes(field) {\n const relevantSubfields = field.subfields.filter(sf => sf.code === 'a' || sf.code === 'd').filter(sf => isLangCodeForALanguage(sf.value, 'field'));\n\n const values = relevantSubfields.map(sf => sf.value).flat();\n\n return [...new Set(values)].sort();\n}\n\nfunction getSharedValues(list1, list2) {\n return list1.filter(val => list2.includes(val));\n}"],
|
|
5
|
+
"mappings": "AAAA,OAAO,uBAAuB;AAE9B,SAAQ,kBAAiB;AAEzB,SAAQ,YAAY,oBAAmB;AAEvC,SAAQ,sBAAqB;AAE7B,MAAM,QAAQ,kBAAkB,qEAAqE;AACrG,MAAM,YAAY,MAAM,OAAO,MAAM;AAerC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,eAAe,IAAI,WAAW,QAAQ,OAAO,kBAAkB;AAErE,eAAW,EAAE,IAAI,YAAY;AAC7B,iBAAa,EAAE,IAAI,YAAY;AAC/B,iBAAa,SAAS,aAAa,OAAO,OAAO,OAAK,CAAC,OAAO,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC;AAIpF,WAAO,CAAC,aAAa,QAAQ,KAAK;AAAA,EACpC;AAAA,EACA,SAAS,CAAC,IAAI,OAAO;AACnB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,cAAU,sBAAsB,KAAK,UAAU,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,EAAE;AAE5E,UAAM,WAAW,WAAW;AAC5B,UAAM,WAAW,WAAW;AAC5B,WAAO,YAAY,WAAW,QAAQ;AAEtC,aAAS,YAAY,OAAO;AAC1B,UAAI,QAAQ,KAAK;AACf,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT;AAEA,aAAS,aAAa;AACpB,YAAM,OAAO,YAAY,GAAG,MAAM;AAClC,YAAM,OAAO,YAAY,GAAG,MAAM;AAElC,UAAI,SAAS,UAAa,SAAS,QAAW;AAC5C,eAAO;AAAA,MACT;AAEA,UAAI,eAAe,IAAI,KAAK,eAAe,IAAI,GAAE;AAE/C,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,MAAM;AACjB,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,SAAS,SAAS,OAAO;AACpC,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT;AAMA,aAAS,aAAa;AACpB,YAAM,QAAQ,aAAa,CAAC;AAC5B,YAAM,QAAQ,aAAa,CAAC;AAE5B,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAC5C,eAAO;AAAA,MACT;AACA,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAE5C,cAAM,gBAAgB,wBAAwB,MAAM,CAAC,CAAC,MAAM,wBAAwB,MAAM,CAAC,CAAC,IAAI,IAAM;AACtG,cAAM,cAAc,yBAAyB,uBAAuB,MAAM,CAAC,CAAC,GAAG,uBAAuB,MAAM,CAAC,CAAC,CAAC;AAC/G,cAAM,YAAY,gBAAgB,IAAM,IAAM,kBAAkB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC;AAClF,eAAO,cAAc,gBAAgB;AAAA,MACvC;AAEA,aAAO,yBAAyB,8BAA8B,CAAC,GAAG,8BAA8B,CAAC,CAAC;AAElG,eAAS,aAAa,QAAQ;AAC5B,YAAI,CAAC,QAAQ;AACX,iBAAO,CAAC;AAAA,QACV;AACA,eAAO,OAAO,OAAO,OAAK,EAAE,QAAQ,KAAK;AAAA,MAC3C;AAAA,IAEF;AAEA,aAAS,wBAAwB,OAAO;AACtC,YAAM,MAAM,MAAM,UAAU,KAAK,QAAM,GAAG,SAAS,GAAG;AACtD,UAAI,CAAC,KAAK;AACR,eAAO;AAAA,MACT;AAEA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,aAAO,IAAI;AAAA,IACb;AAEA,aAAS,kBAAkB,QAAQ,QAAQ;AACzC,UAAI,OAAO,SAAS,OAAO,OAAO,SAAS,OAAM,OAAO,SAAS,OAAO,MAAO;AAC7E,eAAO;AAAA,MACT;AACA,YAAM,yBAA0B,OAAO,IAAI,SAAS,OAAO,IAAI,GAAG;AAClE,aAAO;AAAA,IACT;AAIA,aAAS,eAAe,cAAc;AACpC,aAAO,CAAC,OAAO,OAAO,KAAK,EAAE,SAAS,YAAY;AAAA,IACpD;AAEA,aAAS,yBAAyBA,IAAGC,IAAG;AACtC,UAAID,GAAE,WAAW,KAAKC,GAAE,WAAW,KAAKD,GAAE,MAAM,SAAO,eAAe,GAAG,CAAC,KAAKC,GAAE,MAAM,SAAO,eAAe,GAAG,CAAC,GAAG;AAClH,kBAAU,wBAAwB;AAClC,eAAO;AAAA,MACT;AAGA,YAAM,oBAAoB,CAAC,OAAO,OAAO,OAAO,OAAO,KAAK;AAE5D,UAAID,GAAE,SAAS,KAAK,KAAKC,GAAE,SAAS,KAAK,GAAG;AAC1C,YAAID,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,KAAK,CAACC,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,GAAG;AACzG,iBAAO,yBAAyBD,GAAE,OAAO,SAAO,CAAC,kBAAkB,SAAS,GAAG,CAAC,GAAGC,EAAC;AAAA,QACtF;AACA,YAAIA,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,KAAK,CAACD,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,GAAG;AACzG,iBAAO,yBAAyBA,IAAGC,GAAE,OAAO,SAAO,CAAC,kBAAkB,SAAS,GAAG,CAAC,CAAC;AAAA,QACtF;AAAA,MACF;AAKA,UAAID,GAAE,WAAWC,GAAE,UAAUD,GAAE,MAAM,CAAC,SAAS,UAAU,YAAYC,GAAE,KAAK,CAAC,GAAG;AAC9E,kBAAU,qBAAqB;AAC/B,eAAO;AAAA,MACT;AAIA,UAAID,GAAE,WAAW,KAAKA,GAAE,CAAC,MAAM,SAASC,GAAE,SAAS,GAAG;AACpD,eAAO;AAAA,MACT;AACA,UAAIA,GAAE,WAAW,KAAKA,GAAE,CAAC,MAAM,SAASD,GAAE,SAAS,GAAG;AACpD,eAAO;AAAA,MACT;AAKA,YAAM,eAAe,gBAAgBA,IAAGC,EAAC,EAAE,OAAO,SAAO,CAAC,eAAe,GAAG,CAAC;AAE7E,UAAI,aAAa,SAAS,GAAG;AAC3B,cAAM,+CAA+C;AACrD,eAAO;AAAA,MACT;AAEA,YAAM,EAAC,gBAAgB,qBAAqB,UAAS,IAAI,eAAeD,IAAGC,EAAC;AAE5E,YAAM,wBAAwB,cAAc,IAAI,mBAAmB,yBAAyB;AAE5F,YAAM,qBAAqB,mBAAmB,IAAI,SAAS,EAAE;AAE7D,YAAM,eAAe,YAAY;AACjC,YAAM,gBAAgB,sBAAsB;AAC5C,YAAM,cAAe,YAAY,EAAE;AACnC,YAAM,iBAAkB,aAAa,EAAE;AAEvC,YAAM,oBAAoB,QAAQ,YAAY;AAC9C,YAAM,qBAAqB,QAAQ,sBAAsB;AACzD,YAAM,gCAAiC,iBAAiB,EAAE;AAC1D,YAAM,iCAAkC,kBAAkB,EAAE;AAE5D,YAAM,SAAS,OAAO,OAAO,MAAM,qBAAqB,iBAAiB,EAAE,QAAQ,CAAC,CAAC;AACrF,YAAM,iBAAiB,MAAM,EAAE;AAE/B,aAAO;AAAA,IACT;AAAA,EACF;AAEF;AAEA,SAAS,YAAY,QAAQ,QAAQ,UAAU;AAC7C,MAAI,CAAC,QAAQ;AACX;AAAA,EACF;AACA,QAAM,OAAO,OAAO,KAAK,OAAK,EAAE,QAAQ,KAAK;AAC7C,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,KAAK,SAAS;AAE5B,MAAI,CAAC,OAAO;AACV,cAAU,GAAG,KAAK,4CAA4C,KAAK,GAAG;AACtE,WAAO;AAAA,EACT;AAEA,QAAM,OAAO,MAAM,MAAM,IAAI,EAAE;AAC/B,YAAU,GAAG,KAAK,gBAAgB,IAAI,GAAG;AACzC,SAAO;AACT;AAIA,SAAS,uBAAuB,MAAM,QAAQ,UAAU,WAAW,QAAW;AAC5E,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AAEA,MAAI,CAAC,QAAW,aAAa,WAAW,EAAE,SAAS,QAAQ,KAAK,KAAK,WAAW,GAAG;AACjF,cAAU,GAAG,KAAK,UAAU,IAAI,iDAAiD;AACjF,WAAO;AAAA,EACT;AACA,MAAI,CAAC,UAAU;AAIb,QAAI,SAAS,SAAS,SAAS,OAAQ;AACrC,gBAAU,GAAG,KAAK,UAAU,IAAI,uCAAuC;AACvE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,kBAAkB;AACxB,MAAI,CAAC,gBAAgB,KAAK,IAAI,GAAG;AAC/B,cAAU,GAAG,KAAK,UAAU,IAAI,kCAAkC;AAClE,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAIA,SAAS,8BAA8B,QAAQ;AAK7C,QAAM,SAAS,OAAO,OAAO,OAAK,EAAE,QAAQ,KAAK,EAAE,QAAQ,OAAK,uBAAuB,CAAC,CAAC;AASzF,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,uBAAuB,OAAO;AACrC,QAAM,oBAAoB,MAAM,UAAU,OAAO,QAAM,GAAG,SAAS,OAAO,GAAG,SAAS,GAAG,EAAE,OAAO,QAAM,uBAAuB,GAAG,OAAO,OAAO,CAAC;AAEjJ,QAAM,SAAS,kBAAkB,IAAI,QAAM,GAAG,KAAK,EAAE,KAAK;AAE1D,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,gBAAgB,OAAO,OAAO;AACrC,SAAO,MAAM,OAAO,SAAO,MAAM,SAAS,GAAG,CAAC;AAChD;",
|
|
6
6
|
"names": ["a", "b"]
|
|
7
7
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/match-detection/features/bib/media-type.js"],
|
|
4
|
-
"sourcesContent": ["
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["import createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:media-type');\nconst debugData = debug.extend('data');\n\nexport default () => ({\n name: 'Media type',\n extract: ({record, recordExternal}) => {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n debugData(`Record (${label}): ${JSON.stringify(record)}`);\n debugData(`RecordExternal: ${JSON.stringify(recordExternal)}`);\n const values337 = get337Values();\n debug(`${label}: 337 $b values: ${JSON.stringify(values337)}`);\n\n return values337;\n\n function get337Values() {\n return record.get(/^337$/u)\n .filter(f => f.subfields.some((subfield) => subfield.code === '2' && subfield.value === 'rdamedia'))\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code === 'b')\n .map(({value}) => value);\n }\n },\n compare: (a, b) => {\n debugData(`Comparing ${JSON.stringify(a)} and ${JSON.stringify(b)}`);\n\n // Should we give extra good points if all mediaTypes match?\n // Should we give partial points for partially matching mediaTypes?\n // Should we check whether recordType is 'mixedMaterials'\n // Should we okay typical cases of not totally matching mediaTypes? What would these be?\n\n if (a.every(elem => b.includes(elem))) {\n debug(`All mediaTypes from A are in B`);\n return 1;\n }\n\n if (b.every(elem => a.includes(elem))) {\n debug(`All mediaTypes from B are in A`);\n return 1;\n }\n\n debug(`Mismatch in mediaTypes between A and B`);\n return -1;\n\n }\n});\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,uBAAuB;AAE9B,MAAM,QAAQ,kBAAkB,uEAAuE;AACvG,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,cAAU,WAAW,KAAK,MAAM,KAAK,UAAU,MAAM,CAAC,EAAE;AACxD,cAAU,mBAAmB,KAAK,UAAU,cAAc,CAAC,EAAE;AAC7D,UAAM,YAAY,aAAa;AAC/B,UAAM,GAAG,KAAK,oBAAoB,KAAK,UAAU,SAAS,CAAC,EAAE;AAE7D,WAAO;AAEP,aAAS,eAAe;AACtB,aAAO,OAAO,IAAI,QAAQ,EACvB,OAAO,OAAK,EAAE,UAAU,KAAK,CAAC,aAAa,SAAS,SAAS,OAAO,SAAS,UAAU,UAAU,CAAC,EAClG,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,GAAG,EAC/B,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK;AAAA,IAC3B;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,cAAU,aAAa,KAAK,UAAU,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,EAAE;AAOnE,QAAI,EAAE,MAAM,UAAQ,EAAE,SAAS,IAAI,CAAC,GAAG;AACrC,YAAM,gCAAgC;AACtC,aAAO;AAAA,IACT;AAEA,QAAI,EAAE,MAAM,UAAQ,EAAE,SAAS,IAAI,CAAC,GAAG;AACrC,YAAM,gCAAgC;AACtC,aAAO;AAAA,IACT;AAEA,UAAM,wCAAwC;AAC9C,WAAO;AAAA,EAET;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -78,7 +78,6 @@ export default () => ({
|
|
|
78
78
|
return uniqYears;
|
|
79
79
|
}
|
|
80
80
|
},
|
|
81
|
-
// eslint-disable-next-line max-statements
|
|
82
81
|
compare: (a, b) => {
|
|
83
82
|
const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years-multi");
|
|
84
83
|
debug(`Comparing ${JSON.stringify(a)} to ${JSON.stringify(b)}`);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/match-detection/features/bib/publication-time-allow-cons-years-multi.js"],
|
|
4
|
-
"sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\n// We should also get copyright time and copyright/publication times from 26x\n// We should also get publishing time type from f008\n// We should get reprint times from f500 $a \"Lis\u00E4painos/Lis\u00E4painokset:\"\n\nexport default () => ({\n name: 'Publication time, allow consequent years, years from multiple sources',\n extract: ({record, recordExternal}) => {\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years-multi');\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n\n const f008Values = extractF008Values(record);\n debug(`${label}: f008: ${JSON.stringify(f008Values)}`);\n\n const f26xValues = extractF26xValues(record);\n debug(`${label}: f26x: ${JSON.stringify(f26xValues)}`);\n\n const f500Values = extractF500Years(record);\n debug(`${label}: f500: ${JSON.stringify(f500Values)}`);\n\n // We should get copyrightYear from f008Date2 to copyrightYears when f008YearType = 'r'\n // Is the original year (f008Date2) in f008YearType === 'r' comparable to copyrightYear?\n // We should handle unknown years (here or comparison?)\n // We should handle year ranges for continuing resources / collections\n\n const normalYears = [...new Set(f26xValues.normalYears.concat(f008Values.f008Date1).filter(value => value && value !== ' ' && value !== '||||'))].sort();\n const copyrightYears = [...new Set(f26xValues.copyrightYears)].sort();\n const reprintYears = [...new Set(f500Values)].sort();\n\n const combined = {normalYears, copyrightYears, reprintYears};\n\n debug(`Combined: ${JSON.stringify(combined)}`);\n\n return combined;\n\n function extractF008Values(record) {\n // Record should have only one f008 - in case of several, we handle the first one\n const value = record.get(/^008$/u)?.[0]?.value || undefined;\n if (value && testStringOrNumber(value)) {\n const f008Date1 = extractF008Date1(value);\n const f008Date2 = extractF008Date2(value);\n const f008YearType = extractF008YearType(value);\n return {f008Date1, f008Date2, f008YearType};\n }\n return {f008Date1: undefined, f008Date2: undefined, f008YearType: undefined};\n\n function extractF008Date1(value) {\n return String(value).slice(7, 11);\n }\n\n function extractF008Date2(value) {\n return String(value).slice(11, 15);\n }\n\n function extractF008YearType(value) {\n return String(value).slice(6, 7);\n }\n }\n\n function extractF26xValues(record) {\n const copyrightRegex = /^(?<copyrightPrefix>cop|cop.|c|\u00A9|p|\u2117)/u;\n\n const pubNormalSubFieldValues = extractPubNormalSubfieldValues(record, copyrightRegex);\n debug(`Normal years: ${JSON.stringify(pubNormalSubFieldValues)}`);\n\n const pubF264CopySubFieldValues = extractPubF264CopySubfieldValues(record);\n debug(`F264 copyright years: ${JSON.stringify(pubF264CopySubFieldValues)}`);\n\n const pubF260CopySubFieldValues = extractPubF260CopySubfieldValues(record, copyrightRegex);\n debug(`F260 copyright years: ${JSON.stringify(pubF260CopySubFieldValues)}`);\n\n return {normalYears: pubNormalSubFieldValues, copyrightYears: [...pubF260CopySubFieldValues, ...pubF264CopySubFieldValues]};\n\n function extractPubNormalSubfieldValues(record, copyrightRegex) {\n return record.get(/^26[04]$/u)\n .filter((field) => !(field.tag === '264' && field.ind2 === '4'))\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code && code === 'c')\n .filter(({value}) => value && !copyrightRegex.test(value))\n .map(({value}) => value)\n .map((value) => removeNonAlphaNumeric(value));\n }\n\n function extractPubF264CopySubfieldValues(record, copyrightRegex) {\n return record.get(/^264$/u)\n .filter((field) => field.ind2 === '4')\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code && code === 'c')\n .filter(({value}) => value)\n .map(({value}) => value)\n .map((value) => value.replace(copyrightRegex, ''))\n .map((value) => removeNonAlphaNumeric(value));\n }\n\n function extractPubF260CopySubfieldValues(record, copyrightRegex) {\n return record.get(/^260$/u)\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code && code === 'c')\n .filter(({value}) => value && copyrightRegex.test(value))\n .map(({value}) => value)\n .map((value) => value.replace(copyrightRegex, ''))\n .map((value) => removeNonAlphaNumeric(value));\n }\n\n function removeNonAlphaNumeric(value) {\n debug(`Cleaning: ${JSON.stringify(value)}`);\n const nonAlphaNumericRegex = /[^A-Za-z0-9]/ug;\n return value ? value.replace(nonAlphaNumericRegex, '') : value;\n }\n }\n\n function extractF500Years(record) {\n\n const reprintRegex = /(?<reprint>Lis\u00E4painokset:|Lis\u00E4painos:)/u;\n const reprintFieldContents = record.get(/^500$/u)\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code === 'a')\n .map(({value}) => value);\n //.filter(value => value.test(reprintRegex));\n\n debug(`f500 reprint field contents: ${JSON.stringify(reprintFieldContents)}`);\n\n const filteredF500 = reprintFieldContents.filter((content) => content && content.match(reprintRegex));\n\n debug(`f500 reprint field contents (filtered): ${JSON.stringify(filteredF500)}`);\n\n const reprintYears = extractReprintYears(filteredF500);\n\n return reprintYears;\n }\n\n function extractReprintYears(contents) {\n const yearRegex = /[0-9][0-9][0-9][0-9]/gu;\n const years = contents.map(content => content.match(yearRegex))\n .flat();\n debug(`${JSON.stringify(years)}`);\n const uniqYears = [...new Set(years)].sort();\n debug(`${JSON.stringify(uniqYears)}`);\n return uniqYears;\n }\n\n },\n
|
|
5
|
-
"mappings": "AACA,OAAO,uBAAuB;AAC9B,SAAQ,0BAAyB;AAMjC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,wGAAwG;AACxI,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAE9E,UAAM,aAAa,kBAAkB,MAAM;AAC3C,UAAM,GAAG,KAAK,WAAW,KAAK,UAAU,UAAU,CAAC,EAAE;AAErD,UAAM,aAAa,kBAAkB,MAAM;AAC3C,UAAM,GAAG,KAAK,WAAW,KAAK,UAAU,UAAU,CAAC,EAAE;AAErD,UAAM,aAAa,iBAAiB,MAAM;AAC1C,UAAM,GAAG,KAAK,WAAW,KAAK,UAAU,UAAU,CAAC,EAAE;AAOrD,UAAM,cAAc,CAAC,GAAG,IAAI,IAAI,WAAW,YAAY,OAAO,WAAW,SAAS,EAAE,OAAO,WAAS,SAAS,UAAU,UAAU,UAAU,MAAM,CAAC,CAAC,EAAE,KAAK;AAC1J,UAAM,iBAAiB,CAAC,GAAG,IAAI,IAAI,WAAW,cAAc,CAAC,EAAE,KAAK;AACpE,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,KAAK;AAEnD,UAAM,WAAW,EAAC,aAAa,gBAAgB,aAAY;AAE3D,UAAM,aAAa,KAAK,UAAU,QAAQ,CAAC,EAAE;AAE7C,WAAO;AAEP,aAAS,kBAAkBA,SAAQ;AAEjC,YAAM,QAAQA,QAAO,IAAI,QAAQ,IAAI,CAAC,GAAG,SAAS;AAClD,UAAI,SAAS,mBAAmB,KAAK,GAAG;AACtC,cAAM,YAAY,iBAAiB,KAAK;AACxC,cAAM,YAAY,iBAAiB,KAAK;AACxC,cAAM,eAAe,oBAAoB,KAAK;AAC9C,eAAO,EAAC,WAAW,WAAW,aAAY;AAAA,MAC5C;AACA,aAAO,EAAC,WAAW,QAAW,WAAW,QAAW,cAAc,OAAS;AAE3E,eAAS,iBAAiBC,QAAO;AAC/B,eAAO,OAAOA,MAAK,EAAE,MAAM,GAAG,EAAE;AAAA,MAClC;AAEA,eAAS,iBAAiBA,QAAO;AAC/B,eAAO,OAAOA,MAAK,EAAE,MAAM,IAAI,EAAE;AAAA,MACnC;AAEA,eAAS,oBAAoBA,QAAO;AAClC,eAAO,OAAOA,MAAK,EAAE,MAAM,GAAG,CAAC;AAAA,MACjC;AAAA,IACF;AAEA,aAAS,kBAAkBD,SAAQ;AACjC,YAAM,iBAAiB;AAEvB,YAAM,0BAA0B,+BAA+BA,SAAQ,cAAc;AACrF,YAAM,iBAAiB,KAAK,UAAU,uBAAuB,CAAC,EAAE;AAEhE,YAAM,4BAA4B,iCAAiCA,OAAM;AACzE,YAAM,yBAAyB,KAAK,UAAU,yBAAyB,CAAC,EAAE;AAE1E,YAAM,4BAA4B,iCAAiCA,SAAQ,cAAc;AACzF,YAAM,yBAAyB,KAAK,UAAU,yBAAyB,CAAC,EAAE;AAE1E,aAAO,EAAC,aAAa,yBAAyB,gBAAgB,CAAC,GAAG,2BAA2B,GAAG,yBAAyB,EAAC;AAE1H,eAAS,+BAA+BA,SAAQE,iBAAgB;AAC9D,eAAOF,QAAO,IAAI,WAAW,EAC1B,OAAO,CAAC,UAAU,EAAE,MAAM,QAAQ,SAAS,MAAM,SAAS,IAAI,EAC9D,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,QAAQ,SAAS,GAAG,EACvC,OAAO,CAAC,EAAC,MAAK,MAAM,SAAS,CAACE,gBAAe,KAAK,KAAK,CAAC,EACxD,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,IAAI,CAAC,UAAU,sBAAsB,KAAK,CAAC;AAAA,MAChD;AAEA,eAAS,iCAAiCF,SAAQE,iBAAgB;AAChE,eAAOF,QAAO,IAAI,QAAQ,EACvB,OAAO,CAAC,UAAU,MAAM,SAAS,GAAG,EACpC,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,QAAQ,SAAS,GAAG,EACvC,OAAO,CAAC,EAAC,MAAK,MAAM,KAAK,EACzB,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,IAAI,CAAC,UAAU,MAAM,QAAQE,iBAAgB,EAAE,CAAC,EAChD,IAAI,CAAC,UAAU,sBAAsB,KAAK,CAAC;AAAA,MAChD;AAEA,eAAS,iCAAiCF,SAAQE,iBAAgB;AAChE,eAAOF,QAAO,IAAI,QAAQ,EACvB,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,QAAQ,SAAS,GAAG,EACvC,OAAO,CAAC,EAAC,MAAK,MAAM,SAASE,gBAAe,KAAK,KAAK,CAAC,EACvD,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,IAAI,CAAC,UAAU,MAAM,QAAQA,iBAAgB,EAAE,CAAC,EAChD,IAAI,CAAC,UAAU,sBAAsB,KAAK,CAAC;AAAA,MAChD;AAEA,eAAS,sBAAsB,OAAO;AACpC,cAAM,aAAa,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1C,cAAM,uBAAuB;AAC7B,eAAO,QAAQ,MAAM,QAAQ,sBAAsB,EAAE,IAAI;AAAA,MAC3D;AAAA,IACF;AAEA,aAAS,iBAAiBF,SAAQ;AAEhC,YAAM,eAAe;AACrB,YAAM,uBAAuBA,QAAO,IAAI,QAAQ,EAC7C,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,GAAG,EAC/B,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK;AAGzB,YAAM,gCAAgC,KAAK,UAAU,oBAAoB,CAAC,EAAE;AAE5E,YAAM,eAAe,qBAAqB,OAAO,CAAC,YAAY,WAAW,QAAQ,MAAM,YAAY,CAAC;AAEpG,YAAM,2CAA2C,KAAK,UAAU,YAAY,CAAC,EAAE;AAE/E,YAAMG,gBAAe,oBAAoB,YAAY;AAErD,aAAOA;AAAA,IACT;AAEA,aAAS,oBAAoB,UAAU;AACrC,YAAM,YAAY;AAClB,YAAM,QAAQ,SAAS,IAAI,aAAW,QAAQ,MAAM,SAAS,CAAC,EAC3D,KAAK;AACR,YAAM,GAAG,KAAK,UAAU,KAAK,CAAC,EAAE;AAChC,YAAM,YAAY,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,EAAE,KAAK;AAC3C,YAAM,GAAG,KAAK,UAAU,SAAS,CAAC,EAAE;AACpC,aAAO;AAAA,IACT;AAAA,EAEF;AAAA
|
|
4
|
+
"sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\n// We should also get copyright time and copyright/publication times from 26x\n// We should also get publishing time type from f008\n// We should get reprint times from f500 $a \"Lis\u00E4painos/Lis\u00E4painokset:\"\n\nexport default () => ({\n name: 'Publication time, allow consequent years, years from multiple sources',\n extract: ({record, recordExternal}) => {\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years-multi');\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n\n const f008Values = extractF008Values(record);\n debug(`${label}: f008: ${JSON.stringify(f008Values)}`);\n\n const f26xValues = extractF26xValues(record);\n debug(`${label}: f26x: ${JSON.stringify(f26xValues)}`);\n\n const f500Values = extractF500Years(record);\n debug(`${label}: f500: ${JSON.stringify(f500Values)}`);\n\n // We should get copyrightYear from f008Date2 to copyrightYears when f008YearType = 'r'\n // Is the original year (f008Date2) in f008YearType === 'r' comparable to copyrightYear?\n // We should handle unknown years (here or comparison?)\n // We should handle year ranges for continuing resources / collections\n\n const normalYears = [...new Set(f26xValues.normalYears.concat(f008Values.f008Date1).filter(value => value && value !== ' ' && value !== '||||'))].sort();\n const copyrightYears = [...new Set(f26xValues.copyrightYears)].sort();\n const reprintYears = [...new Set(f500Values)].sort();\n\n const combined = {normalYears, copyrightYears, reprintYears};\n\n debug(`Combined: ${JSON.stringify(combined)}`);\n\n return combined;\n\n function extractF008Values(record) {\n // Record should have only one f008 - in case of several, we handle the first one\n const value = record.get(/^008$/u)?.[0]?.value || undefined;\n if (value && testStringOrNumber(value)) {\n const f008Date1 = extractF008Date1(value);\n const f008Date2 = extractF008Date2(value);\n const f008YearType = extractF008YearType(value);\n return {f008Date1, f008Date2, f008YearType};\n }\n return {f008Date1: undefined, f008Date2: undefined, f008YearType: undefined};\n\n function extractF008Date1(value) {\n return String(value).slice(7, 11);\n }\n\n function extractF008Date2(value) {\n return String(value).slice(11, 15);\n }\n\n function extractF008YearType(value) {\n return String(value).slice(6, 7);\n }\n }\n\n function extractF26xValues(record) {\n const copyrightRegex = /^(?<copyrightPrefix>cop|cop.|c|\u00A9|p|\u2117)/u;\n\n const pubNormalSubFieldValues = extractPubNormalSubfieldValues(record, copyrightRegex);\n debug(`Normal years: ${JSON.stringify(pubNormalSubFieldValues)}`);\n\n const pubF264CopySubFieldValues = extractPubF264CopySubfieldValues(record);\n debug(`F264 copyright years: ${JSON.stringify(pubF264CopySubFieldValues)}`);\n\n const pubF260CopySubFieldValues = extractPubF260CopySubfieldValues(record, copyrightRegex);\n debug(`F260 copyright years: ${JSON.stringify(pubF260CopySubFieldValues)}`);\n\n return {normalYears: pubNormalSubFieldValues, copyrightYears: [...pubF260CopySubFieldValues, ...pubF264CopySubFieldValues]};\n\n function extractPubNormalSubfieldValues(record, copyrightRegex) {\n return record.get(/^26[04]$/u)\n .filter((field) => !(field.tag === '264' && field.ind2 === '4'))\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code && code === 'c')\n .filter(({value}) => value && !copyrightRegex.test(value))\n .map(({value}) => value)\n .map((value) => removeNonAlphaNumeric(value));\n }\n\n function extractPubF264CopySubfieldValues(record, copyrightRegex) {\n return record.get(/^264$/u)\n .filter((field) => field.ind2 === '4')\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code && code === 'c')\n .filter(({value}) => value)\n .map(({value}) => value)\n .map((value) => value.replace(copyrightRegex, ''))\n .map((value) => removeNonAlphaNumeric(value));\n }\n\n function extractPubF260CopySubfieldValues(record, copyrightRegex) {\n return record.get(/^260$/u)\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code && code === 'c')\n .filter(({value}) => value && copyrightRegex.test(value))\n .map(({value}) => value)\n .map((value) => value.replace(copyrightRegex, ''))\n .map((value) => removeNonAlphaNumeric(value));\n }\n\n function removeNonAlphaNumeric(value) {\n debug(`Cleaning: ${JSON.stringify(value)}`);\n const nonAlphaNumericRegex = /[^A-Za-z0-9]/ug;\n return value ? value.replace(nonAlphaNumericRegex, '') : value;\n }\n }\n\n function extractF500Years(record) {\n\n const reprintRegex = /(?<reprint>Lis\u00E4painokset:|Lis\u00E4painos:)/u;\n const reprintFieldContents = record.get(/^500$/u)\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code === 'a')\n .map(({value}) => value);\n //.filter(value => value.test(reprintRegex));\n\n debug(`f500 reprint field contents: ${JSON.stringify(reprintFieldContents)}`);\n\n const filteredF500 = reprintFieldContents.filter((content) => content && content.match(reprintRegex));\n\n debug(`f500 reprint field contents (filtered): ${JSON.stringify(filteredF500)}`);\n\n const reprintYears = extractReprintYears(filteredF500);\n\n return reprintYears;\n }\n\n function extractReprintYears(contents) {\n const yearRegex = /[0-9][0-9][0-9][0-9]/gu;\n const years = contents.map(content => content.match(yearRegex))\n .flat();\n debug(`${JSON.stringify(years)}`);\n const uniqYears = [...new Set(years)].sort();\n debug(`${JSON.stringify(uniqYears)}`);\n return uniqYears;\n }\n\n },\n compare: (a, b) => {\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/publication-time-allow-cons-years-multi');\n debug(`Comparing ${JSON.stringify(a)} to ${JSON.stringify(b)}`);\n\n const [firstA] = a.normalYears ? a.normalYears : a;\n const [firstB] = b.normalYears ? b.normalYears : b;\n\n debug(`Comparing ${JSON.stringify(firstA)} to ${JSON.stringify(firstB)}`);\n\n if (firstA === firstB) {\n return 0.1;\n }\n\n // If either of years is a non string/number, values are not comparable\n if (!testStringOrNumber(firstA) || !testStringOrNumber(firstB)) {\n return 0;\n }\n\n const firstANumber = parseInt(firstA, 10);\n const firstBNumber = parseInt(firstB, 10);\n\n // Handle consequent years as a match\n // see publication-time for a version that does not handle consequent years as a match\n if (!(isNaN(firstANumber) || isNaN(firstBNumber)) &&\n (firstANumber + 1 === firstBNumber || firstANumber - 1 === firstBNumber)) {\n return 0.1;\n }\n\n // We should do something with copyrightYears, too\n\n // Do not give minus points if a normal publishing year is found in normal years\n const bNormalInANormal = a.normalYears.filter(aValue => b.normalYears.some(bValue => aValue === bValue));\n const aNormalInBNormal = b.normalYears.filter(bValue => a.normalYears.some(aValue => bValue === aValue));\n debug(`BNorm in ANorm: ${JSON.stringify(bNormalInANormal)}`);\n debug(`ANorm in BNorm: ${JSON.stringify(aNormalInBNormal)}`);\n\n if (bNormalInANormal > 0 || aNormalInBNormal > 0) {\n return 0;\n }\n\n // Do not give minus points if a normal publishing year is found in reprint years\n const bNormalInAReprint = a.reprintYears.filter(aValue => b.normalYears.some(bValue => aValue === bValue));\n const aNormalInBReprint = b.reprintYears.filter(bValue => a.normalYears.some(aValue => bValue === aValue));\n debug(`BNorm in AReprint: ${JSON.stringify(bNormalInAReprint)}`);\n debug(`ANorm in BReprint: ${JSON.stringify(aNormalInBReprint)}`);\n\n if (bNormalInAReprint > 0 || aNormalInBReprint > 0) {\n return 0;\n }\n\n return -1;\n\n }\n});\n\n// https://www.loc.gov/marc/bibliographic/bd008.html\n// field 008\n// 06 - Type of date/Publication status\n// 07-10 - Date 1\n// 11-14 - Date 2\n//\n// 06 - Type of date/Publication status\n// b - No dates given; B.C. date involved\n// c - Continuing resource currently published\n// d - Continuing resource ceased publication\n// e - Detailed date\n// i - Inclusive dates of collection\n// k - Range of years of bulk of collection\n// m - Multiple dates\n// n - Dates unknown\n// p - Date of distribution/release/issue and production/recording session when different\n// q - Questionable date\n// r - Reprint/reissue date and original date\n// s - Single known date/probable date\n// t - Publication date and copyright date\n// u - Continuing resource status unknown\n// | - No attempt to code\n//\n// 07-10 - Date 1\n// 1-9 - Date digit\n// # - Date element is not applicable\n// u - Date element is totally or partially unknown\n// |||| - No attempt to code\n//\n// 11-14 - Date 2\n// 1-9 - Date digit\n// # - Date element is not applicable\n// u - Date element is totally or partially unknown\n// |||| - No attempt to code\n//\n\n"],
|
|
5
|
+
"mappings": "AACA,OAAO,uBAAuB;AAC9B,SAAQ,0BAAyB;AAMjC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,wGAAwG;AACxI,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAE9E,UAAM,aAAa,kBAAkB,MAAM;AAC3C,UAAM,GAAG,KAAK,WAAW,KAAK,UAAU,UAAU,CAAC,EAAE;AAErD,UAAM,aAAa,kBAAkB,MAAM;AAC3C,UAAM,GAAG,KAAK,WAAW,KAAK,UAAU,UAAU,CAAC,EAAE;AAErD,UAAM,aAAa,iBAAiB,MAAM;AAC1C,UAAM,GAAG,KAAK,WAAW,KAAK,UAAU,UAAU,CAAC,EAAE;AAOrD,UAAM,cAAc,CAAC,GAAG,IAAI,IAAI,WAAW,YAAY,OAAO,WAAW,SAAS,EAAE,OAAO,WAAS,SAAS,UAAU,UAAU,UAAU,MAAM,CAAC,CAAC,EAAE,KAAK;AAC1J,UAAM,iBAAiB,CAAC,GAAG,IAAI,IAAI,WAAW,cAAc,CAAC,EAAE,KAAK;AACpE,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,KAAK;AAEnD,UAAM,WAAW,EAAC,aAAa,gBAAgB,aAAY;AAE3D,UAAM,aAAa,KAAK,UAAU,QAAQ,CAAC,EAAE;AAE7C,WAAO;AAEP,aAAS,kBAAkBA,SAAQ;AAEjC,YAAM,QAAQA,QAAO,IAAI,QAAQ,IAAI,CAAC,GAAG,SAAS;AAClD,UAAI,SAAS,mBAAmB,KAAK,GAAG;AACtC,cAAM,YAAY,iBAAiB,KAAK;AACxC,cAAM,YAAY,iBAAiB,KAAK;AACxC,cAAM,eAAe,oBAAoB,KAAK;AAC9C,eAAO,EAAC,WAAW,WAAW,aAAY;AAAA,MAC5C;AACA,aAAO,EAAC,WAAW,QAAW,WAAW,QAAW,cAAc,OAAS;AAE3E,eAAS,iBAAiBC,QAAO;AAC/B,eAAO,OAAOA,MAAK,EAAE,MAAM,GAAG,EAAE;AAAA,MAClC;AAEA,eAAS,iBAAiBA,QAAO;AAC/B,eAAO,OAAOA,MAAK,EAAE,MAAM,IAAI,EAAE;AAAA,MACnC;AAEA,eAAS,oBAAoBA,QAAO;AAClC,eAAO,OAAOA,MAAK,EAAE,MAAM,GAAG,CAAC;AAAA,MACjC;AAAA,IACF;AAEA,aAAS,kBAAkBD,SAAQ;AACjC,YAAM,iBAAiB;AAEvB,YAAM,0BAA0B,+BAA+BA,SAAQ,cAAc;AACrF,YAAM,iBAAiB,KAAK,UAAU,uBAAuB,CAAC,EAAE;AAEhE,YAAM,4BAA4B,iCAAiCA,OAAM;AACzE,YAAM,yBAAyB,KAAK,UAAU,yBAAyB,CAAC,EAAE;AAE1E,YAAM,4BAA4B,iCAAiCA,SAAQ,cAAc;AACzF,YAAM,yBAAyB,KAAK,UAAU,yBAAyB,CAAC,EAAE;AAE1E,aAAO,EAAC,aAAa,yBAAyB,gBAAgB,CAAC,GAAG,2BAA2B,GAAG,yBAAyB,EAAC;AAE1H,eAAS,+BAA+BA,SAAQE,iBAAgB;AAC9D,eAAOF,QAAO,IAAI,WAAW,EAC1B,OAAO,CAAC,UAAU,EAAE,MAAM,QAAQ,SAAS,MAAM,SAAS,IAAI,EAC9D,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,QAAQ,SAAS,GAAG,EACvC,OAAO,CAAC,EAAC,MAAK,MAAM,SAAS,CAACE,gBAAe,KAAK,KAAK,CAAC,EACxD,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,IAAI,CAAC,UAAU,sBAAsB,KAAK,CAAC;AAAA,MAChD;AAEA,eAAS,iCAAiCF,SAAQE,iBAAgB;AAChE,eAAOF,QAAO,IAAI,QAAQ,EACvB,OAAO,CAAC,UAAU,MAAM,SAAS,GAAG,EACpC,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,QAAQ,SAAS,GAAG,EACvC,OAAO,CAAC,EAAC,MAAK,MAAM,KAAK,EACzB,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,IAAI,CAAC,UAAU,MAAM,QAAQE,iBAAgB,EAAE,CAAC,EAChD,IAAI,CAAC,UAAU,sBAAsB,KAAK,CAAC;AAAA,MAChD;AAEA,eAAS,iCAAiCF,SAAQE,iBAAgB;AAChE,eAAOF,QAAO,IAAI,QAAQ,EACvB,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,QAAQ,SAAS,GAAG,EACvC,OAAO,CAAC,EAAC,MAAK,MAAM,SAASE,gBAAe,KAAK,KAAK,CAAC,EACvD,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK,EACtB,IAAI,CAAC,UAAU,MAAM,QAAQA,iBAAgB,EAAE,CAAC,EAChD,IAAI,CAAC,UAAU,sBAAsB,KAAK,CAAC;AAAA,MAChD;AAEA,eAAS,sBAAsB,OAAO;AACpC,cAAM,aAAa,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1C,cAAM,uBAAuB;AAC7B,eAAO,QAAQ,MAAM,QAAQ,sBAAsB,EAAE,IAAI;AAAA,MAC3D;AAAA,IACF;AAEA,aAAS,iBAAiBF,SAAQ;AAEhC,YAAM,eAAe;AACrB,YAAM,uBAAuBA,QAAO,IAAI,QAAQ,EAC7C,IAAI,CAAC,EAAC,UAAS,MAAM,SAAS,EAC9B,KAAK,EACL,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,GAAG,EAC/B,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK;AAGzB,YAAM,gCAAgC,KAAK,UAAU,oBAAoB,CAAC,EAAE;AAE5E,YAAM,eAAe,qBAAqB,OAAO,CAAC,YAAY,WAAW,QAAQ,MAAM,YAAY,CAAC;AAEpG,YAAM,2CAA2C,KAAK,UAAU,YAAY,CAAC,EAAE;AAE/E,YAAMG,gBAAe,oBAAoB,YAAY;AAErD,aAAOA;AAAA,IACT;AAEA,aAAS,oBAAoB,UAAU;AACrC,YAAM,YAAY;AAClB,YAAM,QAAQ,SAAS,IAAI,aAAW,QAAQ,MAAM,SAAS,CAAC,EAC3D,KAAK;AACR,YAAM,GAAG,KAAK,UAAU,KAAK,CAAC,EAAE;AAChC,YAAM,YAAY,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,EAAE,KAAK;AAC3C,YAAM,GAAG,KAAK,UAAU,SAAS,CAAC,EAAE;AACpC,aAAO;AAAA,IACT;AAAA,EAEF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AACjB,UAAM,QAAQ,kBAAkB,wGAAwG;AACxI,UAAM,aAAa,KAAK,UAAU,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,EAAE;AAE9D,UAAM,CAAC,MAAM,IAAI,EAAE,cAAc,EAAE,cAAc;AACjD,UAAM,CAAC,MAAM,IAAI,EAAE,cAAc,EAAE,cAAc;AAEjD,UAAM,aAAa,KAAK,UAAU,MAAM,CAAC,OAAO,KAAK,UAAU,MAAM,CAAC,EAAE;AAExE,QAAI,WAAW,QAAQ;AACrB,aAAO;AAAA,IACT;AAGA,QAAI,CAAC,mBAAmB,MAAM,KAAK,CAAC,mBAAmB,MAAM,GAAG;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,SAAS,QAAQ,EAAE;AACxC,UAAM,eAAe,SAAS,QAAQ,EAAE;AAIxC,QAAI,EAAE,MAAM,YAAY,KAAK,MAAM,YAAY,OAC5C,eAAe,MAAM,gBAAgB,eAAe,MAAM,eAAe;AAC1E,aAAO;AAAA,IACT;AAKA,UAAM,mBAAmB,EAAE,YAAY,OAAO,YAAU,EAAE,YAAY,KAAK,YAAU,WAAW,MAAM,CAAC;AACvG,UAAM,mBAAmB,EAAE,YAAY,OAAO,YAAU,EAAE,YAAY,KAAK,YAAU,WAAW,MAAM,CAAC;AACvG,UAAM,mBAAmB,KAAK,UAAU,gBAAgB,CAAC,EAAE;AAC3D,UAAM,mBAAmB,KAAK,UAAU,gBAAgB,CAAC,EAAE;AAE3D,QAAI,mBAAmB,KAAK,mBAAmB,GAAG;AAChD,aAAO;AAAA,IACT;AAGA,UAAM,oBAAoB,EAAE,aAAa,OAAO,YAAU,EAAE,YAAY,KAAK,YAAU,WAAW,MAAM,CAAC;AACzG,UAAM,oBAAoB,EAAE,aAAa,OAAO,YAAU,EAAE,YAAY,KAAK,YAAU,WAAW,MAAM,CAAC;AACzG,UAAM,sBAAsB,KAAK,UAAU,iBAAiB,CAAC,EAAE;AAC/D,UAAM,sBAAsB,KAAK,UAAU,iBAAiB,CAAC,EAAE;AAE/D,QAAI,oBAAoB,KAAK,oBAAoB,GAAG;AAClD,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EAET;AACF;",
|
|
6
6
|
"names": ["record", "value", "copyrightRegex", "reprintYears"]
|
|
7
7
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/match-detection/features/bib/standard-identifier-factory.js"],
|
|
4
|
-
"sourcesContent": ["
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["import createDebugLogger from 'debug';\nimport {extractSubfieldsFromField, uniqueSubfields} from '../../../matching-utils.js';\n\n// Note about validity of standardIdentifiers:\n// We have three types of invalid standardIdentifiers:\n// 1. Formally invalid standardIdentifiers (ie. typos either in the resource or the record)\n// 2. Formally valid standardIdentifiers that are used in a wrong resource\n// 3. Canceled standardIdentifiers\n\n// Matcher could and should check that a standardIdentifier found in a subfield for a valid identifier is formally valid, and if it's not formally valid, handle it as an invalid standardIdentifier\n// Formally valid standardIdentifiers found in subfield for invalid identifier cannot be handled as valid standardIdentifiers, because they can be a case of type 2) or 3) invalid standardIdentifiers\n// We could also do a separate handling for formally valid an formally invalid standardIdentifiers\n\nexport default ({pattern, subfieldCodes, identifier, validIdentifierSubfieldCodes = ['a'], invalidIdentifierSubfieldCodes = ['z'], validatorAndNormalizer = undefined}) => {\n const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);\n const debugData = debug.extend('data');\n\n return {extract, compare};\n\n function extract({record, recordExternal}) {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const fields = record.get(pattern);\n debugData(`${label}: ${fields.length} ${identifier}-fields `);\n\n // extractIdentifierSubfield normalizes hyphens away from the subfield values\n const identifiersFromFields = fields.map(field => extractSubfieldsFromField(field, subfieldCodes));\n debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);\n const allIdentifiers = identifiersFromFields.flat();\n debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);\n\n const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({identifierSubs: allIdentifiers, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes});\n\n const identifiers = uniqueSubfields(validatedAndNormalizedIdentifiers);\n\n debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);\n return identifiers;\n\n function validateAndNormalizeIdentifiers({identifierSubs, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {\n if (validatorAndNormalizer) {\n return identifierSubs.map((idSub) => validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}));\n }\n return identifierSubs.map((idSub) => normalizeHyphens(idSub));\n }\n\n function validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {\n const {valid, value} = validatorAndNormalizer(idSub.value);\n if (validIdentifierSubfieldCodes.includes(idSub.code) && valid === false) {\n const [code] = invalidIdentifierSubfieldCodes;\n return {code, value};\n }\n return {code: idSub.code, value};\n }\n\n function normalizeHyphens(idSub) {\n return {code: idSub.code, value: idSub.value.replace(/-/ug, '')};\n }\n\n\n }\n\n function compare(a, b) {\n debug(`Comparing A and B`);\n if (a.length === 0 || b.length === 0) {\n debugData(`No standardidentifiers (${identifier}) to compare`);\n return 0;\n }\n\n debugData(`A: ${JSON.stringify(a)}`);\n debugData(`B: ${JSON.stringify(b)}`);\n\n\n if (bothHaveValidIdentifiers()) {\n // Compare only valid identifiers, if both have valid idenfiers\n const {maxValues, possibleMatchValues, matchingValues} = getValueCount(true);\n if (matchingValues < 1) {\n debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);\n return -0.75;\n }\n debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues}/${possibleMatchValues} valid identifiers match.`);\n // ignore non-matches if there is mismatching amount of values\n debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);\n //we give some kind of penalty for mismatching amount of values instead of simple divide?\n const penaltyForMissing = 0.1 * (maxValues - possibleMatchValues);\n const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues);\n debug(`\\t points: penaltyForMissing: ${penaltyForMissing}`);\n debug(`\\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);\n\n return 0.75 - penaltyForMisMatch - penaltyForMissing;\n //return matchingValues / possibleMatchValues * 0.75;\n }\n // If both do not have valid identifiers, compare all identifiers\n const {maxValues, matchingValues} = getValueCount();\n debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);\n\n return matchingValues / maxValues * 0.2;\n\n function bothHaveValidIdentifiers() {\n const aValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));\n const bValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));\n debug(`A: ${aValues.length} valid ${identifier} identifiers`);\n debug(`B: ${bValues.length} valid ${identifier} identifiers`);\n return aValues.length > 0 && bValues.length > 0;\n }\n\n function getValueCount(validOnly = false) {\n const aValues = getIdentifiers(a, validOnly);\n const bValues = getIdentifiers(b, validOnly);\n\n const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n return {\n maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n matchingValues\n };\n\n function getMatchingValuesAmount(aValues, bValues) {\n if (bValues.length > aValues.length) {\n return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n }\n if (aValues.length > bValues.length) {\n return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n }\n\n // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n // there would be duplicate values\n const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n return aToB < bToA ? aToB : bToA;\n }\n\n function getIdentifiers(values, validOnly) {\n if (validOnly) {\n return values\n .filter(({code}) => validIdentifierSubfieldCodes.includes(code))\n .map(({value}) => value);\n }\n\n return values.map(({value}) => value);\n }\n }\n }\n};\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,uBAAuB;AAC9B,SAAQ,2BAA2B,uBAAsB;AAYzD,eAAe,CAAC,EAAC,SAAS,eAAe,YAAY,+BAA+B,CAAC,GAAG,GAAG,iCAAiC,CAAC,GAAG,GAAG,yBAAyB,OAAS,MAAM;AACzK,QAAM,QAAQ,kBAAkB,mFAAmF,UAAU,EAAE;AAC/H,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,SAAO,EAAC,SAAS,QAAO;AAExB,WAAS,QAAQ,EAAC,QAAQ,eAAc,GAAG;AACzC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,SAAS,OAAO,IAAI,OAAO;AACjC,cAAU,GAAG,KAAK,KAAK,OAAO,MAAM,IAAI,UAAU,UAAU;AAG5D,UAAM,wBAAwB,OAAO,IAAI,WAAS,0BAA0B,OAAO,aAAa,CAAC;AACjG,cAAU,GAAG,KAAK,sBAAsB,sBAAsB,MAAM,MAAM,KAAK,UAAU,qBAAqB,CAAC,EAAE;AACjH,UAAM,iBAAiB,sBAAsB,KAAK;AAClD,cAAU,GAAG,KAAK,2BAA2B,eAAe,MAAM,MAAM,KAAK,UAAU,cAAc,CAAC,EAAE;AAExG,UAAM,oCAAoC,gCAAgC,EAAC,gBAAgB,gBAAgB,wBAAwB,8BAA8B,+BAA8B,CAAC;AAEhM,UAAM,cAAc,gBAAgB,iCAAiC;AAErE,cAAU,GAAG,KAAK,6BAA6B,YAAY,MAAM,MAAM,KAAK,UAAU,WAAW,CAAC,EAAE;AACpG,WAAO;AAEP,aAAS,gCAAgC,EAAC,gBAAgB,wBAAAA,yBAAwB,8BAAAC,+BAA8B,gCAAAC,gCAA8B,GAAG;AAC/I,UAAIF,yBAAwB;AAC1B,eAAO,eAAe,IAAI,CAAC,UAAU,+BAA+B,EAAC,OAAO,wBAAAA,yBAAwB,8BAAAC,+BAA8B,gCAAAC,gCAA8B,CAAC,CAAC;AAAA,MACpK;AACA,aAAO,eAAe,IAAI,CAAC,UAAU,iBAAiB,KAAK,CAAC;AAAA,IAC9D;AAEA,aAAS,+BAA+B,EAAC,OAAO,wBAAAF,yBAAwB,8BAAAC,+BAA8B,gCAAAC,gCAA8B,GAAG;AACrI,YAAM,EAAC,OAAO,MAAK,IAAIF,wBAAuB,MAAM,KAAK;AACzD,UAAIC,8BAA6B,SAAS,MAAM,IAAI,KAAK,UAAU,OAAO;AACxE,cAAM,CAAC,IAAI,IAAIC;AACf,eAAO,EAAC,MAAM,MAAK;AAAA,MACrB;AACA,aAAO,EAAC,MAAM,MAAM,MAAM,MAAK;AAAA,IACjC;AAEA,aAAS,iBAAiB,OAAO;AAC/B,aAAO,EAAC,MAAM,MAAM,MAAM,OAAO,MAAM,MAAM,QAAQ,OAAO,EAAE,EAAC;AAAA,IACjE;AAAA,EAGF;AAEA,WAAS,QAAQ,GAAG,GAAG;AACrB,UAAM,mBAAmB;AACzB,QAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,gBAAU,2BAA2B,UAAU,cAAc;AAC7D,aAAO;AAAA,IACT;AAEA,cAAU,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE;AACnC,cAAU,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE;AAGnC,QAAI,yBAAyB,GAAG;AAE9B,YAAM,EAAC,WAAAC,YAAW,qBAAqB,gBAAAC,gBAAc,IAAI,cAAc,IAAI;AAC3E,UAAIA,kBAAiB,GAAG;AACtB,cAAM,wCAAwC,UAAU,6BAA6B;AACrF,eAAO;AAAA,MACT;AACA,YAAM,wCAAwC,UAAU,MAAMA,eAAc,IAAI,mBAAmB,2BAA2B;AAE9H,YAAM,qBAAqB,mBAAmB,IAAID,UAAS,EAAE;AAE7D,YAAM,oBAAoB,OAAOA,aAAY;AAC7C,YAAM,qBAAqB,OAAO,sBAAsBC;AACxD,YAAM,gCAAiC,iBAAiB,EAAE;AAC1D,YAAM,iCAAkC,kBAAkB,EAAE;AAE5D,aAAO,OAAO,qBAAqB;AAAA,IAErC;AAEA,UAAM,EAAC,WAAW,eAAc,IAAI,cAAc;AAClD,UAAM,+CAA+C,UAAU,MAAM,cAAc,IAAI,SAAS,mCAAmC;AAEnI,WAAO,iBAAiB,YAAY;AAEpC,aAAS,2BAA2B;AAClC,YAAM,UAAU,EAAE,OAAO,CAAC,EAAC,KAAI,MAAM,6BAA6B,SAAS,IAAI,CAAC;AAChF,YAAM,UAAU,EAAE,OAAO,CAAC,EAAC,KAAI,MAAM,6BAA6B,SAAS,IAAI,CAAC;AAChF,YAAM,MAAM,QAAQ,MAAM,UAAU,UAAU,cAAc;AAC5D,YAAM,MAAM,QAAQ,MAAM,UAAU,UAAU,cAAc;AAC5D,aAAO,QAAQ,SAAS,KAAK,QAAQ,SAAS;AAAA,IAChD;AAEA,aAAS,cAAc,YAAY,OAAO;AACxC,YAAM,UAAU,eAAe,GAAG,SAAS;AAC3C,YAAM,UAAU,eAAe,GAAG,SAAS;AAE3C,YAAMA,kBAAiB,wBAAwB,SAAS,OAAO;AAE/D,aAAO;AAAA,QACL,WAAW,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA;AAAA,QAEtE,qBAAqB,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA,QAChF,gBAAAA;AAAA,MACF;AAEA,eAAS,wBAAwBC,UAASC,UAAS;AACjD,YAAIA,SAAQ,SAASD,SAAQ,QAAQ;AACnC,iBAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,QAC7E;AACA,YAAID,SAAQ,SAASC,SAAQ,QAAQ;AACnC,iBAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,QAC7E;AAIA,cAAM,OAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AACjF,cAAM,OAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAEjF,eAAO,OAAO,OAAO,OAAO;AAAA,MAC9B;AAEA,eAAS,eAAe,QAAQE,YAAW;AACzC,YAAIA,YAAW;AACb,iBAAO,OACJ,OAAO,CAAC,EAAC,KAAI,MAAM,6BAA6B,SAAS,IAAI,CAAC,EAC9D,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK;AAAA,QAC3B;AAEA,eAAO,OAAO,IAAI,CAAC,EAAC,MAAK,MAAM,KAAK;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["validatorAndNormalizer", "validIdentifierSubfieldCodes", "invalidIdentifierSubfieldCodes", "maxValues", "matchingValues", "aValues", "bValues", "validOnly"]
|
|
7
7
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/match-detection/features/index.js"],
|
|
4
|
-
"sourcesContent": ["\nimport * as bib from './bib/index.js';\nexport {bib};\n"],
|
|
5
|
-
"mappings": "AACA,YAAY,SAAS;
|
|
4
|
+
"sourcesContent": ["\nimport * as auth from './auth/index.js';\nimport * as bib from './bib/index.js';\n\nexport {auth, bib};\n"],
|
|
5
|
+
"mappings": "AACA,YAAY,UAAU;AACtB,YAAY,SAAS;AAErB,SAAQ,MAAM;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -17,11 +17,7 @@ describe("match-detection", () => {
|
|
|
17
17
|
fixura: {
|
|
18
18
|
reader: READERS.JSON
|
|
19
19
|
},
|
|
20
|
-
callback: ({ getFixture, options, expectedResults, array
|
|
21
|
-
if (!enabled) {
|
|
22
|
-
debug(`*** DISABLED TEST! ***`);
|
|
23
|
-
return;
|
|
24
|
-
}
|
|
20
|
+
callback: ({ getFixture, options, expectedResults, array }) => {
|
|
25
21
|
const detect = createDetectionInterface(formatOptions());
|
|
26
22
|
const recordA = new MarcRecord(getFixture("recordA.json"), { subfieldValues: false });
|
|
27
23
|
debugData(inspect(recordA));
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/match-detection/index.test.js"],
|
|
4
|
-
"sourcesContent": ["import assert from 'node:assert';\nimport {describe} from 'node:test';\nimport createDebugLogger from 'debug';\nimport {inspect} from 'util';\nimport generateTests from '@natlibfi/fixugen';\nimport {READERS} from '@natlibfi/fixura';\nimport {MarcRecord} from '@natlibfi/marc-record';\n\nimport createDetectionInterface from './index.js';\nimport * as features from './features/index.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:test');\nconst debugData = debug.extend('data');\n\ndescribe('match-detection', () => {\n generateTests({\n path: [import.meta.dirname, '..', '..', 'test-fixtures', 'match-detection', 'index'],\n useMetadataFile: true,\n recurse: false,\n fixura: {\n reader: READERS.JSON\n },\n callback: ({getFixture, options, expectedResults, array
|
|
5
|
-
"mappings": "AAAA,OAAO,YAAY;AACnB,SAAQ,gBAAe;AACvB,OAAO,uBAAuB;AAC9B,SAAQ,eAAc;AACtB,OAAO,mBAAmB;AAC1B,SAAQ,eAAc;AACtB,SAAQ,kBAAiB;AAEzB,OAAO,8BAA8B;AACrC,YAAY,cAAc;AAE1B,MAAM,QAAQ,kBAAkB,wDAAwD;AACxF,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,SAAS,mBAAmB,MAAM;AAChC,gBAAc;AAAA,IACZ,MAAM,CAAC,YAAY,SAAS,MAAM,MAAM,iBAAiB,mBAAmB,OAAO;AAAA,IACnF,iBAAiB;AAAA,IACjB,SAAS;AAAA,IACT,QAAQ;AAAA,MACN,QAAQ,QAAQ;AAAA,IAClB;AAAA,IACA,UAAU,CAAC,EAAC,YAAY,SAAS,iBAAiB,
|
|
4
|
+
"sourcesContent": ["import assert from 'node:assert';\nimport {describe} from 'node:test';\nimport createDebugLogger from 'debug';\nimport {inspect} from 'util';\nimport generateTests from '@natlibfi/fixugen';\nimport {READERS} from '@natlibfi/fixura';\nimport {MarcRecord} from '@natlibfi/marc-record';\n\nimport createDetectionInterface from './index.js';\nimport * as features from './features/index.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:test');\nconst debugData = debug.extend('data');\n\ndescribe('match-detection', () => {\n generateTests({\n path: [import.meta.dirname, '..', '..', 'test-fixtures', 'match-detection', 'index'],\n useMetadataFile: true,\n recurse: false,\n fixura: {\n reader: READERS.JSON\n },\n callback: ({getFixture, options, expectedResults, array}) => {\n const detect = createDetectionInterface(formatOptions());\n const recordA = new MarcRecord(getFixture('recordA.json'), {subfieldValues: false});\n debugData(inspect(recordA));\n\n debug(`Our recordB is an array of records: ${array}`);\n const recordB = array\n ? getFixture('recordB.json').map(recordJson => new MarcRecord(recordJson, {subfieldValues: false}))\n : new MarcRecord(getFixture('recordB.json'), {subfieldValues: false});\n debugData(inspect(recordB));\n\n const results = detect({recordA, recordB});\n debugData(`${JSON.stringify(results)}`);\n\n assert.deepStrictEqual(results, expectedResults);\n\n function formatOptions() {\n const contextFeatures = features[options.strategy.type];\n\n return {\n ...options,\n strategy: options.strategy.features.map(v => contextFeatures[v]())\n };\n }\n }\n });\n});\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,YAAY;AACnB,SAAQ,gBAAe;AACvB,OAAO,uBAAuB;AAC9B,SAAQ,eAAc;AACtB,OAAO,mBAAmB;AAC1B,SAAQ,eAAc;AACtB,SAAQ,kBAAiB;AAEzB,OAAO,8BAA8B;AACrC,YAAY,cAAc;AAE1B,MAAM,QAAQ,kBAAkB,wDAAwD;AACxF,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,SAAS,mBAAmB,MAAM;AAChC,gBAAc;AAAA,IACZ,MAAM,CAAC,YAAY,SAAS,MAAM,MAAM,iBAAiB,mBAAmB,OAAO;AAAA,IACnF,iBAAiB;AAAA,IACjB,SAAS;AAAA,IACT,QAAQ;AAAA,MACN,QAAQ,QAAQ;AAAA,IAClB;AAAA,IACA,UAAU,CAAC,EAAC,YAAY,SAAS,iBAAiB,MAAK,MAAM;AAC3D,YAAM,SAAS,yBAAyB,cAAc,CAAC;AACvD,YAAM,UAAU,IAAI,WAAW,WAAW,cAAc,GAAG,EAAC,gBAAgB,MAAK,CAAC;AAClF,gBAAU,QAAQ,OAAO,CAAC;AAE1B,YAAM,uCAAuC,KAAK,EAAE;AACpD,YAAM,UAAU,QACZ,WAAW,cAAc,EAAE,IAAI,gBAAc,IAAI,WAAW,YAAY,EAAC,gBAAgB,MAAK,CAAC,CAAC,IAChG,IAAI,WAAW,WAAW,cAAc,GAAG,EAAC,gBAAgB,MAAK,CAAC;AACtE,gBAAU,QAAQ,OAAO,CAAC;AAE1B,YAAM,UAAU,OAAO,EAAC,SAAS,QAAO,CAAC;AACzC,gBAAU,GAAG,KAAK,UAAU,OAAO,CAAC,EAAE;AAEtC,aAAO,gBAAgB,SAAS,eAAe;AAE/C,eAAS,gBAAgB;AACvB,cAAM,kBAAkB,SAAS,QAAQ,SAAS,IAAI;AAEtD,eAAO;AAAA,UACL,GAAG;AAAA,UACH,UAAU,QAAQ,SAAS,SAAS,IAAI,OAAK,gBAAgB,CAAC,EAAE,CAAC;AAAA,QACnE;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AACH,CAAC;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"url": "https://github.com/NatLibFi/melinda-record-matching-js"
|
|
14
14
|
},
|
|
15
15
|
"license": "MIT",
|
|
16
|
-
"version": "5.0
|
|
16
|
+
"version": "5.1.0",
|
|
17
17
|
"type": "module",
|
|
18
18
|
"main": "./dist/index.js",
|
|
19
19
|
"bin": "./dist/cli.js",
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"@natlibfi/melinda-commons": "^14.0.2",
|
|
46
46
|
"@natlibfi/sru-client": "^7.0.1",
|
|
47
47
|
"debug": "^4.4.3",
|
|
48
|
-
"isbn3": "^2.0.
|
|
48
|
+
"isbn3": "^2.0.7",
|
|
49
49
|
"moment": "^2.30.1",
|
|
50
50
|
"natural": "^8.1.1",
|
|
51
51
|
"uuid": "^14.0.0",
|
|
@@ -57,6 +57,9 @@
|
|
|
57
57
|
"@natlibfi/fixura": "^4.0.1",
|
|
58
58
|
"cross-env": "^10.1.0",
|
|
59
59
|
"esbuild": "^0.28.0",
|
|
60
|
-
"eslint": "^10.
|
|
60
|
+
"eslint": "^10.3.0"
|
|
61
|
+
},
|
|
62
|
+
"overrides": {
|
|
63
|
+
"uuid": "^14.0.0"
|
|
61
64
|
}
|
|
62
65
|
}
|
|
@@ -73,7 +73,6 @@ export default async ({record, searchSpec, url, maxCandidates, maxRecordsPerRequ
|
|
|
73
73
|
|
|
74
74
|
return {search};
|
|
75
75
|
|
|
76
|
-
// eslint-disable-next-line max-statements
|
|
77
76
|
async function search({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) {
|
|
78
77
|
const query = chosenQueryList[queryOffset];
|
|
79
78
|
debug(`Running query ${JSON.stringify(query)} (${queryOffset})`);
|
|
@@ -124,7 +123,6 @@ export default async ({record, searchSpec, url, maxCandidates, maxRecordsPerRequ
|
|
|
124
123
|
|
|
125
124
|
export function retrieveRecords(client, query, resultSetOffset) {
|
|
126
125
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:retrieveRecords');
|
|
127
|
-
// eslint-disable-next-line no-unused-vars
|
|
128
126
|
const debugData = debug.extend('data');
|
|
129
127
|
|
|
130
128
|
return new Promise((resolve, reject) => {
|
|
@@ -19,14 +19,9 @@ describe('candidate-search', () => {
|
|
|
19
19
|
}
|
|
20
20
|
});
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError = false, expectedSearchError = false, enabled = true}) {
|
|
22
|
+
async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError = false, expectedSearchError = false}) {
|
|
24
23
|
const url = 'http://foo.bar';
|
|
25
24
|
|
|
26
|
-
if (!enabled) {
|
|
27
|
-
return;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
25
|
if (expectedFactoryError) {
|
|
31
26
|
debug(`We're expecting an error`);
|
|
32
27
|
if (expectedFactoryError.isCandidateSearchError) {
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import createDebugLogger from 'debug';
|
|
2
|
+
import {toQueries} from '../candidate-search-utils.js';
|
|
3
|
+
|
|
4
|
+
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query-list:auth');
|
|
5
|
+
|
|
6
|
+
export function authURX(record) {
|
|
7
|
+
const a = recordGetAuthURX(record);
|
|
8
|
+
|
|
9
|
+
if (a.length === 0) {
|
|
10
|
+
debug(`No identifiers found, no queries created.`);
|
|
11
|
+
return [];
|
|
12
|
+
}
|
|
13
|
+
return toQueries(a, 'melinda.urx');
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function recordGetAuthURX(record) {
|
|
17
|
+
const f024s = record.get(/024/u).filter(f => f.subfields.some(sf => sf.code === '2' && ['uri', 'urn'].includes(sf.value))); // f024 is repeatable
|
|
18
|
+
if (f024s.length > 0) {
|
|
19
|
+
debug(`${f024s.length} id(s) found`);
|
|
20
|
+
// NB! f024$a is a non-repeatable subfield, but we are not checking that here
|
|
21
|
+
const values = f024s.map(f => f.subfields).flat().filter(sf => sf.code === 'a').map(sf => sf.value);
|
|
22
|
+
return values;
|
|
23
|
+
}
|
|
24
|
+
return [];
|
|
25
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import assert from 'node:assert';
|
|
2
|
+
import {describe} from 'node:test';
|
|
3
|
+
import createDebugLogger from 'debug';
|
|
4
|
+
import generateTests from '@natlibfi/fixugen';
|
|
5
|
+
import {READERS} from '@natlibfi/fixura';
|
|
6
|
+
import {MarcRecord} from '@natlibfi/marc-record';
|
|
7
|
+
import * as generators from './auth.js';
|
|
8
|
+
|
|
9
|
+
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query-list:auth-test');
|
|
10
|
+
const debugData = debug.extend('data');
|
|
11
|
+
|
|
12
|
+
describe('candidate-search/query-list/auth/', () => {
|
|
13
|
+
generateTests({
|
|
14
|
+
path: [import.meta.dirname, '..', '..', '..', 'test-fixtures', 'candidate-search', 'query-list', 'auth'],
|
|
15
|
+
useMetadataFile: true,
|
|
16
|
+
fixura: {
|
|
17
|
+
reader: READERS.JSON
|
|
18
|
+
},
|
|
19
|
+
callback: ({type, inputRecord, expectedQuery, expectedQueryListType}) => {
|
|
20
|
+
const generate = generators[type];
|
|
21
|
+
const record = new MarcRecord(inputRecord, {subfieldValues: false});
|
|
22
|
+
|
|
23
|
+
const result = generate(record);
|
|
24
|
+
debugData(`Result: ${JSON.stringify(result)}`);
|
|
25
|
+
|
|
26
|
+
if (result.queryListType) {
|
|
27
|
+
assert.deepStrictEqual(result.queryList, expectedQuery);
|
|
28
|
+
assert.equal(result.queryListType, expectedQueryListType);
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
assert.deepStrictEqual(result, expectedQuery);
|
|
32
|
+
}
|
|
33
|
+
});
|
|
34
|
+
});
|
|
@@ -332,8 +332,7 @@ export function bibAuthors(record) {
|
|
|
332
332
|
|
|
333
333
|
function getAuthor(record) {
|
|
334
334
|
//debugData(record);
|
|
335
|
-
|
|
336
|
-
const [field] = record.get(/^(100)|(110)|(111)|(700)|(710)|(711)$/u);
|
|
335
|
+
const [field] = record.get(/^(?:100|110|111|700|710|711)$/u);
|
|
337
336
|
//debugData(field);
|
|
338
337
|
|
|
339
338
|
if (field) {
|
|
@@ -16,14 +16,10 @@ describe('candidate-search/query-list/bib/', () => {
|
|
|
16
16
|
fixura: {
|
|
17
17
|
reader: READERS.JSON
|
|
18
18
|
},
|
|
19
|
-
callback: ({type, inputRecord, expectedQuery, expectedQueryListType
|
|
19
|
+
callback: ({type, inputRecord, expectedQuery, expectedQueryListType}) => {
|
|
20
20
|
const generate = generators[type];
|
|
21
21
|
const record = new MarcRecord(inputRecord, {subfieldValues: false});
|
|
22
22
|
|
|
23
|
-
if (!enabled) {
|
|
24
|
-
return;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
23
|
const result = generate(record);
|
|
28
24
|
debugData(`Result: ${JSON.stringify(result)}`);
|
|
29
25
|
|
|
@@ -4,10 +4,10 @@ import {toQueries} from '../candidate-search-utils.js';
|
|
|
4
4
|
import {getSubfieldValues, stringAfter, stringBefore, testStringOrNumber, toMelindaIds} from '../../matching-utils.js';
|
|
5
5
|
import {fieldToString, uniqArray} from '@natlibfi/marc-record-validators-melinda/dist/utils.js';
|
|
6
6
|
|
|
7
|
-
const setTimeoutPromise = promisify(setTimeout);
|
|
7
|
+
const setTimeoutPromise = promisify(setTimeout);
|
|
8
8
|
|
|
9
9
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:component');
|
|
10
|
-
const debugData = debug.extend('data');
|
|
10
|
+
const debugData = debug.extend('data');
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
|
|
@@ -44,7 +44,7 @@ export function parse773g(field) {
|
|
|
44
44
|
if (value.match(/^[0-9]+(?:-[0-9]+)?(?:, [0-9]+(?:-[0-9]+)?)*$/u)) {
|
|
45
45
|
return value;
|
|
46
46
|
}
|
|
47
|
-
const numberPartOnly = value.replace(/^.*(?:p\.|raidat|raita
|
|
47
|
+
const numberPartOnly = value.replace(/^.*(?:p\.|raidat|raita|\bs\.|Seite|sivut?|pages?) /ui, '');
|
|
48
48
|
if (numberPartOnly !== value) {
|
|
49
49
|
return gToPages(numberPartOnly);
|
|
50
50
|
}
|
|
@@ -116,7 +116,7 @@ export function hostIdMelinda(record) { // old function with replaced code
|
|
|
116
116
|
|
|
117
117
|
export async function hostIdOtherSource(record, client) {
|
|
118
118
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:hostIdOtherSource');
|
|
119
|
-
const debugData = debug.extend('data');
|
|
119
|
+
const debugData = debug.extend('data');
|
|
120
120
|
debug(`Creating query for the Other source Id host`);
|
|
121
121
|
|
|
122
122
|
const ids = getHostIds(record);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as bib from './bib.js';
|
|
2
2
|
import * as component from './component.js';
|
|
3
|
+
import * as auth from './auth.js';
|
|
3
4
|
import createDebugLogger from 'debug';
|
|
4
5
|
|
|
5
6
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:index');
|
|
@@ -21,11 +22,16 @@ export const searchTypes = {
|
|
|
21
22
|
hostIdMelinda: 'hostIdMelinda', // 773 $w (FI-MELINDA)
|
|
22
23
|
hostIdOtherSource: 'hostIdOtherSource', // 773 $w !(FI-MELINDA)
|
|
23
24
|
hostIsbn: 'hostIsbn' // 773 $z
|
|
25
|
+
},
|
|
26
|
+
auth: {
|
|
27
|
+
//catalogingRules: 'authCatalogingRules', // 008/10 and 040$f
|
|
28
|
+
//term: 'authTermComparison' // 1XX
|
|
29
|
+
authURX: 'authURX' // 024$a
|
|
24
30
|
}
|
|
25
31
|
};
|
|
26
32
|
|
|
27
33
|
export default async (record, searchSpec, client) => {
|
|
28
|
-
const extractors = {...bib, ...component};
|
|
34
|
+
const extractors = {...auth, ...bib, ...component};
|
|
29
35
|
debugData(`extractors: ${JSON.stringify(extractors)}`);
|
|
30
36
|
debugData(`searchSpec: ${JSON.stringify(searchSpec)}`);
|
|
31
37
|
|
package/src/cli.js
CHANGED
|
@@ -42,7 +42,7 @@ async function cli() {
|
|
|
42
42
|
throw new Error('Setup sru url');
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
-
if (!['IDS', 'STANDARD_IDS', 'COMPONENT', 'CONTENT', 'CONTENTALT'].includes(args.searchType)) {
|
|
45
|
+
if (!['IDS', 'STANDARD_IDS', 'COMPONENT', 'CONTENT', 'CONTENTALT', 'YSO'].includes(args.searchType)) {
|
|
46
46
|
throw new Error('Invalid search type');
|
|
47
47
|
}
|
|
48
48
|
|
|
@@ -136,6 +136,12 @@ async function cli() {
|
|
|
136
136
|
];
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
+
if (['YSO']) {
|
|
140
|
+
return [
|
|
141
|
+
matchDetection.features.auth.yso()
|
|
142
|
+
];
|
|
143
|
+
}
|
|
144
|
+
|
|
139
145
|
throw new Error('Unsupported match validation package');
|
|
140
146
|
}
|
|
141
147
|
|
|
@@ -176,6 +182,12 @@ async function cli() {
|
|
|
176
182
|
];
|
|
177
183
|
}
|
|
178
184
|
|
|
185
|
+
if (searchType === 'YSO') {
|
|
186
|
+
return [
|
|
187
|
+
candidateSearch.searchTypes.auth.authURX
|
|
188
|
+
];
|
|
189
|
+
}
|
|
190
|
+
|
|
179
191
|
throw new Error('Unsupported match validation package');
|
|
180
192
|
}
|
|
181
193
|
}
|
package/src/index.js
CHANGED
|
@@ -161,7 +161,6 @@ export default ({detection: detectionOptions, search: searchOptions, maxMatches
|
|
|
161
161
|
return;
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
-
// eslint-disable-next-line max-statements
|
|
165
164
|
function getMatchState(state, stopReason, matchErrorCount) {
|
|
166
165
|
debugData(`${JSON.stringify(state)}`);
|
|
167
166
|
debug(`We had ${matchErrorCount} retrieved candidates that errored in matchDetection.`);
|
package/src/index.test.js
CHANGED
|
@@ -19,7 +19,6 @@ describe('INDEX', () => {
|
|
|
19
19
|
}
|
|
20
20
|
});
|
|
21
21
|
|
|
22
|
-
// eslint-disable-next-line max-statements
|
|
23
22
|
async function callback({getFixture, options, expectedMatchStatus, expectedStopReason, expectedFailures, expectedCandidateCount}) {
|
|
24
23
|
|
|
25
24
|
const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {default as yso} from './yso.js'
|