@natlibfi/melinda-record-matching 5.1.2 → 5.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/match-detection/features/bib/language.js +37 -19
- package/dist/match-detection/features/bib/language.js.map +2 -2
- package/dist/match-detection/features/bib/record-type.js +26 -2
- package/dist/match-detection/features/bib/record-type.js.map +2 -2
- package/dist/match-detection/features/bib/title.js +80 -38
- package/dist/match-detection/features/bib/title.js.map +3 -3
- package/package.json +3 -3
- package/src/match-detection/features/bib/language.js +44 -28
- package/src/match-detection/features/bib/record-type.js +35 -6
- package/src/match-detection/features/bib/title.js +99 -40
|
@@ -84,9 +84,6 @@ export default () => ({
|
|
|
84
84
|
debug(` Indicator penalty: '${aField.ind1}' vs '${bField.ind1}'`);
|
|
85
85
|
return -0.1;
|
|
86
86
|
}
|
|
87
|
-
function containsNoData(languageCode) {
|
|
88
|
-
return [" ", "|||", "und"].includes(languageCode);
|
|
89
|
-
}
|
|
90
87
|
function compareLanguageCodeLists(a2, b2) {
|
|
91
88
|
if (a2.length === 0 || b2.length === 0 || a2.every((val) => containsNoData(val)) || b2.every((val) => containsNoData(val))) {
|
|
92
89
|
debugData(`No language to compare`);
|
|
@@ -111,25 +108,40 @@ export default () => ({
|
|
|
111
108
|
if (b2.length === 1 && b2[0] === "mul" && a2.length > 1) {
|
|
112
109
|
return 0;
|
|
113
110
|
}
|
|
114
|
-
const sharedValues = getSharedValues(a2, b2)
|
|
111
|
+
const sharedValues = getSharedValues(a2, b2);
|
|
115
112
|
if (sharedValues.length < 1) {
|
|
116
113
|
debug(`Both have languages, but none of these match.`);
|
|
117
114
|
return -1;
|
|
118
115
|
}
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
debug(`
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
116
|
+
const aOnly = getUniqueInFirst(a2, b2);
|
|
117
|
+
const bOnly = getUniqueInFirst(b2, a2);
|
|
118
|
+
debug(`A only: ${aOnly.join(", ")}`);
|
|
119
|
+
debug(`B only: ${bOnly.join(", ")}`);
|
|
120
|
+
return calculatePenalty();
|
|
121
|
+
function calculatePenalty() {
|
|
122
|
+
const basePenalty = calculatePenalty2() / sharedValues.length;
|
|
123
|
+
if (basePenalty < -1) {
|
|
124
|
+
return -1;
|
|
125
|
+
}
|
|
126
|
+
return basePenalty;
|
|
127
|
+
}
|
|
128
|
+
function calculatePenalty2() {
|
|
129
|
+
const n = aOnly.length + bOnly.length;
|
|
130
|
+
if (aOnly.length === 0 || bOnly.length === 0) {
|
|
131
|
+
if (n > 3) {
|
|
132
|
+
return -0.3;
|
|
133
|
+
}
|
|
134
|
+
return 0.1 - n * 0.1;
|
|
135
|
+
}
|
|
136
|
+
if (aOnly.length === 1 && bOnly.length === 1) {
|
|
137
|
+
return -0.1;
|
|
138
|
+
}
|
|
139
|
+
debug(` ${n} differences altogether`);
|
|
140
|
+
if (n > 3) {
|
|
141
|
+
return -1;
|
|
142
|
+
}
|
|
143
|
+
return n * -0.2;
|
|
144
|
+
}
|
|
133
145
|
}
|
|
134
146
|
}
|
|
135
147
|
});
|
|
@@ -181,6 +193,12 @@ function getFieldsLanguageCodes(field) {
|
|
|
181
193
|
return [...new Set(values)].sort();
|
|
182
194
|
}
|
|
183
195
|
function getSharedValues(list1, list2) {
|
|
184
|
-
return list1.filter((val) => list2.includes(val));
|
|
196
|
+
return list1.filter((val) => list2.includes(val) && !containsNoData(val));
|
|
197
|
+
}
|
|
198
|
+
function getUniqueInFirst(list1, list2) {
|
|
199
|
+
return list1.filter((val) => !list2.includes(val) && !containsNoData(val));
|
|
200
|
+
}
|
|
201
|
+
function containsNoData(languageCode) {
|
|
202
|
+
return [" ", "|||", "und"].includes(languageCode);
|
|
185
203
|
}
|
|
186
204
|
//# sourceMappingURL=language.js.map
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/match-detection/features/bib/language.js"],
|
|
4
|
-
"sourcesContent": ["import createDebugLogger from 'debug';\n\nimport {MarcRecord} from '@natlibfi/marc-record';\n\nimport {FixSami041, Remove041zxx} from '@natlibfi/marc-record-validators-melinda';\n\nimport {getMatchCounts} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:language');\nconst debugData = debug.extend('data');\n\n// NB! 2025-12-17: I changed the logic drastically. However, I tried to keep the scores as same as possible.\n// 'extract' no longer extracts anything. Instead it clones the record, and does some preprocessing (using validators) instead.\n// 'compare' had multiple changes\n// - f041 ind1 differences ('0' vs '1') now result in a small penalty\n// - Two sami languages related changes:\n// -- validator adds a 'smi' subfield before a corresponding sma/sme/...subfield, if needed (national)\n// -- 'smi' only vs 'smi'+'sma' does not cause penalty\n// - 'mul' vs 'fin'+'swe' does not cause a penalty. However, 'mul' vs 'fin' alone triggers penalty.\n// - 008/35-37 and f041$a/$d are calculated separately\n// - a threshold is applied: return value is always between -1.0 and +0.1 (as it was before)\n// - we try to handle 041 $2 ISO 639-2 and ISO 639-3 as well.\n// - 'und' vs one other language does not cause a penalty in certain contexts. In other contexts it's treated as a normal \"language code\". (Tune/alleviate penalties later on.)\n\nexport default () => ({\n name: 'Language',\n extract: ({record, recordExternal}) => {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const clonedRecord = new MarcRecord(record, record._validationOptions); // clone(record); // NB! This loses record.get()...\n\n FixSami041().fix(clonedRecord); // Handle 'smi' adding if needed\n Remove041zxx().fix(clonedRecord); // Remove 'zxx' from f041s\n clonedRecord.fields = clonedRecord.fields.filter(f => ['008', '041'].includes(f.tag));\n // Add other language code normalizations?\n\n // Should we return all the fields, or just the relevant fields?\n return [clonedRecord.fields, label];\n },\n compare: (aa, bb) => {\n const [a, aLabel] = aa;\n const [b, bLabel] = bb;\n debugData(`Comparing language ${JSON.stringify(a)} and ${JSON.stringify(b)}`);\n\n const score008 = compare008();\n const score041 = compare041();\n return applyLimits(score008 + score041);\n\n function applyLimits(score) {\n if (score > 0.1) { // Keeps the original max, we might have 0.10 (041) + 0.05 (008/35-37) = 0.15 now\n return 0.1;\n }\n return score;\n }\n\n function compare008() {\n const a008 = get008Value(a, aLabel);\n const b008 = get008Value(b, bLabel);\n // Something seriously wrong with 008:\n if (a008 === undefined || b008 === undefined) {\n return 0.0; // Punish for generic badness?\n }\n\n if (containsNoData(a008) || containsNoData(b008)){\n // Nothing to compare\n return 0.0;\n }\n\n if (a008 === b008) {\n return 0.05;\n }\n\n if (a008 === 'mul' || b008 === 'mul') { // We'll let 041 decide\n return 0.0;\n }\n\n return -0.2;\n }\n\n\n\n\n\n function compare041() {\n const a041s = getFields041(a);\n const b041s = getFields041(b);\n\n if (a041s.length === 0 || b041s.length === 0) {\n return 0.0; // Should we punish these for badness?\n }\n if (a041s.length === 1 && b041s.length === 1) {\n // The language codes are typically same between different sources. Incur a small penalty for differences though...\n const sourcePenalty = getSourceOfLanguageCode(a041s[0]) === getSourceOfLanguageCode(b041s[0]) ? 0.0 : -0.05;\n const mainPenalty = compareLanguageCodeLists(getFieldsLanguageCodes(a041s[0]), getFieldsLanguageCodes(b041s[0]));\n const scoreInd1 = mainPenalty === 0.0 ? 0.0 : indicator1Penalty(a041s[0], b041s[0]);\n return mainPenalty + sourcePenalty + scoreInd1;\n }\n // Things are already complicated (likely to pe penalized, so no nedd to worry about language code sources, I daresay.\n return compareLanguageCodeLists(getLanguageCodesFrom041Fields(a), getLanguageCodesFrom041Fields(b));\n\n function getFields041(fields) {\n if (!fields) {\n return [];\n }\n return fields.filter(f => f.tag === '041');\n }\n\n }\n\n function getSourceOfLanguageCode(field) {\n const sf2 = field.subfields.find(sf => sf.code === '2');\n if (!sf2) {\n return 'MARC';\n }\n // Normalize the two relevant language code names:\n if (sf2.value.match(/639.2/u)) {\n return 'ISO 639-2';\n }\n if (sf2.value.match(/639.3/u)) {\n return 'ISO 639-3';\n }\n return sf2.value; // We don't actually have anything else in Melinda though\n }\n\n function indicator1Penalty(aField, bField) {\n if (aField.ind1 === ' ' || bField.ind1 === ' '|| aField.ind1 === bField.ind1 ) {\n return 0.0;\n }\n debug(`\\t Indicator penalty: '${aField.ind1}' vs '${bField.ind1}'`);\n return -0.1;\n }\n\n\n\n function containsNoData(languageCode) {\n return [' ', '|||', 'und'].includes(languageCode);\n }\n\n function compareLanguageCodeLists(a, b) {\n if (a.length === 0 || b.length === 0 || a.every(val => containsNoData(val)) || b.every(val => containsNoData(val))) {\n debugData(`No language to compare`);\n return 0;\n }\n\n // If either one of the sets has only the generic 'smi', remove the specific sami languages codes from the other as well:\n const samiLanguageCodes = ['sma', 'sme', 'smj', 'smn', 'sms']; // NB! Don't put generic 'smi' here!\n\n if (a.includes('smi') && b.includes('smi')) {\n if (a.some(code => samiLanguageCodes.includes(code)) && !b.some(code => samiLanguageCodes.includes(code))) {\n return compareLanguageCodeLists(a.filter(val => !samiLanguageCodes.includes(val)), b); // recurse\n }\n if (b.some(code => samiLanguageCodes.includes(code)) && !a.some(code => samiLanguageCodes.includes(code))) {\n return compareLanguageCodeLists(a, b.filter(val => !samiLanguageCodes.includes(val))); // recurse\n }\n }\n\n\n\n\n if (a.length === b.length && a.every((element, index) => element === b[index])) {\n debugData(`All languages match`);\n return 0.1;\n }\n\n // Handle 'mul'. (Should we check whether the other array contains mul as well, probably not...)\n // (If 'mul' does not appear alone, then it is very iffy... )\n if (a.length === 1 && a[0] === 'mul' && b.length > 1) {\n return 0;\n }\n if (b.length === 1 && b[0] === 'mul' && a.length > 1) {\n return 0;\n }\n\n\n\n // Damage control:\n const sharedValues = getSharedValues(a, b).filter(val => !containsNoData(val));\n\n if (sharedValues.length < 1) {\n debug(`Both have languages, but none of these match.`); // includes 'mul' vs a single language code\n return -1.0;\n }\n\n const {matchingValues, possibleMatchValues, maxValues} = getMatchCounts(a, b);\n\n debug(`Both have languages, ${matchingValues}/${possibleMatchValues} valid languages match.`);\n // ignore non-matches if there is mismatching amount of values\n debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);\n // we give some kind of penalty for mismatching amount of values instead of simple divide?\n const missingCount = maxValues - possibleMatchValues;\n const misMatchCount = possibleMatchValues - matchingValues;\n debug(`\\t missing: ${missingCount}`);\n debug(`\\t mismatches: ${misMatchCount}`);\n\n const penaltyForMissing = 0.02 * (maxValues - possibleMatchValues);\n const penaltyForMisMatch = 0.05 * (possibleMatchValues - matchingValues);\n debug(`\\t points: penaltyForMissing: ${penaltyForMissing}`);\n debug(`\\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);\n\n const points = Number(Number(0.1 - penaltyForMisMatch - penaltyForMissing).toFixed(2));\n debug(`Total points: ${points}`);\n\n return points;\n }\n }\n\n});\n\nfunction get008Value(fields, label = 'record') {\n if (!fields) {\n return;\n }\n const f008 = fields.find(f => f.tag === '008');\n if (!f008) {\n return undefined;\n }\n const value = f008.value || defaultValue;\n\n if (!value) {\n debugData(`${label}: Failed to extact 008/35-37 value from '${value}'`);\n return defaultValue;\n }\n\n const code = value.slice(35, 38);\n debugData(`${label}: 008 code: '${code}'`);\n return code;\n}\n\n// Check if a string is a possible, validly formed language code for a single language\n// Currently accept also codes in capitals\nfunction isLangCodeForALanguage(code, label = 'record', encoding = undefined) {\n if (!code) {\n return false;\n }\n\n if ([undefined, 'ISO 639-2', 'ISO-639-3'].includes(encoding) && code.length !== 3) {\n debugData(`${label}: Code ${code} is not correct length (3) for a language code.`);\n return false;\n }\n if (!encoding) {\n // 'mul' should be passed as 'mul' can match 'fin' + 'swe' etc.\n // 'zxx' should be removed by a validator\n // '^^^' is Aleph-specific corruption, not supporting it anymore\n if (code === '|||' || code === ' ' ) { // || code === '^^^' || code === 'mul' || code === 'zxx') {\n debugData(`${label}: Code ${code} is not code for a spesific language.`);\n return false;\n }\n }\n // Marc, ISO 639-2 and ISO 639-3 are all three-letters long. Not other language codes seen in Melinda:\n const langCodePattern = /^[a-z][a-z][a-z]$/ui;\n if (!langCodePattern.test(code)) {\n debugData(`${label}: Code ${code} is not valid as a language code`);\n return false;\n }\n return true;\n}\n\n\n\nfunction getLanguageCodesFrom041Fields(fields) {\n // NB! We brutally don't check $2 (language code source) as marc's language codes is practically a subset of ISO 639-2,\n // and also ISO 639-2 and ISO 639-3 overlap to a degree. If we ever run into a trouble with some ISO 639-2 vs 639-3 mismatch, then we'll work it out.\n // Also we could write a validator that converts ISO 639-2 to marc if applicable and maybe even partial support for ISO-639-3.\n // Note that ISO 639-2-B values correspond with marc beteer that ISO 639-2-T. Eg. code for Chinese 'zho' should/code be normalized to 'chi'!\n const values = fields.filter(f => f.tag === '041').flatMap(f => getFieldsLanguageCodes(f));\n /*\n // .filter(({ind2}) => ind2 === ' ')\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code === 'a' || code === 'd')\n .filter(({value}) => value && isLangCodeForALanguage(value))\n .map(({value}) => value);\n */\n return [...new Set(values)].sort();\n}\n\nfunction getFieldsLanguageCodes(field) {\n const relevantSubfields = field.subfields.filter(sf => sf.code === 'a' || sf.code === 'd').filter(sf => isLangCodeForALanguage(sf.value, 'field'));\n\n const values = relevantSubfields.map(sf => sf.value).flat();\n\n return [...new Set(values)].sort();\n}\n\nfunction getSharedValues(list1, list2) {\n return list1.filter(val => list2.includes(val));\n}"],
|
|
5
|
-
"mappings": "AAAA,OAAO,uBAAuB;AAE9B,SAAQ,kBAAiB;AAEzB,SAAQ,YAAY,oBAAmB;AAEvC,SAAQ,sBAAqB;AAE7B,MAAM,QAAQ,kBAAkB,qEAAqE;AACrG,MAAM,YAAY,MAAM,OAAO,MAAM;AAerC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,eAAe,IAAI,WAAW,QAAQ,OAAO,kBAAkB;AAErE,eAAW,EAAE,IAAI,YAAY;AAC7B,iBAAa,EAAE,IAAI,YAAY;AAC/B,iBAAa,SAAS,aAAa,OAAO,OAAO,OAAK,CAAC,OAAO,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC;AAIpF,WAAO,CAAC,aAAa,QAAQ,KAAK;AAAA,EACpC;AAAA,EACA,SAAS,CAAC,IAAI,OAAO;AACnB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,cAAU,sBAAsB,KAAK,UAAU,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,EAAE;AAE5E,UAAM,WAAW,WAAW;AAC5B,UAAM,WAAW,WAAW;AAC5B,WAAO,YAAY,WAAW,QAAQ;AAEtC,aAAS,YAAY,OAAO;AAC1B,UAAI,QAAQ,KAAK;AACf,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT;AAEA,aAAS,aAAa;AACpB,YAAM,OAAO,YAAY,GAAG,MAAM;AAClC,YAAM,OAAO,YAAY,GAAG,MAAM;AAElC,UAAI,SAAS,UAAa,SAAS,QAAW;AAC5C,eAAO;AAAA,MACT;AAEA,UAAI,eAAe,IAAI,KAAK,eAAe,IAAI,GAAE;AAE/C,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,MAAM;AACjB,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,SAAS,SAAS,OAAO;AACpC,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT;AAMA,aAAS,aAAa;AACpB,YAAM,QAAQ,aAAa,CAAC;AAC5B,YAAM,QAAQ,aAAa,CAAC;AAE5B,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAC5C,eAAO;AAAA,MACT;AACA,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAE5C,cAAM,gBAAgB,wBAAwB,MAAM,CAAC,CAAC,MAAM,wBAAwB,MAAM,CAAC,CAAC,IAAI,IAAM;AACtG,cAAM,cAAc,yBAAyB,uBAAuB,MAAM,CAAC,CAAC,GAAG,uBAAuB,MAAM,CAAC,CAAC,CAAC;AAC/G,cAAM,YAAY,gBAAgB,IAAM,IAAM,kBAAkB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC;AAClF,eAAO,cAAc,gBAAgB;AAAA,MACvC;AAEA,aAAO,yBAAyB,8BAA8B,CAAC,GAAG,8BAA8B,CAAC,CAAC;AAElG,eAAS,aAAa,QAAQ;AAC5B,YAAI,CAAC,QAAQ;AACX,iBAAO,CAAC;AAAA,QACV;AACA,eAAO,OAAO,OAAO,OAAK,EAAE,QAAQ,KAAK;AAAA,MAC3C;AAAA,IAEF;AAEA,aAAS,wBAAwB,OAAO;AACtC,YAAM,MAAM,MAAM,UAAU,KAAK,QAAM,GAAG,SAAS,GAAG;AACtD,UAAI,CAAC,KAAK;AACR,eAAO;AAAA,MACT;AAEA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,aAAO,IAAI;AAAA,IACb;AAEA,aAAS,kBAAkB,QAAQ,QAAQ;AACzC,UAAI,OAAO,SAAS,OAAO,OAAO,SAAS,OAAM,OAAO,SAAS,OAAO,MAAO;AAC7E,eAAO;AAAA,MACT;AACA,YAAM,yBAA0B,OAAO,IAAI,SAAS,OAAO,IAAI,GAAG;AAClE,aAAO;AAAA,IACT;
|
|
4
|
+
"sourcesContent": ["import createDebugLogger from 'debug';\n\nimport {MarcRecord} from '@natlibfi/marc-record';\n\nimport {FixSami041, Remove041zxx} from '@natlibfi/marc-record-validators-melinda';\n\nimport {getMatchCounts} from '../../../matching-utils.js';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:language');\nconst debugData = debug.extend('data');\n\n// NB! 2025-12-17: I changed the logic drastically. However, I tried to keep the scores as same as possible.\n// 'extract' no longer extracts anything. Instead it clones the record, and does some preprocessing (using validators) instead.\n// 'compare' had multiple changes\n// - f041 ind1 differences ('0' vs '1') now result in a small penalty\n// - Two sami languages related changes:\n// -- validator adds a 'smi' subfield before a corresponding sma/sme/...subfield, if needed (national)\n// -- 'smi' only vs 'smi'+'sma' does not cause penalty\n// - 'mul' vs 'fin'+'swe' does not cause a penalty. However, 'mul' vs 'fin' alone triggers penalty.\n// - 008/35-37 and f041$a/$d are calculated separately\n// - a threshold is applied: return value is always between -1.0 and +0.1 (as it was before)\n// - we try to handle 041 $2 ISO 639-2 and ISO 639-3 as well.\n// - 'und' vs one other language does not cause a penalty in certain contexts. In other contexts it's treated as a normal \"language code\". (Tune/alleviate penalties later on.)\n\nexport default () => ({\n name: 'Language',\n extract: ({record, recordExternal}) => {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const clonedRecord = new MarcRecord(record, record._validationOptions); // clone(record); // NB! This loses record.get()...\n\n FixSami041().fix(clonedRecord); // Handle 'smi' adding if needed\n Remove041zxx().fix(clonedRecord); // Remove 'zxx' from f041s\n clonedRecord.fields = clonedRecord.fields.filter(f => ['008', '041'].includes(f.tag));\n // Add other language code normalizations?\n\n // Should we return all the fields, or just the relevant fields?\n return [clonedRecord.fields, label];\n },\n compare: (aa, bb) => {\n const [a, aLabel] = aa;\n const [b, bLabel] = bb;\n debugData(`Comparing language ${JSON.stringify(a)} and ${JSON.stringify(b)}`);\n\n const score008 = compare008();\n const score041 = compare041();\n return applyLimits(score008 + score041);\n\n function applyLimits(score) {\n if (score > 0.1) { // Keeps the original max, we might have 0.10 (041) + 0.05 (008/35-37) = 0.15 now\n return 0.1;\n }\n return score;\n }\n\n function compare008() {\n const a008 = get008Value(a, aLabel);\n const b008 = get008Value(b, bLabel);\n // Something seriously wrong with 008:\n if (a008 === undefined || b008 === undefined) {\n return 0.0; // Punish for generic badness?\n }\n\n if (containsNoData(a008) || containsNoData(b008)){\n // Nothing to compare\n return 0.0;\n }\n\n if (a008 === b008) {\n return 0.05;\n }\n\n if (a008 === 'mul' || b008 === 'mul') { // We'll let 041 decide\n return 0.0;\n }\n\n return -0.2;\n }\n\n\n\n\n\n function compare041() {\n const a041s = getFields041(a);\n const b041s = getFields041(b);\n\n if (a041s.length === 0 || b041s.length === 0) {\n return 0.0; // Should we punish these for badness?\n }\n if (a041s.length === 1 && b041s.length === 1) {\n // The language codes are typically same between different sources. Incur a small penalty for differences though...\n const sourcePenalty = getSourceOfLanguageCode(a041s[0]) === getSourceOfLanguageCode(b041s[0]) ? 0.0 : -0.05;\n const mainPenalty = compareLanguageCodeLists(getFieldsLanguageCodes(a041s[0]), getFieldsLanguageCodes(b041s[0]));\n const scoreInd1 = mainPenalty === 0.0 ? 0.0 : indicator1Penalty(a041s[0], b041s[0]);\n return mainPenalty + sourcePenalty + scoreInd1;\n }\n // Things are already complicated (likely to pe penalized, so no nedd to worry about language code sources, I daresay.\n return compareLanguageCodeLists(getLanguageCodesFrom041Fields(a), getLanguageCodesFrom041Fields(b));\n\n function getFields041(fields) {\n if (!fields) {\n return [];\n }\n return fields.filter(f => f.tag === '041');\n }\n\n }\n\n function getSourceOfLanguageCode(field) {\n const sf2 = field.subfields.find(sf => sf.code === '2');\n if (!sf2) {\n return 'MARC';\n }\n // Normalize the two relevant language code names:\n if (sf2.value.match(/639.2/u)) {\n return 'ISO 639-2';\n }\n if (sf2.value.match(/639.3/u)) {\n return 'ISO 639-3';\n }\n return sf2.value; // We don't actually have anything else in Melinda though\n }\n\n function indicator1Penalty(aField, bField) {\n if (aField.ind1 === ' ' || bField.ind1 === ' '|| aField.ind1 === bField.ind1 ) {\n return 0.0;\n }\n debug(`\\t Indicator penalty: '${aField.ind1}' vs '${bField.ind1}'`);\n return -0.1;\n }\n\n function compareLanguageCodeLists(a, b) {\n if (a.length === 0 || b.length === 0 || a.every(val => containsNoData(val)) || b.every(val => containsNoData(val))) {\n debugData(`No language to compare`);\n return 0;\n }\n\n // If either one of the sets has only the generic 'smi', remove the specific sami languages codes from the other as well:\n const samiLanguageCodes = ['sma', 'sme', 'smj', 'smn', 'sms']; // NB! Don't put generic 'smi' here!\n\n if (a.includes('smi') && b.includes('smi')) {\n if (a.some(code => samiLanguageCodes.includes(code)) && !b.some(code => samiLanguageCodes.includes(code))) {\n return compareLanguageCodeLists(a.filter(val => !samiLanguageCodes.includes(val)), b); // recurse\n }\n if (b.some(code => samiLanguageCodes.includes(code)) && !a.some(code => samiLanguageCodes.includes(code))) {\n return compareLanguageCodeLists(a, b.filter(val => !samiLanguageCodes.includes(val))); // recurse\n }\n }\n\n\n\n\n if (a.length === b.length && a.every((element, index) => element === b[index])) {\n debugData(`All languages match`);\n return 0.1;\n }\n\n // Handle 'mul'. (Should we check whether the other array contains mul as well, probably not...)\n // (If 'mul' does not appear alone, then it is very iffy... )\n if (a.length === 1 && a[0] === 'mul' && b.length > 1) {\n return 0;\n }\n if (b.length === 1 && b[0] === 'mul' && a.length > 1) {\n return 0;\n }\n\n // Damage control:\n const sharedValues = getSharedValues(a, b); // NB! excludes meaningless values\n\n if (sharedValues.length < 1) {\n debug(`Both have languages, but none of these match.`);\n return -1.0;\n }\n\n const aOnly = getUniqueInFirst(a, b); // NB! excludes meaningless values\n const bOnly = getUniqueInFirst(b, a);\n\n debug(`A only: ${aOnly.join(', ')}`);\n debug(`B only: ${bOnly.join(', ')}`);\n\n return calculatePenalty();\n\n function calculatePenalty() {\n const basePenalty = calculatePenalty2() / sharedValues.length;\n if (basePenalty < -1.0) {\n return -1.0;\n }\n return basePenalty;\n }\n \n function calculatePenalty2() {\n const n = aOnly.length + bOnly.length;\n // It's reasonably ok for one to have more languages than the other (max penalty -0.3 for that):\n if (aOnly.length === 0 || bOnly.length === 0) {\n if (n > 3) {\n return -0.3;\n }\n return 0.1 - (n * 0.1);\n }\n // Possible typo (should we calculate levenshtein distance for the value?):\n if (aOnly.length === 1 && bOnly.length === 1) {\n return -0.1; // Not too big a penalty\n }\n debug(` ${n} differences altogether`);\n // However, it's worse to have different languages on both sides\n if (n > 3) {\n return -1.0;\n }\n return n * -0.2;\n }\n }\n }\n\n});\n\nfunction get008Value(fields, label = 'record') {\n if (!fields) {\n return;\n }\n const f008 = fields.find(f => f.tag === '008');\n if (!f008) {\n return undefined;\n }\n const value = f008.value || defaultValue;\n\n if (!value) {\n debugData(`${label}: Failed to extact 008/35-37 value from '${value}'`);\n return defaultValue;\n }\n\n const code = value.slice(35, 38);\n debugData(`${label}: 008 code: '${code}'`);\n return code;\n}\n\n// Check if a string is a possible, validly formed language code for a single language\n// Currently accept also codes in capitals\nfunction isLangCodeForALanguage(code, label = 'record', encoding = undefined) {\n if (!code) {\n return false;\n }\n\n if ([undefined, 'ISO 639-2', 'ISO-639-3'].includes(encoding) && code.length !== 3) {\n debugData(`${label}: Code ${code} is not correct length (3) for a language code.`);\n return false;\n }\n if (!encoding) {\n // 'mul' should be passed as 'mul' can match 'fin' + 'swe' etc.\n // 'zxx' should be removed by a validator\n // '^^^' is Aleph-specific corruption, not supporting it anymore\n if (code === '|||' || code === ' ' ) { // || code === '^^^' || code === 'mul' || code === 'zxx') {\n debugData(`${label}: Code ${code} is not code for a spesific language.`);\n return false;\n }\n }\n // Marc, ISO 639-2 and ISO 639-3 are all three-letters long. Not other language codes seen in Melinda:\n const langCodePattern = /^[a-z][a-z][a-z]$/ui;\n if (!langCodePattern.test(code)) {\n debugData(`${label}: Code ${code} is not valid as a language code`);\n return false;\n }\n return true;\n}\n\n\n\nfunction getLanguageCodesFrom041Fields(fields) {\n // NB! We brutally don't check $2 (language code source) as marc's language codes is practically a subset of ISO 639-2,\n // and also ISO 639-2 and ISO 639-3 overlap to a degree. If we ever run into a trouble with some ISO 639-2 vs 639-3 mismatch, then we'll work it out.\n // Also we could write a validator that converts ISO 639-2 to marc if applicable and maybe even partial support for ISO-639-3.\n // Note that ISO 639-2-B values correspond with marc beteer that ISO 639-2-T. Eg. code for Chinese 'zho' should/code be normalized to 'chi'!\n const values = fields.filter(f => f.tag === '041').flatMap(f => getFieldsLanguageCodes(f));\n /*\n // .filter(({ind2}) => ind2 === ' ')\n .map(({subfields}) => subfields)\n .flat()\n .filter(({code}) => code === 'a' || code === 'd')\n .filter(({value}) => value && isLangCodeForALanguage(value))\n .map(({value}) => value);\n */\n return [...new Set(values)].sort();\n}\n\nfunction getFieldsLanguageCodes(field) {\n const relevantSubfields = field.subfields.filter(sf => sf.code === 'a' || sf.code === 'd').filter(sf => isLangCodeForALanguage(sf.value, 'field'));\n\n const values = relevantSubfields.map(sf => sf.value).flat();\n\n return [...new Set(values)].sort();\n}\n\nfunction getSharedValues(list1, list2) {\n return list1.filter(val => list2.includes(val) && !containsNoData(val));\n}\n\nfunction getUniqueInFirst(list1, list2){\n return list1.filter(val => !list2.includes(val) && !containsNoData(val));\n}\n\nfunction containsNoData(languageCode) {\n return [' ', '|||', 'und'].includes(languageCode);\n}"],
|
|
5
|
+
"mappings": "AAAA,OAAO,uBAAuB;AAE9B,SAAQ,kBAAiB;AAEzB,SAAQ,YAAY,oBAAmB;AAEvC,SAAQ,sBAAqB;AAE7B,MAAM,QAAQ,kBAAkB,qEAAqE;AACrG,MAAM,YAAY,MAAM,OAAO,MAAM;AAerC,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,eAAe,IAAI,WAAW,QAAQ,OAAO,kBAAkB;AAErE,eAAW,EAAE,IAAI,YAAY;AAC7B,iBAAa,EAAE,IAAI,YAAY;AAC/B,iBAAa,SAAS,aAAa,OAAO,OAAO,OAAK,CAAC,OAAO,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC;AAIpF,WAAO,CAAC,aAAa,QAAQ,KAAK;AAAA,EACpC;AAAA,EACA,SAAS,CAAC,IAAI,OAAO;AACnB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,UAAM,CAAC,GAAG,MAAM,IAAI;AACpB,cAAU,sBAAsB,KAAK,UAAU,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,EAAE;AAE5E,UAAM,WAAW,WAAW;AAC5B,UAAM,WAAW,WAAW;AAC5B,WAAO,YAAY,WAAW,QAAQ;AAEtC,aAAS,YAAY,OAAO;AAC1B,UAAI,QAAQ,KAAK;AACf,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT;AAEA,aAAS,aAAa;AACpB,YAAM,OAAO,YAAY,GAAG,MAAM;AAClC,YAAM,OAAO,YAAY,GAAG,MAAM;AAElC,UAAI,SAAS,UAAa,SAAS,QAAW;AAC5C,eAAO;AAAA,MACT;AAEA,UAAI,eAAe,IAAI,KAAK,eAAe,IAAI,GAAE;AAE/C,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,MAAM;AACjB,eAAO;AAAA,MACT;AAEA,UAAI,SAAS,SAAS,SAAS,OAAO;AACpC,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT;AAMA,aAAS,aAAa;AACpB,YAAM,QAAQ,aAAa,CAAC;AAC5B,YAAM,QAAQ,aAAa,CAAC;AAE5B,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAC5C,eAAO;AAAA,MACT;AACA,UAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAE5C,cAAM,gBAAgB,wBAAwB,MAAM,CAAC,CAAC,MAAM,wBAAwB,MAAM,CAAC,CAAC,IAAI,IAAM;AACtG,cAAM,cAAc,yBAAyB,uBAAuB,MAAM,CAAC,CAAC,GAAG,uBAAuB,MAAM,CAAC,CAAC,CAAC;AAC/G,cAAM,YAAY,gBAAgB,IAAM,IAAM,kBAAkB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC;AAClF,eAAO,cAAc,gBAAgB;AAAA,MACvC;AAEA,aAAO,yBAAyB,8BAA8B,CAAC,GAAG,8BAA8B,CAAC,CAAC;AAElG,eAAS,aAAa,QAAQ;AAC5B,YAAI,CAAC,QAAQ;AACX,iBAAO,CAAC;AAAA,QACV;AACA,eAAO,OAAO,OAAO,OAAK,EAAE,QAAQ,KAAK;AAAA,MAC3C;AAAA,IAEF;AAEA,aAAS,wBAAwB,OAAO;AACtC,YAAM,MAAM,MAAM,UAAU,KAAK,QAAM,GAAG,SAAS,GAAG;AACtD,UAAI,CAAC,KAAK;AACR,eAAO;AAAA,MACT;AAEA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,UAAI,IAAI,MAAM,MAAM,QAAQ,GAAG;AAC7B,eAAO;AAAA,MACT;AACA,aAAO,IAAI;AAAA,IACb;AAEA,aAAS,kBAAkB,QAAQ,QAAQ;AACzC,UAAI,OAAO,SAAS,OAAO,OAAO,SAAS,OAAM,OAAO,SAAS,OAAO,MAAO;AAC7E,eAAO;AAAA,MACT;AACA,YAAM,yBAA0B,OAAO,IAAI,SAAS,OAAO,IAAI,GAAG;AAClE,aAAO;AAAA,IACT;AAEA,aAAS,yBAAyBA,IAAGC,IAAG;AACtC,UAAID,GAAE,WAAW,KAAKC,GAAE,WAAW,KAAKD,GAAE,MAAM,SAAO,eAAe,GAAG,CAAC,KAAKC,GAAE,MAAM,SAAO,eAAe,GAAG,CAAC,GAAG;AAClH,kBAAU,wBAAwB;AAClC,eAAO;AAAA,MACT;AAGA,YAAM,oBAAoB,CAAC,OAAO,OAAO,OAAO,OAAO,KAAK;AAE5D,UAAID,GAAE,SAAS,KAAK,KAAKC,GAAE,SAAS,KAAK,GAAG;AAC1C,YAAID,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,KAAK,CAACC,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,GAAG;AACzG,iBAAO,yBAAyBD,GAAE,OAAO,SAAO,CAAC,kBAAkB,SAAS,GAAG,CAAC,GAAGC,EAAC;AAAA,QACtF;AACA,YAAIA,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,KAAK,CAACD,GAAE,KAAK,UAAQ,kBAAkB,SAAS,IAAI,CAAC,GAAG;AACzG,iBAAO,yBAAyBA,IAAGC,GAAE,OAAO,SAAO,CAAC,kBAAkB,SAAS,GAAG,CAAC,CAAC;AAAA,QACtF;AAAA,MACF;AAKA,UAAID,GAAE,WAAWC,GAAE,UAAUD,GAAE,MAAM,CAAC,SAAS,UAAU,YAAYC,GAAE,KAAK,CAAC,GAAG;AAC9E,kBAAU,qBAAqB;AAC/B,eAAO;AAAA,MACT;AAIA,UAAID,GAAE,WAAW,KAAKA,GAAE,CAAC,MAAM,SAASC,GAAE,SAAS,GAAG;AACpD,eAAO;AAAA,MACT;AACA,UAAIA,GAAE,WAAW,KAAKA,GAAE,CAAC,MAAM,SAASD,GAAE,SAAS,GAAG;AACpD,eAAO;AAAA,MACT;AAGA,YAAM,eAAe,gBAAgBA,IAAGC,EAAC;AAEzC,UAAI,aAAa,SAAS,GAAG;AAC3B,cAAM,+CAA+C;AACrD,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,iBAAiBD,IAAGC,EAAC;AACnC,YAAM,QAAQ,iBAAiBA,IAAGD,EAAC;AAEnC,YAAM,WAAW,MAAM,KAAK,IAAI,CAAC,EAAE;AACnC,YAAM,WAAW,MAAM,KAAK,IAAI,CAAC,EAAE;AAEnC,aAAO,iBAAiB;AAExB,eAAS,mBAAmB;AAC1B,cAAM,cAAc,kBAAkB,IAAI,aAAa;AACvD,YAAI,cAAc,IAAM;AACtB,iBAAO;AAAA,QACT;AACA,eAAO;AAAA,MACT;AAEA,eAAS,oBAAoB;AAC3B,cAAM,IAAI,MAAM,SAAS,MAAM;AAE/B,YAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAC5C,cAAI,IAAI,GAAG;AACT,mBAAO;AAAA,UACT;AACA,iBAAO,MAAO,IAAI;AAAA,QACpB;AAEA,YAAI,MAAM,WAAW,KAAK,MAAM,WAAW,GAAG;AAC5C,iBAAO;AAAA,QACT;AACA,cAAM,IAAI,CAAC,yBAAyB;AAEpC,YAAI,IAAI,GAAG;AACT,iBAAO;AAAA,QACT;AACA,eAAO,IAAI;AAAA,MACb;AAAA,IACF;AAAA,EACF;AAEF;AAEA,SAAS,YAAY,QAAQ,QAAQ,UAAU;AAC7C,MAAI,CAAC,QAAQ;AACX;AAAA,EACF;AACA,QAAM,OAAO,OAAO,KAAK,OAAK,EAAE,QAAQ,KAAK;AAC7C,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,KAAK,SAAS;AAE5B,MAAI,CAAC,OAAO;AACV,cAAU,GAAG,KAAK,4CAA4C,KAAK,GAAG;AACtE,WAAO;AAAA,EACT;AAEA,QAAM,OAAO,MAAM,MAAM,IAAI,EAAE;AAC/B,YAAU,GAAG,KAAK,gBAAgB,IAAI,GAAG;AACzC,SAAO;AACT;AAIA,SAAS,uBAAuB,MAAM,QAAQ,UAAU,WAAW,QAAW;AAC5E,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AAEA,MAAI,CAAC,QAAW,aAAa,WAAW,EAAE,SAAS,QAAQ,KAAK,KAAK,WAAW,GAAG;AACjF,cAAU,GAAG,KAAK,UAAU,IAAI,iDAAiD;AACjF,WAAO;AAAA,EACT;AACA,MAAI,CAAC,UAAU;AAIb,QAAI,SAAS,SAAS,SAAS,OAAQ;AACrC,gBAAU,GAAG,KAAK,UAAU,IAAI,uCAAuC;AACvE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,kBAAkB;AACxB,MAAI,CAAC,gBAAgB,KAAK,IAAI,GAAG;AAC/B,cAAU,GAAG,KAAK,UAAU,IAAI,kCAAkC;AAClE,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAIA,SAAS,8BAA8B,QAAQ;AAK7C,QAAM,SAAS,OAAO,OAAO,OAAK,EAAE,QAAQ,KAAK,EAAE,QAAQ,OAAK,uBAAuB,CAAC,CAAC;AASzF,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,uBAAuB,OAAO;AACrC,QAAM,oBAAoB,MAAM,UAAU,OAAO,QAAM,GAAG,SAAS,OAAO,GAAG,SAAS,GAAG,EAAE,OAAO,QAAM,uBAAuB,GAAG,OAAO,OAAO,CAAC;AAEjJ,QAAM,SAAS,kBAAkB,IAAI,QAAM,GAAG,KAAK,EAAE,KAAK;AAE1D,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,gBAAgB,OAAO,OAAO;AACrC,SAAO,MAAM,OAAO,SAAO,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,GAAG,CAAC;AACxE;AAEA,SAAS,iBAAiB,OAAO,OAAM;AACrC,SAAO,MAAM,OAAO,SAAO,CAAC,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,GAAG,CAAC;AACzE;AAEA,SAAS,eAAe,cAAc;AACpC,SAAO,CAAC,OAAO,OAAO,KAAK,EAAE,SAAS,YAAY;AACpD;",
|
|
6
6
|
"names": ["a", "b"]
|
|
7
7
|
}
|
|
@@ -1,6 +1,30 @@
|
|
|
1
1
|
export default () => ({
|
|
2
2
|
name: "Record type",
|
|
3
|
-
extract: ({ record }) =>
|
|
4
|
-
|
|
3
|
+
extract: ({ record }) => {
|
|
4
|
+
const recordType = record.leader[6] || null;
|
|
5
|
+
const soitonopas = isSoitonopas();
|
|
6
|
+
return [recordType, soitonopas];
|
|
7
|
+
function isSoitonopas() {
|
|
8
|
+
if (recordType === "c") {
|
|
9
|
+
const f300 = record.get("300");
|
|
10
|
+
if (f300.some((f) => f.subfields?.some((sf) => sf.code === "a" && sf.value.match(/(?:instrumentskol|soitonopas)/ui)))) {
|
|
11
|
+
return true;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
compare: (a, b) => {
|
|
18
|
+
if (a[0] === b[0]) {
|
|
19
|
+
return 0.1;
|
|
20
|
+
}
|
|
21
|
+
if (a[0] === "a" && b[1]) {
|
|
22
|
+
return 0;
|
|
23
|
+
}
|
|
24
|
+
if (b[0] === "a" && a[1]) {
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
27
|
+
return -0.5;
|
|
28
|
+
}
|
|
5
29
|
});
|
|
6
30
|
//# sourceMappingURL=record-type.js.map
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/match-detection/features/bib/record-type.js"],
|
|
4
|
-
"sourcesContent": ["\n//
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["// RecordType\n//\n// Array structure:\n// 0: record type\n// 1: is soitonopas; The English translation for soitionopas is both terrible and unused in Melinda, that I'm not using it (\"musical instrument methods\")\n\nexport default () => ({\n name: 'Record type',\n extract: ({record}) => {\n const recordType = record.leader[6] || null;\n const soitonopas = isSoitonopas();\n\n return [recordType, soitonopas];\n\n function isSoitonopas() {\n if (recordType === 'c') {\n const f300 = record.get('300');\n if (f300.some(f => f.subfields?.some(sf => sf.code === 'a' && sf.value.match(/(?:instrumentskol|soitonopas)/ui)))) {\n return true; \n }\n }\n return false;\n }\n },\n compare: (a, b) => {\n if (a[0] === b[0]) {\n return 0.1;\n }\n // Soitonopas vs text/book: no bonus, no penalty\n if (a[0] === 'a' && b[1]) {\n return 0.0;\n }\n if (b[0] === 'a' && a[1]) {\n return 0.0;\n }\n // Bad:\n return -0.5\n }\n});\n"],
|
|
5
|
+
"mappings": "AAMA,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,OAAM,MAAM;AACrB,UAAM,aAAa,OAAO,OAAO,CAAC,KAAK;AACvC,UAAM,aAAa,aAAa;AAEhC,WAAO,CAAC,YAAY,UAAU;AAE9B,aAAS,eAAe;AACtB,UAAI,eAAe,KAAK;AACtB,cAAM,OAAO,OAAO,IAAI,KAAK;AAC7B,YAAI,KAAK,KAAK,OAAK,EAAE,WAAW,KAAK,QAAM,GAAG,SAAS,OAAO,GAAG,MAAM,MAAM,iCAAiC,CAAC,CAAC,GAAG;AACjH,iBAAO;AAAA,QACT;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,SAAS,CAAC,GAAG,MAAM;AAChB,QAAI,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,QAAI,EAAE,CAAC,MAAM,OAAO,EAAE,CAAC,GAAG;AACxB,aAAO;AAAA,IACT;AACA,QAAI,EAAE,CAAC,MAAM,OAAO,EAAE,CAAC,GAAG;AACxB,aAAO;AAAA,IACT;AAEA,WAAQ;AAAA,EACV;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -2,8 +2,10 @@ import createDebugLogger from "debug";
|
|
|
2
2
|
import naturalPkg from "natural";
|
|
3
3
|
const { LevenshteinDistance: leven } = naturalPkg;
|
|
4
4
|
import { testStringOrNumber } from "../../../matching-utils.js";
|
|
5
|
+
import { subfieldToString } from "@natlibfi/marc-record-validators-melinda/dist/utils.js";
|
|
5
6
|
const debug = createDebugLogger("@natlibfi/melinda-record-matching:match-detection:features/bib/title");
|
|
6
7
|
const debugData = debug.extend("data");
|
|
8
|
+
const TITLE_MAX = 0.5;
|
|
7
9
|
export default ({ threshold = 0.9 } = {}) => ({
|
|
8
10
|
name: "Title",
|
|
9
11
|
extract: ({ record, recordExternal }) => {
|
|
@@ -18,45 +20,85 @@ export default ({ threshold = 0.9 } = {}) => ({
|
|
|
18
20
|
}
|
|
19
21
|
return ["", "", "", ""];
|
|
20
22
|
function normalizeTitle(title) {
|
|
21
|
-
return title.normalize("NFD").replace(/[^\p{Letter}\p{Number}]/gu, "").toLowerCase();
|
|
23
|
+
return title.normalize("NFD").replace(/\b(?:ein|ett|I|one|uno|yksi)\b/uig, "1").replace(/\b(?:dos|kaksi|II|två|two|zwei)\b/uig, "2").replace(/\b(?:drei|III|kolme|three|tre|tres)\b/uig, "3").replace(/\b(?:four|fyra|IV|neljä|quat+ro|vier)\b/uig, "4").replace(/\b(?:five|fünf|fyra|viisi|V)\b/uig, "5").replace(/\b(?:kuusi|sechts|sex|six|VI)\b/uig, "6").replace(/\b(and|ja|och|und)\b/uig, "&").replace(/[^\p{Letter}\p{Number}&]/gu, "").toLowerCase();
|
|
22
24
|
}
|
|
23
25
|
},
|
|
24
|
-
compare: (
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
debug(`'${aFull}' vs '${bFull}': Max length = ${maxLength}, distance = ${distance}, correctness = ${correctness}`);
|
|
37
|
-
if (distance === 0) {
|
|
38
|
-
return 0.5;
|
|
39
|
-
}
|
|
40
|
-
if (correctness >= threshold) {
|
|
41
|
-
return 0.4;
|
|
42
|
-
}
|
|
43
|
-
if (an && bn) {
|
|
44
|
-
if (ab && isEmpty(ap) && bp && isEmpty(bb)) {
|
|
45
|
-
return compare([aa, ap, an, ab], [ba, bb, bn, bp]);
|
|
26
|
+
compare: (origA, origB) => {
|
|
27
|
+
return compare2(origA, origB);
|
|
28
|
+
function compare2(a, b) {
|
|
29
|
+
const [aa, ab, an, ap] = a;
|
|
30
|
+
const [ba, bb, bn, bp] = b;
|
|
31
|
+
const aFull = toFullTitle(a);
|
|
32
|
+
const bFull = toFullTitle(b);
|
|
33
|
+
debug(`COMPARE:
|
|
34
|
+
'${aFull}' vs
|
|
35
|
+
'${bFull}'`);
|
|
36
|
+
if (isEmpty(aa) || isEmpty(ba)) {
|
|
37
|
+
return -1;
|
|
46
38
|
}
|
|
47
|
-
if (
|
|
48
|
-
return
|
|
39
|
+
if (aFull === bFull) {
|
|
40
|
+
return TITLE_MAX;
|
|
49
41
|
}
|
|
42
|
+
if (ab && ab !== "" && ab === bb) {
|
|
43
|
+
debug(`Ignore $b ${ab}`);
|
|
44
|
+
return compare2([aa, "", an, ap], [ba, "", bn, bp]);
|
|
45
|
+
}
|
|
46
|
+
if (an && an !== "" && an === bn) {
|
|
47
|
+
debug(`Ignore $n ${an}`);
|
|
48
|
+
return compare2([aa, ab, "", ap], [ba, bb, "", bp]);
|
|
49
|
+
}
|
|
50
|
+
if (ap && ap !== "" && ap === bp) {
|
|
51
|
+
debug(`Ignore $p ${ap}`);
|
|
52
|
+
return compare2([aa, ab, an, ""], [ba, bb, bn, ""]);
|
|
53
|
+
}
|
|
54
|
+
if (!isEmpty(ab) && ab === bp) {
|
|
55
|
+
return compare2([aa, "", an, ap], [ba, bb, bn, ""]);
|
|
56
|
+
}
|
|
57
|
+
if (!isEmpty(ap) && ap === bp) {
|
|
58
|
+
return compare2([aa, ab, an, ""], [ba, "", bn, bp]);
|
|
59
|
+
}
|
|
60
|
+
if (an && bn && an.match(/[0-9]/u)) {
|
|
61
|
+
debug(`NUMBER FOUND. Compare ${an} and ${bn}`);
|
|
62
|
+
const atmp = an.replace(/[^0-9]/ug, "");
|
|
63
|
+
const btmp = bn.replace(/[^0-9]/ug, "");
|
|
64
|
+
if (atmp === btmp) {
|
|
65
|
+
debug(`Ignore $n '${an}' vs '${bn}'`);
|
|
66
|
+
return compare2([aa, ab, "", ap], [ba, bb, "", bp]);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (an && an !== bn) {
|
|
70
|
+
return -1;
|
|
71
|
+
}
|
|
72
|
+
const [distance, maxLength, correctness] = doLevenshtein(aFull, bFull);
|
|
73
|
+
debug(`'${aFull}' vs '${bFull}': Max length = ${maxLength}, distance = ${distance}, correctness = ${correctness}`);
|
|
74
|
+
if (correctness >= threshold) {
|
|
75
|
+
return TITLE_MAX * 0.8;
|
|
76
|
+
}
|
|
77
|
+
if (ab && bb) {
|
|
78
|
+
if (ab.indexOf(bb) === 0) {
|
|
79
|
+
return compare2([aa, ab.substring(bb.length), an, ap], [ba, "", bn, bp]);
|
|
80
|
+
}
|
|
81
|
+
if (bb.indexOf(ab) === 0) {
|
|
82
|
+
return compare2([aa, "", an, ap], [ba, bb.substring(ab.length), bn, bp]);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (localXor(ap, bp)) {
|
|
86
|
+
const result = compare2([aa, ab, an, ""], [ba, bb, bn, ""]);
|
|
87
|
+
return result > 0 ? result * 0.8 : result;
|
|
88
|
+
}
|
|
89
|
+
if (aa === ba && an === bn) {
|
|
90
|
+
const candScore = TITLE_MAX / 2;
|
|
91
|
+
if (ap === bp && localXor(ab, bb)) {
|
|
92
|
+
debug(`Handle omitted $b using x ${candScore}`);
|
|
93
|
+
return candScore;
|
|
94
|
+
}
|
|
95
|
+
if (ab === bb && localXor(ap, bp)) {
|
|
96
|
+
debug(`Handle omitted $p using x ${candScore}`);
|
|
97
|
+
return candScore;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return -0.5;
|
|
50
101
|
}
|
|
51
|
-
if (localXor(ap, bp)) {
|
|
52
|
-
const result = compare([aa, ab, an, ""], [ba, bb, bn, ""]);
|
|
53
|
-
return result > 0 ? result * 0.8 : result;
|
|
54
|
-
}
|
|
55
|
-
if (isEmpty(ap) && isEmpty(bp) && localXor(ab, bb)) {
|
|
56
|
-
const result = compare([aa, "", an, ""], [ba, "", bn, ""]);
|
|
57
|
-
return result > 0 ? result * 0.8 : result;
|
|
58
|
-
}
|
|
59
|
-
return -0.5;
|
|
60
102
|
function isEmpty(x) {
|
|
61
103
|
return !x || x.length === 0;
|
|
62
104
|
}
|
|
@@ -67,14 +109,14 @@ export default ({ threshold = 0.9 } = {}) => ({
|
|
|
67
109
|
return isEmpty(y);
|
|
68
110
|
}
|
|
69
111
|
function doLevenshtein(string1, string2) {
|
|
70
|
-
const
|
|
112
|
+
const distance = leven(string1, string2);
|
|
71
113
|
const len = getMaxLength(string1, string2);
|
|
72
|
-
const
|
|
73
|
-
return [
|
|
114
|
+
const correctness = 1 - distance / len;
|
|
115
|
+
return [distance, len, correctness];
|
|
74
116
|
}
|
|
75
117
|
function toFullTitle(arr) {
|
|
76
118
|
const relevant = arr.filter((val) => typeof val === "string" && val.length);
|
|
77
|
-
return relevant.join("
|
|
119
|
+
return relevant.join("");
|
|
78
120
|
}
|
|
79
121
|
function getMaxLength(str1, str2) {
|
|
80
122
|
return str1.length > str2.length ? str1.length : str2.length;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/match-detection/features/bib/title.js"],
|
|
4
|
-
"sourcesContent": ["import createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\nimport {testStringOrNumber} from '../../../matching-utils.js';\n\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');\nconst debugData = debug.extend('data');\n\nexport default ({threshold = 0.9} = {}) => ({\n name: 'Title',\n extract: ({record, recordExternal}) => {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const a = getTitle(record, ['a']);\n debug(`${label}: title: ${a}`);\n\n if (a) {\n const b = normalizeTitle(getTitle(record, ['b'])) || '';\n const n = normalizeTitle(getTitle(record, ['n'])) || '';\n const p = normalizeTitle(getTitle(record, ['p'])) || '';\n return [normalizeTitle(a), b, n, p];\n }\n\n return ['', '', '', ''];\n\n function normalizeTitle(title) {\n return title\n // decompose unicode diacritics\n .normalize('NFD')\n // strip non-letters/numbers\n // - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 's\u00E4\u00E4' as 'saa'\n // - we could precompose the Finnish letters back to avoid this\n // - see validator normalize-utf8-diacritics for details\n .replace(/[^\\p{Letter}\\p{Number}]/gu, '')\n .toLowerCase();\n }\n\n },\n compare: (
|
|
5
|
-
"mappings": "AAAA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AACrC,SAAQ,0BAAyB;AAGjC,MAAM,QAAQ,kBAAkB,sEAAsE;AACtG,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,eAAe,CAAC,EAAC,YAAY,IAAG,IAAI,CAAC,OAAO;AAAA,EAC1C,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,IAAI,SAAS,QAAQ,CAAC,GAAG,CAAC;AAChC,UAAM,GAAG,KAAK,YAAY,CAAC,EAAE;AAE7B,QAAI,GAAG;AACL,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,aAAO,CAAC,eAAe,CAAC,GAAG,GAAG,GAAG,CAAC;AAAA,IACpC;AAEA,WAAO,CAAC,IAAI,IAAI,IAAI,EAAE;AAEtB,aAAS,eAAe,OAAO;AAC7B,aAAO,MAEJ,UAAU,KAAK,
|
|
6
|
-
"names": [
|
|
4
|
+
"sourcesContent": ["import createDebugLogger from 'debug';\nimport naturalPkg from 'natural';\nconst {LevenshteinDistance: leven} = naturalPkg;\nimport {testStringOrNumber} from '../../../matching-utils.js';\nimport { subfieldToString } from '@natlibfi/marc-record-validators-melinda/dist/utils.js';\n\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');\nconst debugData = debug.extend('data');\n\nconst TITLE_MAX = 0.5;\nexport default ({threshold = 0.9} = {}) => ({\n name: 'Title',\n extract: ({record, recordExternal}) => {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const a = getTitle(record, ['a']);\n debug(`${label}: title: ${a}`);\n\n if (a) {\n const b = normalizeTitle(getTitle(record, ['b'])) || '';\n const n = normalizeTitle(getTitle(record, ['n'])) || '';\n const p = normalizeTitle(getTitle(record, ['p'])) || '';\n return [normalizeTitle(a), b, n, p];\n }\n\n return ['', '', '', ''];\n\n function normalizeTitle(title) {\n return title\n // decompose unicode diacritics\n .normalize('NFD')\n .replace(/\\b(?:ein|ett|I|one|uno|yksi)\\b/uig, '1')\n .replace(/\\b(?:dos|kaksi|II|tv\u00E5|two|zwei)\\b/uig, '2')\n .replace(/\\b(?:drei|III|kolme|three|tre|tres)\\b/uig, '3')\n .replace(/\\b(?:four|fyra|IV|nelj\u00E4|quat+ro|vier)\\b/uig, '4')\n .replace(/\\b(?:five|f\u00FCnf|fyra|viisi|V)\\b/uig, '5')\n .replace(/\\b(?:kuusi|sechts|sex|six|VI)\\b/uig, '6')\n .replace(/\\b(and|ja|och|und)\\b/uig, '&')\n // strip non-letters/numbers\n // - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 's\u00E4\u00E4' as 'saa'\n // - we could precompose the Finnish letters back to avoid this\n // - see validator normalize-utf8-diacritics for details\n .replace(/[^\\p{Letter}\\p{Number}&]/gu, '')\n .toLowerCase();\n }\n\n },\n compare: (origA, origB) => {\n return compare2(origA, origB);\n\n function compare2(a, b) { // Added this wrapper function as I had issues with recursion\n\n const [aa, ab, an, ap] = a;\n const [ba, bb, bn, bp] = b;\n\n const aFull = toFullTitle(a);\n const bFull = toFullTitle(b);\n\n debug(`COMPARE:\\n '${aFull}' vs \\n '${bFull}'`);\n\n if (isEmpty(aa) || isEmpty(ba)) {\n return -1.0;\n }\n\n if (aFull === bFull) {\n return TITLE_MAX;\n }\n\n\n if (ab && ab !== '' && ab === bb) { // Remove $b from equation (MELKEHITYS-3494)\n debug(`Ignore \\$b ${ab}`);\n return compare2([aa, '', an, ap], [ba, '', bn, bp]);\n }\n if (an && an !== '' && an === bn) { // Remove $n from equation\n debug(`Ignore \\$n ${an}`);\n return compare2([aa, ab, '', ap], [ba, bb, '', bp]);\n }\n if (ap && ap !== '' && ap === bp) { // Remove $p from equation\n debug(`Ignore \\$p ${ap}`);\n return compare2([aa, ab, an, ''], [ba, bb, bn, '']);\n }\n // There seems to be wobble between $b and $p\n if (!isEmpty(ab) && ab === bp) {\n return compare2([aa, '', an, ap], [ba, bb, bn, '']);\n }\n if (!isEmpty(ap) && ap === bp) {\n return compare2([aa, ab, an, ''], [ba, '', bn, bp]);\n }\n\n if (an && bn && an.match(/[0-9]/u)) {\n debug(`NUMBER FOUND. Compare ${an} and ${bn}`);\n const atmp = an.replace(/[^0-9]/ug, '');\n const btmp = bn.replace(/[^0-9]/ug, '');\n if (atmp === btmp) {\n debug(`Ignore \\$n '${an}' vs '${bn}'`);\n return compare2([aa, ab, '', ap], [ba, bb, '', bp]);\n }\n }\n\n\n // f245$n information is critical; it can not mismatch at all (exceptions have already been handled above):\n if (an && an !== bn) {\n return -1.0;\n }\n\n const [distance, maxLength, correctness] = doLevenshtein(aFull, bFull);\n\n debug(`'${aFull}' vs '${bFull}': Max length = ${maxLength}, distance = ${distance}, correctness = ${correctness}`);\n\n\n if (correctness >= threshold) {\n return TITLE_MAX * 0.8;\n }\n\n // Subset removal (MELKEHITYS-3498-ish)\n if (ab && bb) { // At this point ab and bb are never equal...\n // If X is a subset of Y, then remove X and shorten Y (= remove the shared content)\n if (ab.indexOf(bb) === 0) {\n return compare2([aa, ab.substring(bb.length), an, ap], [ba, '', bn, bp]);\n }\n if (bb.indexOf(ab) === 0) {\n return compare2([aa, '', an, ap], [ba, bb.substring(ab.length), bn, bp]);\n }\n }\n \n // Try the same without $p:\n if (localXor(ap, bp)) {\n const result = compare2([aa, ab, an, ''], [ba, bb, bn, '']);\n return result > 0.0 ? result * 0.8 : result;\n }\n \n if (aa === ba && an === bn) {\n // No $n: Don't score $a vs $ab. Return 0, and let other match detectors decide\n const candScore = TITLE_MAX/2;\n if (ap === bp && localXor(ab, bb)) {\n debug(`Handle omitted \\$b using x ${candScore}`);\n return candScore;\n }\n if (ab === bb && localXor(ap, bp)) {\n debug(`Handle omitted \\$p using x ${candScore}`);\n return candScore;\n }\n }\n\n return -0.5; // Not likely\n }\n\n function isEmpty(x) {\n return !x || x.length === 0;\n }\n\n function localXor(x, y) {\n if (isEmpty(x)) {\n return !isEmpty(y);\n }\n // 'x' exists, thus 'y' can not exist:\n return isEmpty(y);\n }\n\n function doLevenshtein(string1, string2) {\n const distance = leven(string1, string2);\n const len = getMaxLength(string1, string2);\n const correctness = 1.0 - (distance / len);\n return [distance, len, correctness];\n }\n\n function toFullTitle(arr) {\n const relevant = arr.filter(val => typeof val === 'string' && val.length);\n // No longer add space between subfields. Due to heavy normalization there are no spaces left in array elements either!\n return relevant.join('');\n }\n\n function getMaxLength(str1, str2) {\n return str1.length > str2.length ? str1.length : str2.length;\n }\n\n }\n});\n\nexport function getTitle(record, targetSubfieldCodes = ['a', 'b', 'n', 'p']) {\n const [field] = record.get(/^245$/u);\n debugData(`titleField: ${JSON.stringify(field)}`);\n\n if (field) {\n const title = field.subfields\n // get also $n:s and $p:s here\n .filter(({code}) => targetSubfieldCodes.includes(code))\n // Would be nice to normalize $n values...\n .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n .join(' ')\n .replace(/\\[[^\\]]*\\]/ug, ' ') // Remove [whatever] stuff. ADD TEST FOR THIS\n .replace(/ [=\\/:](?:$| )/ug, ' ') // Strip punctuation\n // Also \u0111 vs d pairs seen:\n //.replace(/(?:\u0111|\u0227|t\u0304|k\u030C|\u01E7|s\u0306|c\u0306)/ug, '') // Hack. Saamelaisbibliografia has often dropped this wierd characters (oft old articles, no longer used in sami either)\n // trim:\n .replace(/ +/ug, ' ')\n .replace(/^ +/u, '')\n .replace(/ +$/u, '');\n\n // Skip non-filing indicator (note that '9' is a magic indicator value, so we don't do it):\n if (/^[1-8]$/u.test(field.ind2)) { // Skip non-filing characters\n return title.slice(parseInt(field.ind2));\n }\n return title;\n\n }\n return false;\n}"],
|
|
5
|
+
"mappings": "AAAA,OAAO,uBAAuB;AAC9B,OAAO,gBAAgB;AACvB,MAAM,EAAC,qBAAqB,MAAK,IAAI;AACrC,SAAQ,0BAAyB;AACjC,SAAS,wBAAwB;AAGjC,MAAM,QAAQ,kBAAkB,sEAAsE;AACtG,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,MAAM,YAAY;AAClB,eAAe,CAAC,EAAC,YAAY,IAAG,IAAI,CAAC,OAAO;AAAA,EAC1C,MAAM;AAAA,EACN,SAAS,CAAC,EAAC,QAAQ,eAAc,MAAM;AACrC,UAAM,QAAQ,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAC9E,UAAM,IAAI,SAAS,QAAQ,CAAC,GAAG,CAAC;AAChC,UAAM,GAAG,KAAK,YAAY,CAAC,EAAE;AAE7B,QAAI,GAAG;AACL,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,YAAM,IAAI,eAAe,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK;AACrD,aAAO,CAAC,eAAe,CAAC,GAAG,GAAG,GAAG,CAAC;AAAA,IACpC;AAEA,WAAO,CAAC,IAAI,IAAI,IAAI,EAAE;AAEtB,aAAS,eAAe,OAAO;AAC7B,aAAO,MAEJ,UAAU,KAAK,EACf,QAAQ,qCAAqC,GAAG,EAChD,QAAQ,wCAAwC,GAAG,EACnD,QAAQ,4CAA4C,GAAG,EACvD,QAAQ,8CAA8C,GAAG,EACzD,QAAQ,qCAAqC,GAAG,EAChD,QAAQ,sCAAsC,GAAG,EACjD,QAAQ,2BAA2B,GAAG,EAKtC,QAAQ,8BAA8B,EAAE,EACxC,YAAY;AAAA,IACjB;AAAA,EAEF;AAAA,EACA,SAAS,CAAC,OAAO,UAAU;AACzB,WAAO,SAAS,OAAO,KAAK;AAE5B,aAAS,SAAS,GAAG,GAAG;AAEtB,YAAM,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI;AACzB,YAAM,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI;AAEzB,YAAM,QAAQ,YAAY,CAAC;AAC3B,YAAM,QAAQ,YAAY,CAAC;AAE3B,YAAM;AAAA,KAAgB,KAAK;AAAA,KAAc,KAAK,GAAG;AAEjD,UAAI,QAAQ,EAAE,KAAK,QAAQ,EAAE,GAAG;AAC9B,eAAO;AAAA,MACT;AAEA,UAAI,UAAU,OAAO;AACnB,eAAO;AAAA,MACT;AAGA,UAAI,MAAM,OAAO,MAAM,OAAO,IAAI;AAChC,cAAM,aAAc,EAAE,EAAE;AACxB,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AACA,UAAI,MAAM,OAAO,MAAM,OAAO,IAAI;AAChC,cAAM,aAAc,EAAE,EAAE;AACxB,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AACA,UAAI,MAAM,OAAO,MAAM,OAAO,IAAI;AAChC,cAAM,aAAc,EAAE,EAAE;AACxB,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AAEA,UAAI,CAAC,QAAQ,EAAE,KAAK,OAAO,IAAI;AAC7B,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AACA,UAAI,CAAC,QAAQ,EAAE,KAAK,OAAO,IAAI;AAC7B,eAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,MACpD;AAEA,UAAI,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG;AAClC,cAAM,yBAAyB,EAAE,QAAQ,EAAE,EAAE;AAC7C,cAAM,OAAO,GAAG,QAAQ,YAAY,EAAE;AACtC,cAAM,OAAO,GAAG,QAAQ,YAAY,EAAE;AACtC,YAAI,SAAS,MAAM;AACjB,gBAAM,cAAe,EAAE,SAAS,EAAE,GAAG;AACrC,iBAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,QACpD;AAAA,MACF;AAIA,UAAI,MAAM,OAAO,IAAI;AACnB,eAAO;AAAA,MACT;AAEA,YAAM,CAAC,UAAU,WAAW,WAAW,IAAI,cAAc,OAAO,KAAK;AAErE,YAAM,IAAI,KAAK,SAAS,KAAK,mBAAmB,SAAS,gBAAgB,QAAQ,mBAAmB,WAAW,EAAE;AAGjH,UAAI,eAAe,WAAW;AAC5B,eAAO,YAAY;AAAA,MACrB;AAGA,UAAI,MAAM,IAAI;AAEZ,YAAI,GAAG,QAAQ,EAAE,MAAM,GAAG;AACxB,iBAAO,SAAS,CAAC,IAAI,GAAG,UAAU,GAAG,MAAM,GAAG,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAAA,QACzE;AACA,YAAI,GAAG,QAAQ,EAAE,MAAM,GAAG;AACxB,iBAAO,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,UAAU,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;AAAA,QACzE;AAAA,MACF;AAGA,UAAI,SAAS,IAAI,EAAE,GAAG;AACpB,cAAM,SAAS,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;AAC1D,eAAO,SAAS,IAAM,SAAS,MAAM;AAAA,MACvC;AAEA,UAAI,OAAO,MAAM,OAAO,IAAI;AAE1B,cAAM,YAAY,YAAU;AAC5B,YAAI,OAAO,MAAM,SAAS,IAAI,EAAE,GAAG;AACjC,gBAAM,6BAA8B,SAAS,EAAE;AAC/C,iBAAO;AAAA,QACT;AACA,YAAI,OAAO,MAAM,SAAS,IAAI,EAAE,GAAG;AACjC,gBAAM,6BAA8B,SAAS,EAAE;AAC/C,iBAAO;AAAA,QACT;AAAA,MACF;AAEA,aAAO;AAAA,IACT;AAEA,aAAS,QAAQ,GAAG;AAClB,aAAO,CAAC,KAAK,EAAE,WAAW;AAAA,IAC5B;AAEA,aAAS,SAAS,GAAG,GAAG;AACtB,UAAI,QAAQ,CAAC,GAAG;AACd,eAAO,CAAC,QAAQ,CAAC;AAAA,MACnB;AAEA,aAAO,QAAQ,CAAC;AAAA,IAClB;AAEA,aAAS,cAAc,SAAS,SAAS;AACvC,YAAM,WAAW,MAAM,SAAS,OAAO;AACvC,YAAM,MAAM,aAAa,SAAS,OAAO;AACzC,YAAM,cAAc,IAAO,WAAW;AACtC,aAAO,CAAC,UAAU,KAAK,WAAW;AAAA,IACpC;AAEA,aAAS,YAAY,KAAK;AACxB,YAAM,WAAW,IAAI,OAAO,SAAO,OAAO,QAAQ,YAAY,IAAI,MAAM;AAExE,aAAO,SAAS,KAAK,EAAE;AAAA,IACzB;AAEA,aAAS,aAAa,MAAM,MAAM;AAChC,aAAO,KAAK,SAAS,KAAK,SAAS,KAAK,SAAS,KAAK;AAAA,IACxD;AAAA,EAEF;AACF;AAEO,gBAAS,SAAS,QAAQ,sBAAsB,CAAC,KAAK,KAAK,KAAK,GAAG,GAAG;AAC3E,QAAM,CAAC,KAAK,IAAI,OAAO,IAAI,QAAQ;AACnC,YAAU,eAAe,KAAK,UAAU,KAAK,CAAC,EAAE;AAEhD,MAAI,OAAO;AACT,UAAM,QAAQ,MAAM,UAEjB,OAAO,CAAC,EAAC,KAAI,MAAM,oBAAoB,SAAS,IAAI,CAAC,EAErD,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,KAAK,GAAG,EACR,QAAQ,gBAAgB,GAAG,EAC3B,QAAQ,oBAAoB,GAAG,EAI/B,QAAQ,QAAQ,GAAG,EACnB,QAAQ,QAAQ,EAAE,EAClB,QAAQ,QAAQ,EAAE;AAGrB,QAAI,WAAW,KAAK,MAAM,IAAI,GAAG;AAC/B,aAAO,MAAM,MAAM,SAAS,MAAM,IAAI,CAAC;AAAA,IACzC;AACA,WAAO;AAAA,EAET;AACA,SAAO;AACT;",
|
|
6
|
+
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"url": "https://github.com/NatLibFi/melinda-record-matching-js"
|
|
14
14
|
},
|
|
15
15
|
"license": "MIT",
|
|
16
|
-
"version": "5.1.
|
|
16
|
+
"version": "5.1.3",
|
|
17
17
|
"type": "module",
|
|
18
18
|
"main": "./dist/index.js",
|
|
19
19
|
"bin": "./dist/cli.js",
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"@natlibfi/melinda-commons": "^14.0.2",
|
|
46
46
|
"@natlibfi/sru-client": "^7.0.1",
|
|
47
47
|
"debug": "^4.4.3",
|
|
48
|
-
"isbn3": "^2.0.
|
|
48
|
+
"isbn3": "^2.0.8",
|
|
49
49
|
"moment": "^2.30.1",
|
|
50
50
|
"natural": "^8.1.1",
|
|
51
51
|
"uuid": "^14.0.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"@natlibfi/fixura": "^4.0.1",
|
|
58
58
|
"cross-env": "^10.1.0",
|
|
59
59
|
"esbuild": "^0.28.0",
|
|
60
|
-
"eslint": "^10.4.
|
|
60
|
+
"eslint": "^10.4.1"
|
|
61
61
|
},
|
|
62
62
|
"overrides": {
|
|
63
63
|
"uuid": "^14.0.0"
|
|
@@ -129,12 +129,6 @@ export default () => ({
|
|
|
129
129
|
return -0.1;
|
|
130
130
|
}
|
|
131
131
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
function containsNoData(languageCode) {
|
|
135
|
-
return [' ', '|||', 'und'].includes(languageCode);
|
|
136
|
-
}
|
|
137
|
-
|
|
138
132
|
function compareLanguageCodeLists(a, b) {
|
|
139
133
|
if (a.length === 0 || b.length === 0 || a.every(val => containsNoData(val)) || b.every(val => containsNoData(val))) {
|
|
140
134
|
debugData(`No language to compare`);
|
|
@@ -170,36 +164,50 @@ export default () => ({
|
|
|
170
164
|
return 0;
|
|
171
165
|
}
|
|
172
166
|
|
|
173
|
-
|
|
174
|
-
|
|
175
167
|
// Damage control:
|
|
176
|
-
const sharedValues = getSharedValues(a, b)
|
|
168
|
+
const sharedValues = getSharedValues(a, b); // NB! excludes meaningless values
|
|
177
169
|
|
|
178
170
|
if (sharedValues.length < 1) {
|
|
179
|
-
debug(`Both have languages, but none of these match.`);
|
|
171
|
+
debug(`Both have languages, but none of these match.`);
|
|
180
172
|
return -1.0;
|
|
181
173
|
}
|
|
182
174
|
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
debug(`Both have languages, ${matchingValues}/${possibleMatchValues} valid languages match.`);
|
|
186
|
-
// ignore non-matches if there is mismatching amount of values
|
|
187
|
-
debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);
|
|
188
|
-
// we give some kind of penalty for mismatching amount of values instead of simple divide?
|
|
189
|
-
const missingCount = maxValues - possibleMatchValues;
|
|
190
|
-
const misMatchCount = possibleMatchValues - matchingValues;
|
|
191
|
-
debug(`\t missing: ${missingCount}`);
|
|
192
|
-
debug(`\t mismatches: ${misMatchCount}`);
|
|
175
|
+
const aOnly = getUniqueInFirst(a, b); // NB! excludes meaningless values
|
|
176
|
+
const bOnly = getUniqueInFirst(b, a);
|
|
193
177
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
debug(`\t points: penaltyForMissing: ${penaltyForMissing}`);
|
|
197
|
-
debug(`\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);
|
|
178
|
+
debug(`A only: ${aOnly.join(', ')}`);
|
|
179
|
+
debug(`B only: ${bOnly.join(', ')}`);
|
|
198
180
|
|
|
199
|
-
|
|
200
|
-
debug(`Total points: ${points}`);
|
|
181
|
+
return calculatePenalty();
|
|
201
182
|
|
|
202
|
-
|
|
183
|
+
function calculatePenalty() {
|
|
184
|
+
const basePenalty = calculatePenalty2() / sharedValues.length;
|
|
185
|
+
if (basePenalty < -1.0) {
|
|
186
|
+
return -1.0;
|
|
187
|
+
}
|
|
188
|
+
return basePenalty;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function calculatePenalty2() {
|
|
192
|
+
const n = aOnly.length + bOnly.length;
|
|
193
|
+
// It's reasonably ok for one to have more languages than the other (max penalty -0.3 for that):
|
|
194
|
+
if (aOnly.length === 0 || bOnly.length === 0) {
|
|
195
|
+
if (n > 3) {
|
|
196
|
+
return -0.3;
|
|
197
|
+
}
|
|
198
|
+
return 0.1 - (n * 0.1);
|
|
199
|
+
}
|
|
200
|
+
// Possible typo (should we calculate levenshtein distance for the value?):
|
|
201
|
+
if (aOnly.length === 1 && bOnly.length === 1) {
|
|
202
|
+
return -0.1; // Not too big a penalty
|
|
203
|
+
}
|
|
204
|
+
debug(` ${n} differences altogether`);
|
|
205
|
+
// However, it's worse to have different languages on both sides
|
|
206
|
+
if (n > 3) {
|
|
207
|
+
return -1.0;
|
|
208
|
+
}
|
|
209
|
+
return n * -0.2;
|
|
210
|
+
}
|
|
203
211
|
}
|
|
204
212
|
}
|
|
205
213
|
|
|
@@ -282,5 +290,13 @@ function getFieldsLanguageCodes(field) {
|
|
|
282
290
|
}
|
|
283
291
|
|
|
284
292
|
function getSharedValues(list1, list2) {
|
|
285
|
-
return list1.filter(val => list2.includes(val));
|
|
293
|
+
return list1.filter(val => list2.includes(val) && !containsNoData(val));
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function getUniqueInFirst(list1, list2){
|
|
297
|
+
return list1.filter(val => !list2.includes(val) && !containsNoData(val));
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function containsNoData(languageCode) {
|
|
301
|
+
return [' ', '|||', 'und'].includes(languageCode);
|
|
286
302
|
}
|
|
@@ -1,10 +1,39 @@
|
|
|
1
|
-
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
1
|
+
// RecordType
|
|
2
|
+
//
|
|
3
|
+
// Array structure:
|
|
4
|
+
// 0: record type
|
|
5
|
+
// 1: is soitonopas; The English translation for soitionopas is both terrible and unused in Melinda, that I'm not using it ("musical instrument methods")
|
|
5
6
|
|
|
6
7
|
export default () => ({
|
|
7
8
|
name: 'Record type',
|
|
8
|
-
extract: ({record}) =>
|
|
9
|
-
|
|
9
|
+
extract: ({record}) => {
|
|
10
|
+
const recordType = record.leader[6] || null;
|
|
11
|
+
const soitonopas = isSoitonopas();
|
|
12
|
+
|
|
13
|
+
return [recordType, soitonopas];
|
|
14
|
+
|
|
15
|
+
function isSoitonopas() {
|
|
16
|
+
if (recordType === 'c') {
|
|
17
|
+
const f300 = record.get('300');
|
|
18
|
+
if (f300.some(f => f.subfields?.some(sf => sf.code === 'a' && sf.value.match(/(?:instrumentskol|soitonopas)/ui)))) {
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
compare: (a, b) => {
|
|
26
|
+
if (a[0] === b[0]) {
|
|
27
|
+
return 0.1;
|
|
28
|
+
}
|
|
29
|
+
// Soitonopas vs text/book: no bonus, no penalty
|
|
30
|
+
if (a[0] === 'a' && b[1]) {
|
|
31
|
+
return 0.0;
|
|
32
|
+
}
|
|
33
|
+
if (b[0] === 'a' && a[1]) {
|
|
34
|
+
return 0.0;
|
|
35
|
+
}
|
|
36
|
+
// Bad:
|
|
37
|
+
return -0.5
|
|
38
|
+
}
|
|
10
39
|
});
|
|
@@ -2,11 +2,13 @@ import createDebugLogger from 'debug';
|
|
|
2
2
|
import naturalPkg from 'natural';
|
|
3
3
|
const {LevenshteinDistance: leven} = naturalPkg;
|
|
4
4
|
import {testStringOrNumber} from '../../../matching-utils.js';
|
|
5
|
+
import { subfieldToString } from '@natlibfi/marc-record-validators-melinda/dist/utils.js';
|
|
5
6
|
|
|
6
7
|
|
|
7
8
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/title');
|
|
8
9
|
const debugData = debug.extend('data');
|
|
9
10
|
|
|
11
|
+
const TITLE_MAX = 0.5;
|
|
10
12
|
export default ({threshold = 0.9} = {}) => ({
|
|
11
13
|
name: 'Title',
|
|
12
14
|
extract: ({record, recordExternal}) => {
|
|
@@ -27,65 +29,121 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
27
29
|
return title
|
|
28
30
|
// decompose unicode diacritics
|
|
29
31
|
.normalize('NFD')
|
|
32
|
+
.replace(/\b(?:ein|ett|I|one|uno|yksi)\b/uig, '1')
|
|
33
|
+
.replace(/\b(?:dos|kaksi|II|två|two|zwei)\b/uig, '2')
|
|
34
|
+
.replace(/\b(?:drei|III|kolme|three|tre|tres)\b/uig, '3')
|
|
35
|
+
.replace(/\b(?:four|fyra|IV|neljä|quat+ro|vier)\b/uig, '4')
|
|
36
|
+
.replace(/\b(?:five|fünf|fyra|viisi|V)\b/uig, '5')
|
|
37
|
+
.replace(/\b(?:kuusi|sechts|sex|six|VI)\b/uig, '6')
|
|
38
|
+
.replace(/\b(and|ja|och|und)\b/uig, '&')
|
|
30
39
|
// strip non-letters/numbers
|
|
31
40
|
// - note: combined with decomposing unicode diacritics this normalizes both 'saa' and 'sää' as 'saa'
|
|
32
41
|
// - we could precompose the Finnish letters back to avoid this
|
|
33
42
|
// - see validator normalize-utf8-diacritics for details
|
|
34
|
-
.replace(/[^\p{Letter}\p{Number}]/gu, '')
|
|
43
|
+
.replace(/[^\p{Letter}\p{Number}&]/gu, '')
|
|
35
44
|
.toLowerCase();
|
|
36
45
|
}
|
|
37
46
|
|
|
38
47
|
},
|
|
39
|
-
compare: (
|
|
40
|
-
|
|
41
|
-
const [ba, bb, bn, bp] = b;
|
|
48
|
+
compare: (origA, origB) => {
|
|
49
|
+
return compare2(origA, origB);
|
|
42
50
|
|
|
43
|
-
|
|
44
|
-
return -1.0;
|
|
45
|
-
}
|
|
46
|
-
// F245$n information is critical; it can not mismatch at all:
|
|
47
|
-
if (an.length && bn.length && an !== bn) { // If these exists, they must be the same (we might convert Roman numbers to Arabic numbers though)
|
|
48
|
-
return -1.0;
|
|
49
|
-
}
|
|
51
|
+
function compare2(a, b) { // Added this wrapper function as I had issues with recursion
|
|
50
52
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
+
const [aa, ab, an, ap] = a;
|
|
54
|
+
const [ba, bb, bn, bp] = b;
|
|
53
55
|
|
|
54
|
-
|
|
56
|
+
const aFull = toFullTitle(a);
|
|
57
|
+
const bFull = toFullTitle(b);
|
|
55
58
|
|
|
56
|
-
|
|
59
|
+
debug(`COMPARE:\n '${aFull}' vs \n '${bFull}'`);
|
|
57
60
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
+
if (isEmpty(aa) || isEmpty(ba)) {
|
|
62
|
+
return -1.0;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (aFull === bFull) {
|
|
66
|
+
return TITLE_MAX;
|
|
67
|
+
}
|
|
61
68
|
|
|
62
|
-
if (correctness >= threshold) {
|
|
63
|
-
return 0.4;
|
|
64
|
-
}
|
|
65
69
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
return compare([aa, ap, an, ab], [ba, bb, bn, bp]);
|
|
70
|
+
if (ab && ab !== '' && ab === bb) { // Remove $b from equation (MELKEHITYS-3494)
|
|
71
|
+
debug(`Ignore \$b ${ab}`);
|
|
72
|
+
return compare2([aa, '', an, ap], [ba, '', bn, bp]);
|
|
70
73
|
}
|
|
71
|
-
if (
|
|
72
|
-
|
|
74
|
+
if (an && an !== '' && an === bn) { // Remove $n from equation
|
|
75
|
+
debug(`Ignore \$n ${an}`);
|
|
76
|
+
return compare2([aa, ab, '', ap], [ba, bb, '', bp]);
|
|
77
|
+
}
|
|
78
|
+
if (ap && ap !== '' && ap === bp) { // Remove $p from equation
|
|
79
|
+
debug(`Ignore \$p ${ap}`);
|
|
80
|
+
return compare2([aa, ab, an, ''], [ba, bb, bn, '']);
|
|
81
|
+
}
|
|
82
|
+
// There seems to be wobble between $b and $p
|
|
83
|
+
if (!isEmpty(ab) && ab === bp) {
|
|
84
|
+
return compare2([aa, '', an, ap], [ba, bb, bn, '']);
|
|
85
|
+
}
|
|
86
|
+
if (!isEmpty(ap) && ap === bp) {
|
|
87
|
+
return compare2([aa, ab, an, ''], [ba, '', bn, bp]);
|
|
73
88
|
}
|
|
74
|
-
}
|
|
75
89
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
90
|
+
if (an && bn && an.match(/[0-9]/u)) {
|
|
91
|
+
debug(`NUMBER FOUND. Compare ${an} and ${bn}`);
|
|
92
|
+
const atmp = an.replace(/[^0-9]/ug, '');
|
|
93
|
+
const btmp = bn.replace(/[^0-9]/ug, '');
|
|
94
|
+
if (atmp === btmp) {
|
|
95
|
+
debug(`Ignore \$n '${an}' vs '${bn}'`);
|
|
96
|
+
return compare2([aa, ab, '', ap], [ba, bb, '', bp]);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
81
99
|
|
|
82
|
-
if (isEmpty(ap) && isEmpty(bp) && localXor(ab, bb)) {
|
|
83
|
-
// Try the same without $b ($p is not here)
|
|
84
|
-
const result = compare([aa, '', an, ''], [ba, '', bn, '']);
|
|
85
|
-
return result > 0.0 ? result * 0.8 : result;
|
|
86
|
-
}
|
|
87
100
|
|
|
88
|
-
|
|
101
|
+
// f245$n information is critical; it can not mismatch at all (exceptions have already been handled above):
|
|
102
|
+
if (an && an !== bn) {
|
|
103
|
+
return -1.0;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const [distance, maxLength, correctness] = doLevenshtein(aFull, bFull);
|
|
107
|
+
|
|
108
|
+
debug(`'${aFull}' vs '${bFull}': Max length = ${maxLength}, distance = ${distance}, correctness = ${correctness}`);
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
if (correctness >= threshold) {
|
|
112
|
+
return TITLE_MAX * 0.8;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Subset removal (MELKEHITYS-3498-ish)
|
|
116
|
+
if (ab && bb) { // At this point ab and bb are never equal...
|
|
117
|
+
// If X is a subset of Y, then remove X and shorten Y (= remove the shared content)
|
|
118
|
+
if (ab.indexOf(bb) === 0) {
|
|
119
|
+
return compare2([aa, ab.substring(bb.length), an, ap], [ba, '', bn, bp]);
|
|
120
|
+
}
|
|
121
|
+
if (bb.indexOf(ab) === 0) {
|
|
122
|
+
return compare2([aa, '', an, ap], [ba, bb.substring(ab.length), bn, bp]);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Try the same without $p:
|
|
127
|
+
if (localXor(ap, bp)) {
|
|
128
|
+
const result = compare2([aa, ab, an, ''], [ba, bb, bn, '']);
|
|
129
|
+
return result > 0.0 ? result * 0.8 : result;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (aa === ba && an === bn) {
|
|
133
|
+
// No $n: Don't score $a vs $ab. Return 0, and let other match detectors decide
|
|
134
|
+
const candScore = TITLE_MAX/2;
|
|
135
|
+
if (ap === bp && localXor(ab, bb)) {
|
|
136
|
+
debug(`Handle omitted \$b using x ${candScore}`);
|
|
137
|
+
return candScore;
|
|
138
|
+
}
|
|
139
|
+
if (ab === bb && localXor(ap, bp)) {
|
|
140
|
+
debug(`Handle omitted \$p using x ${candScore}`);
|
|
141
|
+
return candScore;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return -0.5; // Not likely
|
|
146
|
+
}
|
|
89
147
|
|
|
90
148
|
function isEmpty(x) {
|
|
91
149
|
return !x || x.length === 0;
|
|
@@ -108,7 +166,8 @@ export default ({threshold = 0.9} = {}) => ({
|
|
|
108
166
|
|
|
109
167
|
function toFullTitle(arr) {
|
|
110
168
|
const relevant = arr.filter(val => typeof val === 'string' && val.length);
|
|
111
|
-
|
|
169
|
+
// No longer add space between subfields. Due to heavy normalization there are no spaces left in array elements either!
|
|
170
|
+
return relevant.join('');
|
|
112
171
|
}
|
|
113
172
|
|
|
114
173
|
function getMaxLength(str1, str2) {
|