@natlibfi/melinda-record-matching 5.0.3 → 5.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -24,44 +24,60 @@ export default () => ({
|
|
|
24
24
|
return 0;
|
|
25
25
|
}
|
|
26
26
|
const [subfieldCodeForGoodValues, subfieldCodeForBadValues] = getSubfieldCodes(aa[0].tag);
|
|
27
|
-
const [
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
27
|
+
const [aValidValuesA, aInvalidValuesA, zValidValuesA, zInvalidValuesA] = getValuesWrapper(aa, "AA");
|
|
28
|
+
const [aValidValuesB, aInvalidValuesB, zValidValuesB, zInvalidValuesB] = getValuesWrapper(bb, "BB");
|
|
29
|
+
function getValuesWrapper(data, prefix) {
|
|
30
|
+
const [aValidValues, aInvalidValues, zValidValues, zInvalidValues] = getValues(data);
|
|
31
|
+
if (aValidValues.length) {
|
|
32
|
+
debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForGoodValues} VALID: ${aValidValues.join(", ")}`);
|
|
33
|
+
}
|
|
34
|
+
if (aInvalidValues.length) {
|
|
35
|
+
debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForGoodValues} INVALID: ${aInvalidValues.join(", ")}`);
|
|
36
|
+
}
|
|
37
|
+
if (zValidValues.length) {
|
|
38
|
+
debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForBadValues} VALID: ${zValidValues.join(", ")}`);
|
|
39
|
+
}
|
|
40
|
+
if (zInvalidValues.length) {
|
|
41
|
+
debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForBadValues} INVALID: ${zInvalidValues.join(", ")}`);
|
|
42
|
+
}
|
|
43
|
+
return [aValidValues, aInvalidValues, zValidValues, zInvalidValues];
|
|
37
44
|
}
|
|
38
|
-
|
|
39
|
-
|
|
45
|
+
const [sharedGoodValues, goodValuesAOnly, goodValuesBOnly] = getUnionData(aValidValuesA, aValidValuesB);
|
|
46
|
+
debug(`GOOD BOTH: ${sharedGoodValues.length}, A only: ${goodValuesAOnly.length}, B only: ${goodValuesBOnly.length}`);
|
|
47
|
+
if (sharedGoodValues.length > 0) {
|
|
48
|
+
return scoreData(MAX_SCORE, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);
|
|
40
49
|
}
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
if (sharedGoodValues.length > 0) {
|
|
50
|
+
const hitScore = scoreSuboptimalHit();
|
|
51
|
+
function scoreSuboptimalHit() {
|
|
52
|
+
if (aValidValuesA.some((valA) => zValidValuesB.includes(valA)) || aValidValuesB.some((valB) => zValidValuesA.includes(valB))) {
|
|
45
53
|
return MAX_SCORE;
|
|
46
54
|
}
|
|
47
|
-
|
|
48
|
-
|
|
55
|
+
const zzValid = zValidValuesA.find((valA) => zValidValuesB.includes(valA));
|
|
56
|
+
if (zzValid) {
|
|
57
|
+
debug(`Both contain a valid value in 020$z: ${zzValid}`);
|
|
58
|
+
return MAX_SCORE * 2 / 3;
|
|
49
59
|
}
|
|
50
|
-
|
|
60
|
+
const aaInvalid = aInvalidValuesA.find((valA) => aInvalidValuesB.includes(valA) || zInvalidValuesB.includes(valA)) || aInvalidValuesB.find((valB) => zInvalidValuesA.includes(valB));
|
|
61
|
+
if (aaInvalid) {
|
|
62
|
+
debug(`Shared invalid value in 020$a and 020$a-or-$z subfields: ${aaInvalid}`);
|
|
51
63
|
return MAX_SCORE * 2 / 3;
|
|
52
64
|
}
|
|
53
65
|
return 0;
|
|
54
66
|
}
|
|
55
|
-
if (sharedGoodValues.length > 0) {
|
|
56
|
-
return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);
|
|
57
|
-
}
|
|
58
67
|
if (hitScore === MAX_SCORE) {
|
|
59
68
|
return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length - 1);
|
|
60
69
|
}
|
|
61
70
|
if (hitScore > 0) {
|
|
62
71
|
return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);
|
|
63
72
|
}
|
|
64
|
-
if (
|
|
73
|
+
if (aValidValuesA.length + aInvalidValuesA.length === 0 || aValidValuesB.length + aInvalidValuesB.length === 0) {
|
|
74
|
+
return 0;
|
|
75
|
+
}
|
|
76
|
+
const allA = [...aValidValuesA, ...aInvalidValuesA, ...zValidValuesA, ...zInvalidValuesA];
|
|
77
|
+
const allB = [...aValidValuesB, ...aInvalidValuesB, ...zValidValuesB, ...zInvalidValuesB];
|
|
78
|
+
const [sharedValues, tmp1, tmp2] = getUnionData(allA, allB);
|
|
79
|
+
debug(`WHATEVER BOTH: ${sharedGoodValues.length}, A only: ${tmp1.length}, B only: ${tmp2.length}`);
|
|
80
|
+
if (sharedValues.length > 0) {
|
|
65
81
|
return 0;
|
|
66
82
|
}
|
|
67
83
|
return -0.75;
|
|
@@ -71,24 +87,17 @@ export default () => ({
|
|
|
71
87
|
}
|
|
72
88
|
return ["a", "z"];
|
|
73
89
|
}
|
|
74
|
-
function looksGood(val) {
|
|
75
|
-
if (/^([0-9]-?){9}[0-9X]$/u.test(val)) {
|
|
76
|
-
return true;
|
|
77
|
-
}
|
|
78
|
-
if (/^([0-9]-?){12}[0-9]$/u.test(val)) {
|
|
79
|
-
return true;
|
|
80
|
-
}
|
|
81
|
-
return false;
|
|
82
|
-
}
|
|
83
90
|
function getValues(fields) {
|
|
84
91
|
const goodValues = fields.flatMap((f) => f.subfields.filter((sf) => sf.code === subfieldCodeForGoodValues)).map((sf) => validatorAndNormalizer(sf.value));
|
|
85
92
|
const trueGoodValues = goodValues.filter((val) => val.valid).map((val) => val.value);
|
|
86
93
|
const wannabeGoodValues = goodValues.filter((val) => !val.valid).map((val) => val.value);
|
|
87
94
|
if (!subfieldCodeForBadValues) {
|
|
88
|
-
return [trueGoodValues, wannabeGoodValues];
|
|
95
|
+
return [trueGoodValues, wannabeGoodValues, [], []];
|
|
89
96
|
}
|
|
90
|
-
const badValues = fields.flatMap((f) => f.subfields.filter((sf) => sf.code === subfieldCodeForBadValues)).map((sf) => sf.value);
|
|
91
|
-
|
|
97
|
+
const badValues = fields.flatMap((f) => f.subfields.filter((sf) => sf.code === subfieldCodeForBadValues)).map((sf) => validatorAndNormalizer(sf.value));
|
|
98
|
+
const validBadValues = badValues.filter((val) => val.valid).map((val) => val.value);
|
|
99
|
+
const invalidBadValues = badValues.filter((val) => !val.valid).map((val) => val.value);
|
|
100
|
+
return [uniqArray(trueGoodValues), uniqArray(wannabeGoodValues), uniqArray(validBadValues), uniqArray(invalidBadValues)];
|
|
92
101
|
}
|
|
93
102
|
function validatorAndNormalizer(string) {
|
|
94
103
|
const string2 = string.replace(/\. -$/u, "");
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/match-detection/features/bib/isbn.js"],
|
|
4
|
-
"sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport {parse as isbnParse} from 'isbn3';\n\nimport {uniqArray} from './issn.js';\nimport {isComponentRecord} from '@natlibfi/melinda-commons';\n\n\nconst debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:ISBN`);\nconst debugData = debug.extend('data');\n\nconst MAX_SCORE = 0.75;\n\nexport default () => ({\n name: 'ISBN',\n extract: ({record/*, recordExternal*/}) => {\n //const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n\n const host = !isComponentRecord(record, false, []);\n\n if (host) {\n return record.get('020').filter(f => f.subfields?.some(sf => ['a', 'z'].includes(sf.code) && sf.value));\n }\n return record.get('773').filter(f => f.subfields?.some(sf => ['z'].includes(sf.code) && sf.value));\n },\n // eslint-disable-next-line max-statements\n compare: (aa, bb) => {\n debugData(`Comparing ISBN sets ${JSON.stringify(aa)} and ${JSON.stringify(bb)}`);\n if (aa.length === 0 || bb.length === 0) {\n // No data for decision\n return 0;\n }\n\n const [subfieldCodeForGoodValues, subfieldCodeForBadValues] = getSubfieldCodes(aa[0].tag);\n\n const [
|
|
5
|
-
"mappings": "AACA,OAAO,uBAAuB;AAC9B,SAAQ,SAAS,iBAAgB;AAEjC,SAAQ,iBAAgB;AACxB,SAAQ,yBAAwB;AAGhC,MAAM,QAAQ,kBAAkB,sFAAsF;AACtH,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,MAAM,YAAY;AAElB,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC;AAAA,IAAC;AAAA;AAAA,EAA0B,MAAM;AAGzC,UAAM,OAAO,CAAC,kBAAkB,QAAQ,OAAO,CAAC,CAAC;AAEjD,QAAI,MAAM;AACR,aAAO,OAAO,IAAI,KAAK,EAAE,OAAO,OAAK,EAAE,WAAW,KAAK,QAAM,CAAC,KAAK,GAAG,EAAE,SAAS,GAAG,IAAI,KAAK,GAAG,KAAK,CAAC;AAAA,IACxG;AACA,WAAO,OAAO,IAAI,KAAK,EAAE,OAAO,OAAK,EAAE,WAAW,KAAK,QAAM,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI,KAAK,GAAG,KAAK,CAAC;AAAA,EACnG;AAAA;AAAA,EAEA,SAAS,CAAC,IAAI,OAAO;AACnB,cAAU,uBAAuB,KAAK,UAAU,EAAE,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC,EAAE;AAC/E,QAAI,GAAG,WAAW,KAAK,GAAG,WAAW,GAAG;AAEtC,aAAO;AAAA,IACT;AAEA,UAAM,CAAC,2BAA2B,wBAAwB,IAAI,iBAAiB,GAAG,CAAC,EAAE,GAAG;AAExF,UAAM,CAAC,
|
|
4
|
+
"sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport {parse as isbnParse} from 'isbn3';\n\nimport {uniqArray} from './issn.js';\nimport {isComponentRecord} from '@natlibfi/melinda-commons';\n\n\nconst debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:ISBN`);\nconst debugData = debug.extend('data');\n\nconst MAX_SCORE = 0.75;\n\nexport default () => ({\n name: 'ISBN',\n extract: ({record/*, recordExternal*/}) => {\n //const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n\n const host = !isComponentRecord(record, false, []);\n\n if (host) {\n return record.get('020').filter(f => f.subfields?.some(sf => ['a', 'z'].includes(sf.code) && sf.value));\n }\n return record.get('773').filter(f => f.subfields?.some(sf => ['z'].includes(sf.code) && sf.value));\n },\n // eslint-disable-next-line max-statements\n compare: (aa, bb) => {\n debugData(`Comparing ISBN sets ${JSON.stringify(aa)} and ${JSON.stringify(bb)}`);\n if (aa.length === 0 || bb.length === 0) {\n // No data for decision\n return 0;\n }\n\n const [subfieldCodeForGoodValues, subfieldCodeForBadValues] = getSubfieldCodes(aa[0].tag);\n\n const [aValidValuesA, aInvalidValuesA, zValidValuesA, zInvalidValuesA] = getValuesWrapper(aa, 'AA'); // initial 'a' and 'z' refer to 020 subfields codes\n const [aValidValuesB, aInvalidValuesB, zValidValuesB, zInvalidValuesB] = getValuesWrapper(bb, 'BB');\n\n function getValuesWrapper(data, prefix) {\n const [aValidValues, aInvalidValues, zValidValues, zInvalidValues] = getValues(data);\n if (aValidValues.length) {\n debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForGoodValues} VALID: ${aValidValues.join(', ')}`);\n }\n if (aInvalidValues.length) {\n debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForGoodValues} INVALID: ${aInvalidValues.join(', ')}`);\n }\n if (zValidValues.length) {\n debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForBadValues} VALID: ${zValidValues.join(', ')}`);\n }\n if (zInvalidValues.length) {\n debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForBadValues} INVALID: ${zInvalidValues.join(', ')}`);\n }\n return [aValidValues, aInvalidValues, zValidValues, zInvalidValues];\n }\n\n const [sharedGoodValues, goodValuesAOnly, goodValuesBOnly] = getUnionData(aValidValuesA, aValidValuesB);\n\n debug(`GOOD\\tBOTH: ${sharedGoodValues.length}, A only: ${goodValuesAOnly.length}, B only: ${goodValuesBOnly.length}`);\n\n if (sharedGoodValues.length > 0) {\n // Third argument (aka 'N times') is >= 0\n return scoreData(MAX_SCORE, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);\n }\n\n const hitScore = scoreSuboptimalHit();\n\n function scoreSuboptimalHit() {\n // One record consider ISBN good and the other record considered it's canceled:\n if (aValidValuesA.some(valA => zValidValuesB.includes(valA)) || aValidValuesB.some(valB => zValidValuesA.includes(valB))) {\n return MAX_SCORE;\n }\n\n // Subfield is for cancelled/whatever values, but the value is syntactically valid:\n // Could happen for two canceled ISBNs for example. I'll give this two thirds of the full score\n const zzValid = zValidValuesA.find(valA => zValidValuesB.includes(valA));\n if (zzValid) {\n debug(`Both contain a valid value in 020$z: ${zzValid}`);\n return MAX_SCORE * 2 / 3;\n }\n // Shared invalid identifiers:\n const aaInvalid = aInvalidValuesA.find(valA => aInvalidValuesB.includes(valA) || zInvalidValuesB.includes(valA)) || aInvalidValuesB.find(valB => zInvalidValuesA.includes(valB));\n if (aaInvalid) {\n debug(`Shared invalid value in 020$a and 020$a-or-$z subfields: ${aaInvalid}`);\n return MAX_SCORE * 2 / 3;\n }\n\n /* // Currently I think that paired invalid idenfiers in 020$z are meaningless...\n const zzInvalid = zInvalidValuesB.find(valB => zInvalidValuesA.includes(valB)) || zInvalidValuesA.find(valA => zInvalidValuesB.includes(valA));\n if (zzInvalid) {\n debug(`Shared invalid value in 020$z subfields: ${zzInvalid}`);\n return MAX_SCORE / 3;\n }\n */\n\n return 0;\n }\n\n\n\n if (hitScore === MAX_SCORE) {\n // -1 is needed to make the third argument >= 0 (otherwise min val would be 0)\n return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length - 1);\n }\n\n if (hitScore > 0) { // Canceled/invalid ISBNs match\n // Note that this is not (currently) penalized for non-matching canceled/invalid ISBNs. Maybe it should be.\n return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);\n }\n\n // No match:\n\n if (aValidValuesA.length + aInvalidValuesA.length === 0 || aValidValuesB.length + aInvalidValuesB.length === 0) { // At least one record did not have any good ISBNs, so not penalizing here! (Invalid 020$as are counted a bad.)\n return 0.0;\n }\n // We have same matching invalid identifiers. Don't penalize:\n const allA = [...aValidValuesA, ...aInvalidValuesA, ...zValidValuesA, ...zInvalidValuesA];\n const allB = [...aValidValuesB, ...aInvalidValuesB, ...zValidValuesB, ...zInvalidValuesB];\n const [sharedValues, tmp1, tmp2] = getUnionData(allA, allB);\n debug(`WHATEVER\\tBOTH: ${sharedGoodValues.length}, A only: ${tmp1.length}, B only: ${tmp2.length}`);\n\n if (sharedValues.length > 0) {\n return 0;\n }\n // We have values but they disagree:\n return -0.75; // Has good ISBNs on both records, but they did not match\n\n\n function getSubfieldCodes(tag) {\n if (tag === '773') {\n return ['z', undefined];\n }\n return ['a', 'z'];\n }\n\n function getValues(fields) {\n // Valid values are normalized to their isbn-13 form. Invalid values get their '-'s removed.\n const goodValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForGoodValues)).map(sf => validatorAndNormalizer(sf.value));\n const trueGoodValues = goodValues.filter(val => val.valid).map(val => val.value);\n const wannabeGoodValues = goodValues.filter(val => !val.valid).map(val => val.value);\n if (!subfieldCodeForBadValues) { // 773\n return [trueGoodValues, wannabeGoodValues, [], []];\n }\n const badValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForBadValues)).map(sf => validatorAndNormalizer(sf.value));\n const validBadValues = badValues.filter(val => val.valid).map(val => val.value);\n const invalidBadValues = badValues.filter(val => !val.valid).map(val => val.value);\n //const badValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForBadValues)).map(sf => validatorAndNormalizer(sf.value).value);\n return [uniqArray(trueGoodValues), uniqArray(wannabeGoodValues), uniqArray(validBadValues), uniqArray(invalidBadValues)];\n }\n\n function validatorAndNormalizer(string) {\n const string2 = string.replace(/\\. -$/u, ''); // Remove punctuation (773$z)\n\n // Hack: Historically we 020$a \"1234567890 sidottu\" etc. Try the LHS alone:\n const string3 = string2.replace(/ .*$/u, '');\n if (string2 !== string3) {\n const altResult = validatorAndNormalizer(string3);\n if (altResult.valid) {\n return altResult;\n }\n }\n\n const isbnParseResult = isbnParse(string2, '') || '';\n debugData(`isbnParseResult: ${JSON.stringify(isbnParseResult)}`);\n if (!isbnParseResult.isValid) {\n debug(`Not parseable ISBN '${string2}', just removing hyphens`);\n return {valid: false, value: string2.replace(/-/ug, '')};\n }\n\n debug(`Parseable ISBN '${string2}', normalizing to ISBN-13 '${isbnParseResult.isbn13}'`);\n return {valid: true, value: isbnParseResult.isbn13};\n\n\n }\n }\n});\n\n// These are outside the default function as I'll probably want to export these later on\n\nfunction getUnionData(set1, set2) {\n const shared = set1.filter(val => set2.includes(val));\n const onlyInSet1 = set1.filter(val => !shared.includes(val));\n const onlyInSet2 = set2.filter(val => !shared.includes(val));\n return [shared, onlyInSet1, onlyInSet2];\n}\n\nfunction scoreData(score, factor, n) {\n return innerScoreData(score, n);\n function innerScoreData(currScore, remaining) {\n if (remaining > 0) {\n return innerScoreData(currScore * factor, remaining-1);\n }\n return Math.round(currScore * 100)/100; // 0.600000000001 => 0.6\n }\n}"],
|
|
5
|
+
"mappings": "AACA,OAAO,uBAAuB;AAC9B,SAAQ,SAAS,iBAAgB;AAEjC,SAAQ,iBAAgB;AACxB,SAAQ,yBAAwB;AAGhC,MAAM,QAAQ,kBAAkB,sFAAsF;AACtH,MAAM,YAAY,MAAM,OAAO,MAAM;AAErC,MAAM,YAAY;AAElB,eAAe,OAAO;AAAA,EACpB,MAAM;AAAA,EACN,SAAS,CAAC;AAAA,IAAC;AAAA;AAAA,EAA0B,MAAM;AAGzC,UAAM,OAAO,CAAC,kBAAkB,QAAQ,OAAO,CAAC,CAAC;AAEjD,QAAI,MAAM;AACR,aAAO,OAAO,IAAI,KAAK,EAAE,OAAO,OAAK,EAAE,WAAW,KAAK,QAAM,CAAC,KAAK,GAAG,EAAE,SAAS,GAAG,IAAI,KAAK,GAAG,KAAK,CAAC;AAAA,IACxG;AACA,WAAO,OAAO,IAAI,KAAK,EAAE,OAAO,OAAK,EAAE,WAAW,KAAK,QAAM,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI,KAAK,GAAG,KAAK,CAAC;AAAA,EACnG;AAAA;AAAA,EAEA,SAAS,CAAC,IAAI,OAAO;AACnB,cAAU,uBAAuB,KAAK,UAAU,EAAE,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC,EAAE;AAC/E,QAAI,GAAG,WAAW,KAAK,GAAG,WAAW,GAAG;AAEtC,aAAO;AAAA,IACT;AAEA,UAAM,CAAC,2BAA2B,wBAAwB,IAAI,iBAAiB,GAAG,CAAC,EAAE,GAAG;AAExF,UAAM,CAAC,eAAe,iBAAiB,eAAe,eAAe,IAAI,iBAAiB,IAAI,IAAI;AAClG,UAAM,CAAC,eAAe,iBAAiB,eAAe,eAAe,IAAI,iBAAiB,IAAI,IAAI;AAElG,aAAS,iBAAiB,MAAM,QAAQ;AACtC,YAAM,CAAC,cAAc,gBAAgB,cAAc,cAAc,IAAI,UAAU,IAAI;AACnF,UAAI,aAAa,QAAQ;AACvB,cAAM,GAAG,MAAM,KAAK,GAAG,CAAC,EAAE,GAAG,IAAI,yBAAyB,WAAW,aAAa,KAAK,IAAI,CAAC,EAAE;AAAA,MAChG;AACA,UAAI,eAAe,QAAQ;AACzB,cAAM,GAAG,MAAM,KAAK,GAAG,CAAC,EAAE,GAAG,IAAI,yBAAyB,aAAa,eAAe,KAAK,IAAI,CAAC,EAAE;AAAA,MACpG;AACA,UAAI,aAAa,QAAQ;AACvB,cAAM,GAAG,MAAM,KAAK,GAAG,CAAC,EAAE,GAAG,IAAI,wBAAwB,WAAW,aAAa,KAAK,IAAI,CAAC,EAAE;AAAA,MAC/F;AACA,UAAI,eAAe,QAAQ;AACzB,cAAM,GAAG,MAAM,KAAK,GAAG,CAAC,EAAE,GAAG,IAAI,wBAAwB,aAAa,eAAe,KAAK,IAAI,CAAC,EAAE;AAAA,MACnG;AACA,aAAO,CAAC,cAAc,gBAAgB,cAAc,cAAc;AAAA,IACpE;AAEA,UAAM,CAAC,kBAAkB,iBAAiB,eAAe,IAAI,aAAa,eAAe,aAAa;AAEtG,UAAM,cAAe,iBAAiB,MAAM,aAAa,gBAAgB,MAAM,aAAa,gBAAgB,MAAM,EAAE;AAEpH,QAAI,iBAAiB,SAAS,GAAG;AAE/B,aAAO,UAAU,WAAW,KAAK,gBAAgB,SAAS,gBAAgB,MAAM;AAAA,IAClF;AAEA,UAAM,WAAW,mBAAmB;AAEpC,aAAS,qBAAqB;AAE5B,UAAI,cAAc,KAAK,UAAQ,cAAc,SAAS,IAAI,CAAC,KAAK,cAAc,KAAK,UAAQ,cAAc,SAAS,IAAI,CAAC,GAAG;AACxH,eAAO;AAAA,MACT;AAIA,YAAM,UAAU,cAAc,KAAK,UAAQ,cAAc,SAAS,IAAI,CAAC;AACvE,UAAI,SAAS;AACX,cAAM,wCAAwC,OAAO,EAAE;AACvD,eAAO,YAAY,IAAI;AAAA,MACzB;AAEA,YAAM,YAAY,gBAAgB,KAAK,UAAQ,gBAAgB,SAAS,IAAI,KAAK,gBAAgB,SAAS,IAAI,CAAC,KAAK,gBAAgB,KAAK,UAAQ,gBAAgB,SAAS,IAAI,CAAC;AAC/K,UAAI,WAAW;AACb,cAAM,4DAA4D,SAAS,EAAE;AAC7E,eAAO,YAAY,IAAI;AAAA,MACzB;AAUA,aAAO;AAAA,IACT;AAIA,QAAI,aAAa,WAAW;AAE1B,aAAO,UAAU,UAAU,KAAK,gBAAgB,SAAS,gBAAgB,SAAS,CAAC;AAAA,IACrF;AAEA,QAAI,WAAW,GAAG;AAEhB,aAAO,UAAU,UAAU,KAAK,gBAAgB,SAAS,gBAAgB,MAAM;AAAA,IACjF;AAIA,QAAI,cAAc,SAAS,gBAAgB,WAAW,KAAK,cAAc,SAAS,gBAAgB,WAAW,GAAG;AAC9G,aAAO;AAAA,IACT;AAEA,UAAM,OAAO,CAAC,GAAG,eAAe,GAAG,iBAAiB,GAAG,eAAe,GAAG,eAAe;AACxF,UAAM,OAAO,CAAC,GAAG,eAAe,GAAG,iBAAiB,GAAG,eAAe,GAAG,eAAe;AACxF,UAAM,CAAC,cAAc,MAAM,IAAI,IAAI,aAAa,MAAM,IAAI;AAC1D,UAAM,kBAAmB,iBAAiB,MAAM,aAAa,KAAK,MAAM,aAAa,KAAK,MAAM,EAAE;AAElG,QAAI,aAAa,SAAS,GAAG;AAC3B,aAAO;AAAA,IACT;AAEA,WAAO;AAGP,aAAS,iBAAiB,KAAK;AAC7B,UAAI,QAAQ,OAAO;AACjB,eAAO,CAAC,KAAK,MAAS;AAAA,MACxB;AACA,aAAO,CAAC,KAAK,GAAG;AAAA,IAClB;AAEA,aAAS,UAAU,QAAQ;AAEzB,YAAM,aAAa,OAAO,QAAQ,OAAK,EAAE,UAAU,OAAO,QAAM,GAAG,SAAS,yBAAyB,CAAC,EAAE,IAAI,QAAM,uBAAuB,GAAG,KAAK,CAAC;AAClJ,YAAM,iBAAiB,WAAW,OAAO,SAAO,IAAI,KAAK,EAAE,IAAI,SAAO,IAAI,KAAK;AAC/E,YAAM,oBAAoB,WAAW,OAAO,SAAO,CAAC,IAAI,KAAK,EAAE,IAAI,SAAO,IAAI,KAAK;AACnF,UAAI,CAAC,0BAA0B;AAC7B,eAAO,CAAC,gBAAgB,mBAAmB,CAAC,GAAG,CAAC,CAAC;AAAA,MACnD;AACA,YAAM,YAAY,OAAO,QAAQ,OAAK,EAAE,UAAU,OAAO,QAAM,GAAG,SAAS,wBAAwB,CAAC,EAAE,IAAI,QAAM,uBAAuB,GAAG,KAAK,CAAC;AAChJ,YAAM,iBAAiB,UAAU,OAAO,SAAO,IAAI,KAAK,EAAE,IAAI,SAAO,IAAI,KAAK;AAC9E,YAAM,mBAAmB,UAAU,OAAO,SAAO,CAAC,IAAI,KAAK,EAAE,IAAI,SAAO,IAAI,KAAK;AAEjF,aAAO,CAAC,UAAU,cAAc,GAAG,UAAU,iBAAiB,GAAG,UAAU,cAAc,GAAG,UAAU,gBAAgB,CAAC;AAAA,IACzH;AAEA,aAAS,uBAAuB,QAAQ;AACtC,YAAM,UAAU,OAAO,QAAQ,UAAU,EAAE;AAG3C,YAAM,UAAU,QAAQ,QAAQ,SAAS,EAAE;AAC3C,UAAI,YAAY,SAAS;AACvB,cAAM,YAAY,uBAAuB,OAAO;AAChD,YAAI,UAAU,OAAO;AACnB,iBAAO;AAAA,QACT;AAAA,MACF;AAEA,YAAM,kBAAkB,UAAU,SAAS,EAAE,KAAK;AAClD,gBAAU,oBAAoB,KAAK,UAAU,eAAe,CAAC,EAAE;AAC/D,UAAI,CAAC,gBAAgB,SAAS;AAC5B,cAAM,uBAAuB,OAAO,0BAA0B;AAC9D,eAAO,EAAC,OAAO,OAAO,OAAO,QAAQ,QAAQ,OAAO,EAAE,EAAC;AAAA,MACzD;AAEA,YAAM,mBAAmB,OAAO,8BAA8B,gBAAgB,MAAM,GAAG;AACvF,aAAO,EAAC,OAAO,MAAM,OAAO,gBAAgB,OAAM;AAAA,IAGpD;AAAA,EACF;AACF;AAIA,SAAS,aAAa,MAAM,MAAM;AAChC,QAAM,SAAS,KAAK,OAAO,SAAO,KAAK,SAAS,GAAG,CAAC;AACpD,QAAM,aAAa,KAAK,OAAO,SAAO,CAAC,OAAO,SAAS,GAAG,CAAC;AAC3D,QAAM,aAAa,KAAK,OAAO,SAAO,CAAC,OAAO,SAAS,GAAG,CAAC;AAC3D,SAAO,CAAC,QAAQ,YAAY,UAAU;AACxC;AAEA,SAAS,UAAU,OAAO,QAAQ,GAAG;AACnC,SAAO,eAAe,OAAO,CAAC;AAC9B,WAAS,eAAe,WAAW,WAAW;AAC5C,QAAI,YAAY,GAAG;AACjB,aAAO,eAAe,YAAY,QAAQ,YAAU,CAAC;AAAA,IACvD;AACA,WAAO,KAAK,MAAM,YAAY,GAAG,IAAE;AAAA,EACrC;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"url": "https://github.com/NatLibFi/melinda-record-matching-js"
|
|
14
14
|
},
|
|
15
15
|
"license": "MIT",
|
|
16
|
-
"version": "5.0.
|
|
16
|
+
"version": "5.0.4",
|
|
17
17
|
"type": "module",
|
|
18
18
|
"main": "./dist/index.js",
|
|
19
19
|
"bin": "./dist/cli.js",
|
|
@@ -32,6 +32,8 @@
|
|
|
32
32
|
"lint:dev": "eslint --fix ./src",
|
|
33
33
|
"test:base": "node --test --test-force-exit --experimental-test-coverage --test-reporter=spec './src/**/*.test.js'",
|
|
34
34
|
"test": "npm run lint && npm run test:base",
|
|
35
|
+
"test:only": "npm run lint && npm run test:base:only",
|
|
36
|
+
"test:base:only": "cross-env DEBUG=@natlibfi/*matching* NODE_ENV=test node --test --test-only --test-force-exit --experimental-test-coverage --test-reporter=spec './src/**/*.test.js'",
|
|
35
37
|
"watch:test": "cross-env DEBUG=@natlibfi/* NODE_ENV=test node --watch --test --experimental-test-coverage --test-reporter=spec './src/**/*.test.js'",
|
|
36
38
|
"dev": "npm run watch:test",
|
|
37
39
|
"dev:debug": "cross-env LOG_LEVEL=debug DEBUG=@natlibfi/* NODE_ENV=test npm run watch:test"
|
|
@@ -50,9 +52,9 @@
|
|
|
50
52
|
"yargs": "^18.0.0"
|
|
51
53
|
},
|
|
52
54
|
"devDependencies": {
|
|
53
|
-
"@natlibfi/fixugen": "^3.0.
|
|
55
|
+
"@natlibfi/fixugen": "^3.0.1",
|
|
54
56
|
"@natlibfi/fixugen-http-client": "^4.0.2",
|
|
55
|
-
"@natlibfi/fixura": "^4.0.
|
|
57
|
+
"@natlibfi/fixura": "^4.0.1",
|
|
56
58
|
"cross-env": "^10.1.0",
|
|
57
59
|
"esbuild": "^0.27.3",
|
|
58
60
|
"eslint": "^10.0.2"
|
|
@@ -33,50 +33,69 @@ export default () => ({
|
|
|
33
33
|
|
|
34
34
|
const [subfieldCodeForGoodValues, subfieldCodeForBadValues] = getSubfieldCodes(aa[0].tag);
|
|
35
35
|
|
|
36
|
-
const [
|
|
37
|
-
|
|
38
|
-
debug(`GOOD VALUES (A): ${goodValuesA.join(', ')}`);
|
|
39
|
-
}
|
|
40
|
-
if (badValuesA.length) {
|
|
41
|
-
debug(`BAD VALUES (A): ${badValuesA.join(', ')}`);
|
|
42
|
-
}
|
|
36
|
+
const [aValidValuesA, aInvalidValuesA, zValidValuesA, zInvalidValuesA] = getValuesWrapper(aa, 'AA'); // initial 'a' and 'z' refer to 020 subfields codes
|
|
37
|
+
const [aValidValuesB, aInvalidValuesB, zValidValuesB, zInvalidValuesB] = getValuesWrapper(bb, 'BB');
|
|
43
38
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
39
|
+
function getValuesWrapper(data, prefix) {
|
|
40
|
+
const [aValidValues, aInvalidValues, zValidValues, zInvalidValues] = getValues(data);
|
|
41
|
+
if (aValidValues.length) {
|
|
42
|
+
debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForGoodValues} VALID: ${aValidValues.join(', ')}`);
|
|
43
|
+
}
|
|
44
|
+
if (aInvalidValues.length) {
|
|
45
|
+
debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForGoodValues} INVALID: ${aInvalidValues.join(', ')}`);
|
|
46
|
+
}
|
|
47
|
+
if (zValidValues.length) {
|
|
48
|
+
debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForBadValues} VALID: ${zValidValues.join(', ')}`);
|
|
49
|
+
}
|
|
50
|
+
if (zInvalidValues.length) {
|
|
51
|
+
debug(`${prefix}: ${aa[0].tag}$${subfieldCodeForBadValues} INVALID: ${zInvalidValues.join(', ')}`);
|
|
52
|
+
}
|
|
53
|
+
return [aValidValues, aInvalidValues, zValidValues, zInvalidValues];
|
|
50
54
|
}
|
|
51
55
|
|
|
52
|
-
const [sharedGoodValues, goodValuesAOnly, goodValuesBOnly] = getUnionData(
|
|
56
|
+
const [sharedGoodValues, goodValuesAOnly, goodValuesBOnly] = getUnionData(aValidValuesA, aValidValuesB);
|
|
53
57
|
|
|
54
|
-
|
|
58
|
+
debug(`GOOD\tBOTH: ${sharedGoodValues.length}, A only: ${goodValuesAOnly.length}, B only: ${goodValuesBOnly.length}`);
|
|
55
59
|
|
|
56
|
-
|
|
60
|
+
if (sharedGoodValues.length > 0) {
|
|
61
|
+
// Third argument (aka 'N times') is >= 0
|
|
62
|
+
return scoreData(MAX_SCORE, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);
|
|
63
|
+
}
|
|
57
64
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
}
|
|
65
|
+
const hitScore = scoreSuboptimalHit();
|
|
66
|
+
|
|
67
|
+
function scoreSuboptimalHit() {
|
|
62
68
|
// One record consider ISBN good and the other record considered it's canceled:
|
|
63
|
-
if (
|
|
69
|
+
if (aValidValuesA.some(valA => zValidValuesB.includes(valA)) || aValidValuesB.some(valB => zValidValuesA.includes(valB))) {
|
|
64
70
|
return MAX_SCORE;
|
|
65
71
|
}
|
|
66
72
|
|
|
67
|
-
//
|
|
73
|
+
// Subfield is for cancelled/whatever values, but the value is syntactically valid:
|
|
68
74
|
// Could happen for two canceled ISBNs for example. I'll give this two thirds of the full score
|
|
69
|
-
|
|
75
|
+
const zzValid = zValidValuesA.find(valA => zValidValuesB.includes(valA));
|
|
76
|
+
if (zzValid) {
|
|
77
|
+
debug(`Both contain a valid value in 020$z: ${zzValid}`);
|
|
78
|
+
return MAX_SCORE * 2 / 3;
|
|
79
|
+
}
|
|
80
|
+
// Shared invalid identifiers:
|
|
81
|
+
const aaInvalid = aInvalidValuesA.find(valA => aInvalidValuesB.includes(valA) || zInvalidValuesB.includes(valA)) || aInvalidValuesB.find(valB => zInvalidValuesA.includes(valB));
|
|
82
|
+
if (aaInvalid) {
|
|
83
|
+
debug(`Shared invalid value in 020$a and 020$a-or-$z subfields: ${aaInvalid}`);
|
|
70
84
|
return MAX_SCORE * 2 / 3;
|
|
71
85
|
}
|
|
72
86
|
|
|
87
|
+
/* // Currently I think that paired invalid idenfiers in 020$z are meaningless...
|
|
88
|
+
const zzInvalid = zInvalidValuesB.find(valB => zInvalidValuesA.includes(valB)) || zInvalidValuesA.find(valA => zInvalidValuesB.includes(valA));
|
|
89
|
+
if (zzInvalid) {
|
|
90
|
+
debug(`Shared invalid value in 020$z subfields: ${zzInvalid}`);
|
|
91
|
+
return MAX_SCORE / 3;
|
|
92
|
+
}
|
|
93
|
+
*/
|
|
94
|
+
|
|
73
95
|
return 0;
|
|
74
96
|
}
|
|
75
97
|
|
|
76
|
-
|
|
77
|
-
// Third argument (aka 'N times') is >= 0
|
|
78
|
-
return scoreData(hitScore, 0.8, goodValuesAOnly.length + goodValuesBOnly.length);
|
|
79
|
-
}
|
|
98
|
+
|
|
80
99
|
|
|
81
100
|
if (hitScore === MAX_SCORE) {
|
|
82
101
|
// -1 is needed to make the third argument >= 0 (otherwise min val would be 0)
|
|
@@ -90,10 +109,19 @@ export default () => ({
|
|
|
90
109
|
|
|
91
110
|
// No match:
|
|
92
111
|
|
|
93
|
-
if (
|
|
112
|
+
if (aValidValuesA.length + aInvalidValuesA.length === 0 || aValidValuesB.length + aInvalidValuesB.length === 0) { // At least one record did not have any good ISBNs, so not penalizing here! (Invalid 020$as are counted a bad.)
|
|
94
113
|
return 0.0;
|
|
95
114
|
}
|
|
115
|
+
// We have same matching invalid identifiers. Don't penalize:
|
|
116
|
+
const allA = [...aValidValuesA, ...aInvalidValuesA, ...zValidValuesA, ...zInvalidValuesA];
|
|
117
|
+
const allB = [...aValidValuesB, ...aInvalidValuesB, ...zValidValuesB, ...zInvalidValuesB];
|
|
118
|
+
const [sharedValues, tmp1, tmp2] = getUnionData(allA, allB);
|
|
119
|
+
debug(`WHATEVER\tBOTH: ${sharedGoodValues.length}, A only: ${tmp1.length}, B only: ${tmp2.length}`);
|
|
96
120
|
|
|
121
|
+
if (sharedValues.length > 0) {
|
|
122
|
+
return 0;
|
|
123
|
+
}
|
|
124
|
+
// We have values but they disagree:
|
|
97
125
|
return -0.75; // Has good ISBNs on both records, but they did not match
|
|
98
126
|
|
|
99
127
|
|
|
@@ -104,27 +132,19 @@ export default () => ({
|
|
|
104
132
|
return ['a', 'z'];
|
|
105
133
|
}
|
|
106
134
|
|
|
107
|
-
function looksGood(val) {
|
|
108
|
-
// isbn10 can end in X:
|
|
109
|
-
if (/^([0-9]-?){9}[0-9X]$/u.test(val)) {
|
|
110
|
-
return true;
|
|
111
|
-
}
|
|
112
|
-
// isbn13 can not:
|
|
113
|
-
if (/^([0-9]-?){12}[0-9]$/u.test(val)) {
|
|
114
|
-
return true;
|
|
115
|
-
}
|
|
116
|
-
return false;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
135
|
function getValues(fields) {
|
|
136
|
+
// Valid values are normalized to their isbn-13 form. Invalid values get their '-'s removed.
|
|
120
137
|
const goodValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForGoodValues)).map(sf => validatorAndNormalizer(sf.value));
|
|
121
138
|
const trueGoodValues = goodValues.filter(val => val.valid).map(val => val.value);
|
|
122
139
|
const wannabeGoodValues = goodValues.filter(val => !val.valid).map(val => val.value);
|
|
123
140
|
if (!subfieldCodeForBadValues) { // 773
|
|
124
|
-
return [trueGoodValues, wannabeGoodValues];
|
|
141
|
+
return [trueGoodValues, wannabeGoodValues, [], []];
|
|
125
142
|
}
|
|
126
|
-
const badValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForBadValues)).map(sf => sf.value);
|
|
127
|
-
|
|
143
|
+
const badValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForBadValues)).map(sf => validatorAndNormalizer(sf.value));
|
|
144
|
+
const validBadValues = badValues.filter(val => val.valid).map(val => val.value);
|
|
145
|
+
const invalidBadValues = badValues.filter(val => !val.valid).map(val => val.value);
|
|
146
|
+
//const badValues = fields.flatMap(f => f.subfields.filter(sf => sf.code === subfieldCodeForBadValues)).map(sf => validatorAndNormalizer(sf.value).value);
|
|
147
|
+
return [uniqArray(trueGoodValues), uniqArray(wannabeGoodValues), uniqArray(validBadValues), uniqArray(invalidBadValues)];
|
|
128
148
|
}
|
|
129
149
|
|
|
130
150
|
function validatorAndNormalizer(string) {
|