@natlibfi/melinda-record-matching 3.0.1-alpha.2 → 3.0.2-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/.github/workflows/melinda-node-tests.yml +2 -2
  2. package/dist/match-detection/features/bib/isbn.js +28 -2
  3. package/dist/match-detection/features/bib/isbn.js.map +1 -1
  4. package/dist/match-detection/features/bib/standard-identifier-factory.js +126 -25
  5. package/dist/match-detection/features/bib/standard-identifier-factory.js.map +1 -1
  6. package/dist/matching-utils.js +26 -0
  7. package/dist/matching-utils.js.map +1 -1
  8. package/package.json +2 -1
  9. package/src/match-detection/features/bib/isbn.js +21 -2
  10. package/src/match-detection/features/bib/standard-identifier-factory.js +106 -23
  11. package/src/matching-utils.js +21 -0
package/.github/workflows/melinda-node-tests.yml CHANGED
@@ -16,7 +16,7 @@ jobs:
16
16
 
17
17
  steps:
18
18
  - name: Checkout the code
19
- uses: actions/checkout@v2
19
+ uses: actions/checkout@v3
20
20
  - name: Use Node.js ${{ matrix.node-version }}
21
21
  uses: actions/setup-node@v3
22
22
  with:
@@ -58,7 +58,7 @@ jobs:
58
58
  if: contains(github.ref, 'refs/tags/')
59
59
 
60
60
  steps:
61
- - uses: actions/checkout@v2
61
+ - uses: actions/checkout@v3
62
62
  # Setup .npmrc file to publish to npm
63
63
  - uses: actions/setup-node@v3
64
64
  with:
package/dist/match-detection/features/bib/isbn.js CHANGED
@@ -5,6 +5,8 @@ Object.defineProperty(exports, "__esModule", {
5
5
  });
6
6
  exports.default = void 0;
7
7
  var _standardIdentifierFactory = _interopRequireDefault(require("./standard-identifier-factory"));
8
+ var _isbn = require("isbn3");
9
+ var _debug = _interopRequireDefault(require("debug"));
8
10
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
9
11
  /**
10
12
  *
@@ -33,18 +35,42 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
33
35
  * for the JavaScript code in this file.
34
36
  *
35
37
  */
38
+
39
+ const debug = (0, _debug.default)(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:ISBN`);
40
+ const debugData = debug.extend('data');
36
41
  var _default = () => {
42
+ const IDENTIFIER_NAME = 'ISBN';
43
+ function validatorAndNormalizer(string) {
44
+ const isbnParseResult = (0, _isbn.parse)(string);
45
+ debugData(`isbnParseResult: ${JSON.stringify(isbnParseResult)}`);
46
+ if (isbnParseResult === null) {
47
+ debug(`Not parseable ISBN, just removing hyphens`);
48
+ return {
49
+ valid: false,
50
+ value: string.replace(/-/ug, '')
51
+ };
52
+ }
53
+ debug(`Parseable ISBN, normalizing to ISBN-13`);
54
+ return {
55
+ valid: true,
56
+ value: isbnParseResult.isbn13
57
+ };
58
+ }
37
59
  const {
38
60
  extract,
39
61
  compare
40
62
  } = (0, _standardIdentifierFactory.default)({
63
+ identifier: IDENTIFIER_NAME,
41
64
  pattern: /^020$/u,
42
- subfieldCodes: ['a', 'z']
65
+ subfieldCodes: ['a', 'z'],
66
+ validIdentifierSubfieldCodes: ['a'],
67
+ invalidIdentifierSubfieldCodes: ['z'],
68
+ validatorAndNormalizer
43
69
  });
44
70
  return {
45
71
  extract,
46
72
  compare,
47
- name: 'ISBN'
73
+ name: IDENTIFIER_NAME
48
74
  };
49
75
  };
50
76
  exports.default = _default;
package/dist/match-detection/features/bib/isbn.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"isbn.js","names":["extract","compare","createInterface","pattern","subfieldCodes","name"],"sources":["../../../../src/match-detection/features/bib/isbn.js"],"sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createInterface from './standard-identifier-factory';\n\nexport default () => {\n const {extract, compare} = createInterface({pattern: /^020$/u, subfieldCodes: ['a', 'z']});\n return {extract, compare, name: 'ISBN'};\n};\n"],"mappings":";;;;;;AA4BA;AAA4D;AA5B5D;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AA1BA,eA8Be,MAAM;EACnB,MAAM;IAACA,OAAO;IAAEC;EAAO,CAAC,GAAG,IAAAC,kCAAe,EAAC;IAACC,OAAO,EAAE,QAAQ;IAAEC,aAAa,EAAE,CAAC,GAAG,EAAE,GAAG;EAAC,CAAC,CAAC;EAC1F,OAAO;IAACJ,OAAO;IAAEC,OAAO;IAAEI,IAAI,EAAE;EAAM,CAAC;AACzC,CAAC;AAAA"}
1
+ {"version":3,"file":"isbn.js","names":["debug","createDebugLogger","debugData","extend","IDENTIFIER_NAME","validatorAndNormalizer","string","isbnParseResult","isbnParse","JSON","stringify","valid","value","replace","isbn13","extract","compare","createInterface","identifier","pattern","subfieldCodes","validIdentifierSubfieldCodes","invalidIdentifierSubfieldCodes","name"],"sources":["../../../../src/match-detection/features/bib/isbn.js"],"sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createInterface from './standard-identifier-factory';\nimport {parse as isbnParse} from 'isbn3';\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:ISBN`);\nconst debugData = debug.extend('data');\n\nexport default () => {\n const IDENTIFIER_NAME = 'ISBN';\n\n function validatorAndNormalizer(string) {\n const isbnParseResult = isbnParse(string);\n debugData(`isbnParseResult: ${JSON.stringify(isbnParseResult)}`);\n if (isbnParseResult === null) {\n debug(`Not parseable ISBN, just removing hyphens`);\n return {valid: false, value: string.replace(/-/ug, '')};\n }\n debug(`Parseable ISBN, normalizing to ISBN-13`);\n return {valid: true, value: isbnParseResult.isbn13};\n }\n\n const {extract, compare} = createInterface({identifier: IDENTIFIER_NAME, pattern: /^020$/u, subfieldCodes: ['a', 'z'], validIdentifierSubfieldCodes: ['a'], invalidIdentifierSubfieldCodes: ['z'], validatorAndNormalizer});\n return {extract, compare, name: IDENTIFIER_NAME};\n};\n\n"],"mappings":";;;;;;AA4BA;AACA;AACA;AAAsC;AA9BtC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAMA,MAAMA,KAAK,GAAG,IAAAC,cAAiB,EAAE,sFAAqF,CAAC;AACvH,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;AAAC,eAExB,MAAM;EACnB,MAAMC,eAAe,GAAG,MAAM;EAE9B,SAASC,sBAAsB,CAACC,MAAM,EAAE;IACtC,MAAMC,eAAe,GAAG,IAAAC,WAAS,EAACF,MAAM,CAAC;IACzCJ,SAAS,CAAE,oBAAmBO,IAAI,CAACC,SAAS,CAACH,eAAe,CAAE,EAAC,CAAC;IAChE,IAAIA,eAAe,KAAK,IAAI,EAAE;MAC5BP,KAAK,CAAE,2CAA0C,CAAC;MAClD,OAAO;QAACW,KAAK,EAAE,KAAK;QAAEC,KAAK,EAAEN,MAAM,CAACO,OAAO,CAAC,KAAK,EAAE,EAAE;MAAC,CAAC;IACzD;IACAb,KAAK,CAAE,wCAAuC,CAAC;IAC/C,OAAO;MAACW,KAAK,EAAE,IAAI;MAAEC,KAAK,EAAEL,eAAe,CAACO;IAAM,CAAC;EACrD;EAEA,MAAM;IAACC,OAAO;IAAEC;EAAO,CAAC,GAAG,IAAAC,kCAAe,EAAC;IAACC,UAAU,EAAEd,eAAe;IAAEe,OAAO,EAAE,QAAQ;IAAEC,aAAa,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;IAAEC,4BAA4B,EAAE,CAAC,GAAG,CAAC;IAAEC,8BAA8B,EAAE,CAAC,GAAG,CAAC;IAAEjB;EAAsB,CAAC,CAAC;EAC3N,OAAO;IAACU,OAAO;IAAEC,OAAO;IAAEO,IAAI,EAAEnB;EAAe,CAAC;AAClD,CAAC;AAAA"}
package/dist/match-detection/features/bib/standard-identifier-factory.js CHANGED
@@ -7,6 +7,7 @@ exports.default = void 0;
7
7
  var _debug = _interopRequireDefault(require("debug"));
8
8
  var _matchingUtils = require("../../../matching-utils");
9
9
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
10
+ /* eslint-disable max-statements */
10
11
  /**
11
12
  *
12
13
  * @licstart The following is the entire license notice for the JavaScript code in this file.
@@ -34,72 +35,172 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
34
35
  * for the JavaScript code in this file.
35
36
  *
36
37
  */
37
-
38
- const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers');
39
- const debugData = debug.extend('data');
38
+ // Note about validity of standardIdentifiers:
39
+ // We have three types of invalid standardIdentifiers:
40
+ // 1. Formally invalid standardIdentifiers (ie. typos either in the resource or the record)
41
+ // 2. Formally valid standardIdentifiers that are used in a wrong resource
42
+ // 3. Canceled standardIdentifiers
43
+ // Matcher could and should check that a standardIdentifier found in a subfield for a valid identifier is formally valid, and if it's not formally valid, handle it as an invalid standardIdentifier
44
+ // Formally valid standardIdentifiers found in subfield for invalid identifier cannot be handled as valid standardIdentifiers, because they can be a case of type 2) or 3) invalid standardIdentifiers
45
+ // We could also do a separate handling for formally valid an formally invalid standardIdentifiers
40
46
  var _default = ({
41
47
  pattern,
42
- subfieldCodes
48
+ subfieldCodes,
49
+ identifier,
50
+ validIdentifierSubfieldCodes = ['a'],
51
+ invalidIdentifierSubfieldCodes = ['z'],
52
+ validatorAndNormalizer = undefined
43
53
  }) => {
54
+ const debug = (0, _debug.default)(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);
55
+ const debugData = debug.extend('data');
44
56
  return {
45
57
  extract,
46
58
  compare
47
59
  };
48
60
  function extract({
49
- record
61
+ record,
62
+ recordExternal
50
63
  }) {
51
- const [field] = record.get(pattern);
52
- if (field) {
53
- return field.subfields.filter(({
54
- code
55
- }) => subfieldCodes.includes(code)).map(({
56
- code,
64
+ const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
65
+ const fields = record.get(pattern);
66
+ debugData(`${label}: ${fields.length} ${identifier}-fields `);
67
+
68
+ // extractIdentifierSubfield normalizes hyphens away from the subfield values
69
+ const identifiersFromFields = fields.map(field => (0, _matchingUtils.extractSubfieldsFromField)(field, subfieldCodes));
70
+ debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);
71
+ const allIdentifiers = identifiersFromFields.flat();
72
+ debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);
73
+ const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({
74
+ identifierSubs: allIdentifiers,
75
+ validatorAndNormalizer,
76
+ validIdentifierSubfieldCodes,
77
+ invalidIdentifierSubfieldCodes
78
+ });
79
+ const identifiers = (0, _matchingUtils.uniqueSubfields)(validatedAndNormalizedIdentifiers);
80
+ debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);
81
+ return identifiers;
82
+ function validateAndNormalizeIdentifiers({
83
+ identifierSubs,
84
+ validatorAndNormalizer,
85
+ validIdentifierSubfieldCodes,
86
+ invalidIdentifierSubfieldCodes
87
+ }) {
88
+ if (validatorAndNormalizer) {
89
+ return identifierSubs.map(idSub => validateAndNormalizeIdentifier({
90
+ idSub,
91
+ validatorAndNormalizer,
92
+ validIdentifierSubfieldCodes,
93
+ invalidIdentifierSubfieldCodes
94
+ }));
95
+ }
96
+ return identifierSubs.map(idSub => normalizeHyphens(idSub));
97
+ }
98
+ function validateAndNormalizeIdentifier({
99
+ idSub,
100
+ validatorAndNormalizer,
101
+ validIdentifierSubfieldCodes,
102
+ invalidIdentifierSubfieldCodes
103
+ }) {
104
+ const {
105
+ valid,
106
+ value
107
+ } = validatorAndNormalizer(idSub.value);
108
+ if (validIdentifierSubfieldCodes.includes(idSub.code) && valid === false) {
109
+ const [code] = invalidIdentifierSubfieldCodes;
110
+ return {
111
+ code,
112
+ value
113
+ };
114
+ }
115
+ return {
116
+ code: idSub.code,
57
117
  value
58
- }) => ({
59
- code,
60
- value: (0, _matchingUtils.testStringOrNumber)(value) ? String(value).replace(/-/ug, '') : ''
61
- })).filter(value => value);
118
+ };
119
+ }
120
+ function normalizeHyphens(idSub) {
121
+ return {
122
+ code: idSub.code,
123
+ value: idSub.value.replace(/-/ug, '')
124
+ };
62
125
  }
63
- return [];
64
126
  }
65
127
  function compare(a, b) {
128
+ debug(`Comparing A and B`);
66
129
  if (a.length === 0 || b.length === 0) {
67
- debugData(`No standardidentifiers to compare`);
130
+ debugData(`No standardidentifiers (${identifier}) to compare`);
68
131
  return 0;
69
132
  }
133
+ debugData(`A: ${JSON.stringify(a)}`);
134
+ debugData(`B: ${JSON.stringify(b)}`);
70
135
  if (bothHaveValidIdentifiers()) {
136
+ // Compare only valid identifiers, if both have valid idenfiers
71
137
  const {
72
138
  maxValues,
139
+ possibleMatchValues,
73
140
  matchingValues
74
141
  } = getValueCount(true);
75
- return matchingValues / maxValues * 0.75;
142
+ if (matchingValues < 1) {
143
+ debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);
144
+ return -0.75;
145
+ }
146
+ debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues}/${possibleMatchValues} valid identifiers match.`);
147
+ // ignore non-matches if there is mismatching amount of values
148
+ debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);
149
+ //we give some kind of penalty for mismatching amount of values instead of simple divide?
150
+ const penaltyForMissing = 0.1 * (maxValues - possibleMatchValues);
151
+ const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues);
152
+ debug(`\t points: penaltyForMissing: ${penaltyForMissing}`);
153
+ debug(`\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);
154
+ return 0.75 - penaltyForMisMatch - penaltyForMissing;
155
+ //return matchingValues / possibleMatchValues * 0.75;
76
156
  }
157
+ // If both do not have valid identifiers, compare all identifiers
77
158
  const {
78
159
  maxValues,
79
160
  matchingValues
80
161
  } = getValueCount();
162
+ debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);
81
163
  return matchingValues / maxValues * 0.2;
82
164
  function bothHaveValidIdentifiers() {
83
165
  const aValues = a.filter(({
84
166
  code
85
- }) => code === 'a');
167
+ }) => validIdentifierSubfieldCodes.includes(code));
86
168
  const bValues = a.filter(({
87
169
  code
88
- }) => code === 'a');
170
+ }) => validIdentifierSubfieldCodes.includes(code));
171
+ debug(`A: ${aValues.length} valid ${identifier} identifiers`);
172
+ debug(`B: ${bValues.length} valid ${identifier} identifiers`);
89
173
  return aValues.length > 0 && bValues.length > 0;
90
174
  }
91
175
  function getValueCount(validOnly = false) {
92
- const aValues = getIdentifiers(a);
93
- const bValues = getIdentifiers(b);
176
+ const aValues = getIdentifiers(a, validOnly);
177
+ const bValues = getIdentifiers(b, validOnly);
178
+ const matchingValues = getMatchingValuesAmount(aValues, bValues);
94
179
  return {
95
180
  maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,
96
- matchingValues: aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length
181
+ // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)
182
+ possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,
183
+ matchingValues
97
184
  };
98
- function getIdentifiers(values) {
185
+ function getMatchingValuesAmount(aValues, bValues) {
186
+ if (bValues.length > aValues.length) {
187
+ return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
188
+ }
189
+ if (aValues.length > bValues.length) {
190
+ return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
191
+ }
192
+
193
+ // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases
194
+ // there would be duplicate values
195
+ const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
196
+ const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
197
+ return aToB < bToA ? aToB : bToA;
198
+ }
199
+ function getIdentifiers(values, validOnly) {
99
200
  if (validOnly) {
100
201
  return values.filter(({
101
202
  code
102
- }) => code === 'a').map(({
203
+ }) => validIdentifierSubfieldCodes.includes(code)).map(({
103
204
  value
104
205
  }) => value);
105
206
  }
package/dist/match-detection/features/bib/standard-identifier-factory.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"standard-identifier-factory.js","names":["debug","createDebugLogger","debugData","extend","pattern","subfieldCodes","extract","compare","record","field","get","subfields","filter","code","includes","map","value","testStringOrNumber","String","replace","a","b","length","bothHaveValidIdentifiers","maxValues","matchingValues","getValueCount","aValues","bValues","validOnly","getIdentifiers","aValue","some","bValue","values"],"sources":["../../../../src/match-detection/features/bib/standard-identifier-factory.js"],"sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport {testStringOrNumber} from '../../../matching-utils';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers');\nconst debugData = debug.extend('data');\n\nexport default ({pattern, subfieldCodes}) => {\n return {extract, compare};\n\n function extract({record}) {\n const [field] = record.get(pattern);\n\n if (field) {\n return field.subfields\n .filter(({code}) => subfieldCodes.includes(code))\n .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value).replace(/-/ug, '') : ''}))\n .filter(value => value);\n }\n\n return [];\n }\n\n function compare(a, b) {\n if (a.length === 0 || b.length === 0) {\n debugData(`No standardidentifiers to compare`);\n return 0;\n }\n\n if (bothHaveValidIdentifiers()) {\n const {maxValues, matchingValues} = getValueCount(true);\n return matchingValues / maxValues * 0.75;\n }\n\n const {maxValues, matchingValues} = getValueCount();\n return matchingValues / maxValues * 0.2;\n\n function bothHaveValidIdentifiers() {\n const aValues = a.filter(({code}) => code === 'a');\n const bValues = a.filter(({code}) => code === 'a');\n return aValues.length > 0 && bValues.length > 0;\n }\n\n function getValueCount(validOnly = false) {\n const aValues = getIdentifiers(a);\n const bValues = getIdentifiers(b);\n\n return {\n maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n matchingValues: aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length\n };\n\n function getIdentifiers(values) {\n if (validOnly) {\n return values\n .filter(({code}) => code === 'a')\n .map(({value}) => value);\n }\n\n return values.map(({value}) => value);\n }\n }\n }\n};\n"],"mappings":";;;;;;AA4BA;AACA;AAA2D;AA7B3D;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAKA,MAAMA,KAAK,GAAG,IAAAC,cAAiB,EAAC,iFAAiF,CAAC;AAClH,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;AAAC,eAExB,CAAC;EAACC,OAAO;EAAEC;AAAa,CAAC,KAAK;EAC3C,OAAO;IAACC,OAAO;IAAEC;EAAO,CAAC;EAEzB,SAASD,OAAO,CAAC;IAACE;EAAM,CAAC,EAAE;IACzB,MAAM,CAACC,KAAK,CAAC,GAAGD,MAAM,CAACE,GAAG,CAACN,OAAO,CAAC;IAEnC,IAAIK,KAAK,EAAE;MACT,OAAOA,KAAK,CAACE,SAAS,CACnBC,MAAM,CAAC,CAAC;QAACC;MAAI,CAAC,KAAKR,aAAa,CAACS,QAAQ,CAACD,IAAI,CAAC,CAAC,CAChDE,GAAG,CAAC,CAAC;QAACF,IAAI;QAAEG;MAAK,CAAC,MAAM;QAACH,IAAI;QAAEG,KAAK,EAAE,IAAAC,iCAAkB,EAACD,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,CAACG,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG;MAAE,CAAC,CAAC,CAAC,CAC1GP,MAAM,CAACI,KAAK,IAAIA,KAAK,CAAC;IAC3B;IAEA,OAAO,EAAE;EACX;EAEA,SAAST,OAAO,CAACa,CAAC,EAAEC,CAAC,EAAE;IACrB,IAAID,CAAC,CAACE,MAAM,KAAK,CAAC,IAAID,CAAC,CAACC,MAAM,KAAK,CAAC,EAAE;MACpCpB,SAAS,CAAE,mCAAkC,CAAC;MAC9C,OAAO,CAAC;IACV;IAEA,IAAIqB,wBAAwB,EAAE,EAAE;MAC9B,MAAM;QAACC,SAAS;QAAEC;MAAc,CAAC,GAAGC,aAAa,CAAC,IAAI,CAAC;MACvD,OAAOD,cAAc,GAAGD,SAAS,GAAG,IAAI;IAC1C;IAEA,MAAM;MAACA,SAAS;MAAEC;IAAc,CAAC,GAAGC,aAAa,EAAE;IACnD,OAAOD,cAAc,GAAGD,SAAS,GAAG,GAAG;IAEvC,SAASD,wBAAwB,GAAG;MAClC,MAAMI,OAAO,GAAGP,CAAC,CAACR,MAAM,CAAC,CAAC;QAACC;MAAI,CAAC,KAAKA,IAAI,KAAK,GAAG,CAAC;MAClD,MAAMe,OAAO,GAAGR,CAAC,CAACR,MAAM,CAAC,CAAC;QAACC;MAAI,CAAC,KAAKA,IAAI,KAAK,GAAG,CAAC;MAClD,OAAOc,OAAO,CAACL,MAAM,GAAG,CAAC,IAAIM,OAAO,CAACN,MAAM,GAAG,CAAC;IACjD;IAEA,SAASI,aAAa,CAACG,SAAS,GAAG,KAAK,EAAE;MACxC,MAAMF,OAAO,GAAGG,cAAc,CAACV,CAAC,CAAC;MACjC,MAAMQ,OAAO,GAAGE,cAAc,CAACT,CAAC,CAAC;MAEjC,OAAO;QACLG,SAAS,EAAEG,OAAO,CAACL,MAAM,GAAGM,OAAO,CAACN,MAAM,GAAGK,OAAO,CAACL,MAAM,GAAGM,OAAO,CAACN,MAAM;QAC5EG,cAAc,EAAEE,OAAO,CAACf,MAAM,CAACmB,MAAM,IAAIH,OAAO,CAACI,IAAI,CAACC,MAAM,IAAIF,MAAM,KAAKE,MAAM,CAAC,CAAC,CAACX;MACtF,CAAC;MAED,SAASQ,cAAc,CAACI,MAAM,EAAE;QAC9B,IAAIL,SAAS,EAAE;UACb,OAAOK,MAAM,CACVtB,MAAM,CAAC,CAAC;YAACC;UAAI,CAAC,KAAKA,IAAI,KAAK,GAAG,CAAC,CAChCE,GAAG,CAAC,CAAC;YAACC;UAAK,CAAC,KAAKA,KAAK,CAAC;QAC5B;QAEA,OAAOkB,MAAM,CAACnB,GAAG,CAAC,CAAC;UAACC;QAAK,CAAC,KAAKA,KAAK,CAAC;MACvC;IACF;EACF;AACF,CAAC;AAAA"}
1
+ {"version":3,"file":"standard-identifier-factory.js","names":["pattern","subfieldCodes","identifier","validIdentifierSubfieldCodes","invalidIdentifierSubfieldCodes","validatorAndNormalizer","undefined","debug","createDebugLogger","debugData","extend","extract","compare","record","recordExternal","label","fields","get","length","identifiersFromFields","map","field","extractSubfieldsFromField","JSON","stringify","allIdentifiers","flat","validatedAndNormalizedIdentifiers","validateAndNormalizeIdentifiers","identifierSubs","identifiers","uniqueSubfields","idSub","validateAndNormalizeIdentifier","normalizeHyphens","valid","value","includes","code","replace","a","b","bothHaveValidIdentifiers","maxValues","possibleMatchValues","matchingValues","getValueCount","penaltyForMissing","penaltyForMisMatch","aValues","filter","bValues","validOnly","getIdentifiers","getMatchingValuesAmount","aValue","some","bValue","aToB","bToA","values"],"sources":["../../../../src/match-detection/features/bib/standard-identifier-factory.js"],"sourcesContent":["/* eslint-disable max-statements */\n/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\nimport {extractSubfieldsFromField, uniqueSubfields} from '../../../matching-utils';\n\n// Note about validity of standardIdentifiers:\n// We have three types of invalid standardIdentifiers:\n// 1. Formally invalid standardIdentifiers (ie. typos either in the resource or the record)\n// 2. Formally valid standardIdentifiers that are used in a wrong resource\n// 3. Canceled standardIdentifiers\n\n// Matcher could and should check that a standardIdentifier found in a subfield for a valid identifier is formally valid, and if it's not formally valid, handle it as an invalid standardIdentifier\n// Formally valid standardIdentifiers found in subfield for invalid identifier cannot be handled as valid standardIdentifiers, because they can be a case of type 2) or 3) invalid standardIdentifiers\n// We could also do a separate handling for formally valid an formally invalid standardIdentifiers\n\nexport default ({pattern, subfieldCodes, identifier, validIdentifierSubfieldCodes = ['a'], invalidIdentifierSubfieldCodes = ['z'], validatorAndNormalizer = undefined}) => {\n const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);\n const debugData = debug.extend('data');\n\n return {extract, compare};\n\n function extract({record, recordExternal}) {\n const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';\n const fields = record.get(pattern);\n debugData(`${label}: ${fields.length} ${identifier}-fields `);\n\n // extractIdentifierSubfield normalizes hyphens away from the subfield values\n const identifiersFromFields = fields.map(field => extractSubfieldsFromField(field, subfieldCodes));\n debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);\n const allIdentifiers = identifiersFromFields.flat();\n debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);\n\n const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({identifierSubs: allIdentifiers, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes});\n\n const identifiers = uniqueSubfields(validatedAndNormalizedIdentifiers);\n\n debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);\n return identifiers;\n\n function validateAndNormalizeIdentifiers({identifierSubs, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {\n if (validatorAndNormalizer) {\n return identifierSubs.map((idSub) => validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}));\n }\n return identifierSubs.map((idSub) => normalizeHyphens(idSub));\n }\n\n function validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {\n const {valid, value} = validatorAndNormalizer(idSub.value);\n if (validIdentifierSubfieldCodes.includes(idSub.code) && valid === false) {\n const [code] = invalidIdentifierSubfieldCodes;\n return {code, value};\n }\n return {code: idSub.code, value};\n }\n\n function normalizeHyphens(idSub) {\n return {code: idSub.code, value: idSub.value.replace(/-/ug, '')};\n }\n\n\n }\n\n function compare(a, b) {\n debug(`Comparing A and B`);\n if (a.length === 0 || b.length === 0) {\n debugData(`No standardidentifiers (${identifier}) to compare`);\n return 0;\n }\n\n debugData(`A: ${JSON.stringify(a)}`);\n debugData(`B: ${JSON.stringify(b)}`);\n\n\n if (bothHaveValidIdentifiers()) {\n // Compare only valid identifiers, if both have valid idenfiers\n const {maxValues, possibleMatchValues, matchingValues} = getValueCount(true);\n if (matchingValues < 1) {\n debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);\n return -0.75;\n }\n debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues}/${possibleMatchValues} valid identifiers match.`);\n // ignore non-matches if there is mismatching amount of values\n debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);\n //we give some kind of penalty for mismatching amount of values instead of simple divide?\n const penaltyForMissing = 0.1 * (maxValues - possibleMatchValues);\n const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues);\n debug(`\\t points: penaltyForMissing: ${penaltyForMissing}`);\n debug(`\\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);\n\n return 0.75 - penaltyForMisMatch - penaltyForMissing;\n //return matchingValues / possibleMatchValues * 0.75;\n }\n // If both do not have valid identifiers, compare all identifiers\n const {maxValues, matchingValues} = getValueCount();\n debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);\n\n return matchingValues / maxValues * 0.2;\n\n function bothHaveValidIdentifiers() {\n const aValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));\n const bValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));\n debug(`A: ${aValues.length} valid ${identifier} identifiers`);\n debug(`B: ${bValues.length} valid ${identifier} identifiers`);\n return aValues.length > 0 && bValues.length > 0;\n }\n\n function getValueCount(validOnly = false) {\n const aValues = getIdentifiers(a, validOnly);\n const bValues = getIdentifiers(b, validOnly);\n\n const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n return {\n maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n matchingValues\n };\n\n function getMatchingValuesAmount(aValues, bValues) {\n if (bValues.length > aValues.length) {\n return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n }\n if (aValues.length > bValues.length) {\n return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n }\n\n // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n // there would be duplicate values\n const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n return aToB < bToA ? aToB : bToA;\n }\n\n function getIdentifiers(values, validOnly) {\n if (validOnly) {\n return values\n .filter(({code}) => validIdentifierSubfieldCodes.includes(code))\n .map(({value}) => value);\n }\n\n return values.map(({value}) => value);\n }\n }\n }\n};\n"],"mappings":";;;;;;AA6BA;AACA;AAAmF;AA9BnF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAKA;AACA;AACA;AACA;AACA;AAEA;AACA;AACA;AAAA,eAEe,CAAC;EAACA,OAAO;EAAEC,aAAa;EAAEC,UAAU;EAAEC,4BAA4B,GAAG,CAAC,GAAG,CAAC;EAAEC,8BAA8B,GAAG,CAAC,GAAG,CAAC;EAAEC,sBAAsB,GAAGC;AAAS,CAAC,KAAK;EACzK,MAAMC,KAAK,GAAG,IAAAC,cAAiB,EAAE,mFAAkFN,UAAW,EAAC,CAAC;EAChI,MAAMO,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtC,OAAO;IAACC,OAAO;IAAEC;EAAO,CAAC;EAEzB,SAASD,OAAO,CAAC;IAACE,MAAM;IAAEC;EAAc,CAAC,EAAE;IACzC,MAAMC,KAAK,GAAGD,cAAc,IAAIA,cAAc,CAACC,KAAK,GAAGD,cAAc,CAACC,KAAK,GAAG,QAAQ;IACtF,MAAMC,MAAM,GAAGH,MAAM,CAACI,GAAG,CAACjB,OAAO,CAAC;IAClCS,SAAS,CAAE,GAAEM,KAAM,KAAIC,MAAM,CAACE,MAAO,IAAGhB,UAAW,UAAS,CAAC;;IAE7D;IACA,MAAMiB,qBAAqB,GAAGH,MAAM,CAACI,GAAG,CAACC,KAAK,IAAI,IAAAC,wCAAyB,EAACD,KAAK,EAAEpB,aAAa,CAAC,CAAC;IAClGQ,SAAS,CAAE,GAAEM,KAAM,sBAAqBI,qBAAqB,CAACD,MAAO,MAAKK,IAAI,CAACC,SAAS,CAACL,qBAAqB,CAAE,EAAC,CAAC;IAClH,MAAMM,cAAc,GAAGN,qBAAqB,CAACO,IAAI,EAAE;IACnDjB,SAAS,CAAE,GAAEM,KAAM,2BAA0BU,cAAc,CAACP,MAAO,MAAKK,IAAI,CAACC,SAAS,CAACC,cAAc,CAAE,EAAC,CAAC;IAEzG,MAAME,iCAAiC,GAAGC,+BAA+B,CAAC;MAACC,cAAc,EAAEJ,cAAc;MAAEpB,sBAAsB;MAAEF,4BAA4B;MAAEC;IAA8B,CAAC,CAAC;IAEjM,MAAM0B,WAAW,GAAG,IAAAC,8BAAe,EAACJ,iCAAiC,CAAC;IAEtElB,SAAS,CAAE,GAAEM,KAAM,6BAA4Be,WAAW,CAACZ,MAAO,MAAKK,IAAI,CAACC,SAAS,CAACM,WAAW,CAAE,EAAC,CAAC;IACrG,OAAOA,WAAW;IAElB,SAASF,+BAA+B,CAAC;MAACC,cAAc;MAAExB,sBAAsB;MAAEF,4BAA4B;MAAEC;IAA8B,CAAC,EAAE;MAC/I,IAAIC,sBAAsB,EAAE;QAC1B,OAAOwB,cAAc,CAACT,GAAG,CAAEY,KAAK,IAAKC,8BAA8B,CAAC;UAACD,KAAK;UAAE3B,sBAAsB;UAAEF,4BAA4B;UAAEC;QAA8B,CAAC,CAAC,CAAC;MACrK;MACA,OAAOyB,cAAc,CAACT,GAAG,CAAEY,KAAK,IAAKE,gBAAgB,CAACF,KAAK,CAAC,CAAC;IAC/D;IAEA,SAASC,8BAA8B,CAAC;MAACD,KAAK;MAAE3B,sBAAsB;MAAEF,4BAA4B;MAAEC;IAA8B,CAAC,EAAE;MACrI,MAAM;QAAC+B,KAAK;QAAEC;MAAK,CAAC,GAAG/B,sBAAsB,CAAC2B,KAAK,CAACI,KAAK,CAAC;MAC1D,IAAIjC,4BAA4B,CAACkC,QAAQ,CAACL,KAAK,CAACM,IAAI,CAAC,IAAIH,KAAK,KAAK,KAAK,EAAE;QACxE,MAAM,CAACG,IAAI,CAAC,GAAGlC,8BAA8B;QAC7C,OAAO;UAACkC,IAAI;UAAEF;QAAK,CAAC;MACtB;MACA,OAAO;QAACE,IAAI,EAAEN,KAAK,CAACM,IAAI;QAAEF;MAAK,CAAC;IAClC;IAEA,SAASF,gBAAgB,CAACF,KAAK,EAAE;MAC/B,OAAO;QAACM,IAAI,EAAEN,KAAK,CAACM,IAAI;QAAEF,KAAK,EAAEJ,KAAK,CAACI,KAAK,CAACG,OAAO,CAAC,KAAK,EAAE,EAAE;MAAC,CAAC;IAClE;EAGF;EAEA,SAAS3B,OAAO,CAAC4B,CAAC,EAAEC,CAAC,EAAE;IACrBlC,KAAK,CAAE,mBAAkB,CAAC;IAC1B,IAAIiC,CAAC,CAACtB,MAAM,KAAK,CAAC,IAAIuB,CAAC,CAACvB,MAAM,KAAK,CAAC,EAAE;MACpCT,SAAS,CAAE,2BAA0BP,UAAW,cAAa,CAAC;MAC9D,OAAO,CAAC;IACV;IAEAO,SAAS,CAAE,MAAKc,IAAI,CAACC,SAAS,CAACgB,CAAC,CAAE,EAAC,CAAC;IACpC/B,SAAS,CAAE,MAAKc,IAAI,CAACC,SAAS,CAACiB,CAAC,CAAE,EAAC,CAAC;IAGpC,IAAIC,wBAAwB,EAAE,EAAE;MAC9B;MACA,MAAM;QAACC,SAAS;QAAEC,mBAAmB;QAAEC;MAAc,CAAC,GAAGC,aAAa,CAAC,IAAI,CAAC;MAC5E,IAAID,cAAc,GAAG,CAAC,EAAE;QACtBtC,KAAK,CAAE,wCAAuCL,UAAW,6BAA4B,CAAC;QACtF,OAAO,CAAC,IAAI;MACd;MACAK,KAAK,CAAE,wCAAuCL,UAAW,MAAK2C,cAAe,IAAGD,mBAAoB,2BAA0B,CAAC;MAC/H;MACArC,KAAK,CAAE,qBAAoBqC,mBAAoB,IAAGD,SAAU,EAAC,CAAC;MAC9D;MACA,MAAMI,iBAAiB,GAAG,GAAG,IAAIJ,SAAS,GAAGC,mBAAmB,CAAC;MACjE,MAAMI,kBAAkB,GAAG,GAAG,IAAIJ,mBAAmB,GAAGC,cAAc,CAAC;MACvEtC,KAAK,CAAE,iCAAgCwC,iBAAkB,EAAC,CAAC;MAC3DxC,KAAK,CAAE,kCAAiCyC,kBAAmB,EAAC,CAAC;MAE7D,OAAO,IAAI,GAAGA,kBAAkB,GAAGD,iBAAiB;MACpD;IACF;IACA;IACA,MAAM;MAACJ,SAAS;MAAEE;IAAc,CAAC,GAAGC,aAAa,EAAE;IACnDvC,KAAK,CAAE,+CAA8CL,UAAW,MAAK2C,cAAe,IAAGF,SAAU,mCAAkC,CAAC;IAEpI,OAAOE,cAAc,GAAGF,SAAS,GAAG,GAAG;IAEvC,SAASD,wBAAwB,GAAG;MAClC,MAAMO,OAAO,GAAGT,CAAC,CAACU,MAAM,CAAC,CAAC;QAACZ;MAAI,CAAC,KAAKnC,4BAA4B,CAACkC,QAAQ,CAACC,IAAI,CAAC,CAAC;MACjF,MAAMa,OAAO,GAAGX,CAAC,CAACU,MAAM,CAAC,CAAC;QAACZ;MAAI,CAAC,KAAKnC,4BAA4B,CAACkC,QAAQ,CAACC,IAAI,CAAC,CAAC;MACjF/B,KAAK,CAAE,MAAK0C,OAAO,CAAC/B,MAAO,UAAShB,UAAW,cAAa,CAAC;MAC7DK,KAAK,CAAE,MAAK4C,OAAO,CAACjC,MAAO,UAAShB,UAAW,cAAa,CAAC;MAC7D,OAAO+C,OAAO,CAAC/B,MAAM,GAAG,CAAC,IAAIiC,OAAO,CAACjC,MAAM,GAAG,CAAC;IACjD;IAEA,SAAS4B,aAAa,CAACM,SAAS,GAAG,KAAK,EAAE;MACxC,MAAMH,OAAO,GAAGI,cAAc,CAACb,CAAC,EAAEY,SAAS,CAAC;MAC5C,MAAMD,OAAO,GAAGE,cAAc,CAACZ,CAAC,EAAEW,SAAS,CAAC;MAE5C,MAAMP,cAAc,GAAGS,uBAAuB,CAACL,OAAO,EAAEE,OAAO,CAAC;MAEhE,OAAO;QACLR,SAAS,EAAEM,OAAO,CAAC/B,MAAM,GAAGiC,OAAO,CAACjC,MAAM,GAAG+B,OAAO,CAAC/B,MAAM,GAAGiC,OAAO,CAACjC,MAAM;QAC5E;QACA0B,mBAAmB,EAAEK,OAAO,CAAC/B,MAAM,GAAGiC,OAAO,CAACjC,MAAM,GAAGiC,OAAO,CAACjC,MAAM,GAAG+B,OAAO,CAAC/B,MAAM;QACtF2B;MACF,CAAC;MAED,SAASS,uBAAuB,CAACL,OAAO,EAAEE,OAAO,EAAE;QACjD,IAAIA,OAAO,CAACjC,MAAM,GAAG+B,OAAO,CAAC/B,MAAM,EAAE;UACnC,OAAO+B,OAAO,CAACC,MAAM,CAACK,MAAM,IAAIJ,OAAO,CAACK,IAAI,CAACC,MAAM,IAAIF,MAAM,KAAKE,MAAM,CAAC,CAAC,CAACvC,MAAM;QACnF;QACA,IAAI+B,OAAO,CAAC/B,MAAM,GAAGiC,OAAO,CAACjC,MAAM,EAAE;UACnC,OAAOiC,OAAO,CAACD,MAAM,CAACO,MAAM,IAAIR,OAAO,CAACO,IAAI,CAACD,MAAM,IAAIE,MAAM,KAAKF,MAAM,CAAC,CAAC,CAACrC,MAAM;QACnF;;QAEA;QACA;QACA,MAAMwC,IAAI,GAAGT,OAAO,CAACC,MAAM,CAACK,MAAM,IAAIJ,OAAO,CAACK,IAAI,CAACC,MAAM,IAAIF,MAAM,KAAKE,MAAM,CAAC,CAAC,CAACvC,MAAM;QACvF,MAAMyC,IAAI,GAAGR,OAAO,CAACD,MAAM,CAACO,MAAM,IAAIR,OAAO,CAACO,IAAI,CAACD,MAAM,IAAIE,MAAM,KAAKF,MAAM,CAAC,CAAC,CAACrC,MAAM;QAEvF,OAAOwC,IAAI,GAAGC,IAAI,GAAGD,IAAI,GAAGC,IAAI;MAClC;MAEA,SAASN,cAAc,CAACO,MAAM,EAAER,SAAS,EAAE;QACzC,IAAIA,SAAS,EAAE;UACb,OAAOQ,MAAM,CACVV,MAAM,CAAC,CAAC;YAACZ;UAAI,CAAC,KAAKnC,4BAA4B,CAACkC,QAAQ,CAACC,IAAI,CAAC,CAAC,CAC/DlB,GAAG,CAAC,CAAC;YAACgB;UAAK,CAAC,KAAKA,KAAK,CAAC;QAC5B;QAEA,OAAOwB,MAAM,CAACxC,GAAG,CAAC,CAAC;UAACgB;QAAK,CAAC,KAAKA,KAAK,CAAC;MACvC;IACF;EACF;AACF,CAAC;AAAA"}
package/dist/matching-utils.js CHANGED
@@ -3,9 +3,11 @@
3
3
  Object.defineProperty(exports, "__esModule", {
4
4
  value: true
5
5
  });
6
+ exports.extractSubfieldsFromField = extractSubfieldsFromField;
6
7
  exports.getMelindaIdsF035 = getMelindaIdsF035;
7
8
  exports.getSubfieldValues = getSubfieldValues;
8
9
  exports.testStringOrNumber = testStringOrNumber;
10
+ exports.uniqueSubfields = uniqueSubfields;
9
11
  exports.validateSidFieldSubfieldCounts = validateSidFieldSubfieldCounts;
10
12
  var _debug = _interopRequireDefault(require("debug"));
11
13
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
@@ -90,4 +92,28 @@ function testStringOrNumber(value) {
90
92
  }
91
93
  return false;
92
94
  }
95
+ function extractSubfieldsFromField(field, subfieldCodes) {
96
+ if (field === undefined || field.subfields === undefined) {
97
+ return [];
98
+ }
99
+ const resultSubfields = field.subfields.filter(({
100
+ code
101
+ }) => subfieldCodes.includes(code)).map(({
102
+ code,
103
+ value
104
+ }) => ({
105
+ code,
106
+ value: testStringOrNumber(value) ? String(value) : ''
107
+ })).filter(value => value);
108
+ return resultSubfields;
109
+ }
110
+ function uniqueSubfields(subfields) {
111
+ return subfields.reduce((arr, e) => {
112
+ if (!arr.find(item => item.code === e.code && item.value === e.value)) {
113
+ const newArr = arr.concat(e);
114
+ return newArr;
115
+ }
116
+ return arr;
117
+ }, []);
118
+ }
93
119
  //# sourceMappingURL=matching-utils.js.map
package/dist/matching-utils.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"matching-utils.js","names":["debug","createDebugLogger","debugData","extend","getMelindaIdsF035","record","f035s","getFields","length","allF035MelindaIds","concat","map","toMelindaIds","f035MelindaIds","Set","JSON","stringify","subfields","melindaIdRegExp","filter","sub","includes","code","testStringOrNumber","value","test","String","replace","validateSidFieldSubfieldCounts","field","countC","countSubfields","countB","subfieldCode","getSubfieldValues"],"sources":["../src/matching-utils.js"],"sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\nexport function getMelindaIdsF035(record) {\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n const debugData = debug.extend('data');\n\n const f035s = record.getFields('035');\n\n if (f035s.length < 1) {\n debug(`No f035s found.`);\n return [];\n }\n\n const allF035MelindaIds = [].concat(...f035s.map(toMelindaIds));\n const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n return f035MelindaIds;\n\n function toMelindaIds({subfields}) {\n const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n return subfields\n .filter(sub => ['a', 'z'].includes(sub.code))\n .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n\n }\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n // Valid SID-fields have just one $c and one $b\n debugData(`Validating SID field ${JSON.stringify(field)}`);\n const countC = countSubfields(field, 'c');\n const countB = countSubfields(field, 'b');\n debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n // debug(`Counting subfields ${subfieldCode}`);\n return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n return field.subfields\n .filter(({code}) => code === subfieldCode)\n .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n if (typeof value === 'string' || typeof value === 'number') {\n return true;\n }\n return false;\n}\n"],"mappings":";;;;;;;;;AA4BA;AAAsC;AA5BtC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAIA,MAAMA,KAAK,GAAG,IAAAC,cAAiB,EAAC,yCAAyC,CAAC;AAC1E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;AAE/B,SAASC,iBAAiB,CAACC,MAAM,EAAE;EAExC,MAAML,KAAK,GAAG,IAAAC,cAAiB,EAAC,8CAA8C,CAAC;EAC/E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtC,MAAMG,KAAK,GAAGD,MAAM,CAACE,SAAS,CAAC,KAAK,CAAC;EAErC,IAAID,KAAK,CAACE,MAAM,GAAG,CAAC,EAAE;IACpBR,KAAK,CAAE,iBAAgB,CAAC;IACxB,OAAO,EAAE;EACX;EAEA,MAAMS,iBAAiB,GAAG,EAAE,CAACC,MAAM,CAAC,GAAGJ,KAAK,CAACK,GAAG,CAACC,YAAY,CAAC,CAAC;EAC/D,MAAMC,cAAc,GAAG,CAAC,GAAG,IAAIC,GAAG,CAACL,iBAAiB,CAAC,CAAC;EAEtDP,SAAS,CAAE,WAAUI,KAAK,CAACE,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACV,KAAK,CAAE,EAAC,CAAC;EAC/DJ,SAAS,CAAE,QAAOO,iBAAiB,CAACD,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACP,iBAAiB,CAAE,EAAC,CAAC;EACpFP,SAAS,CAAE,eAAcW,cAAc,CAACL,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACH,cAAc,CAAE,EAAC,CAAC;EAErF,OAAOA,cAAc;EAErB,SAASD,YAAY,CAAC;IAACK;EAAS,CAAC,EAAE;IACjC,MAAMC,eAAe,GAAG,8CAA8C;IAEtE,OAAOD,SAAS,CACbE,MAAM,CAACC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAACC,QAAQ,CAACD,GAAG,CAACE,IAAI,CAAC,CAAC,CAC5CH,MAAM,CAACC,GAAG,IAAIG,kBAAkB,CAACH,GAAG,CAACI,KAAK,CAAC,IAAIN,eAAe,CAACO,IAAI,CAACC,MAAM,CAACN,GAAG,CAACI,KAAK,CAAC,CAAC,CAAC,CACvFb,GAAG,CAAC,CAAC;MAACa;IAAK,CAAC,KAAKD,kBAAkB,CAACC,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,CAACG,OAAO,CAACT,eAAe,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;EAEvG;AACF;AAEO,SAASU,8BAA8B,CAACC,KAAK,EAAE;EACpD;EACA3B,SAAS,CAAE,wBAAuBa,IAAI,CAACC,SAAS,CAACa,KAAK,CAAE,EAAC,CAAC;EAC1D,MAAMC,MAAM,GAAGC,cAAc,CAACF,KAAK,EAAE,GAAG,CAAC;EACzC,MAAMG,MAAM,GAAGD,cAAc,CAACF,KAAK,EAAE,GAAG,CAAC;EACzC7B,KAAK,CAAE,SAAQ8B,MAAO,eAAcE,MAAO,qBAAoBF,MAAM,KAAK,CAAC,IAAIE,MAAM,KAAK,CAAE,EAAC,CAAC;EAE9F,OAAOF,MAAM,KAAK,CAAC,IAAIE,MAAM,KAAK,CAAC;AACrC;AAEA,SAASD,cAAc,CAACF,KAAK,EAAEI,YAAY,EAAE;EAC3C;EACA,OAAOJ,KAAK,CAACZ,SAAS,CAACE,MAAM,CAAC,CAAC;IAACG;EAAI,CAAC,KAAKA,IAAI,KAAKW,YAAY,CAAC,CAACzB,MAAM;AACzE;AAEO,SAAS0B,iBAAiB,CAACL,KAAK,EAAEI,YAAY,EAAE;EACrD/B,SAAS,CAAE,oBAAmB+B,YAAa,SAAQlB,IAAI,CAACC,SAAS,CAACa,KAAK,CAAE,EAAC,CAAC;EAC3E,OAAOA,KAAK,CAACZ,SAAS,CACnBE,MAAM,CAAC,CAAC;IAACG;EAAI,CAAC,KAAKA,IAAI,KAAKW,YAAY,CAAC,CACzCtB,GAAG,CAAC,CAAC;IAACa;EAAK,CAAC,KAAKD,kBAAkB,CAACC,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,GAAG,EAAE,CAAC,CAChEL,MAAM,CAACK,KAAK,IAAIA,KAAK,CAAC;AAC3B;AAEO,SAASD,kBAAkB,CAACC,KAAK,EAAE;EACxC,IAAI,OAAOA,KAAK,KAAK,QAAQ,IAAI,OAAOA,KAAK,KAAK,QAAQ,EAAE;IAC1D,OAAO,IAAI;EACb;EACA,OAAO,KAAK;AACd"}
1
+ {"version":3,"file":"matching-utils.js","names":["debug","createDebugLogger","debugData","extend","getMelindaIdsF035","record","f035s","getFields","length","allF035MelindaIds","concat","map","toMelindaIds","f035MelindaIds","Set","JSON","stringify","subfields","melindaIdRegExp","filter","sub","includes","code","testStringOrNumber","value","test","String","replace","validateSidFieldSubfieldCounts","field","countC","countSubfields","countB","subfieldCode","getSubfieldValues","extractSubfieldsFromField","subfieldCodes","undefined","resultSubfields","uniqueSubfields","reduce","arr","e","find","item","newArr"],"sources":["../src/matching-utils.js"],"sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\nexport function getMelindaIdsF035(record) {\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n const debugData = debug.extend('data');\n\n const f035s = record.getFields('035');\n\n if (f035s.length < 1) {\n debug(`No f035s found.`);\n return [];\n }\n\n const allF035MelindaIds = [].concat(...f035s.map(toMelindaIds));\n const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n return f035MelindaIds;\n\n function toMelindaIds({subfields}) {\n const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n return subfields\n .filter(sub => ['a', 'z'].includes(sub.code))\n .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n\n }\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n // Valid SID-fields have just one $c and one $b\n debugData(`Validating SID field ${JSON.stringify(field)}`);\n const countC = countSubfields(field, 'c');\n const countB = countSubfields(field, 'b');\n debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n // debug(`Counting subfields ${subfieldCode}`);\n return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n return field.subfields\n .filter(({code}) => code === subfieldCode)\n .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n if (typeof value === 'string' || typeof value === 'number') {\n return true;\n }\n return false;\n}\n\nexport function extractSubfieldsFromField(field, subfieldCodes) {\n if (field === undefined || field.subfields === undefined) {\n return [];\n }\n const resultSubfields = field.subfields\n .filter(({code}) => subfieldCodes.includes(code))\n .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))\n .filter(value => value);\n return resultSubfields;\n}\n\nexport function uniqueSubfields(subfields) {\n return subfields.reduce((arr, e) => {\n if (!arr.find(item => item.code === e.code && item.value === e.value)) {\n const newArr = arr.concat(e);\n return newArr;\n }\n return arr;\n }, []);\n}\n"],"mappings":";;;;;;;;;;;AA4BA;AAAsC;AA5BtC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAIA,MAAMA,KAAK,GAAG,IAAAC,cAAiB,EAAC,yCAAyC,CAAC;AAC1E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;AAE/B,SAASC,iBAAiB,CAACC,MAAM,EAAE;EAExC,MAAML,KAAK,GAAG,IAAAC,cAAiB,EAAC,8CAA8C,CAAC;EAC/E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtC,MAAMG,KAAK,GAAGD,MAAM,CAACE,SAAS,CAAC,KAAK,CAAC;EAErC,IAAID,KAAK,CAACE,MAAM,GAAG,CAAC,EAAE;IACpBR,KAAK,CAAE,iBAAgB,CAAC;IACxB,OAAO,EAAE;EACX;EAEA,MAAMS,iBAAiB,GAAG,EAAE,CAACC,MAAM,CAAC,GAAGJ,KAAK,CAACK,GAAG,CAACC,YAAY,CAAC,CAAC;EAC/D,MAAMC,cAAc,GAAG,CAAC,GAAG,IAAIC,GAAG,CAACL,iBAAiB,CAAC,CAAC;EAEtDP,SAAS,CAAE,WAAUI,KAAK,CAACE,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACV,KAAK,CAAE,EAAC,CAAC;EAC/DJ,SAAS,CAAE,QAAOO,iBAAiB,CAACD,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACP,iBAAiB,CAAE,EAAC,CAAC;EACpFP,SAAS,CAAE,eAAcW,cAAc,CAACL,MAAO,MAAKO,IAAI,CAACC,SAAS,CAACH,cAAc,CAAE,EAAC,CAAC;EAErF,OAAOA,cAAc;EAErB,SAASD,YAAY,CAAC;IAACK;EAAS,CAAC,EAAE;IACjC,MAAMC,eAAe,GAAG,8CAA8C;IAEtE,OAAOD,SAAS,CACbE,MAAM,CAACC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAACC,QAAQ,CAACD,GAAG,CAACE,IAAI,CAAC,CAAC,CAC5CH,MAAM,CAACC,GAAG,IAAIG,kBAAkB,CAACH,GAAG,CAACI,KAAK,CAAC,IAAIN,eAAe,CAACO,IAAI,CAACC,MAAM,CAACN,GAAG,CAACI,KAAK,CAAC,CAAC,CAAC,CACvFb,GAAG,CAAC,CAAC;MAACa;IAAK,CAAC,KAAKD,kBAAkB,CAACC,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,CAACG,OAAO,CAACT,eAAe,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;EAEvG;AACF;AAEO,SAASU,8BAA8B,CAACC,KAAK,EAAE;EACpD;EACA3B,SAAS,CAAE,wBAAuBa,IAAI,CAACC,SAAS,CAACa,KAAK,CAAE,EAAC,CAAC;EAC1D,MAAMC,MAAM,GAAGC,cAAc,CAACF,KAAK,EAAE,GAAG,CAAC;EACzC,MAAMG,MAAM,GAAGD,cAAc,CAACF,KAAK,EAAE,GAAG,CAAC;EACzC7B,KAAK,CAAE,SAAQ8B,MAAO,eAAcE,MAAO,qBAAoBF,MAAM,KAAK,CAAC,IAAIE,MAAM,KAAK,CAAE,EAAC,CAAC;EAE9F,OAAOF,MAAM,KAAK,CAAC,IAAIE,MAAM,KAAK,CAAC;AACrC;AAEA,SAASD,cAAc,CAACF,KAAK,EAAEI,YAAY,EAAE;EAC3C;EACA,OAAOJ,KAAK,CAACZ,SAAS,CAACE,MAAM,CAAC,CAAC;IAACG;EAAI,CAAC,KAAKA,IAAI,KAAKW,YAAY,CAAC,CAACzB,MAAM;AACzE;AAEO,SAAS0B,iBAAiB,CAACL,KAAK,EAAEI,YAAY,EAAE;EACrD/B,SAAS,CAAE,oBAAmB+B,YAAa,SAAQlB,IAAI,CAACC,SAAS,CAACa,KAAK,CAAE,EAAC,CAAC;EAC3E,OAAOA,KAAK,CAACZ,SAAS,CACnBE,MAAM,CAAC,CAAC;IAACG;EAAI,CAAC,KAAKA,IAAI,KAAKW,YAAY,CAAC,CACzCtB,GAAG,CAAC,CAAC;IAACa;EAAK,CAAC,KAAKD,kBAAkB,CAACC,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,GAAG,EAAE,CAAC,CAChEL,MAAM,CAACK,KAAK,IAAIA,KAAK,CAAC;AAC3B;AAEO,SAASD,kBAAkB,CAACC,KAAK,EAAE;EACxC,IAAI,OAAOA,KAAK,KAAK,QAAQ,IAAI,OAAOA,KAAK,KAAK,QAAQ,EAAE;IAC1D,OAAO,IAAI;EACb;EACA,OAAO,KAAK;AACd;AAEO,SAASW,yBAAyB,CAACN,KAAK,EAAEO,aAAa,EAAE;EAC9D,IAAIP,KAAK,KAAKQ,SAAS,IAAIR,KAAK,CAACZ,SAAS,KAAKoB,SAAS,EAAE;IACxD,OAAO,EAAE;EACX;EACA,MAAMC,eAAe,GAAGT,KAAK,CAACZ,SAAS,CACpCE,MAAM,CAAC,CAAC;IAACG;EAAI,CAAC,KAAKc,aAAa,CAACf,QAAQ,CAACC,IAAI,CAAC,CAAC,CAChDX,GAAG,CAAC,CAAC;IAACW,IAAI;IAAEE;EAAK,CAAC,MAAM;IAACF,IAAI;IAAEE,KAAK,EAAED,kBAAkB,CAACC,KAAK,CAAC,GAAGE,MAAM,CAACF,KAAK,CAAC,GAAG;EAAE,CAAC,CAAC,CAAC,CACvFL,MAAM,CAACK,KAAK,IAAIA,KAAK,CAAC;EACzB,OAAOc,eAAe;AACxB;AAEO,SAASC,eAAe,CAACtB,SAAS,EAAE;EACzC,OAAOA,SAAS,CAACuB,MAAM,CAAC,CAACC,GAAG,EAAEC,CAAC,KAAK;IAClC,IAAI,CAACD,GAAG,CAACE,IAAI,CAACC,IAAI,IAAIA,IAAI,CAACtB,IAAI,KAAKoB,CAAC,CAACpB,IAAI,IAAIsB,IAAI,CAACpB,KAAK,KAAKkB,CAAC,CAAClB,KAAK,CAAC,EAAE;MACrE,MAAMqB,MAAM,GAAGJ,GAAG,CAAC/B,MAAM,CAACgC,CAAC,CAAC;MAC5B,OAAOG,MAAM;IACf;IACA,OAAOJ,GAAG;EACZ,CAAC,EAAE,EAAE,CAAC;AACR"}
package/package.json CHANGED
@@ -14,7 +14,7 @@
14
14
  "url": "git@github.com:natlibfi/melinda-record-matching-js.git"
15
15
  },
16
16
  "license": "LGPL-3.0+",
17
- "version": "3.0.1-alpha.2",
17
+ "version": "3.0.2-alpha.1",
18
18
  "main": "./dist/index.js",
19
19
  "engines": {
20
20
  "node": ">=14"
@@ -41,6 +41,7 @@
41
41
  "@natlibfi/melinda-commons": "^12.0.6",
42
42
  "@natlibfi/sru-client": "^5.0.3",
43
43
  "debug": "^4.3.4",
44
+ "isbn3": "^1.1.28",
44
45
  "moment": "^2.29.4",
45
46
  "natural": "^5.2.3",
46
47
  "uuid": "^9.0.0",
package/src/match-detection/features/bib/isbn.js CHANGED
@@ -27,8 +27,27 @@
27
27
  */
28
28
 
29
29
  import createInterface from './standard-identifier-factory';
30
+ import {parse as isbnParse} from 'isbn3';
31
+ import createDebugLogger from 'debug';
32
+
33
+ const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:ISBN`);
34
+ const debugData = debug.extend('data');
30
35
 
31
36
  export default () => {
32
- const {extract, compare} = createInterface({pattern: /^020$/u, subfieldCodes: ['a', 'z']});
33
- return {extract, compare, name: 'ISBN'};
37
+ const IDENTIFIER_NAME = 'ISBN';
38
+
39
+ function validatorAndNormalizer(string) {
40
+ const isbnParseResult = isbnParse(string);
41
+ debugData(`isbnParseResult: ${JSON.stringify(isbnParseResult)}`);
42
+ if (isbnParseResult === null) {
43
+ debug(`Not parseable ISBN, just removing hyphens`);
44
+ return {valid: false, value: string.replace(/-/ug, '')};
45
+ }
46
+ debug(`Parseable ISBN, normalizing to ISBN-13`);
47
+ return {valid: true, value: isbnParseResult.isbn13};
48
+ }
49
+
50
+ const {extract, compare} = createInterface({identifier: IDENTIFIER_NAME, pattern: /^020$/u, subfieldCodes: ['a', 'z'], validIdentifierSubfieldCodes: ['a'], invalidIdentifierSubfieldCodes: ['z'], validatorAndNormalizer});
51
+ return {extract, compare, name: IDENTIFIER_NAME};
34
52
  };
53
+
package/src/match-detection/features/bib/standard-identifier-factory.js CHANGED
@@ -1,3 +1,4 @@
1
+ /* eslint-disable max-statements */
1
2
  /**
2
3
  *
3
4
  * @licstart The following is the entire license notice for the JavaScript code in this file.
@@ -27,60 +28,142 @@
27
28
  */
28
29
 
29
30
  import createDebugLogger from 'debug';
30
- import {testStringOrNumber} from '../../../matching-utils';
31
+ import {extractSubfieldsFromField, uniqueSubfields} from '../../../matching-utils';
31
32
 
32
- const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers');
33
- const debugData = debug.extend('data');
33
+ // Note about validity of standardIdentifiers:
34
+ // We have three types of invalid standardIdentifiers:
35
+ // 1. Formally invalid standardIdentifiers (ie. typos either in the resource or the record)
36
+ // 2. Formally valid standardIdentifiers that are used in a wrong resource
37
+ // 3. Canceled standardIdentifiers
38
+
39
+ // Matcher could and should check that a standardIdentifier found in a subfield for a valid identifier is formally valid, and if it's not formally valid, handle it as an invalid standardIdentifier
40
+ // Formally valid standardIdentifiers found in subfield for invalid identifier cannot be handled as valid standardIdentifiers, because they can be a case of type 2) or 3) invalid standardIdentifiers
41
+ // We could also do a separate handling for formally valid an formally invalid standardIdentifiers
42
+
43
+ export default ({pattern, subfieldCodes, identifier, validIdentifierSubfieldCodes = ['a'], invalidIdentifierSubfieldCodes = ['z'], validatorAndNormalizer = undefined}) => {
44
+ const debug = createDebugLogger(`@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers:${identifier}`);
45
+ const debugData = debug.extend('data');
34
46
 
35
- export default ({pattern, subfieldCodes}) => {
36
47
  return {extract, compare};
37
48
 
38
- function extract({record}) {
39
- const [field] = record.get(pattern);
49
+ function extract({record, recordExternal}) {
50
+ const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
51
+ const fields = record.get(pattern);
52
+ debugData(`${label}: ${fields.length} ${identifier}-fields `);
53
+
54
+ // extractIdentifierSubfield normalizes hyphens away from the subfield values
55
+ const identifiersFromFields = fields.map(field => extractSubfieldsFromField(field, subfieldCodes));
56
+ debugData(`${label}: IDs from fields (${identifiersFromFields.length}): ${JSON.stringify(identifiersFromFields)}`);
57
+ const allIdentifiers = identifiersFromFields.flat();
58
+ debugData(`${label}: Flat IDs from fields (${allIdentifiers.length}): ${JSON.stringify(allIdentifiers)}`);
59
+
60
+ const validatedAndNormalizedIdentifiers = validateAndNormalizeIdentifiers({identifierSubs: allIdentifiers, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes});
40
61
 
41
- if (field) {
42
- return field.subfields
43
- .filter(({code}) => subfieldCodes.includes(code))
44
- .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value).replace(/-/ug, '') : ''}))
45
- .filter(value => value);
62
+ const identifiers = uniqueSubfields(validatedAndNormalizedIdentifiers);
63
+
64
+ debugData(`${label}: Unique IDs from fields (${identifiers.length}): ${JSON.stringify(identifiers)}`);
65
+ return identifiers;
66
+
67
+ function validateAndNormalizeIdentifiers({identifierSubs, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {
68
+ if (validatorAndNormalizer) {
69
+ return identifierSubs.map((idSub) => validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}));
70
+ }
71
+ return identifierSubs.map((idSub) => normalizeHyphens(idSub));
46
72
  }
47
73
 
48
- return [];
74
+ function validateAndNormalizeIdentifier({idSub, validatorAndNormalizer, validIdentifierSubfieldCodes, invalidIdentifierSubfieldCodes}) {
75
+ const {valid, value} = validatorAndNormalizer(idSub.value);
76
+ if (validIdentifierSubfieldCodes.includes(idSub.code) && valid === false) {
77
+ const [code] = invalidIdentifierSubfieldCodes;
78
+ return {code, value};
79
+ }
80
+ return {code: idSub.code, value};
81
+ }
82
+
83
+ function normalizeHyphens(idSub) {
84
+ return {code: idSub.code, value: idSub.value.replace(/-/ug, '')};
85
+ }
86
+
87
+
49
88
  }
50
89
 
51
90
  function compare(a, b) {
91
+ debug(`Comparing A and B`);
52
92
  if (a.length === 0 || b.length === 0) {
53
- debugData(`No standardidentifiers to compare`);
93
+ debugData(`No standardidentifiers (${identifier}) to compare`);
54
94
  return 0;
55
95
  }
56
96
 
97
+ debugData(`A: ${JSON.stringify(a)}`);
98
+ debugData(`B: ${JSON.stringify(b)}`);
99
+
100
+
57
101
  if (bothHaveValidIdentifiers()) {
58
- const {maxValues, matchingValues} = getValueCount(true);
59
- return matchingValues / maxValues * 0.75;
60
- }
102
+ // Compare only valid identifiers, if both have valid idenfiers
103
+ const {maxValues, possibleMatchValues, matchingValues} = getValueCount(true);
104
+ if (matchingValues < 1) {
105
+ debug(`Both have valid standardidentifiers (${identifier}), but none of these match.`);
106
+ return -0.75;
107
+ }
108
+ debug(`Both have valid standardidentifiers (${identifier}), ${matchingValues}/${possibleMatchValues} valid identifiers match.`);
109
+ // ignore non-matches if there is mismatching amount of values
110
+ debug(`Possible matches: ${possibleMatchValues}/${maxValues}`);
111
+ //we give some kind of penalty for mismatching amount of values instead of simple divide?
112
+ const penaltyForMissing = 0.1 * (maxValues - possibleMatchValues);
113
+ const penaltyForMisMatch = 0.2 * (possibleMatchValues - matchingValues);
114
+ debug(`\t points: penaltyForMissing: ${penaltyForMissing}`);
115
+ debug(`\t points: penaltyForMisMatch: ${penaltyForMisMatch}`);
61
116
 
117
+ return 0.75 - penaltyForMisMatch - penaltyForMissing;
118
+ //return matchingValues / possibleMatchValues * 0.75;
119
+ }
120
+ // If both do not have valid identifiers, compare all identifiers
62
121
  const {maxValues, matchingValues} = getValueCount();
122
+ debug(`Both do NOT have valid standardidentifiers (${identifier}), ${matchingValues}/${maxValues} valid/invalid identifiers match.`);
123
+
63
124
  return matchingValues / maxValues * 0.2;
64
125
 
65
126
  function bothHaveValidIdentifiers() {
66
- const aValues = a.filter(({code}) => code === 'a');
67
- const bValues = a.filter(({code}) => code === 'a');
127
+ const aValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));
128
+ const bValues = a.filter(({code}) => validIdentifierSubfieldCodes.includes(code));
129
+ debug(`A: ${aValues.length} valid ${identifier} identifiers`);
130
+ debug(`B: ${bValues.length} valid ${identifier} identifiers`);
68
131
  return aValues.length > 0 && bValues.length > 0;
69
132
  }
70
133
 
71
134
  function getValueCount(validOnly = false) {
72
- const aValues = getIdentifiers(a);
73
- const bValues = getIdentifiers(b);
135
+ const aValues = getIdentifiers(a, validOnly);
136
+ const bValues = getIdentifiers(b, validOnly);
137
+
138
+ const matchingValues = getMatchingValuesAmount(aValues, bValues);
74
139
 
75
140
  return {
76
141
  maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,
77
- matchingValues: aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length
142
+ // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)
143
+ possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,
144
+ matchingValues
78
145
  };
79
146
 
80
- function getIdentifiers(values) {
147
+ function getMatchingValuesAmount(aValues, bValues) {
148
+ if (bValues.length > aValues.length) {
149
+ return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
150
+ }
151
+ if (aValues.length > bValues.length) {
152
+ return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
153
+ }
154
+
155
+ // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases
156
+ // there would be duplicate values
157
+ const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;
158
+ const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;
159
+
160
+ return aToB < bToA ? aToB : bToA;
161
+ }
162
+
163
+ function getIdentifiers(values, validOnly) {
81
164
  if (validOnly) {
82
165
  return values
83
- .filter(({code}) => code === 'a')
166
+ .filter(({code}) => validIdentifierSubfieldCodes.includes(code))
84
167
  .map(({value}) => value);
85
168
  }
86
169
 
package/src/matching-utils.js CHANGED
@@ -92,3 +92,24 @@ export function testStringOrNumber(value) {
92
92
  }
93
93
  return false;
94
94
  }
95
+
96
+ export function extractSubfieldsFromField(field, subfieldCodes) {
97
+ if (field === undefined || field.subfields === undefined) {
98
+ return [];
99
+ }
100
+ const resultSubfields = field.subfields
101
+ .filter(({code}) => subfieldCodes.includes(code))
102
+ .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))
103
+ .filter(value => value);
104
+ return resultSubfields;
105
+ }
106
+
107
+ export function uniqueSubfields(subfields) {
108
+ return subfields.reduce((arr, e) => {
109
+ if (!arr.find(item => item.code === e.code && item.value === e.value)) {
110
+ const newArr = arr.concat(e);
111
+ return newArr;
112
+ }
113
+ return arr;
114
+ }, []);
115
+ }