@natlibfi/melinda-record-matching 4.4.0-alpha.1 → 5.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/melinda-node-tests.yml +13 -9
- package/LICENSE.txt +21 -165
- package/README.md +2 -2
- package/dist/candidate-search/candidate-search-utils.js +6 -43
- package/dist/candidate-search/candidate-search-utils.js.map +7 -1
- package/dist/candidate-search/choose-queries.js +75 -0
- package/dist/candidate-search/choose-queries.js.map +7 -0
- package/dist/candidate-search/index.js +89 -225
- package/dist/candidate-search/index.js.map +7 -1
- package/dist/candidate-search/index.test.js +92 -0
- package/dist/candidate-search/index.test.js.map +7 -0
- package/dist/candidate-search/query-list/bib.js +141 -320
- package/dist/candidate-search/query-list/bib.js.map +7 -1
- package/dist/candidate-search/query-list/bib.test.js +34 -0
- package/dist/candidate-search/query-list/bib.test.js.map +7 -0
- package/dist/candidate-search/query-list/component.js +87 -0
- package/dist/candidate-search/query-list/component.js.map +7 -0
- package/dist/candidate-search/query-list/index.js +39 -52
- package/dist/candidate-search/query-list/index.js.map +7 -1
- package/dist/cli.js +138 -0
- package/dist/cli.js.map +7 -0
- package/dist/index.js +87 -462
- package/dist/index.js.map +7 -1
- package/dist/index.test.js +69 -0
- package/dist/index.test.js.map +7 -0
- package/dist/match-detection/features/bib/all-source-ids.js +20 -91
- package/dist/match-detection/features/bib/all-source-ids.js.map +7 -1
- package/dist/match-detection/features/bib/authors.js +20 -76
- package/dist/match-detection/features/bib/authors.js.map +7 -1
- package/dist/match-detection/features/bib/bibliographic-level.js +4 -40
- package/dist/match-detection/features/bib/bibliographic-level.js.map +7 -1
- package/dist/match-detection/features/bib/host-component.js +5 -41
- package/dist/match-detection/features/bib/host-component.js.map +7 -1
- package/dist/match-detection/features/bib/index.js +17 -119
- package/dist/match-detection/features/bib/index.js.map +7 -1
- package/dist/match-detection/features/bib/index.test.js +40 -0
- package/dist/match-detection/features/bib/index.test.js.map +7 -0
- package/dist/match-detection/features/bib/isbn.js +13 -69
- package/dist/match-detection/features/bib/isbn.js.map +7 -1
- package/dist/match-detection/features/bib/issn.js +5 -50
- package/dist/match-detection/features/bib/issn.js.map +7 -1
- package/dist/match-detection/features/bib/language.js +25 -86
- package/dist/match-detection/features/bib/language.js.map +7 -1
- package/dist/match-detection/features/bib/media-type.js +12 -64
- package/dist/match-detection/features/bib/media-type.js.map +7 -1
- package/dist/match-detection/features/bib/melinda-id.js +5 -47
- package/dist/match-detection/features/bib/melinda-id.js.map +7 -1
- package/dist/match-detection/features/bib/melinda-identifier-factory.js +16 -67
- package/dist/match-detection/features/bib/melinda-identifier-factory.js.map +7 -1
- package/dist/match-detection/features/bib/other-standard-identifier.js +5 -51
- package/dist/match-detection/features/bib/other-standard-identifier.js.map +7 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +49 -189
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +7 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js +10 -53
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map +7 -1
- package/dist/match-detection/features/bib/publication-time.js +8 -46
- package/dist/match-detection/features/bib/publication-time.js.map +7 -1
- package/dist/match-detection/features/bib/record-type.js +4 -43
- package/dist/match-detection/features/bib/record-type.js.map +7 -1
- package/dist/match-detection/features/bib/standard-identifier-factory.js +47 -163
- package/dist/match-detection/features/bib/standard-identifier-factory.js.map +7 -1
- package/dist/match-detection/features/bib/title-version-original.js +11 -53
- package/dist/match-detection/features/bib/title-version-original.js.map +7 -1
- package/dist/match-detection/features/bib/title.js +20 -72
- package/dist/match-detection/features/bib/title.js.map +7 -1
- package/dist/match-detection/features/index.js +3 -11
- package/dist/match-detection/features/index.js.map +7 -1
- package/dist/match-detection/index.js +35 -150
- package/dist/match-detection/index.js.map +7 -1
- package/dist/match-detection/index.test.js +44 -0
- package/dist/match-detection/index.test.js.map +7 -0
- package/dist/matching-utils.js +38 -102
- package/dist/matching-utils.js.map +7 -1
- package/eslint.config.js +52 -0
- package/example.env +4 -0
- package/package.json +29 -78
- package/src/candidate-search/candidate-search-utils.js +1 -28
- package/src/candidate-search/choose-queries.js +93 -0
- package/src/candidate-search/index.js +85 -119
- package/src/candidate-search/{index.spec.js → index.test.js} +30 -43
- package/src/candidate-search/query-list/bib.js +54 -55
- package/src/candidate-search/query-list/bib.test.js +38 -0
- package/src/candidate-search/query-list/component.js +127 -0
- package/src/candidate-search/query-list/index.js +31 -35
- package/src/cli.js +177 -0
- package/src/index.js +27 -68
- package/src/{index.spec.js → index.test.js} +11 -42
- package/src/match-detection/features/bib/all-source-ids.js +2 -29
- package/src/match-detection/features/bib/authors.js +5 -30
- package/src/match-detection/features/bib/bibliographic-level.js +0 -27
- package/src/match-detection/features/bib/host-component.js +0 -27
- package/src/match-detection/features/bib/index.js +16 -43
- package/src/match-detection/features/bib/index.test.js +52 -0
- package/src/match-detection/features/bib/isbn.js +2 -29
- package/src/match-detection/features/bib/issn.js +1 -28
- package/src/match-detection/features/bib/language.js +5 -32
- package/src/match-detection/features/bib/media-type.js +2 -28
- package/src/match-detection/features/bib/melinda-id.js +1 -28
- package/src/match-detection/features/bib/melinda-identifier-factory.js +5 -32
- package/src/match-detection/features/bib/other-standard-identifier.js +1 -28
- package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +7 -34
- package/src/match-detection/features/bib/publication-time-allow-cons-years.js +1 -28
- package/src/match-detection/features/bib/publication-time.js +2 -28
- package/src/match-detection/features/bib/record-type.js +0 -27
- package/src/match-detection/features/bib/standard-identifier-factory.js +1 -28
- package/src/match-detection/features/bib/title-version-original.js +2 -28
- package/src/match-detection/features/bib/title.js +6 -34
- package/src/match-detection/features/index.js +1 -28
- package/src/match-detection/index.js +1 -28
- package/src/match-detection/{index.spec.js → index.test.js} +9 -35
- package/src/matching-utils.js +8 -37
- package/dist/candidate-search/index.spec.js +0 -144
- package/dist/candidate-search/index.spec.js.map +0 -1
- package/dist/candidate-search/query-list/bib.spec.js +0 -64
- package/dist/candidate-search/query-list/bib.spec.js.map +0 -1
- package/dist/index.spec.js +0 -127
- package/dist/index.spec.js.map +0 -1
- package/dist/match-detection/features/bib/index.spec.js +0 -93
- package/dist/match-detection/features/bib/index.spec.js.map +0 -1
- package/dist/match-detection/index.spec.js +0 -91
- package/dist/match-detection/index.spec.js.map +0 -1
- package/src/candidate-search/query-list/bib.spec.js +0 -54
- package/src/match-detection/features/bib/index.spec.js +0 -78
|
@@ -1,101 +1,34 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
exports.bibSourceIds = bibSourceIds;
|
|
11
|
-
exports.bibStandardIdentifiers = bibStandardIdentifiers;
|
|
12
|
-
exports.bibTitle = bibTitle;
|
|
13
|
-
exports.bibTitleAuthor = bibTitleAuthor;
|
|
14
|
-
exports.bibTitleAuthorPublisher = bibTitleAuthorPublisher;
|
|
15
|
-
exports.bibTitleAuthorYear = bibTitleAuthorYear;
|
|
16
|
-
exports.bibYear = bibYear;
|
|
17
|
-
var _debug = _interopRequireDefault(require("debug"));
|
|
18
|
-
var _candidateSearchUtils = require("../candidate-search-utils");
|
|
19
|
-
var _matchingUtils = require("../../matching-utils");
|
|
20
|
-
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
21
|
-
/**
|
|
22
|
-
*
|
|
23
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
24
|
-
*
|
|
25
|
-
* Melinda record matching modules for Javascript
|
|
26
|
-
*
|
|
27
|
-
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
28
|
-
*
|
|
29
|
-
* This file is part of melinda-record-matching-js
|
|
30
|
-
*
|
|
31
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
32
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
33
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
34
|
-
* License, or (at your option) any later version.
|
|
35
|
-
*
|
|
36
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
37
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
38
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
39
|
-
* GNU Lesser General Public License for more details.
|
|
40
|
-
*
|
|
41
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
42
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
43
|
-
*
|
|
44
|
-
* @licend The above is the entire license notice
|
|
45
|
-
* for the JavaScript code in this file.
|
|
46
|
-
*
|
|
47
|
-
*/
|
|
48
|
-
|
|
49
|
-
const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:candidate-search:query');
|
|
50
|
-
function bibSourceIds(record) {
|
|
51
|
-
/* Melinda's SRU-index melinda.sourceid includes source IDs from SID fields in Melinda records
|
|
52
|
-
SID-fields in Melinda have sf $c with local id and sf $b with a code for the local db:
|
|
53
|
-
SID__ $c 123457 $b helka
|
|
54
|
-
SID__ $c (ANDL100020)1077305 $b sata
|
|
55
|
-
SID__ $c VER999999 $ FI-KV
|
|
56
|
-
SID__ $c /10024/508126 $ REPO_THESEUS
|
|
57
|
-
In melinda.sourceid -index case is kept, sourceprefixes in brackets and hyphens are normalized away:
|
|
58
|
-
Note: slashes are not normalized away, but a SRU-search-string including slashes needs to be quoted
|
|
59
|
-
1234567helka
|
|
60
|
-
1077305sata
|
|
61
|
-
VER999999FIKV
|
|
62
|
-
/10024/508126REPO_THESEUS
|
|
63
|
-
Note: All Melinda records that have a matching records in a local db do NOT have SID for that local records,
|
|
64
|
-
existence of a SID field depends on how the record has been added to Melinda and how it has been handled
|
|
65
|
-
afterwards. SIDs are also not reliably maintained. A record might or might not have a SID for a local db
|
|
66
|
-
after the matching record is removed from the local db.
|
|
67
|
-
*/
|
|
68
|
-
|
|
69
|
-
const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:candidate-search:query:source-ids');
|
|
70
|
-
const debugData = debug.extend('data');
|
|
71
|
-
//const debugInfo = debug.extend('info');
|
|
72
|
-
|
|
73
|
-
debug(`Creating queries for sourceid's`);
|
|
74
|
-
const fSids = record.get('SID');
|
|
1
|
+
import createDebugLogger from "debug";
|
|
2
|
+
import { toQueries } from "../candidate-search-utils.js";
|
|
3
|
+
import { getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues, testStringOrNumber } from "../../matching-utils.js";
|
|
4
|
+
const debug = createDebugLogger("@natlibfi/melinda-record-matching:candidate-search:query");
|
|
5
|
+
export function bibSourceIds(record) {
|
|
6
|
+
const debug2 = createDebugLogger("@natlibfi/melinda-record-matching:candidate-search:query:source-ids");
|
|
7
|
+
const debugData = debug2.extend("data");
|
|
8
|
+
debug2(`Creating queries for sourceid's`);
|
|
9
|
+
const fSids = record.get("SID");
|
|
75
10
|
debugData(`SID-fields (${fSids.length}): ${JSON.stringify(fSids)}`);
|
|
76
11
|
return fSids.length > 0 ? toSidQueries(fSids) : [];
|
|
77
|
-
function toSidQueries(
|
|
78
|
-
|
|
79
|
-
const sidStrings = getSidStrings(
|
|
12
|
+
function toSidQueries(fSids2) {
|
|
13
|
+
debug2(`Creating actual queries for sourceid's`);
|
|
14
|
+
const sidStrings = getSidStrings(fSids2);
|
|
80
15
|
if (sidStrings.length < 1) {
|
|
81
|
-
|
|
16
|
+
debug2(`No identifiers found, no queries created.`);
|
|
82
17
|
return [];
|
|
83
18
|
}
|
|
84
|
-
const sidQueries =
|
|
19
|
+
const sidQueries = toQueries(sidStrings, "melinda.sourceid");
|
|
85
20
|
return sidQueries;
|
|
86
|
-
function getSidStrings(
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
const sidStrings = fSids.map(toSidString).filter(nonEmptySid => nonEmptySid);
|
|
91
|
-
return sidStrings;
|
|
21
|
+
function getSidStrings(fSids3) {
|
|
22
|
+
debug2(`Getting Sid strings from SID fields`);
|
|
23
|
+
const sidStrings2 = fSids3.map(toSidString).filter((nonEmptySid) => nonEmptySid);
|
|
24
|
+
return sidStrings2;
|
|
92
25
|
function toSidString(field) {
|
|
93
|
-
|
|
94
|
-
return
|
|
95
|
-
function createSidString(
|
|
96
|
-
|
|
97
|
-
const [sfC] =
|
|
98
|
-
const [sfB] =
|
|
26
|
+
debug2(`Getting string from a field`);
|
|
27
|
+
return validateSidFieldSubfieldCounts(field) ? createSidString(field) : "";
|
|
28
|
+
function createSidString(field2) {
|
|
29
|
+
debug2(`Creating string from a field`);
|
|
30
|
+
const [sfC] = getSubfieldValues(field2, "c");
|
|
31
|
+
const [sfB] = getSubfieldValues(field2, "b");
|
|
99
32
|
const cleanedSfC = removeSourcePrefix(normalizeSidSubfieldValue(sfC));
|
|
100
33
|
const cleanedSfB = normalizeSidSubfieldValue(sfB);
|
|
101
34
|
debugData(`${JSON.stringify(sfC)} + ${JSON.stringify(sfB)}`);
|
|
@@ -103,307 +36,204 @@ function bibSourceIds(record) {
|
|
|
103
36
|
}
|
|
104
37
|
function removeSourcePrefix(subfieldValue) {
|
|
105
38
|
const sourcePrefixRegex = /^(?<sourcePrefix>\([A-Za-z0-9-]+\))(?<id>.+)$/u;
|
|
106
|
-
const normalizedValue =
|
|
39
|
+
const normalizedValue = testStringOrNumber(subfieldValue) ? String(subfieldValue).replace(sourcePrefixRegex, "$<id>") : "";
|
|
107
40
|
debugData(`Normalized ${subfieldValue} to ${normalizedValue}`);
|
|
108
41
|
return normalizedValue;
|
|
109
42
|
}
|
|
110
43
|
function normalizeSidSubfieldValue(subfieldValue) {
|
|
111
44
|
debugData(`Normalizing ${subfieldValue}`);
|
|
112
45
|
const normalizeAwayRegex = /[- ]/u;
|
|
113
|
-
return
|
|
46
|
+
return testStringOrNumber(subfieldValue) ? String(subfieldValue).replace(normalizeAwayRegex, "") : "";
|
|
114
47
|
}
|
|
115
48
|
}
|
|
116
49
|
}
|
|
117
50
|
}
|
|
118
51
|
}
|
|
119
|
-
function bibMelindaIds(record) {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
const
|
|
124
|
-
debug(`Creating queries for MelindaIds`);
|
|
125
|
-
|
|
126
|
-
// Note: Melinda-ID's for search queries are created just from records f035a's and f035z's
|
|
127
|
-
// Both (FI-MELINDA)- and FCC-prefixed forms are found
|
|
128
|
-
// f001 controlnumber is not currently included, even if record's f003 is FI-MELINDA
|
|
129
|
-
const melindaIds = (0, _matchingUtils.getMelindaIdsF035)(record);
|
|
52
|
+
export function bibMelindaIds(record) {
|
|
53
|
+
const debug2 = createDebugLogger("@natlibfi/melinda-record-matching:candidate-search:query:bibMelindaIds");
|
|
54
|
+
const debugData = debug2.extend("data");
|
|
55
|
+
debug2(`Creating queries for MelindaIds`);
|
|
56
|
+
const melindaIds = getMelindaIdsF035(record);
|
|
130
57
|
debugData(`Unique identifiers (${melindaIds.length}): ${JSON.stringify(melindaIds)}`);
|
|
131
58
|
if (melindaIds.length < 1) {
|
|
132
|
-
|
|
59
|
+
debug2(`No identifiers found, no queries created.`);
|
|
133
60
|
return [];
|
|
134
61
|
}
|
|
135
|
-
return
|
|
62
|
+
return toQueries(melindaIds, "melinda.melindaid");
|
|
136
63
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
64
|
+
export function bibHostComponents(record) {
|
|
65
|
+
const debug2 = createDebugLogger("@natlibfi/melinda-record-matching:candidate-search:query:bibHostComponents");
|
|
66
|
+
const debugData = debug2.extend("data");
|
|
67
|
+
debug2(`Creating queries for hostIds`);
|
|
141
68
|
const id = getHostId();
|
|
142
|
-
|
|
69
|
+
debugData(`Found id: ${JSON.stringify(id)}`);
|
|
70
|
+
return testStringOrNumber(id) ? [`melinda.partsofhost=${id}`] : [];
|
|
143
71
|
function getHostId() {
|
|
144
72
|
const [field] = record.get(/^773$/u);
|
|
145
73
|
if (field) {
|
|
146
|
-
const {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
code
|
|
150
|
-
}) => code === 'w') || {};
|
|
151
|
-
if ((0, _matchingUtils.testStringOrNumber)(value) && /^\(FI-MELINDA\)/u.test(String(value))) {
|
|
152
|
-
return String(value).replace(/^\(FI-MELINDA\)/u, '');
|
|
74
|
+
const { value } = field.subfields.find(({ code }) => code === "w") || {};
|
|
75
|
+
if (testStringOrNumber(value) && /^\(FI-MELINDA\)/u.test(String(value))) {
|
|
76
|
+
return String(value).replace(/^\(FI-MELINDA\)/u, "");
|
|
153
77
|
}
|
|
154
|
-
if (
|
|
155
|
-
return String(value).replace(/^\(FIN01\)/u,
|
|
78
|
+
if (testStringOrNumber(value) && /^\(FIN01\)/u.test(String(value))) {
|
|
79
|
+
return String(value).replace(/^\(FIN01\)/u, "");
|
|
156
80
|
}
|
|
157
81
|
return false;
|
|
158
82
|
}
|
|
159
83
|
return false;
|
|
160
84
|
}
|
|
161
85
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
// (probably) to *headings* index TIT
|
|
165
|
-
// - Aleph cannot currently handle headings searches starting with a boolean - in these cases use word search
|
|
166
|
-
|
|
167
|
-
// Headings index TIT drops articles etc. from the start of the title according to the filing indicator
|
|
168
|
-
// Currently filing indicator is not implemented - if the title starts with an article and the Melinda
|
|
169
|
-
// record is correctly catalogued using a filing indicator -> dc.title search won't match
|
|
170
|
-
|
|
171
|
-
function bibTitle(record) {
|
|
172
|
-
// We get author/publisher only when formatted title is shorter than 5 chars
|
|
173
|
-
return bibTitleAuthorPublisher({
|
|
174
|
-
record,
|
|
175
|
-
onlyTitleLength: 5
|
|
176
|
-
});
|
|
86
|
+
export function bibTitle(record) {
|
|
87
|
+
return bibTitleAuthorPublisher({ record, onlyTitleLength: 5 });
|
|
177
88
|
}
|
|
178
|
-
function bibTitleAuthor(record) {
|
|
179
|
-
debug(
|
|
180
|
-
|
|
181
|
-
// get an author or an publisher always
|
|
182
|
-
return bibTitleAuthorPublisher({
|
|
183
|
-
record,
|
|
184
|
-
onlyTitleLength: 100
|
|
185
|
-
});
|
|
89
|
+
export function bibTitleAuthor(record) {
|
|
90
|
+
debug("bibTitleAuthor");
|
|
91
|
+
return bibTitleAuthorPublisher({ record, onlyTitleLength: 100 });
|
|
186
92
|
}
|
|
187
|
-
function bibTitleAuthorYear(record) {
|
|
188
|
-
debug(
|
|
189
|
-
|
|
190
|
-
// get an author or an publisher always
|
|
191
|
-
|
|
192
|
-
return bibTitleAuthorPublisher({
|
|
193
|
-
record,
|
|
194
|
-
onlyTitleLength: 100,
|
|
195
|
-
addYear: true
|
|
196
|
-
});
|
|
93
|
+
export function bibTitleAuthorYear(record) {
|
|
94
|
+
debug("bibTitleAuthorYearAlternates");
|
|
95
|
+
return bibTitleAuthorPublisher({ record, onlyTitleLength: 100, addYear: true });
|
|
197
96
|
}
|
|
198
|
-
function
|
|
199
|
-
|
|
200
|
-
onlyTitleLength,
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
97
|
+
export function bibTitleAuthorYearAlternates(record) {
|
|
98
|
+
debug("bibTitleAuthorYearAlternates");
|
|
99
|
+
const origQueryList = bibTitleAuthorPublisher({ record, onlyTitleLength: 100, addYear: true, alternates: true, alternateQueries: [] });
|
|
100
|
+
debug(`${JSON.stringify(origQueryList)}`);
|
|
101
|
+
return { queryList: Array.from(origQueryList).reverse(), queryListType: "alternates" };
|
|
102
|
+
}
|
|
103
|
+
export function bibTitleAuthorPublisher({ record, onlyTitleLength, addYear = false, alternates = false, alternateQueries = [] }) {
|
|
104
|
+
debug(`bibTitleAuthorPublisher, onlyTitleLength: ${onlyTitleLength}, addYear: ${addYear}, alternates: ${alternates}`);
|
|
204
105
|
const title = getTitle();
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
const
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
// use word search for titles starting with a boolean
|
|
212
|
-
const useWordSearch = booleanStartWords.some(word => formatted.toLowerCase().startsWith(word));
|
|
213
|
-
// Prevent too many matches / SRU crashing by having a minimum length
|
|
214
|
-
// Note that currently this fails matching if there are no matches from previous matchers
|
|
215
|
-
if (formatted.length >= onlyTitleLength) {
|
|
216
|
-
return [`dc.title="${useWordSearch ? '' : '^'}${formatted}*"`];
|
|
106
|
+
if (testStringOrNumber(title)) {
|
|
107
|
+
const formatted = String(title).replace(/[^\w\s\p{Alphabetic}]/gu, "").replace(/ +/gu, " ").trim().slice(0, 30).trim();
|
|
108
|
+
const useWordSearch = checkUseWordSearch(formatted);
|
|
109
|
+
if (formatted.length >= onlyTitleLength && !alternates) {
|
|
110
|
+
return [`dc.title="${useWordSearch ? "" : "^"}${formatted}*"`];
|
|
217
111
|
}
|
|
218
112
|
const queryIsOkAlone = formatted.length >= 5;
|
|
219
|
-
|
|
220
|
-
// use word search without ending * also in combination searches to avoid SRU-server crashes [MRA-189]
|
|
221
|
-
const query = `dc.title="${useWordSearch || !queryIsOkAlone ? '' : '^'}${formatted}${queryIsOkAlone ? '*' : ''}"`;
|
|
113
|
+
const query = `dc.title="${useWordSearch || !queryIsOkAlone ? "" : "^"}${formatted}${queryIsOkAlone ? "*" : ""}"`;
|
|
222
114
|
debug(`query: ${query}`);
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
queryIsOkAlone,
|
|
226
|
-
addYear
|
|
227
|
-
});
|
|
115
|
+
const newAlternateQueries = alternates ? [...alternateQueries, query] : alternateQueries;
|
|
116
|
+
return addAuthorsToSearch({ query, queryIsOkAlone, addYear, alternates, alternateQueries: newAlternateQueries });
|
|
228
117
|
}
|
|
229
118
|
return [];
|
|
230
|
-
function addAuthorsToSearch({
|
|
231
|
-
|
|
232
|
-
queryIsOkAlone = false,
|
|
233
|
-
addYear = false
|
|
234
|
-
}) {
|
|
235
|
-
debug('addAuthorsToSearch');
|
|
119
|
+
function addAuthorsToSearch({ query, queryIsOkAlone = false, addYear: addYear2 = false, alternates: alternates2 = false, alternateQueries: alternateQueries2 = [] }) {
|
|
120
|
+
debug("addAuthorsToSearch");
|
|
236
121
|
const [authorQuery] = bibAuthors(record);
|
|
237
|
-
if (authorQuery !==
|
|
238
|
-
if (
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
queryIsOkAlone: true
|
|
242
|
-
});
|
|
122
|
+
if (authorQuery !== void 0) {
|
|
123
|
+
if (addYear2) {
|
|
124
|
+
const newAlternateQueries = alternates2 ? [...alternateQueries2, `${authorQuery} AND ${query}`] : alternateQueries2;
|
|
125
|
+
return addYearToSearch({ query: `${authorQuery} AND ${query}`, queryIsOkAlone: true, alternates: alternates2, alternateQueries: newAlternateQueries });
|
|
243
126
|
}
|
|
244
|
-
return [`${authorQuery} AND ${query}`];
|
|
127
|
+
return alternates2 ? alternateQueries2 : [`${authorQuery} AND ${query}`];
|
|
245
128
|
}
|
|
246
|
-
return addPublisherToSearch({
|
|
247
|
-
query,
|
|
248
|
-
queryIsOkAlone,
|
|
249
|
-
addYear
|
|
250
|
-
});
|
|
251
|
-
//return [];
|
|
129
|
+
return addPublisherToSearch({ query, queryIsOkAlone, addYear: addYear2, alternates: alternates2, alternateQueries: alternateQueries2 });
|
|
252
130
|
}
|
|
253
|
-
|
|
254
|
-
function addPublisherToSearch({
|
|
255
|
-
query,
|
|
256
|
-
queryIsOkAlone = false,
|
|
257
|
-
addYear = false
|
|
258
|
-
}) {
|
|
131
|
+
function addPublisherToSearch({ query, queryIsOkAlone = false, addYear: addYear2 = false, alternates: alternates2 = false, alternateQueries: alternateQueries2 = [] }) {
|
|
259
132
|
const [publisherQuery] = bibPublishers(record);
|
|
260
|
-
if (publisherQuery !==
|
|
261
|
-
if (
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
queryIsOkAlone: true
|
|
265
|
-
});
|
|
133
|
+
if (publisherQuery !== void 0) {
|
|
134
|
+
if (addYear2) {
|
|
135
|
+
const newAlternateQueries = alternates2 ? [...alternateQueries2, `${publisherQuery} AND ${query}`] : alternateQueries2;
|
|
136
|
+
return addYearToSearch({ query: `${publisherQuery} AND ${query}`, queryIsOkAlone: true, alternates: alternates2, alternateQueries: newAlternateQueries });
|
|
266
137
|
}
|
|
267
|
-
return [`${publisherQuery} AND ${query}`];
|
|
138
|
+
return alternates2 ? alternateQueries2 : [`${publisherQuery} AND ${query}`];
|
|
268
139
|
}
|
|
269
|
-
if (queryIsOkAlone && !
|
|
270
|
-
return [`${query}`];
|
|
140
|
+
if (queryIsOkAlone && !addYear2) {
|
|
141
|
+
return alternates2 ? alternateQueries2 : [`${query}`];
|
|
271
142
|
}
|
|
272
|
-
return addYearToSearch({
|
|
273
|
-
query,
|
|
274
|
-
queryIsOkAlone
|
|
275
|
-
});
|
|
143
|
+
return addYearToSearch({ query, queryIsOkAlone, alternates: alternates2, alternateQueries: alternateQueries2 });
|
|
276
144
|
}
|
|
277
|
-
function addYearToSearch({
|
|
278
|
-
query,
|
|
279
|
-
queryIsOkAlone = false
|
|
280
|
-
}) {
|
|
145
|
+
function addYearToSearch({ query, queryIsOkAlone = false, alternates: alternates2 = false, alternateQueries: alternateQueries2 = [] }) {
|
|
281
146
|
const [yearQuery] = bibYear(record);
|
|
282
|
-
if (yearQuery !==
|
|
283
|
-
|
|
147
|
+
if (yearQuery !== void 0) {
|
|
148
|
+
const newAlternateQueries = alternates2 ? [...alternateQueries2, `${yearQuery} AND ${query}`] : alternateQueries2;
|
|
149
|
+
return alternates2 ? newAlternateQueries : [`${yearQuery} AND ${query}`];
|
|
284
150
|
}
|
|
285
151
|
if (queryIsOkAlone) {
|
|
286
|
-
return [`${query}`];
|
|
152
|
+
return alternates2 ? alternateQueries2 : [`${query}`];
|
|
287
153
|
}
|
|
288
154
|
return [];
|
|
289
155
|
}
|
|
290
156
|
function getTitle() {
|
|
291
157
|
const [field] = record.get(/^245$/u);
|
|
292
158
|
if (field) {
|
|
293
|
-
const titleString = field.subfields
|
|
294
|
-
//.filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))
|
|
295
|
-
.filter(({
|
|
296
|
-
code
|
|
297
|
-
}) => ['a', 'b'].includes(code))
|
|
298
|
-
//.filter(({code}) => ['a'].includes(code))
|
|
299
|
-
.map(({
|
|
300
|
-
value
|
|
301
|
-
}) => (0, _matchingUtils.testStringOrNumber)(value) ? String(value) : '').filter(value => value)
|
|
302
|
-
// In Melinda's index subfield separators are indexed as ' '
|
|
303
|
-
.join(' ');
|
|
159
|
+
const titleString = field.subfields.filter(({ code }) => ["a", "b"].includes(code)).map(({ value }) => testStringOrNumber(value) ? String(value) : "").filter((value) => value).join(" ");
|
|
304
160
|
return titleString;
|
|
305
161
|
}
|
|
306
162
|
return false;
|
|
307
163
|
}
|
|
308
164
|
}
|
|
309
|
-
function bibAuthors(record) {
|
|
310
|
-
const
|
|
311
|
-
const debugData =
|
|
312
|
-
|
|
313
|
-
//debugData(record);
|
|
314
|
-
|
|
165
|
+
export function bibAuthors(record) {
|
|
166
|
+
const debug2 = createDebugLogger("@natlibfi/melinda-record-matching:candidate-search:query:bibAuthors");
|
|
167
|
+
const debugData = debug2.extend("data");
|
|
168
|
+
debug2(`Creating query for the first author`);
|
|
315
169
|
const author = getAuthor(record);
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
const
|
|
319
|
-
// Clean up concurrent spaces from fe. subfield changes
|
|
320
|
-
.replace(/ +/gu, ' ').trim().slice(0, 30).trim();
|
|
321
|
-
|
|
322
|
-
// use word search for authors starting with a boolean
|
|
323
|
-
const useWordSearch = booleanStartWords.some(word => formatted.toLowerCase().startsWith(word));
|
|
324
|
-
// Prevent too many matches by having a minimum length
|
|
170
|
+
if (testStringOrNumber(author)) {
|
|
171
|
+
const formatted = String(author).replace(/[^\w\s\p{Alphabetic}]/gu, "").replace(/ +/gu, " ").trim().slice(0, 30).trim();
|
|
172
|
+
const useWordSearch = checkUseWordSearch(formatted);
|
|
325
173
|
debugData(`Author string: ${formatted}`);
|
|
326
174
|
if (formatted.length >= 5) {
|
|
327
|
-
return [`dc.author="${useWordSearch ?
|
|
175
|
+
return [`dc.author="${useWordSearch ? "" : "^"}${formatted}*"`];
|
|
328
176
|
}
|
|
329
177
|
return [];
|
|
330
178
|
}
|
|
331
179
|
return [];
|
|
332
|
-
function getAuthor(
|
|
333
|
-
|
|
334
|
-
// eslint-disable-next-line prefer-named-capture-group
|
|
335
|
-
const [field] = record.get(/^(100)|(110)|(111)|(700)|(710)|(711)$/u);
|
|
336
|
-
//debugData(field);
|
|
337
|
-
|
|
180
|
+
function getAuthor(record2) {
|
|
181
|
+
const [field] = record2.get(/^(100)|(110)|(111)|(700)|(710)|(711)$/u);
|
|
338
182
|
if (field) {
|
|
339
|
-
const authorString = field.subfields.filter(({
|
|
340
|
-
code
|
|
341
|
-
}) => ['a'].includes(code)).map(({
|
|
342
|
-
value
|
|
343
|
-
}) => (0, _matchingUtils.testStringOrNumber)(value) ? String(value) : '').filter(value => value)
|
|
344
|
-
// In Melinda's index subfield separators are indexed as ' '
|
|
345
|
-
.join(' ');
|
|
183
|
+
const authorString = field.subfields.filter(({ code }) => ["a"].includes(code)).map(({ value }) => testStringOrNumber(value) ? String(value) : "").filter((value) => value).join(" ");
|
|
346
184
|
return authorString;
|
|
347
185
|
}
|
|
348
186
|
return false;
|
|
349
187
|
}
|
|
350
188
|
}
|
|
351
|
-
function bibPublishers(record) {
|
|
352
|
-
const
|
|
353
|
-
const debugData =
|
|
354
|
-
|
|
355
|
-
//debugData(record);
|
|
356
|
-
|
|
189
|
+
export function bibPublishers(record) {
|
|
190
|
+
const debug2 = createDebugLogger("@natlibfi/melinda-record-matching:candidate-search:query:bibPublishers");
|
|
191
|
+
const debugData = debug2.extend("data");
|
|
192
|
+
debug2(`Creating query for the first publisher`);
|
|
357
193
|
const publisher = getPublisher(record);
|
|
358
|
-
if (
|
|
359
|
-
const formatted = String(publisher).replace(/[^\w\s\p{Alphabetic}]/gu,
|
|
360
|
-
// Clean up concurrent spaces from fe. subfield changes
|
|
361
|
-
.replace(/ +/gu, ' ').trim().slice(0, 30).trim();
|
|
194
|
+
if (testStringOrNumber(publisher)) {
|
|
195
|
+
const formatted = String(publisher).replace(/[^\w\s\p{Alphabetic}]/gu, "").replace(/ +/gu, " ").trim().slice(0, 30).trim();
|
|
362
196
|
debugData(`Publisher string: ${formatted}`);
|
|
363
|
-
// use non-wildcard word search from dc.publisher
|
|
364
197
|
return [`dc.publisher="${formatted}"`];
|
|
365
198
|
}
|
|
366
199
|
return [];
|
|
367
|
-
function getPublisher(
|
|
368
|
-
|
|
369
|
-
const [field] = record.get(/^(?:260)|(?:264)$/u);
|
|
370
|
-
//debugData(field);
|
|
371
|
-
|
|
200
|
+
function getPublisher(record2) {
|
|
201
|
+
const [field] = record2.get(/^(?:260)|(?:264)$/u);
|
|
372
202
|
if (field) {
|
|
373
|
-
const publisherString = field.subfields.filter(({
|
|
374
|
-
code
|
|
375
|
-
}) => ['b'].includes(code)).map(({
|
|
376
|
-
value
|
|
377
|
-
}) => (0, _matchingUtils.testStringOrNumber)(value) ? String(value) : '').filter(value => value)
|
|
378
|
-
// In Melinda's index subfield separators are indexed as ' '
|
|
379
|
-
.join(' ');
|
|
203
|
+
const publisherString = field.subfields.filter(({ code }) => ["b"].includes(code)).map(({ value }) => testStringOrNumber(value) ? String(value) : "").filter((value) => value).join(" ");
|
|
380
204
|
return publisherString;
|
|
381
205
|
}
|
|
382
206
|
return false;
|
|
383
207
|
}
|
|
384
208
|
}
|
|
385
|
-
function bibYear(record) {
|
|
386
|
-
const
|
|
387
|
-
const debugData =
|
|
388
|
-
|
|
389
|
-
debugData(record);
|
|
209
|
+
export function bibYear(record) {
|
|
210
|
+
const debug2 = createDebugLogger("@natlibfi/melinda-record-matching:candidate-search:query:bibYear");
|
|
211
|
+
const debugData = debug2.extend("data");
|
|
212
|
+
debug2(`Creating query for the publishing year`);
|
|
390
213
|
const year = getYear(record);
|
|
391
214
|
if (year) {
|
|
392
215
|
return [`dc.date="${year}"`];
|
|
393
216
|
}
|
|
394
217
|
return [];
|
|
395
|
-
function getYear(
|
|
396
|
-
const
|
|
397
|
-
|
|
218
|
+
function getYear(record2) {
|
|
219
|
+
const [f008] = record2.get(/^008$/u);
|
|
220
|
+
if (f008 === void 0) {
|
|
221
|
+
debug2("f008 missing");
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
224
|
+
debugData(`f008: ${JSON.stringify(f008)}`);
|
|
225
|
+
const { value } = f008;
|
|
226
|
+
return testStringOrNumber(value) ? String(value).slice(7, 11) : void 0;
|
|
398
227
|
}
|
|
399
228
|
}
|
|
400
|
-
function
|
|
401
|
-
const
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
229
|
+
export function checkUseWordSearch(formatted) {
|
|
230
|
+
const booleanStartWords = ["and ", "or ", "nor ", "not "];
|
|
231
|
+
return booleanStartWords.some((word) => formatted.toLowerCase().startsWith(word));
|
|
232
|
+
}
|
|
233
|
+
export function bibStandardIdentifiers(record) {
|
|
234
|
+
const debug2 = createDebugLogger("@natlibfi/melinda-record-matching:candidate-search:query:bibStandardIdentifiers");
|
|
235
|
+
const debugData = debug2.extend("data");
|
|
236
|
+
debug2(`Creating queries for standard identifiers`);
|
|
407
237
|
const fields = record.get(/^(?<def>020|022|024)$/u);
|
|
408
238
|
const identifiers = [].concat(...fields.map(toIdentifiers));
|
|
409
239
|
const uniqueIdentifiers = [...new Set(identifiers)];
|
|
@@ -411,29 +241,20 @@ function bibStandardIdentifiers(record) {
|
|
|
411
241
|
debugData(`Identifiers (${identifiers.length}): ${JSON.stringify(identifiers)}`);
|
|
412
242
|
debugData(`Unique identifiers (${uniqueIdentifiers.length}): ${JSON.stringify(uniqueIdentifiers)}`);
|
|
413
243
|
if (uniqueIdentifiers.length < 1) {
|
|
414
|
-
|
|
244
|
+
debug2(`No identifiers found, no queries created.`);
|
|
415
245
|
return [];
|
|
416
246
|
}
|
|
417
|
-
return
|
|
418
|
-
function toIdentifiers({
|
|
419
|
-
tag,
|
|
420
|
-
subfields
|
|
421
|
-
}) {
|
|
247
|
+
return toQueries(uniqueIdentifiers, "dc.identifier");
|
|
248
|
+
function toIdentifiers({ tag, subfields }) {
|
|
422
249
|
const issnIsbnReqExp = /^[A-Za-z0-9-]+$/u;
|
|
423
250
|
const otherIdReqExp = /^[A-Za-z0-9-:]+$/u;
|
|
424
|
-
if (tag ===
|
|
425
|
-
return subfields.filter(sub => [
|
|
426
|
-
value
|
|
427
|
-
}) => String(value));
|
|
251
|
+
if (tag === "022") {
|
|
252
|
+
return subfields.filter((sub) => ["a", "z", "y"].includes(sub.code) && testStringOrNumber(sub.value) && issnIsbnReqExp.test(String(sub.value))).map(({ value }) => String(value));
|
|
428
253
|
}
|
|
429
|
-
if (tag ===
|
|
430
|
-
return subfields.filter(sub => [
|
|
431
|
-
value
|
|
432
|
-
}) => String(value));
|
|
254
|
+
if (tag === "020") {
|
|
255
|
+
return subfields.filter((sub) => ["a", "z"].includes(sub.code) && testStringOrNumber(sub.value) && issnIsbnReqExp.test(String(sub.value))).map(({ value }) => String(value));
|
|
433
256
|
}
|
|
434
|
-
return subfields.filter(sub => [
|
|
435
|
-
value
|
|
436
|
-
}) => String(value));
|
|
257
|
+
return subfields.filter((sub) => ["a", "z"].includes(sub.code) && testStringOrNumber(sub.value) && otherIdReqExp.test(String(sub.value))).map(({ value }) => String(value));
|
|
437
258
|
}
|
|
438
259
|
}
|
|
439
|
-
//# sourceMappingURL=bib.js.map
|
|
260
|
+
//# sourceMappingURL=bib.js.map
|