@natlibfi/melinda-record-matching 4.4.0-alpha.1 → 5.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/melinda-node-tests.yml +13 -9
- package/LICENSE.txt +21 -165
- package/README.md +2 -2
- package/eslint.config.js +52 -0
- package/example.env +4 -0
- package/package.json +29 -79
- package/src/candidate-search/candidate-search-utils.js +1 -28
- package/src/candidate-search/choose-queries.js +93 -0
- package/src/candidate-search/index.js +85 -119
- package/src/candidate-search/{index.spec.js → index.test.js} +30 -43
- package/src/candidate-search/query-list/bib.js +54 -55
- package/src/candidate-search/query-list/bib.test.js +38 -0
- package/src/candidate-search/query-list/component.js +127 -0
- package/src/candidate-search/query-list/index.js +31 -35
- package/src/cli.js +177 -0
- package/src/index.js +27 -68
- package/src/{index.spec.js → index.test.js} +11 -42
- package/src/match-detection/features/bib/all-source-ids.js +2 -29
- package/src/match-detection/features/bib/authors.js +5 -30
- package/src/match-detection/features/bib/bibliographic-level.js +0 -27
- package/src/match-detection/features/bib/host-component.js +0 -27
- package/src/match-detection/features/bib/index.js +16 -43
- package/src/match-detection/features/bib/index.test.js +52 -0
- package/src/match-detection/features/bib/isbn.js +2 -29
- package/src/match-detection/features/bib/issn.js +1 -28
- package/src/match-detection/features/bib/language.js +5 -32
- package/src/match-detection/features/bib/media-type.js +2 -28
- package/src/match-detection/features/bib/melinda-id.js +1 -28
- package/src/match-detection/features/bib/melinda-identifier-factory.js +5 -32
- package/src/match-detection/features/bib/other-standard-identifier.js +1 -28
- package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +7 -34
- package/src/match-detection/features/bib/publication-time-allow-cons-years.js +1 -28
- package/src/match-detection/features/bib/publication-time.js +2 -28
- package/src/match-detection/features/bib/record-type.js +0 -27
- package/src/match-detection/features/bib/standard-identifier-factory.js +1 -28
- package/src/match-detection/features/bib/title-version-original.js +2 -28
- package/src/match-detection/features/bib/title.js +6 -34
- package/src/match-detection/features/index.js +1 -28
- package/src/match-detection/index.js +1 -28
- package/src/match-detection/{index.spec.js → index.test.js} +9 -35
- package/src/matching-utils.js +8 -37
- package/dist/candidate-search/candidate-search-utils.js +0 -55
- package/dist/candidate-search/candidate-search-utils.js.map +0 -1
- package/dist/candidate-search/index.js +0 -254
- package/dist/candidate-search/index.js.map +0 -1
- package/dist/candidate-search/index.spec.js +0 -144
- package/dist/candidate-search/index.spec.js.map +0 -1
- package/dist/candidate-search/query-list/bib.js +0 -439
- package/dist/candidate-search/query-list/bib.js.map +0 -1
- package/dist/candidate-search/query-list/bib.spec.js +0 -64
- package/dist/candidate-search/query-list/bib.spec.js.map +0 -1
- package/dist/candidate-search/query-list/index.js +0 -66
- package/dist/candidate-search/query-list/index.js.map +0 -1
- package/dist/index.js +0 -579
- package/dist/index.js.map +0 -1
- package/dist/index.spec.js +0 -127
- package/dist/index.spec.js.map +0 -1
- package/dist/match-detection/features/bib/all-source-ids.js +0 -119
- package/dist/match-detection/features/bib/all-source-ids.js.map +0 -1
- package/dist/match-detection/features/bib/authors.js +0 -113
- package/dist/match-detection/features/bib/authors.js.map +0 -1
- package/dist/match-detection/features/bib/bibliographic-level.js +0 -42
- package/dist/match-detection/features/bib/bibliographic-level.js.map +0 -1
- package/dist/match-detection/features/bib/host-component.js +0 -42
- package/dist/match-detection/features/bib/host-component.js.map +0 -1
- package/dist/match-detection/features/bib/index.js +0 -119
- package/dist/match-detection/features/bib/index.js.map +0 -1
- package/dist/match-detection/features/bib/index.spec.js +0 -93
- package/dist/match-detection/features/bib/index.spec.js.map +0 -1
- package/dist/match-detection/features/bib/isbn.js +0 -77
- package/dist/match-detection/features/bib/isbn.js.map +0 -1
- package/dist/match-detection/features/bib/issn.js +0 -51
- package/dist/match-detection/features/bib/issn.js.map +0 -1
- package/dist/match-detection/features/bib/language.js +0 -141
- package/dist/match-detection/features/bib/language.js.map +0 -1
- package/dist/match-detection/features/bib/media-type.js +0 -83
- package/dist/match-detection/features/bib/media-type.js.map +0 -1
- package/dist/match-detection/features/bib/melinda-id.js +0 -48
- package/dist/match-detection/features/bib/melinda-id.js.map +0 -1
- package/dist/match-detection/features/bib/melinda-identifier-factory.js +0 -89
- package/dist/match-detection/features/bib/melinda-identifier-factory.js.map +0 -1
- package/dist/match-detection/features/bib/other-standard-identifier.js +0 -52
- package/dist/match-detection/features/bib/other-standard-identifier.js.map +0 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +0 -256
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +0 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js +0 -71
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map +0 -1
- package/dist/match-detection/features/bib/publication-time.js +0 -48
- package/dist/match-detection/features/bib/publication-time.js.map +0 -1
- package/dist/match-detection/features/bib/record-type.js +0 -45
- package/dist/match-detection/features/bib/record-type.js.map +0 -1
- package/dist/match-detection/features/bib/standard-identifier-factory.js +0 -215
- package/dist/match-detection/features/bib/standard-identifier-factory.js.map +0 -1
- package/dist/match-detection/features/bib/title-version-original.js +0 -79
- package/dist/match-detection/features/bib/title-version-original.js.map +0 -1
- package/dist/match-detection/features/bib/title.js +0 -98
- package/dist/match-detection/features/bib/title.js.map +0 -1
- package/dist/match-detection/features/index.js +0 -11
- package/dist/match-detection/features/index.js.map +0 -1
- package/dist/match-detection/index.js +0 -201
- package/dist/match-detection/index.js.map +0 -1
- package/dist/match-detection/index.spec.js +0 -91
- package/dist/match-detection/index.spec.js.map +0 -1
- package/dist/matching-utils.js +0 -143
- package/dist/matching-utils.js.map +0 -1
- package/src/candidate-search/query-list/bib.spec.js +0 -54
- package/src/match-detection/features/bib/index.spec.js +0 -78
|
@@ -1,33 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020-2023 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
|
-
|
|
29
|
-
import {testStringOrNumber} from '../../../matching-utils';
|
|
1
|
+
|
|
30
2
|
import createDebugLogger from 'debug';
|
|
3
|
+
import {testStringOrNumber} from '../../../matching-utils.js';
|
|
31
4
|
|
|
32
5
|
// We should also get copyright time and copyright/publication times from 26x
|
|
33
6
|
// We should also get publishing time type from f008
|
|
@@ -40,13 +13,13 @@ export default () => ({
|
|
|
40
13
|
const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
|
|
41
14
|
|
|
42
15
|
const f008Values = extractF008Values(record);
|
|
43
|
-
debug(`${label} f008: ${JSON.stringify(f008Values)}`);
|
|
16
|
+
debug(`${label}: f008: ${JSON.stringify(f008Values)}`);
|
|
44
17
|
|
|
45
18
|
const f26xValues = extractF26xValues(record);
|
|
46
|
-
debug(`${label} f26x: ${JSON.stringify(f26xValues)}`);
|
|
19
|
+
debug(`${label}: f26x: ${JSON.stringify(f26xValues)}`);
|
|
47
20
|
|
|
48
21
|
const f500Values = extractF500Years(record);
|
|
49
|
-
debug(`${label} f500: ${JSON.stringify(f500Values)}`);
|
|
22
|
+
debug(`${label}: f500: ${JSON.stringify(f500Values)}`);
|
|
50
23
|
|
|
51
24
|
// We should get copyrightYear from f008Date2 to copyrightYears when f008YearType = 'r'
|
|
52
25
|
// Is the original year (f008Date2) in f008YearType === 'r' comparable to copyrightYear?
|
|
@@ -150,7 +123,7 @@ export default () => ({
|
|
|
150
123
|
.flat()
|
|
151
124
|
.filter(({code}) => code === 'a')
|
|
152
125
|
.map(({value}) => value);
|
|
153
|
-
|
|
126
|
+
//.filter(value => value.test(reprintRegex));
|
|
154
127
|
|
|
155
128
|
debug(`f500 reprint field contents: ${JSON.stringify(reprintFieldContents)}`);
|
|
156
129
|
|
|
@@ -199,7 +172,7 @@ export default () => ({
|
|
|
199
172
|
// Handle consequent years as a match
|
|
200
173
|
// see publication-time for a version that does not handle consequent years as a match
|
|
201
174
|
if (!(isNaN(firstANumber) || isNaN(firstBNumber)) &&
|
|
202
|
-
|
|
175
|
+
(firstANumber + 1 === firstBNumber || firstANumber - 1 === firstBNumber)) {
|
|
203
176
|
return 0.1;
|
|
204
177
|
}
|
|
205
178
|
|
|
@@ -1,33 +1,6 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020-2023 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
1
|
|
|
29
|
-
import {testStringOrNumber} from '../../../matching-utils';
|
|
30
2
|
import createDebugLogger from 'debug';
|
|
3
|
+
import {testStringOrNumber} from '../../../matching-utils.js';
|
|
31
4
|
|
|
32
5
|
// We should also get copyright time and copyright/publication times from 26x
|
|
33
6
|
|
|
@@ -1,32 +1,6 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
1
|
|
|
29
|
-
|
|
2
|
+
|
|
3
|
+
import {testStringOrNumber} from '../../../matching-utils.js';
|
|
30
4
|
|
|
31
5
|
// We should also get copyright time and copyright/publication times from 26x
|
|
32
6
|
// see publication-time-allow-cons-years for a version allowing consequent years to match
|
|
@@ -1,30 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
1
|
|
|
29
2
|
// we could handle the case of books/notes
|
|
30
3
|
// Recordtype: LDR/06 - Type of Record
|
|
@@ -1,34 +1,7 @@
|
|
|
1
1
|
/* eslint-disable max-statements */
|
|
2
|
-
/**
|
|
3
|
-
*
|
|
4
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
5
|
-
*
|
|
6
|
-
* Melinda record matching modules for Javascript
|
|
7
|
-
*
|
|
8
|
-
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
9
|
-
*
|
|
10
|
-
* This file is part of melinda-record-matching-js
|
|
11
|
-
*
|
|
12
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
13
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
14
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
15
|
-
* License, or (at your option) any later version.
|
|
16
|
-
*
|
|
17
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
18
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
19
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
20
|
-
* GNU Lesser General Public License for more details.
|
|
21
|
-
*
|
|
22
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
23
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
24
|
-
*
|
|
25
|
-
* @licend The above is the entire license notice
|
|
26
|
-
* for the JavaScript code in this file.
|
|
27
|
-
*
|
|
28
|
-
*/
|
|
29
2
|
|
|
30
3
|
import createDebugLogger from 'debug';
|
|
31
|
-
import {extractSubfieldsFromField, uniqueSubfields} from '../../../matching-utils';
|
|
4
|
+
import {extractSubfieldsFromField, uniqueSubfields} from '../../../matching-utils.js';
|
|
32
5
|
|
|
33
6
|
// Note about validity of standardIdentifiers:
|
|
34
7
|
// We have three types of invalid standardIdentifiers:
|
|
@@ -1,33 +1,7 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
1
|
|
|
29
2
|
import createDebugLogger from 'debug';
|
|
30
|
-
import
|
|
3
|
+
import naturalPkg from 'natural';
|
|
4
|
+
const {LevenshteinDistance: leven} = naturalPkg;
|
|
31
5
|
|
|
32
6
|
export default ({treshold = 10} = {}) => ({
|
|
33
7
|
name: 'titleVersionOriginal',
|
|
@@ -1,35 +1,7 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
import {LevenshteinDistance as leven} from 'natural';
|
|
31
|
-
import {testStringOrNumber} from '../../../matching-utils';
|
|
32
1
|
import createDebugLogger from 'debug';
|
|
2
|
+
import naturalPkg from 'natural';
|
|
3
|
+
const {LevenshteinDistance: leven} = naturalPkg;
|
|
4
|
+
import {testStringOrNumber} from '../../../matching-utils.js';
|
|
33
5
|
|
|
34
6
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:title');
|
|
35
7
|
const debugData = debug.extend('data');
|
|
@@ -40,7 +12,7 @@ export default ({treshold = 10} = {}) => ({
|
|
|
40
12
|
extract: ({record, recordExternal}) => {
|
|
41
13
|
const label = recordExternal && recordExternal.label ? recordExternal.label : 'record';
|
|
42
14
|
const title = getTitle();
|
|
43
|
-
debug(`${label} title: ${title}`);
|
|
15
|
+
debug(`${label}: title: ${title}`);
|
|
44
16
|
|
|
45
17
|
if (testStringOrNumber(title)) {
|
|
46
18
|
const titleAsNormalizedString = String(title)
|
|
@@ -51,7 +23,7 @@ export default ({treshold = 10} = {}) => ({
|
|
|
51
23
|
// - we could precompose the finnish letters back to avoid this
|
|
52
24
|
.replace(/[^\p{Letter}\p{Number}]/gu, '')
|
|
53
25
|
.toLowerCase();
|
|
54
|
-
debug(`${label} titleString: ${titleAsNormalizedString}`);
|
|
26
|
+
debug(`${label}: titleString: ${titleAsNormalizedString}`);
|
|
55
27
|
return [titleAsNormalizedString];
|
|
56
28
|
}
|
|
57
29
|
|
|
@@ -59,7 +31,7 @@ export default ({treshold = 10} = {}) => ({
|
|
|
59
31
|
|
|
60
32
|
function getTitle() {
|
|
61
33
|
const [field] = record.get(/^245$/u);
|
|
62
|
-
debugData(`${label} titleField: ${JSON.stringify(field)}`);
|
|
34
|
+
debugData(`${label}: titleField: ${JSON.stringify(field)}`);
|
|
63
35
|
|
|
64
36
|
if (field) {
|
|
65
37
|
return field.subfields
|
|
@@ -1,30 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
1
|
|
|
29
|
-
import * as bib from './bib';
|
|
2
|
+
import * as bib from './bib/index.js';
|
|
30
3
|
export {bib};
|
|
@@ -1,33 +1,6 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
1
|
|
|
29
2
|
import createDebugLogger from 'debug';
|
|
30
|
-
import * as features from './features';
|
|
3
|
+
import * as features from './features/index.js';
|
|
31
4
|
|
|
32
5
|
export {features};
|
|
33
6
|
|
|
@@ -1,46 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
|
-
|
|
1
|
+
import assert from 'node:assert';
|
|
2
|
+
import {describe} from 'node:test';
|
|
3
|
+
import createDebugLogger from 'debug';
|
|
4
|
+
import {inspect} from 'util';
|
|
29
5
|
import generateTests from '@natlibfi/fixugen';
|
|
30
6
|
import {READERS} from '@natlibfi/fixura';
|
|
31
|
-
import {expect} from 'chai';
|
|
32
7
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
33
|
-
|
|
34
|
-
import createDetectionInterface from '.';
|
|
35
|
-
import
|
|
36
|
-
import createDebugLogger from 'debug';
|
|
8
|
+
|
|
9
|
+
import createDetectionInterface from './index.js';
|
|
10
|
+
import * as features from './features/index.js';
|
|
37
11
|
|
|
38
12
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:test');
|
|
39
13
|
const debugData = debug.extend('data');
|
|
40
14
|
|
|
41
15
|
describe('match-detection', () => {
|
|
42
16
|
generateTests({
|
|
43
|
-
path: [
|
|
17
|
+
path: [import.meta.dirname, '..', '..', 'test-fixtures', 'match-detection', 'index'],
|
|
44
18
|
useMetadataFile: true,
|
|
45
19
|
recurse: false,
|
|
46
20
|
fixura: {
|
|
@@ -66,7 +40,7 @@ describe('match-detection', () => {
|
|
|
66
40
|
const results = detect({recordA, recordB});
|
|
67
41
|
debugData(`${JSON.stringify(results)}`);
|
|
68
42
|
|
|
69
|
-
|
|
43
|
+
assert.deepStrictEqual(results, expectedResults);
|
|
70
44
|
|
|
71
45
|
function formatOptions() {
|
|
72
46
|
const contextFeatures = features[options.strategy.type];
|
package/src/matching-utils.js
CHANGED
|
@@ -1,31 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
4
|
-
*
|
|
5
|
-
* Melinda record matching modules for Javascript
|
|
6
|
-
*
|
|
7
|
-
* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
|
|
8
|
-
*
|
|
9
|
-
* This file is part of melinda-record-matching-js
|
|
10
|
-
*
|
|
11
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
12
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
13
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
* License, or (at your option) any later version.
|
|
15
|
-
*
|
|
16
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
17
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
* GNU Lesser General Public License for more details.
|
|
20
|
-
*
|
|
21
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
23
|
-
*
|
|
24
|
-
* @licend The above is the entire license notice
|
|
25
|
-
* for the JavaScript code in this file.
|
|
26
|
-
*
|
|
27
|
-
*/
|
|
28
|
-
|
|
29
1
|
import createDebugLogger from 'debug';
|
|
30
2
|
|
|
31
3
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');
|
|
@@ -43,7 +15,7 @@ export function getMelindaIdsF035(record) {
|
|
|
43
15
|
return [];
|
|
44
16
|
}
|
|
45
17
|
|
|
46
|
-
const allF035MelindaIds = [].concat(...f035s.map(toMelindaIds));
|
|
18
|
+
const allF035MelindaIds = [].concat(...f035s.map(field => toMelindaIds(field)));
|
|
47
19
|
const f035MelindaIds = [...new Set(allF035MelindaIds)];
|
|
48
20
|
|
|
49
21
|
debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);
|
|
@@ -51,16 +23,15 @@ export function getMelindaIdsF035(record) {
|
|
|
51
23
|
debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);
|
|
52
24
|
|
|
53
25
|
return f035MelindaIds;
|
|
26
|
+
}
|
|
54
27
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
return subfields
|
|
59
|
-
.filter(sub => ['a', 'z'].includes(sub.code))
|
|
60
|
-
.filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))
|
|
61
|
-
.map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');
|
|
28
|
+
export function toMelindaIds({subfields}, subfieldsToParse = ['a', 'z']) {
|
|
29
|
+
const melindaIdRegExp = /^(?<prefix>\(FI-MELINDA\)|FCC)(?<id>\d{9})$/u;
|
|
62
30
|
|
|
63
|
-
|
|
31
|
+
return subfields
|
|
32
|
+
.filter(sub => subfieldsToParse.includes(sub.code))
|
|
33
|
+
.filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))
|
|
34
|
+
.map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');
|
|
64
35
|
}
|
|
65
36
|
|
|
66
37
|
export function validateSidFieldSubfieldCounts(field) {
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.toQueries = toQueries;
|
|
7
|
-
var _debug = _interopRequireDefault(require("debug"));
|
|
8
|
-
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
9
|
-
/**
|
|
10
|
-
*
|
|
11
|
-
* @licstart The following is the entire license notice for the JavaScript code in this file.
|
|
12
|
-
*
|
|
13
|
-
* Melinda record matching modules for Javascript
|
|
14
|
-
*
|
|
15
|
-
* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
|
|
16
|
-
*
|
|
17
|
-
* This file is part of melinda-record-matching-js
|
|
18
|
-
*
|
|
19
|
-
* melinda-record-matching-js program is free software: you can redistribute it and/or modify
|
|
20
|
-
* it under the terms of the GNU Lesser General Public License as
|
|
21
|
-
* published by the Free Software Foundation, either version 3 of the
|
|
22
|
-
* License, or (at your option) any later version.
|
|
23
|
-
*
|
|
24
|
-
* melinda-record-matching-js is distributed in the hope that it will be useful,
|
|
25
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
26
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
27
|
-
* GNU Lesser General Public License for more details.
|
|
28
|
-
*
|
|
29
|
-
* You should have received a copy of the GNU Affero General Public License
|
|
30
|
-
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
31
|
-
*
|
|
32
|
-
* @licend The above is the entire license notice
|
|
33
|
-
* for the JavaScript code in this file.
|
|
34
|
-
*
|
|
35
|
-
*/
|
|
36
|
-
|
|
37
|
-
function toQueries(identifiers, queryString) {
|
|
38
|
-
const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:toQueries');
|
|
39
|
-
const debugData = debug.extend('data');
|
|
40
|
-
const quotedIdentifiers = identifiers.map(identifier => identifier.match(/\//u) ? `"${identifier}"` : `${identifier}`);
|
|
41
|
-
|
|
42
|
-
// Aleph supports only two queries with or -operator (This is not actually true)
|
|
43
|
-
const pairs = toPairs(quotedIdentifiers);
|
|
44
|
-
const queries = pairs.map(([a, b]) => b ? `${queryString}=${a} or ${queryString}=${b}` : `${queryString}=${a}`);
|
|
45
|
-
debugData(`Pairs (${pairs.length}): ${JSON.stringify(pairs)}`);
|
|
46
|
-
debugData(`Queries (${queries.length}): ${JSON.stringify(queries)}`);
|
|
47
|
-
return queries;
|
|
48
|
-
}
|
|
49
|
-
function toPairs(array) {
|
|
50
|
-
if (array.length === 0) {
|
|
51
|
-
return [];
|
|
52
|
-
}
|
|
53
|
-
return [array.slice(0, 2)].concat(toPairs(array.slice(2), 2));
|
|
54
|
-
}
|
|
55
|
-
//# sourceMappingURL=candidate-search-utils.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"candidate-search-utils.js","names":["_debug","_interopRequireDefault","require","obj","__esModule","default","toQueries","identifiers","queryString","debug","createDebugLogger","debugData","extend","quotedIdentifiers","map","identifier","match","pairs","toPairs","queries","a","b","length","JSON","stringify","array","slice","concat"],"sources":["../../src/candidate-search/candidate-search-utils.js"],"sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\n\nexport function toQueries(identifiers, queryString) {\n\n const debug = createDebugLogger('@natlibfi/melinda-record-matching:toQueries');\n const debugData = debug.extend('data');\n\n const quotedIdentifiers = identifiers.map(identifier => identifier.match(/\\//u) ? `\"${identifier}\"` : `${identifier}`);\n\n // Aleph supports only two queries with or -operator (This is not actually true)\n const pairs = toPairs(quotedIdentifiers);\n const queries = pairs.map(([a, b]) => b ? `${queryString}=${a} or ${queryString}=${b}` : `${queryString}=${a}`);\n\n debugData(`Pairs (${pairs.length}): ${JSON.stringify(pairs)}`);\n debugData(`Queries (${queries.length}): ${JSON.stringify(queries)}`);\n\n return queries;\n}\n\nfunction toPairs(array) {\n if (array.length === 0) {\n return [];\n }\n return [array.slice(0, 2)].concat(toPairs(array.slice(2), 2));\n}\n"],"mappings":";;;;;;AA4BA,IAAAA,MAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAsC,SAAAD,uBAAAE,GAAA,WAAAA,GAAA,IAAAA,GAAA,CAAAC,UAAA,GAAAD,GAAA,KAAAE,OAAA,EAAAF,GAAA;AA5BtC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAIO,SAASG,SAASA,CAACC,WAAW,EAAEC,WAAW,EAAE;EAElD,MAAMC,KAAK,GAAG,IAAAC,cAAiB,EAAC,6CAA6C,CAAC;EAC9E,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAM,CAAC,MAAM,CAAC;EAEtC,MAAMC,iBAAiB,GAAGN,WAAW,CAACO,GAAG,CAACC,UAAU,IAAIA,UAAU,CAACC,KAAK,CAAC,KAAK,CAAC,GAAI,IAAGD,UAAW,GAAE,GAAI,GAAEA,UAAW,EAAC,CAAC;;EAEtH;EACA,MAAME,KAAK,GAAGC,OAAO,CAACL,iBAAiB,CAAC;EACxC,MAAMM,OAAO,GAAGF,KAAK,CAACH,GAAG,CAAC,CAAC,CAACM,CAAC,EAAEC,CAAC,CAAC,KAAKA,CAAC,GAAI,GAAEb,WAAY,IAAGY,CAAE,OAAMZ,WAAY,IAAGa,CAAE,EAAC,GAAI,GAAEb,WAAY,IAAGY,CAAE,EAAC,CAAC;EAE/GT,SAAS,CAAE,UAASM,KAAK,CAACK,MAAO,MAAKC,IAAI,CAACC,SAAS,CAACP,KAAK,CAAE,EAAC,CAAC;EAC9DN,SAAS,CAAE,YAAWQ,OAAO,CAACG,MAAO,MAAKC,IAAI,CAACC,SAAS,CAACL,OAAO,CAAE,EAAC,CAAC;EAEpE,OAAOA,OAAO;AAChB;AAEA,SAASD,OAAOA,CAACO,KAAK,EAAE;EACtB,IAAIA,KAAK,CAACH,MAAM,KAAK,CAAC,EAAE;IACtB,OAAO,EAAE;EACX;EACA,OAAO,CAACG,KAAK,CAACC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAACC,MAAM,CAACT,OAAO,CAACO,KAAK,CAACC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/D"}
|