@natlibfi/melinda-record-matching 4.3.5 → 5.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/melinda-node-tests.yml +11 -7
- package/eslint.config.js +52 -0
- package/example.env +4 -0
- package/package.json +22 -90
- package/src/candidate-search/choose-queries.js +1 -3
- package/src/candidate-search/index.js +63 -89
- package/src/candidate-search/{index.spec.js → index.test.js} +19 -18
- package/src/candidate-search/query-list/bib.js +10 -4
- package/src/candidate-search/query-list/{bib.spec.js → bib.test.js} +8 -7
- package/src/candidate-search/query-list/component.js +127 -0
- package/src/candidate-search/query-list/index.js +23 -9
- package/src/cli.js +177 -0
- package/src/index.js +22 -37
- package/src/{index.spec.js → index.test.js} +11 -14
- package/src/match-detection/features/bib/all-source-ids.js +2 -2
- package/src/match-detection/features/bib/authors.js +3 -2
- package/src/match-detection/features/bib/index.js +16 -16
- package/src/match-detection/features/bib/{index.spec.js → index.test.js} +7 -6
- package/src/match-detection/features/bib/isbn.js +2 -2
- package/src/match-detection/features/bib/issn.js +1 -1
- package/src/match-detection/features/bib/language.js +5 -5
- package/src/match-detection/features/bib/media-type.js +1 -1
- package/src/match-detection/features/bib/melinda-id.js +1 -1
- package/src/match-detection/features/bib/melinda-identifier-factory.js +5 -5
- package/src/match-detection/features/bib/other-standard-identifier.js +1 -1
- package/src/match-detection/features/bib/publication-time-allow-cons-years-multi.js +6 -6
- package/src/match-detection/features/bib/publication-time-allow-cons-years.js +1 -1
- package/src/match-detection/features/bib/publication-time.js +1 -1
- package/src/match-detection/features/bib/standard-identifier-factory.js +1 -2
- package/src/match-detection/features/bib/title-version-original.js +2 -1
- package/src/match-detection/features/bib/title.js +6 -5
- package/src/match-detection/features/index.js +1 -1
- package/src/match-detection/index.js +1 -1
- package/src/match-detection/{index.spec.js → index.test.js} +9 -8
- package/src/matching-utils.js +8 -9
- package/dist/candidate-search/candidate-search-utils.js +0 -29
- package/dist/candidate-search/candidate-search-utils.js.map +0 -1
- package/dist/candidate-search/choose-queries.js +0 -122
- package/dist/candidate-search/choose-queries.js.map +0 -1
- package/dist/candidate-search/index.js +0 -260
- package/dist/candidate-search/index.js.map +0 -1
- package/dist/candidate-search/index.spec.js +0 -133
- package/dist/candidate-search/index.spec.js.map +0 -1
- package/dist/candidate-search/query-list/bib.js +0 -458
- package/dist/candidate-search/query-list/bib.js.map +0 -1
- package/dist/candidate-search/query-list/bib.spec.js +0 -46
- package/dist/candidate-search/query-list/bib.spec.js.map +0 -1
- package/dist/candidate-search/query-list/index.js +0 -47
- package/dist/candidate-search/query-list/index.js.map +0 -1
- package/dist/index.js +0 -555
- package/dist/index.js.map +0 -1
- package/dist/index.spec.js +0 -99
- package/dist/index.spec.js.map +0 -1
- package/dist/match-detection/features/bib/all-source-ids.js +0 -91
- package/dist/match-detection/features/bib/all-source-ids.js.map +0 -1
- package/dist/match-detection/features/bib/authors.js +0 -86
- package/dist/match-detection/features/bib/authors.js.map +0 -1
- package/dist/match-detection/features/bib/bibliographic-level.js +0 -15
- package/dist/match-detection/features/bib/bibliographic-level.js.map +0 -1
- package/dist/match-detection/features/bib/host-component.js +0 -15
- package/dist/match-detection/features/bib/host-component.js.map +0 -1
- package/dist/match-detection/features/bib/index.js +0 -119
- package/dist/match-detection/features/bib/index.js.map +0 -1
- package/dist/match-detection/features/bib/index.spec.js +0 -65
- package/dist/match-detection/features/bib/index.spec.js.map +0 -1
- package/dist/match-detection/features/bib/isbn.js +0 -49
- package/dist/match-detection/features/bib/isbn.js.map +0 -1
- package/dist/match-detection/features/bib/issn.js +0 -24
- package/dist/match-detection/features/bib/issn.js.map +0 -1
- package/dist/match-detection/features/bib/language.js +0 -113
- package/dist/match-detection/features/bib/language.js.map +0 -1
- package/dist/match-detection/features/bib/media-type.js +0 -56
- package/dist/match-detection/features/bib/media-type.js.map +0 -1
- package/dist/match-detection/features/bib/melinda-id.js +0 -21
- package/dist/match-detection/features/bib/melinda-id.js.map +0 -1
- package/dist/match-detection/features/bib/melinda-identifier-factory.js +0 -62
- package/dist/match-detection/features/bib/melinda-identifier-factory.js.map +0 -1
- package/dist/match-detection/features/bib/other-standard-identifier.js +0 -25
- package/dist/match-detection/features/bib/other-standard-identifier.js.map +0 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js +0 -229
- package/dist/match-detection/features/bib/publication-time-allow-cons-years-multi.js.map +0 -1
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js +0 -44
- package/dist/match-detection/features/bib/publication-time-allow-cons-years.js.map +0 -1
- package/dist/match-detection/features/bib/publication-time.js +0 -21
- package/dist/match-detection/features/bib/publication-time.js.map +0 -1
- package/dist/match-detection/features/bib/record-type.js +0 -18
- package/dist/match-detection/features/bib/record-type.js.map +0 -1
- package/dist/match-detection/features/bib/standard-identifier-factory.js +0 -188
- package/dist/match-detection/features/bib/standard-identifier-factory.js.map +0 -1
- package/dist/match-detection/features/bib/title-version-original.js +0 -52
- package/dist/match-detection/features/bib/title-version-original.js.map +0 -1
- package/dist/match-detection/features/bib/title.js +0 -70
- package/dist/match-detection/features/bib/title.js.map +0 -1
- package/dist/match-detection/features/index.js +0 -11
- package/dist/match-detection/features/index.js.map +0 -1
- package/dist/match-detection/index.js +0 -174
- package/dist/match-detection/index.js.map +0 -1
- package/dist/match-detection/index.spec.js +0 -63
- package/dist/match-detection/index.spec.js.map +0 -1
- package/dist/matching-utils.js +0 -115
- package/dist/matching-utils.js.map +0 -1
|
@@ -11,14 +11,14 @@ jobs:
|
|
|
11
11
|
|
|
12
12
|
strategy:
|
|
13
13
|
matrix:
|
|
14
|
-
node-version: [
|
|
14
|
+
node-version: [22.x, 24.x]
|
|
15
15
|
# See supported Node.js release schedule at https://nodejs.org/en/about/releases/
|
|
16
16
|
|
|
17
17
|
steps:
|
|
18
18
|
- name: Checkout the code
|
|
19
|
-
uses: actions/checkout@
|
|
19
|
+
uses: actions/checkout@v5
|
|
20
20
|
- name: Use Node.js ${{ matrix.node-version }}
|
|
21
|
-
uses: actions/setup-node@
|
|
21
|
+
uses: actions/setup-node@v5
|
|
22
22
|
with:
|
|
23
23
|
node-version: ${{ matrix.node-version }}
|
|
24
24
|
cache: 'npm'
|
|
@@ -32,10 +32,12 @@ jobs:
|
|
|
32
32
|
njsscan:
|
|
33
33
|
name: Njsscan check
|
|
34
34
|
runs-on: ubuntu-latest
|
|
35
|
+
container:
|
|
36
|
+
image: docker.io/node:22
|
|
35
37
|
|
|
36
38
|
steps:
|
|
37
39
|
- name: Checkout the code
|
|
38
|
-
uses: actions/checkout@
|
|
40
|
+
uses: actions/checkout@v5
|
|
39
41
|
- name: nodejsscan scan
|
|
40
42
|
id: njsscan
|
|
41
43
|
uses: ajinabraham/njsscan-action@master
|
|
@@ -45,9 +47,11 @@ jobs:
|
|
|
45
47
|
license-scan:
|
|
46
48
|
name: License compliance check
|
|
47
49
|
runs-on: ubuntu-latest
|
|
50
|
+
container:
|
|
51
|
+
image: docker.io/node:22
|
|
48
52
|
|
|
49
53
|
steps:
|
|
50
|
-
- uses: actions/checkout@
|
|
54
|
+
- uses: actions/checkout@v5
|
|
51
55
|
- uses: mikaelvesavuori/license-compliance-action@v1.0.3
|
|
52
56
|
with:
|
|
53
57
|
exclude_pattern: /^@natlibfi/
|
|
@@ -58,9 +62,9 @@ jobs:
|
|
|
58
62
|
if: contains(github.ref, 'refs/tags/')
|
|
59
63
|
|
|
60
64
|
steps:
|
|
61
|
-
- uses: actions/checkout@
|
|
65
|
+
- uses: actions/checkout@v5
|
|
62
66
|
# Setup .npmrc file to publish to npm
|
|
63
|
-
- uses: actions/setup-node@
|
|
67
|
+
- uses: actions/setup-node@v5
|
|
64
68
|
with:
|
|
65
69
|
node-version: '20.x'
|
|
66
70
|
registry-url: 'https://registry.npmjs.org'
|
package/eslint.config.js
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Eslint configuration object for src
|
|
2
|
+
const configSrc = {
|
|
3
|
+
files: [
|
|
4
|
+
"src/*"
|
|
5
|
+
],
|
|
6
|
+
linterOptions: {
|
|
7
|
+
reportUnusedDisableDirectives: true,
|
|
8
|
+
},
|
|
9
|
+
rules: {
|
|
10
|
+
"no-console": "warn",
|
|
11
|
+
"eqeqeq": ["error", "always"],
|
|
12
|
+
"no-const-assign": "error",
|
|
13
|
+
"max-depth": ["warn", 4],
|
|
14
|
+
"max-lines": ["warn", 500],
|
|
15
|
+
"max-lines-per-function": ["warn", {"max": 100}],
|
|
16
|
+
"no-else-return": ["error", {allowElseIf: false}],
|
|
17
|
+
"no-plusplus": [
|
|
18
|
+
"error",
|
|
19
|
+
{
|
|
20
|
+
"allowForLoopAfterthoughts": true
|
|
21
|
+
}
|
|
22
|
+
],
|
|
23
|
+
"array-callback-return": [
|
|
24
|
+
"error",
|
|
25
|
+
{
|
|
26
|
+
"checkForEach": false
|
|
27
|
+
}
|
|
28
|
+
],
|
|
29
|
+
"no-unused-vars": [
|
|
30
|
+
"error",
|
|
31
|
+
{
|
|
32
|
+
"argsIgnorePattern": "next"
|
|
33
|
+
}
|
|
34
|
+
],
|
|
35
|
+
"no-warning-comments": "off"
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
// Eslint configuration object for globally ignoring .js files
|
|
40
|
+
// - ignore all files that start with a dot
|
|
41
|
+
// - ignore all files inside directories named 'dist'
|
|
42
|
+
const configIgnores = {
|
|
43
|
+
ignores: [
|
|
44
|
+
"**/.*",
|
|
45
|
+
"**/dist/"
|
|
46
|
+
]
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
export default [
|
|
50
|
+
configSrc,
|
|
51
|
+
configIgnores
|
|
52
|
+
];
|
package/example.env
ADDED
package/package.json
CHANGED
|
@@ -14,115 +14,47 @@
|
|
|
14
14
|
"url": "git@github.com:natlibfi/melinda-record-matching-js.git"
|
|
15
15
|
},
|
|
16
16
|
"license": "MIT",
|
|
17
|
-
"version": "
|
|
17
|
+
"version": "5.0.0-alpha.1",
|
|
18
18
|
"main": "./dist/index.js",
|
|
19
|
+
"type": "module",
|
|
20
|
+
"bin": "./dist/cli.js",
|
|
19
21
|
"engines": {
|
|
20
|
-
"node": ">=
|
|
22
|
+
"node": ">=22"
|
|
21
23
|
},
|
|
22
24
|
"publishConfig": {
|
|
23
25
|
"access": "public"
|
|
24
26
|
},
|
|
25
27
|
"scripts": {
|
|
26
|
-
"prepare": "npm run build",
|
|
27
|
-
"build": "babel src --source-maps --copy-files --delete-dir-on-start --out-dir=dist",
|
|
28
28
|
"start": "node dist/index.js",
|
|
29
|
+
"build": "esbuild \"src/**/*.js\" --sourcemap --platform=node --outdir=dist",
|
|
29
30
|
"cli": "node dist/cli.js",
|
|
30
31
|
"lint": "eslint ./src",
|
|
31
32
|
"lint:dev": "eslint --fix ./src",
|
|
33
|
+
"test:base": "node --test --test-force-exit --experimental-test-coverage --test-reporter=spec './src/*.test.js' './src/**/*.test.js'",
|
|
32
34
|
"test": "npm run lint && npm run test:base",
|
|
33
|
-
"test
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"dev": "nodemon",
|
|
37
|
-
"dev:debug": "cross-env LOG_LEVEL=debug DEBUG=@natlibfi/* NODE_ENV=test nodemon",
|
|
38
|
-
"watch:test": "cross-env DEBUG=1 NODE_ENV=test nodemon -w src -w test-fixtures --exec 'npm run test:dev'"
|
|
35
|
+
"watch:test": "cross-env DEBUG=@natlibfi/* NODE_ENV=test node --watch --test --experimental-test-coverage --test-reporter=spec './src/*.test.js' './src/**/*.test.js'",
|
|
36
|
+
"dev": "npm run watch:test",
|
|
37
|
+
"dev:debug": "cross-env LOG_LEVEL=debug DEBUG=@natlibfi/* NODE_ENV=test npm run watch:test"
|
|
39
38
|
},
|
|
40
39
|
"dependencies": {
|
|
41
|
-
"@natlibfi/marc-record": "^
|
|
42
|
-
"@natlibfi/marc-record-serializers": "^
|
|
43
|
-
"@natlibfi/melinda-commons": "^
|
|
44
|
-
"@natlibfi/sru-client": "^
|
|
45
|
-
"debug": "^4.4.
|
|
46
|
-
"isbn3": "^1.2.
|
|
40
|
+
"@natlibfi/marc-record": "^10.0.0-alpha.1",
|
|
41
|
+
"@natlibfi/marc-record-serializers": "^11.0.0-alpha.2",
|
|
42
|
+
"@natlibfi/melinda-commons": "^14.0.0-alpha.3",
|
|
43
|
+
"@natlibfi/sru-client": "^7.0.0-alpha.1",
|
|
44
|
+
"debug": "^4.4.1",
|
|
45
|
+
"isbn3": "^1.2.14",
|
|
47
46
|
"moment": "^2.30.1",
|
|
48
47
|
"natural": "^8.0.1",
|
|
49
48
|
"uuid": "^11.1.0",
|
|
50
|
-
"winston": "^3.17.0"
|
|
49
|
+
"winston": "^3.17.0",
|
|
50
|
+
"yargs": "^18.0.0"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
|
-
"@
|
|
54
|
-
"@
|
|
55
|
-
"@
|
|
56
|
-
"@babel/preset-env": "^7.26.9",
|
|
57
|
-
"@babel/register": "^7.25.9",
|
|
58
|
-
"@natlibfi/eslint-config-melinda-backend": "^3.0.5",
|
|
59
|
-
"@natlibfi/fixugen": "^2.0.13",
|
|
60
|
-
"@natlibfi/fixugen-http-client": "^3.0.10",
|
|
61
|
-
"@natlibfi/fixura": "^3.0.12",
|
|
62
|
-
"babel-plugin-istanbul": "^6.1.1",
|
|
63
|
-
"babel-plugin-rewire": "^1.2.0",
|
|
64
|
-
"chai": "^4.5.0",
|
|
65
|
-
"chai-as-promised": "^7.1.2",
|
|
53
|
+
"@natlibfi/fixugen": "^3.0.0-alpha.6",
|
|
54
|
+
"@natlibfi/fixugen-http-client": "^4.0.0-alpha.1",
|
|
55
|
+
"@natlibfi/fixura": "^4.0.0-alpha.13",
|
|
66
56
|
"cross-env": "^7.0.3",
|
|
67
|
-
"
|
|
68
|
-
"
|
|
69
|
-
"nodemon": "^3.1.9",
|
|
70
|
-
"nyc": "^17.1.0"
|
|
71
|
-
},
|
|
72
|
-
"eslintConfig": {
|
|
73
|
-
"extends": "@natlibfi/melinda-backend"
|
|
74
|
-
},
|
|
75
|
-
"babel": {
|
|
76
|
-
"presets": [
|
|
77
|
-
[
|
|
78
|
-
"@babel/preset-env",
|
|
79
|
-
{
|
|
80
|
-
"targets": "maintained node versions"
|
|
81
|
-
}
|
|
82
|
-
]
|
|
83
|
-
],
|
|
84
|
-
"env": {
|
|
85
|
-
"test": {
|
|
86
|
-
"plugins": [
|
|
87
|
-
"istanbul",
|
|
88
|
-
"rewire"
|
|
89
|
-
]
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
},
|
|
93
|
-
"mocha": {
|
|
94
|
-
"spec": [
|
|
95
|
-
"src/*.spec.js",
|
|
96
|
-
"src/**/*.spec.js"
|
|
97
|
-
],
|
|
98
|
-
"require": [
|
|
99
|
-
"@babel/register"
|
|
100
|
-
],
|
|
101
|
-
"inline-diffs": true,
|
|
102
|
-
"maxDiffSize": 25000,
|
|
103
|
-
"bail": true,
|
|
104
|
-
"exit": true
|
|
105
|
-
},
|
|
106
|
-
"nodemonConfig": {
|
|
107
|
-
"exec": "npm run test:dev",
|
|
108
|
-
"watch": [
|
|
109
|
-
"src/*",
|
|
110
|
-
"test-fixtures/*"
|
|
111
|
-
]
|
|
112
|
-
},
|
|
113
|
-
"nyc": {
|
|
114
|
-
"exclude": [
|
|
115
|
-
"src/*.spec.js",
|
|
116
|
-
"src/**/*.spec.js"
|
|
117
|
-
],
|
|
118
|
-
"reporter": [
|
|
119
|
-
"text"
|
|
120
|
-
],
|
|
121
|
-
"sourceMap": false,
|
|
122
|
-
"instrument": false,
|
|
123
|
-
"lines": 80,
|
|
124
|
-
"statements": 80,
|
|
125
|
-
"functions": 80,
|
|
126
|
-
"branches": 80
|
|
57
|
+
"esbuild": "^0.25.9",
|
|
58
|
+
"eslint": "^9.35.0"
|
|
127
59
|
}
|
|
128
60
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import createDebugLogger from 'debug';
|
|
2
2
|
import createClient, {SruSearchError} from '@natlibfi/sru-client';
|
|
3
3
|
|
|
4
|
-
export class CandidateSearchError extends Error {}
|
|
4
|
+
export class CandidateSearchError extends Error { }
|
|
5
5
|
|
|
6
6
|
export default async function ({url, queryList, queryListType, maxCandidates = 50}) {
|
|
7
7
|
|
|
@@ -50,14 +50,12 @@ export default async function ({url, queryList, queryListType, maxCandidates = 5
|
|
|
50
50
|
|
|
51
51
|
function retrieveTotal() {
|
|
52
52
|
return new Promise((resolve, reject) => {
|
|
53
|
-
// eslint-disable-next-line functional/no-let
|
|
54
53
|
let totalRecords = 0;
|
|
55
54
|
|
|
56
55
|
debug(`Searching total amount of candidates for query: ${query}`);
|
|
57
56
|
|
|
58
57
|
client.searchRetrieve(query)
|
|
59
58
|
.on('error', err => {
|
|
60
|
-
// eslint-disable-next-line functional/no-conditional-statements
|
|
61
59
|
if (err instanceof SruSearchError) {
|
|
62
60
|
debug(`SRU SruSearchError for query: ${query}: ${err}`);
|
|
63
61
|
reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import createDebugLogger from 'debug';
|
|
2
2
|
import createClient, {SruSearchError} from '@natlibfi/sru-client';
|
|
3
3
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
4
|
-
import {MARCXML} from '@natlibfi/marc-record-serializers';
|
|
5
|
-
import generateQueryList from './query-list';
|
|
6
|
-
import {Error as MatchingError} from '@natlibfi/melinda-commons';
|
|
7
|
-
import chooseQueries from './choose-queries';
|
|
8
4
|
|
|
9
|
-
|
|
5
|
+
import generateQueryList from './query-list/index.js';
|
|
6
|
+
import chooseQueries from './choose-queries.js';
|
|
10
7
|
|
|
11
|
-
export
|
|
8
|
+
export {searchTypes} from './query-list/index.js';
|
|
9
|
+
|
|
10
|
+
export class CandidateSearchError extends Error { }
|
|
12
11
|
|
|
13
12
|
// serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000
|
|
14
13
|
|
|
@@ -27,8 +26,16 @@ export default async ({record, searchSpec, url, maxCandidates, maxRecordsPerRequ
|
|
|
27
26
|
// Do not retrieve more candidates than defined in maxCandidates
|
|
28
27
|
const adjustedMaxRecordsPerRequest = maxRecordsPerRequest >= maxCandidates ? maxCandidates : maxRecordsPerRequest;
|
|
29
28
|
|
|
29
|
+
const client = createClient({
|
|
30
|
+
url,
|
|
31
|
+
maxRecordsPerRequest: adjustedMaxRecordsPerRequest,
|
|
32
|
+
version: '2.0',
|
|
33
|
+
retrieveAll: false,
|
|
34
|
+
metadataFormat: 'marcJson'
|
|
35
|
+
});
|
|
36
|
+
|
|
30
37
|
const inputRecordId = getRecordId(record);
|
|
31
|
-
const queryListResult = generateQueryList(record, searchSpec);
|
|
38
|
+
const queryListResult = await generateQueryList(record, searchSpec, client);
|
|
32
39
|
const queryList = queryListResult[0]?.queryList ? queryListResult[0].queryList : queryListResult;
|
|
33
40
|
const queryListType = queryListResult[0]?.queryListType ? queryListResult[0].queryListType : undefined;
|
|
34
41
|
|
|
@@ -42,13 +49,6 @@ export default async ({record, searchSpec, url, maxCandidates, maxRecordsPerRequ
|
|
|
42
49
|
throw new CandidateSearchError(`Generated query list has invalid type`);
|
|
43
50
|
}
|
|
44
51
|
|
|
45
|
-
const client = createClient({
|
|
46
|
-
url,
|
|
47
|
-
maxRecordsPerRequest: adjustedMaxRecordsPerRequest,
|
|
48
|
-
version: '2.0',
|
|
49
|
-
retrieveAll: false
|
|
50
|
-
});
|
|
51
|
-
|
|
52
52
|
debug(`Searching matches for ${inputRecordId}`);
|
|
53
53
|
const chosenQueryList = await filterQueryList({queryList, queryListType});
|
|
54
54
|
debug(`Chosen queries: ${JSON.stringify(chosenQueryList)}`);
|
|
@@ -75,97 +75,33 @@ export default async ({record, searchSpec, url, maxCandidates, maxRecordsPerRequ
|
|
|
75
75
|
|
|
76
76
|
// eslint-disable-next-line max-statements
|
|
77
77
|
async function search({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) {
|
|
78
|
-
|
|
79
78
|
const query = chosenQueryList[queryOffset];
|
|
80
79
|
debug(`Running query ${JSON.stringify(query)} (${queryOffset})`);
|
|
81
80
|
|
|
82
81
|
if (query) {
|
|
83
|
-
const {records,
|
|
82
|
+
const {records, nextOffset, total} = await retrieveRecords(client, query, resultSetOffset);
|
|
84
83
|
|
|
85
84
|
// If resultSetOffset === 1 this is the first search for the current query
|
|
86
85
|
debugData(`ResultSetOffset: ${resultSetOffset}`);
|
|
87
86
|
const newTotalRecords = resultSetOffset === 1 ? total : totalRecords;
|
|
88
87
|
const newQueryCounter = resultSetOffset === 1 ? queryCounter + 1 : queryCounter;
|
|
89
88
|
const newSearchCounter = resultSetOffset === 1 ? 1 : searchCounter + 1;
|
|
90
|
-
const newQueryCandidateCounter = resultSetOffset === 1 ? records.length
|
|
89
|
+
const newQueryCandidateCounter = resultSetOffset === 1 ? records.length : queryCandidateCounter + records.length;
|
|
91
90
|
|
|
92
91
|
const maxedQuery = resultSetOffset === 1 ? checkMaxedQuery(query, total, serverMaxResult) : undefined;
|
|
93
92
|
const newMaxedQueries = maxedQuery ? maxedQueries.concat(maxedQuery) : maxedQueries;
|
|
94
93
|
|
|
95
94
|
if (typeof nextOffset === 'number') {
|
|
96
95
|
debug(`Next search will be for query ${queryOffset} ${query}, starting from record ${nextOffset}`);
|
|
97
|
-
return {records,
|
|
96
|
+
return {records, queryOffset, resultSetOffset: nextOffset, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
|
|
98
97
|
}
|
|
99
98
|
debug(`Query ${queryOffset} ${query} done.`);
|
|
100
99
|
debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);
|
|
101
|
-
return {records,
|
|
100
|
+
return {records, queryOffset: queryOffset + 1, queriesLeft: queryList.length - (queryOffset + 1), totalRecords: newTotalRecords, query, searchCounter: newSearchCounter, queryCandidateCounter: newQueryCandidateCounter, queryCounter: newQueryCounter, maxedQueries: newMaxedQueries};
|
|
102
101
|
}
|
|
103
102
|
|
|
104
103
|
debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);
|
|
105
|
-
return {records: [],
|
|
106
|
-
|
|
107
|
-
function retrieveRecords() {
|
|
108
|
-
return new Promise((resolve, reject) => {
|
|
109
|
-
const promises = [];
|
|
110
|
-
// eslint-disable-next-line functional/no-let
|
|
111
|
-
let totalRecords = 0;
|
|
112
|
-
|
|
113
|
-
debug(`Searching for candidates with query: ${query} (Offset ${resultSetOffset})`);
|
|
114
|
-
|
|
115
|
-
client.searchRetrieve(query, {startRecord: resultSetOffset})
|
|
116
|
-
.on('error', err => {
|
|
117
|
-
// eslint-disable-next-line functional/no-conditional-statements
|
|
118
|
-
if (err instanceof SruSearchError) {
|
|
119
|
-
debug(`SRU SruSearchError for query: ${query}: ${err}`);
|
|
120
|
-
reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));
|
|
121
|
-
}
|
|
122
|
-
debug(`SRU error for query: ${query}: ${err}`);
|
|
123
|
-
reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));
|
|
124
|
-
})
|
|
125
|
-
.on('total', total => {
|
|
126
|
-
debug(`Got total: ${total}`);
|
|
127
|
-
totalRecords += total;
|
|
128
|
-
})
|
|
129
|
-
.on('end', async nextOffset => {
|
|
130
|
-
try {
|
|
131
|
-
const recordPromises = await Promise.allSettled(promises);
|
|
132
|
-
debugData(`All recordPromises: ${JSON.stringify(recordPromises)}`);
|
|
133
|
-
const filtered = recordPromises.filter(r => r.status === 'fulfilled').map(r => r.value);
|
|
134
|
-
const failures = recordPromises.filter(r => r.status === 'rejected').map(r => ({status: r.reason.status, payload: r.reason.payload}));
|
|
135
|
-
|
|
136
|
-
debug(`Found ${recordPromises.length} records`);
|
|
137
|
-
debug(`Found ${filtered.length} convertable candidates`);
|
|
138
|
-
debug(`Found ${failures.length} NON-convertable candidates`);
|
|
139
|
-
debugData(`Converted: ${JSON.stringify(filtered)}.`);
|
|
140
|
-
debugData(`Not converted: ${JSON.stringify(failures)}.`);
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
resolve({nextOffset, records: filtered, failures, total: totalRecords});
|
|
144
|
-
} catch (err) {
|
|
145
|
-
debug(`Error caught on END`);
|
|
146
|
-
reject(err);
|
|
147
|
-
}
|
|
148
|
-
})
|
|
149
|
-
.on('record', recordXML => {
|
|
150
|
-
promises.push(handleRecord()); // eslint-disable-line functional/immutable-data
|
|
151
|
-
|
|
152
|
-
async function handleRecord() {
|
|
153
|
-
try {
|
|
154
|
-
const recordMarc = await MARCXML.from(recordXML, {subfieldValues: false});
|
|
155
|
-
const recordId = getRecordId(recordMarc);
|
|
156
|
-
|
|
157
|
-
return {record: recordMarc, id: recordId};
|
|
158
|
-
} catch (err) {
|
|
159
|
-
// What should this do?
|
|
160
|
-
const idFromXML = getRecordIdFromXML(recordXML);
|
|
161
|
-
debugData(`Failed converting record: ${err.message}, id: ${idFromXML}, data: ${recordXML}`);
|
|
162
|
-
//return {message: `Failed converting record: ${err.message}`, id: idFromXML, data: recordXML};
|
|
163
|
-
throw new MatchingError(422, {message: `Failed converting record: ${err.message}`, id: idFromXML || '000000000', data: recordXML});
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
});
|
|
167
|
-
});
|
|
168
|
-
}
|
|
104
|
+
return {records: [], queriesLeft: 0, queryCounter, maxedQueries};
|
|
169
105
|
}
|
|
170
106
|
|
|
171
107
|
function checkMaxedQuery(query, total, serverMaxResult) {
|
|
@@ -179,10 +115,48 @@ export default async ({record, searchSpec, url, maxCandidates, maxRecordsPerRequ
|
|
|
179
115
|
const [field] = record.get(/^001$/u);
|
|
180
116
|
return field ? field.value : '';
|
|
181
117
|
}
|
|
182
|
-
|
|
183
|
-
function getRecordIdFromXML(recordXML) {
|
|
184
|
-
//<controlfield tag=\"001\">015376846</controlfield
|
|
185
|
-
debug(`Cannot yet find possible database record id from recordXML (length ${recordXML.length})`);
|
|
186
|
-
return undefined;
|
|
187
|
-
}
|
|
188
118
|
};
|
|
119
|
+
|
|
120
|
+
export function retrieveRecords(client, query, resultSetOffset) {
|
|
121
|
+
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:retrieveRecords');
|
|
122
|
+
// eslint-disable-next-line no-unused-vars
|
|
123
|
+
const debugData = debug.extend('data');
|
|
124
|
+
|
|
125
|
+
return new Promise((resolve, reject) => {
|
|
126
|
+
const records = [];
|
|
127
|
+
let totalRecords = 0;
|
|
128
|
+
|
|
129
|
+
debug(`Searching for candidates with query: ${query} (Offset ${resultSetOffset})`);
|
|
130
|
+
|
|
131
|
+
client.searchRetrieve(query, {startRecord: resultSetOffset})
|
|
132
|
+
.on('error', err => {
|
|
133
|
+
if (err instanceof SruSearchError) {
|
|
134
|
+
debug(`SRU SruSearchError for query: ${query}: ${err}`);
|
|
135
|
+
reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
debug(`SRU error for query: ${query}: ${err}`);
|
|
139
|
+
reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));
|
|
140
|
+
})
|
|
141
|
+
.on('total', total => {
|
|
142
|
+
debug(`Got total: ${total}`);
|
|
143
|
+
totalRecords += total;
|
|
144
|
+
})
|
|
145
|
+
.on('end', nextOffset => {
|
|
146
|
+
try {
|
|
147
|
+
debug(`Found ${records.length} records`);
|
|
148
|
+
|
|
149
|
+
resolve({nextOffset, records, total: totalRecords});
|
|
150
|
+
} catch (err) {
|
|
151
|
+
debug(`Error caught on END`);
|
|
152
|
+
reject(err);
|
|
153
|
+
}
|
|
154
|
+
})
|
|
155
|
+
.on('record', record => {
|
|
156
|
+
const [field] = record.get(/^001$/u);
|
|
157
|
+
debug(field);
|
|
158
|
+
const id = field.value ? field.value : '';
|
|
159
|
+
records.push({record, id});
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import
|
|
1
|
+
import assert from 'node:assert';
|
|
2
|
+
import {describe} from 'node:test';
|
|
2
3
|
import {READERS} from '@natlibfi/fixura';
|
|
3
4
|
import generateTests from '@natlibfi/fixugen-http-client';
|
|
4
5
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
5
6
|
import {Error as MatchingError} from '@natlibfi/melinda-commons';
|
|
6
|
-
import createSearchInterface, {CandidateSearchError} from '.';
|
|
7
|
+
import createSearchInterface, {CandidateSearchError} from './index.js';
|
|
7
8
|
import createDebugLogger from 'debug';
|
|
8
9
|
|
|
9
10
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:test');
|
|
@@ -11,7 +12,7 @@ const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-sea
|
|
|
11
12
|
describe('candidate-search', () => {
|
|
12
13
|
generateTests({
|
|
13
14
|
callback,
|
|
14
|
-
path: [
|
|
15
|
+
path: [import.meta.dirname, '..', '..', 'test-fixtures', 'candidate-search', 'index'],
|
|
15
16
|
recurse: false,
|
|
16
17
|
fixura: {
|
|
17
18
|
reader: READERS.JSON
|
|
@@ -30,26 +31,28 @@ describe('candidate-search', () => {
|
|
|
30
31
|
debug(`We're expecting an error`);
|
|
31
32
|
if (expectedFactoryError.isCandidateSearchError) {
|
|
32
33
|
try {
|
|
33
|
-
const result = createSearchInterface({...formatFactoryOptions(), url});
|
|
34
|
+
const result = await createSearchInterface({...formatFactoryOptions(), url});
|
|
34
35
|
debug(result);
|
|
35
36
|
} catch (err) {
|
|
36
|
-
|
|
37
|
+
assert.equal(err instanceof CandidateSearchError, true);
|
|
38
|
+
assert.match(err.message, new RegExp(expectedFactoryError.message));
|
|
39
|
+
} finally {
|
|
40
|
+
return;
|
|
37
41
|
}
|
|
38
|
-
return;
|
|
39
42
|
}
|
|
40
43
|
|
|
41
44
|
try {
|
|
42
|
-
const result = createSearchInterface({...formatFactoryOptions(), url});
|
|
45
|
+
const result = await createSearchInterface({...formatFactoryOptions(), url});
|
|
43
46
|
debug(result);
|
|
44
47
|
} catch (err) {
|
|
45
|
-
|
|
48
|
+
assert.equal(err instanceof Error, true);
|
|
49
|
+
assert.match(err.message, new RegExp(expectedFactoryError.message));
|
|
50
|
+
} finally {
|
|
51
|
+
return;
|
|
46
52
|
}
|
|
47
|
-
return;
|
|
48
53
|
}
|
|
49
54
|
|
|
50
55
|
const {search} = await createSearchInterface({...formatFactoryOptions(), url});
|
|
51
|
-
// eslint-disable-next-line no-console
|
|
52
|
-
console.log(search);
|
|
53
56
|
await iterate({searchOptions, expectedSearchError});
|
|
54
57
|
|
|
55
58
|
function formatFactoryOptions() {
|
|
@@ -65,30 +68,29 @@ describe('candidate-search', () => {
|
|
|
65
68
|
async function iterate({searchOptions, expectedSearchError, expectedErrorStatus, count = 1}) {
|
|
66
69
|
const expectedResults = getFixture(`expectedResults${count}.json`);
|
|
67
70
|
|
|
68
|
-
if (expectedSearchError) {
|
|
71
|
+
if (expectedSearchError) {
|
|
69
72
|
try {
|
|
70
73
|
await search(searchOptions);
|
|
71
74
|
throw new Error('Expected an error');
|
|
72
75
|
} catch (err) {
|
|
73
76
|
debug(`Got an error: ${err}`);
|
|
74
|
-
|
|
77
|
+
assert(err instanceof Error);
|
|
75
78
|
const errorMessage = err instanceof MatchingError ? err.payload.message : err.message;
|
|
76
79
|
const errorStatus = err instanceof MatchingError ? err.status : undefined;
|
|
77
80
|
debug(`errorMessage: ${errorMessage}, errorStatus: ${errorStatus}`);
|
|
78
|
-
|
|
81
|
+
assert.match(errorMessage, new RegExp(expectedSearchError, 'u'));
|
|
79
82
|
|
|
80
83
|
if (expectedErrorStatus) {
|
|
81
|
-
|
|
84
|
+
assert.equal(errorStatus, expectedErrorStatus);
|
|
82
85
|
return;
|
|
83
86
|
}
|
|
84
87
|
return;
|
|
85
88
|
}
|
|
86
89
|
}
|
|
87
90
|
|
|
88
|
-
// eslint-disable-next-line functional/no-conditional-statements
|
|
89
91
|
if (!expectedSearchError) {
|
|
90
92
|
const results = await search(searchOptions);
|
|
91
|
-
|
|
93
|
+
assert.deepStrictEqual(formatResults(results), expectedResults);
|
|
92
94
|
}
|
|
93
95
|
|
|
94
96
|
function formatResults(results) {
|
|
@@ -98,7 +100,6 @@ describe('candidate-search', () => {
|
|
|
98
100
|
records: results.records.map(({record, id}) => ({id, record: record.toObject()}))
|
|
99
101
|
};
|
|
100
102
|
}
|
|
101
|
-
|
|
102
103
|
}
|
|
103
104
|
}
|
|
104
105
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import createDebugLogger from 'debug';
|
|
2
|
-
import {toQueries} from '../candidate-search-utils';
|
|
3
|
-
import {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues, testStringOrNumber} from '../../matching-utils';
|
|
2
|
+
import {toQueries} from '../candidate-search-utils.js';
|
|
3
|
+
import {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues, testStringOrNumber} from '../../matching-utils.js';
|
|
4
4
|
|
|
5
5
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query');
|
|
6
6
|
|
|
@@ -372,7 +372,6 @@ export function bibYear(record) {
|
|
|
372
372
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibYear');
|
|
373
373
|
const debugData = debug.extend('data');
|
|
374
374
|
debug(`Creating query for the publishing year`);
|
|
375
|
-
debugData(record);
|
|
376
375
|
|
|
377
376
|
const year = getYear(record);
|
|
378
377
|
if (year) {
|
|
@@ -381,7 +380,14 @@ export function bibYear(record) {
|
|
|
381
380
|
return [];
|
|
382
381
|
|
|
383
382
|
function getYear(record) {
|
|
384
|
-
const
|
|
383
|
+
const [f008] = record.get(/^008$/u);
|
|
384
|
+
if (f008 === undefined) {
|
|
385
|
+
debug('f008 missing');
|
|
386
|
+
return false;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
debugData(`f008: ${JSON.stringify(f008)}`);
|
|
390
|
+
const {value} = f008;
|
|
385
391
|
return testStringOrNumber(value) ? String(value).slice(7, 11) : undefined;
|
|
386
392
|
}
|
|
387
393
|
}
|
|
@@ -1,16 +1,17 @@
|
|
|
1
|
+
import assert from 'node:assert';
|
|
2
|
+
import {describe} from 'node:test';
|
|
3
|
+
import createDebugLogger from 'debug';
|
|
1
4
|
import generateTests from '@natlibfi/fixugen';
|
|
2
5
|
import {READERS} from '@natlibfi/fixura';
|
|
3
|
-
import {expect} from 'chai';
|
|
4
6
|
import {MarcRecord} from '@natlibfi/marc-record';
|
|
5
|
-
import * as generators from './bib';
|
|
6
|
-
import createDebugLogger from 'debug';
|
|
7
|
+
import * as generators from './bib.js';
|
|
7
8
|
|
|
8
9
|
const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibHostComponents:test');
|
|
9
10
|
const debugData = debug.extend('data');
|
|
10
11
|
|
|
11
12
|
describe('candidate-search/query-list/bib/', () => {
|
|
12
13
|
generateTests({
|
|
13
|
-
path: [
|
|
14
|
+
path: [import.meta.dirname, '..', '..', '..', 'test-fixtures', 'candidate-search', 'query-list', 'bib'],
|
|
14
15
|
useMetadataFile: true,
|
|
15
16
|
fixura: {
|
|
16
17
|
reader: READERS.JSON
|
|
@@ -27,11 +28,11 @@ describe('candidate-search/query-list/bib/', () => {
|
|
|
27
28
|
debugData(`Result: ${JSON.stringify(result)}`);
|
|
28
29
|
|
|
29
30
|
if (result.queryListType) {
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
assert.deepStrictEqual(result.queryList, expectedQuery);
|
|
32
|
+
assert.equal(result.queryListType, expectedQueryListType);
|
|
32
33
|
return;
|
|
33
34
|
}
|
|
34
|
-
|
|
35
|
+
assert.deepStrictEqual(result, expectedQuery);
|
|
35
36
|
}
|
|
36
37
|
});
|
|
37
38
|
});
|