@natlibfi/melinda-rest-api-commons 6.0.1 → 6.1.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/amqp.js +37 -15
- package/dist/amqp.js.map +2 -2
- package/dist/helpers/conversion.js +1 -2
- package/dist/helpers/conversion.js.map +2 -2
- package/dist/helpers/fix-strip-f884s.js +18 -4
- package/dist/helpers/fix-strip-f884s.js.map +2 -2
- package/dist/helpers/fix.js +12 -8
- package/dist/helpers/fix.js.map +2 -2
- package/dist/helpers/validation.js +1 -1
- package/dist/helpers/validation.js.map +1 -1
- package/dist/mongo.js +200 -109
- package/dist/mongo.js.map +3 -3
- package/dist/mongoLog.js +113 -35
- package/dist/mongoLog.js.map +3 -3
- package/dist/utils.js +33 -4
- package/dist/utils.js.map +2 -2
- package/package.json +11 -17
- package/tests/amqp-error-branches.test.js +535 -0
- package/tests/amqp.healthtest.js +12 -4
- package/tests/amqp.tests.localtest.js +13 -18
- package/tests/helpers/conversion.test.js +16 -0
- package/tests/helpers/fix-edge.test.js +197 -0
- package/tests/helpers/fix.test.js +20 -2
- package/tests/helpers/validation.test.js +6 -2
- package/tests/mongo-check-and-check-state.test.js +3 -1
- package/tests/mongo-check-and-set-import-job-state.test.js +3 -1
- package/tests/mongo-error-branches.test.js +787 -0
- package/tests/mongo-get-stream.test.js +4 -1
- package/tests/mongo-read-content.test.js +10 -0
- package/tests/mongoLog-protect.test.js +181 -0
- package/tests/mongoLog.test.js +113 -6
- package/tests/utils.test.js +360 -0
package/README.md
CHANGED
|
@@ -35,6 +35,22 @@ Shared modules for microservices of Melinda rest api import system
|
|
|
35
35
|
|
|
36
36
|
*IMPORT_JOB_STATEs: EMPTY or IN_QUEUE, PROCESSING -> final: DONE, ERROR, ABORT
|
|
37
37
|
|
|
38
|
+
#### Timeout (ABORT) contract
|
|
39
|
+
|
|
40
|
+
A queue item that has not made progress for longer than the timeout window is
|
|
41
|
+
set to `ABORT` (with `errorStatus` 408 and `errorMessage`
|
|
42
|
+
`Timeout in <state>`). The window is the time since the last write to the
|
|
43
|
+
item's `modificationTime` (each Mongo write done by the importer -
|
|
44
|
+
`pushMessages`, `pushIds`, `addBlobSize`, `setBlobSize`, `setState`,
|
|
45
|
+
`setImportJobState` - refreshes it). The window defaults to **1 minute** and
|
|
46
|
+
can be made longer with the `MONGO_TIMEOUT_MINUTES` env var, which large bulk
|
|
47
|
+
imports (see the CHUNK_SIZE benchmark below: ~9248 records / ~597 sec) may
|
|
48
|
+
need:
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
MONGO_TIMEOUT_MINUTES=10 # abort only after 10 minutes without progress
|
|
52
|
+
```
|
|
53
|
+
|
|
38
54
|
### CHUNK_SIZE
|
|
39
55
|
Chunck size to execution time ratio (From file to queue: ~450 records in 15sec)
|
|
40
56
|
|
|
@@ -63,6 +79,24 @@ Executed on test server using bridge to docker server and back to server
|
|
|
63
79
|
| p_z07_priority_year | pZ07PriorityYear | 1998 | 1990 | 2099 | 2099 | Override indexing priority |
|
|
64
80
|
| p_redirection_field | pRedirectionField | | | | | |
|
|
65
81
|
|
|
82
|
+
## Running the tests
|
|
83
|
+
|
|
84
|
+
There are three test suites. The base suite runs anywhere; the two AMQP
|
|
85
|
+
suites need a running RabbitMQ broker on `127.0.0.1:5672` (guest/guest) and
|
|
86
|
+
are therefore excluded from the base glob and from CI.
|
|
87
|
+
|
|
88
|
+
| Command | What it runs | Requirements |
|
|
89
|
+
|-------------------------|---------------------------------------------------------------------|-----------------------|
|
|
90
|
+
| `npm run test:base` | all `tests/**/*.test.js` with coverage + thresholds | none |
|
|
91
|
+
| `npm test` | `npm run lint` then `test:base` (what CI runs) | none |
|
|
92
|
+
| `npm run test:local` | `tests/amqp.*.localtest.js`, AMQP operator against a live broker | RabbitMQ on localhost |
|
|
93
|
+
| `npm run test:healthcheck` | `tests/amqp.healthtest.js`, healthCheck loop vs live broker | RabbitMQ on localhost |
|
|
94
|
+
|
|
95
|
+
`test:base` enforces minimum coverage thresholds via
|
|
96
|
+
`--test-coverage-lines/--test-coverage-branches/--test-coverage-functions`
|
|
97
|
+
(currently 88 / 88 / 90 over `src`); raise them in [`package.json`](package.json)
|
|
98
|
+
as coverage grows.
|
|
99
|
+
|
|
66
100
|
## License and copyright
|
|
67
101
|
|
|
68
102
|
Copyright (c) 2020-2026 **University Of Helsinki (The National Library Of Finland)**
|
package/dist/amqp.js
CHANGED
|
@@ -3,35 +3,41 @@ import createDebugLogger from "debug";
|
|
|
3
3
|
import { promisify, inspect } from "util";
|
|
4
4
|
import httpStatus from "http-status";
|
|
5
5
|
import { MarcRecord } from "@natlibfi/marc-record";
|
|
6
|
-
import { Error as ApiError } from "@natlibfi/melinda-commons";
|
|
7
|
-
import { createLogger } from "@natlibfi/melinda-backend-commons";
|
|
6
|
+
import { createLogger, Error as ApiError } from "@natlibfi/melinda-commons";
|
|
8
7
|
import { CHUNK_SIZE } from "./constants.js";
|
|
9
8
|
import { logError } from "./utils.js";
|
|
10
|
-
export default async function(AMQP_URL, runHealthCheck = false) {
|
|
9
|
+
export default async function(AMQP_URL, runHealthCheck = false, deps = {}) {
|
|
11
10
|
const logger = createLogger();
|
|
12
11
|
const debug = createDebugLogger("@natlibfi/melinda-rest-api-commons:amqp");
|
|
13
12
|
const debugHC = debug.extend("HC");
|
|
14
13
|
const debugData = debug.extend("data");
|
|
15
14
|
const setTimeoutPromise = promisify(setTimeout);
|
|
16
15
|
debug(`Creating an AMQP operator to ${AMQP_URL}`);
|
|
17
|
-
const connection = await amqplib.connect(AMQP_URL);
|
|
18
|
-
const channel = await connection.createChannel();
|
|
16
|
+
const connection = deps.connection ?? await amqplib.connect(AMQP_URL);
|
|
17
|
+
const channel = deps.channel ?? await connection.createChannel();
|
|
18
|
+
let healthCheckRunning = runHealthCheck;
|
|
19
19
|
const healthCheckLoop = runHealthCheck ? healthCheck() : false;
|
|
20
20
|
debug(`Connection: ${connection}`);
|
|
21
21
|
debug(`Channel: ${channel}`);
|
|
22
22
|
debug(`HealthCheckLoop: ${healthCheckLoop ? "Running health check" : "Not running health check"}`);
|
|
23
23
|
return { checkQueue, consumeChunk, consumeOne, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords, closeChannel, closeConnection };
|
|
24
24
|
async function closeChannel() {
|
|
25
|
+
healthCheckRunning = false;
|
|
25
26
|
debug(`Closing channel`);
|
|
26
27
|
await channel.close();
|
|
27
28
|
debug(`Channel: ${channel}`);
|
|
28
29
|
}
|
|
29
30
|
async function closeConnection() {
|
|
31
|
+
healthCheckRunning = false;
|
|
30
32
|
debug(`Closing connection`);
|
|
31
33
|
await connection.close();
|
|
32
34
|
debug(`Connection: ${connection}`);
|
|
33
35
|
}
|
|
34
36
|
async function healthCheck(wait = false) {
|
|
37
|
+
if (!healthCheckRunning) {
|
|
38
|
+
debugHC(`HealthCheck cancelled, not running`);
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
35
41
|
if (wait) {
|
|
36
42
|
await setTimeoutPromise(wait);
|
|
37
43
|
return healthCheck(false);
|
|
@@ -43,6 +49,10 @@ export default async function(AMQP_URL, runHealthCheck = false) {
|
|
|
43
49
|
return healthCheck(200);
|
|
44
50
|
} catch (error) {
|
|
45
51
|
debugHC(`HealthCheck error ${JSON.stringify(error)}`);
|
|
52
|
+
if (!healthCheckRunning) {
|
|
53
|
+
debugHC(`HealthCheck cancelled while erroring, not throwing`);
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
46
56
|
handleAmqpErrors(error);
|
|
47
57
|
}
|
|
48
58
|
}
|
|
@@ -96,10 +106,14 @@ export default async function(AMQP_URL, runHealthCheck = false) {
|
|
|
96
106
|
try {
|
|
97
107
|
await channel.assertQueue(queue, { durable: true });
|
|
98
108
|
const messages = await getData(queue);
|
|
109
|
+
if (messages.length === 0) {
|
|
110
|
+
debug(`consumeChunk: queue ${queue} drained, returning false`);
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
99
113
|
const headers = getHeaderInfo(messages[0]);
|
|
100
114
|
debug(`consumeChunk (${messages ? messages.length : "0"} from queue ${queue}) ${toRecord ? "to records" : "just messages"}`);
|
|
101
115
|
if (toRecord) {
|
|
102
|
-
const records =
|
|
116
|
+
const records = messagesToRecords(messages);
|
|
103
117
|
return { headers, records, messages };
|
|
104
118
|
}
|
|
105
119
|
return { headers, messages };
|
|
@@ -167,15 +181,17 @@ export default async function(AMQP_URL, runHealthCheck = false) {
|
|
|
167
181
|
}
|
|
168
182
|
}
|
|
169
183
|
async function removeQueue(queue) {
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
await tempChannel.
|
|
176
|
-
|
|
184
|
+
let tempChannel;
|
|
185
|
+
try {
|
|
186
|
+
tempChannel = await connection.createChannel();
|
|
187
|
+
logger.verbose(`Removing queue ${queue}.`);
|
|
188
|
+
debug(`Removing queue ${queue}.`);
|
|
189
|
+
await tempChannel.deleteQueue(queue);
|
|
190
|
+
} finally {
|
|
191
|
+
if (tempChannel) {
|
|
192
|
+
await tempChannel.close();
|
|
193
|
+
}
|
|
177
194
|
}
|
|
178
|
-
return;
|
|
179
195
|
}
|
|
180
196
|
function messagesToRecords(messages) {
|
|
181
197
|
debug(`Parsing messages (${messages.length}) to records`);
|
|
@@ -202,11 +218,17 @@ export default async function(AMQP_URL, runHealthCheck = false) {
|
|
|
202
218
|
return results;
|
|
203
219
|
}
|
|
204
220
|
const message = await channel.get(queue);
|
|
221
|
+
if (!message) {
|
|
222
|
+
return results;
|
|
223
|
+
}
|
|
205
224
|
const identifier = {
|
|
206
225
|
correlationId: message.properties.correlationId,
|
|
207
226
|
deliveryTag: message.fields.deliveryTag
|
|
208
227
|
};
|
|
209
|
-
|
|
228
|
+
const isDuplicate = identifiers.some(
|
|
229
|
+
(i) => i.correlationId === identifier.correlationId && i.deliveryTag === identifier.deliveryTag
|
|
230
|
+
);
|
|
231
|
+
if (isDuplicate) {
|
|
210
232
|
return pump(count - 1, results, identifiers);
|
|
211
233
|
}
|
|
212
234
|
return pump(count - 1, results.concat(message), identifiers.concat(identifier));
|
package/dist/amqp.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/amqp.js"],
|
|
4
|
-
"sourcesContent": ["import amqplib from 'amqplib';\nimport createDebugLogger from 'debug';\nimport {promisify, inspect} from 'util';\nimport httpStatus from 'http-status';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {CHUNK_SIZE} from './constants.js';\nimport {logError} from './utils.js';\n\nexport default async function (AMQP_URL, runHealthCheck = false) {\n const logger = createLogger();\n const debug = createDebugLogger('@natlibfi/melinda-rest-api-commons:amqp');\n const debugHC = debug.extend('HC');\n const debugData = debug.extend('data');\n const setTimeoutPromise = promisify(setTimeout);\n\n debug(`Creating an AMQP operator to ${AMQP_URL}`);\n const connection = await amqplib.connect(AMQP_URL);\n const channel = await connection.createChannel();\n\n const healthCheckLoop = runHealthCheck ? healthCheck() : false;\n\n debug(`Connection: ${connection}`);\n debug(`Channel: ${channel}`);\n debug(`HealthCheckLoop: ${healthCheckLoop ? 'Running health check' : 'Not running health check'}`);\n\n return {checkQueue, consumeChunk, consumeOne, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords, closeChannel, closeConnection};\n\n async function closeChannel() {\n debug(`Closing channel`);\n await channel.close();\n debug(`Channel: ${channel}`);\n }\n\n async function closeConnection() {\n debug(`Closing connection`);\n await connection.close();\n debug(`Connection: ${connection}`);\n }\n\n async function healthCheck(wait = false) {\n if (wait) {\n await setTimeoutPromise(wait);\n return healthCheck(false);\n }\n\n try {\n debugHC(`Health checking amqp by asserting a queue`);\n await channel.assertQueue('HEALTHCHECK', {durable: false});\n debugHC(`Waiting 200ms before running healthCheck next`);\n return healthCheck(200);\n } catch (error) {\n debugHC(`HealthCheck error ${JSON.stringify(error)}`);\n handleAmqpErrors(error);\n }\n }\n\n async function checkQueue({queue, style = 'basic', toRecord = true, purge = false}) {\n debug(`checkQueue: ${queue}, Style: ${style}, toRecord: ${toRecord}, Purge: ${purge}`);\n\n try {\n\n errorUndefinedQueue(queue);\n const channelInfo = await channel.assertQueue(queue, {durable: true});\n\n if (purge) {\n await purgeQueue(purge);\n logger.verbose(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n return checkQueue({queue, style});\n }\n\n if (style === 'messages') {\n return channelInfo.messageCount;\n }\n\n\n // Note: returns one message (+ record, of toRecord: true)\n // note: if toRecord is false returns just plain message / false\n // note: if toRecord is true returns {headers, records, messages} -object / false\n // should this be more consistent?\n if (style === 'one') {\n if (checkMessageCount(channelInfo)) {\n return consumeOne(queue, toRecord);\n }\n return false;\n }\n\n // Note: returns a chunk of (100) messages (+ records, if toRecord: true)\n // returns {headers, records, messages} object or {headers, messages} object depending on toRecord\n if (style === 'basic') {\n if (checkMessageCount(channelInfo)) {\n return consumeChunk(queue, toRecord);\n }\n return false;\n }\n\n // Defaults:\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `CheckQueue does not recognize style ${style}`);\n } catch (error) {\n handleAmqpErrors(error);\n }\n\n function checkMessageCount(channelInfo) {\n if (channelInfo.messageCount < 1) {\n debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n return false;\n }\n debug(`Queue ${queue} has ${channelInfo.messageCount} messages`);\n return true;\n }\n\n function purgeQueue(purge) {\n debug(`Purging queue: ${queue}, purge: ${purge}`);\n if (purge) {\n return channel.purgeQueue(queue);\n }\n }\n }\n\n async function consumeChunk(queue, toRecord) {\n debug(`Prepared to consumeChunk from queue: ${queue}`);\n try {\n await channel.assertQueue(queue, {durable: true});\n\n // getData: next chunk (100) messages\n const messages = await getData(queue);\n\n // Note: headers are from the first message of the chunk\n const headers = getHeaderInfo(messages[0]);\n debug(`consumeChunk (${messages ? messages.length : '0'} from queue ${queue}) ${toRecord ? 'to records' : 'just messages'}`);\n\n if (toRecord) {\n const records = await messagesToRecords(messages);\n return {headers, records, messages};\n }\n\n return {headers, messages};\n } catch (error) {\n handleAmqpErrors(error);\n }\n }\n\n async function consumeOne(queue, toRecord) {\n debug(`Prepared to consumeOne from queue: ${queue}`);\n try {\n await channel.assertQueue(queue, {durable: true});\n\n // Returns false if 0 items in queue\n const message = await channel.get(queue);\n\n debugData(`Message: ${inspect(message, {colors: true, maxArrayLength: 3, depth: 3})}`);\n // Do not spam the logs\n debug(`consumeOne from queue: ${queue} ${toRecord ? 'to records' : 'just the message'}`);\n\n if (message) {\n if (toRecord) {\n const headers = getHeaderInfo(message);\n const records = messagesToRecords([message]);\n return {headers, records, messages: [message]};\n }\n\n return message;\n }\n\n return false;\n } catch (error) {\n handleAmqpErrors(error);\n }\n }\n\n function ackMessages(messages) {\n messages.forEach(message => {\n debug(`Ack message ${message.properties.correlationId}`);\n channel.ack(message);\n });\n }\n\n function nackMessages(messages) {\n messages.forEach(message => {\n debug(`Nack message ${message.properties.correlationId}`);\n channel.nack(message);\n });\n }\n\n async function sendToQueue({queue, correlationId, headers, data}) {\n debug(`sendToQueue`);\n try {\n debug(`Queue ${queue}`);\n debug(`CorrelationId ${correlationId}`);\n debugData(`Data ${JSON.stringify(data)}`);\n debug(`Headers ${JSON.stringify(headers)}`);\n\n errorUndefinedQueue(queue);\n\n debug(`Asserting queue: ${queue}`);\n await channel.assertQueue(queue, {durable: true});\n\n debug(`Actually sendToQueue: ${queue}`);\n await channel.sendToQueue(\n queue,\n Buffer.from(JSON.stringify({data})),\n {\n correlationId,\n persistent: true,\n headers\n }\n );\n debug(`Send message for ${correlationId} to queue: ${queue}`);\n\n\n } catch (error) {\n const errorToThrow = error;\n //if (error instanceof ApiError) {\n // throw error;\n //}\n debug(`SendToQueue errored: ${JSON.stringify(error)}`);\n handleAmqpErrors(errorToThrow);\n }\n }\n\n async function removeQueue(queue) {\n // deleteQueue borks the channel if the queue does not exist\n // -> use throwaway tempChannel to avoid killing actual channel in use\n // this might be doable also with assertQueue before deleteQueue\n const tempChannel = await connection.createChannel();\n logger.verbose(`Removing queue ${queue}.`);\n debug(`Removing queue ${queue}.`);\n await tempChannel.deleteQueue(queue);\n\n if (tempChannel) {\n await tempChannel.close();\n return;\n }\n\n return;\n }\n\n // ----------------\n // Helper functions\n // ----------------\n\n function messagesToRecords(messages) {\n debug(`Parsing messages (${messages.length}) to records`);\n\n return messages.map(message => {\n const content = JSON.parse(message.content.toString());\n // Use subfieldValues: false validationOption here\n return new MarcRecord(content.data, {subfieldValues: false});\n });\n }\n\n async function getData(queue) {\n debug(`Getting queue data from ${queue}`);\n try {\n const {messageCount} = await channel.checkQueue(queue);\n debug(`There is ${messageCount} messages in queue ${queue}`);\n const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n debug(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n const messages = await pump(messagesToGet);\n\n debug(`Returning ${messages.length} unique messages`);\n\n return messages;\n } catch (error) {\n handleAmqpErrors(error);\n }\n\n async function pump(count, results = [], identifiers = []) {\n if (count === 0) {\n return results;\n }\n\n const message = await channel.get(queue);\n const identifier = {\n correlationId: message.properties.correlationId,\n deliveryTag: message.fields.deliveryTag\n };\n // Filter not unique messages\n if (identifiers.includes(identifier)) {\n return pump(count - 1, results, identifiers);\n }\n\n return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n }\n }\n\n function getHeaderInfo(data) {\n return data.properties.headers;\n }\n\n function errorUndefinedQueue(queue) {\n if (queue === undefined || queue === '' || queue === null) {\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Undefined queue!`);\n }\n }\n\n function handleAmqpErrors(error) {\n const newError = error;\n debug(`HandleAmqpErrors got an error: ${JSON.stringify(error)}`);\n if (error instanceof ApiError) {\n debug(`We have an ApiError`);\n throw new ApiError(newError.status, newError.payload);\n }\n debug(`We have a non-ApiError`);\n logError(error);\n throw newError;\n }\n}\n\n\n"],
|
|
5
|
-
"mappings": "AAAA,OAAO,aAAa;AACpB,OAAO,uBAAuB;AAC9B,SAAQ,WAAW,eAAc;AACjC,OAAO,gBAAgB;AACvB,SAAQ,kBAAiB;AACzB,SAAQ,SAAS,gBAAe;
|
|
4
|
+
"sourcesContent": ["import amqplib from 'amqplib';\nimport createDebugLogger from 'debug';\nimport {promisify, inspect} from 'util';\nimport httpStatus from 'http-status';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {createLogger, Error as ApiError} from '@natlibfi/melinda-commons';\nimport {CHUNK_SIZE} from './constants.js';\nimport {logError} from './utils.js';\n\nexport default async function (AMQP_URL, runHealthCheck = false, deps = {}) {\n const logger = createLogger();\n const debug = createDebugLogger('@natlibfi/melinda-rest-api-commons:amqp');\n const debugHC = debug.extend('HC');\n const debugData = debug.extend('data');\n const setTimeoutPromise = promisify(setTimeout);\n\n debug(`Creating an AMQP operator to ${AMQP_URL}`);\n // deps allows injecting stubbed connection/channel in unit tests\n const connection = deps.connection ?? await amqplib.connect(AMQP_URL);\n const channel = deps.channel ?? await connection.createChannel();\n\n // the recursive healthCheck loop is never awaited and kept firing\n // ~every 200 ms even after the connection was closed. This flag is cleared\n // by closeChannel/closeConnection so the loop short-circuits on graceful\n // shutdown instead of running until the closed channel throws.\n let healthCheckRunning = runHealthCheck;\n const healthCheckLoop = runHealthCheck ? healthCheck() : false;\n\n debug(`Connection: ${connection}`);\n debug(`Channel: ${channel}`);\n debug(`HealthCheckLoop: ${healthCheckLoop ? 'Running health check' : 'Not running health check'}`);\n\n return {checkQueue, consumeChunk, consumeOne, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords, closeChannel, closeConnection};\n\n async function closeChannel() {\n // stop the health check loop before closing so it does not keep\n // firing on a closed channel during graceful shutdown\n healthCheckRunning = false;\n debug(`Closing channel`);\n await channel.close();\n debug(`Channel: ${channel}`);\n }\n\n async function closeConnection() {\n // stop the health check loop before closing\n healthCheckRunning = false;\n debug(`Closing connection`);\n await connection.close();\n debug(`Connection: ${connection}`);\n }\n\n async function healthCheck(wait = false) {\n // cancellable - closeChannel/closeConnection set this to false\n if (!healthCheckRunning) {\n debugHC(`HealthCheck cancelled, not running`);\n return false;\n }\n if (wait) {\n await setTimeoutPromise(wait);\n return healthCheck(false);\n }\n\n try {\n debugHC(`Health checking amqp by asserting a queue`);\n await channel.assertQueue('HEALTHCHECK', {durable: false});\n debugHC(`Waiting 200ms before running healthCheck next`);\n return healthCheck(200);\n } catch (error) {\n debugHC(`HealthCheck error ${JSON.stringify(error)}`);\n // if we were asked to stop, do not rethrow - a closed channel\n // throwing during shutdown would keep the process alive / crash it\n if (!healthCheckRunning) {\n debugHC(`HealthCheck cancelled while erroring, not throwing`);\n return false;\n }\n handleAmqpErrors(error);\n }\n }\n\n async function checkQueue({queue, style = 'basic', toRecord = true, purge = false}) {\n debug(`checkQueue: ${queue}, Style: ${style}, toRecord: ${toRecord}, Purge: ${purge}`);\n\n try {\n\n errorUndefinedQueue(queue);\n const channelInfo = await channel.assertQueue(queue, {durable: true});\n\n if (purge) {\n await purgeQueue(purge);\n logger.verbose(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n return checkQueue({queue, style});\n }\n\n if (style === 'messages') {\n return channelInfo.messageCount;\n }\n\n\n // Note: returns one message (+ record, of toRecord: true)\n // note: if toRecord is false returns just plain message / false\n // note: if toRecord is true returns {headers, records, messages} -object / false\n // should this be more consistent?\n if (style === 'one') {\n if (checkMessageCount(channelInfo)) {\n return consumeOne(queue, toRecord);\n }\n return false;\n }\n\n // Note: returns a chunk of (100) messages (+ records, if toRecord: true)\n // returns {headers, records, messages} object or {headers, messages} object depending on toRecord\n if (style === 'basic') {\n if (checkMessageCount(channelInfo)) {\n return consumeChunk(queue, toRecord);\n }\n return false;\n }\n\n // Defaults:\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `CheckQueue does not recognize style ${style}`);\n } catch (error) {\n handleAmqpErrors(error);\n }\n\n function checkMessageCount(channelInfo) {\n if (channelInfo.messageCount < 1) {\n debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n return false;\n }\n debug(`Queue ${queue} has ${channelInfo.messageCount} messages`);\n return true;\n }\n\n function purgeQueue(purge) {\n debug(`Purging queue: ${queue}, purge: ${purge}`);\n if (purge) {\n return channel.purgeQueue(queue);\n }\n }\n }\n\n async function consumeChunk(queue, toRecord) {\n debug(`Prepared to consumeChunk from queue: ${queue}`);\n try {\n await channel.assertQueue(queue, {durable: true});\n\n // getData: next chunk (100) messages\n const messages = await getData(queue);\n\n // queue can drain before/during the pump -> nothing to return,\n // matches the `false` contract checkQueue uses for an empty queue\n if (messages.length === 0) {\n debug(`consumeChunk: queue ${queue} drained, returning false`);\n return false;\n }\n\n // Note: headers are from the first message of the chunk\n const headers = getHeaderInfo(messages[0]);\n debug(`consumeChunk (${messages ? messages.length : '0'} from queue ${queue}) ${toRecord ? 'to records' : 'just messages'}`);\n\n if (toRecord) {\n const records = messagesToRecords(messages);\n return {headers, records, messages};\n }\n\n return {headers, messages};\n } catch (error) {\n handleAmqpErrors(error);\n }\n }\n\n async function consumeOne(queue, toRecord) {\n debug(`Prepared to consumeOne from queue: ${queue}`);\n try {\n await channel.assertQueue(queue, {durable: true});\n\n // Returns false if 0 items in queue\n const message = await channel.get(queue);\n\n debugData(`Message: ${inspect(message, {colors: true, maxArrayLength: 3, depth: 3})}`);\n // Do not spam the logs\n debug(`consumeOne from queue: ${queue} ${toRecord ? 'to records' : 'just the message'}`);\n\n if (message) {\n if (toRecord) {\n const headers = getHeaderInfo(message);\n const records = messagesToRecords([message]);\n return {headers, records, messages: [message]};\n }\n\n return message;\n }\n\n return false;\n } catch (error) {\n handleAmqpErrors(error);\n }\n }\n\n function ackMessages(messages) {\n messages.forEach(message => {\n debug(`Ack message ${message.properties.correlationId}`);\n channel.ack(message);\n });\n }\n\n function nackMessages(messages) {\n messages.forEach(message => {\n debug(`Nack message ${message.properties.correlationId}`);\n channel.nack(message);\n });\n }\n\n async function sendToQueue({queue, correlationId, headers, data}) {\n debug(`sendToQueue`);\n try {\n debug(`Queue ${queue}`);\n debug(`CorrelationId ${correlationId}`);\n debugData(`Data ${JSON.stringify(data)}`);\n debug(`Headers ${JSON.stringify(headers)}`);\n\n errorUndefinedQueue(queue);\n\n debug(`Asserting queue: ${queue}`);\n await channel.assertQueue(queue, {durable: true});\n\n // `correlationId` is deliberately delivered ONLY via AMQP message\n // properties (headers) \u2014 it is NOT included in the message body\n // (which is just `JSON.stringify({data})`). Consumers MUST read it from\n // `message.properties.correlationId`, not from the body. Do not add it\n // to the body without changing the message contract.\n debug(`Actually sendToQueue: ${queue}`);\n await channel.sendToQueue(\n queue,\n Buffer.from(JSON.stringify({data})),\n {\n correlationId,\n persistent: true,\n headers\n }\n );\n debug(`Send message for ${correlationId} to queue: ${queue}`);\n\n\n } catch (error) {\n const errorToThrow = error;\n //if (error instanceof ApiError) {\n // throw error;\n //}\n debug(`SendToQueue errored: ${JSON.stringify(error)}`);\n handleAmqpErrors(errorToThrow);\n }\n }\n\n async function removeQueue(queue) {\n // deleteQueue borks the channel if the queue does not exist\n // -> use throwaway tempChannel to avoid killing actual channel in use\n // this might be doable also with assertQueue before deleteQueue\n let tempChannel;\n try {\n tempChannel = await connection.createChannel();\n logger.verbose(`Removing queue ${queue}.`);\n debug(`Removing queue ${queue}.`);\n await tempChannel.deleteQueue(queue);\n } finally {\n // if deleteQueue throws (e.g. the queue does not exist) the\n // throwaway channel used to leak -> always close it\n if (tempChannel) {\n await tempChannel.close();\n }\n }\n }\n\n // ----------------\n // Helper functions\n // ----------------\n\n function messagesToRecords(messages) {\n debug(`Parsing messages (${messages.length}) to records`);\n\n return messages.map(message => {\n const content = JSON.parse(message.content.toString());\n // Use subfieldValues: false validationOption here\n return new MarcRecord(content.data, {subfieldValues: false});\n });\n }\n\n async function getData(queue) {\n debug(`Getting queue data from ${queue}`);\n try {\n const {messageCount} = await channel.checkQueue(queue);\n debug(`There is ${messageCount} messages in queue ${queue}`);\n const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n debug(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n const messages = await pump(messagesToGet);\n\n debug(`Returning ${messages.length} unique messages`);\n\n return messages;\n } catch (error) {\n handleAmqpErrors(error);\n }\n\n async function pump(count, results = [], identifiers = []) {\n if (count === 0) {\n return results;\n }\n\n const message = await channel.get(queue);\n // queue can drain between the messageCount read and channel.get\n // (race, purge, other consumer) -> channel.get returns null\n if (!message) {\n return results;\n }\n\n const identifier = {\n correlationId: message.properties.correlationId,\n deliveryTag: message.fields.deliveryTag\n };\n // filter not unique messages (compare by key, not by object reference)\n const isDuplicate = identifiers.some(\n i => i.correlationId === identifier.correlationId && i.deliveryTag === identifier.deliveryTag\n );\n if (isDuplicate) {\n return pump(count - 1, results, identifiers);\n }\n\n return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n }\n }\n\n function getHeaderInfo(data) {\n return data.properties.headers;\n }\n\n function errorUndefinedQueue(queue) {\n if (queue === undefined || queue === '' || queue === null) {\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Undefined queue!`);\n }\n }\n\n function handleAmqpErrors(error) {\n const newError = error;\n debug(`HandleAmqpErrors got an error: ${JSON.stringify(error)}`);\n if (error instanceof ApiError) {\n debug(`We have an ApiError`);\n throw new ApiError(newError.status, newError.payload);\n }\n debug(`We have a non-ApiError`);\n logError(error);\n throw newError;\n }\n}\n\n\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,aAAa;AACpB,OAAO,uBAAuB;AAC9B,SAAQ,WAAW,eAAc;AACjC,OAAO,gBAAgB;AACvB,SAAQ,kBAAiB;AACzB,SAAQ,cAAc,SAAS,gBAAe;AAC9C,SAAQ,kBAAiB;AACzB,SAAQ,gBAAe;AAEvB,8BAA+B,UAAU,iBAAiB,OAAO,OAAO,CAAC,GAAG;AAC1E,QAAM,SAAS,aAAa;AAC5B,QAAM,QAAQ,kBAAkB,yCAAyC;AACzE,QAAM,UAAU,MAAM,OAAO,IAAI;AACjC,QAAM,YAAY,MAAM,OAAO,MAAM;AACrC,QAAM,oBAAoB,UAAU,UAAU;AAE9C,QAAM,gCAAgC,QAAQ,EAAE;AAEhD,QAAM,aAAa,KAAK,cAAc,MAAM,QAAQ,QAAQ,QAAQ;AACpE,QAAM,UAAU,KAAK,WAAW,MAAM,WAAW,cAAc;AAM/D,MAAI,qBAAqB;AACzB,QAAM,kBAAkB,iBAAiB,YAAY,IAAI;AAEzD,QAAM,eAAe,UAAU,EAAE;AACjC,QAAM,YAAY,OAAO,EAAE;AAC3B,QAAM,oBAAoB,kBAAkB,yBAAyB,0BAA0B,EAAE;AAEjG,SAAO,EAAC,YAAY,cAAc,YAAY,aAAa,cAAc,aAAa,aAAa,mBAAmB,cAAc,gBAAe;AAEnJ,iBAAe,eAAe;AAG5B,yBAAqB;AACrB,UAAM,iBAAiB;AACvB,UAAM,QAAQ,MAAM;AACpB,UAAM,YAAY,OAAO,EAAE;AAAA,EAC7B;AAEA,iBAAe,kBAAkB;AAE/B,yBAAqB;AACrB,UAAM,oBAAoB;AAC1B,UAAM,WAAW,MAAM;AACvB,UAAM,eAAe,UAAU,EAAE;AAAA,EACnC;AAEA,iBAAe,YAAY,OAAO,OAAO;AAEvC,QAAI,CAAC,oBAAoB;AACvB,cAAQ,oCAAoC;AAC5C,aAAO;AAAA,IACT;AACA,QAAI,MAAM;AACR,YAAM,kBAAkB,IAAI;AAC5B,aAAO,YAAY,KAAK;AAAA,IAC1B;AAEA,QAAI;AACF,cAAQ,2CAA2C;AACnD,YAAM,QAAQ,YAAY,eAAe,EAAC,SAAS,MAAK,CAAC;AACzD,cAAQ,+CAA+C;AACvD,aAAO,YAAY,GAAG;AAAA,IACxB,SAAS,OAAO;AACd,cAAQ,qBAAqB,KAAK,UAAU,KAAK,CAAC,EAAE;AAGpD,UAAI,CAAC,oBAAoB;AACvB,gBAAQ,oDAAoD;AAC5D,eAAO;AAAA,MACT;AACA,uBAAiB,KAAK;AAAA,IACxB;AAAA,EACF;AAEA,iBAAe,WAAW,EAAC,OAAO,QAAQ,SAAS,WAAW,MAAM,QAAQ,MAAK,GAAG;AAClF,UAAM,eAAe,KAAK,YAAY,KAAK,eAAe,QAAQ,YAAY,KAAK,EAAE;AAErF,QAAI;AAEF,0BAAoB,KAAK;AACzB,YAAM,cAAc,MAAM,QAAQ,YAAY,OAAO,EAAC,SAAS,KAAI,CAAC;AAEpE,UAAI,OAAO;AACT,cAAM,WAAW,KAAK;AACtB,eAAO,QAAQ,SAAS,KAAK,eAAe,YAAY,YAAY,WAAW;AAC/E,cAAM,SAAS,KAAK,eAAe,YAAY,YAAY,WAAW;AACtE,eAAO,WAAW,EAAC,OAAO,MAAK,CAAC;AAAA,MAClC;AAEA,UAAI,UAAU,YAAY;AACxB,eAAO,YAAY;AAAA,MACrB;AAOA,UAAI,UAAU,OAAO;AACnB,YAAI,kBAAkB,WAAW,GAAG;AAClC,iBAAO,WAAW,OAAO,QAAQ;AAAA,QACnC;AACA,eAAO;AAAA,MACT;AAIA,UAAI,UAAU,SAAS;AACrB,YAAI,kBAAkB,WAAW,GAAG;AAClC,iBAAO,aAAa,OAAO,QAAQ;AAAA,QACrC;AACA,eAAO;AAAA,MACT;AAGA,YAAM,IAAI,SAAS,WAAW,uBAAuB,uCAAuC,KAAK,EAAE;AAAA,IACrG,SAAS,OAAO;AACd,uBAAiB,KAAK;AAAA,IACxB;AAEA,aAAS,kBAAkB,aAAa;AACtC,UAAI,YAAY,eAAe,GAAG;AAChC,cAAM,eAAe,YAAY,YAAY,MAAM,KAAK,WAAW;AACnE,eAAO;AAAA,MACT;AACA,YAAM,SAAS,KAAK,QAAQ,YAAY,YAAY,WAAW;AAC/D,aAAO;AAAA,IACT;AAEA,aAAS,WAAWA,QAAO;AACzB,YAAM,kBAAkB,KAAK,YAAYA,MAAK,EAAE;AAChD,UAAIA,QAAO;AACT,eAAO,QAAQ,WAAW,KAAK;AAAA,MACjC;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,aAAa,OAAO,UAAU;AAC3C,UAAM,wCAAwC,KAAK,EAAE;AACrD,QAAI;AACF,YAAM,QAAQ,YAAY,OAAO,EAAC,SAAS,KAAI,CAAC;AAGhD,YAAM,WAAW,MAAM,QAAQ,KAAK;AAIpC,UAAI,SAAS,WAAW,GAAG;AACzB,cAAM,uBAAuB,KAAK,2BAA2B;AAC7D,eAAO;AAAA,MACT;AAGA,YAAM,UAAU,cAAc,SAAS,CAAC,CAAC;AACzC,YAAM,iBAAiB,WAAW,SAAS,SAAS,GAAG,eAAe,KAAK,KAAK,WAAW,eAAe,eAAe,EAAE;AAE3H,UAAI,UAAU;AACZ,cAAM,UAAU,kBAAkB,QAAQ;AAC1C,eAAO,EAAC,SAAS,SAAS,SAAQ;AAAA,MACpC;AAEA,aAAO,EAAC,SAAS,SAAQ;AAAA,IAC3B,SAAS,OAAO;AACd,uBAAiB,KAAK;AAAA,IACxB;AAAA,EACF;AAEA,iBAAe,WAAW,OAAO,UAAU;AACzC,UAAM,sCAAsC,KAAK,EAAE;AACnD,QAAI;AACF,YAAM,QAAQ,YAAY,OAAO,EAAC,SAAS,KAAI,CAAC;AAGhD,YAAM,UAAU,MAAM,QAAQ,IAAI,KAAK;AAEvC,gBAAU,YAAY,QAAQ,SAAS,EAAC,QAAQ,MAAM,gBAAgB,GAAG,OAAO,EAAC,CAAC,CAAC,EAAE;AAErF,YAAM,0BAA0B,KAAK,IAAI,WAAW,eAAe,kBAAkB,EAAE;AAEvF,UAAI,SAAS;AACX,YAAI,UAAU;AACZ,gBAAM,UAAU,cAAc,OAAO;AACrC,gBAAM,UAAU,kBAAkB,CAAC,OAAO,CAAC;AAC3C,iBAAO,EAAC,SAAS,SAAS,UAAU,CAAC,OAAO,EAAC;AAAA,QAC/C;AAEA,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT,SAAS,OAAO;AACd,uBAAiB,KAAK;AAAA,IACxB;AAAA,EACF;AAEA,WAAS,YAAY,UAAU;AAC7B,aAAS,QAAQ,aAAW;AAC1B,YAAM,eAAe,QAAQ,WAAW,aAAa,EAAE;AACvD,cAAQ,IAAI,OAAO;AAAA,IACrB,CAAC;AAAA,EACH;AAEA,WAAS,aAAa,UAAU;AAC9B,aAAS,QAAQ,aAAW;AAC1B,YAAM,gBAAgB,QAAQ,WAAW,aAAa,EAAE;AACxD,cAAQ,KAAK,OAAO;AAAA,IACtB,CAAC;AAAA,EACH;AAEA,iBAAe,YAAY,EAAC,OAAO,eAAe,SAAS,KAAI,GAAG;AAChE,UAAM,aAAa;AACnB,QAAI;AACF,YAAM,SAAS,KAAK,EAAE;AACtB,YAAM,iBAAiB,aAAa,EAAE;AACtC,gBAAU,QAAQ,KAAK,UAAU,IAAI,CAAC,EAAE;AACxC,YAAM,WAAW,KAAK,UAAU,OAAO,CAAC,EAAE;AAE1C,0BAAoB,KAAK;AAEzB,YAAM,oBAAoB,KAAK,EAAE;AACjC,YAAM,QAAQ,YAAY,OAAO,EAAC,SAAS,KAAI,CAAC;AAOhD,YAAM,yBAAyB,KAAK,EAAE;AACtC,YAAM,QAAQ;AAAA,QACZ;AAAA,QACA,OAAO,KAAK,KAAK,UAAU,EAAC,KAAI,CAAC,CAAC;AAAA,QAClC;AAAA,UACE;AAAA,UACA,YAAY;AAAA,UACZ;AAAA,QACF;AAAA,MACF;AACA,YAAM,oBAAoB,aAAa,cAAc,KAAK,EAAE;AAAA,IAG9D,SAAS,OAAO;AACd,YAAM,eAAe;AAIrB,YAAM,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AACrD,uBAAiB,YAAY;AAAA,IAC/B;AAAA,EACF;AAEA,iBAAe,YAAY,OAAO;AAIhC,QAAI;AACJ,QAAI;AACF,oBAAc,MAAM,WAAW,cAAc;AAC7C,aAAO,QAAQ,kBAAkB,KAAK,GAAG;AACzC,YAAM,kBAAkB,KAAK,GAAG;AAChC,YAAM,YAAY,YAAY,KAAK;AAAA,IACrC,UAAE;AAGA,UAAI,aAAa;AACf,cAAM,YAAY,MAAM;AAAA,MAC1B;AAAA,IACF;AAAA,EACF;AAMA,WAAS,kBAAkB,UAAU;AACnC,UAAM,qBAAqB,SAAS,MAAM,cAAc;AAExD,WAAO,SAAS,IAAI,aAAW;AAC7B,YAAM,UAAU,KAAK,MAAM,QAAQ,QAAQ,SAAS,CAAC;AAErD,aAAO,IAAI,WAAW,QAAQ,MAAM,EAAC,gBAAgB,MAAK,CAAC;AAAA,IAC7D,CAAC;AAAA,EACH;AAEA,iBAAe,QAAQ,OAAO;AAC5B,UAAM,2BAA2B,KAAK,EAAE;AACxC,QAAI;AACF,YAAM,EAAC,aAAY,IAAI,MAAM,QAAQ,WAAW,KAAK;AACrD,YAAM,YAAY,YAAY,sBAAsB,KAAK,EAAE;AAC3D,YAAM,gBAAgB,gBAAgB,aAAa,aAAa;AAChE,YAAM,WAAW,aAAa,wBAAwB,KAAK,EAAE;AAE7D,YAAM,WAAW,MAAM,KAAK,aAAa;AAEzC,YAAM,aAAa,SAAS,MAAM,kBAAkB;AAEpD,aAAO;AAAA,IACT,SAAS,OAAO;AACd,uBAAiB,KAAK;AAAA,IACxB;AAEA,mBAAe,KAAK,OAAO,UAAU,CAAC,GAAG,cAAc,CAAC,GAAG;AACzD,UAAI,UAAU,GAAG;AACf,eAAO;AAAA,MACT;AAEA,YAAM,UAAU,MAAM,QAAQ,IAAI,KAAK;AAGvC,UAAI,CAAC,SAAS;AACZ,eAAO;AAAA,MACT;AAEA,YAAM,aAAa;AAAA,QACjB,eAAe,QAAQ,WAAW;AAAA,QAClC,aAAa,QAAQ,OAAO;AAAA,MAC9B;AAEA,YAAM,cAAc,YAAY;AAAA,QAC9B,OAAK,EAAE,kBAAkB,WAAW,iBAAiB,EAAE,gBAAgB,WAAW;AAAA,MACpF;AACA,UAAI,aAAa;AACf,eAAO,KAAK,QAAQ,GAAG,SAAS,WAAW;AAAA,MAC7C;AAEA,aAAO,KAAK,QAAQ,GAAG,QAAQ,OAAO,OAAO,GAAG,YAAY,OAAO,UAAU,CAAC;AAAA,IAChF;AAAA,EACF;AAEA,WAAS,cAAc,MAAM;AAC3B,WAAO,KAAK,WAAW;AAAA,EACzB;AAEA,WAAS,oBAAoB,OAAO;AAClC,QAAI,UAAU,UAAa,UAAU,MAAM,UAAU,MAAM;AACzD,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB;AAAA,IACzE;AAAA,EACF;AAEA,WAAS,iBAAiB,OAAO;AAC/B,UAAM,WAAW;AACjB,UAAM,kCAAkC,KAAK,UAAU,KAAK,CAAC,EAAE;AAC/D,QAAI,iBAAiB,UAAU;AAC7B,YAAM,qBAAqB;AAC3B,YAAM,IAAI,SAAS,SAAS,QAAQ,SAAS,OAAO;AAAA,IACtD;AACA,UAAM,wBAAwB;AAC9B,aAAS,KAAK;AACd,UAAM;AAAA,EACR;AACF;",
|
|
6
6
|
"names": ["purge"]
|
|
7
7
|
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import httpStatus from "http-status";
|
|
2
2
|
import { MARCXML, ISO2709, Json, AlephSequential } from "@natlibfi/marc-record-serializers";
|
|
3
|
-
import { createLogger } from "@natlibfi/melinda-
|
|
4
|
-
import { Error as ConversionError } from "@natlibfi/melinda-commons";
|
|
3
|
+
import { createLogger, Error as ConversionError } from "@natlibfi/melinda-commons";
|
|
5
4
|
import { CONVERSION_FORMATS } from "../constants.js";
|
|
6
5
|
import { logError } from "../utils.js";
|
|
7
6
|
export default function() {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/helpers/conversion.js"],
|
|
4
|
-
"sourcesContent": ["\n\nimport httpStatus from 'http-status';\nimport {MARCXML, ISO2709, Json, AlephSequential} from '@natlibfi/marc-record-serializers';\nimport {createLogger
|
|
5
|
-
"mappings": "AAEA,OAAO,gBAAgB;AACvB,SAAQ,SAAS,SAAS,MAAM,uBAAsB;AACtD,SAAQ,
|
|
4
|
+
"sourcesContent": ["\n\nimport httpStatus from 'http-status';\nimport {MARCXML, ISO2709, Json, AlephSequential} from '@natlibfi/marc-record-serializers';\nimport {createLogger, Error as ConversionError} from '@natlibfi/melinda-commons';\nimport {CONVERSION_FORMATS} from '../constants.js';\nimport {logError} from '../utils.js';\n\nexport default function () {\n const logger = createLogger();\n\n return {serialize, unserialize};\n\n // This uses format as a number form contants CONVERSION_FORMATS\n function serialize(record, format) {\n logger.verbose(`Serializing record to ${format}`);\n try {\n if (format === CONVERSION_FORMATS.MARCXML) {\n return MARCXML.to(record);\n }\n\n if (format === CONVERSION_FORMATS.ISO2709) {\n return ISO2709.to(record);\n }\n\n if (format === CONVERSION_FORMATS.JSON) {\n return Json.to(record);\n }\n\n if (format === CONVERSION_FORMATS.ALEPHSEQ) {\n return AlephSequential.to(record);\n }\n\n throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n } catch (err) {\n logError(err);\n const message = err.message || err.payload?.message || err.payload;\n logger.debug(`${message}`);\n if (err instanceof ConversionError) {\n throw err;\n }\n throw new ConversionError(httpStatus.BAD_REQUEST, `Error while serializing record. ${message}`);\n }\n }\n\n function unserialize(data, format, validationOptions = {subfieldValues: false}) {\n logger.verbose(`Unserializing record from ${format}`);\n logger.silly(`Format: ${format}`);\n logger.debug(`Validation options: ${JSON.stringify(validationOptions)}`);\n logger.silly(`Data: ${JSON.stringify(data)}`);\n try {\n if (format === CONVERSION_FORMATS.MARCXML) {\n logger.silly('Unserialize format marcxml');\n return MARCXML.from(data, validationOptions);\n }\n\n if (format === CONVERSION_FORMATS.ISO2709) {\n logger.silly('Unserialize format iso2709');\n return ISO2709.from(data, validationOptions);\n }\n\n if (format === CONVERSION_FORMATS.JSON) {\n logger.silly('Unserialize format json');\n return Json.from(data, validationOptions);\n }\n\n if (format === CONVERSION_FORMATS.ALEPHSEQ) {\n logger.silly('Unserialize format aleph sequential');\n return AlephSequential.from(data, validationOptions);\n }\n\n throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n } catch (err) {\n logError(err);\n const message = err.message || err.payload?.message || err.payload;\n logger.debug(`${message}`);\n if (err instanceof ConversionError) {\n throw err;\n }\n throw new ConversionError(httpStatus.BAD_REQUEST, `Error while unserializing record. ${message}`);\n }\n }\n}\n"],
|
|
5
|
+
"mappings": "AAEA,OAAO,gBAAgB;AACvB,SAAQ,SAAS,SAAS,MAAM,uBAAsB;AACtD,SAAQ,cAAc,SAAS,uBAAsB;AACrD,SAAQ,0BAAyB;AACjC,SAAQ,gBAAe;AAEvB,0BAA2B;AACzB,QAAM,SAAS,aAAa;AAE5B,SAAO,EAAC,WAAW,YAAW;AAG9B,WAAS,UAAU,QAAQ,QAAQ;AACjC,WAAO,QAAQ,yBAAyB,MAAM,EAAE;AAChD,QAAI;AACF,UAAI,WAAW,mBAAmB,SAAS;AACzC,eAAO,QAAQ,GAAG,MAAM;AAAA,MAC1B;AAEA,UAAI,WAAW,mBAAmB,SAAS;AACzC,eAAO,QAAQ,GAAG,MAAM;AAAA,MAC1B;AAEA,UAAI,WAAW,mBAAmB,MAAM;AACtC,eAAO,KAAK,GAAG,MAAM;AAAA,MACvB;AAEA,UAAI,WAAW,mBAAmB,UAAU;AAC1C,eAAO,gBAAgB,GAAG,MAAM;AAAA,MAClC;AAEA,YAAM,IAAI,gBAAgB,WAAW,sBAAsB;AAAA,IAC7D,SAAS,KAAK;AACZ,eAAS,GAAG;AACZ,YAAM,UAAU,IAAI,WAAW,IAAI,SAAS,WAAW,IAAI;AAC3D,aAAO,MAAM,GAAG,OAAO,EAAE;AACzB,UAAI,eAAe,iBAAiB;AAClC,cAAM;AAAA,MACR;AACA,YAAM,IAAI,gBAAgB,WAAW,aAAa,mCAAmC,OAAO,EAAE;AAAA,IAChG;AAAA,EACF;AAEA,WAAS,YAAY,MAAM,QAAQ,oBAAoB,EAAC,gBAAgB,MAAK,GAAG;AAC9E,WAAO,QAAQ,6BAA6B,MAAM,EAAE;AACpD,WAAO,MAAM,WAAW,MAAM,EAAE;AAChC,WAAO,MAAM,uBAAuB,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACvE,WAAO,MAAM,SAAS,KAAK,UAAU,IAAI,CAAC,EAAE;AAC5C,QAAI;AACF,UAAI,WAAW,mBAAmB,SAAS;AACzC,eAAO,MAAM,4BAA4B;AACzC,eAAO,QAAQ,KAAK,MAAM,iBAAiB;AAAA,MAC7C;AAEA,UAAI,WAAW,mBAAmB,SAAS;AACzC,eAAO,MAAM,4BAA4B;AACzC,eAAO,QAAQ,KAAK,MAAM,iBAAiB;AAAA,MAC7C;AAEA,UAAI,WAAW,mBAAmB,MAAM;AACtC,eAAO,MAAM,yBAAyB;AACtC,eAAO,KAAK,KAAK,MAAM,iBAAiB;AAAA,MAC1C;AAEA,UAAI,WAAW,mBAAmB,UAAU;AAC1C,eAAO,MAAM,qCAAqC;AAClD,eAAO,gBAAgB,KAAK,MAAM,iBAAiB;AAAA,MACrD;AAEA,YAAM,IAAI,gBAAgB,WAAW,sBAAsB;AAAA,IAC7D,SAAS,KAAK;AACZ,eAAS,GAAG;AACZ,YAAM,UAAU,IAAI,WAAW,IAAI,SAAS,WAAW,IAAI;AAC3D,aAAO,MAAM,GAAG,OAAO,EAAE;AACzB,UAAI,eAAe,iBAAiB;AAClC,cAAM;AAAA,MACR;AACA,YAAM,IAAI,gBAAgB,WAAW,aAAa,qCAAqC,OAAO,EAAE;AAAA,IAClG;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import createDebugLogger from "debug";
|
|
2
2
|
import { inspect } from "util";
|
|
3
|
-
import { MarcRecord } from "@natlibfi/marc-record";
|
|
4
3
|
export function stripF884s(newRecord, options) {
|
|
5
4
|
const debug = createDebugLogger("@natlibfi/melinda-rest-api-commons:fixRecord:stripF884s");
|
|
6
5
|
const debugData = debug.extend("data");
|
|
@@ -104,9 +103,24 @@ export function stripF884s(newRecord, options) {
|
|
|
104
103
|
const uniq884s = uniqFields(editedf884s);
|
|
105
104
|
debug(`UniqFields (${uniq884s.length})`);
|
|
106
105
|
debugData(inspect(uniq884s, { depth: 3 }));
|
|
106
|
+
function canonicalize(value) {
|
|
107
|
+
if (Array.isArray(value)) {
|
|
108
|
+
return value.map(canonicalize);
|
|
109
|
+
}
|
|
110
|
+
if (value && typeof value === "object") {
|
|
111
|
+
return Object.keys(value).sort().reduce((canonical, key) => {
|
|
112
|
+
canonical[key] = canonicalize(value[key]);
|
|
113
|
+
return canonical;
|
|
114
|
+
}, {});
|
|
115
|
+
}
|
|
116
|
+
return value;
|
|
117
|
+
}
|
|
118
|
+
function fieldsAreEqual(a, b) {
|
|
119
|
+
return JSON.stringify(canonicalize(a)) === JSON.stringify(canonicalize(b));
|
|
120
|
+
}
|
|
107
121
|
function uniqFields(fields) {
|
|
108
122
|
return fields.reduce((uniq, field) => {
|
|
109
|
-
if (!uniq.some((f) =>
|
|
123
|
+
if (!uniq.some((f) => fieldsAreEqual(f, field))) {
|
|
110
124
|
uniq.push(field);
|
|
111
125
|
}
|
|
112
126
|
return uniq;
|
|
@@ -123,7 +137,7 @@ export function stripF884s(newRecord, options) {
|
|
|
123
137
|
}
|
|
124
138
|
function findSource(f884) {
|
|
125
139
|
const [sfK] = f884.subfields.filter((subfield) => subfield.code === "k").map((subfield) => subfield.value);
|
|
126
|
-
const source = sfK.replace(/:.*$/u, "");
|
|
140
|
+
const source = (sfK ?? "").replace(/:.*$/u, "");
|
|
127
141
|
if (source && source.length !== 64) {
|
|
128
142
|
debugData(`Source: "${source}"`);
|
|
129
143
|
return source;
|
|
@@ -134,7 +148,7 @@ export function stripF884s(newRecord, options) {
|
|
|
134
148
|
}
|
|
135
149
|
function findHash(f884) {
|
|
136
150
|
const [sfK] = f884.subfields.filter((subfield) => subfield.code === "k").map((subfield) => subfield.value);
|
|
137
|
-
const hash = sfK.replace(/^.*:/u, "");
|
|
151
|
+
const hash = (sfK ?? "").replace(/^.*:/u, "");
|
|
138
152
|
if (hash && hash.length === 64) {
|
|
139
153
|
debugData(`Hash: "${hash}"`);
|
|
140
154
|
return hash;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/helpers/fix-strip-f884s.js"],
|
|
4
|
-
"sourcesContent": ["import createDebugLogger from 'debug';\n//import {sortFieldsBySubfieldValue, removeSubfield} from './fix-utils';\nimport {inspect} from 'util';\
|
|
5
|
-
"mappings": "AAAA,OAAO,uBAAuB;AAE9B,SAAQ,eAAc;
|
|
4
|
+
"sourcesContent": ["import createDebugLogger from 'debug';\n//import {sortFieldsBySubfieldValue, removeSubfield} from './fix-utils';\nimport {inspect} from 'util';\n\nexport function stripF884s(newRecord, options) {\n const debug = createDebugLogger('@natlibfi/melinda-rest-api-commons:fixRecord:stripF884s');\n const debugData = debug.extend('data');\n\n if (options !== true) {\n debug(`NOT running stripF884s fixer, no options`);\n return newRecord;\n }\n\n debug(`Running stripF884s fixer`);\n debugData(`Options for stripF884s: ${JSON.stringify(options)}`);\n\n // Handle only f884 with $5 MELINDA - let's no remove random f884s\n const isMelindaf884 = f => f.tag === '884' && f.subfields.some(({code, value}) => code === '5' && value === 'MELINDA');\n\n const f884Melindas = newRecord.fields.filter(isMelindaf884);\n debugData(`Melinda's f884s (${f884Melindas.length}) \\n ${JSON.stringify(f884Melindas)}`);\n //debugData(`Melinda's f884s (${f884Melindas.length}) \\n ${inspect(f884Melindas, {depth: 3})}`);\n\n if (f884Melindas.length < 2) {\n debug(`Not enough Melinda's f884s to filter (${f884Melindas.length})`);\n return newRecord;\n }\n\n // Pick data (firstDate, lastDate, hash with latest date) for all conversion : source combinations\n const results = f884Melindas.reduce((allResults, f884, index) => {\n debugData(`All results: ${JSON.stringify(allResults)}`);\n //debugData(`Handling field (${index}): ${inspect(f884, {depth: 3})}`);\n debugData(`Handling field *(${index})*: ${JSON.stringify(f884)}`);\n\n const conversion = findConversion(f884);\n const source = findSource(f884);\n const convSource = `${conversion}:${source}`;\n const {firstDate, lastDate} = findDates(f884);\n const hash = findHash(f884);\n\n // Find currently existing data for conversion : source combination\n const currResult = allResults[convSource];\n debugData(JSON.stringify(currResult));\n\n // If we do not have any current data, add data from field\n if (!currResult) {\n debug(`Add a new convSource ${convSource} to results`);\n const newResult = {\n conversion,\n source,\n firstDate,\n lastDate,\n hash\n };\n debugData(`- New result: ${JSON.stringify(newResult)}`);\n return {\n ...allResults,\n [convSource]: newResult\n };\n }\n // If we have current data, update it from the data from field, if needed\n debug(`Update result for ${convSource}`);\n debugData(`- Current result: ${JSON.stringify(currResult)}`);\n\n // '00000000' is the deliberate no-date sentinel for a missing $g (see\n // findDates): it must never win a date comparison, and both sides must\n // agree on this sentinel\n const updatedResult = {\n ...currResult,\n firstDate: currResult.firstDate > firstDate && firstDate !== '00000000' ? firstDate : currResult.firstDate,\n lastDate: currResult.lastDate < lastDate ? lastDate : currResult.lastDate,\n hash: currResult.lastDate < lastDate && lastDate !== '00000000' && hash !== '0000000000000000000000000000000000000000000000000000000000000000' ? hash : currResult.hash\n };\n debugData(`- Updated result: ${JSON.stringify(updatedResult)}`);\n return {\n ...allResults,\n [convSource]: updatedResult\n };\n }, {});\n\n debugData(`RESULTS: ${JSON.stringify(results)}`);\n const editedf884s = editFields(f884Melindas, results);\n\n debug(`Edited fields (${editedf884s.length})`);\n //debugData(inspect(editedf884s, {depth: 3}));\n debugData(`${JSON.stringify(editedf884s)}`);\n\n // We update each field based on the results we got from fields\n // Note: we do not add $g:s or $k:s if the field doesn't have them\n function editFields(fields, results) {\n debug(`EDITING FIELDS (${fields.length})`);\n const editedFields = fields.map((field) => {\n const convSource = `${findConversion(field)}:${findSource(field)}`;\n debug(`Looking for: ${convSource}`);\n\n if (convSource && results[convSource]) {\n debugData(`Found: ${JSON.stringify(results[convSource])}`);\n const firstDate = results[convSource].firstDate && results[convSource].firstDate !== '00000000' ? results[convSource].firstDate : '';\n const lastDate = results[convSource].lastDate && results[convSource].lastDate !== '00000000' ? results[convSource].lastDate : '';\n\n const source = results[convSource].source && results[convSource].source !== 'NO_SOURCE' ? results[convSource].source : '';\n const hash = results[convSource].hash && results[convSource].hash !== '0000000000000000000000000000000000000000000000000000000000000000' ? results[convSource].hash : '';\n\n const sfGAll = firstDate === lastDate ? `${firstDate}` : `${firstDate} - ${lastDate}`;\n debugData(`EditedDates: ${sfGAll}`);\n const sfKAll = `${source}:${hash}`;\n\n const sfG = sfGAll.replace(/^ - /u, '').replace(/ - $/u, '');\n const sfK = sfKAll.replace(/^:/u, '').replace(/:$/u, '');\n\n return {\n ...field,\n subfields: field.subfields.map(subfield => {\n\n // we don't get $g or $k if the original field doesn't have them\n if (subfield.code === 'g') {\n return {\n code: subfield.code,\n value: sfG\n };\n }\n\n if (subfield.code === 'k') {\n return {\n code: subfield.code,\n value: sfK\n };\n }\n\n return subfield;\n })\n\n };\n }\n // Defensive: results is keyed from the same f884Melindas array, so\n // convSource is always present when truthy.\n debug(`Not found: ${convSource}`);\n return field;\n });\n return editedFields;\n }\n\n // we'll need to uniq these too\n\n const uniq884s = uniqFields(editedf884s);\n debug(`UniqFields (${uniq884s.length})`);\n debugData(inspect(uniq884s, {depth: 3}));\n\n\n // Canonicalize a raw field object ({tag, ind1, ind2, subfields: [...]}) so\n // that two structurally equal fields stringify identically: object keys are\n // sorted recursively, arrays keep their order, primitives pass through.\n function canonicalize(value) {\n if (Array.isArray(value)) {\n return value.map(canonicalize);\n }\n if (value && typeof value === 'object') {\n return Object.keys(value).sort().reduce((canonical, key) => {\n canonical[key] = canonicalize(value[key]);\n return canonical;\n }, {});\n }\n return value;\n }\n\n function fieldsAreEqual(a, b) {\n return JSON.stringify(canonicalize(a)) === JSON.stringify(canonicalize(b));\n }\n\n function uniqFields(fields) {\n return fields.reduce((uniq, field) => {\n if (!uniq.some(f => fieldsAreEqual(f, field))) {\n uniq.push(field);\n }\n\n return uniq;\n }, []);\n }\n // Replace original Melinda-f884s with remaining Melinda-f884s\n // NOTE: this sorts MELINDA-f884s after possible other f884s\n newRecord.removeFields(f884Melindas);\n newRecord.insertFields(uniq884s);\n\n return newRecord;\n\n\n // Find conversion:source pairs\n // Find first date\n // Find last date and last hash\n\n\n function findConversion(f884) {\n const [sfA] = f884.subfields.filter((subfield) => subfield.code === 'a').map(subfield => subfield.value);\n const conversion = sfA;\n debugData(`Conversion: \"${conversion}\"`);\n return conversion;\n }\n\n function findSource(f884) {\n const [sfK] = f884.subfields.filter((subfield) => subfield.code === 'k').map(subfield => subfield.value);\n // a Melinda f884 may have no $k subfield -> sfK is undefined\n const source = (sfK ?? '').replace(/:.*$/u, '');\n if (source && source.length !== 64) {\n debugData(`Source: \"${source}\"`);\n return source;\n }\n const emptySource = 'NO_SOURCE';\n debugData(`Source: \"${emptySource}\"`);\n return emptySource;\n }\n\n function findHash(f884) {\n const [sfK] = f884.subfields.filter((subfield) => subfield.code === 'k').map(subfield => subfield.value);\n // a Melinda f884 may have no $k subfield -> sfK is undefined\n const hash = (sfK ?? '').replace(/^.*:/u, '');\n if (hash && hash.length === 64) {\n debugData(`Hash: \"${hash}\"`);\n return hash;\n }\n const emptyHash = '0000000000000000000000000000000000000000000000000000000000000000';\n debugData(`Hash: \"${emptyHash}\"`);\n return emptyHash;\n }\n\n function findDates(f884) {\n const [sfG] = f884.subfields.filter((subfield) => subfield.code === 'g').map(subfield => subfield.value);\n if (sfG && (/ - /u).test(sfG)) {\n const firstDate = sfG.replace(/ - .*$/u, '').padStart(8, '0');\n const lastDate = sfG.replace(/^.* - /u, '').padStart(8, '0');\n debugData(`FirstDate: \"${firstDate}\", LastDate: \"${lastDate}\"`);\n return {firstDate, lastDate};\n }\n // '00000000' is the deliberate no-date sentinel for a missing $g; the\n // consumer side (updatedResult / editFields above) must keep agreeing on\n // this exact sentinel\n const firstDate = sfG ? sfG.padStart(8, '0') : '00000000';\n const lastDate = sfG ? sfG.padStart(8, '0') : '00000000';\n\n debugData(`FirstDate: \"${firstDate}\", LastDate: \"${lastDate}\"`);\n return {firstDate, lastDate};\n }\n\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,uBAAuB;AAE9B,SAAQ,eAAc;AAEf,gBAAS,WAAW,WAAW,SAAS;AAC7C,QAAM,QAAQ,kBAAkB,yDAAyD;AACzF,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,MAAI,YAAY,MAAM;AACpB,UAAM,0CAA0C;AAChD,WAAO;AAAA,EACT;AAEA,QAAM,0BAA0B;AAChC,YAAU,2BAA2B,KAAK,UAAU,OAAO,CAAC,EAAE;AAG9D,QAAM,gBAAgB,OAAK,EAAE,QAAQ,SAAS,EAAE,UAAU,KAAK,CAAC,EAAC,MAAM,MAAK,MAAM,SAAS,OAAO,UAAU,SAAS;AAErH,QAAM,eAAe,UAAU,OAAO,OAAO,aAAa;AAC1D,YAAU,oBAAoB,aAAa,MAAM;AAAA,GAAQ,KAAK,UAAU,YAAY,CAAC,EAAE;AAGvF,MAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,yCAAyC,aAAa,MAAM,GAAG;AACrE,WAAO;AAAA,EACT;AAGA,QAAM,UAAU,aAAa,OAAO,CAAC,YAAY,MAAM,UAAU;AAC/D,cAAU,gBAAgB,KAAK,UAAU,UAAU,CAAC,EAAE;AAEtD,cAAU,oBAAoB,KAAK,OAAO,KAAK,UAAU,IAAI,CAAC,EAAE;AAEhE,UAAM,aAAa,eAAe,IAAI;AACtC,UAAM,SAAS,WAAW,IAAI;AAC9B,UAAM,aAAa,GAAG,UAAU,IAAI,MAAM;AAC1C,UAAM,EAAC,WAAW,SAAQ,IAAI,UAAU,IAAI;AAC5C,UAAM,OAAO,SAAS,IAAI;AAG1B,UAAM,aAAa,WAAW,UAAU;AACxC,cAAU,KAAK,UAAU,UAAU,CAAC;AAGpC,QAAI,CAAC,YAAY;AACf,YAAM,wBAAwB,UAAU,aAAa;AACrD,YAAM,YAAY;AAAA,QAChB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,gBAAU,iBAAiB,KAAK,UAAU,SAAS,CAAC,EAAE;AACtD,aAAO;AAAA,QACL,GAAG;AAAA,QACH,CAAC,UAAU,GAAG;AAAA,MAChB;AAAA,IACF;AAEA,UAAM,qBAAqB,UAAU,EAAE;AACvC,cAAU,qBAAqB,KAAK,UAAU,UAAU,CAAC,EAAE;AAK3D,UAAM,gBAAgB;AAAA,MACpB,GAAG;AAAA,MACH,WAAW,WAAW,YAAY,aAAa,cAAc,aAAa,YAAY,WAAW;AAAA,MACjG,UAAU,WAAW,WAAW,WAAW,WAAW,WAAW;AAAA,MACjE,MAAM,WAAW,WAAW,YAAY,aAAa,cAAc,SAAS,qEAAqE,OAAO,WAAW;AAAA,IACrK;AACA,cAAU,qBAAqB,KAAK,UAAU,aAAa,CAAC,EAAE;AAC9D,WAAO;AAAA,MACL,GAAG;AAAA,MACH,CAAC,UAAU,GAAG;AAAA,IAChB;AAAA,EACF,GAAG,CAAC,CAAC;AAEL,YAAU,YAAY,KAAK,UAAU,OAAO,CAAC,EAAE;AAC/C,QAAM,cAAc,WAAW,cAAc,OAAO;AAEpD,QAAM,kBAAkB,YAAY,MAAM,GAAG;AAE7C,YAAU,GAAG,KAAK,UAAU,WAAW,CAAC,EAAE;AAI1C,WAAS,WAAW,QAAQA,UAAS;AACnC,UAAM,mBAAmB,OAAO,MAAM,GAAG;AACzC,UAAM,eAAe,OAAO,IAAI,CAAC,UAAU;AACzC,YAAM,aAAa,GAAG,eAAe,KAAK,CAAC,IAAI,WAAW,KAAK,CAAC;AAChE,YAAM,gBAAgB,UAAU,EAAE;AAElC,UAAI,cAAcA,SAAQ,UAAU,GAAG;AACrC,kBAAU,UAAU,KAAK,UAAUA,SAAQ,UAAU,CAAC,CAAC,EAAE;AACzD,cAAM,YAAYA,SAAQ,UAAU,EAAE,aAAaA,SAAQ,UAAU,EAAE,cAAc,aAAaA,SAAQ,UAAU,EAAE,YAAY;AAClI,cAAM,WAAWA,SAAQ,UAAU,EAAE,YAAYA,SAAQ,UAAU,EAAE,aAAa,aAAaA,SAAQ,UAAU,EAAE,WAAW;AAE9H,cAAM,SAASA,SAAQ,UAAU,EAAE,UAAUA,SAAQ,UAAU,EAAE,WAAW,cAAcA,SAAQ,UAAU,EAAE,SAAS;AACvH,cAAM,OAAOA,SAAQ,UAAU,EAAE,QAAQA,SAAQ,UAAU,EAAE,SAAS,qEAAqEA,SAAQ,UAAU,EAAE,OAAO;AAEtK,cAAM,SAAS,cAAc,WAAW,GAAG,SAAS,KAAK,GAAG,SAAS,MAAM,QAAQ;AACnF,kBAAU,gBAAgB,MAAM,EAAE;AAClC,cAAM,SAAS,GAAG,MAAM,IAAI,IAAI;AAEhC,cAAM,MAAM,OAAO,QAAQ,SAAS,EAAE,EAAE,QAAQ,SAAS,EAAE;AAC3D,cAAM,MAAM,OAAO,QAAQ,OAAO,EAAE,EAAE,QAAQ,OAAO,EAAE;AAEvD,eAAO;AAAA,UACL,GAAG;AAAA,UACH,WAAW,MAAM,UAAU,IAAI,cAAY;AAGzC,gBAAI,SAAS,SAAS,KAAK;AACzB,qBAAO;AAAA,gBACL,MAAM,SAAS;AAAA,gBACf,OAAO;AAAA,cACT;AAAA,YACF;AAEA,gBAAI,SAAS,SAAS,KAAK;AACzB,qBAAO;AAAA,gBACL,MAAM,SAAS;AAAA,gBACf,OAAO;AAAA,cACT;AAAA,YACF;AAEA,mBAAO;AAAA,UACT,CAAC;AAAA,QAEH;AAAA,MACF;AAGA,YAAM,cAAc,UAAU,EAAE;AAChC,aAAO;AAAA,IACT,CAAC;AACD,WAAO;AAAA,EACT;AAIA,QAAM,WAAW,WAAW,WAAW;AACvC,QAAM,eAAe,SAAS,MAAM,GAAG;AACvC,YAAU,QAAQ,UAAU,EAAC,OAAO,EAAC,CAAC,CAAC;AAMvC,WAAS,aAAa,OAAO;AAC3B,QAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,aAAO,MAAM,IAAI,YAAY;AAAA,IAC/B;AACA,QAAI,SAAS,OAAO,UAAU,UAAU;AACtC,aAAO,OAAO,KAAK,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,WAAW,QAAQ;AAC1D,kBAAU,GAAG,IAAI,aAAa,MAAM,GAAG,CAAC;AACxC,eAAO;AAAA,MACT,GAAG,CAAC,CAAC;AAAA,IACP;AACA,WAAO;AAAA,EACT;AAEA,WAAS,eAAe,GAAG,GAAG;AAC5B,WAAO,KAAK,UAAU,aAAa,CAAC,CAAC,MAAM,KAAK,UAAU,aAAa,CAAC,CAAC;AAAA,EAC3E;AAEA,WAAS,WAAW,QAAQ;AAC1B,WAAO,OAAO,OAAO,CAAC,MAAM,UAAU;AACpC,UAAI,CAAC,KAAK,KAAK,OAAK,eAAe,GAAG,KAAK,CAAC,GAAG;AAC7C,aAAK,KAAK,KAAK;AAAA,MACjB;AAEA,aAAO;AAAA,IACT,GAAG,CAAC,CAAC;AAAA,EACP;AAGA,YAAU,aAAa,YAAY;AACnC,YAAU,aAAa,QAAQ;AAE/B,SAAO;AAQP,WAAS,eAAe,MAAM;AAC5B,UAAM,CAAC,GAAG,IAAI,KAAK,UAAU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,EAAE,IAAI,cAAY,SAAS,KAAK;AACvG,UAAM,aAAa;AACnB,cAAU,gBAAgB,UAAU,GAAG;AACvC,WAAO;AAAA,EACT;AAEA,WAAS,WAAW,MAAM;AACxB,UAAM,CAAC,GAAG,IAAI,KAAK,UAAU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,EAAE,IAAI,cAAY,SAAS,KAAK;AAEvG,UAAM,UAAU,OAAO,IAAI,QAAQ,SAAS,EAAE;AAC9C,QAAI,UAAU,OAAO,WAAW,IAAI;AAClC,gBAAU,YAAY,MAAM,GAAG;AAC/B,aAAO;AAAA,IACT;AACA,UAAM,cAAc;AACpB,cAAU,YAAY,WAAW,GAAG;AACpC,WAAO;AAAA,EACT;AAEA,WAAS,SAAS,MAAM;AACtB,UAAM,CAAC,GAAG,IAAI,KAAK,UAAU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,EAAE,IAAI,cAAY,SAAS,KAAK;AAEvG,UAAM,QAAQ,OAAO,IAAI,QAAQ,SAAS,EAAE;AAC5C,QAAI,QAAQ,KAAK,WAAW,IAAI;AAC9B,gBAAU,UAAU,IAAI,GAAG;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,YAAY;AAClB,cAAU,UAAU,SAAS,GAAG;AAChC,WAAO;AAAA,EACT;AAEA,WAAS,UAAU,MAAM;AACvB,UAAM,CAAC,GAAG,IAAI,KAAK,UAAU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,EAAE,IAAI,cAAY,SAAS,KAAK;AACvG,QAAI,OAAQ,OAAQ,KAAK,GAAG,GAAG;AAC7B,YAAMC,aAAY,IAAI,QAAQ,WAAW,EAAE,EAAE,SAAS,GAAG,GAAG;AAC5D,YAAMC,YAAW,IAAI,QAAQ,WAAW,EAAE,EAAE,SAAS,GAAG,GAAG;AAC3D,gBAAU,eAAeD,UAAS,iBAAiBC,SAAQ,GAAG;AAC9D,aAAO,EAAC,WAAAD,YAAW,UAAAC,UAAQ;AAAA,IAC7B;AAIA,UAAM,YAAY,MAAM,IAAI,SAAS,GAAG,GAAG,IAAI;AAC/C,UAAM,WAAW,MAAM,IAAI,SAAS,GAAG,GAAG,IAAI;AAE9C,cAAU,eAAe,SAAS,iBAAiB,QAAQ,GAAG;AAC9D,WAAO,EAAC,WAAW,SAAQ;AAAA,EAC7B;AAEF;",
|
|
6
6
|
"names": ["results", "firstDate", "lastDate"]
|
|
7
7
|
}
|
package/dist/helpers/fix.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import createDebugLogger from "debug";
|
|
2
2
|
import { MarcRecord } from "@natlibfi/marc-record";
|
|
3
|
-
import { createLogger } from "@natlibfi/melinda-
|
|
3
|
+
import { createLogger } from "@natlibfi/melinda-commons";
|
|
4
4
|
import { FieldExclusion, SubfieldExclusion } from "@natlibfi/marc-record-validators-melinda";
|
|
5
5
|
import { handleTempUrns } from "./fix-handle-tempurns.js";
|
|
6
6
|
import { stripF884s } from "./fix-strip-f884s.js";
|
|
@@ -50,8 +50,12 @@ function replacePrefix(record, options) {
|
|
|
50
50
|
function generateMissingSIDs(record, options) {
|
|
51
51
|
const f035Filters = options && options.f035Filters ? options.f035Filters : [];
|
|
52
52
|
const fSIDFilters = options && options.fSIDFilters ? options.fSIDFilters : [];
|
|
53
|
-
if (f035Filters.length !== fSIDFilters.length
|
|
54
|
-
debug(`NOT running
|
|
53
|
+
if (f035Filters.length !== fSIDFilters.length) {
|
|
54
|
+
debug(`NOT running generateMissingSIDs: f035Filters (${f035Filters.length}) and fSIDFilters (${fSIDFilters.length}) lengths differ`);
|
|
55
|
+
return record;
|
|
56
|
+
}
|
|
57
|
+
if (f035Filters.length < 1) {
|
|
58
|
+
debug(`NOT running generateMissingSIDs, no options`);
|
|
55
59
|
return record;
|
|
56
60
|
}
|
|
57
61
|
debug(`Running generateMissingSIDs fixer`);
|
|
@@ -80,10 +84,11 @@ function generateMissingSIDs(record, options) {
|
|
|
80
84
|
}
|
|
81
85
|
function f035ToSidInfo(f035s2) {
|
|
82
86
|
return f035s2.flatMap(({ subfields }) => {
|
|
83
|
-
const
|
|
87
|
+
const wantedSub = subfields.find((sub) => sub.code === "a" && f035Filters.findIndex((regexp) => regexp.test(sub.value)) > -1);
|
|
84
88
|
if (wantedSub) {
|
|
89
|
+
const filterIndex = f035Filters.findIndex((regexp) => regexp.test(wantedSub.value));
|
|
85
90
|
return {
|
|
86
|
-
SID: fSIDFilters[
|
|
91
|
+
SID: fSIDFilters[filterIndex],
|
|
87
92
|
value: wantedSub.value.slice(wantedSub.value.indexOf(")") + 1)
|
|
88
93
|
};
|
|
89
94
|
}
|
|
@@ -104,7 +109,6 @@ function stripF984s(record, stripF984sSettings) {
|
|
|
104
109
|
]
|
|
105
110
|
}
|
|
106
111
|
];
|
|
107
|
-
debug(`Running removeF984AFields`);
|
|
108
112
|
debugData(`removeF984AFieldsSettings: ${JSON.stringify(stripF984sSettings)}`);
|
|
109
113
|
if (!stripF984sSettings || !F984A_PATTERN || record.get("984").length < 1) {
|
|
110
114
|
debug(`No settings or pattern for removeF984AFields or f984 in record`);
|
|
@@ -120,8 +124,8 @@ function stripMelindaTempSubfields(record, stripMelindaTempSubfieldsSettings) {
|
|
|
120
124
|
debug(`No settings for stripMelindaTempSubfields`);
|
|
121
125
|
return record;
|
|
122
126
|
}
|
|
123
|
-
const tagPattern =
|
|
124
|
-
const subfieldCodePattern =
|
|
127
|
+
const tagPattern = /^856$/u;
|
|
128
|
+
const subfieldCodePattern = /^9$/u;
|
|
125
129
|
const subfieldValuePattern = /^MELINDA<TEMP>$/u;
|
|
126
130
|
const config = [
|
|
127
131
|
{
|
package/dist/helpers/fix.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/helpers/fix.js"],
|
|
4
|
-
"sourcesContent": ["import createDebugLogger from 'debug';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {createLogger} from '@natlibfi/melinda-
|
|
5
|
-
"mappings": "AAAA,OAAO,uBAAuB;AAC9B,SAAQ,kBAAiB;AACzB,SAAQ,oBAAmB;AAE3B,SAAQ,gBAAgB,yBAAwB;AAEhD,SAAQ,sBAAqB;AAC7B,SAAQ,kBAAiB;AAEzB,cAAc;AAEd,MAAM,SAAS,aAAa;AAC5B,MAAM,QAAQ,kBAAkB,8CAA8C;AAC9E,MAAM,YAAY,MAAM,OAAO,MAAM;AAE9B,gBAAS,UAAU,QAAQ,WAAW,CAAC,GAAG;AAE/C,SAAO,QAAQ,iEAAiE,KAAK,UAAU,QAAQ,CAAC,EAAE;AAC1G,QAAM,YAAY,WAAW,MAAM,QAAQ,EAAC,gBAAgB,MAAK,CAAC;AAClE,YAAU,aAAa,KAAK,UAAU,QAAQ,CAAC,EAAE;AAKjD,QAAM,6BAA6B,SAAS,uBAAuB;AACnE,QAAM,aAAa,oBAAoB,WAAW,0BAA0B;AAG5E,QAAM,yBAAyB,SAAS,mBAAmB,CAAC;AAC5D,QAAM,aAAa,mBAAmB,YAAY,sBAAsB;AAGxE,QAAM,aAAa,eAAe,YAAY,SAAS,cAAc;AAGrE,QAAM,aAAa,WAAW,YAAY,SAAS,UAAU;AAG7D,QAAM,aAAa,WAAW,YAAY,SAAS,cAAc,KAAK;AAGtE,QAAM,aAAa,0BAA0B,YAAY,SAAS,6BAA6B,KAAK;AAEpG,YAAU,UAAU;AAEpB,SAAO,WAAW,SAAS;AAE7B;AAGA,SAAS,mBAAmB,QAAQ,wBAAwB;AAC1D,MAAI,uBAAuB,SAAS,GAAG;AACrC,UAAM,yCAAyC;AAC/C,WAAO;AAAA,EACT;AAEA,QAAM,+BAA+B;AACrC,YAAU,2BAA2B,KAAK,UAAU,sBAAsB,CAAC,EAAE;AAE7E,yBAAuB,QAAQ,aAAW;AACxC,kBAAc,QAAQ,OAAO;AAAA,EAC/B,CAAC;AACD,SAAO;AACT;AAIA,SAAS,cAAc,QAAQ,SAAS;AACtC,QAAM,EAAC,WAAW,WAAW,mBAAkB,IAAI;AACnD,QAAM,aAAa,SAAS,SAAS,SAAS,iBAAiB,kBAAkB,EAAE;AACnF,QAAM,UAAU,IAAI,SAAS;AAC7B,QAAM,cAAc,IAAI,SAAS;AACjC,SAAO,cAAc,EAClB,QAAQ,WAAS;AAChB,UAAM,UACH,OAAO,CAAC,EAAC,KAAI,MAAM,mBAAmB,SAAS,IAAI,CAAC,EACpD,QAAQ,cAAY;AACnB,eAAS,QAAQ,SAAS,MAAM,QAAQ,SAAS,WAAW;AAAA,IAC9D,CAAC;AAAA,EACL,CAAC;AACH,SAAO;AACT;AAGA,SAAS,oBAAoB,QAAQ,SAAS;AAE5C,QAAM,cAAc,WAAW,QAAQ,cAAc,QAAQ,cAAc,CAAC;AAC5E,QAAM,cAAc,WAAW,QAAQ,cAAc,QAAQ,cAAc,CAAC;AAE5E,MAAI,YAAY,WAAW,YAAY,
|
|
4
|
+
"sourcesContent": ["import createDebugLogger from 'debug';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {createLogger} from '@natlibfi/melinda-commons';\n\nimport {FieldExclusion, SubfieldExclusion} from '@natlibfi/marc-record-validators-melinda';\n\nimport {handleTempUrns} from './fix-handle-tempurns.js';\nimport {stripF884s} from './fix-strip-f884s.js';\n\nexport * from './fix-constants.js';\n\nconst logger = createLogger();\nconst debug = createDebugLogger('@natlibfi/melinda-rest-api-commons:fixRecord');\nconst debugData = debug.extend('data');\n\nexport function fixRecord(record, settings = {}) {\n\n logger.verbose(`We will apply formatting to the record according to settings: ${JSON.stringify(settings)}`);\n const newRecord = MarcRecord.clone(record, {subfieldValues: false});\n debugData(`settings: ${JSON.stringify(settings)}`);\n\n // We should handle these in some prettier way ...\n\n // Fix 1, generate SIDs\n const generateMissingSIDsOptions = settings.generateMissingSIDs || undefined;\n const newRecord1 = generateMissingSIDs(newRecord, generateMissingSIDsOptions);\n\n // Fix 2, replace identifier prefixes\n const replacePrefixesOptions = settings.replacePrefixes || [];\n const newRecord2 = replaceAllPrefixes(newRecord1, replacePrefixesOptions);\n\n // Fix 3, handle temporary URNs from legal deposit record import\n const newRecord3 = handleTempUrns(newRecord2, settings.handleTempUrns);\n\n // Fix 4, strip extra field 884s\n const newRecord4 = stripF884s(newRecord3, settings.stripF884s);\n\n // Fix 5, strip extra field 984s\n const newRecord5 = stripF984s(newRecord4, settings.stripF984s || false);\n\n // Fix 6, strip $9 MELINDA<TEMP> subfields\n const newRecord6 = stripMelindaTempSubfields(newRecord5, settings.stripMelindaTempSubfields || false);\n\n debugData(newRecord6);\n\n return newRecord6.toObject();\n\n}\n\n\nfunction replaceAllPrefixes(record, replacePrefixesOptions) {\n if (replacePrefixesOptions.length < 1) {\n debug(`NOT running replacePrefixes, no options`);\n return record;\n }\n\n debug(`Running replacePrefixes fixer`);\n debugData(`replacePrefixesOptions: ${JSON.stringify(replacePrefixesOptions)}`);\n\n replacePrefixesOptions.forEach(options => {\n replacePrefix(record, options);\n });\n return record;\n}\n\n\n// Replace prefix in all specified subfields\nfunction replacePrefix(record, options) {\n const {oldPrefix, newPrefix, prefixReplaceCodes} = options;\n debug(`Replacing ${oldPrefix} with ${newPrefix} in subfields ${prefixReplaceCodes}`);\n const pattern = `(${oldPrefix})`;\n const replacement = `(${newPrefix})`;\n record.getDatafields()\n .forEach(field => {\n field.subfields\n .filter(({code}) => prefixReplaceCodes.includes(code))\n .forEach(subfield => {\n subfield.value = subfield.value.replace(pattern, replacement);\n });\n });\n return record;\n}\n\n\nfunction generateMissingSIDs(record, options) {\n\n const f035Filters = options && options.f035Filters ? options.f035Filters : [];\n const fSIDFilters = options && options.fSIDFilters ? options.fSIDFilters : [];\n\n if (f035Filters.length !== fSIDFilters.length) {\n debug(`NOT running generateMissingSIDs: f035Filters (${f035Filters.length}) and fSIDFilters (${fSIDFilters.length}) lengths differ`);\n return record;\n }\n if (f035Filters.length < 1) {\n debug(`NOT running generateMissingSIDs, no options`);\n return record;\n }\n\n debug(`Running generateMissingSIDs fixer`);\n debugData(`generateMissingSIDs options: ${JSON.stringify(options)}`);\n\n const f035s = f035ToSidInfo(record.get(/^035$/u));\n const fSIDs = sidsToSidInfo(record.get(/^SID$/u));\n\n const sidsToBeAdded = genNewSids(fSIDs.length === 0\n ? f035s\n // test that SIDs are not there yet\n : f035s.filter(f035SidInfo => !fSIDs.some(fSIDInfo => f035SidInfo.SID === fSIDInfo.SID && f035SidInfo.value === fSIDInfo.value)));\n\n debugData(`Adding (${sidsToBeAdded.length}) SIDs: ${JSON.stringify(sidsToBeAdded)}`);\n\n // Add new SIDs\n sidsToBeAdded.forEach(sidField => record.insertField(sidField));\n return record;\n\n function genNewSids(sidsToBeAdded) {\n return sidsToBeAdded.map(sidInfo => ({tag: 'SID', ind1: ' ', ind2: ' ', subfields: [{code: 'c', value: sidInfo.value}, {code: 'b', value: sidInfo.SID}]}));\n }\n\n function sidsToSidInfo(SIDs) {\n return SIDs.flatMap(({subfields}) => {\n const [SID] = subfields.filter(sub => sub.code === 'b' && fSIDFilters.includes(sub.value)).map(sub => sub.value);\n const [value] = subfields.filter(sub => sub.code === 'c' && sub.value !== undefined).map(sub => sub.value);\n\n if (SID && value) {\n return {\n SID,\n value\n };\n }\n\n return undefined;\n }).filter(value => value !== undefined);\n }\n\n function f035ToSidInfo(f035s) {\n return f035s.flatMap(({subfields}) => {\n // single pass over the filters - the subfield AND the SID are\n // both chosen by the SAME first matching filter index (previously the\n // subfield matched any filter but the SID lookup re-scanned with\n // findIndex, which was order-sensitive and O(n^2) per field)\n const wantedSub = subfields.find(sub => sub.code === 'a' && f035Filters.findIndex(regexp => regexp.test(sub.value)) > -1);\n\n if (wantedSub) {\n const filterIndex = f035Filters.findIndex(regexp => regexp.test(wantedSub.value));\n return {\n SID: fSIDFilters[filterIndex],\n value: wantedSub.value.slice(wantedSub.value.indexOf(')') + 1)\n };\n }\n\n return undefined;\n }).filter(value => value !== undefined);\n }\n}\n\n\nfunction stripF984s(record, stripF984sSettings) {\n const F984A_PATTERN = /^(?:ALWAYS|NEVER)-PREFER-IN-MERGE$/u;\n const config = [\n {\n tag: /^984$/u,\n subfields: [\n {\n code: /a/u,\n value: F984A_PATTERN\n }\n ]\n }\n ];\n\n debugData(`removeF984AFieldsSettings: ${JSON.stringify(stripF984sSettings)}`);\n\n if (!stripF984sSettings || !F984A_PATTERN || record.get('984').length < 1) {\n debug(`No settings or pattern for removeF984AFields or f984 in record`);\n return record;\n }\n\n debug(`Removing f984 fields with subfield $a consisting values ${F984A_PATTERN.toString()}`);\n const validator = FieldExclusion(config);\n validator.fix(record);\n return record;\n\n}\n\nfunction stripMelindaTempSubfields(record, stripMelindaTempSubfieldsSettings) {\n if (!stripMelindaTempSubfieldsSettings) {\n debug(`No settings for stripMelindaTempSubfields`);\n return record;\n }\n // NOTE: we remove $9 MELINDA<TEMP> subfields only from f856\n // Anchored so e.g. tag '8560'-ish values or multi-char subfield codes can\n // never over-match (SubfieldExclusion tests these RegExps, not replaces)\n const tagPattern = /^856$/u;\n const subfieldCodePattern = /^9$/u;\n const subfieldValuePattern = /^MELINDA<TEMP>$/u;\n const config = [\n {\n tag: tagPattern,\n subfields: [\n {\n code: subfieldCodePattern,\n value: subfieldValuePattern\n }\n ]\n }\n ];\n\n debug(`Running stripMelindaTempSubfields`);\n\n const validator = SubfieldExclusion(config);\n validator.fix(record);\n return record;\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,uBAAuB;AAC9B,SAAQ,kBAAiB;AACzB,SAAQ,oBAAmB;AAE3B,SAAQ,gBAAgB,yBAAwB;AAEhD,SAAQ,sBAAqB;AAC7B,SAAQ,kBAAiB;AAEzB,cAAc;AAEd,MAAM,SAAS,aAAa;AAC5B,MAAM,QAAQ,kBAAkB,8CAA8C;AAC9E,MAAM,YAAY,MAAM,OAAO,MAAM;AAE9B,gBAAS,UAAU,QAAQ,WAAW,CAAC,GAAG;AAE/C,SAAO,QAAQ,iEAAiE,KAAK,UAAU,QAAQ,CAAC,EAAE;AAC1G,QAAM,YAAY,WAAW,MAAM,QAAQ,EAAC,gBAAgB,MAAK,CAAC;AAClE,YAAU,aAAa,KAAK,UAAU,QAAQ,CAAC,EAAE;AAKjD,QAAM,6BAA6B,SAAS,uBAAuB;AACnE,QAAM,aAAa,oBAAoB,WAAW,0BAA0B;AAG5E,QAAM,yBAAyB,SAAS,mBAAmB,CAAC;AAC5D,QAAM,aAAa,mBAAmB,YAAY,sBAAsB;AAGxE,QAAM,aAAa,eAAe,YAAY,SAAS,cAAc;AAGrE,QAAM,aAAa,WAAW,YAAY,SAAS,UAAU;AAG7D,QAAM,aAAa,WAAW,YAAY,SAAS,cAAc,KAAK;AAGtE,QAAM,aAAa,0BAA0B,YAAY,SAAS,6BAA6B,KAAK;AAEpG,YAAU,UAAU;AAEpB,SAAO,WAAW,SAAS;AAE7B;AAGA,SAAS,mBAAmB,QAAQ,wBAAwB;AAC1D,MAAI,uBAAuB,SAAS,GAAG;AACrC,UAAM,yCAAyC;AAC/C,WAAO;AAAA,EACT;AAEA,QAAM,+BAA+B;AACrC,YAAU,2BAA2B,KAAK,UAAU,sBAAsB,CAAC,EAAE;AAE7E,yBAAuB,QAAQ,aAAW;AACxC,kBAAc,QAAQ,OAAO;AAAA,EAC/B,CAAC;AACD,SAAO;AACT;AAIA,SAAS,cAAc,QAAQ,SAAS;AACtC,QAAM,EAAC,WAAW,WAAW,mBAAkB,IAAI;AACnD,QAAM,aAAa,SAAS,SAAS,SAAS,iBAAiB,kBAAkB,EAAE;AACnF,QAAM,UAAU,IAAI,SAAS;AAC7B,QAAM,cAAc,IAAI,SAAS;AACjC,SAAO,cAAc,EAClB,QAAQ,WAAS;AAChB,UAAM,UACH,OAAO,CAAC,EAAC,KAAI,MAAM,mBAAmB,SAAS,IAAI,CAAC,EACpD,QAAQ,cAAY;AACnB,eAAS,QAAQ,SAAS,MAAM,QAAQ,SAAS,WAAW;AAAA,IAC9D,CAAC;AAAA,EACL,CAAC;AACH,SAAO;AACT;AAGA,SAAS,oBAAoB,QAAQ,SAAS;AAE5C,QAAM,cAAc,WAAW,QAAQ,cAAc,QAAQ,cAAc,CAAC;AAC5E,QAAM,cAAc,WAAW,QAAQ,cAAc,QAAQ,cAAc,CAAC;AAE5E,MAAI,YAAY,WAAW,YAAY,QAAQ;AAC7C,UAAM,iDAAiD,YAAY,MAAM,sBAAsB,YAAY,MAAM,kBAAkB;AACnI,WAAO;AAAA,EACT;AACA,MAAI,YAAY,SAAS,GAAG;AAC1B,UAAM,6CAA6C;AACnD,WAAO;AAAA,EACT;AAEA,QAAM,mCAAmC;AACzC,YAAU,gCAAgC,KAAK,UAAU,OAAO,CAAC,EAAE;AAEnE,QAAM,QAAQ,cAAc,OAAO,IAAI,QAAQ,CAAC;AAChD,QAAM,QAAQ,cAAc,OAAO,IAAI,QAAQ,CAAC;AAEhD,QAAM,gBAAgB,WAAW,MAAM,WAAW,IAC9C,QAEA,MAAM,OAAO,iBAAe,CAAC,MAAM,KAAK,cAAY,YAAY,QAAQ,SAAS,OAAO,YAAY,UAAU,SAAS,KAAK,CAAC,CAAC;AAElI,YAAU,WAAW,cAAc,MAAM,WAAW,KAAK,UAAU,aAAa,CAAC,EAAE;AAGnF,gBAAc,QAAQ,cAAY,OAAO,YAAY,QAAQ,CAAC;AAC9D,SAAO;AAEP,WAAS,WAAWA,gBAAe;AACjC,WAAOA,eAAc,IAAI,cAAY,EAAC,KAAK,OAAO,MAAM,KAAK,MAAM,KAAK,WAAW,CAAC,EAAC,MAAM,KAAK,OAAO,QAAQ,MAAK,GAAG,EAAC,MAAM,KAAK,OAAO,QAAQ,IAAG,CAAC,EAAC,EAAE;AAAA,EAC3J;AAEA,WAAS,cAAc,MAAM;AAC3B,WAAO,KAAK,QAAQ,CAAC,EAAC,UAAS,MAAM;AACnC,YAAM,CAAC,GAAG,IAAI,UAAU,OAAO,SAAO,IAAI,SAAS,OAAO,YAAY,SAAS,IAAI,KAAK,CAAC,EAAE,IAAI,SAAO,IAAI,KAAK;AAC/G,YAAM,CAAC,KAAK,IAAI,UAAU,OAAO,SAAO,IAAI,SAAS,OAAO,IAAI,UAAU,MAAS,EAAE,IAAI,SAAO,IAAI,KAAK;AAEzG,UAAI,OAAO,OAAO;AAChB,eAAO;AAAA,UACL;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAEA,aAAO;AAAA,IACT,CAAC,EAAE,OAAO,WAAS,UAAU,MAAS;AAAA,EACxC;AAEA,WAAS,cAAcC,QAAO;AAC5B,WAAOA,OAAM,QAAQ,CAAC,EAAC,UAAS,MAAM;AAKpC,YAAM,YAAY,UAAU,KAAK,SAAO,IAAI,SAAS,OAAO,YAAY,UAAU,YAAU,OAAO,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE;AAExH,UAAI,WAAW;AACb,cAAM,cAAc,YAAY,UAAU,YAAU,OAAO,KAAK,UAAU,KAAK,CAAC;AAChF,eAAO;AAAA,UACL,KAAK,YAAY,WAAW;AAAA,UAC5B,OAAO,UAAU,MAAM,MAAM,UAAU,MAAM,QAAQ,GAAG,IAAI,CAAC;AAAA,QAC/D;AAAA,MACF;AAEA,aAAO;AAAA,IACT,CAAC,EAAE,OAAO,WAAS,UAAU,MAAS;AAAA,EACxC;AACF;AAGA,SAAS,WAAW,QAAQ,oBAAoB;AAC9C,QAAM,gBAAgB;AACtB,QAAM,SAAS;AAAA,IACb;AAAA,MACE,KAAK;AAAA,MACL,WAAW;AAAA,QACT;AAAA,UACE,MAAM;AAAA,UACN,OAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,YAAU,8BAA8B,KAAK,UAAU,kBAAkB,CAAC,EAAE;AAE5E,MAAI,CAAC,sBAAsB,CAAC,iBAAiB,OAAO,IAAI,KAAK,EAAE,SAAS,GAAG;AACzE,UAAM,gEAAgE;AACtE,WAAO;AAAA,EACT;AAEA,QAAM,2DAA2D,cAAc,SAAS,CAAC,EAAE;AAC3F,QAAM,YAAY,eAAe,MAAM;AACvC,YAAU,IAAI,MAAM;AACpB,SAAO;AAET;AAEA,SAAS,0BAA0B,QAAQ,mCAAmC;AAC5E,MAAI,CAAC,mCAAmC;AACtC,UAAM,2CAA2C;AACjD,WAAO;AAAA,EACT;AAIA,QAAM,aAAa;AACnB,QAAM,sBAAsB;AAC5B,QAAM,uBAAuB;AAC7B,QAAM,SAAS;AAAA,IACb;AAAA,MACE,KAAK;AAAA,MACL,WAAW;AAAA,QACT;AAAA,UACE,MAAM;AAAA,UACN,OAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,mCAAmC;AAEzC,QAAM,YAAY,kBAAkB,MAAM;AAC1C,YAAU,IAAI,MAAM;AACpB,SAAO;AACT;",
|
|
6
6
|
"names": ["sidsToBeAdded", "f035s"]
|
|
7
7
|
}
|
|
@@ -2,7 +2,7 @@ import validateFactory from "@natlibfi/marc-record-validate";
|
|
|
2
2
|
import {
|
|
3
3
|
FieldStructure as fieldStructure
|
|
4
4
|
} from "@natlibfi/marc-record-validators-melinda";
|
|
5
|
-
import { createLogger } from "@natlibfi/melinda-
|
|
5
|
+
import { createLogger } from "@natlibfi/melinda-commons";
|
|
6
6
|
export default async () => {
|
|
7
7
|
const logger = createLogger();
|
|
8
8
|
logger.verbose("Running inbuilt record validations");
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/helpers/validation.js"],
|
|
4
|
-
"sourcesContent": ["import validateFactory from '@natlibfi/marc-record-validate';\nimport {\n FieldStructure as fieldStructure\n} from '@natlibfi/marc-record-validators-melinda';\nimport {createLogger} from '@natlibfi/melinda-
|
|
4
|
+
"sourcesContent": ["import validateFactory from '@natlibfi/marc-record-validate';\nimport {\n FieldStructure as fieldStructure\n} from '@natlibfi/marc-record-validators-melinda';\nimport {createLogger} from '@natlibfi/melinda-commons';\n\nexport default async () => {\n const logger = createLogger();\n\n logger.verbose('Running inbuilt record validations');\n const validate = validateFactory([await fieldStructure([{tag: /^003$/u, valuePattern: /^FI-MELINDA$/u}])]);\n\n return async unvalidRecord => {\n const {record, valid, report} = await validate(unvalidRecord, {fix: false, validateFixes: false}, {subfieldValues: false});\n\n return {\n record,\n failed: valid === false,\n messages: report\n };\n };\n};\n"],
|
|
5
5
|
"mappings": "AAAA,OAAO,qBAAqB;AAC5B;AAAA,EACE,kBAAkB;AAAA,OACb;AACP,SAAQ,oBAAmB;AAE3B,eAAe,YAAY;AACzB,QAAM,SAAS,aAAa;AAE5B,SAAO,QAAQ,oCAAoC;AACnD,QAAM,WAAW,gBAAgB,CAAC,MAAM,eAAe,CAAC,EAAC,KAAK,UAAU,cAAc,gBAAe,CAAC,CAAC,CAAC,CAAC;AAEzG,SAAO,OAAM,kBAAiB;AAC5B,UAAM,EAAC,QAAQ,OAAO,OAAM,IAAI,MAAM,SAAS,eAAe,EAAC,KAAK,OAAO,eAAe,MAAK,GAAG,EAAC,gBAAgB,MAAK,CAAC;AAEzH,WAAO;AAAA,MACL;AAAA,MACA,QAAQ,UAAU;AAAA,MAClB,UAAU;AAAA,IACZ;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|