@natlibfi/melinda-rest-api-commons 6.1.0-alpha.0 → 6.2.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/melinda-node-tests-and-publish.yml +4 -2
- package/README.md +211 -14
- package/dist/amqp.d.ts +41 -0
- package/dist/amqp.d.ts.map +1 -0
- package/dist/amqp.js +251 -235
- package/dist/amqp.js.map +1 -7
- package/dist/constants.d.ts +49 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +38 -38
- package/dist/constants.js.map +1 -7
- package/dist/helpers/conversion.d.ts +8 -0
- package/dist/helpers/conversion.d.ts.map +1 -0
- package/dist/helpers/conversion.js +70 -64
- package/dist/helpers/conversion.js.map +1 -7
- package/dist/helpers/fix-constants.d.ts +76 -0
- package/dist/helpers/fix-constants.d.ts.map +1 -0
- package/dist/helpers/fix-constants.js +41 -41
- package/dist/helpers/fix-constants.js.map +1 -7
- package/dist/helpers/fix-handle-tempurns.d.ts +3 -0
- package/dist/helpers/fix-handle-tempurns.d.ts.map +1 -0
- package/dist/helpers/fix-handle-tempurns.js +73 -71
- package/dist/helpers/fix-handle-tempurns.js.map +1 -7
- package/dist/helpers/fix-strip-f884s.d.ts +10 -0
- package/dist/helpers/fix-strip-f884s.d.ts.map +1 -0
- package/dist/helpers/fix-strip-f884s.js +161 -163
- package/dist/helpers/fix-strip-f884s.js.map +1 -7
- package/dist/helpers/fix.d.ts +6 -0
- package/dist/helpers/fix.d.ts.map +1 -0
- package/dist/helpers/fix.js +133 -121
- package/dist/helpers/fix.js.map +1 -7
- package/dist/helpers/validation.d.ts +9 -0
- package/dist/helpers/validation.d.ts.map +1 -0
- package/dist/helpers/validation.js +14 -16
- package/dist/helpers/validation.js.map +1 -7
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -7
- package/dist/mongo.d.ts +165 -0
- package/dist/mongo.d.ts.map +1 -0
- package/dist/mongo.js +595 -547
- package/dist/mongo.js.map +1 -7
- package/dist/mongoError.d.ts +2 -0
- package/dist/mongoError.d.ts.map +1 -0
- package/dist/mongoError.js +19 -0
- package/dist/mongoError.js.map +1 -0
- package/dist/mongoLog.d.ts +50 -0
- package/dist/mongoLog.d.ts.map +1 -0
- package/dist/mongoLog.js +274 -242
- package/dist/mongoLog.js.map +1 -7
- package/dist/utils.d.ts +36 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +112 -110
- package/dist/utils.js.map +1 -7
- package/eslint.config.ts +66 -0
- package/package.json +17 -11
- package/{tests/amqp-error-branches.test.js → test/amqp-error-branches.test.ts} +135 -61
- package/{tests/amqp.healthtest.js → test/amqp.healthtest.ts} +19 -11
- package/{tests/amqp.tests.localtest.js → test/amqp.tests.localtest.ts} +54 -35
- package/{tests/helpers/conversion.test.js → test/helpers/conversion.test.ts} +26 -14
- package/{tests/helpers/fix-edge.test.js → test/helpers/fix-edge.test.ts} +16 -12
- package/{tests/helpers/fix.test.js → test/helpers/fix.test.ts} +14 -8
- package/{tests/helpers/validation.test.js → test/helpers/validation.test.ts} +7 -6
- package/{tests/mongo-add-blob-size.test.js → test/mongo-add-blob-size.test.ts} +14 -15
- package/{tests/mongo-check-and-check-state.test.js → test/mongo-check-and-check-state.test.ts} +6 -8
- package/{tests/mongo-check-and-set-import-job-state.test.js → test/mongo-check-and-set-import-job-state.test.ts} +6 -9
- package/{tests/mongo-check-timeout.test.js → test/mongo-check-timeout.test.ts} +6 -9
- package/{tests/mongo-create-bulk.test.js → test/mongo-create-bulk.test.ts} +6 -7
- package/{tests/mongo-create-prio.test.js → test/mongo-create-prio.test.ts} +6 -6
- package/{tests/mongo-error-branches.test.js → test/mongo-error-branches.test.ts} +166 -65
- package/{tests/mongo-get-one.test.js → test/mongo-get-one.test.ts} +6 -6
- package/{tests/mongo-get-stream.test.js → test/mongo-get-stream.test.ts} +6 -8
- package/{tests/mongo-push-ids.test.js → test/mongo-push-ids.test.ts} +6 -6
- package/{tests/mongo-push-messages.test.js → test/mongo-push-messages.test.ts} +6 -6
- package/{tests/mongo-query-by-id.test.js → test/mongo-query-by-id.test.ts} +6 -9
- package/{tests/mongo-query.test.js → test/mongo-query.test.ts} +6 -7
- package/{tests/mongo-read-content.test.js → test/mongo-read-content.test.ts} +6 -9
- package/{tests/mongo-remove.test.js → test/mongo-remove.test.ts} +6 -8
- package/{tests/mongo-set-blob-size.test.js → test/mongo-set-blob-size.test.ts} +6 -6
- package/{tests/mongo-set-import-job-state.test.js → test/mongo-set-import-job-state.test.ts} +6 -6
- package/{tests/mongo-set-operation.test.js → test/mongo-set-operation.test.ts} +6 -6
- package/{tests/mongo-set-operations.test.js → test/mongo-set-operations.test.ts} +6 -6
- package/{tests/mongo-set-state.test.js → test/mongo-set-state.test.ts} +6 -6
- package/{tests/mongoLog-protect.test.js → test/mongoLog-protect.test.ts} +18 -17
- package/{tests/mongoLog.test.js → test/mongoLog.test.ts} +8 -8
- package/{tests/testUtils.js → test/testUtils.ts} +49 -15
- package/{tests/utils.test.js → test/utils.test.ts} +5 -5
- package/tsconfig.json +62 -0
- package/tsconfig.test.json +17 -0
- package/dist/helpers/fix-utils.js +0 -39
- package/dist/helpers/fix-utils.js.map +0 -7
- package/eslint.config.js +0 -53
|
@@ -28,7 +28,6 @@ jobs:
|
|
|
28
28
|
cache: 'npm'
|
|
29
29
|
env:
|
|
30
30
|
NPM_CONFIG_IGNORE_SCRIPTS: true
|
|
31
|
-
- run: npm update
|
|
32
31
|
- run: npm audit --package-lock-only --production --audit-level=moderate
|
|
33
32
|
- run: npm i
|
|
34
33
|
- run: npm test
|
|
@@ -41,8 +40,11 @@ jobs:
|
|
|
41
40
|
|
|
42
41
|
steps:
|
|
43
42
|
- uses: actions/checkout@v7
|
|
44
|
-
- uses: mikaelvesavuori/license-compliance-action@v1
|
|
43
|
+
- uses: mikaelvesavuori/license-compliance-action@v1.0.3
|
|
45
44
|
with:
|
|
45
|
+
# default action list + BlueOak-1.0.0 (sax, via xml2js in
|
|
46
|
+
# marc-record-serializers) + MIT-0 — OSI-approved permissive
|
|
47
|
+
allow_licenses: 'MIT;ISC;0BSD;BSD-2-Clause;BSD-3-Clause;Apache-2.0;BlueOak-1.0.0;MIT-0'
|
|
46
48
|
exclude_pattern: /^@natlibfi/
|
|
47
49
|
|
|
48
50
|
njsscan:
|
package/README.md
CHANGED
|
@@ -4,6 +4,154 @@
|
|
|
4
4
|
|
|
5
5
|
Shared modules for microservices of Melinda rest api import system
|
|
6
6
|
|
|
7
|
+
## API
|
|
8
|
+
|
|
9
|
+
Everything is exported from the package root (`@natlibfi/melinda-rest-api-commons`):
|
|
10
|
+
|
|
11
|
+
- three **operator factories** (async, `await` them to get the operator object):
|
|
12
|
+
`mongoFactory`, `mongoLogFactory`, `amqpFactory`
|
|
13
|
+
- **MARC helpers**: `conversions()`, `validations()`, and the `fixes` namespace
|
|
14
|
+
(`fixRecord` + fix settings/constants)
|
|
15
|
+
- **utility functions**: `logError`, `createImportJobState`,
|
|
16
|
+
`createRecordResponseItem`, `addRecordResponseItem`, `addRecordResponseItems`
|
|
17
|
+
- **constants**: `QUEUE_ITEM_STATE`, `IMPORT_JOB_STATE`, `OPERATIONS`,
|
|
18
|
+
`CHUNK_SIZE`, `LOG_ITEM_TYPE`, `CONVERSION_FORMATS` (see
|
|
19
|
+
[Constants](#constants) below)
|
|
20
|
+
- **types** (TypeScript consumers): `MongoOperator`, `QueueItem`,
|
|
21
|
+
`ImportJobStateValue`, `MongoLogOperator`, `LogItem`, `LogItemType`,
|
|
22
|
+
`AmqpOperator`, `QueueItemRecordPayload`, `QueueItemMessagePayload`
|
|
23
|
+
|
|
24
|
+
Invalid input is rejected with an `ApiError` 400; a Mongo failure rejects
|
|
25
|
+
with an `ApiError` 500. Both `ApiError`s carry an HTTP status you can map to
|
|
26
|
+
a response.
|
|
27
|
+
|
|
28
|
+
### `mongoFactory(MONGO_URI, collection, db?)`
|
|
29
|
+
|
|
30
|
+
Creates the queue-item operator on a Mongo collection. `db` defaults to
|
|
31
|
+
`'rest-api'`.
|
|
32
|
+
|
|
33
|
+
| Method | Description | Returns |
|
|
34
|
+
|--------|-------------|---------|
|
|
35
|
+
| `createPrio(params)` | Create a prio (single-record) queue item | `void` |
|
|
36
|
+
| `createBulk(params)` | Create a bulk queue item, with or without an input content stream | `{correlationId, acknowledged, queueItemState}` |
|
|
37
|
+
| `checkAndSetState(params)` | Check the queue item has not timed out, then set its state | `CheckAndSetResult` |
|
|
38
|
+
| `checkAndSetImportJobState(params)` | Same, for one operation's `importJobState` | `CheckAndSetResult` |
|
|
39
|
+
| `query(params, showParams?)` | Query queue items with a trusted internal query object (limit, skip, Mongo operators) | `QueueItem[]` |
|
|
40
|
+
| `createProjection(showParams?)` | Build the Mongo projection used by `query` (internal fields hidden unless shown) | `Record<string, number>` |
|
|
41
|
+
| `queryById({correlationId, checkModTime?})` | Get a queue item by `correlationId` | `QueueItem \| null` |
|
|
42
|
+
| `checkTimeOut(params)` | Abort the queue item when it has not made progress within the [timeout window](#timeout-abort-contract) (sets `ABORT`, `errorStatus` 408) | `boolean` (true when it aborted) |
|
|
43
|
+
| `remove({correlationId})` | Remove a queue item and its GridFS content | `{removed, deletedContent}` |
|
|
44
|
+
| `readContent(correlationId)` | Read the GridFS content of a queue item | `{contentType, readStream}` |
|
|
45
|
+
| `removeContent({correlationId})` | Remove the GridFS content of a queue item | `boolean` |
|
|
46
|
+
| `getOne({queueItemState?, importJobState?})` | Get one queue item filtered by `queueItemState` and/or `importJobState` | `QueueItem \| null` |
|
|
47
|
+
| `getStream(correlationId)` | Download stream of the GridFS content; rejects with 404 when the file is missing | `Readable` (async) |
|
|
48
|
+
| `setState(params)` | Set the queue item state (plus optional `errorMessage` / `errorStatus`) | `QueueItem \| null` |
|
|
49
|
+
| `setImportJobState(params)` | Set the `importJobState` of one operation | `QueueItem \| null` |
|
|
50
|
+
| `pushIds({correlationId, handledIds, rejectedIds})` | Push handled and rejected ids to the queue item | `void` |
|
|
51
|
+
| `pushMessages({correlationId, messages, messageField?})` | Push message records to a queue item field (`messageField` defaults to `messages`) | `void` |
|
|
52
|
+
| `setOperation({correlationId, operation})` | Set the operation of a queue item; rejects with 404 when the item does not exist | `QueueItem` |
|
|
53
|
+
| `setOperations({correlationId, addOperation, removeOperation?})` | Add and/or remove operations on a queue item | `QueueItem` |
|
|
54
|
+
| `addBlobSize({correlationId})` | Increment the `blobSize` of a queue item waiting for records | `QueueItem \| null` |
|
|
55
|
+
| `setBlobSize({correlationId, blobSize})` | Set the `blobSize` of a queue item | `QueueItem \| null` |
|
|
56
|
+
|
|
57
|
+
**`CheckAndSetResult`** — the discriminated result of
|
|
58
|
+
`checkAndSetState` / `checkAndSetImportJobState` (branch on `status`, read
|
|
59
|
+
the item from `queueItem`):
|
|
60
|
+
|
|
61
|
+
| `status` | Meaning |
|
|
62
|
+
|----------|---------|
|
|
63
|
+
| `updated` | the state was set; `queueItem` is the resulting document |
|
|
64
|
+
| `timeout` | the item was aborted by `checkTimeOut`; `queueItem` is the resulting document |
|
|
65
|
+
| `unchanged` | the item was already `ABORT` / `ERROR` / `DONE`, or no longer exists; `queueItem` is `null` when the item is gone |
|
|
66
|
+
|
|
67
|
+
**`createBulk` result** — `{correlationId, acknowledged, queueItemState}` on
|
|
68
|
+
both the stream and no-stream paths (`queueItemState` is `UPLOADING` for the
|
|
69
|
+
stream path, `WAITING_FOR_RECORDS` for the no-stream path).
|
|
70
|
+
|
|
71
|
+
### `mongoLogFactory(MONGO_URI, dbName?)`
|
|
72
|
+
|
|
73
|
+
Creates the log operator on the `logs` collection of a Mongo db. `dbName`
|
|
74
|
+
defaults to `'rest-api'`.
|
|
75
|
+
|
|
76
|
+
| Method | Description | Returns |
|
|
77
|
+
|--------|-------------|---------|
|
|
78
|
+
| `addLogItem(logItem)` | Add a log item | `void` |
|
|
79
|
+
| `query(params)` | Query log items with a trusted internal query object (defaults: limit 5, skip 0) | `LogItem[]` |
|
|
80
|
+
| `queryById(correlationId, logItemType?, skip?, limit?)` | Get log items for one `correlationId` (defaults: `logItemType` `MERGE_LOG`, limit 1) | `LogItem[]` |
|
|
81
|
+
| `getListOfLogs(logItemType?)` | Get the list of `correlationId`s that have logs of a given type (default `MERGE_LOG`) | `string[]` |
|
|
82
|
+
| `getListOfCatalogers()` | Get the list of catalogers over all logs | `string[]` |
|
|
83
|
+
| `getListOfCorrelationIds()` | Get the list of `correlationId`s over all logs | `string[]` |
|
|
84
|
+
| `getExpandedListOfLogs(params)` | Aggregated log list per `correlationId` + `logItemType` with cataloger, creation time and record count; optional filters `logItemTypes`, `catalogers`, `dateBefore`, `dateAfter` (a provided `logItemTypes` is validated element-wise, unknown element rejects with 400) | `ExpandedLogListItem[]` |
|
|
85
|
+
| `protect(correlationId, blobSequence?)` | Mark matching log items as protected (idempotent) | `UpdateResult` |
|
|
86
|
+
| `unprotect(correlationId, blobSequence?)` | Mark matching log items as unprotected, so regular cleaning can remove them | `UpdateResult` |
|
|
87
|
+
| `remove(correlationId, force?)` | Remove log items for a `correlationId`; protected items are kept unless `force` | `DeleteResult` |
|
|
88
|
+
| `removeBySequences(correlationId, sequences?, force?)` | Remove log items for a `correlationId` matching blob sequences; protected items are kept unless `force` | `DeleteResult` |
|
|
89
|
+
|
|
90
|
+
### `amqpFactory(AMQP_URL, runHealthCheck?)`
|
|
91
|
+
|
|
92
|
+
Creates the AMQP operator on a RabbitMQ broker. With `runHealthCheck: true`,
|
|
93
|
+
a background loop asserts a temporary `HEALTHCHECK` queue about every 200 ms.
|
|
94
|
+
This is a deliberate fast-fail: when the broker is unreachable the error
|
|
95
|
+
escapes as an unhandled rejection — the operator does not catch, retry or
|
|
96
|
+
reconnect. Deploy the process with `unhandledRejection` /
|
|
97
|
+
`uncaughtException` handlers that log and exit, so the orchestrator restarts
|
|
98
|
+
it.
|
|
99
|
+
|
|
100
|
+
| Method | Description | Returns |
|
|
101
|
+
|--------|-------------|---------|
|
|
102
|
+
| `checkQueue({queue, style?, toRecord?, purge?})` | Check a queue: consume one message or a chunk, return the message count, or purge (defaults: `style` `basic`, `toRecord` true) | `false` (empty) \| message count (`style` `messages`) \| message / records / chunk object (`style` `one` / `basic`) |
|
|
103
|
+
| `consumeChunk(queue, toRecord)` | Consume a chunk (up to `CHUNK_SIZE` messages) from a queue | `false` \| `{headers, [records,] messages}` |
|
|
104
|
+
| `consumeOne(queue, toRecord)` | Consume one message from a queue | `false` \| message \| `{headers, records, messages}` |
|
|
105
|
+
| `ackMessages(messages)` | Acknowledge consumed messages | `void` |
|
|
106
|
+
| `nackMessages(messages)` | Nack consumed messages (not requeued) | `void` |
|
|
107
|
+
| `sendToQueue({queue, correlationId, headers?, data})` | Send a message to a queue; rejects with 500 when `queue` is missing | `void` |
|
|
108
|
+
| `removeQueue(queue)` | Remove a queue | `void` |
|
|
109
|
+
| `messagesToRecords(messages)` | Parse the message bodies of consumed messages into MARC records | `MarcRecord[]` |
|
|
110
|
+
| `closeChannel()` | Close the channel | `void` |
|
|
111
|
+
| `closeConnection()` | Close the connection | `void` |
|
|
112
|
+
|
|
113
|
+
The consumed-payload shape depends on `toRecord`: `{headers, records,
|
|
114
|
+
messages}` when true, `{headers, messages}` when false (`headers` is the
|
|
115
|
+
AMQP properties headers of the first message of the chunk; `records` is the
|
|
116
|
+
MARC records parsed from the message bodies). `consumeOne` with `toRecord`
|
|
117
|
+
false returns the plain amqplib message instead of a payload object.
|
|
118
|
+
|
|
119
|
+
### MARC helpers
|
|
120
|
+
|
|
121
|
+
| Export | Description |
|
|
122
|
+
|--------|-------------|
|
|
123
|
+
| `conversions()` | Factory returning `{serialize, unserialize}`. `serialize(record, format)` returns the record as a string; `unserialize(data, format, validationOptions?)` returns a `MarcRecord`. `format` is a `CONVERSION_FORMATS` value (`MARCXML`, `ISO2709`, `JSON`, `ALEPHSEQ`); an unknown format rejects with 415, malformed data with 400 |
|
|
124
|
+
| `validations()` | Async factory returning an async validator: `(record) => {record, failed, messages}` — the (fixed) record, whether validation failed, and the validation report |
|
|
125
|
+
| `fixes.fixRecord(record, settings?)` | Apply the configured record fixes (`FixSettings`) to a clone of the record (the input is never mutated); returns a `MarcRecordObject` |
|
|
126
|
+
| `fixes.*` | Fix settings and constants: `BIB_F035_TO_SID`, `REPLACE_PREFIXES`, `BIB_FORMAT_SETTINGS`, `BIB_PREVALIDATION_FIX_SETTINGS`, `BIB_POSTMERGE_FIX_SETTINGS`, `BIB_PREIMPORT_FIX_SETTINGS`, `AUTNAME_PREIMPORT_FIX_SETTINGS` |
|
|
127
|
+
|
|
128
|
+
### Utilities
|
|
129
|
+
|
|
130
|
+
| Export | Description |
|
|
131
|
+
|--------|-------------|
|
|
132
|
+
| `logError(err)` | Log an error (`ApiError`s are stringified, the SIGINT marker as-is, everything else with its stack when it has one) |
|
|
133
|
+
| `createImportJobState(operation, state, queryImportJobState?)` | Build an `importJobState` object; the optional third argument keys it by the dotted query form (`importJobState.<operation>`) |
|
|
134
|
+
| `createRecordResponseItem({responsePayload, responseStatus, recordMetadata?, id?})` | Build one record response item (the payload must be an object or a string) |
|
|
135
|
+
| `addRecordResponseItem({recordResponseItem, correlationId, mongoOperator})` | Add one record response item to the `records` field of a queue item |
|
|
136
|
+
| `addRecordResponseItems({recordResponseItems, correlationId, mongoOperator})` | Add several record response items in one Mongo operation (avoids write conflicts) |
|
|
137
|
+
|
|
138
|
+
### Breaking changes since 6.0
|
|
139
|
+
|
|
140
|
+
If you upgrade from 5.x or 6.0, verify these methods — their return shape or
|
|
141
|
+
calling convention changed:
|
|
142
|
+
|
|
143
|
+
1. `checkAndSetState` / `checkAndSetImportJobState` no longer return a
|
|
144
|
+
boolean; they return the `CheckAndSetResult` object above. Branch on
|
|
145
|
+
`result.status` instead of the truthiness of the return value.
|
|
146
|
+
2. `createBulk` returns the normalized `{correlationId, acknowledged,
|
|
147
|
+
queueItemState}` object on both the stream and the no-stream path; do not
|
|
148
|
+
read driver result fields like `insertedId` anymore.
|
|
149
|
+
3. `getStream` is now async (returns a Promise) and rejects with `ApiError`
|
|
150
|
+
404 when the GridFS file is missing, instead of surfacing the missing
|
|
151
|
+
file as an async `error` event on the stream.
|
|
152
|
+
4. `getOne` is now async (returns a Promise); invalid input is rejected with
|
|
153
|
+
`ApiError` 400 before any query is issued.
|
|
154
|
+
|
|
7
155
|
## Constants
|
|
8
156
|
|
|
9
157
|
### QUEUE_ITEM_STATE
|
|
@@ -39,20 +187,55 @@ Shared modules for microservices of Melinda rest api import system
|
|
|
39
187
|
|
|
40
188
|
A queue item that has not made progress for longer than the timeout window is
|
|
41
189
|
set to `ABORT` (with `errorStatus` 408 and `errorMessage`
|
|
42
|
-
`Timeout in <state>`). The
|
|
43
|
-
|
|
44
|
-
`
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
190
|
+
`Timeout in <state>`). The check is not automatic: it only runs when a
|
|
191
|
+
consumer calls `checkTimeOut` or the `checkAndSetState` /
|
|
192
|
+
`checkAndSetImportJobState` wrappers. The window is the time since the last
|
|
193
|
+
write to the item's `modificationTime` (every Mongo write done through the
|
|
194
|
+
operator - `pushMessages`, `pushIds`, `addBlobSize`, `setBlobSize`,
|
|
195
|
+
`setState`, `setImportJobState` - refreshes it). The window defaults to
|
|
196
|
+
**1 minute** and can be made longer with the `MONGO_TIMEOUT_MINUTES` env var
|
|
197
|
+
when a single state phase may legitimately take longer:
|
|
49
198
|
|
|
50
199
|
```sh
|
|
51
200
|
MONGO_TIMEOUT_MINUTES=10 # abort only after 10 minutes without progress
|
|
52
201
|
```
|
|
53
202
|
|
|
203
|
+
#### Full `QUEUE_ITEM_STATE` reference
|
|
204
|
+
|
|
205
|
+
The flow tables above show the happy-path sequence per service. The full set of `queueItemState` values (see `src/constants.ts`):
|
|
206
|
+
|
|
207
|
+
| State | Group | Meaning |
|
|
208
|
+
|-------|-------|---------|
|
|
209
|
+
| `VALIDATOR.PENDING_QUEUING` | VALIDATOR | Bulk job accepted (http); waiting to be picked up for validation. |
|
|
210
|
+
| `VALIDATOR.PENDING_VALIDATION` | VALIDATOR | Prio job accepted (http); waiting to be picked up for validation. |
|
|
211
|
+
| `VALIDATOR.QUEUING_IN_PROGRESS` | VALIDATOR | Validator is distributing records to the per-operation import queues. |
|
|
212
|
+
| `VALIDATOR.UPLOADING` | VALIDATOR | Stream-bulk upload in progress (records written to the GridFS blob). |
|
|
213
|
+
| `VALIDATOR.VALIDATING` | VALIDATOR | Validator is validating records. |
|
|
214
|
+
| `VALIDATOR.WAITING_FOR_RECORDS` | VALIDATOR | Stream-bulk job: partial records received, waiting for the remainder. |
|
|
215
|
+
| `IMPORTER.IN_QUEUE` | IMPORTER | Records are queued for import, awaiting the importer. |
|
|
216
|
+
| `IMPORTER.IMPORTING` | IMPORTER | Importer has the job and is importing it. |
|
|
217
|
+
| `IMPORTER.IN_PROCESS` | IMPORTER | Importer is processing the job. |
|
|
218
|
+
| `DONE` | terminal | Job completed successfully. |
|
|
219
|
+
| `ERROR` | terminal | Job failed; see `errorStatus` / `errorMessage`. |
|
|
220
|
+
| `ABORT` | terminal | Job aborted (e.g. timeout on a stalled job; `errorStatus` 408). |
|
|
221
|
+
|
|
222
|
+
#### Full `IMPORT_JOB_STATE` reference
|
|
223
|
+
|
|
224
|
+
Each queue item also carries a per-operation `importJobState` subdocument (`CREATE` / `UPDATE` / `FIX`). The full set of values:
|
|
225
|
+
|
|
226
|
+
| State | Meaning |
|
|
227
|
+
|-------|---------|
|
|
228
|
+
| `EMPTY` | No such operation in the job (coalesced default when the subdocument is absent). |
|
|
229
|
+
| `QUEUING` | Validator is queuing this operation's records. |
|
|
230
|
+
| `IN_QUEUE` | This operation's records are queued for import. |
|
|
231
|
+
| `IMPORTING` | Importer is importing this operation. |
|
|
232
|
+
| `PROCESSING` | Importer is processing this operation (record-load). |
|
|
233
|
+
| `DONE` | This operation completed. |
|
|
234
|
+
| `ERROR` | This operation failed. |
|
|
235
|
+
| `ABORT` | This operation aborted (e.g. timeout). |
|
|
236
|
+
|
|
54
237
|
### CHUNK_SIZE
|
|
55
|
-
|
|
238
|
+
Chunk size to execution time ratio (From file to queue: ~450 records in 15sec)
|
|
56
239
|
|
|
57
240
|
| Total records | Chunk size | Chunks | Execution time | Time / chunk | Operation |
|
|
58
241
|
|---------------|------------|--------|----------------|--------------|-----------|
|
|
@@ -79,18 +262,32 @@ Executed on test server using bridge to docker server and back to server
|
|
|
79
262
|
| p_z07_priority_year | pZ07PriorityYear | 1998 | 1990 | 2099 | 2099 | Override indexing priority |
|
|
80
263
|
| p_redirection_field | pRedirectionField | | | | | |
|
|
81
264
|
|
|
265
|
+
## TypeScript toolchain
|
|
266
|
+
|
|
267
|
+
The source is TypeScript (Node >= 24, native type stripping for tests).
|
|
268
|
+
|
|
269
|
+
- `npm run build` runs `tsc` (`tsconfig.json`): `src/` -> `dist/`, emitting
|
|
270
|
+
`.js` + `.js.map` + `.d.ts` + `.d.ts.map` for every module. The published
|
|
271
|
+
entry point stays `./dist/index.js`; the `.d.ts` files are additive for
|
|
272
|
+
TypeScript consumers.
|
|
273
|
+
- `npm run typecheck` runs `tsc -p tsconfig.test.json` (typechecks `src/`
|
|
274
|
+
and `test/`, no emit).
|
|
275
|
+
- `npm test` runs `npm run lint` then `npm run typecheck` then
|
|
276
|
+
`npm run test:base` (what CI runs).
|
|
277
|
+
|
|
82
278
|
## Running the tests
|
|
83
279
|
|
|
84
280
|
There are three test suites. The base suite runs anywhere; the two AMQP
|
|
85
281
|
suites need a running RabbitMQ broker on `127.0.0.1:5672` (guest/guest) and
|
|
86
|
-
are therefore excluded from the base glob and from CI.
|
|
282
|
+
are therefore excluded from the base glob and from CI. Tests are `.ts`
|
|
283
|
+
files run directly by `node --test` (no loader).
|
|
87
284
|
|
|
88
285
|
| Command | What it runs | Requirements |
|
|
89
286
|
|-------------------------|---------------------------------------------------------------------|-----------------------|
|
|
90
|
-
| `npm run test:base` | all `
|
|
91
|
-
| `npm test` | `npm run lint` then `test:base` (what CI runs)
|
|
92
|
-
| `npm run test:local` | `
|
|
93
|
-
| `npm run test:healthcheck` | `
|
|
287
|
+
| `npm run test:base` | all `test/**/*.test.ts` with coverage + thresholds | none |
|
|
288
|
+
| `npm test` | `npm run lint`, `npm run typecheck`, then `test:base` (what CI runs) | none |
|
|
289
|
+
| `npm run test:local` | `test/amqp.*.localtest.ts`, AMQP operator against a live broker | RabbitMQ on localhost |
|
|
290
|
+
| `npm run test:healthcheck` | `test/amqp.healthtest.ts`, healthCheck loop vs live broker | RabbitMQ on localhost |
|
|
94
291
|
|
|
95
292
|
`test:base` enforces minimum coverage thresholds via
|
|
96
293
|
`--test-coverage-lines/--test-coverage-branches/--test-coverage-functions`
|
|
@@ -99,6 +296,6 @@ as coverage grows.
|
|
|
99
296
|
|
|
100
297
|
## License and copyright
|
|
101
298
|
|
|
102
|
-
Copyright (c)
|
|
299
|
+
Copyright (c) 2022-current **University of Helsinki (National Library of Finland)**
|
|
103
300
|
|
|
104
301
|
This project's source code is licensed under the terms of **MIT** or any later version.
|
package/dist/amqp.d.ts
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { MarcRecord } from '@natlibfi/marc-record';
|
|
2
|
+
import type { GetMessage, Message } from 'amqplib';
|
|
3
|
+
export interface QueueItemRecordPayload {
|
|
4
|
+
headers: Record<string, unknown> | undefined;
|
|
5
|
+
records: MarcRecord[];
|
|
6
|
+
messages: Message[];
|
|
7
|
+
}
|
|
8
|
+
export interface QueueItemMessagePayload {
|
|
9
|
+
headers: Record<string, unknown> | undefined;
|
|
10
|
+
messages: Message[];
|
|
11
|
+
}
|
|
12
|
+
export interface CheckQueueParams {
|
|
13
|
+
queue: string;
|
|
14
|
+
style?: string;
|
|
15
|
+
toRecord?: boolean;
|
|
16
|
+
purge?: boolean;
|
|
17
|
+
}
|
|
18
|
+
export type CheckQueueResult = number | boolean | QueueItemMessagePayload | QueueItemRecordPayload | GetMessage;
|
|
19
|
+
export interface SendToQueueParams {
|
|
20
|
+
queue?: string;
|
|
21
|
+
correlationId: string;
|
|
22
|
+
headers?: Record<string, unknown>;
|
|
23
|
+
data: unknown;
|
|
24
|
+
}
|
|
25
|
+
export interface AmqpOperator {
|
|
26
|
+
checkQueue: (params: CheckQueueParams) => Promise<CheckQueueResult>;
|
|
27
|
+
consumeChunk: (queue: string, toRecord: boolean) => Promise<false | QueueItemMessagePayload | QueueItemRecordPayload>;
|
|
28
|
+
consumeOne: (queue: string, toRecord: boolean) => Promise<false | GetMessage | QueueItemRecordPayload>;
|
|
29
|
+
ackMessages: (messages: Message[]) => void;
|
|
30
|
+
nackMessages: (messages: Message[]) => void;
|
|
31
|
+
sendToQueue: (params: SendToQueueParams) => Promise<void>;
|
|
32
|
+
removeQueue: (queue: string) => Promise<void>;
|
|
33
|
+
messagesToRecords: (messages: Message[]) => MarcRecord[];
|
|
34
|
+
closeChannel: () => Promise<void>;
|
|
35
|
+
closeConnection: () => Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
export default function (AMQP_URL: string, runHealthCheck?: boolean, deps?: {
|
|
38
|
+
connection?: unknown;
|
|
39
|
+
channel?: unknown;
|
|
40
|
+
}): Promise<AmqpOperator>;
|
|
41
|
+
//# sourceMappingURL=amqp.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"amqp.d.ts","sourceRoot":"","sources":["../src/amqp.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAC;AAGjD,OAAO,KAAK,EAAwB,UAAU,EAAE,OAAO,EAAU,MAAM,SAAS,CAAC;AAcjF,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC7C,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAMD,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC7C,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAMD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAOD,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,OAAO,GAAG,uBAAuB,GAAG,sBAAsB,GAAG,UAAU,CAAC;AAMhH,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,IAAI,EAAE,OAAO,CAAC;CACf;AAKD,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACpE,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,KAAK,GAAG,uBAAuB,GAAG,sBAAsB,CAAC,CAAC;IACtH,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,KAAK,GAAG,UAAU,GAAG,sBAAsB,CAAC,CAAC;IACvG,WAAW,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC3C,YAAY,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC5C,WAAW,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1D,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,iBAAiB,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;IACzD,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,eAAe,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAwBD,yBAA+B,QAAQ,EAAE,MAAM,EAAE,cAAc,UAAQ,EAAE,IAAI,GAAE;IAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CA+apJ"}
|