@natlibfi/melinda-rest-api-commons 6.1.0-alpha.0 → 6.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/melinda-node-tests-and-publish.yml +4 -2
- package/README.md +53 -5
- package/dist/amqp.d.ts +41 -0
- package/dist/amqp.d.ts.map +1 -0
- package/dist/amqp.js +237 -235
- package/dist/amqp.js.map +1 -7
- package/dist/constants.d.ts +49 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +38 -38
- package/dist/constants.js.map +1 -7
- package/dist/helpers/conversion.d.ts +8 -0
- package/dist/helpers/conversion.d.ts.map +1 -0
- package/dist/helpers/conversion.js +70 -64
- package/dist/helpers/conversion.js.map +1 -7
- package/dist/helpers/fix-constants.d.ts +76 -0
- package/dist/helpers/fix-constants.d.ts.map +1 -0
- package/dist/helpers/fix-constants.js +41 -41
- package/dist/helpers/fix-constants.js.map +1 -7
- package/dist/helpers/fix-handle-tempurns.d.ts +3 -0
- package/dist/helpers/fix-handle-tempurns.d.ts.map +1 -0
- package/dist/helpers/fix-handle-tempurns.js +73 -71
- package/dist/helpers/fix-handle-tempurns.js.map +1 -7
- package/dist/helpers/fix-strip-f884s.d.ts +10 -0
- package/dist/helpers/fix-strip-f884s.d.ts.map +1 -0
- package/dist/helpers/fix-strip-f884s.js +161 -163
- package/dist/helpers/fix-strip-f884s.js.map +1 -7
- package/dist/helpers/fix.d.ts +6 -0
- package/dist/helpers/fix.d.ts.map +1 -0
- package/dist/helpers/fix.js +133 -121
- package/dist/helpers/fix.js.map +1 -7
- package/dist/helpers/validation.d.ts +9 -0
- package/dist/helpers/validation.d.ts.map +1 -0
- package/dist/helpers/validation.js +14 -16
- package/dist/helpers/validation.js.map +1 -7
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -7
- package/dist/mongo.d.ts +165 -0
- package/dist/mongo.d.ts.map +1 -0
- package/dist/mongo.js +595 -547
- package/dist/mongo.js.map +1 -7
- package/dist/mongoError.d.ts +2 -0
- package/dist/mongoError.d.ts.map +1 -0
- package/dist/mongoError.js +19 -0
- package/dist/mongoError.js.map +1 -0
- package/dist/mongoLog.d.ts +50 -0
- package/dist/mongoLog.d.ts.map +1 -0
- package/dist/mongoLog.js +274 -242
- package/dist/mongoLog.js.map +1 -7
- package/dist/utils.d.ts +36 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +112 -110
- package/dist/utils.js.map +1 -7
- package/eslint.config.ts +66 -0
- package/package.json +15 -11
- package/{tests/amqp-error-branches.test.js → test/amqp-error-branches.test.ts} +68 -52
- package/{tests/amqp.healthtest.js → test/amqp.healthtest.ts} +7 -3
- package/{tests/amqp.tests.localtest.js → test/amqp.tests.localtest.ts} +54 -35
- package/{tests/helpers/conversion.test.js → test/helpers/conversion.test.ts} +26 -14
- package/{tests/helpers/fix-edge.test.js → test/helpers/fix-edge.test.ts} +16 -12
- package/{tests/helpers/fix.test.js → test/helpers/fix.test.ts} +14 -8
- package/{tests/helpers/validation.test.js → test/helpers/validation.test.ts} +7 -6
- package/{tests/mongo-add-blob-size.test.js → test/mongo-add-blob-size.test.ts} +14 -15
- package/{tests/mongo-check-and-check-state.test.js → test/mongo-check-and-check-state.test.ts} +6 -8
- package/{tests/mongo-check-and-set-import-job-state.test.js → test/mongo-check-and-set-import-job-state.test.ts} +6 -9
- package/{tests/mongo-check-timeout.test.js → test/mongo-check-timeout.test.ts} +6 -9
- package/{tests/mongo-create-bulk.test.js → test/mongo-create-bulk.test.ts} +6 -7
- package/{tests/mongo-create-prio.test.js → test/mongo-create-prio.test.ts} +6 -6
- package/{tests/mongo-error-branches.test.js → test/mongo-error-branches.test.ts} +147 -51
- package/{tests/mongo-get-one.test.js → test/mongo-get-one.test.ts} +6 -6
- package/{tests/mongo-get-stream.test.js → test/mongo-get-stream.test.ts} +6 -8
- package/{tests/mongo-push-ids.test.js → test/mongo-push-ids.test.ts} +6 -6
- package/{tests/mongo-push-messages.test.js → test/mongo-push-messages.test.ts} +6 -6
- package/{tests/mongo-query-by-id.test.js → test/mongo-query-by-id.test.ts} +6 -9
- package/{tests/mongo-query.test.js → test/mongo-query.test.ts} +6 -7
- package/{tests/mongo-read-content.test.js → test/mongo-read-content.test.ts} +6 -9
- package/{tests/mongo-remove.test.js → test/mongo-remove.test.ts} +6 -8
- package/{tests/mongo-set-blob-size.test.js → test/mongo-set-blob-size.test.ts} +6 -6
- package/{tests/mongo-set-import-job-state.test.js → test/mongo-set-import-job-state.test.ts} +6 -6
- package/{tests/mongo-set-operation.test.js → test/mongo-set-operation.test.ts} +6 -6
- package/{tests/mongo-set-operations.test.js → test/mongo-set-operations.test.ts} +6 -6
- package/{tests/mongo-set-state.test.js → test/mongo-set-state.test.ts} +6 -6
- package/{tests/mongoLog-protect.test.js → test/mongoLog-protect.test.ts} +18 -17
- package/{tests/mongoLog.test.js → test/mongoLog.test.ts} +8 -8
- package/{tests/testUtils.js → test/testUtils.ts} +49 -15
- package/{tests/utils.test.js → test/utils.test.ts} +5 -5
- package/tsconfig.json +62 -0
- package/tsconfig.test.json +17 -0
- package/dist/helpers/fix-utils.js +0 -39
- package/dist/helpers/fix-utils.js.map +0 -7
- package/eslint.config.js +0 -53
|
@@ -28,7 +28,6 @@ jobs:
|
|
|
28
28
|
cache: 'npm'
|
|
29
29
|
env:
|
|
30
30
|
NPM_CONFIG_IGNORE_SCRIPTS: true
|
|
31
|
-
- run: npm update
|
|
32
31
|
- run: npm audit --package-lock-only --production --audit-level=moderate
|
|
33
32
|
- run: npm i
|
|
34
33
|
- run: npm test
|
|
@@ -41,8 +40,11 @@ jobs:
|
|
|
41
40
|
|
|
42
41
|
steps:
|
|
43
42
|
- uses: actions/checkout@v7
|
|
44
|
-
- uses: mikaelvesavuori/license-compliance-action@v1
|
|
43
|
+
- uses: mikaelvesavuori/license-compliance-action@v1.0.3
|
|
45
44
|
with:
|
|
45
|
+
# default action list + BlueOak-1.0.0 (sax, via xml2js in
|
|
46
|
+
# marc-record-serializers) + MIT-0 — OSI-approved permissive
|
|
47
|
+
allow_licenses: 'MIT;ISC;0BSD;BSD-2-Clause;BSD-3-Clause;Apache-2.0;BlueOak-1.0.0;MIT-0'
|
|
46
48
|
exclude_pattern: /^@natlibfi/
|
|
47
49
|
|
|
48
50
|
njsscan:
|
package/README.md
CHANGED
|
@@ -51,6 +51,40 @@ need:
|
|
|
51
51
|
MONGO_TIMEOUT_MINUTES=10 # abort only after 10 minutes without progress
|
|
52
52
|
```
|
|
53
53
|
|
|
54
|
+
#### Full `QUEUE_ITEM_STATE` reference
|
|
55
|
+
|
|
56
|
+
The flow tables above show the happy-path sequence per service. The full set of `queueItemState` values (see `src/constants.ts`):
|
|
57
|
+
|
|
58
|
+
| State | Group | Meaning |
|
|
59
|
+
|-------|-------|---------|
|
|
60
|
+
| `VALIDATOR.PENDING_QUEUING` | VALIDATOR | Bulk job accepted (http); waiting to be picked up for validation. |
|
|
61
|
+
| `VALIDATOR.PENDING_VALIDATION` | VALIDATOR | Prio job accepted (http); waiting to be picked up for validation. |
|
|
62
|
+
| `VALIDATOR.QUEUING_IN_PROGRESS` | VALIDATOR | Validator is distributing records to the per-operation import queues. |
|
|
63
|
+
| `VALIDATOR.UPLOADING` | VALIDATOR | Stream-bulk upload in progress (records written to the GridFS blob). |
|
|
64
|
+
| `VALIDATOR.VALIDATING` | VALIDATOR | Validator is validating records. |
|
|
65
|
+
| `VALIDATOR.WAITING_FOR_RECORDS` | VALIDATOR | Stream-bulk job: partial records received, waiting for the remainder. |
|
|
66
|
+
| `IMPORTER.IN_QUEUE` | IMPORTER | Records are queued for import, awaiting the importer. |
|
|
67
|
+
| `IMPORTER.IMPORTING` | IMPORTER | Importer has the job and is importing it. |
|
|
68
|
+
| `IMPORTER.IN_PROCESS` | IMPORTER | Importer is processing the job. |
|
|
69
|
+
| `DONE` | terminal | Job completed successfully. |
|
|
70
|
+
| `ERROR` | terminal | Job failed; see `errorStatus` / `errorMessage`. |
|
|
71
|
+
| `ABORT` | terminal | Job aborted (e.g. timeout on a stalled job; `errorStatus` 408). |
|
|
72
|
+
|
|
73
|
+
#### Full `IMPORT_JOB_STATE` reference
|
|
74
|
+
|
|
75
|
+
Each queue item also carries a per-operation `importJobState` subdocument (`CREATE` / `UPDATE` / `FIX`). The full set of values:
|
|
76
|
+
|
|
77
|
+
| State | Meaning |
|
|
78
|
+
|-------|---------|
|
|
79
|
+
| `EMPTY` | No such operation in the job (coalesced default when the subdocument is absent). |
|
|
80
|
+
| `QUEUING` | Validator is queuing this operation's records. |
|
|
81
|
+
| `IN_QUEUE` | This operation's records are queued for import. |
|
|
82
|
+
| `IMPORTING` | Importer is importing this operation. |
|
|
83
|
+
| `PROCESSING` | Importer is processing this operation (record-load). |
|
|
84
|
+
| `DONE` | This operation completed. |
|
|
85
|
+
| `ERROR` | This operation failed. |
|
|
86
|
+
| `ABORT` | This operation aborted (e.g. timeout). |
|
|
87
|
+
|
|
54
88
|
### CHUNK_SIZE
|
|
55
89
|
Chunck size to execution time ratio (From file to queue: ~450 records in 15sec)
|
|
56
90
|
|
|
@@ -79,18 +113,32 @@ Executed on test server using bridge to docker server and back to server
|
|
|
79
113
|
| p_z07_priority_year | pZ07PriorityYear | 1998 | 1990 | 2099 | 2099 | Override indexing priority |
|
|
80
114
|
| p_redirection_field | pRedirectionField | | | | | |
|
|
81
115
|
|
|
116
|
+
## TypeScript toolchain
|
|
117
|
+
|
|
118
|
+
The source is TypeScript (Node >= 24, native type stripping for tests).
|
|
119
|
+
|
|
120
|
+
- `npm run build` runs `tsc` (`tsconfig.json`): `src/` -> `dist/`, emitting
|
|
121
|
+
`.js` + `.js.map` + `.d.ts` + `.d.ts.map` for every module. The published
|
|
122
|
+
entry point stays `./dist/index.js`; the `.d.ts` files are additive for
|
|
123
|
+
TypeScript consumers.
|
|
124
|
+
- `npm run typecheck` runs `tsc -p tsconfig.test.json` (typechecks `src/`
|
|
125
|
+
and `test/`, no emit).
|
|
126
|
+
- `npm test` runs `npm run lint` then `npm run typecheck` then
|
|
127
|
+
`npm run test:base` (what CI runs).
|
|
128
|
+
|
|
82
129
|
## Running the tests
|
|
83
130
|
|
|
84
131
|
There are three test suites. The base suite runs anywhere; the two AMQP
|
|
85
132
|
suites need a running RabbitMQ broker on `127.0.0.1:5672` (guest/guest) and
|
|
86
|
-
are therefore excluded from the base glob and from CI.
|
|
133
|
+
are therefore excluded from the base glob and from CI. Tests are `.ts`
|
|
134
|
+
files run directly by `node --test` (no loader).
|
|
87
135
|
|
|
88
136
|
| Command | What it runs | Requirements |
|
|
89
137
|
|-------------------------|---------------------------------------------------------------------|-----------------------|
|
|
90
|
-
| `npm run test:base` | all `tests/**/*.test.
|
|
91
|
-
| `npm test` | `npm run lint` then `test:base` (what CI runs)
|
|
92
|
-
| `npm run test:local` | `tests/amqp.*.localtest.
|
|
93
|
-
| `npm run test:healthcheck` | `tests/amqp.healthtest.
|
|
138
|
+
| `npm run test:base` | all `tests/**/*.test.ts` with coverage + thresholds | none |
|
|
139
|
+
| `npm test` | `npm run lint`, `npm run typecheck`, then `test:base` (what CI runs) | none |
|
|
140
|
+
| `npm run test:local` | `tests/amqp.*.localtest.ts`, AMQP operator against a live broker | RabbitMQ on localhost |
|
|
141
|
+
| `npm run test:healthcheck` | `tests/amqp.healthtest.ts`, healthCheck loop vs live broker | RabbitMQ on localhost |
|
|
94
142
|
|
|
95
143
|
`test:base` enforces minimum coverage thresholds via
|
|
96
144
|
`--test-coverage-lines/--test-coverage-branches/--test-coverage-functions`
|
package/dist/amqp.d.ts
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { MarcRecord } from '@natlibfi/marc-record';
|
|
2
|
+
import type { GetMessage, Message } from 'amqplib';
|
|
3
|
+
export interface QueueItemRecordPayload {
|
|
4
|
+
headers: Record<string, unknown> | undefined;
|
|
5
|
+
records: MarcRecord[];
|
|
6
|
+
messages: Message[];
|
|
7
|
+
}
|
|
8
|
+
export interface QueueItemMessagePayload {
|
|
9
|
+
headers: Record<string, unknown> | undefined;
|
|
10
|
+
messages: Message[];
|
|
11
|
+
}
|
|
12
|
+
export interface CheckQueueParams {
|
|
13
|
+
queue: string;
|
|
14
|
+
style?: string;
|
|
15
|
+
toRecord?: boolean;
|
|
16
|
+
purge?: boolean;
|
|
17
|
+
}
|
|
18
|
+
export type CheckQueueResult = number | boolean | QueueItemMessagePayload | QueueItemRecordPayload | GetMessage;
|
|
19
|
+
export interface SendToQueueParams {
|
|
20
|
+
queue?: string;
|
|
21
|
+
correlationId: string;
|
|
22
|
+
headers?: Record<string, unknown>;
|
|
23
|
+
data: unknown;
|
|
24
|
+
}
|
|
25
|
+
export interface AmqpOperator {
|
|
26
|
+
checkQueue: (params: CheckQueueParams) => Promise<CheckQueueResult>;
|
|
27
|
+
consumeChunk: (queue: string, toRecord: boolean) => Promise<false | QueueItemMessagePayload | QueueItemRecordPayload>;
|
|
28
|
+
consumeOne: (queue: string, toRecord: boolean) => Promise<false | GetMessage | QueueItemRecordPayload>;
|
|
29
|
+
ackMessages: (messages: Message[]) => void;
|
|
30
|
+
nackMessages: (messages: Message[]) => void;
|
|
31
|
+
sendToQueue: (params: SendToQueueParams) => Promise<void>;
|
|
32
|
+
removeQueue: (queue: string) => Promise<void>;
|
|
33
|
+
messagesToRecords: (messages: Message[]) => MarcRecord[];
|
|
34
|
+
closeChannel: () => Promise<void>;
|
|
35
|
+
closeConnection: () => Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
export default function (AMQP_URL: string, runHealthCheck?: boolean, deps?: {
|
|
38
|
+
connection?: unknown;
|
|
39
|
+
channel?: unknown;
|
|
40
|
+
}): Promise<AmqpOperator>;
|
|
41
|
+
//# sourceMappingURL=amqp.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"amqp.d.ts","sourceRoot":"","sources":["../src/amqp.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAC;AAGjD,OAAO,KAAK,EAAwB,UAAU,EAAE,OAAO,EAAU,MAAM,SAAS,CAAC;AAcjF,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC7C,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAMD,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC7C,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAMD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAOD,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,OAAO,GAAG,uBAAuB,GAAG,sBAAsB,GAAG,UAAU,CAAC;AAMhH,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,IAAI,EAAE,OAAO,CAAC;CACf;AAKD,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACpE,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,KAAK,GAAG,uBAAuB,GAAG,sBAAsB,CAAC,CAAC;IACtH,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,KAAK,GAAG,UAAU,GAAG,sBAAsB,CAAC,CAAC;IACvG,WAAW,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC3C,YAAY,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC5C,WAAW,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1D,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,iBAAiB,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;IACzD,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,eAAe,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAiBD,yBAA+B,QAAQ,EAAE,MAAM,EAAE,cAAc,UAAQ,EAAE,IAAI,GAAE;IAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CA2ZpJ"}
|
package/dist/amqp.js
CHANGED
|
@@ -1,257 +1,259 @@
|
|
|
1
|
-
import amqplib from
|
|
2
|
-
import createDebugLogger from
|
|
3
|
-
import { promisify, inspect } from
|
|
4
|
-
import httpStatus from
|
|
5
|
-
import { MarcRecord } from
|
|
6
|
-
import { createLogger, Error as ApiError } from
|
|
1
|
+
import amqplib from 'amqplib';
|
|
2
|
+
import createDebugLogger from 'debug';
|
|
3
|
+
import { promisify, inspect } from 'util';
|
|
4
|
+
import httpStatus from 'http-status';
|
|
5
|
+
import { MarcRecord } from '@natlibfi/marc-record';
|
|
6
|
+
import { createLogger, Error as ApiError } from '@natlibfi/melinda-commons';
|
|
7
7
|
import { CHUNK_SIZE } from "./constants.js";
|
|
8
8
|
import { logError } from "./utils.js";
|
|
9
|
-
export default async function(AMQP_URL, runHealthCheck = false, deps = {}) {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
debug(`Connection: ${connection}`);
|
|
21
|
-
debug(`Channel: ${channel}`);
|
|
22
|
-
debug(`HealthCheckLoop: ${healthCheckLoop ? "Running health check" : "Not running health check"}`);
|
|
23
|
-
return { checkQueue, consumeChunk, consumeOne, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords, closeChannel, closeConnection };
|
|
24
|
-
async function closeChannel() {
|
|
25
|
-
healthCheckRunning = false;
|
|
26
|
-
debug(`Closing channel`);
|
|
27
|
-
await channel.close();
|
|
28
|
-
debug(`Channel: ${channel}`);
|
|
29
|
-
}
|
|
30
|
-
async function closeConnection() {
|
|
31
|
-
healthCheckRunning = false;
|
|
32
|
-
debug(`Closing connection`);
|
|
33
|
-
await connection.close();
|
|
9
|
+
export default async function (AMQP_URL, runHealthCheck = false, deps = {}) {
|
|
10
|
+
const logger = createLogger();
|
|
11
|
+
const debug = createDebugLogger('@natlibfi/melinda-rest-api-commons:amqp');
|
|
12
|
+
const debugHC = debug.extend('HC');
|
|
13
|
+
const debugData = debug.extend('data');
|
|
14
|
+
const setTimeoutPromise = promisify(setTimeout);
|
|
15
|
+
debug(`Creating an AMQP operator to ${AMQP_URL}`);
|
|
16
|
+
const connection = deps.connection ?? await amqplib.connect(AMQP_URL);
|
|
17
|
+
const channel = deps.channel ?? await connection.createChannel();
|
|
18
|
+
let healthCheckRunning = runHealthCheck;
|
|
19
|
+
const healthCheckLoop = runHealthCheck ? healthCheck() : false;
|
|
34
20
|
debug(`Connection: ${connection}`);
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
21
|
+
debug(`Channel: ${channel}`);
|
|
22
|
+
debug(`HealthCheckLoop: ${healthCheckLoop ? 'Running health check' : 'Not running health check'}`);
|
|
23
|
+
return { checkQueue, consumeChunk, consumeOne, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords, closeChannel, closeConnection };
|
|
24
|
+
async function closeChannel() {
|
|
25
|
+
healthCheckRunning = false;
|
|
26
|
+
debug(`Closing channel`);
|
|
27
|
+
await channel.close();
|
|
28
|
+
debug(`Channel: ${channel}`);
|
|
29
|
+
}
|
|
30
|
+
async function closeConnection() {
|
|
31
|
+
healthCheckRunning = false;
|
|
32
|
+
debug(`Closing connection`);
|
|
33
|
+
await connection.close();
|
|
34
|
+
debug(`Connection: ${connection}`);
|
|
40
35
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
36
|
+
async function healthCheck(wait = false) {
|
|
37
|
+
if (!healthCheckRunning) {
|
|
38
|
+
debugHC(`HealthCheck cancelled, not running`);
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
if (wait) {
|
|
42
|
+
await setTimeoutPromise(wait);
|
|
43
|
+
return healthCheck(false);
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
debugHC(`Health checking amqp by asserting a queue`);
|
|
47
|
+
await channel.assertQueue('HEALTHCHECK', { durable: false });
|
|
48
|
+
debugHC(`Waiting 200ms before running healthCheck next`);
|
|
49
|
+
return healthCheck(200);
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
debugHC(`HealthCheck error ${JSON.stringify(error)}`);
|
|
53
|
+
if (!healthCheckRunning) {
|
|
54
|
+
debugHC(`HealthCheck cancelled while erroring, not throwing`);
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
handleAmqpErrors(error);
|
|
58
|
+
}
|
|
44
59
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
60
|
+
async function checkQueue({ queue, style = 'basic', toRecord = true, purge = false }) {
|
|
61
|
+
debug(`checkQueue: ${queue}, Style: ${style}, toRecord: ${toRecord}, Purge: ${purge}`);
|
|
62
|
+
try {
|
|
63
|
+
errorUndefinedQueue(queue);
|
|
64
|
+
const channelInfo = await channel.assertQueue(queue, { durable: true });
|
|
65
|
+
if (purge) {
|
|
66
|
+
await purgeQueue(purge);
|
|
67
|
+
logger.verbose(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
|
|
68
|
+
debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
|
|
69
|
+
return checkQueue({ queue, style });
|
|
70
|
+
}
|
|
71
|
+
if (style === 'messages') {
|
|
72
|
+
return channelInfo.messageCount;
|
|
73
|
+
}
|
|
74
|
+
if (style === 'one') {
|
|
75
|
+
if (checkMessageCount(channelInfo)) {
|
|
76
|
+
return consumeOne(queue, toRecord);
|
|
77
|
+
}
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
if (style === 'basic') {
|
|
81
|
+
if (checkMessageCount(channelInfo)) {
|
|
82
|
+
return consumeChunk(queue, toRecord);
|
|
83
|
+
}
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `CheckQueue does not recognize style ${style}`);
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
handleAmqpErrors(error);
|
|
90
|
+
}
|
|
91
|
+
function checkMessageCount(channelInfo) {
|
|
92
|
+
if (channelInfo.messageCount < 1) {
|
|
93
|
+
debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
debug(`Queue ${queue} has ${channelInfo.messageCount} messages`);
|
|
97
|
+
return true;
|
|
98
|
+
}
|
|
99
|
+
function purgeQueue(purge) {
|
|
100
|
+
debug(`Purging queue: ${queue}, purge: ${purge}`);
|
|
101
|
+
if (purge) {
|
|
102
|
+
return channel.purgeQueue(queue);
|
|
103
|
+
}
|
|
104
|
+
return undefined;
|
|
105
|
+
}
|
|
57
106
|
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
if (checkMessageCount(channelInfo)) {
|
|
75
|
-
return consumeOne(queue, toRecord);
|
|
107
|
+
async function consumeChunk(queue, toRecord) {
|
|
108
|
+
debug(`Prepared to consumeChunk from queue: ${queue}`);
|
|
109
|
+
try {
|
|
110
|
+
await channel.assertQueue(queue, { durable: true });
|
|
111
|
+
const messages = await getData(queue);
|
|
112
|
+
if (messages.length === 0) {
|
|
113
|
+
debug(`consumeChunk: queue ${queue} drained, returning false`);
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
const headers = getHeaderInfo(messages[0]);
|
|
117
|
+
debug(`consumeChunk (${messages ? messages.length : '0'} from queue ${queue}) ${toRecord ? 'to records' : 'just messages'}`);
|
|
118
|
+
if (toRecord) {
|
|
119
|
+
const records = messagesToRecords(messages);
|
|
120
|
+
return { headers, records, messages };
|
|
121
|
+
}
|
|
122
|
+
return { headers, messages };
|
|
76
123
|
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
if (style === "basic") {
|
|
80
|
-
if (checkMessageCount(channelInfo)) {
|
|
81
|
-
return consumeChunk(queue, toRecord);
|
|
124
|
+
catch (error) {
|
|
125
|
+
handleAmqpErrors(error);
|
|
82
126
|
}
|
|
83
|
-
return false;
|
|
84
|
-
}
|
|
85
|
-
throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `CheckQueue does not recognize style ${style}`);
|
|
86
|
-
} catch (error) {
|
|
87
|
-
handleAmqpErrors(error);
|
|
88
127
|
}
|
|
89
|
-
function
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
128
|
+
async function consumeOne(queue, toRecord) {
|
|
129
|
+
debug(`Prepared to consumeOne from queue: ${queue}`);
|
|
130
|
+
try {
|
|
131
|
+
await channel.assertQueue(queue, { durable: true });
|
|
132
|
+
const message = await channel.get(queue);
|
|
133
|
+
debugData(`Message: ${inspect(message, { colors: true, maxArrayLength: 3, depth: 3 })}`);
|
|
134
|
+
debug(`consumeOne from queue: ${queue} ${toRecord ? 'to records' : 'just the message'}`);
|
|
135
|
+
if (message) {
|
|
136
|
+
if (toRecord) {
|
|
137
|
+
const headers = getHeaderInfo(message);
|
|
138
|
+
const records = messagesToRecords([message]);
|
|
139
|
+
return { headers, records, messages: [message] };
|
|
140
|
+
}
|
|
141
|
+
return message;
|
|
142
|
+
}
|
|
143
|
+
return false;
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
handleAmqpErrors(error);
|
|
147
|
+
}
|
|
96
148
|
}
|
|
97
|
-
function
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
149
|
+
function ackMessages(messages) {
|
|
150
|
+
messages.forEach(message => {
|
|
151
|
+
debug(`Ack message ${message.properties.correlationId}`);
|
|
152
|
+
channel.ack(message);
|
|
153
|
+
});
|
|
102
154
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
const messages = await getData(queue);
|
|
109
|
-
if (messages.length === 0) {
|
|
110
|
-
debug(`consumeChunk: queue ${queue} drained, returning false`);
|
|
111
|
-
return false;
|
|
112
|
-
}
|
|
113
|
-
const headers = getHeaderInfo(messages[0]);
|
|
114
|
-
debug(`consumeChunk (${messages ? messages.length : "0"} from queue ${queue}) ${toRecord ? "to records" : "just messages"}`);
|
|
115
|
-
if (toRecord) {
|
|
116
|
-
const records = messagesToRecords(messages);
|
|
117
|
-
return { headers, records, messages };
|
|
118
|
-
}
|
|
119
|
-
return { headers, messages };
|
|
120
|
-
} catch (error) {
|
|
121
|
-
handleAmqpErrors(error);
|
|
155
|
+
function nackMessages(messages) {
|
|
156
|
+
messages.forEach(message => {
|
|
157
|
+
debug(`Nack message ${message.properties.correlationId}`);
|
|
158
|
+
channel.nack(message);
|
|
159
|
+
});
|
|
122
160
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
161
|
+
async function sendToQueue({ queue, correlationId, headers, data }) {
|
|
162
|
+
debug(`sendToQueue`);
|
|
163
|
+
try {
|
|
164
|
+
debug(`Queue ${queue}`);
|
|
165
|
+
debug(`CorrelationId ${correlationId}`);
|
|
166
|
+
debugData(`Data ${JSON.stringify(data)}`);
|
|
167
|
+
debug(`Headers ${JSON.stringify(headers)}`);
|
|
168
|
+
errorUndefinedQueue(queue);
|
|
169
|
+
debug(`Asserting queue: ${queue}`);
|
|
170
|
+
await channel.assertQueue(queue, { durable: true });
|
|
171
|
+
debug(`Actually sendToQueue: ${queue}`);
|
|
172
|
+
await channel.sendToQueue(queue, Buffer.from(JSON.stringify({ data })), {
|
|
173
|
+
correlationId,
|
|
174
|
+
persistent: true,
|
|
175
|
+
headers
|
|
176
|
+
});
|
|
177
|
+
debug(`Send message for ${correlationId} to queue: ${queue}`);
|
|
178
|
+
}
|
|
179
|
+
catch (error) {
|
|
180
|
+
const errorToThrow = error;
|
|
181
|
+
debug(`SendToQueue errored: ${JSON.stringify(error)}`);
|
|
182
|
+
handleAmqpErrors(errorToThrow);
|
|
136
183
|
}
|
|
137
|
-
return message;
|
|
138
|
-
}
|
|
139
|
-
return false;
|
|
140
|
-
} catch (error) {
|
|
141
|
-
handleAmqpErrors(error);
|
|
142
184
|
}
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
}
|
|
156
|
-
async function sendToQueue({ queue, correlationId, headers, data }) {
|
|
157
|
-
debug(`sendToQueue`);
|
|
158
|
-
try {
|
|
159
|
-
debug(`Queue ${queue}`);
|
|
160
|
-
debug(`CorrelationId ${correlationId}`);
|
|
161
|
-
debugData(`Data ${JSON.stringify(data)}`);
|
|
162
|
-
debug(`Headers ${JSON.stringify(headers)}`);
|
|
163
|
-
errorUndefinedQueue(queue);
|
|
164
|
-
debug(`Asserting queue: ${queue}`);
|
|
165
|
-
await channel.assertQueue(queue, { durable: true });
|
|
166
|
-
debug(`Actually sendToQueue: ${queue}`);
|
|
167
|
-
await channel.sendToQueue(
|
|
168
|
-
queue,
|
|
169
|
-
Buffer.from(JSON.stringify({ data })),
|
|
170
|
-
{
|
|
171
|
-
correlationId,
|
|
172
|
-
persistent: true,
|
|
173
|
-
headers
|
|
185
|
+
async function removeQueue(queue) {
|
|
186
|
+
let tempChannel;
|
|
187
|
+
try {
|
|
188
|
+
tempChannel = await connection.createChannel();
|
|
189
|
+
logger.verbose(`Removing queue ${queue}.`);
|
|
190
|
+
debug(`Removing queue ${queue}.`);
|
|
191
|
+
await tempChannel.deleteQueue(queue);
|
|
192
|
+
}
|
|
193
|
+
finally {
|
|
194
|
+
if (tempChannel) {
|
|
195
|
+
await tempChannel.close();
|
|
196
|
+
}
|
|
174
197
|
}
|
|
175
|
-
);
|
|
176
|
-
debug(`Send message for ${correlationId} to queue: ${queue}`);
|
|
177
|
-
} catch (error) {
|
|
178
|
-
const errorToThrow = error;
|
|
179
|
-
debug(`SendToQueue errored: ${JSON.stringify(error)}`);
|
|
180
|
-
handleAmqpErrors(errorToThrow);
|
|
181
198
|
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
debug(`Removing queue ${queue}.`);
|
|
189
|
-
await tempChannel.deleteQueue(queue);
|
|
190
|
-
} finally {
|
|
191
|
-
if (tempChannel) {
|
|
192
|
-
await tempChannel.close();
|
|
193
|
-
}
|
|
199
|
+
function messagesToRecords(messages) {
|
|
200
|
+
debug(`Parsing messages (${messages.length}) to records`);
|
|
201
|
+
return messages.map(message => {
|
|
202
|
+
const content = JSON.parse(message.content.toString());
|
|
203
|
+
return new MarcRecord(content.data, { subfieldValues: false });
|
|
204
|
+
});
|
|
194
205
|
}
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
206
|
+
async function getData(queue) {
|
|
207
|
+
debug(`Getting queue data from ${queue}`);
|
|
208
|
+
try {
|
|
209
|
+
const { messageCount } = await channel.checkQueue(queue);
|
|
210
|
+
debug(`There is ${messageCount} messages in queue ${queue}`);
|
|
211
|
+
const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;
|
|
212
|
+
debug(`Getting ${messagesToGet} messages from queue ${queue}`);
|
|
213
|
+
const messages = await pump(messagesToGet);
|
|
214
|
+
debug(`Returning ${messages.length} unique messages`);
|
|
215
|
+
return messages;
|
|
216
|
+
}
|
|
217
|
+
catch (error) {
|
|
218
|
+
handleAmqpErrors(error);
|
|
219
|
+
}
|
|
220
|
+
async function pump(count, results = [], identifiers = []) {
|
|
221
|
+
if (count === 0) {
|
|
222
|
+
return results;
|
|
223
|
+
}
|
|
224
|
+
const message = await channel.get(queue);
|
|
225
|
+
if (!message) {
|
|
226
|
+
return results;
|
|
227
|
+
}
|
|
228
|
+
const identifier = {
|
|
229
|
+
correlationId: message.properties.correlationId,
|
|
230
|
+
deliveryTag: message.fields.deliveryTag
|
|
231
|
+
};
|
|
232
|
+
const isDuplicate = identifiers.some(i => i.correlationId === identifier.correlationId && i.deliveryTag === identifier.deliveryTag);
|
|
233
|
+
if (isDuplicate) {
|
|
234
|
+
return pump(count - 1, results, identifiers);
|
|
235
|
+
}
|
|
236
|
+
return pump(count - 1, results.concat(message), identifiers.concat(identifier));
|
|
237
|
+
}
|
|
215
238
|
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
return results;
|
|
219
|
-
}
|
|
220
|
-
const message = await channel.get(queue);
|
|
221
|
-
if (!message) {
|
|
222
|
-
return results;
|
|
223
|
-
}
|
|
224
|
-
const identifier = {
|
|
225
|
-
correlationId: message.properties.correlationId,
|
|
226
|
-
deliveryTag: message.fields.deliveryTag
|
|
227
|
-
};
|
|
228
|
-
const isDuplicate = identifiers.some(
|
|
229
|
-
(i) => i.correlationId === identifier.correlationId && i.deliveryTag === identifier.deliveryTag
|
|
230
|
-
);
|
|
231
|
-
if (isDuplicate) {
|
|
232
|
-
return pump(count - 1, results, identifiers);
|
|
233
|
-
}
|
|
234
|
-
return pump(count - 1, results.concat(message), identifiers.concat(identifier));
|
|
239
|
+
function getHeaderInfo(data) {
|
|
240
|
+
return data.properties.headers;
|
|
235
241
|
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
function errorUndefinedQueue(queue) {
|
|
241
|
-
if (queue === void 0 || queue === "" || queue === null) {
|
|
242
|
-
throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Undefined queue!`);
|
|
242
|
+
function errorUndefinedQueue(queue) {
|
|
243
|
+
if (queue === undefined || queue === '' || queue === null) {
|
|
244
|
+
throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Undefined queue!`);
|
|
245
|
+
}
|
|
243
246
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
247
|
+
function handleAmqpErrors(error) {
|
|
248
|
+
const newError = error;
|
|
249
|
+
debug(`HandleAmqpErrors got an error: ${JSON.stringify(error)}`);
|
|
250
|
+
if (error instanceof ApiError) {
|
|
251
|
+
debug(`We have an ApiError`);
|
|
252
|
+
throw new ApiError(error.status, error.payload);
|
|
253
|
+
}
|
|
254
|
+
debug(`We have a non-ApiError`);
|
|
255
|
+
logError(error);
|
|
256
|
+
throw newError;
|
|
251
257
|
}
|
|
252
|
-
debug(`We have a non-ApiError`);
|
|
253
|
-
logError(error);
|
|
254
|
-
throw newError;
|
|
255
|
-
}
|
|
256
258
|
}
|
|
257
|
-
//# sourceMappingURL=amqp.js.map
|
|
259
|
+
//# sourceMappingURL=amqp.js.map
|