@natlibfi/melinda-rest-api-commons 6.1.0-alpha.0 → 6.2.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/.github/workflows/melinda-node-tests-and-publish.yml +4 -2
  2. package/README.md +53 -5
  3. package/dist/amqp.d.ts +41 -0
  4. package/dist/amqp.d.ts.map +1 -0
  5. package/dist/amqp.js +237 -235
  6. package/dist/amqp.js.map +1 -7
  7. package/dist/constants.d.ts +49 -0
  8. package/dist/constants.d.ts.map +1 -0
  9. package/dist/constants.js +38 -38
  10. package/dist/constants.js.map +1 -7
  11. package/dist/helpers/conversion.d.ts +8 -0
  12. package/dist/helpers/conversion.d.ts.map +1 -0
  13. package/dist/helpers/conversion.js +70 -64
  14. package/dist/helpers/conversion.js.map +1 -7
  15. package/dist/helpers/fix-constants.d.ts +76 -0
  16. package/dist/helpers/fix-constants.d.ts.map +1 -0
  17. package/dist/helpers/fix-constants.js +41 -41
  18. package/dist/helpers/fix-constants.js.map +1 -7
  19. package/dist/helpers/fix-handle-tempurns.d.ts +3 -0
  20. package/dist/helpers/fix-handle-tempurns.d.ts.map +1 -0
  21. package/dist/helpers/fix-handle-tempurns.js +73 -71
  22. package/dist/helpers/fix-handle-tempurns.js.map +1 -7
  23. package/dist/helpers/fix-strip-f884s.d.ts +10 -0
  24. package/dist/helpers/fix-strip-f884s.d.ts.map +1 -0
  25. package/dist/helpers/fix-strip-f884s.js +161 -163
  26. package/dist/helpers/fix-strip-f884s.js.map +1 -7
  27. package/dist/helpers/fix.d.ts +6 -0
  28. package/dist/helpers/fix.d.ts.map +1 -0
  29. package/dist/helpers/fix.js +133 -121
  30. package/dist/helpers/fix.js.map +1 -7
  31. package/dist/helpers/validation.d.ts +9 -0
  32. package/dist/helpers/validation.d.ts.map +1 -0
  33. package/dist/helpers/validation.js +14 -16
  34. package/dist/helpers/validation.js.map +1 -7
  35. package/dist/index.d.ts +13 -0
  36. package/dist/index.d.ts.map +1 -0
  37. package/dist/index.js +1 -1
  38. package/dist/index.js.map +1 -7
  39. package/dist/mongo.d.ts +165 -0
  40. package/dist/mongo.d.ts.map +1 -0
  41. package/dist/mongo.js +595 -547
  42. package/dist/mongo.js.map +1 -7
  43. package/dist/mongoError.d.ts +2 -0
  44. package/dist/mongoError.d.ts.map +1 -0
  45. package/dist/mongoError.js +19 -0
  46. package/dist/mongoError.js.map +1 -0
  47. package/dist/mongoLog.d.ts +50 -0
  48. package/dist/mongoLog.d.ts.map +1 -0
  49. package/dist/mongoLog.js +274 -242
  50. package/dist/mongoLog.js.map +1 -7
  51. package/dist/utils.d.ts +36 -0
  52. package/dist/utils.d.ts.map +1 -0
  53. package/dist/utils.js +112 -110
  54. package/dist/utils.js.map +1 -7
  55. package/eslint.config.ts +66 -0
  56. package/package.json +15 -11
  57. package/{tests/amqp-error-branches.test.js → test/amqp-error-branches.test.ts} +68 -52
  58. package/{tests/amqp.healthtest.js → test/amqp.healthtest.ts} +7 -3
  59. package/{tests/amqp.tests.localtest.js → test/amqp.tests.localtest.ts} +54 -35
  60. package/{tests/helpers/conversion.test.js → test/helpers/conversion.test.ts} +26 -14
  61. package/{tests/helpers/fix-edge.test.js → test/helpers/fix-edge.test.ts} +16 -12
  62. package/{tests/helpers/fix.test.js → test/helpers/fix.test.ts} +14 -8
  63. package/{tests/helpers/validation.test.js → test/helpers/validation.test.ts} +7 -6
  64. package/{tests/mongo-add-blob-size.test.js → test/mongo-add-blob-size.test.ts} +14 -15
  65. package/{tests/mongo-check-and-check-state.test.js → test/mongo-check-and-check-state.test.ts} +6 -8
  66. package/{tests/mongo-check-and-set-import-job-state.test.js → test/mongo-check-and-set-import-job-state.test.ts} +6 -9
  67. package/{tests/mongo-check-timeout.test.js → test/mongo-check-timeout.test.ts} +6 -9
  68. package/{tests/mongo-create-bulk.test.js → test/mongo-create-bulk.test.ts} +6 -7
  69. package/{tests/mongo-create-prio.test.js → test/mongo-create-prio.test.ts} +6 -6
  70. package/{tests/mongo-error-branches.test.js → test/mongo-error-branches.test.ts} +147 -51
  71. package/{tests/mongo-get-one.test.js → test/mongo-get-one.test.ts} +6 -6
  72. package/{tests/mongo-get-stream.test.js → test/mongo-get-stream.test.ts} +6 -8
  73. package/{tests/mongo-push-ids.test.js → test/mongo-push-ids.test.ts} +6 -6
  74. package/{tests/mongo-push-messages.test.js → test/mongo-push-messages.test.ts} +6 -6
  75. package/{tests/mongo-query-by-id.test.js → test/mongo-query-by-id.test.ts} +6 -9
  76. package/{tests/mongo-query.test.js → test/mongo-query.test.ts} +6 -7
  77. package/{tests/mongo-read-content.test.js → test/mongo-read-content.test.ts} +6 -9
  78. package/{tests/mongo-remove.test.js → test/mongo-remove.test.ts} +6 -8
  79. package/{tests/mongo-set-blob-size.test.js → test/mongo-set-blob-size.test.ts} +6 -6
  80. package/{tests/mongo-set-import-job-state.test.js → test/mongo-set-import-job-state.test.ts} +6 -6
  81. package/{tests/mongo-set-operation.test.js → test/mongo-set-operation.test.ts} +6 -6
  82. package/{tests/mongo-set-operations.test.js → test/mongo-set-operations.test.ts} +6 -6
  83. package/{tests/mongo-set-state.test.js → test/mongo-set-state.test.ts} +6 -6
  84. package/{tests/mongoLog-protect.test.js → test/mongoLog-protect.test.ts} +18 -17
  85. package/{tests/mongoLog.test.js → test/mongoLog.test.ts} +8 -8
  86. package/{tests/testUtils.js → test/testUtils.ts} +49 -15
  87. package/{tests/utils.test.js → test/utils.test.ts} +5 -5
  88. package/tsconfig.json +62 -0
  89. package/tsconfig.test.json +17 -0
  90. package/dist/helpers/fix-utils.js +0 -39
  91. package/dist/helpers/fix-utils.js.map +0 -7
  92. package/eslint.config.js +0 -53
@@ -28,7 +28,6 @@ jobs:
28
28
  cache: 'npm'
29
29
  env:
30
30
  NPM_CONFIG_IGNORE_SCRIPTS: true
31
- - run: npm update
32
31
  - run: npm audit --package-lock-only --production --audit-level=moderate
33
32
  - run: npm i
34
33
  - run: npm test
@@ -41,8 +40,11 @@ jobs:
41
40
 
42
41
  steps:
43
42
  - uses: actions/checkout@v7
44
- - uses: mikaelvesavuori/license-compliance-action@v1
43
+ - uses: mikaelvesavuori/license-compliance-action@v1.0.3
45
44
  with:
45
+ # default action list + BlueOak-1.0.0 (sax, via xml2js in
46
+ # marc-record-serializers) + MIT-0 — OSI-approved permissive
47
+ allow_licenses: 'MIT;ISC;0BSD;BSD-2-Clause;BSD-3-Clause;Apache-2.0;BlueOak-1.0.0;MIT-0'
46
48
  exclude_pattern: /^@natlibfi/
47
49
 
48
50
  njsscan:
package/README.md CHANGED
@@ -51,6 +51,40 @@ need:
51
51
  MONGO_TIMEOUT_MINUTES=10 # abort only after 10 minutes without progress
52
52
  ```
53
53
 
54
+ #### Full `QUEUE_ITEM_STATE` reference
55
+
56
+ The flow tables above show the happy-path sequence per service. The full set of `queueItemState` values (see `src/constants.ts`):
57
+
58
+ | State | Group | Meaning |
59
+ |-------|-------|---------|
60
+ | `VALIDATOR.PENDING_QUEUING` | VALIDATOR | Bulk job accepted (http); waiting to be picked up for validation. |
61
+ | `VALIDATOR.PENDING_VALIDATION` | VALIDATOR | Prio job accepted (http); waiting to be picked up for validation. |
62
+ | `VALIDATOR.QUEUING_IN_PROGRESS` | VALIDATOR | Validator is distributing records to the per-operation import queues. |
63
+ | `VALIDATOR.UPLOADING` | VALIDATOR | Stream-bulk upload in progress (records written to the GridFS blob). |
64
+ | `VALIDATOR.VALIDATING` | VALIDATOR | Validator is validating records. |
65
+ | `VALIDATOR.WAITING_FOR_RECORDS` | VALIDATOR | Stream-bulk job: partial records received, waiting for the remainder. |
66
+ | `IMPORTER.IN_QUEUE` | IMPORTER | Records are queued for import, awaiting the importer. |
67
+ | `IMPORTER.IMPORTING` | IMPORTER | Importer has the job and is importing it. |
68
+ | `IMPORTER.IN_PROCESS` | IMPORTER | Importer is processing the job. |
69
+ | `DONE` | terminal | Job completed successfully. |
70
+ | `ERROR` | terminal | Job failed; see `errorStatus` / `errorMessage`. |
71
+ | `ABORT` | terminal | Job aborted (e.g. timeout on a stalled job; `errorStatus` 408). |
72
+
73
+ #### Full `IMPORT_JOB_STATE` reference
74
+
75
+ Each queue item also carries a per-operation `importJobState` subdocument (`CREATE` / `UPDATE` / `FIX`). The full set of values:
76
+
77
+ | State | Meaning |
78
+ |-------|---------|
79
+ | `EMPTY` | No such operation in the job (coalesced default when the subdocument is absent). |
80
+ | `QUEUING` | Validator is queuing this operation's records. |
81
+ | `IN_QUEUE` | This operation's records are queued for import. |
82
+ | `IMPORTING` | Importer is importing this operation. |
83
+ | `PROCESSING` | Importer is processing this operation (record-load). |
84
+ | `DONE` | This operation completed. |
85
+ | `ERROR` | This operation failed. |
86
+ | `ABORT` | This operation aborted (e.g. timeout). |
87
+
54
88
  ### CHUNK_SIZE
55
89
  Chunck size to execution time ratio (From file to queue: ~450 records in 15sec)
56
90
 
@@ -79,18 +113,32 @@ Executed on test server using bridge to docker server and back to server
79
113
  | p_z07_priority_year | pZ07PriorityYear | 1998 | 1990 | 2099 | 2099 | Override indexing priority |
80
114
  | p_redirection_field | pRedirectionField | | | | | |
81
115
 
116
+ ## TypeScript toolchain
117
+
118
+ The source is TypeScript (Node >= 24, native type stripping for tests).
119
+
120
+ - `npm run build` runs `tsc` (`tsconfig.json`): `src/` -> `dist/`, emitting
121
+ `.js` + `.js.map` + `.d.ts` + `.d.ts.map` for every module. The published
122
+ entry point stays `./dist/index.js`; the `.d.ts` files are additive for
123
+ TypeScript consumers.
124
+ - `npm run typecheck` runs `tsc -p tsconfig.test.json` (typechecks `src/`
125
+ and `test/`, no emit).
126
+ - `npm test` runs `npm run lint` then `npm run typecheck` then
127
+ `npm run test:base` (what CI runs).
128
+
82
129
  ## Running the tests
83
130
 
84
131
  There are three test suites. The base suite runs anywhere; the two AMQP
85
132
  suites need a running RabbitMQ broker on `127.0.0.1:5672` (guest/guest) and
86
- are therefore excluded from the base glob and from CI.
133
+ are therefore excluded from the base glob and from CI. Tests are `.ts`
134
+ files run directly by `node --test` (no loader).
87
135
 
88
136
  | Command | What it runs | Requirements |
89
137
  |-------------------------|---------------------------------------------------------------------|-----------------------|
90
- | `npm run test:base` | all `tests/**/*.test.js` with coverage + thresholds | none |
91
- | `npm test` | `npm run lint` then `test:base` (what CI runs) | none |
92
- | `npm run test:local` | `tests/amqp.*.localtest.js`, AMQP operator against a live broker | RabbitMQ on localhost |
93
- | `npm run test:healthcheck` | `tests/amqp.healthtest.js`, healthCheck loop vs live broker | RabbitMQ on localhost |
138
+ | `npm run test:base` | all `tests/**/*.test.ts` with coverage + thresholds | none |
139
+ | `npm test` | `npm run lint`, `npm run typecheck`, then `test:base` (what CI runs) | none |
140
+ | `npm run test:local` | `tests/amqp.*.localtest.ts`, AMQP operator against a live broker | RabbitMQ on localhost |
141
+ | `npm run test:healthcheck` | `tests/amqp.healthtest.ts`, healthCheck loop vs live broker | RabbitMQ on localhost |
94
142
 
95
143
  `test:base` enforces minimum coverage thresholds via
96
144
  `--test-coverage-lines/--test-coverage-branches/--test-coverage-functions`
package/dist/amqp.d.ts ADDED
@@ -0,0 +1,41 @@
1
+ import { MarcRecord } from '@natlibfi/marc-record';
2
+ import type { GetMessage, Message } from 'amqplib';
3
+ export interface QueueItemRecordPayload {
4
+ headers: Record<string, unknown> | undefined;
5
+ records: MarcRecord[];
6
+ messages: Message[];
7
+ }
8
+ export interface QueueItemMessagePayload {
9
+ headers: Record<string, unknown> | undefined;
10
+ messages: Message[];
11
+ }
12
+ export interface CheckQueueParams {
13
+ queue: string;
14
+ style?: string;
15
+ toRecord?: boolean;
16
+ purge?: boolean;
17
+ }
18
+ export type CheckQueueResult = number | boolean | QueueItemMessagePayload | QueueItemRecordPayload | GetMessage;
19
+ export interface SendToQueueParams {
20
+ queue?: string;
21
+ correlationId: string;
22
+ headers?: Record<string, unknown>;
23
+ data: unknown;
24
+ }
25
+ export interface AmqpOperator {
26
+ checkQueue: (params: CheckQueueParams) => Promise<CheckQueueResult>;
27
+ consumeChunk: (queue: string, toRecord: boolean) => Promise<false | QueueItemMessagePayload | QueueItemRecordPayload>;
28
+ consumeOne: (queue: string, toRecord: boolean) => Promise<false | GetMessage | QueueItemRecordPayload>;
29
+ ackMessages: (messages: Message[]) => void;
30
+ nackMessages: (messages: Message[]) => void;
31
+ sendToQueue: (params: SendToQueueParams) => Promise<void>;
32
+ removeQueue: (queue: string) => Promise<void>;
33
+ messagesToRecords: (messages: Message[]) => MarcRecord[];
34
+ closeChannel: () => Promise<void>;
35
+ closeConnection: () => Promise<void>;
36
+ }
37
+ export default function (AMQP_URL: string, runHealthCheck?: boolean, deps?: {
38
+ connection?: unknown;
39
+ channel?: unknown;
40
+ }): Promise<AmqpOperator>;
41
+ //# sourceMappingURL=amqp.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"amqp.d.ts","sourceRoot":"","sources":["../src/amqp.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAC;AAGjD,OAAO,KAAK,EAAwB,UAAU,EAAE,OAAO,EAAU,MAAM,SAAS,CAAC;AAcjF,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC7C,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAMD,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC7C,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAMD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAOD,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,OAAO,GAAG,uBAAuB,GAAG,sBAAsB,GAAG,UAAU,CAAC;AAMhH,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,IAAI,EAAE,OAAO,CAAC;CACf;AAKD,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACpE,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,KAAK,GAAG,uBAAuB,GAAG,sBAAsB,CAAC,CAAC;IACtH,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,KAAK,GAAG,UAAU,GAAG,sBAAsB,CAAC,CAAC;IACvG,WAAW,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC3C,YAAY,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC5C,WAAW,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1D,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,iBAAiB,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;IACzD,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,eAAe,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAiBD,yBAA+B,QAAQ,EAAE,MAAM,EAAE,cAAc,UAAQ,EAAE,IAAI,GAAE;IAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CA2ZpJ"}
package/dist/amqp.js CHANGED
@@ -1,257 +1,259 @@
1
- import amqplib from "amqplib";
2
- import createDebugLogger from "debug";
3
- import { promisify, inspect } from "util";
4
- import httpStatus from "http-status";
5
- import { MarcRecord } from "@natlibfi/marc-record";
6
- import { createLogger, Error as ApiError } from "@natlibfi/melinda-commons";
1
+ import amqplib from 'amqplib';
2
+ import createDebugLogger from 'debug';
3
+ import { promisify, inspect } from 'util';
4
+ import httpStatus from 'http-status';
5
+ import { MarcRecord } from '@natlibfi/marc-record';
6
+ import { createLogger, Error as ApiError } from '@natlibfi/melinda-commons';
7
7
  import { CHUNK_SIZE } from "./constants.js";
8
8
  import { logError } from "./utils.js";
9
- export default async function(AMQP_URL, runHealthCheck = false, deps = {}) {
10
- const logger = createLogger();
11
- const debug = createDebugLogger("@natlibfi/melinda-rest-api-commons:amqp");
12
- const debugHC = debug.extend("HC");
13
- const debugData = debug.extend("data");
14
- const setTimeoutPromise = promisify(setTimeout);
15
- debug(`Creating an AMQP operator to ${AMQP_URL}`);
16
- const connection = deps.connection ?? await amqplib.connect(AMQP_URL);
17
- const channel = deps.channel ?? await connection.createChannel();
18
- let healthCheckRunning = runHealthCheck;
19
- const healthCheckLoop = runHealthCheck ? healthCheck() : false;
20
- debug(`Connection: ${connection}`);
21
- debug(`Channel: ${channel}`);
22
- debug(`HealthCheckLoop: ${healthCheckLoop ? "Running health check" : "Not running health check"}`);
23
- return { checkQueue, consumeChunk, consumeOne, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords, closeChannel, closeConnection };
24
- async function closeChannel() {
25
- healthCheckRunning = false;
26
- debug(`Closing channel`);
27
- await channel.close();
28
- debug(`Channel: ${channel}`);
29
- }
30
- async function closeConnection() {
31
- healthCheckRunning = false;
32
- debug(`Closing connection`);
33
- await connection.close();
9
+ export default async function (AMQP_URL, runHealthCheck = false, deps = {}) {
10
+ const logger = createLogger();
11
+ const debug = createDebugLogger('@natlibfi/melinda-rest-api-commons:amqp');
12
+ const debugHC = debug.extend('HC');
13
+ const debugData = debug.extend('data');
14
+ const setTimeoutPromise = promisify(setTimeout);
15
+ debug(`Creating an AMQP operator to ${AMQP_URL}`);
16
+ const connection = deps.connection ?? await amqplib.connect(AMQP_URL);
17
+ const channel = deps.channel ?? await connection.createChannel();
18
+ let healthCheckRunning = runHealthCheck;
19
+ const healthCheckLoop = runHealthCheck ? healthCheck() : false;
34
20
  debug(`Connection: ${connection}`);
35
- }
36
- async function healthCheck(wait = false) {
37
- if (!healthCheckRunning) {
38
- debugHC(`HealthCheck cancelled, not running`);
39
- return false;
21
+ debug(`Channel: ${channel}`);
22
+ debug(`HealthCheckLoop: ${healthCheckLoop ? 'Running health check' : 'Not running health check'}`);
23
+ return { checkQueue, consumeChunk, consumeOne, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords, closeChannel, closeConnection };
24
+ async function closeChannel() {
25
+ healthCheckRunning = false;
26
+ debug(`Closing channel`);
27
+ await channel.close();
28
+ debug(`Channel: ${channel}`);
29
+ }
30
+ async function closeConnection() {
31
+ healthCheckRunning = false;
32
+ debug(`Closing connection`);
33
+ await connection.close();
34
+ debug(`Connection: ${connection}`);
40
35
  }
41
- if (wait) {
42
- await setTimeoutPromise(wait);
43
- return healthCheck(false);
36
+ async function healthCheck(wait = false) {
37
+ if (!healthCheckRunning) {
38
+ debugHC(`HealthCheck cancelled, not running`);
39
+ return false;
40
+ }
41
+ if (wait) {
42
+ await setTimeoutPromise(wait);
43
+ return healthCheck(false);
44
+ }
45
+ try {
46
+ debugHC(`Health checking amqp by asserting a queue`);
47
+ await channel.assertQueue('HEALTHCHECK', { durable: false });
48
+ debugHC(`Waiting 200ms before running healthCheck next`);
49
+ return healthCheck(200);
50
+ }
51
+ catch (error) {
52
+ debugHC(`HealthCheck error ${JSON.stringify(error)}`);
53
+ if (!healthCheckRunning) {
54
+ debugHC(`HealthCheck cancelled while erroring, not throwing`);
55
+ return false;
56
+ }
57
+ handleAmqpErrors(error);
58
+ }
44
59
  }
45
- try {
46
- debugHC(`Health checking amqp by asserting a queue`);
47
- await channel.assertQueue("HEALTHCHECK", { durable: false });
48
- debugHC(`Waiting 200ms before running healthCheck next`);
49
- return healthCheck(200);
50
- } catch (error) {
51
- debugHC(`HealthCheck error ${JSON.stringify(error)}`);
52
- if (!healthCheckRunning) {
53
- debugHC(`HealthCheck cancelled while erroring, not throwing`);
54
- return false;
55
- }
56
- handleAmqpErrors(error);
60
+ async function checkQueue({ queue, style = 'basic', toRecord = true, purge = false }) {
61
+ debug(`checkQueue: ${queue}, Style: ${style}, toRecord: ${toRecord}, Purge: ${purge}`);
62
+ try {
63
+ errorUndefinedQueue(queue);
64
+ const channelInfo = await channel.assertQueue(queue, { durable: true });
65
+ if (purge) {
66
+ await purgeQueue(purge);
67
+ logger.verbose(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
68
+ debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
69
+ return checkQueue({ queue, style });
70
+ }
71
+ if (style === 'messages') {
72
+ return channelInfo.messageCount;
73
+ }
74
+ if (style === 'one') {
75
+ if (checkMessageCount(channelInfo)) {
76
+ return consumeOne(queue, toRecord);
77
+ }
78
+ return false;
79
+ }
80
+ if (style === 'basic') {
81
+ if (checkMessageCount(channelInfo)) {
82
+ return consumeChunk(queue, toRecord);
83
+ }
84
+ return false;
85
+ }
86
+ throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `CheckQueue does not recognize style ${style}`);
87
+ }
88
+ catch (error) {
89
+ handleAmqpErrors(error);
90
+ }
91
+ function checkMessageCount(channelInfo) {
92
+ if (channelInfo.messageCount < 1) {
93
+ debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);
94
+ return false;
95
+ }
96
+ debug(`Queue ${queue} has ${channelInfo.messageCount} messages`);
97
+ return true;
98
+ }
99
+ function purgeQueue(purge) {
100
+ debug(`Purging queue: ${queue}, purge: ${purge}`);
101
+ if (purge) {
102
+ return channel.purgeQueue(queue);
103
+ }
104
+ return undefined;
105
+ }
57
106
  }
58
- }
59
- async function checkQueue({ queue, style = "basic", toRecord = true, purge = false }) {
60
- debug(`checkQueue: ${queue}, Style: ${style}, toRecord: ${toRecord}, Purge: ${purge}`);
61
- try {
62
- errorUndefinedQueue(queue);
63
- const channelInfo = await channel.assertQueue(queue, { durable: true });
64
- if (purge) {
65
- await purgeQueue(purge);
66
- logger.verbose(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
67
- debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
68
- return checkQueue({ queue, style });
69
- }
70
- if (style === "messages") {
71
- return channelInfo.messageCount;
72
- }
73
- if (style === "one") {
74
- if (checkMessageCount(channelInfo)) {
75
- return consumeOne(queue, toRecord);
107
+ async function consumeChunk(queue, toRecord) {
108
+ debug(`Prepared to consumeChunk from queue: ${queue}`);
109
+ try {
110
+ await channel.assertQueue(queue, { durable: true });
111
+ const messages = await getData(queue);
112
+ if (messages.length === 0) {
113
+ debug(`consumeChunk: queue ${queue} drained, returning false`);
114
+ return false;
115
+ }
116
+ const headers = getHeaderInfo(messages[0]);
117
+ debug(`consumeChunk (${messages ? messages.length : '0'} from queue ${queue}) ${toRecord ? 'to records' : 'just messages'}`);
118
+ if (toRecord) {
119
+ const records = messagesToRecords(messages);
120
+ return { headers, records, messages };
121
+ }
122
+ return { headers, messages };
76
123
  }
77
- return false;
78
- }
79
- if (style === "basic") {
80
- if (checkMessageCount(channelInfo)) {
81
- return consumeChunk(queue, toRecord);
124
+ catch (error) {
125
+ handleAmqpErrors(error);
82
126
  }
83
- return false;
84
- }
85
- throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `CheckQueue does not recognize style ${style}`);
86
- } catch (error) {
87
- handleAmqpErrors(error);
88
127
  }
89
- function checkMessageCount(channelInfo) {
90
- if (channelInfo.messageCount < 1) {
91
- debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);
92
- return false;
93
- }
94
- debug(`Queue ${queue} has ${channelInfo.messageCount} messages`);
95
- return true;
128
+ async function consumeOne(queue, toRecord) {
129
+ debug(`Prepared to consumeOne from queue: ${queue}`);
130
+ try {
131
+ await channel.assertQueue(queue, { durable: true });
132
+ const message = await channel.get(queue);
133
+ debugData(`Message: ${inspect(message, { colors: true, maxArrayLength: 3, depth: 3 })}`);
134
+ debug(`consumeOne from queue: ${queue} ${toRecord ? 'to records' : 'just the message'}`);
135
+ if (message) {
136
+ if (toRecord) {
137
+ const headers = getHeaderInfo(message);
138
+ const records = messagesToRecords([message]);
139
+ return { headers, records, messages: [message] };
140
+ }
141
+ return message;
142
+ }
143
+ return false;
144
+ }
145
+ catch (error) {
146
+ handleAmqpErrors(error);
147
+ }
96
148
  }
97
- function purgeQueue(purge2) {
98
- debug(`Purging queue: ${queue}, purge: ${purge2}`);
99
- if (purge2) {
100
- return channel.purgeQueue(queue);
101
- }
149
+ function ackMessages(messages) {
150
+ messages.forEach(message => {
151
+ debug(`Ack message ${message.properties.correlationId}`);
152
+ channel.ack(message);
153
+ });
102
154
  }
103
- }
104
- async function consumeChunk(queue, toRecord) {
105
- debug(`Prepared to consumeChunk from queue: ${queue}`);
106
- try {
107
- await channel.assertQueue(queue, { durable: true });
108
- const messages = await getData(queue);
109
- if (messages.length === 0) {
110
- debug(`consumeChunk: queue ${queue} drained, returning false`);
111
- return false;
112
- }
113
- const headers = getHeaderInfo(messages[0]);
114
- debug(`consumeChunk (${messages ? messages.length : "0"} from queue ${queue}) ${toRecord ? "to records" : "just messages"}`);
115
- if (toRecord) {
116
- const records = messagesToRecords(messages);
117
- return { headers, records, messages };
118
- }
119
- return { headers, messages };
120
- } catch (error) {
121
- handleAmqpErrors(error);
155
+ function nackMessages(messages) {
156
+ messages.forEach(message => {
157
+ debug(`Nack message ${message.properties.correlationId}`);
158
+ channel.nack(message);
159
+ });
122
160
  }
123
- }
124
- async function consumeOne(queue, toRecord) {
125
- debug(`Prepared to consumeOne from queue: ${queue}`);
126
- try {
127
- await channel.assertQueue(queue, { durable: true });
128
- const message = await channel.get(queue);
129
- debugData(`Message: ${inspect(message, { colors: true, maxArrayLength: 3, depth: 3 })}`);
130
- debug(`consumeOne from queue: ${queue} ${toRecord ? "to records" : "just the message"}`);
131
- if (message) {
132
- if (toRecord) {
133
- const headers = getHeaderInfo(message);
134
- const records = messagesToRecords([message]);
135
- return { headers, records, messages: [message] };
161
+ async function sendToQueue({ queue, correlationId, headers, data }) {
162
+ debug(`sendToQueue`);
163
+ try {
164
+ debug(`Queue ${queue}`);
165
+ debug(`CorrelationId ${correlationId}`);
166
+ debugData(`Data ${JSON.stringify(data)}`);
167
+ debug(`Headers ${JSON.stringify(headers)}`);
168
+ errorUndefinedQueue(queue);
169
+ debug(`Asserting queue: ${queue}`);
170
+ await channel.assertQueue(queue, { durable: true });
171
+ debug(`Actually sendToQueue: ${queue}`);
172
+ await channel.sendToQueue(queue, Buffer.from(JSON.stringify({ data })), {
173
+ correlationId,
174
+ persistent: true,
175
+ headers
176
+ });
177
+ debug(`Send message for ${correlationId} to queue: ${queue}`);
178
+ }
179
+ catch (error) {
180
+ const errorToThrow = error;
181
+ debug(`SendToQueue errored: ${JSON.stringify(error)}`);
182
+ handleAmqpErrors(errorToThrow);
136
183
  }
137
- return message;
138
- }
139
- return false;
140
- } catch (error) {
141
- handleAmqpErrors(error);
142
184
  }
143
- }
144
- function ackMessages(messages) {
145
- messages.forEach((message) => {
146
- debug(`Ack message ${message.properties.correlationId}`);
147
- channel.ack(message);
148
- });
149
- }
150
- function nackMessages(messages) {
151
- messages.forEach((message) => {
152
- debug(`Nack message ${message.properties.correlationId}`);
153
- channel.nack(message);
154
- });
155
- }
156
- async function sendToQueue({ queue, correlationId, headers, data }) {
157
- debug(`sendToQueue`);
158
- try {
159
- debug(`Queue ${queue}`);
160
- debug(`CorrelationId ${correlationId}`);
161
- debugData(`Data ${JSON.stringify(data)}`);
162
- debug(`Headers ${JSON.stringify(headers)}`);
163
- errorUndefinedQueue(queue);
164
- debug(`Asserting queue: ${queue}`);
165
- await channel.assertQueue(queue, { durable: true });
166
- debug(`Actually sendToQueue: ${queue}`);
167
- await channel.sendToQueue(
168
- queue,
169
- Buffer.from(JSON.stringify({ data })),
170
- {
171
- correlationId,
172
- persistent: true,
173
- headers
185
+ async function removeQueue(queue) {
186
+ let tempChannel;
187
+ try {
188
+ tempChannel = await connection.createChannel();
189
+ logger.verbose(`Removing queue ${queue}.`);
190
+ debug(`Removing queue ${queue}.`);
191
+ await tempChannel.deleteQueue(queue);
192
+ }
193
+ finally {
194
+ if (tempChannel) {
195
+ await tempChannel.close();
196
+ }
174
197
  }
175
- );
176
- debug(`Send message for ${correlationId} to queue: ${queue}`);
177
- } catch (error) {
178
- const errorToThrow = error;
179
- debug(`SendToQueue errored: ${JSON.stringify(error)}`);
180
- handleAmqpErrors(errorToThrow);
181
198
  }
182
- }
183
- async function removeQueue(queue) {
184
- let tempChannel;
185
- try {
186
- tempChannel = await connection.createChannel();
187
- logger.verbose(`Removing queue ${queue}.`);
188
- debug(`Removing queue ${queue}.`);
189
- await tempChannel.deleteQueue(queue);
190
- } finally {
191
- if (tempChannel) {
192
- await tempChannel.close();
193
- }
199
+ function messagesToRecords(messages) {
200
+ debug(`Parsing messages (${messages.length}) to records`);
201
+ return messages.map(message => {
202
+ const content = JSON.parse(message.content.toString());
203
+ return new MarcRecord(content.data, { subfieldValues: false });
204
+ });
194
205
  }
195
- }
196
- function messagesToRecords(messages) {
197
- debug(`Parsing messages (${messages.length}) to records`);
198
- return messages.map((message) => {
199
- const content = JSON.parse(message.content.toString());
200
- return new MarcRecord(content.data, { subfieldValues: false });
201
- });
202
- }
203
- async function getData(queue) {
204
- debug(`Getting queue data from ${queue}`);
205
- try {
206
- const { messageCount } = await channel.checkQueue(queue);
207
- debug(`There is ${messageCount} messages in queue ${queue}`);
208
- const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;
209
- debug(`Getting ${messagesToGet} messages from queue ${queue}`);
210
- const messages = await pump(messagesToGet);
211
- debug(`Returning ${messages.length} unique messages`);
212
- return messages;
213
- } catch (error) {
214
- handleAmqpErrors(error);
206
+ async function getData(queue) {
207
+ debug(`Getting queue data from ${queue}`);
208
+ try {
209
+ const { messageCount } = await channel.checkQueue(queue);
210
+ debug(`There is ${messageCount} messages in queue ${queue}`);
211
+ const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;
212
+ debug(`Getting ${messagesToGet} messages from queue ${queue}`);
213
+ const messages = await pump(messagesToGet);
214
+ debug(`Returning ${messages.length} unique messages`);
215
+ return messages;
216
+ }
217
+ catch (error) {
218
+ handleAmqpErrors(error);
219
+ }
220
+ async function pump(count, results = [], identifiers = []) {
221
+ if (count === 0) {
222
+ return results;
223
+ }
224
+ const message = await channel.get(queue);
225
+ if (!message) {
226
+ return results;
227
+ }
228
+ const identifier = {
229
+ correlationId: message.properties.correlationId,
230
+ deliveryTag: message.fields.deliveryTag
231
+ };
232
+ const isDuplicate = identifiers.some(i => i.correlationId === identifier.correlationId && i.deliveryTag === identifier.deliveryTag);
233
+ if (isDuplicate) {
234
+ return pump(count - 1, results, identifiers);
235
+ }
236
+ return pump(count - 1, results.concat(message), identifiers.concat(identifier));
237
+ }
215
238
  }
216
- async function pump(count, results = [], identifiers = []) {
217
- if (count === 0) {
218
- return results;
219
- }
220
- const message = await channel.get(queue);
221
- if (!message) {
222
- return results;
223
- }
224
- const identifier = {
225
- correlationId: message.properties.correlationId,
226
- deliveryTag: message.fields.deliveryTag
227
- };
228
- const isDuplicate = identifiers.some(
229
- (i) => i.correlationId === identifier.correlationId && i.deliveryTag === identifier.deliveryTag
230
- );
231
- if (isDuplicate) {
232
- return pump(count - 1, results, identifiers);
233
- }
234
- return pump(count - 1, results.concat(message), identifiers.concat(identifier));
239
+ function getHeaderInfo(data) {
240
+ return data.properties.headers;
235
241
  }
236
- }
237
- function getHeaderInfo(data) {
238
- return data.properties.headers;
239
- }
240
- function errorUndefinedQueue(queue) {
241
- if (queue === void 0 || queue === "" || queue === null) {
242
- throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Undefined queue!`);
242
+ function errorUndefinedQueue(queue) {
243
+ if (queue === undefined || queue === '' || queue === null) {
244
+ throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Undefined queue!`);
245
+ }
243
246
  }
244
- }
245
- function handleAmqpErrors(error) {
246
- const newError = error;
247
- debug(`HandleAmqpErrors got an error: ${JSON.stringify(error)}`);
248
- if (error instanceof ApiError) {
249
- debug(`We have an ApiError`);
250
- throw new ApiError(newError.status, newError.payload);
247
+ function handleAmqpErrors(error) {
248
+ const newError = error;
249
+ debug(`HandleAmqpErrors got an error: ${JSON.stringify(error)}`);
250
+ if (error instanceof ApiError) {
251
+ debug(`We have an ApiError`);
252
+ throw new ApiError(error.status, error.payload);
253
+ }
254
+ debug(`We have a non-ApiError`);
255
+ logError(error);
256
+ throw newError;
251
257
  }
252
- debug(`We have a non-ApiError`);
253
- logError(error);
254
- throw newError;
255
- }
256
258
  }
257
- //# sourceMappingURL=amqp.js.map
259
+ //# sourceMappingURL=amqp.js.map