@onlineapps/mq-client-core 2.0.1-rc.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +153 -0
- package/README.md +491 -8
- package/package.json +12 -8
- package/src/BaseClient.js +953 -80
- package/src/buffer/InMemoryBuffer.js +50 -9
- package/src/buffer/MessageBuffer.js +20 -52
- package/src/config/composeConfig.js +60 -0
- package/src/config/configSchema.js +398 -17
- package/src/config/defaultConfig.js +169 -15
- package/src/config/deliveryPolicy.js +165 -0
- package/src/config/queueConfig.js +738 -64
- package/src/config.js +29 -0
- package/src/defaults.js +43 -0
- package/src/index.js +91 -2
- package/src/layers/PublishLayer.js +83 -37
- package/src/monitoring/PublishMonitor.js +11 -4
- package/src/monitoring-publish.js +81 -54
- package/src/transports/rabbitmqClient.js +2698 -738
- package/src/transports/transportFactory.js +9 -2
- package/src/utils/errorHandler.js +83 -4
- package/src/utils/nearestKey.js +101 -0
- package/src/utils/publishErrors.js +95 -10
- package/src/utils/redactCredentials.js +106 -0
- package/src/utils/serializer.js +12 -2
- package/src/workers/RecoveryWorker.js +58 -81
- package/src/buffer/RedisBuffer.js +0 -57
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
const { assertLogger } = require('@onlineapps/logger-contract');
|
|
4
|
+
|
|
3
5
|
/**
|
|
4
6
|
* Jednoduchý in-memory buffer pro zprávy, které čekají na bezpečné odeslání.
|
|
5
7
|
* - Omezený počtem položek (maxSize)
|
|
@@ -10,15 +12,54 @@
|
|
|
10
12
|
|
|
11
13
|
class InMemoryBuffer {
|
|
12
14
|
/**
|
|
15
|
+
* Both numbers are REQUIRED and have no stand-in here.
|
|
16
|
+
*
|
|
17
|
+
* They are declared and defaulted one floor up — `publishBufferMaxSize` and
|
|
18
|
+
* `publishBufferTtlMs` in `config/defaultConfig.js`, composed before validation
|
|
19
|
+
* and handed down by the transport through `MessageBuffer`. Until d.311 this
|
|
20
|
+
* constructor wrote `options.maxSize || 100` and `options.ttlMs || 5 * 60 * 1000`,
|
|
21
|
+
* a second copy of two values that already had an owner — and `||` means "if
|
|
22
|
+
* falsy", so `publishBufferMaxSize: 0` ("hold nothing") arrived here as 100.
|
|
23
|
+
* That is the defect d.292 §5 removed from the transport and recorded as still
|
|
24
|
+
* open one floor down.
|
|
25
|
+
*
|
|
26
|
+
* Absence is therefore a defect of the caller, named as one
|
|
27
|
+
* (`architecture-principles.md` §3, §4, §5), not an invitation to invent a size.
|
|
28
|
+
*
|
|
13
29
|
* @param {Object} options
|
|
14
|
-
* @param {number}
|
|
15
|
-
* @param {number}
|
|
16
|
-
* @param {
|
|
30
|
+
* @param {number} options.maxSize - Max messages held; 0 means hold none
|
|
31
|
+
* @param {number} options.ttlMs - How long one message may wait, in ms
|
|
32
|
+
* @param {Object} options.logger - Logger s info/warn/error/debug (povinný)
|
|
17
33
|
*/
|
|
18
34
|
constructor(options = {}) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
35
|
+
// The logger first, so a caller who forgot it gets the contract message
|
|
36
|
+
// rather than a complaint about the next missing thing (same order as
|
|
37
|
+
// `BaseClient`).
|
|
38
|
+
this._logger = assertLogger(
|
|
39
|
+
'InMemoryBuffer',
|
|
40
|
+
options.logger,
|
|
41
|
+
'the buffer reports every message it drops when it is full or expired',
|
|
42
|
+
'pass options.logger (MessageBuffer forwards the one your service built)'
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
if (!Number.isFinite(options.maxSize)) {
|
|
46
|
+
throw new Error(
|
|
47
|
+
'[InMemoryBuffer] Missing required option - maxSize is required and has no default here. '
|
|
48
|
+
+ 'Expected: the value the client resolved from publishBufferMaxSize (config/defaultConfig.js). '
|
|
49
|
+
+ 'Fix: build this buffer through MessageBuffer, which is handed the resolved configuration, '
|
|
50
|
+
+ 'or pass maxSize explicitly.'
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
if (!Number.isFinite(options.ttlMs)) {
|
|
54
|
+
throw new Error(
|
|
55
|
+
'[InMemoryBuffer] Missing required option - ttlMs is required and has no default here. '
|
|
56
|
+
+ 'Expected: the value the client resolved from publishBufferTtlMs (config/defaultConfig.js). '
|
|
57
|
+
+ 'Fix: build this buffer through MessageBuffer, which is handed the resolved configuration, '
|
|
58
|
+
+ 'or pass ttlMs explicitly.'
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
this._maxSize = options.maxSize;
|
|
62
|
+
this._ttlMs = options.ttlMs;
|
|
22
63
|
|
|
23
64
|
/** @type {Array<{queue:string, buffer:Buffer, options:Object, createdAt:number, priority:string}>} */
|
|
24
65
|
this._items = [];
|
|
@@ -34,7 +75,7 @@ class InMemoryBuffer {
|
|
|
34
75
|
|
|
35
76
|
if (this._items.length >= this._maxSize) {
|
|
36
77
|
const dropped = this._items.shift();
|
|
37
|
-
this._logger
|
|
78
|
+
this._logger.warn(
|
|
38
79
|
`[InMemoryBuffer] Buffer full (${this._maxSize}), dropping oldest message for queue '${dropped.queue}'`
|
|
39
80
|
);
|
|
40
81
|
}
|
|
@@ -81,7 +122,7 @@ class InMemoryBuffer {
|
|
|
81
122
|
} catch (err) {
|
|
82
123
|
// Pokud flush selže, vrátíme zprávu zpět do bufferu (na konec fronty),
|
|
83
124
|
// aby ji mohl zpracovat další pokus / worker.
|
|
84
|
-
this._logger
|
|
125
|
+
this._logger.warn(
|
|
85
126
|
`[InMemoryBuffer] Failed to flush message for queue '${item.queue}': ${err.message}`
|
|
86
127
|
);
|
|
87
128
|
await this.add(item.queue, item.buffer, item.options, item.priority);
|
|
@@ -106,7 +147,7 @@ class InMemoryBuffer {
|
|
|
106
147
|
);
|
|
107
148
|
const removed = before - this._items.length;
|
|
108
149
|
if (removed > 0) {
|
|
109
|
-
this._logger
|
|
150
|
+
this._logger.info(
|
|
110
151
|
`[InMemoryBuffer] Removed ${removed} expired buffered messages`
|
|
111
152
|
);
|
|
112
153
|
}
|
|
@@ -1,25 +1,35 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
const InMemoryBuffer = require('./InMemoryBuffer');
|
|
4
|
-
const
|
|
4
|
+
const { assertLogger } = require('@onlineapps/logger-contract');
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* MessageBuffer –
|
|
7
|
+
* MessageBuffer – the buffer a failed publish is held in until the connection is
|
|
8
|
+
* back.
|
|
8
9
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
10
|
+
* ONE buffer, `InMemoryBuffer`, bounded by `publishBufferMaxSize` and
|
|
11
|
+
* `publishBufferTtlMs`. Until d.343 this class also routed critical-priority
|
|
12
|
+
* messages to a `RedisBuffer` whose `add()` and `flush()` stored and returned
|
|
13
|
+
* nothing, so `persistentBufferEnabled: true` DROPPED exactly the messages it
|
|
14
|
+
* claimed to protect — a declaration with no mechanism
|
|
15
|
+
* (`automation-gates.md` §5). The stub, its two config keys and this branch went
|
|
16
|
+
* out together; the four questions of `change-discipline.md` § Removing are
|
|
17
|
+
* answered in the commit that removed them.
|
|
11
18
|
*/
|
|
12
19
|
|
|
13
20
|
class MessageBuffer {
|
|
14
21
|
/**
|
|
15
22
|
* @param {Object} options
|
|
16
23
|
* @param {Object} [options.inMemory] - In-memory buffer options
|
|
17
|
-
* @param {Object}
|
|
18
|
-
* @param {boolean} [options.persistent.enabled=false] - Zapnutí persistentního bufferu
|
|
19
|
-
* @param {Object} [options.logger] - Logger (console-like)
|
|
24
|
+
* @param {Object} options.logger - Logger s info/warn/error/debug (povinny)
|
|
20
25
|
*/
|
|
21
26
|
constructor(options = {}) {
|
|
22
|
-
this._logger =
|
|
27
|
+
this._logger = assertLogger(
|
|
28
|
+
'MessageBuffer',
|
|
29
|
+
options.logger,
|
|
30
|
+
'the buffer reports every message it holds, drops or replays',
|
|
31
|
+
'pass options.logger (PublishLayer forwards the one your service built)'
|
|
32
|
+
);
|
|
23
33
|
|
|
24
34
|
this._inMemory = new InMemoryBuffer({
|
|
25
35
|
maxSize: options.inMemory?.maxSize,
|
|
@@ -27,19 +37,6 @@ class MessageBuffer {
|
|
|
27
37
|
logger: this._logger,
|
|
28
38
|
});
|
|
29
39
|
|
|
30
|
-
const persistentEnabled = !!options.persistent?.enabled;
|
|
31
|
-
this._persistent = persistentEnabled
|
|
32
|
-
? new RedisBuffer({
|
|
33
|
-
redisClient: options.persistent?.redisClient,
|
|
34
|
-
logger: this._logger,
|
|
35
|
-
})
|
|
36
|
-
: null;
|
|
37
|
-
|
|
38
|
-
if (!persistentEnabled) {
|
|
39
|
-
this._logger?.info?.(
|
|
40
|
-
'[MessageBuffer] Persistent buffer is disabled (persistent.enabled=false)'
|
|
41
|
-
);
|
|
42
|
-
}
|
|
43
40
|
}
|
|
44
41
|
|
|
45
42
|
/**
|
|
@@ -49,22 +46,9 @@ class MessageBuffer {
|
|
|
49
46
|
* @param {Object} options
|
|
50
47
|
* @param {Object} [meta]
|
|
51
48
|
* @param {string} [meta.priority='normal'] - 'normal' | 'critical'
|
|
52
|
-
* @param {boolean} [meta.persistent=false] - zda preferovat persistentní buffer
|
|
53
49
|
*/
|
|
54
50
|
async add(queue, buffer, options = {}, meta = {}) {
|
|
55
51
|
const priority = meta.priority || 'normal';
|
|
56
|
-
const wantPersistent = !!meta.persistent;
|
|
57
|
-
|
|
58
|
-
if (wantPersistent && this._persistent) {
|
|
59
|
-
try {
|
|
60
|
-
await this._persistent.add(queue, buffer, options, priority);
|
|
61
|
-
return;
|
|
62
|
-
} catch (err) {
|
|
63
|
-
this._logger?.warn?.(
|
|
64
|
-
`[MessageBuffer] Failed to add message to persistent buffer, falling back to in-memory: ${err.message}`
|
|
65
|
-
);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
52
|
|
|
69
53
|
await this._inMemory.add(queue, buffer, options, priority);
|
|
70
54
|
}
|
|
@@ -72,26 +56,10 @@ class MessageBuffer {
|
|
|
72
56
|
/**
|
|
73
57
|
* Flushne všechny buffered zprávy přes poskytnutou publish funkci.
|
|
74
58
|
* @param {Function} flushFn - async (queue, buffer, options) => void
|
|
75
|
-
* @returns {Promise<
|
|
59
|
+
* @returns {Promise<number>} how many messages were replayed
|
|
76
60
|
*/
|
|
77
61
|
async flush(flushFn) {
|
|
78
|
-
|
|
79
|
-
if (this._persistent) {
|
|
80
|
-
try {
|
|
81
|
-
persistentFlushed = await this._persistent.flush(flushFn);
|
|
82
|
-
} catch (err) {
|
|
83
|
-
this._logger?.error?.(
|
|
84
|
-
`[MessageBuffer] Failed to flush persistent buffer: ${err.message}`
|
|
85
|
-
);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
const inMemoryFlushed = await this._inMemory.flush(flushFn);
|
|
90
|
-
|
|
91
|
-
return {
|
|
92
|
-
inMemory: inMemoryFlushed,
|
|
93
|
-
persistent: persistentFlushed,
|
|
94
|
-
};
|
|
62
|
+
return await this._inMemory.flush(flushFn);
|
|
95
63
|
}
|
|
96
64
|
|
|
97
65
|
/**
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* composeConfig.js
|
|
5
|
+
*
|
|
6
|
+
* How this client builds ONE configuration object out of several sources
|
|
7
|
+
* (module defaults, the caller's config, the overrides `connect()` was given).
|
|
8
|
+
*
|
|
9
|
+
* SHALLOW, AND THAT IS THE POINT. `BaseClient` used `lodash.merge`, which walks
|
|
10
|
+
* into every plain object it is handed and copies it key by key. A config here
|
|
11
|
+
* carries two different kinds of value, and deep-merging treats them the same:
|
|
12
|
+
*
|
|
13
|
+
* - configuration keys — all 42 of them flat (`configSchema.js`), so there has
|
|
14
|
+
* never been a nested branch for a deep merge to merge;
|
|
15
|
+
* - INJECTED DEPENDENCIES — the `logger` every layer of this client writes
|
|
16
|
+
* through, and the callbacks (`onFatal`, `healthReportCallback`,
|
|
17
|
+
* `healthCriticalCallback`, …).
|
|
18
|
+
*
|
|
19
|
+
* Measured 2026-09-13: a dependency written as a plain object came out of that
|
|
20
|
+
* merge as a DIFFERENT object (`out.plainOne === plain` → false) while a class
|
|
21
|
+
* instance passed through by reference. So a dependency built as an object
|
|
22
|
+
* literal — a wrapper, a double, anything not built with `new` — reached the
|
|
23
|
+
* component that needs it as a copy: the same shape, none of the identity, and
|
|
24
|
+
* none of whatever the copy could not carry. It looked configured and was not.
|
|
25
|
+
* What the caller injects is what the mechanism must get
|
|
26
|
+
* (`architecture-principles.md` §1). (The example that measurement was written
|
|
27
|
+
* against was `persistentRedisClient`; the key went out with the stub it fed,
|
|
28
|
+
* d.343, and the rule it proved did not.)
|
|
29
|
+
*
|
|
30
|
+
* A key whose value is `undefined` is treated as NOT PROVIDED and never
|
|
31
|
+
* overwrites what an earlier source set. That is not a fallback, it is what
|
|
32
|
+
* "absent" means for an optional key in a config object literal: callers build
|
|
33
|
+
* their config with optional keys present and unset (`@onlineapps/service-wrapper`
|
|
34
|
+
* passes `onFatal` whether the service declared one or not), and `lodash.merge`
|
|
35
|
+
* skipped them the same way.
|
|
36
|
+
*
|
|
37
|
+
* The caller's own object is never written into: composition returns a new object
|
|
38
|
+
* and the sources are only read (`automation-gates.md` §1 requirement 3, the same
|
|
39
|
+
* idea applied to a library).
|
|
40
|
+
*
|
|
41
|
+
* @param {...(Object|null|undefined)} sources - Read left to right; a later
|
|
42
|
+
* source wins over an earlier one, except where its value is `undefined`.
|
|
43
|
+
* @returns {Object} A new object carrying every provided key, values by reference.
|
|
44
|
+
*/
|
|
45
|
+
function composeConfig(...sources) {
|
|
46
|
+
const composed = {};
|
|
47
|
+
|
|
48
|
+
for (const source of sources) {
|
|
49
|
+
if (source === null || source === undefined) continue;
|
|
50
|
+
|
|
51
|
+
for (const key of Object.keys(source)) {
|
|
52
|
+
if (source[key] === undefined) continue;
|
|
53
|
+
composed[key] = source[key];
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return composed;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
module.exports = composeConfig;
|