@onlineapps/mq-client-core 1.0.37 → 1.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/buffer/InMemoryBuffer.js +118 -0
- package/src/buffer/MessageBuffer.js +107 -0
- package/src/buffer/RedisBuffer.js +57 -0
- package/src/index.js +12 -0
- package/src/layers/PublishLayer.js +242 -0
- package/src/monitoring/PublishMonitor.js +138 -0
- package/src/transports/rabbitmqClient.js +149 -198
- package/src/utils/publishErrors.js +105 -0
- package/src/workers/RecoveryWorker.js +99 -0
package/package.json
CHANGED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Jednoduchý in-memory buffer pro zprávy, které čekají na bezpečné odeslání.
|
|
5
|
+
* - Omezený počtem položek (maxSize)
|
|
6
|
+
* - TTL pro jednotlivé položky (ttlMs)
|
|
7
|
+
*
|
|
8
|
+
* Primárně určený pro transient scénáře (health checky apod.).
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
class InMemoryBuffer {
|
|
12
|
+
/**
|
|
13
|
+
* @param {Object} options
|
|
14
|
+
* @param {number} [options.maxSize=100] - Max počet zpráv v bufferu
|
|
15
|
+
* @param {number} [options.ttlMs=300000] - TTL jedné zprávy v ms (default 5 minut)
|
|
16
|
+
* @param {Function} [options.logger] - Volitelný logger (console-like)
|
|
17
|
+
*/
|
|
18
|
+
constructor(options = {}) {
|
|
19
|
+
this._maxSize = options.maxSize || 100;
|
|
20
|
+
this._ttlMs = options.ttlMs || 5 * 60 * 1000;
|
|
21
|
+
this._logger = options.logger || console;
|
|
22
|
+
|
|
23
|
+
/** @type {Array<{queue:string, buffer:Buffer, options:Object, createdAt:number, priority:string}>} */
|
|
24
|
+
this._items = [];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Přidá zprávu do bufferu.
|
|
29
|
+
* Staré/expirující zprávy jsou průběžně čištěny, při překročení kapacity odhazujeme nejstarší.
|
|
30
|
+
*/
|
|
31
|
+
async add(queue, buffer, options = {}, priority = 'normal') {
|
|
32
|
+
const now = Date.now();
|
|
33
|
+
this._cleanupExpired(now);
|
|
34
|
+
|
|
35
|
+
if (this._items.length >= this._maxSize) {
|
|
36
|
+
const dropped = this._items.shift();
|
|
37
|
+
this._logger?.warn?.(
|
|
38
|
+
`[InMemoryBuffer] Buffer full (${this._maxSize}), dropping oldest message for queue '${dropped.queue}'`
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
this._items.push({
|
|
43
|
+
queue,
|
|
44
|
+
buffer,
|
|
45
|
+
options,
|
|
46
|
+
createdAt: now,
|
|
47
|
+
priority,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Flushne všechny aktuálně platné zprávy do poskytnuté publish funkce (FIFO, priority-aware).
|
|
53
|
+
* @param {Function} flushFn - async (queue, buffer, options) => void
|
|
54
|
+
* @returns {Promise<number>} - počet úspěšně flushnutých zpráv
|
|
55
|
+
*/
|
|
56
|
+
async flush(flushFn) {
|
|
57
|
+
const now = Date.now();
|
|
58
|
+
this._cleanupExpired(now);
|
|
59
|
+
|
|
60
|
+
if (this._items.length === 0) {
|
|
61
|
+
return 0;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Zkopírujeme a vyprázdníme interní pole, abychom se vyhnuli reentrancy problémům
|
|
65
|
+
const items = this._items.slice();
|
|
66
|
+
this._items = [];
|
|
67
|
+
|
|
68
|
+
// Kritické zprávy (priority === 'critical') házíme dopředu
|
|
69
|
+
items.sort((a, b) => {
|
|
70
|
+
if (a.priority === b.priority) return a.createdAt - b.createdAt;
|
|
71
|
+
if (a.priority === 'critical') return -1;
|
|
72
|
+
if (b.priority === 'critical') return 1;
|
|
73
|
+
return a.createdAt - b.createdAt;
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
let flushed = 0;
|
|
77
|
+
for (const item of items) {
|
|
78
|
+
try {
|
|
79
|
+
await flushFn(item.queue, item.buffer, item.options || {});
|
|
80
|
+
flushed++;
|
|
81
|
+
} catch (err) {
|
|
82
|
+
// Pokud flush selže, vrátíme zprávu zpět do bufferu (na konec fronty),
|
|
83
|
+
// aby ji mohl zpracovat další pokus / worker.
|
|
84
|
+
this._logger?.warn?.(
|
|
85
|
+
`[InMemoryBuffer] Failed to flush message for queue '${item.queue}': ${err.message}`
|
|
86
|
+
);
|
|
87
|
+
await this.add(item.queue, item.buffer, item.options, item.priority);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return flushed;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Aktuální velikost bufferu.
|
|
96
|
+
*/
|
|
97
|
+
size() {
|
|
98
|
+
this._cleanupExpired(Date.now());
|
|
99
|
+
return this._items.length;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
_cleanupExpired(now) {
|
|
103
|
+
const before = this._items.length;
|
|
104
|
+
this._items = this._items.filter(
|
|
105
|
+
(item) => now - item.createdAt <= this._ttlMs
|
|
106
|
+
);
|
|
107
|
+
const removed = before - this._items.length;
|
|
108
|
+
if (removed > 0) {
|
|
109
|
+
this._logger?.info?.(
|
|
110
|
+
`[InMemoryBuffer] Removed ${removed} expired buffered messages`
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
module.exports = InMemoryBuffer;
|
|
117
|
+
|
|
118
|
+
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const InMemoryBuffer = require('./InMemoryBuffer');
|
|
4
|
+
const RedisBuffer = require('./RedisBuffer');
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* MessageBuffer – sjednocený interface nad in-memory a (potenciálně) persistentním bufferem.
|
|
8
|
+
*
|
|
9
|
+
* - InMemoryBuffer: pro transient zprávy (např. health checky)
|
|
10
|
+
* - RedisBuffer: pro kritické zprávy (workflow.completed apod.) – aktuálně jen připravený hook
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
class MessageBuffer {
|
|
14
|
+
/**
|
|
15
|
+
* @param {Object} options
|
|
16
|
+
* @param {Object} [options.inMemory] - In-memory buffer options
|
|
17
|
+
* @param {Object} [options.persistent] - Persistent buffer options (Redis)
|
|
18
|
+
* @param {boolean} [options.persistent.enabled=false] - Zapnutí persistentního bufferu
|
|
19
|
+
* @param {Object} [options.logger] - Logger (console-like)
|
|
20
|
+
*/
|
|
21
|
+
constructor(options = {}) {
|
|
22
|
+
this._logger = options.logger || console;
|
|
23
|
+
|
|
24
|
+
this._inMemory = new InMemoryBuffer({
|
|
25
|
+
maxSize: options.inMemory?.maxSize,
|
|
26
|
+
ttlMs: options.inMemory?.ttlMs,
|
|
27
|
+
logger: this._logger,
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
const persistentEnabled = !!options.persistent?.enabled;
|
|
31
|
+
this._persistent = persistentEnabled
|
|
32
|
+
? new RedisBuffer({
|
|
33
|
+
redisClient: options.persistent?.redisClient,
|
|
34
|
+
logger: this._logger,
|
|
35
|
+
})
|
|
36
|
+
: null;
|
|
37
|
+
|
|
38
|
+
if (!persistentEnabled) {
|
|
39
|
+
this._logger?.info?.(
|
|
40
|
+
'[MessageBuffer] Persistent buffer is disabled (persistent.enabled=false)'
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Přidá zprávu do bufferu.
|
|
47
|
+
* @param {string} queue
|
|
48
|
+
* @param {Buffer} buffer
|
|
49
|
+
* @param {Object} options
|
|
50
|
+
* @param {Object} [meta]
|
|
51
|
+
* @param {string} [meta.priority='normal'] - 'normal' | 'critical'
|
|
52
|
+
* @param {boolean} [meta.persistent=false] - zda preferovat persistentní buffer
|
|
53
|
+
*/
|
|
54
|
+
async add(queue, buffer, options = {}, meta = {}) {
|
|
55
|
+
const priority = meta.priority || 'normal';
|
|
56
|
+
const wantPersistent = !!meta.persistent;
|
|
57
|
+
|
|
58
|
+
if (wantPersistent && this._persistent) {
|
|
59
|
+
try {
|
|
60
|
+
await this._persistent.add(queue, buffer, options, priority);
|
|
61
|
+
return;
|
|
62
|
+
} catch (err) {
|
|
63
|
+
this._logger?.warn?.(
|
|
64
|
+
`[MessageBuffer] Failed to add message to persistent buffer, falling back to in-memory: ${err.message}`
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
await this._inMemory.add(queue, buffer, options, priority);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Flushne všechny buffered zprávy přes poskytnutou publish funkci.
|
|
74
|
+
* @param {Function} flushFn - async (queue, buffer, options) => void
|
|
75
|
+
* @returns {Promise<{inMemory:number,persistent:number}>}
|
|
76
|
+
*/
|
|
77
|
+
async flush(flushFn) {
|
|
78
|
+
let persistentFlushed = 0;
|
|
79
|
+
if (this._persistent) {
|
|
80
|
+
try {
|
|
81
|
+
persistentFlushed = await this._persistent.flush(flushFn);
|
|
82
|
+
} catch (err) {
|
|
83
|
+
this._logger?.error?.(
|
|
84
|
+
`[MessageBuffer] Failed to flush persistent buffer: ${err.message}`
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const inMemoryFlushed = await this._inMemory.flush(flushFn);
|
|
90
|
+
|
|
91
|
+
return {
|
|
92
|
+
inMemory: inMemoryFlushed,
|
|
93
|
+
persistent: persistentFlushed,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Aktuální velikost in-memory bufferu.
|
|
99
|
+
*/
|
|
100
|
+
size() {
|
|
101
|
+
return this._inMemory.size();
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
module.exports = MessageBuffer;
|
|
106
|
+
|
|
107
|
+
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Placeholder Redis buffer.
|
|
5
|
+
*
|
|
6
|
+
* Záměrně JE minimalistický: v tomto projektu zatím nemáme přímou Redis závislost v mq-client-core.
|
|
7
|
+
* Třída je připravená na budoucí rozšíření – aktuálně pouze no-op / deleguje na in-memory fallback.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
class RedisBuffer {
|
|
11
|
+
/**
|
|
12
|
+
* @param {Object} options
|
|
13
|
+
* @param {Object} [options.redisClient] - Volitelný Redis klient (s metodami set/get/lpush/brpop atd.)
|
|
14
|
+
* @param {Function} [options.logger] - Logger (console-like)
|
|
15
|
+
*/
|
|
16
|
+
constructor(options = {}) {
|
|
17
|
+
this._redis = options.redisClient || null;
|
|
18
|
+
this._logger = options.logger || console;
|
|
19
|
+
|
|
20
|
+
if (!this._redis) {
|
|
21
|
+
this._logger?.info?.(
|
|
22
|
+
'[RedisBuffer] No redisClient provided - RedisBuffer is effectively disabled (will not persist messages)'
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Přidá zprávu do persistentního bufferu.
|
|
29
|
+
* Aktuální implementace je no-op, pokud není k dispozici redisClient.
|
|
30
|
+
*/
|
|
31
|
+
async add(/* queue, buffer, options, priority */) {
|
|
32
|
+
if (!this._redis) {
|
|
33
|
+
// No-op; fallback na InMemoryBuffer zajišťuje MessageBuffer
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
// Budoucí rozšíření: implementace zápisu do Redis seznamu/streamu.
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Flushne všechny zprávy z Redis bufferu přes poskytnutý flushFn.
|
|
41
|
+
* Aktuální implementace je no-op, pokud není k dispozici redisClient.
|
|
42
|
+
*
|
|
43
|
+
* @param {Function} flushFn - async (queue, buffer, options) => void
|
|
44
|
+
* @returns {Promise<number>} - počet flushnutých zpráv
|
|
45
|
+
*/
|
|
46
|
+
async flush(flushFn) {
|
|
47
|
+
if (!this._redis) {
|
|
48
|
+
return 0;
|
|
49
|
+
}
|
|
50
|
+
// Budoucí rozšíření: čtení z Redis (např. list/stream) a volání flushFn.
|
|
51
|
+
return 0;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
module.exports = RedisBuffer;
|
|
56
|
+
|
|
57
|
+
|
package/src/index.js
CHANGED
|
@@ -19,6 +19,12 @@ const {
|
|
|
19
19
|
ConsumeError,
|
|
20
20
|
SerializationError,
|
|
21
21
|
} = require('./utils/errorHandler');
|
|
22
|
+
const {
|
|
23
|
+
TransientPublishError,
|
|
24
|
+
PermanentPublishError,
|
|
25
|
+
QueueNotFoundError,
|
|
26
|
+
classifyPublishError,
|
|
27
|
+
} = require('./utils/publishErrors');
|
|
22
28
|
|
|
23
29
|
// Export BaseClient as default (constructor), with additional named exports
|
|
24
30
|
// NOTE: When destructuring, use: const { BaseClient } = require('@onlineapps/mq-client-core');
|
|
@@ -34,4 +40,10 @@ module.exports.errors = {
|
|
|
34
40
|
ConsumeError,
|
|
35
41
|
SerializationError,
|
|
36
42
|
};
|
|
43
|
+
module.exports.publishErrors = {
|
|
44
|
+
TransientPublishError,
|
|
45
|
+
PermanentPublishError,
|
|
46
|
+
QueueNotFoundError,
|
|
47
|
+
classifyPublishError,
|
|
48
|
+
};
|
|
37
49
|
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const {
|
|
4
|
+
TransientPublishError,
|
|
5
|
+
PermanentPublishError,
|
|
6
|
+
QueueNotFoundError,
|
|
7
|
+
} = require('../utils/publishErrors');
|
|
8
|
+
const MessageBuffer = require('../buffer/MessageBuffer');
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* PublishLayer - čistá vrstva pro publish s retry + buffer
|
|
12
|
+
*
|
|
13
|
+
* Odděluje:
|
|
14
|
+
* - Retry logiku (exponential backoff)
|
|
15
|
+
* - Bufferování při transient chybách
|
|
16
|
+
* - Delegaci na RabbitMQClient._publishOnce() (čisté publish bez retry)
|
|
17
|
+
*/
|
|
18
|
+
class PublishLayer {
|
|
19
|
+
/**
|
|
20
|
+
* @param {Object} options
|
|
21
|
+
* @param {Object} options.client - Instance RabbitMQClient
|
|
22
|
+
* @param {Object} [options.logger] - Logger
|
|
23
|
+
* @param {Object} [options.bufferConfig] - Konfigurace pro MessageBuffer
|
|
24
|
+
* @param {boolean} [options.retryEnabled=true] - Zapnout retry
|
|
25
|
+
* @param {number} [options.maxRetries=3] - Max počet pokusů
|
|
26
|
+
* @param {number} [options.retryBaseDelay=100] - Base delay v ms
|
|
27
|
+
* @param {number} [options.retryMaxDelay=5000] - Max delay v ms
|
|
28
|
+
* @param {number} [options.retryBackoffMultiplier=2] - Backoff multiplikátor
|
|
29
|
+
*/
|
|
30
|
+
constructor(options = {}) {
|
|
31
|
+
if (!options.client) {
|
|
32
|
+
throw new Error('PublishLayer requires a client instance');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
this._client = options.client;
|
|
36
|
+
this._logger = options.logger || console;
|
|
37
|
+
|
|
38
|
+
// Retry konfigurace - bere z client config nebo options
|
|
39
|
+
this._retryEnabled = options.retryEnabled !== undefined
|
|
40
|
+
? options.retryEnabled
|
|
41
|
+
: (this._client._publishRetryEnabled !== false);
|
|
42
|
+
this._maxRetries = options.maxRetries || this._client._publishMaxRetries || 3;
|
|
43
|
+
this._retryBaseDelay = options.retryBaseDelay || this._client._publishRetryBaseDelay || 100;
|
|
44
|
+
this._retryMaxDelay = options.retryMaxDelay || this._client._publishRetryMaxDelay || 5000;
|
|
45
|
+
this._retryBackoffMultiplier = options.retryBackoffMultiplier || this._client._publishRetryBackoffMultiplier || 2;
|
|
46
|
+
|
|
47
|
+
// Buffer
|
|
48
|
+
this._buffer = new MessageBuffer({
|
|
49
|
+
inMemory: options.bufferConfig?.inMemory,
|
|
50
|
+
persistent: options.bufferConfig?.persistent,
|
|
51
|
+
logger: this._logger,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Publish s retry + buffer logikou
|
|
57
|
+
* @param {string} queue
|
|
58
|
+
* @param {Buffer} buffer
|
|
59
|
+
* @param {Object} [options]
|
|
60
|
+
*/
|
|
61
|
+
async publish(queue, buffer, options = {}) {
|
|
62
|
+
const priority = options.priority || 'normal';
|
|
63
|
+
const usePersistentBuffer =
|
|
64
|
+
priority === 'critical' && !!this._client._config?.persistentBufferEnabled;
|
|
65
|
+
|
|
66
|
+
if (this._retryEnabled) {
|
|
67
|
+
return await this._publishWithRetry(queue, buffer, options, priority, usePersistentBuffer);
|
|
68
|
+
} else {
|
|
69
|
+
return await this._publishOnce(queue, buffer, options, priority, usePersistentBuffer);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Single publish attempt - deleguje na RabbitMQClient._publishOnce()
|
|
75
|
+
* @private
|
|
76
|
+
*/
|
|
77
|
+
async _publishOnce(queue, buffer, options, priority, usePersistentBuffer) {
|
|
78
|
+
try {
|
|
79
|
+
await this._client._publishOnce(queue, buffer, options);
|
|
80
|
+
} catch (err) {
|
|
81
|
+
return await this._handlePublishError(err, queue, buffer, options, priority, usePersistentBuffer);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Publish s retry logikou
|
|
87
|
+
* @private
|
|
88
|
+
*/
|
|
89
|
+
async _publishWithRetry(queue, buffer, options, priority, usePersistentBuffer) {
|
|
90
|
+
let lastError = null;
|
|
91
|
+
let attempt = 0;
|
|
92
|
+
|
|
93
|
+
while (attempt < this._maxRetries) {
|
|
94
|
+
attempt++;
|
|
95
|
+
|
|
96
|
+
try {
|
|
97
|
+
// Track attempt (i při prvním pokusu pro monitoring)
|
|
98
|
+
this._client.emit('publish:retry', {
|
|
99
|
+
queue,
|
|
100
|
+
attempt,
|
|
101
|
+
maxRetries: this._maxRetries,
|
|
102
|
+
lastError: lastError?.message
|
|
103
|
+
});
|
|
104
|
+
if (attempt > 1) {
|
|
105
|
+
this._logger?.log?.(`[PublishLayer] Retry attempt ${attempt}/${this._maxRetries} for queue "${queue}"`);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
await this._client._publishOnce(queue, buffer, options);
|
|
109
|
+
|
|
110
|
+
// Track success (always emit, even for first attempt)
|
|
111
|
+
this._client.emit('publish:success', {
|
|
112
|
+
queue,
|
|
113
|
+
attempt,
|
|
114
|
+
totalAttempts: attempt
|
|
115
|
+
});
|
|
116
|
+
if (attempt > 1) {
|
|
117
|
+
this._logger?.log?.(`[PublishLayer] ✓ Published successfully after ${attempt} attempts for queue "${queue}"`);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return; // Success
|
|
121
|
+
|
|
122
|
+
} catch (err) {
|
|
123
|
+
lastError = err;
|
|
124
|
+
|
|
125
|
+
// Non-retryable errors - fail immediately
|
|
126
|
+
if (err instanceof PermanentPublishError || err instanceof QueueNotFoundError) {
|
|
127
|
+
this._client.emit('publish:failed', {
|
|
128
|
+
queue,
|
|
129
|
+
attempt,
|
|
130
|
+
error: err.message,
|
|
131
|
+
retryable: false
|
|
132
|
+
});
|
|
133
|
+
throw err;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Transient errors - check if we should retry or buffer
|
|
137
|
+
if (err instanceof TransientPublishError) {
|
|
138
|
+
// If we have more attempts, retry
|
|
139
|
+
if (attempt < this._maxRetries) {
|
|
140
|
+
// Wait for reconnection if in progress
|
|
141
|
+
if (this._client._reconnecting) {
|
|
142
|
+
this._logger?.log?.(`[PublishLayer] Connection reconnecting, waiting before retry ${attempt + 1}/${this._maxRetries}...`);
|
|
143
|
+
try {
|
|
144
|
+
await this._client._waitForReconnection();
|
|
145
|
+
this._logger?.log?.(`[PublishLayer] ✓ Reconnection completed, proceeding with retry ${attempt + 1}/${this._maxRetries}`);
|
|
146
|
+
continue; // Retry immediately after reconnect
|
|
147
|
+
} catch (reconnectErr) {
|
|
148
|
+
// Reconnection failed - buffer and fail
|
|
149
|
+
await this._bufferMessage(queue, buffer, options, priority, usePersistentBuffer, err);
|
|
150
|
+
this._client.emit('publish:failed', {
|
|
151
|
+
queue,
|
|
152
|
+
attempt,
|
|
153
|
+
error: `Reconnection failed: ${reconnectErr.message}`,
|
|
154
|
+
retryable: false,
|
|
155
|
+
reconnectFailed: true
|
|
156
|
+
});
|
|
157
|
+
throw new Error(`Publish failed: reconnection failed after ${attempt} attempts for queue "${queue}": ${reconnectErr.message}`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Exponential backoff
|
|
162
|
+
const delay = Math.min(
|
|
163
|
+
this._retryBaseDelay * Math.pow(this._retryBackoffMultiplier, attempt - 1),
|
|
164
|
+
this._retryMaxDelay
|
|
165
|
+
);
|
|
166
|
+
this._logger?.warn?.(`[PublishLayer] Retryable error for queue "${queue}" (attempt ${attempt}/${this._maxRetries}), waiting ${delay}ms before retry: ${err.message}`);
|
|
167
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Max retries exceeded - buffer and fail
|
|
172
|
+
await this._bufferMessage(queue, buffer, options, priority, usePersistentBuffer, err);
|
|
173
|
+
this._client.emit('publish:failed', {
|
|
174
|
+
queue,
|
|
175
|
+
attempt,
|
|
176
|
+
error: err.message,
|
|
177
|
+
retryable: true,
|
|
178
|
+
maxRetriesExceeded: true
|
|
179
|
+
});
|
|
180
|
+
throw new Error(`Publish failed after ${attempt} attempts for queue "${queue}": ${err.message}`);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Unknown error - fail immediately
|
|
184
|
+
throw err;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
throw lastError || new Error(`Publish failed for queue "${queue}" after ${attempt} attempts`);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Handle publish error - buffer transient errors
|
|
193
|
+
* @private
|
|
194
|
+
*/
|
|
195
|
+
async _handlePublishError(err, queue, buffer, options, priority, usePersistentBuffer) {
|
|
196
|
+
if (err instanceof QueueNotFoundError) {
|
|
197
|
+
this._logger?.error?.(`[PublishLayer] QueueNotFoundError for '${queue}' (infra=${err.isInfrastructure}): ${err.message}`);
|
|
198
|
+
throw err;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (err instanceof TransientPublishError) {
|
|
202
|
+
await this._bufferMessage(queue, buffer, options, priority, usePersistentBuffer, err);
|
|
203
|
+
throw err;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
throw err;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Buffer message při transient error
|
|
211
|
+
* @private
|
|
212
|
+
*/
|
|
213
|
+
async _bufferMessage(queue, buffer, options, priority, usePersistentBuffer, err) {
|
|
214
|
+
try {
|
|
215
|
+
await this._buffer.add(queue, buffer, options, {
|
|
216
|
+
priority,
|
|
217
|
+
persistent: usePersistentBuffer,
|
|
218
|
+
});
|
|
219
|
+
this._logger?.warn?.(`[PublishLayer] Buffered message for queue '${queue}' due to transient error: ${err.message}`);
|
|
220
|
+
this._client.emit('publish:buffered', { queue });
|
|
221
|
+
} catch (bufferErr) {
|
|
222
|
+
this._logger?.error?.(`[PublishLayer] Failed to buffer message for queue '${queue}': ${bufferErr.message}`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Flush buffered messages po reconnectu
|
|
228
|
+
*/
|
|
229
|
+
async flushBuffered(flushOptions = {}) {
|
|
230
|
+
const result = await this._buffer.flush(async (queue, buf, opts) => {
|
|
231
|
+
const mergedOptions = Object.assign({}, opts, flushOptions);
|
|
232
|
+
await this._client._publishOnce(queue, buf, mergedOptions);
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
this._logger?.info?.(`[PublishLayer] Flushed buffered messages (inMemory=${result.inMemory}, persistent=${result.persistent})`);
|
|
236
|
+
return result;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
module.exports = PublishLayer;
|
|
241
|
+
|
|
242
|
+
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* PublishMonitor - tracking a metriky pro publish operace
|
|
5
|
+
*/
|
|
6
|
+
class PublishMonitor {
|
|
7
|
+
/**
|
|
8
|
+
* @param {Object} options
|
|
9
|
+
* @param {Object} [options.logger] - Logger
|
|
10
|
+
*/
|
|
11
|
+
constructor(options = {}) {
|
|
12
|
+
this._logger = options.logger || console;
|
|
13
|
+
|
|
14
|
+
// Metrics
|
|
15
|
+
this._metrics = {
|
|
16
|
+
attempts: 0,
|
|
17
|
+
successes: 0,
|
|
18
|
+
failures: 0,
|
|
19
|
+
retries: 0,
|
|
20
|
+
buffered: 0,
|
|
21
|
+
flushed: 0,
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
// Per-queue metrics
|
|
25
|
+
this._queueMetrics = new Map(); // queue -> { attempts, successes, failures, retries }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Track publish attempt
|
|
30
|
+
*/
|
|
31
|
+
trackAttempt(queue) {
|
|
32
|
+
this._metrics.attempts++;
|
|
33
|
+
this._updateQueueMetrics(queue, 'attempts');
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Track publish success
|
|
38
|
+
*/
|
|
39
|
+
trackSuccess(queue, attempt = 1) {
|
|
40
|
+
this._metrics.successes++;
|
|
41
|
+
if (attempt > 1) {
|
|
42
|
+
this._metrics.retries += (attempt - 1);
|
|
43
|
+
}
|
|
44
|
+
this._updateQueueMetrics(queue, 'successes', attempt);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Track publish failure
|
|
49
|
+
*/
|
|
50
|
+
trackFailure(queue, retryable = false) {
|
|
51
|
+
this._metrics.failures++;
|
|
52
|
+
this._updateQueueMetrics(queue, 'failures', 0, retryable);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Track buffered message
|
|
57
|
+
*/
|
|
58
|
+
trackBuffered(queue) {
|
|
59
|
+
this._metrics.buffered++;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Track flushed messages
|
|
64
|
+
*/
|
|
65
|
+
trackFlushed(count) {
|
|
66
|
+
this._metrics.flushed += count;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Get metrics
|
|
71
|
+
*/
|
|
72
|
+
getMetrics() {
|
|
73
|
+
return {
|
|
74
|
+
...this._metrics,
|
|
75
|
+
queues: Array.from(this._queueMetrics.entries()).map(([queue, metrics]) => ({
|
|
76
|
+
queue,
|
|
77
|
+
...metrics
|
|
78
|
+
}))
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Get Prometheus metrics format
|
|
84
|
+
*/
|
|
85
|
+
getPrometheusMetrics() {
|
|
86
|
+
const lines = [];
|
|
87
|
+
lines.push(`# HELP mq_publish_attempts Total publish attempts`);
|
|
88
|
+
lines.push(`# TYPE mq_publish_attempts counter`);
|
|
89
|
+
lines.push(`mq_publish_attempts ${this._metrics.attempts}`);
|
|
90
|
+
|
|
91
|
+
lines.push(`# HELP mq_publish_successes Total successful publishes`);
|
|
92
|
+
lines.push(`# TYPE mq_publish_successes counter`);
|
|
93
|
+
lines.push(`mq_publish_successes ${this._metrics.successes}`);
|
|
94
|
+
|
|
95
|
+
lines.push(`# HELP mq_publish_failures Total failed publishes`);
|
|
96
|
+
lines.push(`# TYPE mq_publish_failures counter`);
|
|
97
|
+
lines.push(`mq_publish_failures ${this._metrics.failures}`);
|
|
98
|
+
|
|
99
|
+
lines.push(`# HELP mq_publish_retries Total retry attempts`);
|
|
100
|
+
lines.push(`# TYPE mq_publish_retries counter`);
|
|
101
|
+
lines.push(`mq_publish_retries ${this._metrics.retries}`);
|
|
102
|
+
|
|
103
|
+
lines.push(`# HELP mq_publish_buffered Total buffered messages`);
|
|
104
|
+
lines.push(`# TYPE mq_publish_buffered counter`);
|
|
105
|
+
lines.push(`mq_publish_buffered ${this._metrics.buffered}`);
|
|
106
|
+
|
|
107
|
+
lines.push(`# HELP mq_publish_flushed Total flushed messages`);
|
|
108
|
+
lines.push(`# TYPE mq_publish_flushed counter`);
|
|
109
|
+
lines.push(`mq_publish_flushed ${this._metrics.flushed}`);
|
|
110
|
+
|
|
111
|
+
return lines.join('\n');
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Update queue metrics
|
|
116
|
+
* @private
|
|
117
|
+
*/
|
|
118
|
+
_updateQueueMetrics(queue, type, attempt = 1, retryable = false) {
|
|
119
|
+
if (!this._queueMetrics.has(queue)) {
|
|
120
|
+
this._queueMetrics.set(queue, {
|
|
121
|
+
attempts: 0,
|
|
122
|
+
successes: 0,
|
|
123
|
+
failures: 0,
|
|
124
|
+
retries: 0,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const metrics = this._queueMetrics.get(queue);
|
|
129
|
+
metrics[type]++;
|
|
130
|
+
|
|
131
|
+
if (type === 'successes' && attempt > 1) {
|
|
132
|
+
metrics.retries += (attempt - 1);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
module.exports = PublishMonitor;
|
|
138
|
+
|
|
@@ -8,6 +8,15 @@
|
|
|
8
8
|
|
|
9
9
|
const amqp = require('amqplib');
|
|
10
10
|
const EventEmitter = require('events');
|
|
11
|
+
const {
|
|
12
|
+
TransientPublishError,
|
|
13
|
+
PermanentPublishError,
|
|
14
|
+
QueueNotFoundError,
|
|
15
|
+
classifyPublishError,
|
|
16
|
+
} = require('../utils/publishErrors');
|
|
17
|
+
const PublishLayer = require('../layers/PublishLayer');
|
|
18
|
+
const RecoveryWorker = require('../workers/RecoveryWorker');
|
|
19
|
+
const PublishMonitor = require('../monitoring/PublishMonitor');
|
|
11
20
|
|
|
12
21
|
class RabbitMQClient extends EventEmitter {
|
|
13
22
|
/**
|
|
@@ -88,6 +97,89 @@ class RabbitMQClient extends EventEmitter {
|
|
|
88
97
|
this._criticalHealthShutdown = this._config.criticalHealthShutdown !== false; // Default: true - shutdown on critical health
|
|
89
98
|
this._criticalHealthShutdownDelay = this._config.criticalHealthShutdownDelay || 60000; // Default: 60s delay before shutdown
|
|
90
99
|
this._criticalHealthStartTime = null; // Track when critical health started
|
|
100
|
+
|
|
101
|
+
// Publish layer (retry + buffer)
|
|
102
|
+
this._publishLayer = new PublishLayer({
|
|
103
|
+
client: this,
|
|
104
|
+
logger: console,
|
|
105
|
+
bufferConfig: {
|
|
106
|
+
inMemory: {
|
|
107
|
+
maxSize: this._config.publishBufferMaxSize || 100,
|
|
108
|
+
ttlMs: this._config.publishBufferTtlMs || 5 * 60 * 1000,
|
|
109
|
+
},
|
|
110
|
+
persistent: {
|
|
111
|
+
enabled: !!this._config.persistentBufferEnabled,
|
|
112
|
+
redisClient: this._config.persistentRedisClient || null,
|
|
113
|
+
},
|
|
114
|
+
},
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
// Recovery worker (connection recovery, queue creation pro business)
|
|
118
|
+
this._recoveryWorker = new RecoveryWorker({
|
|
119
|
+
client: this,
|
|
120
|
+
scope: this._config.recoveryScope || 'infrastructure',
|
|
121
|
+
queueCreationFilter: this._config.queueCreationFilter || null,
|
|
122
|
+
logger: console,
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
// Publish monitor (metriky)
|
|
126
|
+
this._publishMonitor = new PublishMonitor({
|
|
127
|
+
logger: console,
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
// Event listeners pro monitoring
|
|
131
|
+
this._setupPublishMonitoring();
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Setup event listeners pro monitoring a recovery
|
|
136
|
+
* @private
|
|
137
|
+
*/
|
|
138
|
+
_setupPublishMonitoring() {
|
|
139
|
+
// Monitoring events
|
|
140
|
+
this.on('publish:retry', (data) => {
|
|
141
|
+
if (data.attempt === 1) {
|
|
142
|
+
this._publishMonitor.trackAttempt(data.queue);
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
this.on('publish:success', (data) => {
|
|
147
|
+
this._publishMonitor.trackSuccess(data.queue, data.totalAttempts);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
this.on('publish:failed', (data) => {
|
|
151
|
+
this._publishMonitor.trackFailure(data.queue, data.retryable);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
// Track buffered messages
|
|
155
|
+
this.on('publish:buffered', () => {
|
|
156
|
+
this._publishMonitor.trackBuffered();
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
// Recovery worker - handle QueueNotFoundError via error event
|
|
160
|
+
this.on('error', async (err) => {
|
|
161
|
+
if (err instanceof QueueNotFoundError) {
|
|
162
|
+
try {
|
|
163
|
+
await this._recoveryWorker.handleQueueNotFound(err, { queue: err.queueName });
|
|
164
|
+
} catch (recoveryErr) {
|
|
165
|
+
// Recovery failed - error already thrown
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Get publish metrics
|
|
173
|
+
*/
|
|
174
|
+
getPublishMetrics() {
|
|
175
|
+
return this._publishMonitor.getMetrics();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Get Prometheus metrics
|
|
180
|
+
*/
|
|
181
|
+
getPrometheusMetrics() {
|
|
182
|
+
return this._publishMonitor.getPrometheusMetrics();
|
|
91
183
|
}
|
|
92
184
|
|
|
93
185
|
/**
|
|
@@ -450,7 +542,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
450
542
|
console.log('[RabbitMQClient] [mq-client-core] ✓ Publisher channel recreated');
|
|
451
543
|
} catch (err) {
|
|
452
544
|
this._channel = null;
|
|
453
|
-
|
|
545
|
+
|
|
546
|
+
const msg = err && err.message ? err.message : String(err);
|
|
547
|
+
|
|
548
|
+
// Pokud se connection právě zavírá, považujeme to za přechodný stav – explicitně vyhodíme
|
|
549
|
+
// TransientPublishError, aby vyšší vrstva (retry / worker) věděla, že má počkat na reconnect.
|
|
550
|
+
if (msg.includes('Connection closing') || msg.includes('Connection closed')) {
|
|
551
|
+
console.warn(
|
|
552
|
+
`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection closing): ${msg}`
|
|
553
|
+
);
|
|
554
|
+
throw new TransientPublishError(
|
|
555
|
+
'Publisher channel cannot be recreated because connection is closing/closed (transient condition)',
|
|
556
|
+
err
|
|
557
|
+
);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// Ostatní chyby považujeme za permanentní problém na úrovni kanálu/spojení.
|
|
561
|
+
throw new PermanentPublishError(`Failed to recreate publisher channel: ${msg}`, err);
|
|
454
562
|
}
|
|
455
563
|
}
|
|
456
564
|
|
|
@@ -868,12 +976,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
868
976
|
* @throws {Error} If publish fails after all retry attempts.
|
|
869
977
|
*/
|
|
870
978
|
async publish(queue, buffer, options = {}) {
|
|
871
|
-
//
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
} else {
|
|
875
|
-
return await this._publishOnce(queue, buffer, options);
|
|
876
|
-
}
|
|
979
|
+
// PublishLayer zachovává stávající retry logiku a přidává možnost bufferování a
|
|
980
|
+
// jasnější klasifikaci chyb. API zůstává stejné.
|
|
981
|
+
return await this._publishLayer.publish(queue, buffer, options);
|
|
877
982
|
}
|
|
878
983
|
|
|
879
984
|
/**
|
|
@@ -909,9 +1014,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
909
1014
|
await this._queueChannel.checkQueue(queue);
|
|
910
1015
|
// Queue exists - proceed to publish
|
|
911
1016
|
} catch (checkErr) {
|
|
912
|
-
// If queue doesn't exist (404), this
|
|
913
|
-
// Infrastructure queues (workflow.*, registry.*, infrastructure.*, monitoring.*, validation.*)
|
|
914
|
-
//
|
|
1017
|
+
// If queue doesn't exist (404), this je ERROR – ale rozlišujeme infra vs. non-infra:
|
|
1018
|
+
// - Infrastructure queues (workflow.*, registry.*, infrastructure.*, monitoring.*, validation.*)
|
|
1019
|
+
// musí být vytvořené infra službami předem → QueueNotFoundError(isInfrastructure=true)
|
|
1020
|
+
// - Non-infrastructure queues můžeme (pro business scénáře) vytvořit s default parametry
|
|
915
1021
|
if (checkErr.code === 404) {
|
|
916
1022
|
// Check if this is an infrastructure queue using queueConfig
|
|
917
1023
|
let isInfraQueue = false;
|
|
@@ -928,13 +1034,18 @@ class RabbitMQClient extends EventEmitter {
|
|
|
928
1034
|
}
|
|
929
1035
|
|
|
930
1036
|
if (isInfraQueue) {
|
|
931
|
-
|
|
1037
|
+
// Jasný, hlasitý signál pro infra služby – fronta chybí, je to programátorská chyba
|
|
1038
|
+
throw new QueueNotFoundError(queue, true, checkErr);
|
|
932
1039
|
}
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
1040
|
+
// For non-infrastructure queues, allow auto-creation with default options
|
|
1041
|
+
const queueOptions = options.queueOptions || { durable: this._config.durable };
|
|
1042
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Auto-creating non-infrastructure queue ${queue} with default options (no TTL). This should be avoided for production.`);
|
|
1043
|
+
// Channel could have been nulled out by close handlers; ensure it's available
|
|
1044
|
+
if (!this._queueChannel) {
|
|
1045
|
+
await this._ensureQueueChannel();
|
|
1046
|
+
}
|
|
1047
|
+
this._trackChannelOperation(this._queueChannel, `assertQueue ${queue}`);
|
|
1048
|
+
await this._queueChannel.assertQueue(queue, queueOptions);
|
|
938
1049
|
} else {
|
|
939
1050
|
// Other error - rethrow
|
|
940
1051
|
throw checkErr;
|
|
@@ -1073,198 +1184,29 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1073
1184
|
await confirmPromise;
|
|
1074
1185
|
}
|
|
1075
1186
|
} catch (err) {
|
|
1076
|
-
//
|
|
1077
|
-
|
|
1078
|
-
|
|
1187
|
+
// Pokud je kanál zavřený, stále se pokusíme o jedno interní obnovení a retry,
|
|
1188
|
+
// aby byl publish co nejodolnější i předtím, než vstoupí do vyšší retry/recovery vrstvy.
|
|
1189
|
+
if (err && err.message && (err.message.includes('Channel closed') || err.message.includes('channel is closed') || this._channel?.closed)) {
|
|
1190
|
+
console.warn('[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during publish, recreating and retrying once...');
|
|
1079
1191
|
try {
|
|
1080
1192
|
await this._ensurePublisherChannel();
|
|
1081
|
-
// Retry publish once
|
|
1193
|
+
// Retry publish once (může znovu vyhodit – už dojde do retry/recovery vrstvy)
|
|
1082
1194
|
return await this.publish(queue, buffer, options);
|
|
1083
1195
|
} catch (retryErr) {
|
|
1084
|
-
console.error('[RabbitMQClient] [mq-client-core] [PUBLISH] Retry failed:', retryErr.message);
|
|
1085
|
-
|
|
1196
|
+
console.error('[RabbitMQClient] [mq-client-core] [PUBLISH] Retry after channel recreation failed:', retryErr.message);
|
|
1197
|
+
const classifiedRetryErr = classifyPublishError(retryErr);
|
|
1198
|
+
this.emit('error', classifiedRetryErr);
|
|
1199
|
+
throw classifiedRetryErr;
|
|
1086
1200
|
}
|
|
1087
1201
|
}
|
|
1088
|
-
this.emit('error', err);
|
|
1089
|
-
throw err;
|
|
1090
|
-
}
|
|
1091
|
-
}
|
|
1092
1202
|
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
* @param {Buffer} buffer - Message payload
|
|
1098
|
-
* @param {Object} options - Publish options
|
|
1099
|
-
* @returns {Promise<void>}
|
|
1100
|
-
*/
|
|
1101
|
-
async _publishWithRetry(queue, buffer, options = {}) {
|
|
1102
|
-
let lastError = null;
|
|
1103
|
-
let attempt = 0;
|
|
1104
|
-
|
|
1105
|
-
while (attempt < this._publishMaxRetries) {
|
|
1106
|
-
attempt++;
|
|
1107
|
-
|
|
1108
|
-
try {
|
|
1109
|
-
// Emit retry attempt event for monitoring
|
|
1110
|
-
if (attempt > 1) {
|
|
1111
|
-
this.emit('publish:retry', {
|
|
1112
|
-
queue,
|
|
1113
|
-
attempt,
|
|
1114
|
-
maxRetries: this._publishMaxRetries,
|
|
1115
|
-
lastError: lastError?.message
|
|
1116
|
-
});
|
|
1117
|
-
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] Retry attempt ${attempt}/${this._publishMaxRetries} for queue "${queue}"`);
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
// Attempt publish
|
|
1121
|
-
await this._publishOnce(queue, buffer, options);
|
|
1122
|
-
|
|
1123
|
-
// Success - emit success event if this was a retry
|
|
1124
|
-
if (attempt > 1) {
|
|
1125
|
-
this.emit('publish:success', {
|
|
1126
|
-
queue,
|
|
1127
|
-
attempt,
|
|
1128
|
-
totalAttempts: attempt
|
|
1129
|
-
});
|
|
1130
|
-
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Published successfully after ${attempt} attempts for queue "${queue}"`);
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
|
-
return; // Success - exit retry loop
|
|
1134
|
-
|
|
1135
|
-
} catch (err) {
|
|
1136
|
-
lastError = err;
|
|
1137
|
-
|
|
1138
|
-
// Check if error is retryable
|
|
1139
|
-
const isRetryable = this._isPublishErrorRetryable(err);
|
|
1140
|
-
|
|
1141
|
-
if (!isRetryable) {
|
|
1142
|
-
// Non-retryable error - fail immediately
|
|
1143
|
-
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Non-retryable error for queue "${queue}":`, err.message);
|
|
1144
|
-
this.emit('publish:failed', {
|
|
1145
|
-
queue,
|
|
1146
|
-
attempt,
|
|
1147
|
-
error: err.message,
|
|
1148
|
-
retryable: false
|
|
1149
|
-
});
|
|
1150
|
-
throw err;
|
|
1151
|
-
}
|
|
1152
|
-
|
|
1153
|
-
// Retryable error - check if we have more attempts
|
|
1154
|
-
if (attempt >= this._publishMaxRetries) {
|
|
1155
|
-
// Max retries exceeded
|
|
1156
|
-
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✗ Failed after ${attempt} attempts for queue "${queue}":`, err.message);
|
|
1157
|
-
this.emit('publish:failed', {
|
|
1158
|
-
queue,
|
|
1159
|
-
attempt,
|
|
1160
|
-
error: err.message,
|
|
1161
|
-
retryable: true,
|
|
1162
|
-
maxRetriesExceeded: true
|
|
1163
|
-
});
|
|
1164
|
-
throw new Error(`Publish failed after ${attempt} attempts for queue "${queue}": ${err.message}`);
|
|
1165
|
-
}
|
|
1166
|
-
|
|
1167
|
-
// CRITICAL: If reconnecting, wait for reconnection to complete before retry
|
|
1168
|
-
// This ensures we don't waste retry attempts while connection is being restored
|
|
1169
|
-
if (this._reconnecting) {
|
|
1170
|
-
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] Connection is reconnecting, waiting for reconnection before retry ${attempt + 1}/${this._publishMaxRetries}...`);
|
|
1171
|
-
try {
|
|
1172
|
-
await this._waitForReconnection();
|
|
1173
|
-
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Reconnection completed, proceeding with retry ${attempt + 1}/${this._publishMaxRetries}`);
|
|
1174
|
-
// Continue to retry immediately after reconnection (no additional delay needed)
|
|
1175
|
-
continue;
|
|
1176
|
-
} catch (reconnectErr) {
|
|
1177
|
-
// Reconnection failed - this is a critical error
|
|
1178
|
-
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Reconnection failed:`, reconnectErr.message);
|
|
1179
|
-
this.emit('publish:failed', {
|
|
1180
|
-
queue,
|
|
1181
|
-
attempt,
|
|
1182
|
-
error: `Reconnection failed: ${reconnectErr.message}`,
|
|
1183
|
-
retryable: false,
|
|
1184
|
-
reconnectFailed: true
|
|
1185
|
-
});
|
|
1186
|
-
throw new Error(`Publish failed: reconnection failed after ${attempt} attempts for queue "${queue}": ${reconnectErr.message}`);
|
|
1187
|
-
}
|
|
1188
|
-
}
|
|
1189
|
-
|
|
1190
|
-
// Calculate exponential backoff delay (only if not reconnecting)
|
|
1191
|
-
const delay = Math.min(
|
|
1192
|
-
this._publishRetryBaseDelay * Math.pow(this._publishRetryBackoffMultiplier, attempt - 1),
|
|
1193
|
-
this._publishRetryMaxDelay
|
|
1194
|
-
);
|
|
1195
|
-
|
|
1196
|
-
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Retryable error for queue "${queue}" (attempt ${attempt}/${this._publishMaxRetries}), waiting ${delay}ms before retry:`, err.message);
|
|
1197
|
-
|
|
1198
|
-
// Wait before retry
|
|
1199
|
-
await new Promise(resolve => setTimeout(resolve, delay));
|
|
1200
|
-
}
|
|
1203
|
+
// Všechny ostatní chyby projdou jednotnou klasifikací do Transient/Permanent/QueueNotFound.
|
|
1204
|
+
const classifiedErr = classifyPublishError(err);
|
|
1205
|
+
this.emit('error', classifiedErr);
|
|
1206
|
+
throw classifiedErr;
|
|
1201
1207
|
}
|
|
1202
|
-
|
|
1203
|
-
// Should never reach here, but just in case
|
|
1204
|
-
throw lastError || new Error(`Publish failed for queue "${queue}" after ${attempt} attempts`);
|
|
1205
1208
|
}
|
|
1206
1209
|
|
|
1207
|
-
/**
|
|
1208
|
-
* Determine if a publish error is retryable
|
|
1209
|
-
* @private
|
|
1210
|
-
* @param {Error} err - Error to check
|
|
1211
|
-
* @returns {boolean} True if error is retryable
|
|
1212
|
-
*/
|
|
1213
|
-
_isPublishErrorRetryable(err) {
|
|
1214
|
-
if (!err || !err.message) {
|
|
1215
|
-
return false;
|
|
1216
|
-
}
|
|
1217
|
-
|
|
1218
|
-
const message = err.message.toLowerCase();
|
|
1219
|
-
|
|
1220
|
-
// Retryable errors: connection/channel issues, timeouts, transient broker errors
|
|
1221
|
-
const retryablePatterns = [
|
|
1222
|
-
'channel closed',
|
|
1223
|
-
'channel ended',
|
|
1224
|
-
'connection closed',
|
|
1225
|
-
'connection ended',
|
|
1226
|
-
'timeout',
|
|
1227
|
-
'confirmation timeout',
|
|
1228
|
-
'delivery tag invalid',
|
|
1229
|
-
'unknown delivery tag',
|
|
1230
|
-
'precondition_failed.*delivery tag',
|
|
1231
|
-
'not connected',
|
|
1232
|
-
'reconnecting'
|
|
1233
|
-
];
|
|
1234
|
-
|
|
1235
|
-
// Non-retryable errors: permanent failures, validation errors
|
|
1236
|
-
const nonRetryablePatterns = [
|
|
1237
|
-
'queue does not exist',
|
|
1238
|
-
'exchange does not exist',
|
|
1239
|
-
'access refused',
|
|
1240
|
-
'not found',
|
|
1241
|
-
'permission denied',
|
|
1242
|
-
'invalid',
|
|
1243
|
-
'malformed'
|
|
1244
|
-
];
|
|
1245
|
-
|
|
1246
|
-
// Check non-retryable first
|
|
1247
|
-
for (const pattern of nonRetryablePatterns) {
|
|
1248
|
-
if (message.includes(pattern)) {
|
|
1249
|
-
return false;
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
|
|
1253
|
-
// Check retryable
|
|
1254
|
-
for (const pattern of retryablePatterns) {
|
|
1255
|
-
if (message.includes(pattern)) {
|
|
1256
|
-
return true;
|
|
1257
|
-
}
|
|
1258
|
-
}
|
|
1259
|
-
|
|
1260
|
-
// Default: if we're reconnecting, it's retryable
|
|
1261
|
-
if (this._reconnecting) {
|
|
1262
|
-
return true;
|
|
1263
|
-
}
|
|
1264
|
-
|
|
1265
|
-
// Default: unknown errors are not retryable (fail fast)
|
|
1266
|
-
return false;
|
|
1267
|
-
}
|
|
1268
1210
|
|
|
1269
1211
|
/**
|
|
1270
1212
|
* Starts consuming messages from the specified queue.
|
|
@@ -1610,6 +1552,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1610
1552
|
console.log('[RabbitMQClient] ✓ Connection-level recovery completed successfully - all channels ready');
|
|
1611
1553
|
this.emit('reconnected');
|
|
1612
1554
|
|
|
1555
|
+
// Flush buffered messages po reconnectu
|
|
1556
|
+
try {
|
|
1557
|
+
const flushResult = await this._publishLayer.flushBuffered();
|
|
1558
|
+
this._publishMonitor.trackFlushed(flushResult.inMemory + flushResult.persistent);
|
|
1559
|
+
console.log(`[RabbitMQClient] ✓ Flushed ${flushResult.inMemory + flushResult.persistent} buffered messages after reconnection`);
|
|
1560
|
+
} catch (flushErr) {
|
|
1561
|
+
console.warn(`[RabbitMQClient] Failed to flush buffered messages after reconnection: ${flushErr.message}`);
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1613
1564
|
return; // Success - exit reconnection loop
|
|
1614
1565
|
} catch (err) {
|
|
1615
1566
|
this._reconnectAttempts++;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Specialized error types for publish/reconnect flow.
|
|
5
|
+
* These errors umožní čisté oddělení publish vrstvy a recovery workeru.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
class TransientPublishError extends Error {
|
|
9
|
+
/**
|
|
10
|
+
* @param {string} message - Human readable message
|
|
11
|
+
* @param {Error} [cause] - Underlying error
|
|
12
|
+
*/
|
|
13
|
+
constructor(message, cause) {
|
|
14
|
+
super(message);
|
|
15
|
+
this.name = 'TransientPublishError';
|
|
16
|
+
this.cause = cause || null;
|
|
17
|
+
this.retryable = true;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
class PermanentPublishError extends Error {
|
|
22
|
+
/**
|
|
23
|
+
* @param {string} message - Human readable message
|
|
24
|
+
* @param {Error} [cause] - Underlying error
|
|
25
|
+
*/
|
|
26
|
+
constructor(message, cause) {
|
|
27
|
+
super(message);
|
|
28
|
+
this.name = 'PermanentPublishError';
|
|
29
|
+
this.cause = cause || null;
|
|
30
|
+
this.retryable = false;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
class QueueNotFoundError extends PermanentPublishError {
|
|
35
|
+
/**
|
|
36
|
+
* @param {string} queueName
|
|
37
|
+
* @param {boolean} isInfrastructure
|
|
38
|
+
* @param {Error} [cause]
|
|
39
|
+
*/
|
|
40
|
+
constructor(queueName, isInfrastructure, cause) {
|
|
41
|
+
const baseMessage = isInfrastructure
|
|
42
|
+
? `Cannot publish to infrastructure queue ${queueName}: queue does not exist. Infrastructure queues must be created explicitly with correct arguments (TTL, max-length, etc.) before publishing.`
|
|
43
|
+
: `Queue ${queueName} does not exist`;
|
|
44
|
+
super(baseMessage, cause);
|
|
45
|
+
this.name = 'QueueNotFoundError';
|
|
46
|
+
this.queueName = queueName;
|
|
47
|
+
this.isInfrastructure = !!isInfrastructure;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Best-effort klasifikace chyb z publishu do specializovaných typů.
|
|
53
|
+
* Vrací původní chybu, pokud neodpovídá žádnému známému patternu.
|
|
54
|
+
*
|
|
55
|
+
* @param {Error} err
|
|
56
|
+
* @returns {Error} - buď speciální error, nebo původní err
|
|
57
|
+
*/
|
|
58
|
+
function classifyPublishError(err) {
|
|
59
|
+
if (!err) {
|
|
60
|
+
return err;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Už je to náš specializovaný error – ponecháme jak je
|
|
64
|
+
if (
|
|
65
|
+
err instanceof TransientPublishError ||
|
|
66
|
+
err instanceof PermanentPublishError ||
|
|
67
|
+
err instanceof QueueNotFoundError
|
|
68
|
+
) {
|
|
69
|
+
return err;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const message = (err.message || String(err)).toLowerCase();
|
|
73
|
+
|
|
74
|
+
// Queue neexistuje – trvalý problém, nerozbitný retry
|
|
75
|
+
if (message.includes('queue does not exist') || message.includes('no queue') || err.code === 404) {
|
|
76
|
+
// Nevíme, zda je to infra/biz – necháme isInfrastructure=false, publish vrstva si může dovodit sama
|
|
77
|
+
return new QueueNotFoundError('unknown', false, err);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Typicky transientní problémy connection/channel
|
|
81
|
+
if (
|
|
82
|
+
message.includes('connection closing') ||
|
|
83
|
+
message.includes('connection closed') ||
|
|
84
|
+
message.includes('connection ended') ||
|
|
85
|
+
message.includes('channel closed') ||
|
|
86
|
+
message.includes('channel ended') ||
|
|
87
|
+
message.includes('timeout') ||
|
|
88
|
+
message.includes('confirmation timeout') ||
|
|
89
|
+
message.includes('reconnecting')
|
|
90
|
+
) {
|
|
91
|
+
return new TransientPublishError(err.message || String(err), err);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Ostatní považujeme za permanentní – ale explicitně označíme typ
|
|
95
|
+
return new PermanentPublishError(err.message || String(err), err);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
module.exports = {
|
|
99
|
+
TransientPublishError,
|
|
100
|
+
PermanentPublishError,
|
|
101
|
+
QueueNotFoundError,
|
|
102
|
+
classifyPublishError,
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { QueueNotFoundError } = require('../utils/publishErrors');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* RecoveryWorker - řeší problémy s publish (connection recovery, queue creation)
|
|
7
|
+
*
|
|
8
|
+
* Scope:
|
|
9
|
+
* - infrastructure: connection recovery, channel recreation, consumer re-registration (NENÍ queue creation)
|
|
10
|
+
* - business: connection recovery, channel recreation, consumer re-registration, queue creation (pokud není infra queue)
|
|
11
|
+
*/
|
|
12
|
+
class RecoveryWorker {
|
|
13
|
+
/**
|
|
14
|
+
* @param {Object} options
|
|
15
|
+
* @param {Object} options.client - RabbitMQClient instance
|
|
16
|
+
* @param {string} [options.scope='infrastructure'] - 'infrastructure' nebo 'business'
|
|
17
|
+
* @param {Object} [options.logger] - Logger
|
|
18
|
+
*/
|
|
19
|
+
constructor(options = {}) {
|
|
20
|
+
if (!options.client) {
|
|
21
|
+
throw new Error('RecoveryWorker requires a client instance');
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
this._client = options.client;
|
|
25
|
+
this._scope = options.scope || 'infrastructure';
|
|
26
|
+
this._logger = options.logger || console;
|
|
27
|
+
|
|
28
|
+
// Scope configuration
|
|
29
|
+
this._queueCreationEnabled = this._scope === 'business';
|
|
30
|
+
this._queueCreationFilter = options.queueCreationFilter || null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Handle transient publish error
|
|
35
|
+
* @param {Error} error - TransientPublishError
|
|
36
|
+
* @param {Object} context - { queue, buffer, options }
|
|
37
|
+
*/
|
|
38
|
+
async handleTransientError(error, context) {
|
|
39
|
+
// Transient errors jsou už bufferované v PublishLayer
|
|
40
|
+
// Recovery worker jen loguje a může triggerovat další akce
|
|
41
|
+
this._logger?.warn?.(`[RecoveryWorker] Transient error handled (buffered): ${error.message}`, context);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Handle QueueNotFoundError
|
|
46
|
+
* @param {QueueNotFoundError} error
|
|
47
|
+
* @param {Object} context - { queue, buffer, options }
|
|
48
|
+
*/
|
|
49
|
+
async handleQueueNotFound(error, context) {
|
|
50
|
+
if (!(error instanceof QueueNotFoundError)) {
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Infrastructure queues must exist - cannot create
|
|
55
|
+
if (error.isInfrastructure) {
|
|
56
|
+
this._logger?.error?.(`[RecoveryWorker] Cannot create infrastructure queue '${error.queueName}' - must exist before publishing`);
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Business queues - create if scope allows
|
|
61
|
+
if (this._queueCreationEnabled) {
|
|
62
|
+
// Check filter if provided
|
|
63
|
+
if (this._queueCreationFilter && !this._queueCreationFilter(error.queueName)) {
|
|
64
|
+
this._logger?.warn?.(`[RecoveryWorker] Queue creation filtered out for '${error.queueName}'`);
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
try {
|
|
69
|
+
await this.createQueue(error.queueName, context.options);
|
|
70
|
+
this._logger?.info?.(`[RecoveryWorker] ✓ Created queue '${error.queueName}'`);
|
|
71
|
+
} catch (createErr) {
|
|
72
|
+
this._logger?.error?.(`[RecoveryWorker] Failed to create queue '${error.queueName}': ${createErr.message}`);
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
75
|
+
} else {
|
|
76
|
+
this._logger?.error?.(`[RecoveryWorker] Cannot create queue '${error.queueName}' - scope is '${this._scope}' (queue creation disabled)`);
|
|
77
|
+
throw error;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Create queue (business services only)
|
|
83
|
+
* @private
|
|
84
|
+
*/
|
|
85
|
+
async createQueue(queueName, options = {}) {
|
|
86
|
+
if (!this._queueCreationEnabled) {
|
|
87
|
+
throw new Error(`Queue creation not enabled for scope '${this._scope}'`);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Use queue channel to create queue
|
|
91
|
+
await this._client._ensureQueueChannel();
|
|
92
|
+
|
|
93
|
+
const queueOptions = options.queueOptions || { durable: this._client._config.durable };
|
|
94
|
+
await this._client._queueChannel.assertQueue(queueName, queueOptions);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
module.exports = RecoveryWorker;
|
|
99
|
+
|