@onlineapps/mq-client-core 1.0.36 → 1.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/buffer/InMemoryBuffer.js +118 -0
- package/src/buffer/MessageBuffer.js +107 -0
- package/src/buffer/RedisBuffer.js +57 -0
- package/src/index.js +12 -0
- package/src/layers/PublishLayer.js +242 -0
- package/src/monitoring/PublishMonitor.js +138 -0
- package/src/transports/rabbitmqClient.js +910 -183
- package/src/utils/publishErrors.js +105 -0
- package/src/workers/RecoveryWorker.js +99 -0
|
@@ -8,6 +8,15 @@
|
|
|
8
8
|
|
|
9
9
|
const amqp = require('amqplib');
|
|
10
10
|
const EventEmitter = require('events');
|
|
11
|
+
const {
|
|
12
|
+
TransientPublishError,
|
|
13
|
+
PermanentPublishError,
|
|
14
|
+
QueueNotFoundError,
|
|
15
|
+
classifyPublishError,
|
|
16
|
+
} = require('../utils/publishErrors');
|
|
17
|
+
const PublishLayer = require('../layers/PublishLayer');
|
|
18
|
+
const RecoveryWorker = require('../workers/RecoveryWorker');
|
|
19
|
+
const PublishMonitor = require('../monitoring/PublishMonitor');
|
|
11
20
|
|
|
12
21
|
class RabbitMQClient extends EventEmitter {
|
|
13
22
|
/**
|
|
@@ -39,13 +48,44 @@ class RabbitMQClient extends EventEmitter {
|
|
|
39
48
|
// Track active consumers for re-registration after channel recreation
|
|
40
49
|
this._activeConsumers = new Map(); // queue -> { handler, options, consumerTag }
|
|
41
50
|
|
|
42
|
-
//
|
|
51
|
+
// Connection-level recovery
|
|
43
52
|
this._reconnecting = false;
|
|
44
53
|
this._reconnectAttempts = 0;
|
|
54
|
+
this._maxReconnectAttempts = this._config.maxReconnectAttempts || 10; // Max 10 attempts
|
|
55
|
+
this._reconnectBaseDelay = this._config.reconnectBaseDelay || 1000; // Start with 1 second
|
|
56
|
+
this._reconnectMaxDelay = this._config.reconnectMaxDelay || 30000; // Max 30 seconds
|
|
57
|
+
this._reconnectEnabled = this._config.reconnectEnabled !== false; // Default: enabled
|
|
58
|
+
this._reconnectTimer = null;
|
|
59
|
+
|
|
60
|
+
// Publisher retry configuration
|
|
61
|
+
this._publishRetryEnabled = this._config.publishRetryEnabled !== false; // Default: enabled
|
|
62
|
+
this._publishMaxRetries = this._config.publishMaxRetries || 3; // Default: 3 attempts
|
|
63
|
+
this._publishRetryBaseDelay = this._config.publishRetryBaseDelay || 100; // Default: 100ms
|
|
64
|
+
this._publishRetryMaxDelay = this._config.publishRetryMaxDelay || 5000; // Default: 5s
|
|
65
|
+
this._publishRetryBackoffMultiplier = this._config.publishRetryBackoffMultiplier || 2; // Default: 2 (exponential)
|
|
66
|
+
this._publishConfirmationTimeout = this._config.publishConfirmationTimeout || 5000; // Default: 5s per attempt
|
|
45
67
|
|
|
46
68
|
// Channel close hooks - called when channel closes with detailed information
|
|
47
69
|
this._channelCloseHooks = []; // Array of { type: 'publisher'|'queue'|'consumer', callback: (details) => void }
|
|
48
70
|
|
|
71
|
+
// Channel thrashing detection - track close frequency per channel type
|
|
72
|
+
this._channelCloseHistory = {
|
|
73
|
+
publisher: [], // Array of { timestamp: number, reason: string }
|
|
74
|
+
queue: [],
|
|
75
|
+
consumer: []
|
|
76
|
+
};
|
|
77
|
+
this._thrashingThreshold = this._config.thrashingThreshold || 5; // Max closes per minute
|
|
78
|
+
this._thrashingWindowMs = this._config.thrashingWindowMs || 60000; // 1 minute window
|
|
79
|
+
this._thrashingAlertCallback = this._config.thrashingAlertCallback || null; // (type, count, windowMs) => void
|
|
80
|
+
this._thrashingAlertsSent = new Set(); // Track which alerts we've already sent to avoid spam
|
|
81
|
+
|
|
82
|
+
// Prefetch monitoring - track utilization per queue
|
|
83
|
+
this._prefetchTracking = new Map(); // queue -> { prefetchCount: number, inFlight: number, lastCheck: number }
|
|
84
|
+
this._prefetchUtilizationThreshold = this._config.prefetchUtilizationThreshold || 0.8; // 80% threshold
|
|
85
|
+
this._prefetchCheckInterval = this._config.prefetchCheckInterval || 10000; // Check every 10s
|
|
86
|
+
this._prefetchAlertCallback = this._config.prefetchAlertCallback || null; // (queue, utilization, inFlight, prefetchCount) => void
|
|
87
|
+
this._prefetchCheckTimer = null;
|
|
88
|
+
|
|
49
89
|
// Health monitoring
|
|
50
90
|
this._healthCheckInterval = null;
|
|
51
91
|
this._healthCheckIntervalMs = this._config.healthCheckInterval || 30000; // Default 30s
|
|
@@ -57,6 +97,89 @@ class RabbitMQClient extends EventEmitter {
|
|
|
57
97
|
this._criticalHealthShutdown = this._config.criticalHealthShutdown !== false; // Default: true - shutdown on critical health
|
|
58
98
|
this._criticalHealthShutdownDelay = this._config.criticalHealthShutdownDelay || 60000; // Default: 60s delay before shutdown
|
|
59
99
|
this._criticalHealthStartTime = null; // Track when critical health started
|
|
100
|
+
|
|
101
|
+
// Publish layer (retry + buffer)
|
|
102
|
+
this._publishLayer = new PublishLayer({
|
|
103
|
+
client: this,
|
|
104
|
+
logger: console,
|
|
105
|
+
bufferConfig: {
|
|
106
|
+
inMemory: {
|
|
107
|
+
maxSize: this._config.publishBufferMaxSize || 100,
|
|
108
|
+
ttlMs: this._config.publishBufferTtlMs || 5 * 60 * 1000,
|
|
109
|
+
},
|
|
110
|
+
persistent: {
|
|
111
|
+
enabled: !!this._config.persistentBufferEnabled,
|
|
112
|
+
redisClient: this._config.persistentRedisClient || null,
|
|
113
|
+
},
|
|
114
|
+
},
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
// Recovery worker (connection recovery, queue creation pro business)
|
|
118
|
+
this._recoveryWorker = new RecoveryWorker({
|
|
119
|
+
client: this,
|
|
120
|
+
scope: this._config.recoveryScope || 'infrastructure',
|
|
121
|
+
queueCreationFilter: this._config.queueCreationFilter || null,
|
|
122
|
+
logger: console,
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
// Publish monitor (metriky)
|
|
126
|
+
this._publishMonitor = new PublishMonitor({
|
|
127
|
+
logger: console,
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
// Event listeners pro monitoring
|
|
131
|
+
this._setupPublishMonitoring();
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Setup event listeners pro monitoring a recovery
|
|
136
|
+
* @private
|
|
137
|
+
*/
|
|
138
|
+
_setupPublishMonitoring() {
|
|
139
|
+
// Monitoring events
|
|
140
|
+
this.on('publish:retry', (data) => {
|
|
141
|
+
if (data.attempt === 1) {
|
|
142
|
+
this._publishMonitor.trackAttempt(data.queue);
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
this.on('publish:success', (data) => {
|
|
147
|
+
this._publishMonitor.trackSuccess(data.queue, data.totalAttempts);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
this.on('publish:failed', (data) => {
|
|
151
|
+
this._publishMonitor.trackFailure(data.queue, data.retryable);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
// Track buffered messages
|
|
155
|
+
this.on('publish:buffered', () => {
|
|
156
|
+
this._publishMonitor.trackBuffered();
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
// Recovery worker - handle QueueNotFoundError via error event
|
|
160
|
+
this.on('error', async (err) => {
|
|
161
|
+
if (err instanceof QueueNotFoundError) {
|
|
162
|
+
try {
|
|
163
|
+
await this._recoveryWorker.handleQueueNotFound(err, { queue: err.queueName });
|
|
164
|
+
} catch (recoveryErr) {
|
|
165
|
+
// Recovery failed - error already thrown
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Get publish metrics
|
|
173
|
+
*/
|
|
174
|
+
getPublishMetrics() {
|
|
175
|
+
return this._publishMonitor.getMetrics();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Get Prometheus metrics
|
|
180
|
+
*/
|
|
181
|
+
getPrometheusMetrics() {
|
|
182
|
+
return this._publishMonitor.getPrometheusMetrics();
|
|
60
183
|
}
|
|
61
184
|
|
|
62
185
|
/**
|
|
@@ -91,6 +214,76 @@ class RabbitMQClient extends EventEmitter {
|
|
|
91
214
|
this._channelCloseHooks.push({ type, callback });
|
|
92
215
|
}
|
|
93
216
|
|
|
217
|
+
/**
|
|
218
|
+
* Track channel close event for thrashing detection
|
|
219
|
+
* @private
|
|
220
|
+
* @param {string} channelType - 'publisher', 'queue', or 'consumer'
|
|
221
|
+
* @param {string} reason - Human-readable reason
|
|
222
|
+
*/
|
|
223
|
+
_trackChannelClose(channelType, reason) {
|
|
224
|
+
const now = Date.now();
|
|
225
|
+
const history = this._channelCloseHistory[channelType];
|
|
226
|
+
|
|
227
|
+
if (!history) {
|
|
228
|
+
console.warn(`[RabbitMQClient] Unknown channel type for thrashing tracking: ${channelType}`);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// Add new close event
|
|
233
|
+
history.push({
|
|
234
|
+
timestamp: now,
|
|
235
|
+
reason: reason || 'unknown'
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
// Remove old events outside the window
|
|
239
|
+
const cutoff = now - this._thrashingWindowMs;
|
|
240
|
+
while (history.length > 0 && history[0].timestamp < cutoff) {
|
|
241
|
+
history.shift();
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Check for thrashing
|
|
245
|
+
if (history.length >= this._thrashingThreshold) {
|
|
246
|
+
const alertKey = `${channelType}:${Math.floor(now / this._thrashingWindowMs)}`;
|
|
247
|
+
|
|
248
|
+
// Only alert once per window to avoid spam
|
|
249
|
+
if (!this._thrashingAlertsSent.has(alertKey)) {
|
|
250
|
+
this._thrashingAlertsSent.add(alertKey);
|
|
251
|
+
|
|
252
|
+
// Clean up old alert keys (keep only last 10 windows)
|
|
253
|
+
if (this._thrashingAlertsSent.size > 10) {
|
|
254
|
+
const oldestKey = Array.from(this._thrashingAlertsSent).sort()[0];
|
|
255
|
+
this._thrashingAlertsSent.delete(oldestKey);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const message = `Channel thrashing detected: ${channelType} channel closed ${history.length} times in ${Math.round(this._thrashingWindowMs / 1000)}s (threshold: ${this._thrashingThreshold})`;
|
|
259
|
+
console.error(`[RabbitMQClient] [mq-client-core] 🚨 ${message}`);
|
|
260
|
+
|
|
261
|
+
// Call alert callback if provided
|
|
262
|
+
if (this._thrashingAlertCallback) {
|
|
263
|
+
try {
|
|
264
|
+
this._thrashingAlertCallback(channelType, history.length, this._thrashingWindowMs, {
|
|
265
|
+
recentCloses: history.slice(-this._thrashingThreshold),
|
|
266
|
+
threshold: this._thrashingThreshold,
|
|
267
|
+
windowMs: this._thrashingWindowMs
|
|
268
|
+
});
|
|
269
|
+
} catch (alertErr) {
|
|
270
|
+
console.error(`[RabbitMQClient] [mq-client-core] Thrashing alert callback error:`, alertErr.message);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// Emit event for external listeners
|
|
275
|
+
this.emit('channel:thrashing', {
|
|
276
|
+
type: channelType,
|
|
277
|
+
count: history.length,
|
|
278
|
+
windowMs: this._thrashingWindowMs,
|
|
279
|
+
threshold: this._thrashingThreshold,
|
|
280
|
+
recentCloses: history.slice(-this._thrashingThreshold),
|
|
281
|
+
timestamp: new Date().toISOString()
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
94
287
|
/**
|
|
95
288
|
* Call all registered channel close hooks
|
|
96
289
|
* @private
|
|
@@ -100,6 +293,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
100
293
|
* @param {string} reason - Human-readable reason
|
|
101
294
|
*/
|
|
102
295
|
_callChannelCloseHooks(channelType, channel, error, reason) {
|
|
296
|
+
// Track channel close for thrashing detection
|
|
297
|
+
this._trackChannelClose(channelType, reason);
|
|
298
|
+
|
|
103
299
|
const details = {
|
|
104
300
|
type: channelType,
|
|
105
301
|
channel: channel,
|
|
@@ -152,10 +348,40 @@ class RabbitMQClient extends EventEmitter {
|
|
|
152
348
|
console.log('[RabbitMQClient] Starting connection race...');
|
|
153
349
|
this._connection = await Promise.race([connectPromise, timeoutPromise]);
|
|
154
350
|
console.log('[RabbitMQClient] Connection established');
|
|
155
|
-
this.
|
|
351
|
+
this._reconnectAttempts = 0; // Reset reconnect attempts on successful connection
|
|
352
|
+
|
|
353
|
+
this._connection.on('error', (err) => {
|
|
354
|
+
console.error('[RabbitMQClient] Connection error:', err.message);
|
|
355
|
+
this.emit('error', err);
|
|
356
|
+
// Connection errors may lead to close event - let close handler handle reconnection
|
|
357
|
+
});
|
|
358
|
+
|
|
156
359
|
this._connection.on('close', () => {
|
|
157
|
-
|
|
360
|
+
console.warn('[RabbitMQClient] Connection closed unexpectedly - close handler STARTED');
|
|
361
|
+
console.log(`[RabbitMQClient] Close handler: reconnectEnabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
|
|
362
|
+
|
|
363
|
+
// Mark all channels as closed
|
|
364
|
+
this._channel = null;
|
|
365
|
+
this._queueChannel = null;
|
|
366
|
+
this._consumerChannel = null;
|
|
367
|
+
|
|
368
|
+
// CONNECTION-LEVEL RECOVERY: Attempt to reconnect with exponential backoff
|
|
369
|
+
// PREDICTABLE: Check conditions and log why reconnect is/isn't starting
|
|
370
|
+
// IMPORTANT: Do this BEFORE emitting error, so reconnect can start even if error handler throws
|
|
371
|
+
if (this._reconnectEnabled && !this._reconnecting) {
|
|
372
|
+
console.log('[RabbitMQClient] ✓ Conditions met - starting connection-level recovery from initial connect() close handler');
|
|
373
|
+
this._reconnectWithBackoff().catch(err => {
|
|
374
|
+
console.error('[RabbitMQClient] Reconnection failed after all attempts:', err.message);
|
|
375
|
+
this.emit('error', err);
|
|
376
|
+
});
|
|
377
|
+
} else {
|
|
378
|
+
console.warn(`[RabbitMQClient] ✗ Reconnect NOT starting: enabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// Emit connection close event AFTER starting reconnect (so reconnect isn't blocked by error handler)
|
|
158
382
|
this.emit('error', new Error('RabbitMQ connection closed unexpectedly'));
|
|
383
|
+
|
|
384
|
+
console.log('[RabbitMQClient] Close handler FINISHED');
|
|
159
385
|
});
|
|
160
386
|
|
|
161
387
|
// Use ConfirmChannel to enable publisher confirms for publish operations
|
|
@@ -167,37 +393,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
167
393
|
this._channel._lastOperation = 'Initial creation';
|
|
168
394
|
this._channel._type = 'publisher';
|
|
169
395
|
|
|
170
|
-
//
|
|
171
|
-
this._channel
|
|
172
|
-
const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
173
|
-
this._channel._closeReason = reason;
|
|
174
|
-
console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
175
|
-
|
|
176
|
-
// Call close hooks with detailed information
|
|
177
|
-
this._callChannelCloseHooks('publisher', this._channel, err, reason);
|
|
178
|
-
|
|
179
|
-
// Auto-recreate channel if connection is still alive
|
|
180
|
-
await this._ensurePublisherChannel();
|
|
181
|
-
this.emit('error', err);
|
|
182
|
-
});
|
|
183
|
-
this._channel.on('close', async () => {
|
|
184
|
-
const reason = this._channel._closeReason || 'Publisher channel closed unexpectedly';
|
|
185
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
|
|
186
|
-
|
|
187
|
-
// Call close hooks with detailed information
|
|
188
|
-
this._callChannelCloseHooks('publisher', this._channel, null, reason);
|
|
189
|
-
|
|
190
|
-
// Mark channel as null - will be recreated on next publish
|
|
191
|
-
const closedChannel = this._channel;
|
|
192
|
-
this._channel = null;
|
|
193
|
-
|
|
194
|
-
// Try to recreate immediately if connection is alive
|
|
195
|
-
await this._ensurePublisherChannel();
|
|
196
|
-
});
|
|
197
|
-
// Set up publisher confirms callback
|
|
198
|
-
this._channel.on('drain', () => {
|
|
199
|
-
console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
|
|
200
|
-
});
|
|
396
|
+
// Attach event handlers
|
|
397
|
+
this._attachPublisherChannelHandlers(this._channel);
|
|
201
398
|
|
|
202
399
|
// Create a separate regular channel for queue operations (assertQueue, checkQueue)
|
|
203
400
|
// This avoids RPC reply queue issues with ConfirmChannel.assertQueue()
|
|
@@ -209,31 +406,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
209
406
|
this._queueChannel._lastOperation = 'Initial creation';
|
|
210
407
|
this._queueChannel._type = 'queue';
|
|
211
408
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
this._queueChannel._closeReason = reason;
|
|
215
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
216
|
-
|
|
217
|
-
// Call close hooks with detailed information
|
|
218
|
-
this._callChannelCloseHooks('queue', this._queueChannel, err, reason);
|
|
219
|
-
|
|
220
|
-
// Auto-recreate channel if connection is still alive
|
|
221
|
-
await this._ensureQueueChannel();
|
|
222
|
-
});
|
|
223
|
-
this._queueChannel.on('close', async () => {
|
|
224
|
-
const reason = this._queueChannel._closeReason || 'Queue channel closed unexpectedly';
|
|
225
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
|
|
226
|
-
|
|
227
|
-
// Call close hooks with detailed information
|
|
228
|
-
this._callChannelCloseHooks('queue', this._queueChannel, null, reason);
|
|
229
|
-
|
|
230
|
-
// Mark channel as null - will be recreated on next queue operation
|
|
231
|
-
const closedChannel = this._queueChannel;
|
|
232
|
-
this._queueChannel = null;
|
|
233
|
-
|
|
234
|
-
// Try to recreate immediately if connection is alive
|
|
235
|
-
await this._ensureQueueChannel();
|
|
236
|
-
});
|
|
409
|
+
// Attach event handlers
|
|
410
|
+
this._attachQueueChannelHandlers(this._queueChannel);
|
|
237
411
|
|
|
238
412
|
// Create a dedicated channel for consume operations
|
|
239
413
|
// This prevents channel conflicts between publish (ConfirmChannel) and consume operations
|
|
@@ -245,32 +419,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
245
419
|
this._consumerChannel._lastOperation = 'Initial creation';
|
|
246
420
|
this._consumerChannel._type = 'consumer';
|
|
247
421
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
this._consumerChannel._closeReason = reason;
|
|
251
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
252
|
-
|
|
253
|
-
// Call close hooks with detailed information
|
|
254
|
-
this._callChannelCloseHooks('consumer', this._consumerChannel, err, reason);
|
|
255
|
-
|
|
256
|
-
// Auto-recreate channel and re-register consumers
|
|
257
|
-
await this._ensureConsumerChannel();
|
|
258
|
-
this.emit('error', err);
|
|
259
|
-
});
|
|
260
|
-
this._consumerChannel.on('close', async () => {
|
|
261
|
-
const reason = this._consumerChannel._closeReason || 'Consumer channel closed unexpectedly';
|
|
262
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
|
|
263
|
-
|
|
264
|
-
// Call close hooks with detailed information
|
|
265
|
-
this._callChannelCloseHooks('consumer', this._consumerChannel, null, reason);
|
|
266
|
-
|
|
267
|
-
// Mark channel as null - will be recreated and consumers re-registered
|
|
268
|
-
const closedChannel = this._consumerChannel;
|
|
269
|
-
this._consumerChannel = null;
|
|
270
|
-
|
|
271
|
-
// Try to recreate and re-register consumers immediately if connection is alive
|
|
272
|
-
await this._ensureConsumerChannel();
|
|
273
|
-
});
|
|
422
|
+
// Attach event handlers
|
|
423
|
+
this._attachConsumerChannelHandlers(this._consumerChannel);
|
|
274
424
|
|
|
275
425
|
// Start health monitoring if enabled
|
|
276
426
|
if (this._healthCheckEnabled) {
|
|
@@ -290,6 +440,50 @@ class RabbitMQClient extends EventEmitter {
|
|
|
290
440
|
}
|
|
291
441
|
}
|
|
292
442
|
|
|
443
|
+
/**
|
|
444
|
+
* Wait for reconnection to complete
|
|
445
|
+
* Used by both retry mechanism and channel creation
|
|
446
|
+
* @private
|
|
447
|
+
* @returns {Promise<void>}
|
|
448
|
+
* @throws {Error} If reconnection fails or times out
|
|
449
|
+
*/
|
|
450
|
+
async _waitForReconnection() {
|
|
451
|
+
if (!this._reconnecting) {
|
|
452
|
+
throw new Error('Cannot wait for reconnection: not currently reconnecting');
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// Wait for 'reconnected' event
|
|
456
|
+
return new Promise((resolve, reject) => {
|
|
457
|
+
const timeout = setTimeout(() => {
|
|
458
|
+
this.removeListener('reconnected', onReconnected);
|
|
459
|
+
this.removeListener('error', onError);
|
|
460
|
+
reject(new Error('Reconnection timeout after 30 seconds'));
|
|
461
|
+
}, 30000); // 30 seconds timeout
|
|
462
|
+
|
|
463
|
+
const onReconnected = () => {
|
|
464
|
+
clearTimeout(timeout);
|
|
465
|
+
this.removeListener('reconnected', onReconnected);
|
|
466
|
+
this.removeListener('error', onError);
|
|
467
|
+
// Wait a bit for channels to be fully recreated
|
|
468
|
+
setTimeout(resolve, 500);
|
|
469
|
+
};
|
|
470
|
+
|
|
471
|
+
const onError = (error) => {
|
|
472
|
+
// Ignore connection errors during reconnection
|
|
473
|
+
if (error.message && error.message.includes('Connection closed unexpectedly')) {
|
|
474
|
+
return; // Expected during reconnection
|
|
475
|
+
}
|
|
476
|
+
clearTimeout(timeout);
|
|
477
|
+
this.removeListener('reconnected', onReconnected);
|
|
478
|
+
this.removeListener('error', onError);
|
|
479
|
+
reject(error);
|
|
480
|
+
};
|
|
481
|
+
|
|
482
|
+
this.once('reconnected', onReconnected);
|
|
483
|
+
this.on('error', onError);
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
|
|
293
487
|
/**
|
|
294
488
|
* Ensure publisher channel exists and is open
|
|
295
489
|
* Auto-recreates if closed
|
|
@@ -302,7 +496,25 @@ class RabbitMQClient extends EventEmitter {
|
|
|
302
496
|
}
|
|
303
497
|
|
|
304
498
|
if (!this._connection || this._connection.closed) {
|
|
305
|
-
|
|
499
|
+
// Connection is closed - wait for reconnection
|
|
500
|
+
console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate publisher channel: connection is closed (waiting for reconnection)');
|
|
501
|
+
|
|
502
|
+
// Wait for reconnection to complete
|
|
503
|
+
if (this._reconnecting) {
|
|
504
|
+
await this._waitForReconnection();
|
|
505
|
+
|
|
506
|
+
// After reconnection, try again
|
|
507
|
+
if (this._channel && !this._channel.closed) {
|
|
508
|
+
return; // Channel was recreated during reconnection
|
|
509
|
+
}
|
|
510
|
+
// If still no channel, connection might have failed - check again
|
|
511
|
+
if (!this._connection || this._connection.closed) {
|
|
512
|
+
throw new Error('Connection reconnection failed - cannot create publisher channel');
|
|
513
|
+
}
|
|
514
|
+
} else {
|
|
515
|
+
// Not reconnecting - connection is closed and not recovering
|
|
516
|
+
throw new Error('Connection is closed and not reconnecting - cannot create publisher channel');
|
|
517
|
+
}
|
|
306
518
|
}
|
|
307
519
|
|
|
308
520
|
try {
|
|
@@ -324,36 +536,29 @@ class RabbitMQClient extends EventEmitter {
|
|
|
324
536
|
this._channel._lastOperation = 'Recreated via _ensurePublisherChannel()';
|
|
325
537
|
this._channel._type = 'publisher';
|
|
326
538
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
this._channel._closeReason = reason;
|
|
330
|
-
console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
331
|
-
|
|
332
|
-
// Call close hooks
|
|
333
|
-
this._callChannelCloseHooks('publisher', this._channel, err, reason);
|
|
334
|
-
|
|
335
|
-
await this._ensurePublisherChannel();
|
|
336
|
-
this.emit('error', err);
|
|
337
|
-
});
|
|
338
|
-
this._channel.on('close', async () => {
|
|
339
|
-
const reason = this._channel._closeReason || 'Publisher channel closed unexpectedly';
|
|
340
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
|
|
341
|
-
|
|
342
|
-
// Call close hooks
|
|
343
|
-
this._callChannelCloseHooks('publisher', this._channel, null, reason);
|
|
344
|
-
|
|
345
|
-
const closedChannel = this._channel;
|
|
346
|
-
this._channel = null;
|
|
347
|
-
await this._ensurePublisherChannel();
|
|
348
|
-
});
|
|
349
|
-
this._channel.on('drain', () => {
|
|
350
|
-
console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
|
|
351
|
-
});
|
|
539
|
+
// Attach event handlers
|
|
540
|
+
this._attachPublisherChannelHandlers(this._channel);
|
|
352
541
|
|
|
353
542
|
console.log('[RabbitMQClient] [mq-client-core] ✓ Publisher channel recreated');
|
|
354
543
|
} catch (err) {
|
|
355
544
|
this._channel = null;
|
|
356
|
-
|
|
545
|
+
|
|
546
|
+
const msg = err && err.message ? err.message : String(err);
|
|
547
|
+
|
|
548
|
+
// Pokud se connection právě zavírá, považujeme to za přechodný stav – explicitně vyhodíme
|
|
549
|
+
// TransientPublishError, aby vyšší vrstva (retry / worker) věděla, že má počkat na reconnect.
|
|
550
|
+
if (msg.includes('Connection closing') || msg.includes('Connection closed')) {
|
|
551
|
+
console.warn(
|
|
552
|
+
`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection closing): ${msg}`
|
|
553
|
+
);
|
|
554
|
+
throw new TransientPublishError(
|
|
555
|
+
'Publisher channel cannot be recreated because connection is closing/closed (transient condition)',
|
|
556
|
+
err
|
|
557
|
+
);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// Ostatní chyby považujeme za permanentní problém na úrovni kanálu/spojení.
|
|
561
|
+
throw new PermanentPublishError(`Failed to recreate publisher channel: ${msg}`, err);
|
|
357
562
|
}
|
|
358
563
|
}
|
|
359
564
|
|
|
@@ -369,7 +574,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
369
574
|
}
|
|
370
575
|
|
|
371
576
|
if (!this._connection || this._connection.closed) {
|
|
372
|
-
|
|
577
|
+
// Connection is closed - cannot recreate channel
|
|
578
|
+
// Reconnection logic will handle this - don't throw, just return
|
|
579
|
+
// The channel will be recreated after reconnection completes
|
|
580
|
+
console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate queue channel: connection is closed (reconnection in progress)');
|
|
581
|
+
return;
|
|
373
582
|
}
|
|
374
583
|
|
|
375
584
|
try {
|
|
@@ -391,32 +600,18 @@ class RabbitMQClient extends EventEmitter {
|
|
|
391
600
|
this._queueChannel._lastOperation = 'Recreated via _ensureQueueChannel()';
|
|
392
601
|
this._queueChannel._type = 'queue';
|
|
393
602
|
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
this._queueChannel._closeReason = reason;
|
|
397
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
398
|
-
|
|
399
|
-
// Call close hooks
|
|
400
|
-
this._callChannelCloseHooks('queue', this._queueChannel, err, reason);
|
|
401
|
-
|
|
402
|
-
await this._ensureQueueChannel();
|
|
403
|
-
});
|
|
404
|
-
this._queueChannel.on('close', async () => {
|
|
405
|
-
const reason = this._queueChannel._closeReason || 'Queue channel closed unexpectedly';
|
|
406
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
|
|
407
|
-
|
|
408
|
-
// Call close hooks
|
|
409
|
-
this._callChannelCloseHooks('queue', this._queueChannel, null, reason);
|
|
410
|
-
|
|
411
|
-
const closedChannel = this._queueChannel;
|
|
412
|
-
this._queueChannel = null;
|
|
413
|
-
await this._ensureQueueChannel();
|
|
414
|
-
});
|
|
603
|
+
// Attach event handlers
|
|
604
|
+
this._attachQueueChannelHandlers(this._queueChannel);
|
|
415
605
|
|
|
416
606
|
console.log('[RabbitMQClient] [mq-client-core] ✓ Queue channel recreated');
|
|
417
607
|
} catch (err) {
|
|
418
608
|
this._queueChannel = null;
|
|
419
|
-
|
|
609
|
+
// If connection is closed, don't throw - reconnection will handle it
|
|
610
|
+
if (this._connection && !this._connection.closed) {
|
|
611
|
+
throw new Error(`Failed to recreate queue channel: ${err.message}`);
|
|
612
|
+
}
|
|
613
|
+
// Connection is closed - reconnection will recreate channel
|
|
614
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel (connection closed): ${err.message}`);
|
|
420
615
|
}
|
|
421
616
|
}
|
|
422
617
|
|
|
@@ -432,7 +627,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
432
627
|
}
|
|
433
628
|
|
|
434
629
|
if (!this._connection || this._connection.closed) {
|
|
435
|
-
|
|
630
|
+
// Connection is closed - cannot recreate channel
|
|
631
|
+
// Reconnection logic will handle this - don't throw, just return
|
|
632
|
+
// The channel will be recreated after reconnection completes
|
|
633
|
+
console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate consumer channel: connection is closed (reconnection in progress)');
|
|
634
|
+
return;
|
|
436
635
|
}
|
|
437
636
|
|
|
438
637
|
try {
|
|
@@ -454,28 +653,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
454
653
|
this._consumerChannel._lastOperation = 'Recreated via _ensureConsumerChannel()';
|
|
455
654
|
this._consumerChannel._type = 'consumer';
|
|
456
655
|
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
this._consumerChannel._closeReason = reason;
|
|
460
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
461
|
-
|
|
462
|
-
// Call close hooks
|
|
463
|
-
this._callChannelCloseHooks('consumer', this._consumerChannel, err, reason);
|
|
464
|
-
|
|
465
|
-
await this._ensureConsumerChannel();
|
|
466
|
-
this.emit('error', err);
|
|
467
|
-
});
|
|
468
|
-
this._consumerChannel.on('close', async () => {
|
|
469
|
-
const reason = this._consumerChannel._closeReason || 'Consumer channel closed unexpectedly';
|
|
470
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
|
|
471
|
-
|
|
472
|
-
// Call close hooks
|
|
473
|
-
this._callChannelCloseHooks('consumer', this._consumerChannel, null, reason);
|
|
474
|
-
|
|
475
|
-
const closedChannel = this._consumerChannel;
|
|
476
|
-
this._consumerChannel = null;
|
|
477
|
-
await this._ensureConsumerChannel();
|
|
478
|
-
});
|
|
656
|
+
// Attach event handlers
|
|
657
|
+
this._attachConsumerChannelHandlers(this._consumerChannel);
|
|
479
658
|
|
|
480
659
|
// Re-register all active consumers
|
|
481
660
|
if (this._activeConsumers.size > 0) {
|
|
@@ -737,9 +916,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
737
916
|
// Stop health monitoring
|
|
738
917
|
this._stopHealthMonitoring();
|
|
739
918
|
|
|
919
|
+
// Stop prefetch monitoring
|
|
920
|
+
this._stopPrefetchMonitoring();
|
|
921
|
+
|
|
740
922
|
// Clear active consumers tracking
|
|
741
923
|
this._activeConsumers.clear();
|
|
742
924
|
|
|
925
|
+
// Clear prefetch tracking
|
|
926
|
+
this._prefetchTracking.clear();
|
|
927
|
+
|
|
743
928
|
try {
|
|
744
929
|
if (this._consumerChannel) {
|
|
745
930
|
await this._consumerChannel.close();
|
|
@@ -771,19 +956,36 @@ class RabbitMQClient extends EventEmitter {
|
|
|
771
956
|
this._connection = null;
|
|
772
957
|
}
|
|
773
958
|
} catch (err) {
|
|
774
|
-
|
|
959
|
+
// Don't emit error for expected connection closure during disconnect
|
|
960
|
+
// This is a predictable, intentional closure
|
|
961
|
+
if (err.message && err.message.includes('Connection closed (by client)')) {
|
|
962
|
+
console.log('[RabbitMQClient] Connection closed during disconnect (expected)');
|
|
963
|
+
} else {
|
|
964
|
+
this.emit('error', err);
|
|
965
|
+
}
|
|
775
966
|
}
|
|
776
967
|
}
|
|
777
968
|
|
|
778
969
|
/**
|
|
779
970
|
* Publishes a message buffer to the specified queue (or default queue) or exchange.
|
|
971
|
+
* Implements systematic retry with exponential backoff for transient failures.
|
|
780
972
|
* @param {string} queue - Target queue name.
|
|
781
973
|
* @param {Buffer} buffer - Message payload as Buffer.
|
|
782
974
|
* @param {Object} [options] - Overrides: routingKey, persistent, headers.
|
|
783
975
|
* @returns {Promise<void>}
|
|
784
|
-
* @throws {Error} If publish fails
|
|
976
|
+
* @throws {Error} If publish fails after all retry attempts.
|
|
785
977
|
*/
|
|
786
978
|
async publish(queue, buffer, options = {}) {
|
|
979
|
+
// PublishLayer zachovává stávající retry logiku a přidává možnost bufferování a
|
|
980
|
+
// jasnější klasifikaci chyb. API zůstává stejné.
|
|
981
|
+
return await this._publishLayer.publish(queue, buffer, options);
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
/**
|
|
985
|
+
* Internal publish implementation - single attempt without retry
|
|
986
|
+
* @private
|
|
987
|
+
*/
|
|
988
|
+
async _publishOnce(queue, buffer, options = {}) {
|
|
787
989
|
// Ensure publisher channel exists and is open (auto-recreates if closed)
|
|
788
990
|
await this._ensurePublisherChannel();
|
|
789
991
|
|
|
@@ -812,9 +1014,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
812
1014
|
await this._queueChannel.checkQueue(queue);
|
|
813
1015
|
// Queue exists - proceed to publish
|
|
814
1016
|
} catch (checkErr) {
|
|
815
|
-
// If queue doesn't exist (404), this
|
|
816
|
-
// Infrastructure queues (workflow.*, registry.*, infrastructure.*, monitoring.*, validation.*)
|
|
817
|
-
//
|
|
1017
|
+
// If queue doesn't exist (404), this je ERROR – ale rozlišujeme infra vs. non-infra:
|
|
1018
|
+
// - Infrastructure queues (workflow.*, registry.*, infrastructure.*, monitoring.*, validation.*)
|
|
1019
|
+
// musí být vytvořené infra službami předem → QueueNotFoundError(isInfrastructure=true)
|
|
1020
|
+
// - Non-infrastructure queues můžeme (pro business scénáře) vytvořit s default parametry
|
|
818
1021
|
if (checkErr.code === 404) {
|
|
819
1022
|
// Check if this is an infrastructure queue using queueConfig
|
|
820
1023
|
let isInfraQueue = false;
|
|
@@ -831,7 +1034,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
831
1034
|
}
|
|
832
1035
|
|
|
833
1036
|
if (isInfraQueue) {
|
|
834
|
-
|
|
1037
|
+
// Jasný, hlasitý signál pro infra služby – fronta chybí, je to programátorská chyba
|
|
1038
|
+
throw new QueueNotFoundError(queue, true, checkErr);
|
|
835
1039
|
}
|
|
836
1040
|
// For non-infrastructure queues, allow auto-creation with default options
|
|
837
1041
|
const queueOptions = options.queueOptions || { durable: this._config.durable };
|
|
@@ -849,16 +1053,103 @@ class RabbitMQClient extends EventEmitter {
|
|
|
849
1053
|
|
|
850
1054
|
// Use callback-based confirmation - kanály jsou spolehlivé, takže callback vždy dorazí
|
|
851
1055
|
const confirmPromise = new Promise((resolve, reject) => {
|
|
1056
|
+
// Set timeout for publish confirmation (configurable)
|
|
1057
|
+
const timeout = setTimeout(() => {
|
|
1058
|
+
reject(new Error(`Publish confirmation timeout for queue "${queue}" after ${this._publishConfirmationTimeout}ms`));
|
|
1059
|
+
}, this._publishConfirmationTimeout);
|
|
1060
|
+
|
|
1061
|
+
// Check if channel is still valid before sending
|
|
1062
|
+
if (!this._channel || this._channel.closed) {
|
|
1063
|
+
clearTimeout(timeout);
|
|
1064
|
+
reject(new Error(`Cannot publish: channel is closed for queue "${queue}"`));
|
|
1065
|
+
return;
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
// Track original channel to detect if it was closed/recreated during publish
|
|
1069
|
+
const originalChannel = this._channel;
|
|
1070
|
+
const channelId = originalChannel ? originalChannel._createdAt : null;
|
|
1071
|
+
let callbackInvoked = false;
|
|
1072
|
+
|
|
852
1073
|
// Send message with callback
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
1074
|
+
try {
|
|
1075
|
+
originalChannel.sendToQueue(queue, buffer, { persistent, headers }, (err, ok) => {
|
|
1076
|
+
callbackInvoked = true;
|
|
1077
|
+
clearTimeout(timeout);
|
|
1078
|
+
|
|
1079
|
+
// CRITICAL: Check if channel was closed or recreated during publish
|
|
1080
|
+
// If channel was recreated, the delivery tag is invalid - ignore this callback
|
|
1081
|
+
if (!this._channel || this._channel.closed || this._channel !== originalChannel ||
|
|
1082
|
+
(this._channel._createdAt && this._channel._createdAt !== channelId)) {
|
|
1083
|
+
// Channel was closed or recreated - delivery tag is invalid
|
|
1084
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed/recreated during publish to queue "${queue}", ignoring callback`);
|
|
1085
|
+
// If we're reconnecting, wait and retry
|
|
1086
|
+
if (this._reconnecting) {
|
|
1087
|
+
this._waitForReconnection().then(() => {
|
|
1088
|
+
// Retry publish after reconnection
|
|
1089
|
+
return this.publish(queue, buffer, options);
|
|
1090
|
+
}).then(resolve).catch(reject);
|
|
1091
|
+
} else {
|
|
1092
|
+
// Not reconnecting - this is a real error
|
|
1093
|
+
// We cannot guarantee the message was delivered - must fail
|
|
1094
|
+
reject(new Error(`Channel closed during publish to queue "${queue}" and not reconnecting - message delivery not guaranteed`));
|
|
1095
|
+
}
|
|
1096
|
+
return;
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
if (err) {
|
|
1100
|
+
// Check if error is about invalid delivery tag (channel was closed)
|
|
1101
|
+
if (err.message && (err.message.includes('unknown delivery tag') || err.message.includes('PRECONDITION_FAILED'))) {
|
|
1102
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Delivery tag invalid (channel may have been closed) for queue "${queue}"`);
|
|
1103
|
+
// If we're reconnecting, wait and retry
|
|
1104
|
+
if (this._reconnecting) {
|
|
1105
|
+
this._waitForReconnection().then(() => {
|
|
1106
|
+
// Retry publish after reconnection
|
|
1107
|
+
return this.publish(queue, buffer, options);
|
|
1108
|
+
}).then(resolve).catch(reject);
|
|
1109
|
+
} else {
|
|
1110
|
+
// Not reconnecting - delivery tag is invalid, message delivery not guaranteed
|
|
1111
|
+
reject(new Error(`Delivery tag invalid for queue "${queue}" (channel closed) and not reconnecting - message delivery not guaranteed`));
|
|
1112
|
+
}
|
|
1113
|
+
} else {
|
|
1114
|
+
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
|
|
1115
|
+
reject(err);
|
|
1116
|
+
}
|
|
1117
|
+
} else {
|
|
1118
|
+
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Message confirmed for queue "${queue}"`);
|
|
1119
|
+
resolve();
|
|
1120
|
+
}
|
|
1121
|
+
});
|
|
1122
|
+
|
|
1123
|
+
// Set a safety timeout - if callback wasn't invoked and channel closed, retry
|
|
1124
|
+
setTimeout(() => {
|
|
1125
|
+
if (!callbackInvoked && (!this._channel || this._channel.closed || this._channel !== originalChannel)) {
|
|
1126
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Callback timeout and channel closed for queue "${queue}", will retry after reconnection`);
|
|
1127
|
+
if (this._reconnecting) {
|
|
1128
|
+
clearTimeout(timeout);
|
|
1129
|
+
this._waitForReconnection().then(() => {
|
|
1130
|
+
return this.publish(queue, buffer, options);
|
|
1131
|
+
}).then(resolve).catch(reject);
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
}, 1000);
|
|
1135
|
+
} catch (sendErr) {
|
|
1136
|
+
clearTimeout(timeout);
|
|
1137
|
+
// If channel is closed, try to wait for reconnection and retry
|
|
1138
|
+
if (sendErr.message && (sendErr.message.includes('Channel ended') || sendErr.message.includes('Channel closed'))) {
|
|
1139
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during sendToQueue, will retry after reconnection`);
|
|
1140
|
+
// Wait for reconnection and retry
|
|
1141
|
+
if (this._reconnecting) {
|
|
1142
|
+
this._waitForReconnection().then(() => {
|
|
1143
|
+
// Retry publish after reconnection
|
|
1144
|
+
return this.publish(queue, buffer, options);
|
|
1145
|
+
}).then(resolve).catch(reject);
|
|
1146
|
+
} else {
|
|
1147
|
+
reject(sendErr);
|
|
1148
|
+
}
|
|
857
1149
|
} else {
|
|
858
|
-
|
|
859
|
-
resolve();
|
|
1150
|
+
reject(sendErr);
|
|
860
1151
|
}
|
|
861
|
-
}
|
|
1152
|
+
}
|
|
862
1153
|
});
|
|
863
1154
|
|
|
864
1155
|
await confirmPromise;
|
|
@@ -869,7 +1160,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
869
1160
|
|
|
870
1161
|
// Use callback-based confirmation - kanály jsou spolehlivé
|
|
871
1162
|
const confirmPromise = new Promise((resolve, reject) => {
|
|
1163
|
+
// Set timeout for exchange publish confirmation
|
|
1164
|
+
const timeout = setTimeout(() => {
|
|
1165
|
+
reject(new Error(`Exchange publish confirmation timeout for exchange "${exchange}" after ${this._publishConfirmationTimeout}ms`));
|
|
1166
|
+
}, this._publishConfirmationTimeout);
|
|
1167
|
+
|
|
872
1168
|
this._channel.publish(exchange, routingKey, buffer, { persistent, headers }, (err, ok) => {
|
|
1169
|
+
clearTimeout(timeout);
|
|
873
1170
|
if (err) {
|
|
874
1171
|
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Exchange publish callback error:`, err.message);
|
|
875
1172
|
reject(err);
|
|
@@ -883,23 +1180,30 @@ class RabbitMQClient extends EventEmitter {
|
|
|
883
1180
|
await confirmPromise;
|
|
884
1181
|
}
|
|
885
1182
|
} catch (err) {
|
|
886
|
-
//
|
|
887
|
-
|
|
888
|
-
|
|
1183
|
+
// Pokud je kanál zavřený, stále se pokusíme o jedno interní obnovení a retry,
|
|
1184
|
+
// aby byl publish co nejodolnější i předtím, než vstoupí do vyšší retry/recovery vrstvy.
|
|
1185
|
+
if (err && err.message && (err.message.includes('Channel closed') || err.message.includes('channel is closed') || this._channel?.closed)) {
|
|
1186
|
+
console.warn('[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during publish, recreating and retrying once...');
|
|
889
1187
|
try {
|
|
890
1188
|
await this._ensurePublisherChannel();
|
|
891
|
-
// Retry publish once
|
|
1189
|
+
// Retry publish once (může znovu vyhodit – už dojde do retry/recovery vrstvy)
|
|
892
1190
|
return await this.publish(queue, buffer, options);
|
|
893
1191
|
} catch (retryErr) {
|
|
894
|
-
console.error('[RabbitMQClient] [mq-client-core] [PUBLISH] Retry failed:', retryErr.message);
|
|
895
|
-
|
|
1192
|
+
console.error('[RabbitMQClient] [mq-client-core] [PUBLISH] Retry after channel recreation failed:', retryErr.message);
|
|
1193
|
+
const classifiedRetryErr = classifyPublishError(retryErr);
|
|
1194
|
+
this.emit('error', classifiedRetryErr);
|
|
1195
|
+
throw classifiedRetryErr;
|
|
896
1196
|
}
|
|
897
1197
|
}
|
|
898
|
-
|
|
899
|
-
|
|
1198
|
+
|
|
1199
|
+
// Všechny ostatní chyby projdou jednotnou klasifikací do Transient/Permanent/QueueNotFound.
|
|
1200
|
+
const classifiedErr = classifyPublishError(err);
|
|
1201
|
+
this.emit('error', classifiedErr);
|
|
1202
|
+
throw classifiedErr;
|
|
900
1203
|
}
|
|
901
1204
|
}
|
|
902
1205
|
|
|
1206
|
+
|
|
903
1207
|
/**
|
|
904
1208
|
* Starts consuming messages from the specified queue.
|
|
905
1209
|
* @param {string} queue - Queue name to consume from.
|
|
@@ -919,6 +1223,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
919
1223
|
const prefetch = options.prefetch !== undefined ? options.prefetch : this._config.prefetch;
|
|
920
1224
|
const noAck = options.noAck !== undefined ? options.noAck : this._config.noAck;
|
|
921
1225
|
|
|
1226
|
+
// Initialize queueOptions with default value
|
|
1227
|
+
let queueOptions = options.queueOptions || { durable };
|
|
1228
|
+
|
|
922
1229
|
try {
|
|
923
1230
|
// Skip assertQueue for reply queues (they're already created with specific settings)
|
|
924
1231
|
// Reply queues start with 'rpc.reply.' and are created as non-durable
|
|
@@ -938,8 +1245,6 @@ class RabbitMQClient extends EventEmitter {
|
|
|
938
1245
|
const isInfraQueue = queueConfig ? queueConfig.isInfrastructureQueue(queue) : false;
|
|
939
1246
|
const isBusinessQueue = queueConfig ? queueConfig.isBusinessQueue(queue) : false;
|
|
940
1247
|
|
|
941
|
-
let queueOptions = options.queueOptions || { durable };
|
|
942
|
-
|
|
943
1248
|
if (queueConfig) {
|
|
944
1249
|
if (isInfraQueue) {
|
|
945
1250
|
// Infrastructure queue - use central config
|
|
@@ -992,6 +1297,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
992
1297
|
|
|
993
1298
|
try {
|
|
994
1299
|
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] About to call assertQueue(${queue}, ${JSON.stringify(queueOptions)})`);
|
|
1300
|
+
|
|
1301
|
+
// Ensure queue channel is available
|
|
1302
|
+
await this._ensureQueueChannel();
|
|
1303
|
+
if (!this._queueChannel) {
|
|
1304
|
+
throw new Error('Queue channel is not available (connection may be closed)');
|
|
1305
|
+
}
|
|
1306
|
+
|
|
995
1307
|
this._trackChannelOperation(this._queueChannel, `assertQueue ${queue}`);
|
|
996
1308
|
await this._queueChannel.assertQueue(queue, queueOptions);
|
|
997
1309
|
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Queue ${queue} asserted successfully`);
|
|
@@ -1013,6 +1325,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1013
1325
|
// Set prefetch if provided (on consumer channel)
|
|
1014
1326
|
if (typeof prefetch === 'number') {
|
|
1015
1327
|
this._consumerChannel.prefetch(prefetch);
|
|
1328
|
+
|
|
1329
|
+
// Track prefetch for monitoring
|
|
1330
|
+
if (!this._prefetchTracking.has(queue)) {
|
|
1331
|
+
this._prefetchTracking.set(queue, {
|
|
1332
|
+
prefetchCount: prefetch,
|
|
1333
|
+
inFlight: 0,
|
|
1334
|
+
lastCheck: Date.now()
|
|
1335
|
+
});
|
|
1336
|
+
} else {
|
|
1337
|
+
const tracking = this._prefetchTracking.get(queue);
|
|
1338
|
+
tracking.prefetchCount = prefetch;
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
// Start prefetch monitoring if not already started
|
|
1342
|
+
if (!this._prefetchCheckTimer) {
|
|
1343
|
+
this._startPrefetchMonitoring();
|
|
1344
|
+
}
|
|
1016
1345
|
}
|
|
1017
1346
|
|
|
1018
1347
|
// CRITICAL: Log before calling amqplib's consume()
|
|
@@ -1024,18 +1353,47 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1024
1353
|
|
|
1025
1354
|
// Use dedicated consumer channel for consume operations
|
|
1026
1355
|
// This prevents conflicts with ConfirmChannel used for publish operations
|
|
1356
|
+
// Wrap handler to track message processing and prefetch utilization
|
|
1027
1357
|
const messageHandler = async (msg) => {
|
|
1028
1358
|
if (msg === null) {
|
|
1029
|
-
return;
|
|
1359
|
+
return; // Consumer cancellation
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
// Track in-flight message (increment)
|
|
1363
|
+
const tracking = this._prefetchTracking.get(queue);
|
|
1364
|
+
if (tracking) {
|
|
1365
|
+
tracking.inFlight++;
|
|
1366
|
+
tracking.lastCheck = Date.now();
|
|
1367
|
+
this._checkPrefetchUtilization(queue, tracking);
|
|
1030
1368
|
}
|
|
1369
|
+
|
|
1031
1370
|
try {
|
|
1032
1371
|
await onMessage(msg);
|
|
1372
|
+
// Acknowledge message after successful processing
|
|
1033
1373
|
if (!noAck) {
|
|
1034
1374
|
this._consumerChannel.ack(msg);
|
|
1375
|
+
// Track ack (decrement in-flight)
|
|
1376
|
+
if (tracking) {
|
|
1377
|
+
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
1378
|
+
tracking.lastCheck = Date.now();
|
|
1379
|
+
}
|
|
1035
1380
|
}
|
|
1036
1381
|
} catch (handlerErr) {
|
|
1037
1382
|
// Negative acknowledge and requeue by default
|
|
1038
|
-
|
|
1383
|
+
// Check if channel is still valid before nacking
|
|
1384
|
+
if (this._consumerChannel && !this._consumerChannel.closed) {
|
|
1385
|
+
try {
|
|
1386
|
+
this._consumerChannel.nack(msg, false, true);
|
|
1387
|
+
} catch (nackErr) {
|
|
1388
|
+
// Channel may have closed during nack - ignore
|
|
1389
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to nack message (channel may be closed): ${nackErr.message}`);
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
// Track nack (decrement in-flight)
|
|
1393
|
+
if (tracking) {
|
|
1394
|
+
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
1395
|
+
tracking.lastCheck = Date.now();
|
|
1396
|
+
}
|
|
1039
1397
|
}
|
|
1040
1398
|
};
|
|
1041
1399
|
|
|
@@ -1062,6 +1420,375 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1062
1420
|
}
|
|
1063
1421
|
}
|
|
1064
1422
|
|
|
1423
|
+
/**
|
|
1424
|
+
* Reconnect to RabbitMQ with exponential backoff
|
|
1425
|
+
* CONNECTION-LEVEL RECOVERY: Recreates connection and all channels, re-registers consumers
|
|
1426
|
+
* @private
|
|
1427
|
+
* @returns {Promise<void>}
|
|
1428
|
+
*/
|
|
1429
|
+
async _reconnectWithBackoff() {
|
|
1430
|
+
if (this._reconnecting) {
|
|
1431
|
+
console.log('[RabbitMQClient] Reconnection already in progress, skipping');
|
|
1432
|
+
return;
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
this._reconnecting = true;
|
|
1436
|
+
console.log(`[RabbitMQClient] Starting connection-level recovery (attempt ${this._reconnectAttempts + 1}/${this._maxReconnectAttempts})...`);
|
|
1437
|
+
|
|
1438
|
+
while (this._reconnectAttempts < this._maxReconnectAttempts) {
|
|
1439
|
+
try {
|
|
1440
|
+
// Calculate exponential backoff delay: baseDelay * 2^attempts, capped at maxDelay
|
|
1441
|
+
const delay = Math.min(
|
|
1442
|
+
this._reconnectBaseDelay * Math.pow(2, this._reconnectAttempts),
|
|
1443
|
+
this._reconnectMaxDelay
|
|
1444
|
+
);
|
|
1445
|
+
|
|
1446
|
+
console.log(`[RabbitMQClient] Waiting ${delay}ms before reconnection attempt ${this._reconnectAttempts + 1}...`);
|
|
1447
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
1448
|
+
|
|
1449
|
+
// Attempt to reconnect
|
|
1450
|
+
console.log(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts + 1}...`);
|
|
1451
|
+
|
|
1452
|
+
// Close old connection if exists
|
|
1453
|
+
if (this._connection) {
|
|
1454
|
+
try {
|
|
1455
|
+
await this._connection.close();
|
|
1456
|
+
} catch (_) {
|
|
1457
|
+
// Ignore errors when closing already-closed connection
|
|
1458
|
+
}
|
|
1459
|
+
this._connection = null;
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
// Reconnect
|
|
1463
|
+
const rawTarget = this._config.host || this._config.url;
|
|
1464
|
+
const heartbeat = this._config.heartbeat ?? 30;
|
|
1465
|
+
const connectionName =
|
|
1466
|
+
this._config.connectionName ||
|
|
1467
|
+
this._config.clientName ||
|
|
1468
|
+
`oa-client:${process.env.SERVICE_NAME || 'unknown'}:${process.pid}`;
|
|
1469
|
+
|
|
1470
|
+
const connectArgs = typeof rawTarget === 'string'
|
|
1471
|
+
? [rawTarget, { heartbeat, clientProperties: { connection_name: connectionName } }]
|
|
1472
|
+
: [{ ...rawTarget, heartbeat, clientProperties: { connection_name: connectionName } }];
|
|
1473
|
+
|
|
1474
|
+
const connectPromise = amqp.connect(...connectArgs);
|
|
1475
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
1476
|
+
setTimeout(() => reject(new Error('Connection timeout after 10 seconds')), 10000);
|
|
1477
|
+
});
|
|
1478
|
+
|
|
1479
|
+
this._connection = await Promise.race([connectPromise, timeoutPromise]);
|
|
1480
|
+
console.log('[RabbitMQClient] ✓ Connection re-established');
|
|
1481
|
+
|
|
1482
|
+
// Re-attach connection event handlers
|
|
1483
|
+
this._connection.on('error', (err) => {
|
|
1484
|
+
console.error('[RabbitMQClient] Connection error:', err.message);
|
|
1485
|
+
this.emit('error', err);
|
|
1486
|
+
});
|
|
1487
|
+
|
|
1488
|
+
this._connection.on('close', () => {
|
|
1489
|
+
console.warn('[RabbitMQClient] Connection closed unexpectedly');
|
|
1490
|
+
this._channel = null;
|
|
1491
|
+
this._queueChannel = null;
|
|
1492
|
+
this._consumerChannel = null;
|
|
1493
|
+
this.emit('error', new Error('RabbitMQ connection closed unexpectedly'));
|
|
1494
|
+
|
|
1495
|
+
// Attempt to reconnect again if enabled
|
|
1496
|
+
if (this._reconnectEnabled && !this._reconnecting) {
|
|
1497
|
+
this._reconnectWithBackoff().catch(err => {
|
|
1498
|
+
console.error('[RabbitMQClient] Reconnection failed after all attempts:', err.message);
|
|
1499
|
+
this.emit('error', err);
|
|
1500
|
+
});
|
|
1501
|
+
}
|
|
1502
|
+
});
|
|
1503
|
+
|
|
1504
|
+
// Recreate all channels
|
|
1505
|
+
console.log('[RabbitMQClient] Recreating channels...');
|
|
1506
|
+
|
|
1507
|
+
// Recreate publisher channel
|
|
1508
|
+
this._channel = await this._connection.createConfirmChannel();
|
|
1509
|
+
this._channel._createdAt = new Date().toISOString();
|
|
1510
|
+
this._channel._closeReason = null;
|
|
1511
|
+
this._channel._lastOperation = 'Recreated via _reconnectWithBackoff()';
|
|
1512
|
+
this._channel._type = 'publisher';
|
|
1513
|
+
this._attachPublisherChannelHandlers(this._channel);
|
|
1514
|
+
|
|
1515
|
+
// Recreate queue channel
|
|
1516
|
+
this._queueChannel = await this._connection.createChannel();
|
|
1517
|
+
this._queueChannel._createdAt = new Date().toISOString();
|
|
1518
|
+
this._queueChannel._closeReason = null;
|
|
1519
|
+
this._queueChannel._lastOperation = 'Recreated via _reconnectWithBackoff()';
|
|
1520
|
+
this._queueChannel._type = 'queue';
|
|
1521
|
+
this._attachQueueChannelHandlers(this._queueChannel);
|
|
1522
|
+
|
|
1523
|
+
// Recreate consumer channel and re-register all consumers
|
|
1524
|
+
this._consumerChannel = await this._connection.createChannel();
|
|
1525
|
+
this._consumerChannel._createdAt = new Date().toISOString();
|
|
1526
|
+
this._consumerChannel._closeReason = null;
|
|
1527
|
+
// IMPORTANT: Don't re-register consumers here - _ensureConsumerChannel() will do it
|
|
1528
|
+
// This prevents duplicate re-registration (once in _reconnectWithBackoff, once in _ensureConsumerChannel)
|
|
1529
|
+
// Just ensure consumer channel is created - it will automatically re-register all active consumers
|
|
1530
|
+
await this._ensureConsumerChannel();
|
|
1531
|
+
|
|
1532
|
+
// Reset reconnect state
|
|
1533
|
+
this._reconnectAttempts = 0;
|
|
1534
|
+
this._reconnecting = false;
|
|
1535
|
+
|
|
1536
|
+
// CRITICAL: Verify all channels are ready before emitting 'reconnected'
|
|
1537
|
+
// This ensures publish/consume operations can proceed immediately after reconnection
|
|
1538
|
+
if (!this._channel || this._channel.closed) {
|
|
1539
|
+
throw new Error('Publisher channel not ready after reconnection');
|
|
1540
|
+
}
|
|
1541
|
+
if (!this._queueChannel || this._queueChannel.closed) {
|
|
1542
|
+
throw new Error('Queue channel not ready after reconnection');
|
|
1543
|
+
}
|
|
1544
|
+
if (!this._consumerChannel || this._consumerChannel.closed) {
|
|
1545
|
+
throw new Error('Consumer channel not ready after reconnection');
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
console.log('[RabbitMQClient] ✓ Connection-level recovery completed successfully - all channels ready');
|
|
1549
|
+
this.emit('reconnected');
|
|
1550
|
+
|
|
1551
|
+
// Flush buffered messages po reconnectu
|
|
1552
|
+
try {
|
|
1553
|
+
const flushResult = await this._publishLayer.flushBuffered();
|
|
1554
|
+
this._publishMonitor.trackFlushed(flushResult.inMemory + flushResult.persistent);
|
|
1555
|
+
console.log(`[RabbitMQClient] ✓ Flushed ${flushResult.inMemory + flushResult.persistent} buffered messages after reconnection`);
|
|
1556
|
+
} catch (flushErr) {
|
|
1557
|
+
console.warn(`[RabbitMQClient] Failed to flush buffered messages after reconnection: ${flushErr.message}`);
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
return; // Success - exit reconnection loop
|
|
1561
|
+
} catch (err) {
|
|
1562
|
+
this._reconnectAttempts++;
|
|
1563
|
+
console.error(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts} failed:`, err.message);
|
|
1564
|
+
|
|
1565
|
+
if (this._reconnectAttempts >= this._maxReconnectAttempts) {
|
|
1566
|
+
console.error(`[RabbitMQClient] ✗ Connection-level recovery failed after ${this._maxReconnectAttempts} attempts`);
|
|
1567
|
+
this._reconnecting = false;
|
|
1568
|
+
throw new Error(`Failed to reconnect after ${this._maxReconnectAttempts} attempts: ${err.message}`);
|
|
1569
|
+
}
|
|
1570
|
+
// Continue to next attempt
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
/**
|
|
1576
|
+
* Attach event handlers to publisher channel
|
|
1577
|
+
* @private
|
|
1578
|
+
*/
|
|
1579
|
+
_attachPublisherChannelHandlers(channel) {
|
|
1580
|
+
channel.on('error', async (err) => {
|
|
1581
|
+
const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
1582
|
+
channel._closeReason = reason;
|
|
1583
|
+
|
|
1584
|
+
// "unknown delivery tag" errors occur when channel closes during publish
|
|
1585
|
+
// This is expected - the publish callback will handle retry
|
|
1586
|
+
// We still log it but don't try to recreate channel (it's already closing)
|
|
1587
|
+
if (err.message && err.message.includes('unknown delivery tag')) {
|
|
1588
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Publisher channel error (channel closing during publish): ${err.message}`);
|
|
1589
|
+
// Don't try to recreate - channel is already closing
|
|
1590
|
+
// Don't emit error - publish callback will handle retry
|
|
1591
|
+
return;
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1594
|
+
console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
1595
|
+
this._callChannelCloseHooks('publisher', channel, err, reason);
|
|
1596
|
+
|
|
1597
|
+
// Only try to recreate if connection is available
|
|
1598
|
+
// If connection is closed, reconnection logic will handle channel recreation
|
|
1599
|
+
if (this._connection && !this._connection.closed) {
|
|
1600
|
+
try {
|
|
1601
|
+
await this._ensurePublisherChannel();
|
|
1602
|
+
} catch (recreateErr) {
|
|
1603
|
+
// Ignore errors during channel recreation if connection is closing
|
|
1604
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${recreateErr.message}`);
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
this.emit('error', err);
|
|
1609
|
+
});
|
|
1610
|
+
|
|
1611
|
+
channel.on('close', async () => {
|
|
1612
|
+
const reason = channel._closeReason || 'Publisher channel closed unexpectedly';
|
|
1613
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
|
|
1614
|
+
this._callChannelCloseHooks('publisher', channel, null, reason);
|
|
1615
|
+
const closedChannel = this._channel;
|
|
1616
|
+
this._channel = null;
|
|
1617
|
+
|
|
1618
|
+
// Only try to recreate if connection is available
|
|
1619
|
+
// If connection is closed, reconnection logic will handle channel recreation
|
|
1620
|
+
if (this._connection && !this._connection.closed) {
|
|
1621
|
+
try {
|
|
1622
|
+
await this._ensurePublisherChannel();
|
|
1623
|
+
} catch (err) {
|
|
1624
|
+
// Ignore errors during channel recreation if connection is closing
|
|
1625
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${err.message}`);
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
});
|
|
1629
|
+
|
|
1630
|
+
channel.on('drain', () => {
|
|
1631
|
+
console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
|
|
1632
|
+
});
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
/**
|
|
1636
|
+
* Attach event handlers to queue channel
|
|
1637
|
+
* @private
|
|
1638
|
+
*/
|
|
1639
|
+
_attachQueueChannelHandlers(channel) {
|
|
1640
|
+
channel.on('error', async (err) => {
|
|
1641
|
+
const reason = `Queue channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
1642
|
+
channel._closeReason = reason;
|
|
1643
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
1644
|
+
this._callChannelCloseHooks('queue', channel, err, reason);
|
|
1645
|
+
await this._ensureQueueChannel();
|
|
1646
|
+
});
|
|
1647
|
+
|
|
1648
|
+
channel.on('close', async () => {
|
|
1649
|
+
const reason = channel._closeReason || 'Queue channel closed unexpectedly';
|
|
1650
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
|
|
1651
|
+
this._callChannelCloseHooks('queue', channel, null, reason);
|
|
1652
|
+
const closedChannel = this._queueChannel;
|
|
1653
|
+
this._queueChannel = null;
|
|
1654
|
+
|
|
1655
|
+
// PREDICTABLE: Only recreate if connection is available and not closing
|
|
1656
|
+
// If connection is closed, reconnection logic will handle channel recreation
|
|
1657
|
+
if (this._connection && !this._connection.closed && !this._reconnecting) {
|
|
1658
|
+
try {
|
|
1659
|
+
await this._ensureQueueChannel();
|
|
1660
|
+
} catch (err) {
|
|
1661
|
+
// If connection is closing, this is expected - don't throw
|
|
1662
|
+
if (err.message && err.message.includes('Connection closed')) {
|
|
1663
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Queue channel close during connection closure (expected - reconnection will handle)`);
|
|
1664
|
+
} else {
|
|
1665
|
+
// Log but don't throw - channel will be recreated on next use or during reconnection
|
|
1666
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel: ${err.message} (will be recreated on next use)`);
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
} else {
|
|
1670
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Queue channel closed - will be recreated during reconnection or on next use`);
|
|
1671
|
+
}
|
|
1672
|
+
});
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
/**
|
|
1676
|
+
* Attach event handlers to consumer channel
|
|
1677
|
+
* @private
|
|
1678
|
+
*/
|
|
1679
|
+
_attachConsumerChannelHandlers(channel) {
|
|
1680
|
+
channel.on('error', async (err) => {
|
|
1681
|
+
const reason = `Consumer channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
1682
|
+
channel._closeReason = reason;
|
|
1683
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
1684
|
+
this._callChannelCloseHooks('consumer', channel, err, reason);
|
|
1685
|
+
await this._ensureConsumerChannel();
|
|
1686
|
+
this.emit('error', err);
|
|
1687
|
+
});
|
|
1688
|
+
|
|
1689
|
+
channel.on('close', async () => {
|
|
1690
|
+
const reason = channel._closeReason || 'Consumer channel closed unexpectedly';
|
|
1691
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
|
|
1692
|
+
this._callChannelCloseHooks('consumer', channel, null, reason);
|
|
1693
|
+
const closedChannel = this._consumerChannel;
|
|
1694
|
+
this._consumerChannel = null;
|
|
1695
|
+
|
|
1696
|
+
// Only try to recreate if connection is available
|
|
1697
|
+
// If connection is closed, reconnection logic will handle channel recreation
|
|
1698
|
+
if (this._connection && !this._connection.closed) {
|
|
1699
|
+
try {
|
|
1700
|
+
await this._ensureConsumerChannel();
|
|
1701
|
+
} catch (err) {
|
|
1702
|
+
// Ignore errors during channel recreation if connection is closing
|
|
1703
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel (connection may be closing): ${err.message}`);
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
});
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
/**
|
|
1710
|
+
* Check prefetch utilization for a queue and alert if threshold exceeded
|
|
1711
|
+
* @private
|
|
1712
|
+
* @param {string} queue - Queue name
|
|
1713
|
+
* @param {Object} tracking - Prefetch tracking object
|
|
1714
|
+
*/
|
|
1715
|
+
_checkPrefetchUtilization(queue, tracking) {
|
|
1716
|
+
if (!tracking || tracking.prefetchCount === 0) {
|
|
1717
|
+
return; // No prefetch set or tracking not available
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
const utilization = tracking.inFlight / tracking.prefetchCount;
|
|
1721
|
+
|
|
1722
|
+
if (utilization >= this._prefetchUtilizationThreshold) {
|
|
1723
|
+
const message = `Prefetch utilization high: queue '${queue}' has ${tracking.inFlight}/${tracking.prefetchCount} messages in-flight (${Math.round(utilization * 100)}%, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`;
|
|
1724
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ⚠️ ${message}`);
|
|
1725
|
+
|
|
1726
|
+
// Call alert callback if provided
|
|
1727
|
+
if (this._prefetchAlertCallback) {
|
|
1728
|
+
try {
|
|
1729
|
+
this._prefetchAlertCallback(queue, utilization, tracking.inFlight, tracking.prefetchCount);
|
|
1730
|
+
} catch (alertErr) {
|
|
1731
|
+
console.error(`[RabbitMQClient] [mq-client-core] Prefetch alert callback error:`, alertErr.message);
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
|
|
1735
|
+
// Emit event for external listeners
|
|
1736
|
+
this.emit('prefetch:high-utilization', {
|
|
1737
|
+
queue,
|
|
1738
|
+
utilization,
|
|
1739
|
+
inFlight: tracking.inFlight,
|
|
1740
|
+
prefetchCount: tracking.prefetchCount,
|
|
1741
|
+
threshold: this._prefetchUtilizationThreshold,
|
|
1742
|
+
timestamp: new Date().toISOString()
|
|
1743
|
+
});
|
|
1744
|
+
}
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
/**
|
|
1748
|
+
* Start periodic prefetch monitoring
|
|
1749
|
+
* @private
|
|
1750
|
+
*/
|
|
1751
|
+
_startPrefetchMonitoring() {
|
|
1752
|
+
if (this._prefetchCheckTimer) {
|
|
1753
|
+
return; // Already started
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
this._prefetchCheckTimer = setInterval(() => {
|
|
1757
|
+
const now = Date.now();
|
|
1758
|
+
for (const [queue, tracking] of this._prefetchTracking.entries()) {
|
|
1759
|
+
// Only check if we have recent activity (within last 30s)
|
|
1760
|
+
if (now - tracking.lastCheck < 30000) {
|
|
1761
|
+
this._checkPrefetchUtilization(queue, tracking);
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
}, this._prefetchCheckInterval);
|
|
1765
|
+
|
|
1766
|
+
console.log(`[RabbitMQClient] [mq-client-core] Prefetch monitoring started (interval: ${this._prefetchCheckInterval}ms, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`);
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
/**
|
|
1770
|
+
* Stop prefetch monitoring
|
|
1771
|
+
* @private
|
|
1772
|
+
*/
|
|
1773
|
+
_stopPrefetchMonitoring() {
|
|
1774
|
+
if (this._prefetchCheckTimer) {
|
|
1775
|
+
clearInterval(this._prefetchCheckTimer);
|
|
1776
|
+
this._prefetchCheckTimer = null;
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
|
|
1780
|
+
/**
|
|
1781
|
+
* Track channel operation for debugging
|
|
1782
|
+
* @private
|
|
1783
|
+
* @param {Object} channel - Channel object
|
|
1784
|
+
* @param {string} operation - Operation description
|
|
1785
|
+
*/
|
|
1786
|
+
_trackChannelOperation(channel, operation) {
|
|
1787
|
+
if (channel && typeof channel === 'object') {
|
|
1788
|
+
channel._lastOperation = operation;
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
|
|
1065
1792
|
/**
|
|
1066
1793
|
* Acknowledges a message.
|
|
1067
1794
|
* @param {Object} msg - RabbitMQ message object.
|