@onlineapps/mq-client-core 1.0.36 → 1.0.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,6 +8,15 @@
8
8
 
9
9
  const amqp = require('amqplib');
10
10
  const EventEmitter = require('events');
11
+ const {
12
+ TransientPublishError,
13
+ PermanentPublishError,
14
+ QueueNotFoundError,
15
+ classifyPublishError,
16
+ } = require('../utils/publishErrors');
17
+ const PublishLayer = require('../layers/PublishLayer');
18
+ const RecoveryWorker = require('../workers/RecoveryWorker');
19
+ const PublishMonitor = require('../monitoring/PublishMonitor');
11
20
 
12
21
  class RabbitMQClient extends EventEmitter {
13
22
  /**
@@ -39,13 +48,44 @@ class RabbitMQClient extends EventEmitter {
39
48
  // Track active consumers for re-registration after channel recreation
40
49
  this._activeConsumers = new Map(); // queue -> { handler, options, consumerTag }
41
50
 
42
- // Auto-reconnect flags
51
+ // Connection-level recovery
43
52
  this._reconnecting = false;
44
53
  this._reconnectAttempts = 0;
54
+ this._maxReconnectAttempts = this._config.maxReconnectAttempts || 10; // Max 10 attempts
55
+ this._reconnectBaseDelay = this._config.reconnectBaseDelay || 1000; // Start with 1 second
56
+ this._reconnectMaxDelay = this._config.reconnectMaxDelay || 30000; // Max 30 seconds
57
+ this._reconnectEnabled = this._config.reconnectEnabled !== false; // Default: enabled
58
+ this._reconnectTimer = null;
59
+
60
+ // Publisher retry configuration
61
+ this._publishRetryEnabled = this._config.publishRetryEnabled !== false; // Default: enabled
62
+ this._publishMaxRetries = this._config.publishMaxRetries || 3; // Default: 3 attempts
63
+ this._publishRetryBaseDelay = this._config.publishRetryBaseDelay || 100; // Default: 100ms
64
+ this._publishRetryMaxDelay = this._config.publishRetryMaxDelay || 5000; // Default: 5s
65
+ this._publishRetryBackoffMultiplier = this._config.publishRetryBackoffMultiplier || 2; // Default: 2 (exponential)
66
+ this._publishConfirmationTimeout = this._config.publishConfirmationTimeout || 5000; // Default: 5s per attempt
45
67
 
46
68
  // Channel close hooks - called when channel closes with detailed information
47
69
  this._channelCloseHooks = []; // Array of { type: 'publisher'|'queue'|'consumer', callback: (details) => void }
48
70
 
71
+ // Channel thrashing detection - track close frequency per channel type
72
+ this._channelCloseHistory = {
73
+ publisher: [], // Array of { timestamp: number, reason: string }
74
+ queue: [],
75
+ consumer: []
76
+ };
77
+ this._thrashingThreshold = this._config.thrashingThreshold || 5; // Max closes per minute
78
+ this._thrashingWindowMs = this._config.thrashingWindowMs || 60000; // 1 minute window
79
+ this._thrashingAlertCallback = this._config.thrashingAlertCallback || null; // (type, count, windowMs) => void
80
+ this._thrashingAlertsSent = new Set(); // Track which alerts we've already sent to avoid spam
81
+
82
+ // Prefetch monitoring - track utilization per queue
83
+ this._prefetchTracking = new Map(); // queue -> { prefetchCount: number, inFlight: number, lastCheck: number }
84
+ this._prefetchUtilizationThreshold = this._config.prefetchUtilizationThreshold || 0.8; // 80% threshold
85
+ this._prefetchCheckInterval = this._config.prefetchCheckInterval || 10000; // Check every 10s
86
+ this._prefetchAlertCallback = this._config.prefetchAlertCallback || null; // (queue, utilization, inFlight, prefetchCount) => void
87
+ this._prefetchCheckTimer = null;
88
+
49
89
  // Health monitoring
50
90
  this._healthCheckInterval = null;
51
91
  this._healthCheckIntervalMs = this._config.healthCheckInterval || 30000; // Default 30s
@@ -57,6 +97,89 @@ class RabbitMQClient extends EventEmitter {
57
97
  this._criticalHealthShutdown = this._config.criticalHealthShutdown !== false; // Default: true - shutdown on critical health
58
98
  this._criticalHealthShutdownDelay = this._config.criticalHealthShutdownDelay || 60000; // Default: 60s delay before shutdown
59
99
  this._criticalHealthStartTime = null; // Track when critical health started
100
+
101
+ // Publish layer (retry + buffer)
102
+ this._publishLayer = new PublishLayer({
103
+ client: this,
104
+ logger: console,
105
+ bufferConfig: {
106
+ inMemory: {
107
+ maxSize: this._config.publishBufferMaxSize || 100,
108
+ ttlMs: this._config.publishBufferTtlMs || 5 * 60 * 1000,
109
+ },
110
+ persistent: {
111
+ enabled: !!this._config.persistentBufferEnabled,
112
+ redisClient: this._config.persistentRedisClient || null,
113
+ },
114
+ },
115
+ });
116
+
117
+ // Recovery worker (connection recovery, queue creation pro business)
118
+ this._recoveryWorker = new RecoveryWorker({
119
+ client: this,
120
+ scope: this._config.recoveryScope || 'infrastructure',
121
+ queueCreationFilter: this._config.queueCreationFilter || null,
122
+ logger: console,
123
+ });
124
+
125
+ // Publish monitor (metriky)
126
+ this._publishMonitor = new PublishMonitor({
127
+ logger: console,
128
+ });
129
+
130
+ // Event listeners pro monitoring
131
+ this._setupPublishMonitoring();
132
+ }
133
+
134
+ /**
135
+ * Setup event listeners pro monitoring a recovery
136
+ * @private
137
+ */
138
+ _setupPublishMonitoring() {
139
+ // Monitoring events
140
+ this.on('publish:retry', (data) => {
141
+ if (data.attempt === 1) {
142
+ this._publishMonitor.trackAttempt(data.queue);
143
+ }
144
+ });
145
+
146
+ this.on('publish:success', (data) => {
147
+ this._publishMonitor.trackSuccess(data.queue, data.totalAttempts);
148
+ });
149
+
150
+ this.on('publish:failed', (data) => {
151
+ this._publishMonitor.trackFailure(data.queue, data.retryable);
152
+ });
153
+
154
+ // Track buffered messages
155
+ this.on('publish:buffered', () => {
156
+ this._publishMonitor.trackBuffered();
157
+ });
158
+
159
+ // Recovery worker - handle QueueNotFoundError via error event
160
+ this.on('error', async (err) => {
161
+ if (err instanceof QueueNotFoundError) {
162
+ try {
163
+ await this._recoveryWorker.handleQueueNotFound(err, { queue: err.queueName });
164
+ } catch (recoveryErr) {
165
+ // Recovery failed - error already thrown
166
+ }
167
+ }
168
+ });
169
+ }
170
+
171
+ /**
172
+ * Get publish metrics
173
+ */
174
+ getPublishMetrics() {
175
+ return this._publishMonitor.getMetrics();
176
+ }
177
+
178
+ /**
179
+ * Get Prometheus metrics
180
+ */
181
+ getPrometheusMetrics() {
182
+ return this._publishMonitor.getPrometheusMetrics();
60
183
  }
61
184
 
62
185
  /**
@@ -91,6 +214,76 @@ class RabbitMQClient extends EventEmitter {
91
214
  this._channelCloseHooks.push({ type, callback });
92
215
  }
93
216
 
217
+ /**
218
+ * Track channel close event for thrashing detection
219
+ * @private
220
+ * @param {string} channelType - 'publisher', 'queue', or 'consumer'
221
+ * @param {string} reason - Human-readable reason
222
+ */
223
+ _trackChannelClose(channelType, reason) {
224
+ const now = Date.now();
225
+ const history = this._channelCloseHistory[channelType];
226
+
227
+ if (!history) {
228
+ console.warn(`[RabbitMQClient] Unknown channel type for thrashing tracking: ${channelType}`);
229
+ return;
230
+ }
231
+
232
+ // Add new close event
233
+ history.push({
234
+ timestamp: now,
235
+ reason: reason || 'unknown'
236
+ });
237
+
238
+ // Remove old events outside the window
239
+ const cutoff = now - this._thrashingWindowMs;
240
+ while (history.length > 0 && history[0].timestamp < cutoff) {
241
+ history.shift();
242
+ }
243
+
244
+ // Check for thrashing
245
+ if (history.length >= this._thrashingThreshold) {
246
+ const alertKey = `${channelType}:${Math.floor(now / this._thrashingWindowMs)}`;
247
+
248
+ // Only alert once per window to avoid spam
249
+ if (!this._thrashingAlertsSent.has(alertKey)) {
250
+ this._thrashingAlertsSent.add(alertKey);
251
+
252
+ // Clean up old alert keys (keep only last 10 windows)
253
+ if (this._thrashingAlertsSent.size > 10) {
254
+ const oldestKey = Array.from(this._thrashingAlertsSent).sort()[0];
255
+ this._thrashingAlertsSent.delete(oldestKey);
256
+ }
257
+
258
+ const message = `Channel thrashing detected: ${channelType} channel closed ${history.length} times in ${Math.round(this._thrashingWindowMs / 1000)}s (threshold: ${this._thrashingThreshold})`;
259
+ console.error(`[RabbitMQClient] [mq-client-core] 🚨 ${message}`);
260
+
261
+ // Call alert callback if provided
262
+ if (this._thrashingAlertCallback) {
263
+ try {
264
+ this._thrashingAlertCallback(channelType, history.length, this._thrashingWindowMs, {
265
+ recentCloses: history.slice(-this._thrashingThreshold),
266
+ threshold: this._thrashingThreshold,
267
+ windowMs: this._thrashingWindowMs
268
+ });
269
+ } catch (alertErr) {
270
+ console.error(`[RabbitMQClient] [mq-client-core] Thrashing alert callback error:`, alertErr.message);
271
+ }
272
+ }
273
+
274
+ // Emit event for external listeners
275
+ this.emit('channel:thrashing', {
276
+ type: channelType,
277
+ count: history.length,
278
+ windowMs: this._thrashingWindowMs,
279
+ threshold: this._thrashingThreshold,
280
+ recentCloses: history.slice(-this._thrashingThreshold),
281
+ timestamp: new Date().toISOString()
282
+ });
283
+ }
284
+ }
285
+ }
286
+
94
287
  /**
95
288
  * Call all registered channel close hooks
96
289
  * @private
@@ -100,6 +293,9 @@ class RabbitMQClient extends EventEmitter {
100
293
  * @param {string} reason - Human-readable reason
101
294
  */
102
295
  _callChannelCloseHooks(channelType, channel, error, reason) {
296
+ // Track channel close for thrashing detection
297
+ this._trackChannelClose(channelType, reason);
298
+
103
299
  const details = {
104
300
  type: channelType,
105
301
  channel: channel,
@@ -152,10 +348,40 @@ class RabbitMQClient extends EventEmitter {
152
348
  console.log('[RabbitMQClient] Starting connection race...');
153
349
  this._connection = await Promise.race([connectPromise, timeoutPromise]);
154
350
  console.log('[RabbitMQClient] Connection established');
155
- this._connection.on('error', (err) => this.emit('error', err));
351
+ this._reconnectAttempts = 0; // Reset reconnect attempts on successful connection
352
+
353
+ this._connection.on('error', (err) => {
354
+ console.error('[RabbitMQClient] Connection error:', err.message);
355
+ this.emit('error', err);
356
+ // Connection errors may lead to close event - let close handler handle reconnection
357
+ });
358
+
156
359
  this._connection.on('close', () => {
157
- // Emit a connection close error to notify listeners
360
+ console.warn('[RabbitMQClient] Connection closed unexpectedly - close handler STARTED');
361
+ console.log(`[RabbitMQClient] Close handler: reconnectEnabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
362
+
363
+ // Mark all channels as closed
364
+ this._channel = null;
365
+ this._queueChannel = null;
366
+ this._consumerChannel = null;
367
+
368
+ // CONNECTION-LEVEL RECOVERY: Attempt to reconnect with exponential backoff
369
+ // PREDICTABLE: Check conditions and log why reconnect is/isn't starting
370
+ // IMPORTANT: Do this BEFORE emitting error, so reconnect can start even if error handler throws
371
+ if (this._reconnectEnabled && !this._reconnecting) {
372
+ console.log('[RabbitMQClient] ✓ Conditions met - starting connection-level recovery from initial connect() close handler');
373
+ this._reconnectWithBackoff().catch(err => {
374
+ console.error('[RabbitMQClient] Reconnection failed after all attempts:', err.message);
375
+ this.emit('error', err);
376
+ });
377
+ } else {
378
+ console.warn(`[RabbitMQClient] ✗ Reconnect NOT starting: enabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
379
+ }
380
+
381
+ // Emit connection close event AFTER starting reconnect (so reconnect isn't blocked by error handler)
158
382
  this.emit('error', new Error('RabbitMQ connection closed unexpectedly'));
383
+
384
+ console.log('[RabbitMQClient] Close handler FINISHED');
159
385
  });
160
386
 
161
387
  // Use ConfirmChannel to enable publisher confirms for publish operations
@@ -167,37 +393,8 @@ class RabbitMQClient extends EventEmitter {
167
393
  this._channel._lastOperation = 'Initial creation';
168
394
  this._channel._type = 'publisher';
169
395
 
170
- // Enable publisher confirms explicitly
171
- this._channel.on('error', async (err) => {
172
- const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
173
- this._channel._closeReason = reason;
174
- console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
175
-
176
- // Call close hooks with detailed information
177
- this._callChannelCloseHooks('publisher', this._channel, err, reason);
178
-
179
- // Auto-recreate channel if connection is still alive
180
- await this._ensurePublisherChannel();
181
- this.emit('error', err);
182
- });
183
- this._channel.on('close', async () => {
184
- const reason = this._channel._closeReason || 'Publisher channel closed unexpectedly';
185
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
186
-
187
- // Call close hooks with detailed information
188
- this._callChannelCloseHooks('publisher', this._channel, null, reason);
189
-
190
- // Mark channel as null - will be recreated on next publish
191
- const closedChannel = this._channel;
192
- this._channel = null;
193
-
194
- // Try to recreate immediately if connection is alive
195
- await this._ensurePublisherChannel();
196
- });
197
- // Set up publisher confirms callback
198
- this._channel.on('drain', () => {
199
- console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
200
- });
396
+ // Attach event handlers
397
+ this._attachPublisherChannelHandlers(this._channel);
201
398
 
202
399
  // Create a separate regular channel for queue operations (assertQueue, checkQueue)
203
400
  // This avoids RPC reply queue issues with ConfirmChannel.assertQueue()
@@ -209,31 +406,8 @@ class RabbitMQClient extends EventEmitter {
209
406
  this._queueChannel._lastOperation = 'Initial creation';
210
407
  this._queueChannel._type = 'queue';
211
408
 
212
- this._queueChannel.on('error', async (err) => {
213
- const reason = `Queue channel error: ${err.message} (code: ${err.code || 'unknown'})`;
214
- this._queueChannel._closeReason = reason;
215
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
216
-
217
- // Call close hooks with detailed information
218
- this._callChannelCloseHooks('queue', this._queueChannel, err, reason);
219
-
220
- // Auto-recreate channel if connection is still alive
221
- await this._ensureQueueChannel();
222
- });
223
- this._queueChannel.on('close', async () => {
224
- const reason = this._queueChannel._closeReason || 'Queue channel closed unexpectedly';
225
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
226
-
227
- // Call close hooks with detailed information
228
- this._callChannelCloseHooks('queue', this._queueChannel, null, reason);
229
-
230
- // Mark channel as null - will be recreated on next queue operation
231
- const closedChannel = this._queueChannel;
232
- this._queueChannel = null;
233
-
234
- // Try to recreate immediately if connection is alive
235
- await this._ensureQueueChannel();
236
- });
409
+ // Attach event handlers
410
+ this._attachQueueChannelHandlers(this._queueChannel);
237
411
 
238
412
  // Create a dedicated channel for consume operations
239
413
  // This prevents channel conflicts between publish (ConfirmChannel) and consume operations
@@ -245,32 +419,8 @@ class RabbitMQClient extends EventEmitter {
245
419
  this._consumerChannel._lastOperation = 'Initial creation';
246
420
  this._consumerChannel._type = 'consumer';
247
421
 
248
- this._consumerChannel.on('error', async (err) => {
249
- const reason = `Consumer channel error: ${err.message} (code: ${err.code || 'unknown'})`;
250
- this._consumerChannel._closeReason = reason;
251
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
252
-
253
- // Call close hooks with detailed information
254
- this._callChannelCloseHooks('consumer', this._consumerChannel, err, reason);
255
-
256
- // Auto-recreate channel and re-register consumers
257
- await this._ensureConsumerChannel();
258
- this.emit('error', err);
259
- });
260
- this._consumerChannel.on('close', async () => {
261
- const reason = this._consumerChannel._closeReason || 'Consumer channel closed unexpectedly';
262
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
263
-
264
- // Call close hooks with detailed information
265
- this._callChannelCloseHooks('consumer', this._consumerChannel, null, reason);
266
-
267
- // Mark channel as null - will be recreated and consumers re-registered
268
- const closedChannel = this._consumerChannel;
269
- this._consumerChannel = null;
270
-
271
- // Try to recreate and re-register consumers immediately if connection is alive
272
- await this._ensureConsumerChannel();
273
- });
422
+ // Attach event handlers
423
+ this._attachConsumerChannelHandlers(this._consumerChannel);
274
424
 
275
425
  // Start health monitoring if enabled
276
426
  if (this._healthCheckEnabled) {
@@ -290,6 +440,50 @@ class RabbitMQClient extends EventEmitter {
290
440
  }
291
441
  }
292
442
 
443
+ /**
444
+ * Wait for reconnection to complete
445
+ * Used by both retry mechanism and channel creation
446
+ * @private
447
+ * @returns {Promise<void>}
448
+ * @throws {Error} If reconnection fails or times out
449
+ */
450
+ async _waitForReconnection() {
451
+ if (!this._reconnecting) {
452
+ throw new Error('Cannot wait for reconnection: not currently reconnecting');
453
+ }
454
+
455
+ // Wait for 'reconnected' event
456
+ return new Promise((resolve, reject) => {
457
+ const timeout = setTimeout(() => {
458
+ this.removeListener('reconnected', onReconnected);
459
+ this.removeListener('error', onError);
460
+ reject(new Error('Reconnection timeout after 30 seconds'));
461
+ }, 30000); // 30 seconds timeout
462
+
463
+ const onReconnected = () => {
464
+ clearTimeout(timeout);
465
+ this.removeListener('reconnected', onReconnected);
466
+ this.removeListener('error', onError);
467
+ // Wait a bit for channels to be fully recreated
468
+ setTimeout(resolve, 500);
469
+ };
470
+
471
+ const onError = (error) => {
472
+ // Ignore connection errors during reconnection
473
+ if (error.message && error.message.includes('Connection closed unexpectedly')) {
474
+ return; // Expected during reconnection
475
+ }
476
+ clearTimeout(timeout);
477
+ this.removeListener('reconnected', onReconnected);
478
+ this.removeListener('error', onError);
479
+ reject(error);
480
+ };
481
+
482
+ this.once('reconnected', onReconnected);
483
+ this.on('error', onError);
484
+ });
485
+ }
486
+
293
487
  /**
294
488
  * Ensure publisher channel exists and is open
295
489
  * Auto-recreates if closed
@@ -302,7 +496,25 @@ class RabbitMQClient extends EventEmitter {
302
496
  }
303
497
 
304
498
  if (!this._connection || this._connection.closed) {
305
- throw new Error('Cannot recreate publisher channel: connection is closed');
499
+ // Connection is closed - wait for reconnection
500
+ console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate publisher channel: connection is closed (waiting for reconnection)');
501
+
502
+ // Wait for reconnection to complete
503
+ if (this._reconnecting) {
504
+ await this._waitForReconnection();
505
+
506
+ // After reconnection, try again
507
+ if (this._channel && !this._channel.closed) {
508
+ return; // Channel was recreated during reconnection
509
+ }
510
+ // If still no channel, connection might have failed - check again
511
+ if (!this._connection || this._connection.closed) {
512
+ throw new Error('Connection reconnection failed - cannot create publisher channel');
513
+ }
514
+ } else {
515
+ // Not reconnecting - connection is closed and not recovering
516
+ throw new Error('Connection is closed and not reconnecting - cannot create publisher channel');
517
+ }
306
518
  }
307
519
 
308
520
  try {
@@ -324,36 +536,29 @@ class RabbitMQClient extends EventEmitter {
324
536
  this._channel._lastOperation = 'Recreated via _ensurePublisherChannel()';
325
537
  this._channel._type = 'publisher';
326
538
 
327
- this._channel.on('error', async (err) => {
328
- const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
329
- this._channel._closeReason = reason;
330
- console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
331
-
332
- // Call close hooks
333
- this._callChannelCloseHooks('publisher', this._channel, err, reason);
334
-
335
- await this._ensurePublisherChannel();
336
- this.emit('error', err);
337
- });
338
- this._channel.on('close', async () => {
339
- const reason = this._channel._closeReason || 'Publisher channel closed unexpectedly';
340
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
341
-
342
- // Call close hooks
343
- this._callChannelCloseHooks('publisher', this._channel, null, reason);
344
-
345
- const closedChannel = this._channel;
346
- this._channel = null;
347
- await this._ensurePublisherChannel();
348
- });
349
- this._channel.on('drain', () => {
350
- console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
351
- });
539
+ // Attach event handlers
540
+ this._attachPublisherChannelHandlers(this._channel);
352
541
 
353
542
  console.log('[RabbitMQClient] [mq-client-core] ✓ Publisher channel recreated');
354
543
  } catch (err) {
355
544
  this._channel = null;
356
- throw new Error(`Failed to recreate publisher channel: ${err.message}`);
545
+
546
+ const msg = err && err.message ? err.message : String(err);
547
+
548
+ // Pokud se connection právě zavírá, považujeme to za přechodný stav – explicitně vyhodíme
549
+ // TransientPublishError, aby vyšší vrstva (retry / worker) věděla, že má počkat na reconnect.
550
+ if (msg.includes('Connection closing') || msg.includes('Connection closed')) {
551
+ console.warn(
552
+ `[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection closing): ${msg}`
553
+ );
554
+ throw new TransientPublishError(
555
+ 'Publisher channel cannot be recreated because connection is closing/closed (transient condition)',
556
+ err
557
+ );
558
+ }
559
+
560
+ // Ostatní chyby považujeme za permanentní problém na úrovni kanálu/spojení.
561
+ throw new PermanentPublishError(`Failed to recreate publisher channel: ${msg}`, err);
357
562
  }
358
563
  }
359
564
 
@@ -369,7 +574,11 @@ class RabbitMQClient extends EventEmitter {
369
574
  }
370
575
 
371
576
  if (!this._connection || this._connection.closed) {
372
- throw new Error('Cannot recreate queue channel: connection is closed');
577
+ // Connection is closed - cannot recreate channel
578
+ // Reconnection logic will handle this - don't throw, just return
579
+ // The channel will be recreated after reconnection completes
580
+ console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate queue channel: connection is closed (reconnection in progress)');
581
+ return;
373
582
  }
374
583
 
375
584
  try {
@@ -391,32 +600,18 @@ class RabbitMQClient extends EventEmitter {
391
600
  this._queueChannel._lastOperation = 'Recreated via _ensureQueueChannel()';
392
601
  this._queueChannel._type = 'queue';
393
602
 
394
- this._queueChannel.on('error', async (err) => {
395
- const reason = `Queue channel error: ${err.message} (code: ${err.code || 'unknown'})`;
396
- this._queueChannel._closeReason = reason;
397
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
398
-
399
- // Call close hooks
400
- this._callChannelCloseHooks('queue', this._queueChannel, err, reason);
401
-
402
- await this._ensureQueueChannel();
403
- });
404
- this._queueChannel.on('close', async () => {
405
- const reason = this._queueChannel._closeReason || 'Queue channel closed unexpectedly';
406
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
407
-
408
- // Call close hooks
409
- this._callChannelCloseHooks('queue', this._queueChannel, null, reason);
410
-
411
- const closedChannel = this._queueChannel;
412
- this._queueChannel = null;
413
- await this._ensureQueueChannel();
414
- });
603
+ // Attach event handlers
604
+ this._attachQueueChannelHandlers(this._queueChannel);
415
605
 
416
606
  console.log('[RabbitMQClient] [mq-client-core] ✓ Queue channel recreated');
417
607
  } catch (err) {
418
608
  this._queueChannel = null;
419
- throw new Error(`Failed to recreate queue channel: ${err.message}`);
609
+ // If connection is closed, don't throw - reconnection will handle it
610
+ if (this._connection && !this._connection.closed) {
611
+ throw new Error(`Failed to recreate queue channel: ${err.message}`);
612
+ }
613
+ // Connection is closed - reconnection will recreate channel
614
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel (connection closed): ${err.message}`);
420
615
  }
421
616
  }
422
617
 
@@ -432,7 +627,11 @@ class RabbitMQClient extends EventEmitter {
432
627
  }
433
628
 
434
629
  if (!this._connection || this._connection.closed) {
435
- throw new Error('Cannot recreate consumer channel: connection is closed');
630
+ // Connection is closed - cannot recreate channel
631
+ // Reconnection logic will handle this - don't throw, just return
632
+ // The channel will be recreated after reconnection completes
633
+ console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate consumer channel: connection is closed (reconnection in progress)');
634
+ return;
436
635
  }
437
636
 
438
637
  try {
@@ -454,28 +653,8 @@ class RabbitMQClient extends EventEmitter {
454
653
  this._consumerChannel._lastOperation = 'Recreated via _ensureConsumerChannel()';
455
654
  this._consumerChannel._type = 'consumer';
456
655
 
457
- this._consumerChannel.on('error', async (err) => {
458
- const reason = `Consumer channel error: ${err.message} (code: ${err.code || 'unknown'})`;
459
- this._consumerChannel._closeReason = reason;
460
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
461
-
462
- // Call close hooks
463
- this._callChannelCloseHooks('consumer', this._consumerChannel, err, reason);
464
-
465
- await this._ensureConsumerChannel();
466
- this.emit('error', err);
467
- });
468
- this._consumerChannel.on('close', async () => {
469
- const reason = this._consumerChannel._closeReason || 'Consumer channel closed unexpectedly';
470
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
471
-
472
- // Call close hooks
473
- this._callChannelCloseHooks('consumer', this._consumerChannel, null, reason);
474
-
475
- const closedChannel = this._consumerChannel;
476
- this._consumerChannel = null;
477
- await this._ensureConsumerChannel();
478
- });
656
+ // Attach event handlers
657
+ this._attachConsumerChannelHandlers(this._consumerChannel);
479
658
 
480
659
  // Re-register all active consumers
481
660
  if (this._activeConsumers.size > 0) {
@@ -737,9 +916,15 @@ class RabbitMQClient extends EventEmitter {
737
916
  // Stop health monitoring
738
917
  this._stopHealthMonitoring();
739
918
 
919
+ // Stop prefetch monitoring
920
+ this._stopPrefetchMonitoring();
921
+
740
922
  // Clear active consumers tracking
741
923
  this._activeConsumers.clear();
742
924
 
925
+ // Clear prefetch tracking
926
+ this._prefetchTracking.clear();
927
+
743
928
  try {
744
929
  if (this._consumerChannel) {
745
930
  await this._consumerChannel.close();
@@ -771,19 +956,36 @@ class RabbitMQClient extends EventEmitter {
771
956
  this._connection = null;
772
957
  }
773
958
  } catch (err) {
774
- this.emit('error', err);
959
+ // Don't emit error for expected connection closure during disconnect
960
+ // This is a predictable, intentional closure
961
+ if (err.message && err.message.includes('Connection closed (by client)')) {
962
+ console.log('[RabbitMQClient] Connection closed during disconnect (expected)');
963
+ } else {
964
+ this.emit('error', err);
965
+ }
775
966
  }
776
967
  }
777
968
 
778
969
  /**
779
970
  * Publishes a message buffer to the specified queue (or default queue) or exchange.
971
+ * Implements systematic retry with exponential backoff for transient failures.
780
972
  * @param {string} queue - Target queue name.
781
973
  * @param {Buffer} buffer - Message payload as Buffer.
782
974
  * @param {Object} [options] - Overrides: routingKey, persistent, headers.
783
975
  * @returns {Promise<void>}
784
- * @throws {Error} If publish fails or channel is not available.
976
+ * @throws {Error} If publish fails after all retry attempts.
785
977
  */
786
978
  async publish(queue, buffer, options = {}) {
979
+ // PublishLayer zachovává stávající retry logiku a přidává možnost bufferování a
980
+ // jasnější klasifikaci chyb. API zůstává stejné.
981
+ return await this._publishLayer.publish(queue, buffer, options);
982
+ }
983
+
984
+ /**
985
+ * Internal publish implementation - single attempt without retry
986
+ * @private
987
+ */
988
+ async _publishOnce(queue, buffer, options = {}) {
787
989
  // Ensure publisher channel exists and is open (auto-recreates if closed)
788
990
  await this._ensurePublisherChannel();
789
991
 
@@ -812,9 +1014,10 @@ class RabbitMQClient extends EventEmitter {
812
1014
  await this._queueChannel.checkQueue(queue);
813
1015
  // Queue exists - proceed to publish
814
1016
  } catch (checkErr) {
815
- // If queue doesn't exist (404), this is an ERROR for infrastructure queues
816
- // Infrastructure queues (workflow.*, registry.*, infrastructure.*, monitoring.*, validation.*) must be created explicitly with correct arguments
817
- // We should NOT auto-create them here, as we don't have access to queueConfig in mq-client-core
1017
+ // If queue doesn't exist (404), this je ERROR ale rozlišujeme infra vs. non-infra:
1018
+ // - Infrastructure queues (workflow.*, registry.*, infrastructure.*, monitoring.*, validation.*)
1019
+ // musí být vytvořené infra službami předem QueueNotFoundError(isInfrastructure=true)
1020
+ // - Non-infrastructure queues můžeme (pro business scénáře) vytvořit s default parametry
818
1021
  if (checkErr.code === 404) {
819
1022
  // Check if this is an infrastructure queue using queueConfig
820
1023
  let isInfraQueue = false;
@@ -831,7 +1034,8 @@ class RabbitMQClient extends EventEmitter {
831
1034
  }
832
1035
 
833
1036
  if (isInfraQueue) {
834
- throw new Error(`Cannot publish to infrastructure queue ${queue}: queue does not exist. Infrastructure queues must be created explicitly with correct arguments (TTL, max-length, etc.) before publishing.`);
1037
+ // Jasný, hlasitý signál pro infra služby fronta chybí, je to programátorská chyba
1038
+ throw new QueueNotFoundError(queue, true, checkErr);
835
1039
  }
836
1040
  // For non-infrastructure queues, allow auto-creation with default options
837
1041
  const queueOptions = options.queueOptions || { durable: this._config.durable };
@@ -849,16 +1053,103 @@ class RabbitMQClient extends EventEmitter {
849
1053
 
850
1054
  // Use callback-based confirmation - kanály jsou spolehlivé, takže callback vždy dorazí
851
1055
  const confirmPromise = new Promise((resolve, reject) => {
1056
+ // Set timeout for publish confirmation (configurable)
1057
+ const timeout = setTimeout(() => {
1058
+ reject(new Error(`Publish confirmation timeout for queue "${queue}" after ${this._publishConfirmationTimeout}ms`));
1059
+ }, this._publishConfirmationTimeout);
1060
+
1061
+ // Check if channel is still valid before sending
1062
+ if (!this._channel || this._channel.closed) {
1063
+ clearTimeout(timeout);
1064
+ reject(new Error(`Cannot publish: channel is closed for queue "${queue}"`));
1065
+ return;
1066
+ }
1067
+
1068
+ // Track original channel to detect if it was closed/recreated during publish
1069
+ const originalChannel = this._channel;
1070
+ const channelId = originalChannel ? originalChannel._createdAt : null;
1071
+ let callbackInvoked = false;
1072
+
852
1073
  // Send message with callback
853
- this._channel.sendToQueue(queue, buffer, { persistent, headers }, (err, ok) => {
854
- if (err) {
855
- console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
856
- reject(err);
1074
+ try {
1075
+ originalChannel.sendToQueue(queue, buffer, { persistent, headers }, (err, ok) => {
1076
+ callbackInvoked = true;
1077
+ clearTimeout(timeout);
1078
+
1079
+ // CRITICAL: Check if channel was closed or recreated during publish
1080
+ // If channel was recreated, the delivery tag is invalid - ignore this callback
1081
+ if (!this._channel || this._channel.closed || this._channel !== originalChannel ||
1082
+ (this._channel._createdAt && this._channel._createdAt !== channelId)) {
1083
+ // Channel was closed or recreated - delivery tag is invalid
1084
+ console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed/recreated during publish to queue "${queue}", ignoring callback`);
1085
+ // If we're reconnecting, wait and retry
1086
+ if (this._reconnecting) {
1087
+ this._waitForReconnection().then(() => {
1088
+ // Retry publish after reconnection
1089
+ return this.publish(queue, buffer, options);
1090
+ }).then(resolve).catch(reject);
1091
+ } else {
1092
+ // Not reconnecting - this is a real error
1093
+ // We cannot guarantee the message was delivered - must fail
1094
+ reject(new Error(`Channel closed during publish to queue "${queue}" and not reconnecting - message delivery not guaranteed`));
1095
+ }
1096
+ return;
1097
+ }
1098
+
1099
+ if (err) {
1100
+ // Check if error is about invalid delivery tag (channel was closed)
1101
+ if (err.message && (err.message.includes('unknown delivery tag') || err.message.includes('PRECONDITION_FAILED'))) {
1102
+ console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Delivery tag invalid (channel may have been closed) for queue "${queue}"`);
1103
+ // If we're reconnecting, wait and retry
1104
+ if (this._reconnecting) {
1105
+ this._waitForReconnection().then(() => {
1106
+ // Retry publish after reconnection
1107
+ return this.publish(queue, buffer, options);
1108
+ }).then(resolve).catch(reject);
1109
+ } else {
1110
+ // Not reconnecting - delivery tag is invalid, message delivery not guaranteed
1111
+ reject(new Error(`Delivery tag invalid for queue "${queue}" (channel closed) and not reconnecting - message delivery not guaranteed`));
1112
+ }
1113
+ } else {
1114
+ console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
1115
+ reject(err);
1116
+ }
1117
+ } else {
1118
+ console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Message confirmed for queue "${queue}"`);
1119
+ resolve();
1120
+ }
1121
+ });
1122
+
1123
+ // Set a safety timeout - if callback wasn't invoked and channel closed, retry
1124
+ setTimeout(() => {
1125
+ if (!callbackInvoked && (!this._channel || this._channel.closed || this._channel !== originalChannel)) {
1126
+ console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Callback timeout and channel closed for queue "${queue}", will retry after reconnection`);
1127
+ if (this._reconnecting) {
1128
+ clearTimeout(timeout);
1129
+ this._waitForReconnection().then(() => {
1130
+ return this.publish(queue, buffer, options);
1131
+ }).then(resolve).catch(reject);
1132
+ }
1133
+ }
1134
+ }, 1000);
1135
+ } catch (sendErr) {
1136
+ clearTimeout(timeout);
1137
+ // If channel is closed, try to wait for reconnection and retry
1138
+ if (sendErr.message && (sendErr.message.includes('Channel ended') || sendErr.message.includes('Channel closed'))) {
1139
+ console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during sendToQueue, will retry after reconnection`);
1140
+ // Wait for reconnection and retry
1141
+ if (this._reconnecting) {
1142
+ this._waitForReconnection().then(() => {
1143
+ // Retry publish after reconnection
1144
+ return this.publish(queue, buffer, options);
1145
+ }).then(resolve).catch(reject);
1146
+ } else {
1147
+ reject(sendErr);
1148
+ }
857
1149
  } else {
858
- console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Message confirmed for queue "${queue}"`);
859
- resolve();
1150
+ reject(sendErr);
860
1151
  }
861
- });
1152
+ }
862
1153
  });
863
1154
 
864
1155
  await confirmPromise;
@@ -869,7 +1160,13 @@ class RabbitMQClient extends EventEmitter {
869
1160
 
870
1161
  // Use callback-based confirmation - kanály jsou spolehlivé
871
1162
  const confirmPromise = new Promise((resolve, reject) => {
1163
+ // Set timeout for exchange publish confirmation
1164
+ const timeout = setTimeout(() => {
1165
+ reject(new Error(`Exchange publish confirmation timeout for exchange "${exchange}" after ${this._publishConfirmationTimeout}ms`));
1166
+ }, this._publishConfirmationTimeout);
1167
+
872
1168
  this._channel.publish(exchange, routingKey, buffer, { persistent, headers }, (err, ok) => {
1169
+ clearTimeout(timeout);
873
1170
  if (err) {
874
1171
  console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Exchange publish callback error:`, err.message);
875
1172
  reject(err);
@@ -883,23 +1180,30 @@ class RabbitMQClient extends EventEmitter {
883
1180
  await confirmPromise;
884
1181
  }
885
1182
  } catch (err) {
886
- // If channel was closed, try to recreate and retry once
887
- if (err.message && (err.message.includes('Channel closed') || err.message.includes('channel is closed') || this._channel?.closed)) {
888
- console.warn('[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during publish, recreating and retrying...');
1183
+ // Pokud je kanál zavřený, stále se pokusíme o jedno interní obnovení a retry,
1184
+ // aby byl publish co nejodolnější i předtím, než vstoupí do vyšší retry/recovery vrstvy.
1185
+ if (err && err.message && (err.message.includes('Channel closed') || err.message.includes('channel is closed') || this._channel?.closed)) {
1186
+ console.warn('[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during publish, recreating and retrying once...');
889
1187
  try {
890
1188
  await this._ensurePublisherChannel();
891
- // Retry publish once
1189
+ // Retry publish once (může znovu vyhodit – už dojde do retry/recovery vrstvy)
892
1190
  return await this.publish(queue, buffer, options);
893
1191
  } catch (retryErr) {
894
- console.error('[RabbitMQClient] [mq-client-core] [PUBLISH] Retry failed:', retryErr.message);
895
- throw retryErr;
1192
+ console.error('[RabbitMQClient] [mq-client-core] [PUBLISH] Retry after channel recreation failed:', retryErr.message);
1193
+ const classifiedRetryErr = classifyPublishError(retryErr);
1194
+ this.emit('error', classifiedRetryErr);
1195
+ throw classifiedRetryErr;
896
1196
  }
897
1197
  }
898
- this.emit('error', err);
899
- throw err;
1198
+
1199
+ // Všechny ostatní chyby projdou jednotnou klasifikací do Transient/Permanent/QueueNotFound.
1200
+ const classifiedErr = classifyPublishError(err);
1201
+ this.emit('error', classifiedErr);
1202
+ throw classifiedErr;
900
1203
  }
901
1204
  }
902
1205
 
1206
+
903
1207
  /**
904
1208
  * Starts consuming messages from the specified queue.
905
1209
  * @param {string} queue - Queue name to consume from.
@@ -919,6 +1223,9 @@ class RabbitMQClient extends EventEmitter {
919
1223
  const prefetch = options.prefetch !== undefined ? options.prefetch : this._config.prefetch;
920
1224
  const noAck = options.noAck !== undefined ? options.noAck : this._config.noAck;
921
1225
 
1226
+ // Initialize queueOptions with default value
1227
+ let queueOptions = options.queueOptions || { durable };
1228
+
922
1229
  try {
923
1230
  // Skip assertQueue for reply queues (they're already created with specific settings)
924
1231
  // Reply queues start with 'rpc.reply.' and are created as non-durable
@@ -938,8 +1245,6 @@ class RabbitMQClient extends EventEmitter {
938
1245
  const isInfraQueue = queueConfig ? queueConfig.isInfrastructureQueue(queue) : false;
939
1246
  const isBusinessQueue = queueConfig ? queueConfig.isBusinessQueue(queue) : false;
940
1247
 
941
- let queueOptions = options.queueOptions || { durable };
942
-
943
1248
  if (queueConfig) {
944
1249
  if (isInfraQueue) {
945
1250
  // Infrastructure queue - use central config
@@ -992,6 +1297,13 @@ class RabbitMQClient extends EventEmitter {
992
1297
 
993
1298
  try {
994
1299
  console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] About to call assertQueue(${queue}, ${JSON.stringify(queueOptions)})`);
1300
+
1301
+ // Ensure queue channel is available
1302
+ await this._ensureQueueChannel();
1303
+ if (!this._queueChannel) {
1304
+ throw new Error('Queue channel is not available (connection may be closed)');
1305
+ }
1306
+
995
1307
  this._trackChannelOperation(this._queueChannel, `assertQueue ${queue}`);
996
1308
  await this._queueChannel.assertQueue(queue, queueOptions);
997
1309
  console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Queue ${queue} asserted successfully`);
@@ -1013,6 +1325,23 @@ class RabbitMQClient extends EventEmitter {
1013
1325
  // Set prefetch if provided (on consumer channel)
1014
1326
  if (typeof prefetch === 'number') {
1015
1327
  this._consumerChannel.prefetch(prefetch);
1328
+
1329
+ // Track prefetch for monitoring
1330
+ if (!this._prefetchTracking.has(queue)) {
1331
+ this._prefetchTracking.set(queue, {
1332
+ prefetchCount: prefetch,
1333
+ inFlight: 0,
1334
+ lastCheck: Date.now()
1335
+ });
1336
+ } else {
1337
+ const tracking = this._prefetchTracking.get(queue);
1338
+ tracking.prefetchCount = prefetch;
1339
+ }
1340
+
1341
+ // Start prefetch monitoring if not already started
1342
+ if (!this._prefetchCheckTimer) {
1343
+ this._startPrefetchMonitoring();
1344
+ }
1016
1345
  }
1017
1346
 
1018
1347
  // CRITICAL: Log before calling amqplib's consume()
@@ -1024,18 +1353,47 @@ class RabbitMQClient extends EventEmitter {
1024
1353
 
1025
1354
  // Use dedicated consumer channel for consume operations
1026
1355
  // This prevents conflicts with ConfirmChannel used for publish operations
1356
+ // Wrap handler to track message processing and prefetch utilization
1027
1357
  const messageHandler = async (msg) => {
1028
1358
  if (msg === null) {
1029
- return;
1359
+ return; // Consumer cancellation
1360
+ }
1361
+
1362
+ // Track in-flight message (increment)
1363
+ const tracking = this._prefetchTracking.get(queue);
1364
+ if (tracking) {
1365
+ tracking.inFlight++;
1366
+ tracking.lastCheck = Date.now();
1367
+ this._checkPrefetchUtilization(queue, tracking);
1030
1368
  }
1369
+
1031
1370
  try {
1032
1371
  await onMessage(msg);
1372
+ // Acknowledge message after successful processing
1033
1373
  if (!noAck) {
1034
1374
  this._consumerChannel.ack(msg);
1375
+ // Track ack (decrement in-flight)
1376
+ if (tracking) {
1377
+ tracking.inFlight = Math.max(0, tracking.inFlight - 1);
1378
+ tracking.lastCheck = Date.now();
1379
+ }
1035
1380
  }
1036
1381
  } catch (handlerErr) {
1037
1382
  // Negative acknowledge and requeue by default
1038
- this._consumerChannel.nack(msg, false, true);
1383
+ // Check if channel is still valid before nacking
1384
+ if (this._consumerChannel && !this._consumerChannel.closed) {
1385
+ try {
1386
+ this._consumerChannel.nack(msg, false, true);
1387
+ } catch (nackErr) {
1388
+ // Channel may have closed during nack - ignore
1389
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to nack message (channel may be closed): ${nackErr.message}`);
1390
+ }
1391
+ }
1392
+ // Track nack (decrement in-flight)
1393
+ if (tracking) {
1394
+ tracking.inFlight = Math.max(0, tracking.inFlight - 1);
1395
+ tracking.lastCheck = Date.now();
1396
+ }
1039
1397
  }
1040
1398
  };
1041
1399
 
@@ -1062,6 +1420,375 @@ class RabbitMQClient extends EventEmitter {
1062
1420
  }
1063
1421
  }
1064
1422
 
1423
+ /**
1424
+ * Reconnect to RabbitMQ with exponential backoff
1425
+ * CONNECTION-LEVEL RECOVERY: Recreates connection and all channels, re-registers consumers
1426
+ * @private
1427
+ * @returns {Promise<void>}
1428
+ */
1429
+ async _reconnectWithBackoff() {
1430
+ if (this._reconnecting) {
1431
+ console.log('[RabbitMQClient] Reconnection already in progress, skipping');
1432
+ return;
1433
+ }
1434
+
1435
+ this._reconnecting = true;
1436
+ console.log(`[RabbitMQClient] Starting connection-level recovery (attempt ${this._reconnectAttempts + 1}/${this._maxReconnectAttempts})...`);
1437
+
1438
+ while (this._reconnectAttempts < this._maxReconnectAttempts) {
1439
+ try {
1440
+ // Calculate exponential backoff delay: baseDelay * 2^attempts, capped at maxDelay
1441
+ const delay = Math.min(
1442
+ this._reconnectBaseDelay * Math.pow(2, this._reconnectAttempts),
1443
+ this._reconnectMaxDelay
1444
+ );
1445
+
1446
+ console.log(`[RabbitMQClient] Waiting ${delay}ms before reconnection attempt ${this._reconnectAttempts + 1}...`);
1447
+ await new Promise(resolve => setTimeout(resolve, delay));
1448
+
1449
+ // Attempt to reconnect
1450
+ console.log(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts + 1}...`);
1451
+
1452
+ // Close old connection if exists
1453
+ if (this._connection) {
1454
+ try {
1455
+ await this._connection.close();
1456
+ } catch (_) {
1457
+ // Ignore errors when closing already-closed connection
1458
+ }
1459
+ this._connection = null;
1460
+ }
1461
+
1462
+ // Reconnect
1463
+ const rawTarget = this._config.host || this._config.url;
1464
+ const heartbeat = this._config.heartbeat ?? 30;
1465
+ const connectionName =
1466
+ this._config.connectionName ||
1467
+ this._config.clientName ||
1468
+ `oa-client:${process.env.SERVICE_NAME || 'unknown'}:${process.pid}`;
1469
+
1470
+ const connectArgs = typeof rawTarget === 'string'
1471
+ ? [rawTarget, { heartbeat, clientProperties: { connection_name: connectionName } }]
1472
+ : [{ ...rawTarget, heartbeat, clientProperties: { connection_name: connectionName } }];
1473
+
1474
+ const connectPromise = amqp.connect(...connectArgs);
1475
+ const timeoutPromise = new Promise((_, reject) => {
1476
+ setTimeout(() => reject(new Error('Connection timeout after 10 seconds')), 10000);
1477
+ });
1478
+
1479
+ this._connection = await Promise.race([connectPromise, timeoutPromise]);
1480
+ console.log('[RabbitMQClient] ✓ Connection re-established');
1481
+
1482
+ // Re-attach connection event handlers
1483
+ this._connection.on('error', (err) => {
1484
+ console.error('[RabbitMQClient] Connection error:', err.message);
1485
+ this.emit('error', err);
1486
+ });
1487
+
1488
+ this._connection.on('close', () => {
1489
+ console.warn('[RabbitMQClient] Connection closed unexpectedly');
1490
+ this._channel = null;
1491
+ this._queueChannel = null;
1492
+ this._consumerChannel = null;
1493
+ this.emit('error', new Error('RabbitMQ connection closed unexpectedly'));
1494
+
1495
+ // Attempt to reconnect again if enabled
1496
+ if (this._reconnectEnabled && !this._reconnecting) {
1497
+ this._reconnectWithBackoff().catch(err => {
1498
+ console.error('[RabbitMQClient] Reconnection failed after all attempts:', err.message);
1499
+ this.emit('error', err);
1500
+ });
1501
+ }
1502
+ });
1503
+
1504
+ // Recreate all channels
1505
+ console.log('[RabbitMQClient] Recreating channels...');
1506
+
1507
+ // Recreate publisher channel
1508
+ this._channel = await this._connection.createConfirmChannel();
1509
+ this._channel._createdAt = new Date().toISOString();
1510
+ this._channel._closeReason = null;
1511
+ this._channel._lastOperation = 'Recreated via _reconnectWithBackoff()';
1512
+ this._channel._type = 'publisher';
1513
+ this._attachPublisherChannelHandlers(this._channel);
1514
+
1515
+ // Recreate queue channel
1516
+ this._queueChannel = await this._connection.createChannel();
1517
+ this._queueChannel._createdAt = new Date().toISOString();
1518
+ this._queueChannel._closeReason = null;
1519
+ this._queueChannel._lastOperation = 'Recreated via _reconnectWithBackoff()';
1520
+ this._queueChannel._type = 'queue';
1521
+ this._attachQueueChannelHandlers(this._queueChannel);
1522
+
1523
+ // Recreate consumer channel and re-register all consumers
1524
+ this._consumerChannel = await this._connection.createChannel();
1525
+ this._consumerChannel._createdAt = new Date().toISOString();
1526
+ this._consumerChannel._closeReason = null;
1527
+ // IMPORTANT: Don't re-register consumers here - _ensureConsumerChannel() will do it
1528
+ // This prevents duplicate re-registration (once in _reconnectWithBackoff, once in _ensureConsumerChannel)
1529
+ // Just ensure consumer channel is created - it will automatically re-register all active consumers
1530
+ await this._ensureConsumerChannel();
1531
+
1532
+ // Reset reconnect state
1533
+ this._reconnectAttempts = 0;
1534
+ this._reconnecting = false;
1535
+
1536
+ // CRITICAL: Verify all channels are ready before emitting 'reconnected'
1537
+ // This ensures publish/consume operations can proceed immediately after reconnection
1538
+ if (!this._channel || this._channel.closed) {
1539
+ throw new Error('Publisher channel not ready after reconnection');
1540
+ }
1541
+ if (!this._queueChannel || this._queueChannel.closed) {
1542
+ throw new Error('Queue channel not ready after reconnection');
1543
+ }
1544
+ if (!this._consumerChannel || this._consumerChannel.closed) {
1545
+ throw new Error('Consumer channel not ready after reconnection');
1546
+ }
1547
+
1548
+ console.log('[RabbitMQClient] ✓ Connection-level recovery completed successfully - all channels ready');
1549
+ this.emit('reconnected');
1550
+
1551
+ // Flush buffered messages po reconnectu
1552
+ try {
1553
+ const flushResult = await this._publishLayer.flushBuffered();
1554
+ this._publishMonitor.trackFlushed(flushResult.inMemory + flushResult.persistent);
1555
+ console.log(`[RabbitMQClient] ✓ Flushed ${flushResult.inMemory + flushResult.persistent} buffered messages after reconnection`);
1556
+ } catch (flushErr) {
1557
+ console.warn(`[RabbitMQClient] Failed to flush buffered messages after reconnection: ${flushErr.message}`);
1558
+ }
1559
+
1560
+ return; // Success - exit reconnection loop
1561
+ } catch (err) {
1562
+ this._reconnectAttempts++;
1563
+ console.error(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts} failed:`, err.message);
1564
+
1565
+ if (this._reconnectAttempts >= this._maxReconnectAttempts) {
1566
+ console.error(`[RabbitMQClient] ✗ Connection-level recovery failed after ${this._maxReconnectAttempts} attempts`);
1567
+ this._reconnecting = false;
1568
+ throw new Error(`Failed to reconnect after ${this._maxReconnectAttempts} attempts: ${err.message}`);
1569
+ }
1570
+ // Continue to next attempt
1571
+ }
1572
+ }
1573
+ }
1574
+
1575
+ /**
1576
+ * Attach event handlers to publisher channel
1577
+ * @private
1578
+ */
1579
+ _attachPublisherChannelHandlers(channel) {
1580
+ channel.on('error', async (err) => {
1581
+ const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
1582
+ channel._closeReason = reason;
1583
+
1584
+ // "unknown delivery tag" errors occur when channel closes during publish
1585
+ // This is expected - the publish callback will handle retry
1586
+ // We still log it but don't try to recreate channel (it's already closing)
1587
+ if (err.message && err.message.includes('unknown delivery tag')) {
1588
+ console.warn(`[RabbitMQClient] [mq-client-core] Publisher channel error (channel closing during publish): ${err.message}`);
1589
+ // Don't try to recreate - channel is already closing
1590
+ // Don't emit error - publish callback will handle retry
1591
+ return;
1592
+ }
1593
+
1594
+ console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
1595
+ this._callChannelCloseHooks('publisher', channel, err, reason);
1596
+
1597
+ // Only try to recreate if connection is available
1598
+ // If connection is closed, reconnection logic will handle channel recreation
1599
+ if (this._connection && !this._connection.closed) {
1600
+ try {
1601
+ await this._ensurePublisherChannel();
1602
+ } catch (recreateErr) {
1603
+ // Ignore errors during channel recreation if connection is closing
1604
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${recreateErr.message}`);
1605
+ }
1606
+ }
1607
+
1608
+ this.emit('error', err);
1609
+ });
1610
+
1611
+ channel.on('close', async () => {
1612
+ const reason = channel._closeReason || 'Publisher channel closed unexpectedly';
1613
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
1614
+ this._callChannelCloseHooks('publisher', channel, null, reason);
1615
+ const closedChannel = this._channel;
1616
+ this._channel = null;
1617
+
1618
+ // Only try to recreate if connection is available
1619
+ // If connection is closed, reconnection logic will handle channel recreation
1620
+ if (this._connection && !this._connection.closed) {
1621
+ try {
1622
+ await this._ensurePublisherChannel();
1623
+ } catch (err) {
1624
+ // Ignore errors during channel recreation if connection is closing
1625
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${err.message}`);
1626
+ }
1627
+ }
1628
+ });
1629
+
1630
+ channel.on('drain', () => {
1631
+ console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
1632
+ });
1633
+ }
1634
+
1635
+ /**
1636
+ * Attach event handlers to queue channel
1637
+ * @private
1638
+ */
1639
+ _attachQueueChannelHandlers(channel) {
1640
+ channel.on('error', async (err) => {
1641
+ const reason = `Queue channel error: ${err.message} (code: ${err.code || 'unknown'})`;
1642
+ channel._closeReason = reason;
1643
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
1644
+ this._callChannelCloseHooks('queue', channel, err, reason);
1645
+ await this._ensureQueueChannel();
1646
+ });
1647
+
1648
+ channel.on('close', async () => {
1649
+ const reason = channel._closeReason || 'Queue channel closed unexpectedly';
1650
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
1651
+ this._callChannelCloseHooks('queue', channel, null, reason);
1652
+ const closedChannel = this._queueChannel;
1653
+ this._queueChannel = null;
1654
+
1655
+ // PREDICTABLE: Only recreate if connection is available and not closing
1656
+ // If connection is closed, reconnection logic will handle channel recreation
1657
+ if (this._connection && !this._connection.closed && !this._reconnecting) {
1658
+ try {
1659
+ await this._ensureQueueChannel();
1660
+ } catch (err) {
1661
+ // If connection is closing, this is expected - don't throw
1662
+ if (err.message && err.message.includes('Connection closed')) {
1663
+ console.warn(`[RabbitMQClient] [mq-client-core] Queue channel close during connection closure (expected - reconnection will handle)`);
1664
+ } else {
1665
+ // Log but don't throw - channel will be recreated on next use or during reconnection
1666
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel: ${err.message} (will be recreated on next use)`);
1667
+ }
1668
+ }
1669
+ } else {
1670
+ console.warn(`[RabbitMQClient] [mq-client-core] Queue channel closed - will be recreated during reconnection or on next use`);
1671
+ }
1672
+ });
1673
+ }
1674
+
1675
+ /**
1676
+ * Attach event handlers to consumer channel
1677
+ * @private
1678
+ */
1679
+ _attachConsumerChannelHandlers(channel) {
1680
+ channel.on('error', async (err) => {
1681
+ const reason = `Consumer channel error: ${err.message} (code: ${err.code || 'unknown'})`;
1682
+ channel._closeReason = reason;
1683
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
1684
+ this._callChannelCloseHooks('consumer', channel, err, reason);
1685
+ await this._ensureConsumerChannel();
1686
+ this.emit('error', err);
1687
+ });
1688
+
1689
+ channel.on('close', async () => {
1690
+ const reason = channel._closeReason || 'Consumer channel closed unexpectedly';
1691
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
1692
+ this._callChannelCloseHooks('consumer', channel, null, reason);
1693
+ const closedChannel = this._consumerChannel;
1694
+ this._consumerChannel = null;
1695
+
1696
+ // Only try to recreate if connection is available
1697
+ // If connection is closed, reconnection logic will handle channel recreation
1698
+ if (this._connection && !this._connection.closed) {
1699
+ try {
1700
+ await this._ensureConsumerChannel();
1701
+ } catch (err) {
1702
+ // Ignore errors during channel recreation if connection is closing
1703
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel (connection may be closing): ${err.message}`);
1704
+ }
1705
+ }
1706
+ });
1707
+ }
1708
+
1709
+ /**
1710
+ * Check prefetch utilization for a queue and alert if threshold exceeded
1711
+ * @private
1712
+ * @param {string} queue - Queue name
1713
+ * @param {Object} tracking - Prefetch tracking object
1714
+ */
1715
+ _checkPrefetchUtilization(queue, tracking) {
1716
+ if (!tracking || tracking.prefetchCount === 0) {
1717
+ return; // No prefetch set or tracking not available
1718
+ }
1719
+
1720
+ const utilization = tracking.inFlight / tracking.prefetchCount;
1721
+
1722
+ if (utilization >= this._prefetchUtilizationThreshold) {
1723
+ const message = `Prefetch utilization high: queue '${queue}' has ${tracking.inFlight}/${tracking.prefetchCount} messages in-flight (${Math.round(utilization * 100)}%, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`;
1724
+ console.warn(`[RabbitMQClient] [mq-client-core] ⚠️ ${message}`);
1725
+
1726
+ // Call alert callback if provided
1727
+ if (this._prefetchAlertCallback) {
1728
+ try {
1729
+ this._prefetchAlertCallback(queue, utilization, tracking.inFlight, tracking.prefetchCount);
1730
+ } catch (alertErr) {
1731
+ console.error(`[RabbitMQClient] [mq-client-core] Prefetch alert callback error:`, alertErr.message);
1732
+ }
1733
+ }
1734
+
1735
+ // Emit event for external listeners
1736
+ this.emit('prefetch:high-utilization', {
1737
+ queue,
1738
+ utilization,
1739
+ inFlight: tracking.inFlight,
1740
+ prefetchCount: tracking.prefetchCount,
1741
+ threshold: this._prefetchUtilizationThreshold,
1742
+ timestamp: new Date().toISOString()
1743
+ });
1744
+ }
1745
+ }
1746
+
1747
+ /**
1748
+ * Start periodic prefetch monitoring
1749
+ * @private
1750
+ */
1751
+ _startPrefetchMonitoring() {
1752
+ if (this._prefetchCheckTimer) {
1753
+ return; // Already started
1754
+ }
1755
+
1756
+ this._prefetchCheckTimer = setInterval(() => {
1757
+ const now = Date.now();
1758
+ for (const [queue, tracking] of this._prefetchTracking.entries()) {
1759
+ // Only check if we have recent activity (within last 30s)
1760
+ if (now - tracking.lastCheck < 30000) {
1761
+ this._checkPrefetchUtilization(queue, tracking);
1762
+ }
1763
+ }
1764
+ }, this._prefetchCheckInterval);
1765
+
1766
+ console.log(`[RabbitMQClient] [mq-client-core] Prefetch monitoring started (interval: ${this._prefetchCheckInterval}ms, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`);
1767
+ }
1768
+
1769
+ /**
1770
+ * Stop prefetch monitoring
1771
+ * @private
1772
+ */
1773
+ _stopPrefetchMonitoring() {
1774
+ if (this._prefetchCheckTimer) {
1775
+ clearInterval(this._prefetchCheckTimer);
1776
+ this._prefetchCheckTimer = null;
1777
+ }
1778
+ }
1779
+
1780
+ /**
1781
+ * Track channel operation for debugging
1782
+ * @private
1783
+ * @param {Object} channel - Channel object
1784
+ * @param {string} operation - Operation description
1785
+ */
1786
+ _trackChannelOperation(channel, operation) {
1787
+ if (channel && typeof channel === 'object') {
1788
+ channel._lastOperation = operation;
1789
+ }
1790
+ }
1791
+
1065
1792
  /**
1066
1793
  * Acknowledges a message.
1067
1794
  * @param {Object} msg - RabbitMQ message object.