@onlineapps/mq-client-core 1.0.36 → 1.0.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@onlineapps/mq-client-core",
3
- "version": "1.0.36",
3
+ "version": "1.0.37",
4
4
  "description": "Core MQ client library for RabbitMQ - shared by infrastructure services and connectors",
5
5
  "main": "src/index.js",
6
6
  "scripts": {
@@ -39,13 +39,44 @@ class RabbitMQClient extends EventEmitter {
39
39
  // Track active consumers for re-registration after channel recreation
40
40
  this._activeConsumers = new Map(); // queue -> { handler, options, consumerTag }
41
41
 
42
- // Auto-reconnect flags
42
+ // Connection-level recovery
43
43
  this._reconnecting = false;
44
44
  this._reconnectAttempts = 0;
45
+ this._maxReconnectAttempts = this._config.maxReconnectAttempts || 10; // Max 10 attempts
46
+ this._reconnectBaseDelay = this._config.reconnectBaseDelay || 1000; // Start with 1 second
47
+ this._reconnectMaxDelay = this._config.reconnectMaxDelay || 30000; // Max 30 seconds
48
+ this._reconnectEnabled = this._config.reconnectEnabled !== false; // Default: enabled
49
+ this._reconnectTimer = null;
50
+
51
+ // Publisher retry configuration
52
+ this._publishRetryEnabled = this._config.publishRetryEnabled !== false; // Default: enabled
53
+ this._publishMaxRetries = this._config.publishMaxRetries || 3; // Default: 3 attempts
54
+ this._publishRetryBaseDelay = this._config.publishRetryBaseDelay || 100; // Default: 100ms
55
+ this._publishRetryMaxDelay = this._config.publishRetryMaxDelay || 5000; // Default: 5s
56
+ this._publishRetryBackoffMultiplier = this._config.publishRetryBackoffMultiplier || 2; // Default: 2 (exponential)
57
+ this._publishConfirmationTimeout = this._config.publishConfirmationTimeout || 5000; // Default: 5s per attempt
45
58
 
46
59
  // Channel close hooks - called when channel closes with detailed information
47
60
  this._channelCloseHooks = []; // Array of { type: 'publisher'|'queue'|'consumer', callback: (details) => void }
48
61
 
62
+ // Channel thrashing detection - track close frequency per channel type
63
+ this._channelCloseHistory = {
64
+ publisher: [], // Array of { timestamp: number, reason: string }
65
+ queue: [],
66
+ consumer: []
67
+ };
68
+ this._thrashingThreshold = this._config.thrashingThreshold || 5; // Max closes per minute
69
+ this._thrashingWindowMs = this._config.thrashingWindowMs || 60000; // 1 minute window
70
+ this._thrashingAlertCallback = this._config.thrashingAlertCallback || null; // (type, count, windowMs) => void
71
+ this._thrashingAlertsSent = new Set(); // Track which alerts we've already sent to avoid spam
72
+
73
+ // Prefetch monitoring - track utilization per queue
74
+ this._prefetchTracking = new Map(); // queue -> { prefetchCount: number, inFlight: number, lastCheck: number }
75
+ this._prefetchUtilizationThreshold = this._config.prefetchUtilizationThreshold || 0.8; // 80% threshold
76
+ this._prefetchCheckInterval = this._config.prefetchCheckInterval || 10000; // Check every 10s
77
+ this._prefetchAlertCallback = this._config.prefetchAlertCallback || null; // (queue, utilization, inFlight, prefetchCount) => void
78
+ this._prefetchCheckTimer = null;
79
+
49
80
  // Health monitoring
50
81
  this._healthCheckInterval = null;
51
82
  this._healthCheckIntervalMs = this._config.healthCheckInterval || 30000; // Default 30s
@@ -91,6 +122,76 @@ class RabbitMQClient extends EventEmitter {
91
122
  this._channelCloseHooks.push({ type, callback });
92
123
  }
93
124
 
125
+ /**
126
+ * Track channel close event for thrashing detection
127
+ * @private
128
+ * @param {string} channelType - 'publisher', 'queue', or 'consumer'
129
+ * @param {string} reason - Human-readable reason
130
+ */
131
+ _trackChannelClose(channelType, reason) {
132
+ const now = Date.now();
133
+ const history = this._channelCloseHistory[channelType];
134
+
135
+ if (!history) {
136
+ console.warn(`[RabbitMQClient] Unknown channel type for thrashing tracking: ${channelType}`);
137
+ return;
138
+ }
139
+
140
+ // Add new close event
141
+ history.push({
142
+ timestamp: now,
143
+ reason: reason || 'unknown'
144
+ });
145
+
146
+ // Remove old events outside the window
147
+ const cutoff = now - this._thrashingWindowMs;
148
+ while (history.length > 0 && history[0].timestamp < cutoff) {
149
+ history.shift();
150
+ }
151
+
152
+ // Check for thrashing
153
+ if (history.length >= this._thrashingThreshold) {
154
+ const alertKey = `${channelType}:${Math.floor(now / this._thrashingWindowMs)}`;
155
+
156
+ // Only alert once per window to avoid spam
157
+ if (!this._thrashingAlertsSent.has(alertKey)) {
158
+ this._thrashingAlertsSent.add(alertKey);
159
+
160
+ // Clean up old alert keys (keep only last 10 windows)
161
+ if (this._thrashingAlertsSent.size > 10) {
162
+ const oldestKey = Array.from(this._thrashingAlertsSent).sort()[0];
163
+ this._thrashingAlertsSent.delete(oldestKey);
164
+ }
165
+
166
+ const message = `Channel thrashing detected: ${channelType} channel closed ${history.length} times in ${Math.round(this._thrashingWindowMs / 1000)}s (threshold: ${this._thrashingThreshold})`;
167
+ console.error(`[RabbitMQClient] [mq-client-core] 🚨 ${message}`);
168
+
169
+ // Call alert callback if provided
170
+ if (this._thrashingAlertCallback) {
171
+ try {
172
+ this._thrashingAlertCallback(channelType, history.length, this._thrashingWindowMs, {
173
+ recentCloses: history.slice(-this._thrashingThreshold),
174
+ threshold: this._thrashingThreshold,
175
+ windowMs: this._thrashingWindowMs
176
+ });
177
+ } catch (alertErr) {
178
+ console.error(`[RabbitMQClient] [mq-client-core] Thrashing alert callback error:`, alertErr.message);
179
+ }
180
+ }
181
+
182
+ // Emit event for external listeners
183
+ this.emit('channel:thrashing', {
184
+ type: channelType,
185
+ count: history.length,
186
+ windowMs: this._thrashingWindowMs,
187
+ threshold: this._thrashingThreshold,
188
+ recentCloses: history.slice(-this._thrashingThreshold),
189
+ timestamp: new Date().toISOString()
190
+ });
191
+ }
192
+ }
193
+ }
194
+
94
195
  /**
95
196
  * Call all registered channel close hooks
96
197
  * @private
@@ -100,6 +201,9 @@ class RabbitMQClient extends EventEmitter {
100
201
  * @param {string} reason - Human-readable reason
101
202
  */
102
203
  _callChannelCloseHooks(channelType, channel, error, reason) {
204
+ // Track channel close for thrashing detection
205
+ this._trackChannelClose(channelType, reason);
206
+
103
207
  const details = {
104
208
  type: channelType,
105
209
  channel: channel,
@@ -152,10 +256,40 @@ class RabbitMQClient extends EventEmitter {
152
256
  console.log('[RabbitMQClient] Starting connection race...');
153
257
  this._connection = await Promise.race([connectPromise, timeoutPromise]);
154
258
  console.log('[RabbitMQClient] Connection established');
155
- this._connection.on('error', (err) => this.emit('error', err));
259
+ this._reconnectAttempts = 0; // Reset reconnect attempts on successful connection
260
+
261
+ this._connection.on('error', (err) => {
262
+ console.error('[RabbitMQClient] Connection error:', err.message);
263
+ this.emit('error', err);
264
+ // Connection errors may lead to close event - let close handler handle reconnection
265
+ });
266
+
156
267
  this._connection.on('close', () => {
157
- // Emit a connection close error to notify listeners
268
+ console.warn('[RabbitMQClient] Connection closed unexpectedly - close handler STARTED');
269
+ console.log(`[RabbitMQClient] Close handler: reconnectEnabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
270
+
271
+ // Mark all channels as closed
272
+ this._channel = null;
273
+ this._queueChannel = null;
274
+ this._consumerChannel = null;
275
+
276
+ // CONNECTION-LEVEL RECOVERY: Attempt to reconnect with exponential backoff
277
+ // PREDICTABLE: Check conditions and log why reconnect is/isn't starting
278
+ // IMPORTANT: Do this BEFORE emitting error, so reconnect can start even if error handler throws
279
+ if (this._reconnectEnabled && !this._reconnecting) {
280
+ console.log('[RabbitMQClient] ✓ Conditions met - starting connection-level recovery from initial connect() close handler');
281
+ this._reconnectWithBackoff().catch(err => {
282
+ console.error('[RabbitMQClient] Reconnection failed after all attempts:', err.message);
283
+ this.emit('error', err);
284
+ });
285
+ } else {
286
+ console.warn(`[RabbitMQClient] ✗ Reconnect NOT starting: enabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
287
+ }
288
+
289
+ // Emit connection close event AFTER starting reconnect (so reconnect isn't blocked by error handler)
158
290
  this.emit('error', new Error('RabbitMQ connection closed unexpectedly'));
291
+
292
+ console.log('[RabbitMQClient] Close handler FINISHED');
159
293
  });
160
294
 
161
295
  // Use ConfirmChannel to enable publisher confirms for publish operations
@@ -167,37 +301,8 @@ class RabbitMQClient extends EventEmitter {
167
301
  this._channel._lastOperation = 'Initial creation';
168
302
  this._channel._type = 'publisher';
169
303
 
170
- // Enable publisher confirms explicitly
171
- this._channel.on('error', async (err) => {
172
- const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
173
- this._channel._closeReason = reason;
174
- console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
175
-
176
- // Call close hooks with detailed information
177
- this._callChannelCloseHooks('publisher', this._channel, err, reason);
178
-
179
- // Auto-recreate channel if connection is still alive
180
- await this._ensurePublisherChannel();
181
- this.emit('error', err);
182
- });
183
- this._channel.on('close', async () => {
184
- const reason = this._channel._closeReason || 'Publisher channel closed unexpectedly';
185
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
186
-
187
- // Call close hooks with detailed information
188
- this._callChannelCloseHooks('publisher', this._channel, null, reason);
189
-
190
- // Mark channel as null - will be recreated on next publish
191
- const closedChannel = this._channel;
192
- this._channel = null;
193
-
194
- // Try to recreate immediately if connection is alive
195
- await this._ensurePublisherChannel();
196
- });
197
- // Set up publisher confirms callback
198
- this._channel.on('drain', () => {
199
- console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
200
- });
304
+ // Attach event handlers
305
+ this._attachPublisherChannelHandlers(this._channel);
201
306
 
202
307
  // Create a separate regular channel for queue operations (assertQueue, checkQueue)
203
308
  // This avoids RPC reply queue issues with ConfirmChannel.assertQueue()
@@ -209,31 +314,8 @@ class RabbitMQClient extends EventEmitter {
209
314
  this._queueChannel._lastOperation = 'Initial creation';
210
315
  this._queueChannel._type = 'queue';
211
316
 
212
- this._queueChannel.on('error', async (err) => {
213
- const reason = `Queue channel error: ${err.message} (code: ${err.code || 'unknown'})`;
214
- this._queueChannel._closeReason = reason;
215
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
216
-
217
- // Call close hooks with detailed information
218
- this._callChannelCloseHooks('queue', this._queueChannel, err, reason);
219
-
220
- // Auto-recreate channel if connection is still alive
221
- await this._ensureQueueChannel();
222
- });
223
- this._queueChannel.on('close', async () => {
224
- const reason = this._queueChannel._closeReason || 'Queue channel closed unexpectedly';
225
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
226
-
227
- // Call close hooks with detailed information
228
- this._callChannelCloseHooks('queue', this._queueChannel, null, reason);
229
-
230
- // Mark channel as null - will be recreated on next queue operation
231
- const closedChannel = this._queueChannel;
232
- this._queueChannel = null;
233
-
234
- // Try to recreate immediately if connection is alive
235
- await this._ensureQueueChannel();
236
- });
317
+ // Attach event handlers
318
+ this._attachQueueChannelHandlers(this._queueChannel);
237
319
 
238
320
  // Create a dedicated channel for consume operations
239
321
  // This prevents channel conflicts between publish (ConfirmChannel) and consume operations
@@ -245,32 +327,8 @@ class RabbitMQClient extends EventEmitter {
245
327
  this._consumerChannel._lastOperation = 'Initial creation';
246
328
  this._consumerChannel._type = 'consumer';
247
329
 
248
- this._consumerChannel.on('error', async (err) => {
249
- const reason = `Consumer channel error: ${err.message} (code: ${err.code || 'unknown'})`;
250
- this._consumerChannel._closeReason = reason;
251
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
252
-
253
- // Call close hooks with detailed information
254
- this._callChannelCloseHooks('consumer', this._consumerChannel, err, reason);
255
-
256
- // Auto-recreate channel and re-register consumers
257
- await this._ensureConsumerChannel();
258
- this.emit('error', err);
259
- });
260
- this._consumerChannel.on('close', async () => {
261
- const reason = this._consumerChannel._closeReason || 'Consumer channel closed unexpectedly';
262
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
263
-
264
- // Call close hooks with detailed information
265
- this._callChannelCloseHooks('consumer', this._consumerChannel, null, reason);
266
-
267
- // Mark channel as null - will be recreated and consumers re-registered
268
- const closedChannel = this._consumerChannel;
269
- this._consumerChannel = null;
270
-
271
- // Try to recreate and re-register consumers immediately if connection is alive
272
- await this._ensureConsumerChannel();
273
- });
330
+ // Attach event handlers
331
+ this._attachConsumerChannelHandlers(this._consumerChannel);
274
332
 
275
333
  // Start health monitoring if enabled
276
334
  if (this._healthCheckEnabled) {
@@ -290,6 +348,50 @@ class RabbitMQClient extends EventEmitter {
290
348
  }
291
349
  }
292
350
 
351
+ /**
352
+ * Wait for reconnection to complete
353
+ * Used by both retry mechanism and channel creation
354
+ * @private
355
+ * @returns {Promise<void>}
356
+ * @throws {Error} If reconnection fails or times out
357
+ */
358
+ async _waitForReconnection() {
359
+ if (!this._reconnecting) {
360
+ throw new Error('Cannot wait for reconnection: not currently reconnecting');
361
+ }
362
+
363
+ // Wait for 'reconnected' event
364
+ return new Promise((resolve, reject) => {
365
+ const timeout = setTimeout(() => {
366
+ this.removeListener('reconnected', onReconnected);
367
+ this.removeListener('error', onError);
368
+ reject(new Error('Reconnection timeout after 30 seconds'));
369
+ }, 30000); // 30 seconds timeout
370
+
371
+ const onReconnected = () => {
372
+ clearTimeout(timeout);
373
+ this.removeListener('reconnected', onReconnected);
374
+ this.removeListener('error', onError);
375
+ // Wait a bit for channels to be fully recreated
376
+ setTimeout(resolve, 500);
377
+ };
378
+
379
+ const onError = (error) => {
380
+ // Ignore connection errors during reconnection
381
+ if (error.message && error.message.includes('Connection closed unexpectedly')) {
382
+ return; // Expected during reconnection
383
+ }
384
+ clearTimeout(timeout);
385
+ this.removeListener('reconnected', onReconnected);
386
+ this.removeListener('error', onError);
387
+ reject(error);
388
+ };
389
+
390
+ this.once('reconnected', onReconnected);
391
+ this.on('error', onError);
392
+ });
393
+ }
394
+
293
395
  /**
294
396
  * Ensure publisher channel exists and is open
295
397
  * Auto-recreates if closed
@@ -302,7 +404,25 @@ class RabbitMQClient extends EventEmitter {
302
404
  }
303
405
 
304
406
  if (!this._connection || this._connection.closed) {
305
- throw new Error('Cannot recreate publisher channel: connection is closed');
407
+ // Connection is closed - wait for reconnection
408
+ console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate publisher channel: connection is closed (waiting for reconnection)');
409
+
410
+ // Wait for reconnection to complete
411
+ if (this._reconnecting) {
412
+ await this._waitForReconnection();
413
+
414
+ // After reconnection, try again
415
+ if (this._channel && !this._channel.closed) {
416
+ return; // Channel was recreated during reconnection
417
+ }
418
+ // If still no channel, connection might have failed - check again
419
+ if (!this._connection || this._connection.closed) {
420
+ throw new Error('Connection reconnection failed - cannot create publisher channel');
421
+ }
422
+ } else {
423
+ // Not reconnecting - connection is closed and not recovering
424
+ throw new Error('Connection is closed and not reconnecting - cannot create publisher channel');
425
+ }
306
426
  }
307
427
 
308
428
  try {
@@ -324,31 +444,8 @@ class RabbitMQClient extends EventEmitter {
324
444
  this._channel._lastOperation = 'Recreated via _ensurePublisherChannel()';
325
445
  this._channel._type = 'publisher';
326
446
 
327
- this._channel.on('error', async (err) => {
328
- const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
329
- this._channel._closeReason = reason;
330
- console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
331
-
332
- // Call close hooks
333
- this._callChannelCloseHooks('publisher', this._channel, err, reason);
334
-
335
- await this._ensurePublisherChannel();
336
- this.emit('error', err);
337
- });
338
- this._channel.on('close', async () => {
339
- const reason = this._channel._closeReason || 'Publisher channel closed unexpectedly';
340
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
341
-
342
- // Call close hooks
343
- this._callChannelCloseHooks('publisher', this._channel, null, reason);
344
-
345
- const closedChannel = this._channel;
346
- this._channel = null;
347
- await this._ensurePublisherChannel();
348
- });
349
- this._channel.on('drain', () => {
350
- console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
351
- });
447
+ // Attach event handlers
448
+ this._attachPublisherChannelHandlers(this._channel);
352
449
 
353
450
  console.log('[RabbitMQClient] [mq-client-core] ✓ Publisher channel recreated');
354
451
  } catch (err) {
@@ -369,7 +466,11 @@ class RabbitMQClient extends EventEmitter {
369
466
  }
370
467
 
371
468
  if (!this._connection || this._connection.closed) {
372
- throw new Error('Cannot recreate queue channel: connection is closed');
469
+ // Connection is closed - cannot recreate channel
470
+ // Reconnection logic will handle this - don't throw, just return
471
+ // The channel will be recreated after reconnection completes
472
+ console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate queue channel: connection is closed (reconnection in progress)');
473
+ return;
373
474
  }
374
475
 
375
476
  try {
@@ -391,32 +492,18 @@ class RabbitMQClient extends EventEmitter {
391
492
  this._queueChannel._lastOperation = 'Recreated via _ensureQueueChannel()';
392
493
  this._queueChannel._type = 'queue';
393
494
 
394
- this._queueChannel.on('error', async (err) => {
395
- const reason = `Queue channel error: ${err.message} (code: ${err.code || 'unknown'})`;
396
- this._queueChannel._closeReason = reason;
397
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
398
-
399
- // Call close hooks
400
- this._callChannelCloseHooks('queue', this._queueChannel, err, reason);
401
-
402
- await this._ensureQueueChannel();
403
- });
404
- this._queueChannel.on('close', async () => {
405
- const reason = this._queueChannel._closeReason || 'Queue channel closed unexpectedly';
406
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
407
-
408
- // Call close hooks
409
- this._callChannelCloseHooks('queue', this._queueChannel, null, reason);
410
-
411
- const closedChannel = this._queueChannel;
412
- this._queueChannel = null;
413
- await this._ensureQueueChannel();
414
- });
495
+ // Attach event handlers
496
+ this._attachQueueChannelHandlers(this._queueChannel);
415
497
 
416
498
  console.log('[RabbitMQClient] [mq-client-core] ✓ Queue channel recreated');
417
499
  } catch (err) {
418
500
  this._queueChannel = null;
419
- throw new Error(`Failed to recreate queue channel: ${err.message}`);
501
+ // If connection is closed, don't throw - reconnection will handle it
502
+ if (this._connection && !this._connection.closed) {
503
+ throw new Error(`Failed to recreate queue channel: ${err.message}`);
504
+ }
505
+ // Connection is closed - reconnection will recreate channel
506
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel (connection closed): ${err.message}`);
420
507
  }
421
508
  }
422
509
 
@@ -432,7 +519,11 @@ class RabbitMQClient extends EventEmitter {
432
519
  }
433
520
 
434
521
  if (!this._connection || this._connection.closed) {
435
- throw new Error('Cannot recreate consumer channel: connection is closed');
522
+ // Connection is closed - cannot recreate channel
523
+ // Reconnection logic will handle this - don't throw, just return
524
+ // The channel will be recreated after reconnection completes
525
+ console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate consumer channel: connection is closed (reconnection in progress)');
526
+ return;
436
527
  }
437
528
 
438
529
  try {
@@ -454,28 +545,8 @@ class RabbitMQClient extends EventEmitter {
454
545
  this._consumerChannel._lastOperation = 'Recreated via _ensureConsumerChannel()';
455
546
  this._consumerChannel._type = 'consumer';
456
547
 
457
- this._consumerChannel.on('error', async (err) => {
458
- const reason = `Consumer channel error: ${err.message} (code: ${err.code || 'unknown'})`;
459
- this._consumerChannel._closeReason = reason;
460
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
461
-
462
- // Call close hooks
463
- this._callChannelCloseHooks('consumer', this._consumerChannel, err, reason);
464
-
465
- await this._ensureConsumerChannel();
466
- this.emit('error', err);
467
- });
468
- this._consumerChannel.on('close', async () => {
469
- const reason = this._consumerChannel._closeReason || 'Consumer channel closed unexpectedly';
470
- console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
471
-
472
- // Call close hooks
473
- this._callChannelCloseHooks('consumer', this._consumerChannel, null, reason);
474
-
475
- const closedChannel = this._consumerChannel;
476
- this._consumerChannel = null;
477
- await this._ensureConsumerChannel();
478
- });
548
+ // Attach event handlers
549
+ this._attachConsumerChannelHandlers(this._consumerChannel);
479
550
 
480
551
  // Re-register all active consumers
481
552
  if (this._activeConsumers.size > 0) {
@@ -737,9 +808,15 @@ class RabbitMQClient extends EventEmitter {
737
808
  // Stop health monitoring
738
809
  this._stopHealthMonitoring();
739
810
 
811
+ // Stop prefetch monitoring
812
+ this._stopPrefetchMonitoring();
813
+
740
814
  // Clear active consumers tracking
741
815
  this._activeConsumers.clear();
742
816
 
817
+ // Clear prefetch tracking
818
+ this._prefetchTracking.clear();
819
+
743
820
  try {
744
821
  if (this._consumerChannel) {
745
822
  await this._consumerChannel.close();
@@ -771,19 +848,39 @@ class RabbitMQClient extends EventEmitter {
771
848
  this._connection = null;
772
849
  }
773
850
  } catch (err) {
774
- this.emit('error', err);
851
+ // Don't emit error for expected connection closure during disconnect
852
+ // This is a predictable, intentional closure
853
+ if (err.message && err.message.includes('Connection closed (by client)')) {
854
+ console.log('[RabbitMQClient] Connection closed during disconnect (expected)');
855
+ } else {
856
+ this.emit('error', err);
857
+ }
775
858
  }
776
859
  }
777
860
 
778
861
  /**
779
862
  * Publishes a message buffer to the specified queue (or default queue) or exchange.
863
+ * Implements systematic retry with exponential backoff for transient failures.
780
864
  * @param {string} queue - Target queue name.
781
865
  * @param {Buffer} buffer - Message payload as Buffer.
782
866
  * @param {Object} [options] - Overrides: routingKey, persistent, headers.
783
867
  * @returns {Promise<void>}
784
- * @throws {Error} If publish fails or channel is not available.
868
+ * @throws {Error} If publish fails after all retry attempts.
785
869
  */
786
870
  async publish(queue, buffer, options = {}) {
871
+ // Wrap in retry logic if enabled
872
+ if (this._publishRetryEnabled) {
873
+ return await this._publishWithRetry(queue, buffer, options);
874
+ } else {
875
+ return await this._publishOnce(queue, buffer, options);
876
+ }
877
+ }
878
+
879
+ /**
880
+ * Internal publish implementation - single attempt without retry
881
+ * @private
882
+ */
883
+ async _publishOnce(queue, buffer, options = {}) {
787
884
  // Ensure publisher channel exists and is open (auto-recreates if closed)
788
885
  await this._ensurePublisherChannel();
789
886
 
@@ -849,16 +946,103 @@ class RabbitMQClient extends EventEmitter {
849
946
 
850
947
  // Use callback-based confirmation - kanály jsou spolehlivé, takže callback vždy dorazí
851
948
  const confirmPromise = new Promise((resolve, reject) => {
949
+ // Set timeout for publish confirmation (configurable)
950
+ const timeout = setTimeout(() => {
951
+ reject(new Error(`Publish confirmation timeout for queue "${queue}" after ${this._publishConfirmationTimeout}ms`));
952
+ }, this._publishConfirmationTimeout);
953
+
954
+ // Check if channel is still valid before sending
955
+ if (!this._channel || this._channel.closed) {
956
+ clearTimeout(timeout);
957
+ reject(new Error(`Cannot publish: channel is closed for queue "${queue}"`));
958
+ return;
959
+ }
960
+
961
+ // Track original channel to detect if it was closed/recreated during publish
962
+ const originalChannel = this._channel;
963
+ const channelId = originalChannel ? originalChannel._createdAt : null;
964
+ let callbackInvoked = false;
965
+
852
966
  // Send message with callback
853
- this._channel.sendToQueue(queue, buffer, { persistent, headers }, (err, ok) => {
854
- if (err) {
855
- console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
856
- reject(err);
967
+ try {
968
+ originalChannel.sendToQueue(queue, buffer, { persistent, headers }, (err, ok) => {
969
+ callbackInvoked = true;
970
+ clearTimeout(timeout);
971
+
972
+ // CRITICAL: Check if channel was closed or recreated during publish
973
+ // If channel was recreated, the delivery tag is invalid - ignore this callback
974
+ if (!this._channel || this._channel.closed || this._channel !== originalChannel ||
975
+ (this._channel._createdAt && this._channel._createdAt !== channelId)) {
976
+ // Channel was closed or recreated - delivery tag is invalid
977
+ console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed/recreated during publish to queue "${queue}", ignoring callback`);
978
+ // If we're reconnecting, wait and retry
979
+ if (this._reconnecting) {
980
+ this._waitForReconnection().then(() => {
981
+ // Retry publish after reconnection
982
+ return this.publish(queue, buffer, options);
983
+ }).then(resolve).catch(reject);
984
+ } else {
985
+ // Not reconnecting - this is a real error
986
+ // We cannot guarantee the message was delivered - must fail
987
+ reject(new Error(`Channel closed during publish to queue "${queue}" and not reconnecting - message delivery not guaranteed`));
988
+ }
989
+ return;
990
+ }
991
+
992
+ if (err) {
993
+ // Check if error is about invalid delivery tag (channel was closed)
994
+ if (err.message && (err.message.includes('unknown delivery tag') || err.message.includes('PRECONDITION_FAILED'))) {
995
+ console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Delivery tag invalid (channel may have been closed) for queue "${queue}"`);
996
+ // If we're reconnecting, wait and retry
997
+ if (this._reconnecting) {
998
+ this._waitForReconnection().then(() => {
999
+ // Retry publish after reconnection
1000
+ return this.publish(queue, buffer, options);
1001
+ }).then(resolve).catch(reject);
1002
+ } else {
1003
+ // Not reconnecting - delivery tag is invalid, message delivery not guaranteed
1004
+ reject(new Error(`Delivery tag invalid for queue "${queue}" (channel closed) and not reconnecting - message delivery not guaranteed`));
1005
+ }
1006
+ } else {
1007
+ console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
1008
+ reject(err);
1009
+ }
1010
+ } else {
1011
+ console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Message confirmed for queue "${queue}"`);
1012
+ resolve();
1013
+ }
1014
+ });
1015
+
1016
+ // Set a safety timeout - if callback wasn't invoked and channel closed, retry
1017
+ setTimeout(() => {
1018
+ if (!callbackInvoked && (!this._channel || this._channel.closed || this._channel !== originalChannel)) {
1019
+ console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Callback timeout and channel closed for queue "${queue}", will retry after reconnection`);
1020
+ if (this._reconnecting) {
1021
+ clearTimeout(timeout);
1022
+ this._waitForReconnection().then(() => {
1023
+ return this.publish(queue, buffer, options);
1024
+ }).then(resolve).catch(reject);
1025
+ }
1026
+ }
1027
+ }, 1000);
1028
+ } catch (sendErr) {
1029
+ clearTimeout(timeout);
1030
+ // If channel is closed, try to wait for reconnection and retry
1031
+ if (sendErr.message && (sendErr.message.includes('Channel ended') || sendErr.message.includes('Channel closed'))) {
1032
+ console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during sendToQueue, will retry after reconnection`);
1033
+ // Wait for reconnection and retry
1034
+ if (this._reconnecting) {
1035
+ this._waitForReconnection().then(() => {
1036
+ // Retry publish after reconnection
1037
+ return this.publish(queue, buffer, options);
1038
+ }).then(resolve).catch(reject);
1039
+ } else {
1040
+ reject(sendErr);
1041
+ }
857
1042
  } else {
858
- console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Message confirmed for queue "${queue}"`);
859
- resolve();
1043
+ reject(sendErr);
860
1044
  }
861
- });
1045
+ }
862
1046
  });
863
1047
 
864
1048
  await confirmPromise;
@@ -869,7 +1053,13 @@ class RabbitMQClient extends EventEmitter {
869
1053
 
870
1054
  // Use callback-based confirmation - kanály jsou spolehlivé
871
1055
  const confirmPromise = new Promise((resolve, reject) => {
1056
+ // Set timeout for exchange publish confirmation
1057
+ const timeout = setTimeout(() => {
1058
+ reject(new Error(`Exchange publish confirmation timeout for exchange "${exchange}" after ${this._publishConfirmationTimeout}ms`));
1059
+ }, this._publishConfirmationTimeout);
1060
+
872
1061
  this._channel.publish(exchange, routingKey, buffer, { persistent, headers }, (err, ok) => {
1062
+ clearTimeout(timeout);
873
1063
  if (err) {
874
1064
  console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Exchange publish callback error:`, err.message);
875
1065
  reject(err);
@@ -900,6 +1090,182 @@ class RabbitMQClient extends EventEmitter {
900
1090
  }
901
1091
  }
902
1092
 
1093
+ /**
1094
+ * Publish with systematic retry and exponential backoff
1095
+ * @private
1096
+ * @param {string} queue - Target queue name
1097
+ * @param {Buffer} buffer - Message payload
1098
+ * @param {Object} options - Publish options
1099
+ * @returns {Promise<void>}
1100
+ */
1101
+ async _publishWithRetry(queue, buffer, options = {}) {
1102
+ let lastError = null;
1103
+ let attempt = 0;
1104
+
1105
+ while (attempt < this._publishMaxRetries) {
1106
+ attempt++;
1107
+
1108
+ try {
1109
+ // Emit retry attempt event for monitoring
1110
+ if (attempt > 1) {
1111
+ this.emit('publish:retry', {
1112
+ queue,
1113
+ attempt,
1114
+ maxRetries: this._publishMaxRetries,
1115
+ lastError: lastError?.message
1116
+ });
1117
+ console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] Retry attempt ${attempt}/${this._publishMaxRetries} for queue "${queue}"`);
1118
+ }
1119
+
1120
+ // Attempt publish
1121
+ await this._publishOnce(queue, buffer, options);
1122
+
1123
+ // Success - emit success event if this was a retry
1124
+ if (attempt > 1) {
1125
+ this.emit('publish:success', {
1126
+ queue,
1127
+ attempt,
1128
+ totalAttempts: attempt
1129
+ });
1130
+ console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Published successfully after ${attempt} attempts for queue "${queue}"`);
1131
+ }
1132
+
1133
+ return; // Success - exit retry loop
1134
+
1135
+ } catch (err) {
1136
+ lastError = err;
1137
+
1138
+ // Check if error is retryable
1139
+ const isRetryable = this._isPublishErrorRetryable(err);
1140
+
1141
+ if (!isRetryable) {
1142
+ // Non-retryable error - fail immediately
1143
+ console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Non-retryable error for queue "${queue}":`, err.message);
1144
+ this.emit('publish:failed', {
1145
+ queue,
1146
+ attempt,
1147
+ error: err.message,
1148
+ retryable: false
1149
+ });
1150
+ throw err;
1151
+ }
1152
+
1153
+ // Retryable error - check if we have more attempts
1154
+ if (attempt >= this._publishMaxRetries) {
1155
+ // Max retries exceeded
1156
+ console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✗ Failed after ${attempt} attempts for queue "${queue}":`, err.message);
1157
+ this.emit('publish:failed', {
1158
+ queue,
1159
+ attempt,
1160
+ error: err.message,
1161
+ retryable: true,
1162
+ maxRetriesExceeded: true
1163
+ });
1164
+ throw new Error(`Publish failed after ${attempt} attempts for queue "${queue}": ${err.message}`);
1165
+ }
1166
+
1167
+ // CRITICAL: If reconnecting, wait for reconnection to complete before retry
1168
+ // This ensures we don't waste retry attempts while connection is being restored
1169
+ if (this._reconnecting) {
1170
+ console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] Connection is reconnecting, waiting for reconnection before retry ${attempt + 1}/${this._publishMaxRetries}...`);
1171
+ try {
1172
+ await this._waitForReconnection();
1173
+ console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Reconnection completed, proceeding with retry ${attempt + 1}/${this._publishMaxRetries}`);
1174
+ // Continue to retry immediately after reconnection (no additional delay needed)
1175
+ continue;
1176
+ } catch (reconnectErr) {
1177
+ // Reconnection failed - this is a critical error
1178
+ console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Reconnection failed:`, reconnectErr.message);
1179
+ this.emit('publish:failed', {
1180
+ queue,
1181
+ attempt,
1182
+ error: `Reconnection failed: ${reconnectErr.message}`,
1183
+ retryable: false,
1184
+ reconnectFailed: true
1185
+ });
1186
+ throw new Error(`Publish failed: reconnection failed after ${attempt} attempts for queue "${queue}": ${reconnectErr.message}`);
1187
+ }
1188
+ }
1189
+
1190
+ // Calculate exponential backoff delay (only if not reconnecting)
1191
+ const delay = Math.min(
1192
+ this._publishRetryBaseDelay * Math.pow(this._publishRetryBackoffMultiplier, attempt - 1),
1193
+ this._publishRetryMaxDelay
1194
+ );
1195
+
1196
+ console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Retryable error for queue "${queue}" (attempt ${attempt}/${this._publishMaxRetries}), waiting ${delay}ms before retry:`, err.message);
1197
+
1198
+ // Wait before retry
1199
+ await new Promise(resolve => setTimeout(resolve, delay));
1200
+ }
1201
+ }
1202
+
1203
+ // Should never reach here, but just in case
1204
+ throw lastError || new Error(`Publish failed for queue "${queue}" after ${attempt} attempts`);
1205
+ }
1206
+
1207
+ /**
1208
+ * Determine if a publish error is retryable
1209
+ * @private
1210
+ * @param {Error} err - Error to check
1211
+ * @returns {boolean} True if error is retryable
1212
+ */
1213
+ _isPublishErrorRetryable(err) {
1214
+ if (!err || !err.message) {
1215
+ return false;
1216
+ }
1217
+
1218
+ const message = err.message.toLowerCase();
1219
+
1220
+ // Retryable errors: connection/channel issues, timeouts, transient broker errors
1221
+ const retryablePatterns = [
1222
+ 'channel closed',
1223
+ 'channel ended',
1224
+ 'connection closed',
1225
+ 'connection ended',
1226
+ 'timeout',
1227
+ 'confirmation timeout',
1228
+ 'delivery tag invalid',
1229
+ 'unknown delivery tag',
1230
+ 'precondition_failed.*delivery tag',
1231
+ 'not connected',
1232
+ 'reconnecting'
1233
+ ];
1234
+
1235
+ // Non-retryable errors: permanent failures, validation errors
1236
+ const nonRetryablePatterns = [
1237
+ 'queue does not exist',
1238
+ 'exchange does not exist',
1239
+ 'access refused',
1240
+ 'not found',
1241
+ 'permission denied',
1242
+ 'invalid',
1243
+ 'malformed'
1244
+ ];
1245
+
1246
+ // Check non-retryable first
1247
+ for (const pattern of nonRetryablePatterns) {
1248
+ if (message.includes(pattern)) {
1249
+ return false;
1250
+ }
1251
+ }
1252
+
1253
+ // Check retryable
1254
+ for (const pattern of retryablePatterns) {
1255
+ if (message.includes(pattern)) {
1256
+ return true;
1257
+ }
1258
+ }
1259
+
1260
+ // Default: if we're reconnecting, it's retryable
1261
+ if (this._reconnecting) {
1262
+ return true;
1263
+ }
1264
+
1265
+ // Default: unknown errors are not retryable (fail fast)
1266
+ return false;
1267
+ }
1268
+
903
1269
  /**
904
1270
  * Starts consuming messages from the specified queue.
905
1271
  * @param {string} queue - Queue name to consume from.
@@ -919,6 +1285,9 @@ class RabbitMQClient extends EventEmitter {
919
1285
  const prefetch = options.prefetch !== undefined ? options.prefetch : this._config.prefetch;
920
1286
  const noAck = options.noAck !== undefined ? options.noAck : this._config.noAck;
921
1287
 
1288
+ // Initialize queueOptions with default value
1289
+ let queueOptions = options.queueOptions || { durable };
1290
+
922
1291
  try {
923
1292
  // Skip assertQueue for reply queues (they're already created with specific settings)
924
1293
  // Reply queues start with 'rpc.reply.' and are created as non-durable
@@ -938,8 +1307,6 @@ class RabbitMQClient extends EventEmitter {
938
1307
  const isInfraQueue = queueConfig ? queueConfig.isInfrastructureQueue(queue) : false;
939
1308
  const isBusinessQueue = queueConfig ? queueConfig.isBusinessQueue(queue) : false;
940
1309
 
941
- let queueOptions = options.queueOptions || { durable };
942
-
943
1310
  if (queueConfig) {
944
1311
  if (isInfraQueue) {
945
1312
  // Infrastructure queue - use central config
@@ -992,6 +1359,13 @@ class RabbitMQClient extends EventEmitter {
992
1359
 
993
1360
  try {
994
1361
  console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] About to call assertQueue(${queue}, ${JSON.stringify(queueOptions)})`);
1362
+
1363
+ // Ensure queue channel is available
1364
+ await this._ensureQueueChannel();
1365
+ if (!this._queueChannel) {
1366
+ throw new Error('Queue channel is not available (connection may be closed)');
1367
+ }
1368
+
995
1369
  this._trackChannelOperation(this._queueChannel, `assertQueue ${queue}`);
996
1370
  await this._queueChannel.assertQueue(queue, queueOptions);
997
1371
  console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Queue ${queue} asserted successfully`);
@@ -1013,6 +1387,23 @@ class RabbitMQClient extends EventEmitter {
1013
1387
  // Set prefetch if provided (on consumer channel)
1014
1388
  if (typeof prefetch === 'number') {
1015
1389
  this._consumerChannel.prefetch(prefetch);
1390
+
1391
+ // Track prefetch for monitoring
1392
+ if (!this._prefetchTracking.has(queue)) {
1393
+ this._prefetchTracking.set(queue, {
1394
+ prefetchCount: prefetch,
1395
+ inFlight: 0,
1396
+ lastCheck: Date.now()
1397
+ });
1398
+ } else {
1399
+ const tracking = this._prefetchTracking.get(queue);
1400
+ tracking.prefetchCount = prefetch;
1401
+ }
1402
+
1403
+ // Start prefetch monitoring if not already started
1404
+ if (!this._prefetchCheckTimer) {
1405
+ this._startPrefetchMonitoring();
1406
+ }
1016
1407
  }
1017
1408
 
1018
1409
  // CRITICAL: Log before calling amqplib's consume()
@@ -1024,18 +1415,47 @@ class RabbitMQClient extends EventEmitter {
1024
1415
 
1025
1416
  // Use dedicated consumer channel for consume operations
1026
1417
  // This prevents conflicts with ConfirmChannel used for publish operations
1418
+ // Wrap handler to track message processing and prefetch utilization
1027
1419
  const messageHandler = async (msg) => {
1028
1420
  if (msg === null) {
1029
- return;
1421
+ return; // Consumer cancellation
1030
1422
  }
1423
+
1424
+ // Track in-flight message (increment)
1425
+ const tracking = this._prefetchTracking.get(queue);
1426
+ if (tracking) {
1427
+ tracking.inFlight++;
1428
+ tracking.lastCheck = Date.now();
1429
+ this._checkPrefetchUtilization(queue, tracking);
1430
+ }
1431
+
1031
1432
  try {
1032
1433
  await onMessage(msg);
1434
+ // Acknowledge message after successful processing
1033
1435
  if (!noAck) {
1034
1436
  this._consumerChannel.ack(msg);
1437
+ // Track ack (decrement in-flight)
1438
+ if (tracking) {
1439
+ tracking.inFlight = Math.max(0, tracking.inFlight - 1);
1440
+ tracking.lastCheck = Date.now();
1441
+ }
1035
1442
  }
1036
1443
  } catch (handlerErr) {
1037
1444
  // Negative acknowledge and requeue by default
1038
- this._consumerChannel.nack(msg, false, true);
1445
+ // Check if channel is still valid before nacking
1446
+ if (this._consumerChannel && !this._consumerChannel.closed) {
1447
+ try {
1448
+ this._consumerChannel.nack(msg, false, true);
1449
+ } catch (nackErr) {
1450
+ // Channel may have closed during nack - ignore
1451
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to nack message (channel may be closed): ${nackErr.message}`);
1452
+ }
1453
+ }
1454
+ // Track nack (decrement in-flight)
1455
+ if (tracking) {
1456
+ tracking.inFlight = Math.max(0, tracking.inFlight - 1);
1457
+ tracking.lastCheck = Date.now();
1458
+ }
1039
1459
  }
1040
1460
  };
1041
1461
 
@@ -1062,6 +1482,366 @@ class RabbitMQClient extends EventEmitter {
1062
1482
  }
1063
1483
  }
1064
1484
 
1485
+ /**
1486
+ * Reconnect to RabbitMQ with exponential backoff
1487
+ * CONNECTION-LEVEL RECOVERY: Recreates connection and all channels, re-registers consumers
1488
+ * @private
1489
+ * @returns {Promise<void>}
1490
+ */
1491
+ async _reconnectWithBackoff() {
1492
+ if (this._reconnecting) {
1493
+ console.log('[RabbitMQClient] Reconnection already in progress, skipping');
1494
+ return;
1495
+ }
1496
+
1497
+ this._reconnecting = true;
1498
+ console.log(`[RabbitMQClient] Starting connection-level recovery (attempt ${this._reconnectAttempts + 1}/${this._maxReconnectAttempts})...`);
1499
+
1500
+ while (this._reconnectAttempts < this._maxReconnectAttempts) {
1501
+ try {
1502
+ // Calculate exponential backoff delay: baseDelay * 2^attempts, capped at maxDelay
1503
+ const delay = Math.min(
1504
+ this._reconnectBaseDelay * Math.pow(2, this._reconnectAttempts),
1505
+ this._reconnectMaxDelay
1506
+ );
1507
+
1508
+ console.log(`[RabbitMQClient] Waiting ${delay}ms before reconnection attempt ${this._reconnectAttempts + 1}...`);
1509
+ await new Promise(resolve => setTimeout(resolve, delay));
1510
+
1511
+ // Attempt to reconnect
1512
+ console.log(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts + 1}...`);
1513
+
1514
+ // Close old connection if exists
1515
+ if (this._connection) {
1516
+ try {
1517
+ await this._connection.close();
1518
+ } catch (_) {
1519
+ // Ignore errors when closing already-closed connection
1520
+ }
1521
+ this._connection = null;
1522
+ }
1523
+
1524
+ // Reconnect
1525
+ const rawTarget = this._config.host || this._config.url;
1526
+ const heartbeat = this._config.heartbeat ?? 30;
1527
+ const connectionName =
1528
+ this._config.connectionName ||
1529
+ this._config.clientName ||
1530
+ `oa-client:${process.env.SERVICE_NAME || 'unknown'}:${process.pid}`;
1531
+
1532
+ const connectArgs = typeof rawTarget === 'string'
1533
+ ? [rawTarget, { heartbeat, clientProperties: { connection_name: connectionName } }]
1534
+ : [{ ...rawTarget, heartbeat, clientProperties: { connection_name: connectionName } }];
1535
+
1536
+ const connectPromise = amqp.connect(...connectArgs);
1537
+ const timeoutPromise = new Promise((_, reject) => {
1538
+ setTimeout(() => reject(new Error('Connection timeout after 10 seconds')), 10000);
1539
+ });
1540
+
1541
+ this._connection = await Promise.race([connectPromise, timeoutPromise]);
1542
+ console.log('[RabbitMQClient] ✓ Connection re-established');
1543
+
1544
+ // Re-attach connection event handlers
1545
+ this._connection.on('error', (err) => {
1546
+ console.error('[RabbitMQClient] Connection error:', err.message);
1547
+ this.emit('error', err);
1548
+ });
1549
+
1550
+ this._connection.on('close', () => {
1551
+ console.warn('[RabbitMQClient] Connection closed unexpectedly');
1552
+ this._channel = null;
1553
+ this._queueChannel = null;
1554
+ this._consumerChannel = null;
1555
+ this.emit('error', new Error('RabbitMQ connection closed unexpectedly'));
1556
+
1557
+ // Attempt to reconnect again if enabled
1558
+ if (this._reconnectEnabled && !this._reconnecting) {
1559
+ this._reconnectWithBackoff().catch(err => {
1560
+ console.error('[RabbitMQClient] Reconnection failed after all attempts:', err.message);
1561
+ this.emit('error', err);
1562
+ });
1563
+ }
1564
+ });
1565
+
1566
+ // Recreate all channels
1567
+ console.log('[RabbitMQClient] Recreating channels...');
1568
+
1569
+ // Recreate publisher channel
1570
+ this._channel = await this._connection.createConfirmChannel();
1571
+ this._channel._createdAt = new Date().toISOString();
1572
+ this._channel._closeReason = null;
1573
+ this._channel._lastOperation = 'Recreated via _reconnectWithBackoff()';
1574
+ this._channel._type = 'publisher';
1575
+ this._attachPublisherChannelHandlers(this._channel);
1576
+
1577
+ // Recreate queue channel
1578
+ this._queueChannel = await this._connection.createChannel();
1579
+ this._queueChannel._createdAt = new Date().toISOString();
1580
+ this._queueChannel._closeReason = null;
1581
+ this._queueChannel._lastOperation = 'Recreated via _reconnectWithBackoff()';
1582
+ this._queueChannel._type = 'queue';
1583
+ this._attachQueueChannelHandlers(this._queueChannel);
1584
+
1585
+ // Recreate consumer channel and re-register all consumers
1586
+ this._consumerChannel = await this._connection.createChannel();
1587
+ this._consumerChannel._createdAt = new Date().toISOString();
1588
+ this._consumerChannel._closeReason = null;
1589
+ // IMPORTANT: Don't re-register consumers here - _ensureConsumerChannel() will do it
1590
+ // This prevents duplicate re-registration (once in _reconnectWithBackoff, once in _ensureConsumerChannel)
1591
+ // Just ensure consumer channel is created - it will automatically re-register all active consumers
1592
+ await this._ensureConsumerChannel();
1593
+
1594
+ // Reset reconnect state
1595
+ this._reconnectAttempts = 0;
1596
+ this._reconnecting = false;
1597
+
1598
+ // CRITICAL: Verify all channels are ready before emitting 'reconnected'
1599
+ // This ensures publish/consume operations can proceed immediately after reconnection
1600
+ if (!this._channel || this._channel.closed) {
1601
+ throw new Error('Publisher channel not ready after reconnection');
1602
+ }
1603
+ if (!this._queueChannel || this._queueChannel.closed) {
1604
+ throw new Error('Queue channel not ready after reconnection');
1605
+ }
1606
+ if (!this._consumerChannel || this._consumerChannel.closed) {
1607
+ throw new Error('Consumer channel not ready after reconnection');
1608
+ }
1609
+
1610
+ console.log('[RabbitMQClient] ✓ Connection-level recovery completed successfully - all channels ready');
1611
+ this.emit('reconnected');
1612
+
1613
+ return; // Success - exit reconnection loop
1614
+ } catch (err) {
1615
+ this._reconnectAttempts++;
1616
+ console.error(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts} failed:`, err.message);
1617
+
1618
+ if (this._reconnectAttempts >= this._maxReconnectAttempts) {
1619
+ console.error(`[RabbitMQClient] ✗ Connection-level recovery failed after ${this._maxReconnectAttempts} attempts`);
1620
+ this._reconnecting = false;
1621
+ throw new Error(`Failed to reconnect after ${this._maxReconnectAttempts} attempts: ${err.message}`);
1622
+ }
1623
+ // Continue to next attempt
1624
+ }
1625
+ }
1626
+ }
1627
+
1628
+ /**
1629
+ * Attach event handlers to publisher channel
1630
+ * @private
1631
+ */
1632
+ _attachPublisherChannelHandlers(channel) {
1633
+ channel.on('error', async (err) => {
1634
+ const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
1635
+ channel._closeReason = reason;
1636
+
1637
+ // "unknown delivery tag" errors occur when channel closes during publish
1638
+ // This is expected - the publish callback will handle retry
1639
+ // We still log it but don't try to recreate channel (it's already closing)
1640
+ if (err.message && err.message.includes('unknown delivery tag')) {
1641
+ console.warn(`[RabbitMQClient] [mq-client-core] Publisher channel error (channel closing during publish): ${err.message}`);
1642
+ // Don't try to recreate - channel is already closing
1643
+ // Don't emit error - publish callback will handle retry
1644
+ return;
1645
+ }
1646
+
1647
+ console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
1648
+ this._callChannelCloseHooks('publisher', channel, err, reason);
1649
+
1650
+ // Only try to recreate if connection is available
1651
+ // If connection is closed, reconnection logic will handle channel recreation
1652
+ if (this._connection && !this._connection.closed) {
1653
+ try {
1654
+ await this._ensurePublisherChannel();
1655
+ } catch (recreateErr) {
1656
+ // Ignore errors during channel recreation if connection is closing
1657
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${recreateErr.message}`);
1658
+ }
1659
+ }
1660
+
1661
+ this.emit('error', err);
1662
+ });
1663
+
1664
+ channel.on('close', async () => {
1665
+ const reason = channel._closeReason || 'Publisher channel closed unexpectedly';
1666
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
1667
+ this._callChannelCloseHooks('publisher', channel, null, reason);
1668
+ const closedChannel = this._channel;
1669
+ this._channel = null;
1670
+
1671
+ // Only try to recreate if connection is available
1672
+ // If connection is closed, reconnection logic will handle channel recreation
1673
+ if (this._connection && !this._connection.closed) {
1674
+ try {
1675
+ await this._ensurePublisherChannel();
1676
+ } catch (err) {
1677
+ // Ignore errors during channel recreation if connection is closing
1678
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${err.message}`);
1679
+ }
1680
+ }
1681
+ });
1682
+
1683
+ channel.on('drain', () => {
1684
+ console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
1685
+ });
1686
+ }
1687
+
1688
+ /**
1689
+ * Attach event handlers to queue channel
1690
+ * @private
1691
+ */
1692
+ _attachQueueChannelHandlers(channel) {
1693
+ channel.on('error', async (err) => {
1694
+ const reason = `Queue channel error: ${err.message} (code: ${err.code || 'unknown'})`;
1695
+ channel._closeReason = reason;
1696
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
1697
+ this._callChannelCloseHooks('queue', channel, err, reason);
1698
+ await this._ensureQueueChannel();
1699
+ });
1700
+
1701
+ channel.on('close', async () => {
1702
+ const reason = channel._closeReason || 'Queue channel closed unexpectedly';
1703
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
1704
+ this._callChannelCloseHooks('queue', channel, null, reason);
1705
+ const closedChannel = this._queueChannel;
1706
+ this._queueChannel = null;
1707
+
1708
+ // PREDICTABLE: Only recreate if connection is available and not closing
1709
+ // If connection is closed, reconnection logic will handle channel recreation
1710
+ if (this._connection && !this._connection.closed && !this._reconnecting) {
1711
+ try {
1712
+ await this._ensureQueueChannel();
1713
+ } catch (err) {
1714
+ // If connection is closing, this is expected - don't throw
1715
+ if (err.message && err.message.includes('Connection closed')) {
1716
+ console.warn(`[RabbitMQClient] [mq-client-core] Queue channel close during connection closure (expected - reconnection will handle)`);
1717
+ } else {
1718
+ // Log but don't throw - channel will be recreated on next use or during reconnection
1719
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel: ${err.message} (will be recreated on next use)`);
1720
+ }
1721
+ }
1722
+ } else {
1723
+ console.warn(`[RabbitMQClient] [mq-client-core] Queue channel closed - will be recreated during reconnection or on next use`);
1724
+ }
1725
+ });
1726
+ }
1727
+
1728
+ /**
1729
+ * Attach event handlers to consumer channel
1730
+ * @private
1731
+ */
1732
+ _attachConsumerChannelHandlers(channel) {
1733
+ channel.on('error', async (err) => {
1734
+ const reason = `Consumer channel error: ${err.message} (code: ${err.code || 'unknown'})`;
1735
+ channel._closeReason = reason;
1736
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
1737
+ this._callChannelCloseHooks('consumer', channel, err, reason);
1738
+ await this._ensureConsumerChannel();
1739
+ this.emit('error', err);
1740
+ });
1741
+
1742
+ channel.on('close', async () => {
1743
+ const reason = channel._closeReason || 'Consumer channel closed unexpectedly';
1744
+ console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
1745
+ this._callChannelCloseHooks('consumer', channel, null, reason);
1746
+ const closedChannel = this._consumerChannel;
1747
+ this._consumerChannel = null;
1748
+
1749
+ // Only try to recreate if connection is available
1750
+ // If connection is closed, reconnection logic will handle channel recreation
1751
+ if (this._connection && !this._connection.closed) {
1752
+ try {
1753
+ await this._ensureConsumerChannel();
1754
+ } catch (err) {
1755
+ // Ignore errors during channel recreation if connection is closing
1756
+ console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel (connection may be closing): ${err.message}`);
1757
+ }
1758
+ }
1759
+ });
1760
+ }
1761
+
1762
+ /**
1763
+ * Check prefetch utilization for a queue and alert if threshold exceeded
1764
+ * @private
1765
+ * @param {string} queue - Queue name
1766
+ * @param {Object} tracking - Prefetch tracking object
1767
+ */
1768
+ _checkPrefetchUtilization(queue, tracking) {
1769
+ if (!tracking || tracking.prefetchCount === 0) {
1770
+ return; // No prefetch set or tracking not available
1771
+ }
1772
+
1773
+ const utilization = tracking.inFlight / tracking.prefetchCount;
1774
+
1775
+ if (utilization >= this._prefetchUtilizationThreshold) {
1776
+ const message = `Prefetch utilization high: queue '${queue}' has ${tracking.inFlight}/${tracking.prefetchCount} messages in-flight (${Math.round(utilization * 100)}%, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`;
1777
+ console.warn(`[RabbitMQClient] [mq-client-core] ⚠️ ${message}`);
1778
+
1779
+ // Call alert callback if provided
1780
+ if (this._prefetchAlertCallback) {
1781
+ try {
1782
+ this._prefetchAlertCallback(queue, utilization, tracking.inFlight, tracking.prefetchCount);
1783
+ } catch (alertErr) {
1784
+ console.error(`[RabbitMQClient] [mq-client-core] Prefetch alert callback error:`, alertErr.message);
1785
+ }
1786
+ }
1787
+
1788
+ // Emit event for external listeners
1789
+ this.emit('prefetch:high-utilization', {
1790
+ queue,
1791
+ utilization,
1792
+ inFlight: tracking.inFlight,
1793
+ prefetchCount: tracking.prefetchCount,
1794
+ threshold: this._prefetchUtilizationThreshold,
1795
+ timestamp: new Date().toISOString()
1796
+ });
1797
+ }
1798
+ }
1799
+
1800
+ /**
1801
+ * Start periodic prefetch monitoring
1802
+ * @private
1803
+ */
1804
+ _startPrefetchMonitoring() {
1805
+ if (this._prefetchCheckTimer) {
1806
+ return; // Already started
1807
+ }
1808
+
1809
+ this._prefetchCheckTimer = setInterval(() => {
1810
+ const now = Date.now();
1811
+ for (const [queue, tracking] of this._prefetchTracking.entries()) {
1812
+ // Only check if we have recent activity (within last 30s)
1813
+ if (now - tracking.lastCheck < 30000) {
1814
+ this._checkPrefetchUtilization(queue, tracking);
1815
+ }
1816
+ }
1817
+ }, this._prefetchCheckInterval);
1818
+
1819
+ console.log(`[RabbitMQClient] [mq-client-core] Prefetch monitoring started (interval: ${this._prefetchCheckInterval}ms, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`);
1820
+ }
1821
+
1822
+ /**
1823
+ * Stop prefetch monitoring
1824
+ * @private
1825
+ */
1826
+ _stopPrefetchMonitoring() {
1827
+ if (this._prefetchCheckTimer) {
1828
+ clearInterval(this._prefetchCheckTimer);
1829
+ this._prefetchCheckTimer = null;
1830
+ }
1831
+ }
1832
+
1833
+ /**
1834
+ * Track channel operation for debugging
1835
+ * @private
1836
+ * @param {Object} channel - Channel object
1837
+ * @param {string} operation - Operation description
1838
+ */
1839
+ _trackChannelOperation(channel, operation) {
1840
+ if (channel && typeof channel === 'object') {
1841
+ channel._lastOperation = operation;
1842
+ }
1843
+ }
1844
+
1065
1845
  /**
1066
1846
  * Acknowledges a message.
1067
1847
  * @param {Object} msg - RabbitMQ message object.