@onlineapps/mq-client-core 1.0.36 → 1.0.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/transports/rabbitmqClient.js +950 -170
package/package.json
CHANGED
|
@@ -39,13 +39,44 @@ class RabbitMQClient extends EventEmitter {
|
|
|
39
39
|
// Track active consumers for re-registration after channel recreation
|
|
40
40
|
this._activeConsumers = new Map(); // queue -> { handler, options, consumerTag }
|
|
41
41
|
|
|
42
|
-
//
|
|
42
|
+
// Connection-level recovery
|
|
43
43
|
this._reconnecting = false;
|
|
44
44
|
this._reconnectAttempts = 0;
|
|
45
|
+
this._maxReconnectAttempts = this._config.maxReconnectAttempts || 10; // Max 10 attempts
|
|
46
|
+
this._reconnectBaseDelay = this._config.reconnectBaseDelay || 1000; // Start with 1 second
|
|
47
|
+
this._reconnectMaxDelay = this._config.reconnectMaxDelay || 30000; // Max 30 seconds
|
|
48
|
+
this._reconnectEnabled = this._config.reconnectEnabled !== false; // Default: enabled
|
|
49
|
+
this._reconnectTimer = null;
|
|
50
|
+
|
|
51
|
+
// Publisher retry configuration
|
|
52
|
+
this._publishRetryEnabled = this._config.publishRetryEnabled !== false; // Default: enabled
|
|
53
|
+
this._publishMaxRetries = this._config.publishMaxRetries || 3; // Default: 3 attempts
|
|
54
|
+
this._publishRetryBaseDelay = this._config.publishRetryBaseDelay || 100; // Default: 100ms
|
|
55
|
+
this._publishRetryMaxDelay = this._config.publishRetryMaxDelay || 5000; // Default: 5s
|
|
56
|
+
this._publishRetryBackoffMultiplier = this._config.publishRetryBackoffMultiplier || 2; // Default: 2 (exponential)
|
|
57
|
+
this._publishConfirmationTimeout = this._config.publishConfirmationTimeout || 5000; // Default: 5s per attempt
|
|
45
58
|
|
|
46
59
|
// Channel close hooks - called when channel closes with detailed information
|
|
47
60
|
this._channelCloseHooks = []; // Array of { type: 'publisher'|'queue'|'consumer', callback: (details) => void }
|
|
48
61
|
|
|
62
|
+
// Channel thrashing detection - track close frequency per channel type
|
|
63
|
+
this._channelCloseHistory = {
|
|
64
|
+
publisher: [], // Array of { timestamp: number, reason: string }
|
|
65
|
+
queue: [],
|
|
66
|
+
consumer: []
|
|
67
|
+
};
|
|
68
|
+
this._thrashingThreshold = this._config.thrashingThreshold || 5; // Max closes per minute
|
|
69
|
+
this._thrashingWindowMs = this._config.thrashingWindowMs || 60000; // 1 minute window
|
|
70
|
+
this._thrashingAlertCallback = this._config.thrashingAlertCallback || null; // (type, count, windowMs) => void
|
|
71
|
+
this._thrashingAlertsSent = new Set(); // Track which alerts we've already sent to avoid spam
|
|
72
|
+
|
|
73
|
+
// Prefetch monitoring - track utilization per queue
|
|
74
|
+
this._prefetchTracking = new Map(); // queue -> { prefetchCount: number, inFlight: number, lastCheck: number }
|
|
75
|
+
this._prefetchUtilizationThreshold = this._config.prefetchUtilizationThreshold || 0.8; // 80% threshold
|
|
76
|
+
this._prefetchCheckInterval = this._config.prefetchCheckInterval || 10000; // Check every 10s
|
|
77
|
+
this._prefetchAlertCallback = this._config.prefetchAlertCallback || null; // (queue, utilization, inFlight, prefetchCount) => void
|
|
78
|
+
this._prefetchCheckTimer = null;
|
|
79
|
+
|
|
49
80
|
// Health monitoring
|
|
50
81
|
this._healthCheckInterval = null;
|
|
51
82
|
this._healthCheckIntervalMs = this._config.healthCheckInterval || 30000; // Default 30s
|
|
@@ -91,6 +122,76 @@ class RabbitMQClient extends EventEmitter {
|
|
|
91
122
|
this._channelCloseHooks.push({ type, callback });
|
|
92
123
|
}
|
|
93
124
|
|
|
125
|
+
/**
|
|
126
|
+
* Track channel close event for thrashing detection
|
|
127
|
+
* @private
|
|
128
|
+
* @param {string} channelType - 'publisher', 'queue', or 'consumer'
|
|
129
|
+
* @param {string} reason - Human-readable reason
|
|
130
|
+
*/
|
|
131
|
+
_trackChannelClose(channelType, reason) {
|
|
132
|
+
const now = Date.now();
|
|
133
|
+
const history = this._channelCloseHistory[channelType];
|
|
134
|
+
|
|
135
|
+
if (!history) {
|
|
136
|
+
console.warn(`[RabbitMQClient] Unknown channel type for thrashing tracking: ${channelType}`);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Add new close event
|
|
141
|
+
history.push({
|
|
142
|
+
timestamp: now,
|
|
143
|
+
reason: reason || 'unknown'
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
// Remove old events outside the window
|
|
147
|
+
const cutoff = now - this._thrashingWindowMs;
|
|
148
|
+
while (history.length > 0 && history[0].timestamp < cutoff) {
|
|
149
|
+
history.shift();
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Check for thrashing
|
|
153
|
+
if (history.length >= this._thrashingThreshold) {
|
|
154
|
+
const alertKey = `${channelType}:${Math.floor(now / this._thrashingWindowMs)}`;
|
|
155
|
+
|
|
156
|
+
// Only alert once per window to avoid spam
|
|
157
|
+
if (!this._thrashingAlertsSent.has(alertKey)) {
|
|
158
|
+
this._thrashingAlertsSent.add(alertKey);
|
|
159
|
+
|
|
160
|
+
// Clean up old alert keys (keep only last 10 windows)
|
|
161
|
+
if (this._thrashingAlertsSent.size > 10) {
|
|
162
|
+
const oldestKey = Array.from(this._thrashingAlertsSent).sort()[0];
|
|
163
|
+
this._thrashingAlertsSent.delete(oldestKey);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const message = `Channel thrashing detected: ${channelType} channel closed ${history.length} times in ${Math.round(this._thrashingWindowMs / 1000)}s (threshold: ${this._thrashingThreshold})`;
|
|
167
|
+
console.error(`[RabbitMQClient] [mq-client-core] 🚨 ${message}`);
|
|
168
|
+
|
|
169
|
+
// Call alert callback if provided
|
|
170
|
+
if (this._thrashingAlertCallback) {
|
|
171
|
+
try {
|
|
172
|
+
this._thrashingAlertCallback(channelType, history.length, this._thrashingWindowMs, {
|
|
173
|
+
recentCloses: history.slice(-this._thrashingThreshold),
|
|
174
|
+
threshold: this._thrashingThreshold,
|
|
175
|
+
windowMs: this._thrashingWindowMs
|
|
176
|
+
});
|
|
177
|
+
} catch (alertErr) {
|
|
178
|
+
console.error(`[RabbitMQClient] [mq-client-core] Thrashing alert callback error:`, alertErr.message);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Emit event for external listeners
|
|
183
|
+
this.emit('channel:thrashing', {
|
|
184
|
+
type: channelType,
|
|
185
|
+
count: history.length,
|
|
186
|
+
windowMs: this._thrashingWindowMs,
|
|
187
|
+
threshold: this._thrashingThreshold,
|
|
188
|
+
recentCloses: history.slice(-this._thrashingThreshold),
|
|
189
|
+
timestamp: new Date().toISOString()
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
94
195
|
/**
|
|
95
196
|
* Call all registered channel close hooks
|
|
96
197
|
* @private
|
|
@@ -100,6 +201,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
100
201
|
* @param {string} reason - Human-readable reason
|
|
101
202
|
*/
|
|
102
203
|
_callChannelCloseHooks(channelType, channel, error, reason) {
|
|
204
|
+
// Track channel close for thrashing detection
|
|
205
|
+
this._trackChannelClose(channelType, reason);
|
|
206
|
+
|
|
103
207
|
const details = {
|
|
104
208
|
type: channelType,
|
|
105
209
|
channel: channel,
|
|
@@ -152,10 +256,40 @@ class RabbitMQClient extends EventEmitter {
|
|
|
152
256
|
console.log('[RabbitMQClient] Starting connection race...');
|
|
153
257
|
this._connection = await Promise.race([connectPromise, timeoutPromise]);
|
|
154
258
|
console.log('[RabbitMQClient] Connection established');
|
|
155
|
-
this.
|
|
259
|
+
this._reconnectAttempts = 0; // Reset reconnect attempts on successful connection
|
|
260
|
+
|
|
261
|
+
this._connection.on('error', (err) => {
|
|
262
|
+
console.error('[RabbitMQClient] Connection error:', err.message);
|
|
263
|
+
this.emit('error', err);
|
|
264
|
+
// Connection errors may lead to close event - let close handler handle reconnection
|
|
265
|
+
});
|
|
266
|
+
|
|
156
267
|
this._connection.on('close', () => {
|
|
157
|
-
|
|
268
|
+
console.warn('[RabbitMQClient] Connection closed unexpectedly - close handler STARTED');
|
|
269
|
+
console.log(`[RabbitMQClient] Close handler: reconnectEnabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
|
|
270
|
+
|
|
271
|
+
// Mark all channels as closed
|
|
272
|
+
this._channel = null;
|
|
273
|
+
this._queueChannel = null;
|
|
274
|
+
this._consumerChannel = null;
|
|
275
|
+
|
|
276
|
+
// CONNECTION-LEVEL RECOVERY: Attempt to reconnect with exponential backoff
|
|
277
|
+
// PREDICTABLE: Check conditions and log why reconnect is/isn't starting
|
|
278
|
+
// IMPORTANT: Do this BEFORE emitting error, so reconnect can start even if error handler throws
|
|
279
|
+
if (this._reconnectEnabled && !this._reconnecting) {
|
|
280
|
+
console.log('[RabbitMQClient] ✓ Conditions met - starting connection-level recovery from initial connect() close handler');
|
|
281
|
+
this._reconnectWithBackoff().catch(err => {
|
|
282
|
+
console.error('[RabbitMQClient] Reconnection failed after all attempts:', err.message);
|
|
283
|
+
this.emit('error', err);
|
|
284
|
+
});
|
|
285
|
+
} else {
|
|
286
|
+
console.warn(`[RabbitMQClient] ✗ Reconnect NOT starting: enabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// Emit connection close event AFTER starting reconnect (so reconnect isn't blocked by error handler)
|
|
158
290
|
this.emit('error', new Error('RabbitMQ connection closed unexpectedly'));
|
|
291
|
+
|
|
292
|
+
console.log('[RabbitMQClient] Close handler FINISHED');
|
|
159
293
|
});
|
|
160
294
|
|
|
161
295
|
// Use ConfirmChannel to enable publisher confirms for publish operations
|
|
@@ -167,37 +301,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
167
301
|
this._channel._lastOperation = 'Initial creation';
|
|
168
302
|
this._channel._type = 'publisher';
|
|
169
303
|
|
|
170
|
-
//
|
|
171
|
-
this._channel
|
|
172
|
-
const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
173
|
-
this._channel._closeReason = reason;
|
|
174
|
-
console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
175
|
-
|
|
176
|
-
// Call close hooks with detailed information
|
|
177
|
-
this._callChannelCloseHooks('publisher', this._channel, err, reason);
|
|
178
|
-
|
|
179
|
-
// Auto-recreate channel if connection is still alive
|
|
180
|
-
await this._ensurePublisherChannel();
|
|
181
|
-
this.emit('error', err);
|
|
182
|
-
});
|
|
183
|
-
this._channel.on('close', async () => {
|
|
184
|
-
const reason = this._channel._closeReason || 'Publisher channel closed unexpectedly';
|
|
185
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
|
|
186
|
-
|
|
187
|
-
// Call close hooks with detailed information
|
|
188
|
-
this._callChannelCloseHooks('publisher', this._channel, null, reason);
|
|
189
|
-
|
|
190
|
-
// Mark channel as null - will be recreated on next publish
|
|
191
|
-
const closedChannel = this._channel;
|
|
192
|
-
this._channel = null;
|
|
193
|
-
|
|
194
|
-
// Try to recreate immediately if connection is alive
|
|
195
|
-
await this._ensurePublisherChannel();
|
|
196
|
-
});
|
|
197
|
-
// Set up publisher confirms callback
|
|
198
|
-
this._channel.on('drain', () => {
|
|
199
|
-
console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
|
|
200
|
-
});
|
|
304
|
+
// Attach event handlers
|
|
305
|
+
this._attachPublisherChannelHandlers(this._channel);
|
|
201
306
|
|
|
202
307
|
// Create a separate regular channel for queue operations (assertQueue, checkQueue)
|
|
203
308
|
// This avoids RPC reply queue issues with ConfirmChannel.assertQueue()
|
|
@@ -209,31 +314,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
209
314
|
this._queueChannel._lastOperation = 'Initial creation';
|
|
210
315
|
this._queueChannel._type = 'queue';
|
|
211
316
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
this._queueChannel._closeReason = reason;
|
|
215
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
216
|
-
|
|
217
|
-
// Call close hooks with detailed information
|
|
218
|
-
this._callChannelCloseHooks('queue', this._queueChannel, err, reason);
|
|
219
|
-
|
|
220
|
-
// Auto-recreate channel if connection is still alive
|
|
221
|
-
await this._ensureQueueChannel();
|
|
222
|
-
});
|
|
223
|
-
this._queueChannel.on('close', async () => {
|
|
224
|
-
const reason = this._queueChannel._closeReason || 'Queue channel closed unexpectedly';
|
|
225
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
|
|
226
|
-
|
|
227
|
-
// Call close hooks with detailed information
|
|
228
|
-
this._callChannelCloseHooks('queue', this._queueChannel, null, reason);
|
|
229
|
-
|
|
230
|
-
// Mark channel as null - will be recreated on next queue operation
|
|
231
|
-
const closedChannel = this._queueChannel;
|
|
232
|
-
this._queueChannel = null;
|
|
233
|
-
|
|
234
|
-
// Try to recreate immediately if connection is alive
|
|
235
|
-
await this._ensureQueueChannel();
|
|
236
|
-
});
|
|
317
|
+
// Attach event handlers
|
|
318
|
+
this._attachQueueChannelHandlers(this._queueChannel);
|
|
237
319
|
|
|
238
320
|
// Create a dedicated channel for consume operations
|
|
239
321
|
// This prevents channel conflicts between publish (ConfirmChannel) and consume operations
|
|
@@ -245,32 +327,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
245
327
|
this._consumerChannel._lastOperation = 'Initial creation';
|
|
246
328
|
this._consumerChannel._type = 'consumer';
|
|
247
329
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
this._consumerChannel._closeReason = reason;
|
|
251
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
252
|
-
|
|
253
|
-
// Call close hooks with detailed information
|
|
254
|
-
this._callChannelCloseHooks('consumer', this._consumerChannel, err, reason);
|
|
255
|
-
|
|
256
|
-
// Auto-recreate channel and re-register consumers
|
|
257
|
-
await this._ensureConsumerChannel();
|
|
258
|
-
this.emit('error', err);
|
|
259
|
-
});
|
|
260
|
-
this._consumerChannel.on('close', async () => {
|
|
261
|
-
const reason = this._consumerChannel._closeReason || 'Consumer channel closed unexpectedly';
|
|
262
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
|
|
263
|
-
|
|
264
|
-
// Call close hooks with detailed information
|
|
265
|
-
this._callChannelCloseHooks('consumer', this._consumerChannel, null, reason);
|
|
266
|
-
|
|
267
|
-
// Mark channel as null - will be recreated and consumers re-registered
|
|
268
|
-
const closedChannel = this._consumerChannel;
|
|
269
|
-
this._consumerChannel = null;
|
|
270
|
-
|
|
271
|
-
// Try to recreate and re-register consumers immediately if connection is alive
|
|
272
|
-
await this._ensureConsumerChannel();
|
|
273
|
-
});
|
|
330
|
+
// Attach event handlers
|
|
331
|
+
this._attachConsumerChannelHandlers(this._consumerChannel);
|
|
274
332
|
|
|
275
333
|
// Start health monitoring if enabled
|
|
276
334
|
if (this._healthCheckEnabled) {
|
|
@@ -290,6 +348,50 @@ class RabbitMQClient extends EventEmitter {
|
|
|
290
348
|
}
|
|
291
349
|
}
|
|
292
350
|
|
|
351
|
+
/**
|
|
352
|
+
* Wait for reconnection to complete
|
|
353
|
+
* Used by both retry mechanism and channel creation
|
|
354
|
+
* @private
|
|
355
|
+
* @returns {Promise<void>}
|
|
356
|
+
* @throws {Error} If reconnection fails or times out
|
|
357
|
+
*/
|
|
358
|
+
async _waitForReconnection() {
|
|
359
|
+
if (!this._reconnecting) {
|
|
360
|
+
throw new Error('Cannot wait for reconnection: not currently reconnecting');
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// Wait for 'reconnected' event
|
|
364
|
+
return new Promise((resolve, reject) => {
|
|
365
|
+
const timeout = setTimeout(() => {
|
|
366
|
+
this.removeListener('reconnected', onReconnected);
|
|
367
|
+
this.removeListener('error', onError);
|
|
368
|
+
reject(new Error('Reconnection timeout after 30 seconds'));
|
|
369
|
+
}, 30000); // 30 seconds timeout
|
|
370
|
+
|
|
371
|
+
const onReconnected = () => {
|
|
372
|
+
clearTimeout(timeout);
|
|
373
|
+
this.removeListener('reconnected', onReconnected);
|
|
374
|
+
this.removeListener('error', onError);
|
|
375
|
+
// Wait a bit for channels to be fully recreated
|
|
376
|
+
setTimeout(resolve, 500);
|
|
377
|
+
};
|
|
378
|
+
|
|
379
|
+
const onError = (error) => {
|
|
380
|
+
// Ignore connection errors during reconnection
|
|
381
|
+
if (error.message && error.message.includes('Connection closed unexpectedly')) {
|
|
382
|
+
return; // Expected during reconnection
|
|
383
|
+
}
|
|
384
|
+
clearTimeout(timeout);
|
|
385
|
+
this.removeListener('reconnected', onReconnected);
|
|
386
|
+
this.removeListener('error', onError);
|
|
387
|
+
reject(error);
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
this.once('reconnected', onReconnected);
|
|
391
|
+
this.on('error', onError);
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
|
|
293
395
|
/**
|
|
294
396
|
* Ensure publisher channel exists and is open
|
|
295
397
|
* Auto-recreates if closed
|
|
@@ -302,7 +404,25 @@ class RabbitMQClient extends EventEmitter {
|
|
|
302
404
|
}
|
|
303
405
|
|
|
304
406
|
if (!this._connection || this._connection.closed) {
|
|
305
|
-
|
|
407
|
+
// Connection is closed - wait for reconnection
|
|
408
|
+
console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate publisher channel: connection is closed (waiting for reconnection)');
|
|
409
|
+
|
|
410
|
+
// Wait for reconnection to complete
|
|
411
|
+
if (this._reconnecting) {
|
|
412
|
+
await this._waitForReconnection();
|
|
413
|
+
|
|
414
|
+
// After reconnection, try again
|
|
415
|
+
if (this._channel && !this._channel.closed) {
|
|
416
|
+
return; // Channel was recreated during reconnection
|
|
417
|
+
}
|
|
418
|
+
// If still no channel, connection might have failed - check again
|
|
419
|
+
if (!this._connection || this._connection.closed) {
|
|
420
|
+
throw new Error('Connection reconnection failed - cannot create publisher channel');
|
|
421
|
+
}
|
|
422
|
+
} else {
|
|
423
|
+
// Not reconnecting - connection is closed and not recovering
|
|
424
|
+
throw new Error('Connection is closed and not reconnecting - cannot create publisher channel');
|
|
425
|
+
}
|
|
306
426
|
}
|
|
307
427
|
|
|
308
428
|
try {
|
|
@@ -324,31 +444,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
324
444
|
this._channel._lastOperation = 'Recreated via _ensurePublisherChannel()';
|
|
325
445
|
this._channel._type = 'publisher';
|
|
326
446
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
this._channel._closeReason = reason;
|
|
330
|
-
console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
331
|
-
|
|
332
|
-
// Call close hooks
|
|
333
|
-
this._callChannelCloseHooks('publisher', this._channel, err, reason);
|
|
334
|
-
|
|
335
|
-
await this._ensurePublisherChannel();
|
|
336
|
-
this.emit('error', err);
|
|
337
|
-
});
|
|
338
|
-
this._channel.on('close', async () => {
|
|
339
|
-
const reason = this._channel._closeReason || 'Publisher channel closed unexpectedly';
|
|
340
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
|
|
341
|
-
|
|
342
|
-
// Call close hooks
|
|
343
|
-
this._callChannelCloseHooks('publisher', this._channel, null, reason);
|
|
344
|
-
|
|
345
|
-
const closedChannel = this._channel;
|
|
346
|
-
this._channel = null;
|
|
347
|
-
await this._ensurePublisherChannel();
|
|
348
|
-
});
|
|
349
|
-
this._channel.on('drain', () => {
|
|
350
|
-
console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
|
|
351
|
-
});
|
|
447
|
+
// Attach event handlers
|
|
448
|
+
this._attachPublisherChannelHandlers(this._channel);
|
|
352
449
|
|
|
353
450
|
console.log('[RabbitMQClient] [mq-client-core] ✓ Publisher channel recreated');
|
|
354
451
|
} catch (err) {
|
|
@@ -369,7 +466,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
369
466
|
}
|
|
370
467
|
|
|
371
468
|
if (!this._connection || this._connection.closed) {
|
|
372
|
-
|
|
469
|
+
// Connection is closed - cannot recreate channel
|
|
470
|
+
// Reconnection logic will handle this - don't throw, just return
|
|
471
|
+
// The channel will be recreated after reconnection completes
|
|
472
|
+
console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate queue channel: connection is closed (reconnection in progress)');
|
|
473
|
+
return;
|
|
373
474
|
}
|
|
374
475
|
|
|
375
476
|
try {
|
|
@@ -391,32 +492,18 @@ class RabbitMQClient extends EventEmitter {
|
|
|
391
492
|
this._queueChannel._lastOperation = 'Recreated via _ensureQueueChannel()';
|
|
392
493
|
this._queueChannel._type = 'queue';
|
|
393
494
|
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
this._queueChannel._closeReason = reason;
|
|
397
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
398
|
-
|
|
399
|
-
// Call close hooks
|
|
400
|
-
this._callChannelCloseHooks('queue', this._queueChannel, err, reason);
|
|
401
|
-
|
|
402
|
-
await this._ensureQueueChannel();
|
|
403
|
-
});
|
|
404
|
-
this._queueChannel.on('close', async () => {
|
|
405
|
-
const reason = this._queueChannel._closeReason || 'Queue channel closed unexpectedly';
|
|
406
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
|
|
407
|
-
|
|
408
|
-
// Call close hooks
|
|
409
|
-
this._callChannelCloseHooks('queue', this._queueChannel, null, reason);
|
|
410
|
-
|
|
411
|
-
const closedChannel = this._queueChannel;
|
|
412
|
-
this._queueChannel = null;
|
|
413
|
-
await this._ensureQueueChannel();
|
|
414
|
-
});
|
|
495
|
+
// Attach event handlers
|
|
496
|
+
this._attachQueueChannelHandlers(this._queueChannel);
|
|
415
497
|
|
|
416
498
|
console.log('[RabbitMQClient] [mq-client-core] ✓ Queue channel recreated');
|
|
417
499
|
} catch (err) {
|
|
418
500
|
this._queueChannel = null;
|
|
419
|
-
|
|
501
|
+
// If connection is closed, don't throw - reconnection will handle it
|
|
502
|
+
if (this._connection && !this._connection.closed) {
|
|
503
|
+
throw new Error(`Failed to recreate queue channel: ${err.message}`);
|
|
504
|
+
}
|
|
505
|
+
// Connection is closed - reconnection will recreate channel
|
|
506
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel (connection closed): ${err.message}`);
|
|
420
507
|
}
|
|
421
508
|
}
|
|
422
509
|
|
|
@@ -432,7 +519,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
432
519
|
}
|
|
433
520
|
|
|
434
521
|
if (!this._connection || this._connection.closed) {
|
|
435
|
-
|
|
522
|
+
// Connection is closed - cannot recreate channel
|
|
523
|
+
// Reconnection logic will handle this - don't throw, just return
|
|
524
|
+
// The channel will be recreated after reconnection completes
|
|
525
|
+
console.warn('[RabbitMQClient] [mq-client-core] Cannot recreate consumer channel: connection is closed (reconnection in progress)');
|
|
526
|
+
return;
|
|
436
527
|
}
|
|
437
528
|
|
|
438
529
|
try {
|
|
@@ -454,28 +545,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
454
545
|
this._consumerChannel._lastOperation = 'Recreated via _ensureConsumerChannel()';
|
|
455
546
|
this._consumerChannel._type = 'consumer';
|
|
456
547
|
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
this._consumerChannel._closeReason = reason;
|
|
460
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
461
|
-
|
|
462
|
-
// Call close hooks
|
|
463
|
-
this._callChannelCloseHooks('consumer', this._consumerChannel, err, reason);
|
|
464
|
-
|
|
465
|
-
await this._ensureConsumerChannel();
|
|
466
|
-
this.emit('error', err);
|
|
467
|
-
});
|
|
468
|
-
this._consumerChannel.on('close', async () => {
|
|
469
|
-
const reason = this._consumerChannel._closeReason || 'Consumer channel closed unexpectedly';
|
|
470
|
-
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
|
|
471
|
-
|
|
472
|
-
// Call close hooks
|
|
473
|
-
this._callChannelCloseHooks('consumer', this._consumerChannel, null, reason);
|
|
474
|
-
|
|
475
|
-
const closedChannel = this._consumerChannel;
|
|
476
|
-
this._consumerChannel = null;
|
|
477
|
-
await this._ensureConsumerChannel();
|
|
478
|
-
});
|
|
548
|
+
// Attach event handlers
|
|
549
|
+
this._attachConsumerChannelHandlers(this._consumerChannel);
|
|
479
550
|
|
|
480
551
|
// Re-register all active consumers
|
|
481
552
|
if (this._activeConsumers.size > 0) {
|
|
@@ -737,9 +808,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
737
808
|
// Stop health monitoring
|
|
738
809
|
this._stopHealthMonitoring();
|
|
739
810
|
|
|
811
|
+
// Stop prefetch monitoring
|
|
812
|
+
this._stopPrefetchMonitoring();
|
|
813
|
+
|
|
740
814
|
// Clear active consumers tracking
|
|
741
815
|
this._activeConsumers.clear();
|
|
742
816
|
|
|
817
|
+
// Clear prefetch tracking
|
|
818
|
+
this._prefetchTracking.clear();
|
|
819
|
+
|
|
743
820
|
try {
|
|
744
821
|
if (this._consumerChannel) {
|
|
745
822
|
await this._consumerChannel.close();
|
|
@@ -771,19 +848,39 @@ class RabbitMQClient extends EventEmitter {
|
|
|
771
848
|
this._connection = null;
|
|
772
849
|
}
|
|
773
850
|
} catch (err) {
|
|
774
|
-
|
|
851
|
+
// Don't emit error for expected connection closure during disconnect
|
|
852
|
+
// This is a predictable, intentional closure
|
|
853
|
+
if (err.message && err.message.includes('Connection closed (by client)')) {
|
|
854
|
+
console.log('[RabbitMQClient] Connection closed during disconnect (expected)');
|
|
855
|
+
} else {
|
|
856
|
+
this.emit('error', err);
|
|
857
|
+
}
|
|
775
858
|
}
|
|
776
859
|
}
|
|
777
860
|
|
|
778
861
|
/**
|
|
779
862
|
* Publishes a message buffer to the specified queue (or default queue) or exchange.
|
|
863
|
+
* Implements systematic retry with exponential backoff for transient failures.
|
|
780
864
|
* @param {string} queue - Target queue name.
|
|
781
865
|
* @param {Buffer} buffer - Message payload as Buffer.
|
|
782
866
|
* @param {Object} [options] - Overrides: routingKey, persistent, headers.
|
|
783
867
|
* @returns {Promise<void>}
|
|
784
|
-
* @throws {Error} If publish fails
|
|
868
|
+
* @throws {Error} If publish fails after all retry attempts.
|
|
785
869
|
*/
|
|
786
870
|
async publish(queue, buffer, options = {}) {
|
|
871
|
+
// Wrap in retry logic if enabled
|
|
872
|
+
if (this._publishRetryEnabled) {
|
|
873
|
+
return await this._publishWithRetry(queue, buffer, options);
|
|
874
|
+
} else {
|
|
875
|
+
return await this._publishOnce(queue, buffer, options);
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
/**
|
|
880
|
+
* Internal publish implementation - single attempt without retry
|
|
881
|
+
* @private
|
|
882
|
+
*/
|
|
883
|
+
async _publishOnce(queue, buffer, options = {}) {
|
|
787
884
|
// Ensure publisher channel exists and is open (auto-recreates if closed)
|
|
788
885
|
await this._ensurePublisherChannel();
|
|
789
886
|
|
|
@@ -849,16 +946,103 @@ class RabbitMQClient extends EventEmitter {
|
|
|
849
946
|
|
|
850
947
|
// Use callback-based confirmation - kanály jsou spolehlivé, takže callback vždy dorazí
|
|
851
948
|
const confirmPromise = new Promise((resolve, reject) => {
|
|
949
|
+
// Set timeout for publish confirmation (configurable)
|
|
950
|
+
const timeout = setTimeout(() => {
|
|
951
|
+
reject(new Error(`Publish confirmation timeout for queue "${queue}" after ${this._publishConfirmationTimeout}ms`));
|
|
952
|
+
}, this._publishConfirmationTimeout);
|
|
953
|
+
|
|
954
|
+
// Check if channel is still valid before sending
|
|
955
|
+
if (!this._channel || this._channel.closed) {
|
|
956
|
+
clearTimeout(timeout);
|
|
957
|
+
reject(new Error(`Cannot publish: channel is closed for queue "${queue}"`));
|
|
958
|
+
return;
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
// Track original channel to detect if it was closed/recreated during publish
|
|
962
|
+
const originalChannel = this._channel;
|
|
963
|
+
const channelId = originalChannel ? originalChannel._createdAt : null;
|
|
964
|
+
let callbackInvoked = false;
|
|
965
|
+
|
|
852
966
|
// Send message with callback
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
967
|
+
try {
|
|
968
|
+
originalChannel.sendToQueue(queue, buffer, { persistent, headers }, (err, ok) => {
|
|
969
|
+
callbackInvoked = true;
|
|
970
|
+
clearTimeout(timeout);
|
|
971
|
+
|
|
972
|
+
// CRITICAL: Check if channel was closed or recreated during publish
|
|
973
|
+
// If channel was recreated, the delivery tag is invalid - ignore this callback
|
|
974
|
+
if (!this._channel || this._channel.closed || this._channel !== originalChannel ||
|
|
975
|
+
(this._channel._createdAt && this._channel._createdAt !== channelId)) {
|
|
976
|
+
// Channel was closed or recreated - delivery tag is invalid
|
|
977
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed/recreated during publish to queue "${queue}", ignoring callback`);
|
|
978
|
+
// If we're reconnecting, wait and retry
|
|
979
|
+
if (this._reconnecting) {
|
|
980
|
+
this._waitForReconnection().then(() => {
|
|
981
|
+
// Retry publish after reconnection
|
|
982
|
+
return this.publish(queue, buffer, options);
|
|
983
|
+
}).then(resolve).catch(reject);
|
|
984
|
+
} else {
|
|
985
|
+
// Not reconnecting - this is a real error
|
|
986
|
+
// We cannot guarantee the message was delivered - must fail
|
|
987
|
+
reject(new Error(`Channel closed during publish to queue "${queue}" and not reconnecting - message delivery not guaranteed`));
|
|
988
|
+
}
|
|
989
|
+
return;
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
if (err) {
|
|
993
|
+
// Check if error is about invalid delivery tag (channel was closed)
|
|
994
|
+
if (err.message && (err.message.includes('unknown delivery tag') || err.message.includes('PRECONDITION_FAILED'))) {
|
|
995
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Delivery tag invalid (channel may have been closed) for queue "${queue}"`);
|
|
996
|
+
// If we're reconnecting, wait and retry
|
|
997
|
+
if (this._reconnecting) {
|
|
998
|
+
this._waitForReconnection().then(() => {
|
|
999
|
+
// Retry publish after reconnection
|
|
1000
|
+
return this.publish(queue, buffer, options);
|
|
1001
|
+
}).then(resolve).catch(reject);
|
|
1002
|
+
} else {
|
|
1003
|
+
// Not reconnecting - delivery tag is invalid, message delivery not guaranteed
|
|
1004
|
+
reject(new Error(`Delivery tag invalid for queue "${queue}" (channel closed) and not reconnecting - message delivery not guaranteed`));
|
|
1005
|
+
}
|
|
1006
|
+
} else {
|
|
1007
|
+
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
|
|
1008
|
+
reject(err);
|
|
1009
|
+
}
|
|
1010
|
+
} else {
|
|
1011
|
+
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Message confirmed for queue "${queue}"`);
|
|
1012
|
+
resolve();
|
|
1013
|
+
}
|
|
1014
|
+
});
|
|
1015
|
+
|
|
1016
|
+
// Set a safety timeout - if callback wasn't invoked and channel closed, retry
|
|
1017
|
+
setTimeout(() => {
|
|
1018
|
+
if (!callbackInvoked && (!this._channel || this._channel.closed || this._channel !== originalChannel)) {
|
|
1019
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Callback timeout and channel closed for queue "${queue}", will retry after reconnection`);
|
|
1020
|
+
if (this._reconnecting) {
|
|
1021
|
+
clearTimeout(timeout);
|
|
1022
|
+
this._waitForReconnection().then(() => {
|
|
1023
|
+
return this.publish(queue, buffer, options);
|
|
1024
|
+
}).then(resolve).catch(reject);
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
}, 1000);
|
|
1028
|
+
} catch (sendErr) {
|
|
1029
|
+
clearTimeout(timeout);
|
|
1030
|
+
// If channel is closed, try to wait for reconnection and retry
|
|
1031
|
+
if (sendErr.message && (sendErr.message.includes('Channel ended') || sendErr.message.includes('Channel closed'))) {
|
|
1032
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during sendToQueue, will retry after reconnection`);
|
|
1033
|
+
// Wait for reconnection and retry
|
|
1034
|
+
if (this._reconnecting) {
|
|
1035
|
+
this._waitForReconnection().then(() => {
|
|
1036
|
+
// Retry publish after reconnection
|
|
1037
|
+
return this.publish(queue, buffer, options);
|
|
1038
|
+
}).then(resolve).catch(reject);
|
|
1039
|
+
} else {
|
|
1040
|
+
reject(sendErr);
|
|
1041
|
+
}
|
|
857
1042
|
} else {
|
|
858
|
-
|
|
859
|
-
resolve();
|
|
1043
|
+
reject(sendErr);
|
|
860
1044
|
}
|
|
861
|
-
}
|
|
1045
|
+
}
|
|
862
1046
|
});
|
|
863
1047
|
|
|
864
1048
|
await confirmPromise;
|
|
@@ -869,7 +1053,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
869
1053
|
|
|
870
1054
|
// Use callback-based confirmation - kanály jsou spolehlivé
|
|
871
1055
|
const confirmPromise = new Promise((resolve, reject) => {
|
|
1056
|
+
// Set timeout for exchange publish confirmation
|
|
1057
|
+
const timeout = setTimeout(() => {
|
|
1058
|
+
reject(new Error(`Exchange publish confirmation timeout for exchange "${exchange}" after ${this._publishConfirmationTimeout}ms`));
|
|
1059
|
+
}, this._publishConfirmationTimeout);
|
|
1060
|
+
|
|
872
1061
|
this._channel.publish(exchange, routingKey, buffer, { persistent, headers }, (err, ok) => {
|
|
1062
|
+
clearTimeout(timeout);
|
|
873
1063
|
if (err) {
|
|
874
1064
|
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Exchange publish callback error:`, err.message);
|
|
875
1065
|
reject(err);
|
|
@@ -900,6 +1090,182 @@ class RabbitMQClient extends EventEmitter {
|
|
|
900
1090
|
}
|
|
901
1091
|
}
|
|
902
1092
|
|
|
1093
|
+
/**
|
|
1094
|
+
* Publish with systematic retry and exponential backoff
|
|
1095
|
+
* @private
|
|
1096
|
+
* @param {string} queue - Target queue name
|
|
1097
|
+
* @param {Buffer} buffer - Message payload
|
|
1098
|
+
* @param {Object} options - Publish options
|
|
1099
|
+
* @returns {Promise<void>}
|
|
1100
|
+
*/
|
|
1101
|
+
async _publishWithRetry(queue, buffer, options = {}) {
|
|
1102
|
+
let lastError = null;
|
|
1103
|
+
let attempt = 0;
|
|
1104
|
+
|
|
1105
|
+
while (attempt < this._publishMaxRetries) {
|
|
1106
|
+
attempt++;
|
|
1107
|
+
|
|
1108
|
+
try {
|
|
1109
|
+
// Emit retry attempt event for monitoring
|
|
1110
|
+
if (attempt > 1) {
|
|
1111
|
+
this.emit('publish:retry', {
|
|
1112
|
+
queue,
|
|
1113
|
+
attempt,
|
|
1114
|
+
maxRetries: this._publishMaxRetries,
|
|
1115
|
+
lastError: lastError?.message
|
|
1116
|
+
});
|
|
1117
|
+
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] Retry attempt ${attempt}/${this._publishMaxRetries} for queue "${queue}"`);
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// Attempt publish
|
|
1121
|
+
await this._publishOnce(queue, buffer, options);
|
|
1122
|
+
|
|
1123
|
+
// Success - emit success event if this was a retry
|
|
1124
|
+
if (attempt > 1) {
|
|
1125
|
+
this.emit('publish:success', {
|
|
1126
|
+
queue,
|
|
1127
|
+
attempt,
|
|
1128
|
+
totalAttempts: attempt
|
|
1129
|
+
});
|
|
1130
|
+
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Published successfully after ${attempt} attempts for queue "${queue}"`);
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
return; // Success - exit retry loop
|
|
1134
|
+
|
|
1135
|
+
} catch (err) {
|
|
1136
|
+
lastError = err;
|
|
1137
|
+
|
|
1138
|
+
// Check if error is retryable
|
|
1139
|
+
const isRetryable = this._isPublishErrorRetryable(err);
|
|
1140
|
+
|
|
1141
|
+
if (!isRetryable) {
|
|
1142
|
+
// Non-retryable error - fail immediately
|
|
1143
|
+
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Non-retryable error for queue "${queue}":`, err.message);
|
|
1144
|
+
this.emit('publish:failed', {
|
|
1145
|
+
queue,
|
|
1146
|
+
attempt,
|
|
1147
|
+
error: err.message,
|
|
1148
|
+
retryable: false
|
|
1149
|
+
});
|
|
1150
|
+
throw err;
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
// Retryable error - check if we have more attempts
|
|
1154
|
+
if (attempt >= this._publishMaxRetries) {
|
|
1155
|
+
// Max retries exceeded
|
|
1156
|
+
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✗ Failed after ${attempt} attempts for queue "${queue}":`, err.message);
|
|
1157
|
+
this.emit('publish:failed', {
|
|
1158
|
+
queue,
|
|
1159
|
+
attempt,
|
|
1160
|
+
error: err.message,
|
|
1161
|
+
retryable: true,
|
|
1162
|
+
maxRetriesExceeded: true
|
|
1163
|
+
});
|
|
1164
|
+
throw new Error(`Publish failed after ${attempt} attempts for queue "${queue}": ${err.message}`);
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
// CRITICAL: If reconnecting, wait for reconnection to complete before retry
|
|
1168
|
+
// This ensures we don't waste retry attempts while connection is being restored
|
|
1169
|
+
if (this._reconnecting) {
|
|
1170
|
+
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] Connection is reconnecting, waiting for reconnection before retry ${attempt + 1}/${this._publishMaxRetries}...`);
|
|
1171
|
+
try {
|
|
1172
|
+
await this._waitForReconnection();
|
|
1173
|
+
console.log(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Reconnection completed, proceeding with retry ${attempt + 1}/${this._publishMaxRetries}`);
|
|
1174
|
+
// Continue to retry immediately after reconnection (no additional delay needed)
|
|
1175
|
+
continue;
|
|
1176
|
+
} catch (reconnectErr) {
|
|
1177
|
+
// Reconnection failed - this is a critical error
|
|
1178
|
+
console.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Reconnection failed:`, reconnectErr.message);
|
|
1179
|
+
this.emit('publish:failed', {
|
|
1180
|
+
queue,
|
|
1181
|
+
attempt,
|
|
1182
|
+
error: `Reconnection failed: ${reconnectErr.message}`,
|
|
1183
|
+
retryable: false,
|
|
1184
|
+
reconnectFailed: true
|
|
1185
|
+
});
|
|
1186
|
+
throw new Error(`Publish failed: reconnection failed after ${attempt} attempts for queue "${queue}": ${reconnectErr.message}`);
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
// Calculate exponential backoff delay (only if not reconnecting)
|
|
1191
|
+
const delay = Math.min(
|
|
1192
|
+
this._publishRetryBaseDelay * Math.pow(this._publishRetryBackoffMultiplier, attempt - 1),
|
|
1193
|
+
this._publishRetryMaxDelay
|
|
1194
|
+
);
|
|
1195
|
+
|
|
1196
|
+
console.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Retryable error for queue "${queue}" (attempt ${attempt}/${this._publishMaxRetries}), waiting ${delay}ms before retry:`, err.message);
|
|
1197
|
+
|
|
1198
|
+
// Wait before retry
|
|
1199
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
// Should never reach here, but just in case
|
|
1204
|
+
throw lastError || new Error(`Publish failed for queue "${queue}" after ${attempt} attempts`);
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
/**
|
|
1208
|
+
* Determine if a publish error is retryable
|
|
1209
|
+
* @private
|
|
1210
|
+
* @param {Error} err - Error to check
|
|
1211
|
+
* @returns {boolean} True if error is retryable
|
|
1212
|
+
*/
|
|
1213
|
+
_isPublishErrorRetryable(err) {
|
|
1214
|
+
if (!err || !err.message) {
|
|
1215
|
+
return false;
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
const message = err.message.toLowerCase();
|
|
1219
|
+
|
|
1220
|
+
// Retryable errors: connection/channel issues, timeouts, transient broker errors
|
|
1221
|
+
const retryablePatterns = [
|
|
1222
|
+
'channel closed',
|
|
1223
|
+
'channel ended',
|
|
1224
|
+
'connection closed',
|
|
1225
|
+
'connection ended',
|
|
1226
|
+
'timeout',
|
|
1227
|
+
'confirmation timeout',
|
|
1228
|
+
'delivery tag invalid',
|
|
1229
|
+
'unknown delivery tag',
|
|
1230
|
+
'precondition_failed.*delivery tag',
|
|
1231
|
+
'not connected',
|
|
1232
|
+
'reconnecting'
|
|
1233
|
+
];
|
|
1234
|
+
|
|
1235
|
+
// Non-retryable errors: permanent failures, validation errors
|
|
1236
|
+
const nonRetryablePatterns = [
|
|
1237
|
+
'queue does not exist',
|
|
1238
|
+
'exchange does not exist',
|
|
1239
|
+
'access refused',
|
|
1240
|
+
'not found',
|
|
1241
|
+
'permission denied',
|
|
1242
|
+
'invalid',
|
|
1243
|
+
'malformed'
|
|
1244
|
+
];
|
|
1245
|
+
|
|
1246
|
+
// Check non-retryable first
|
|
1247
|
+
for (const pattern of nonRetryablePatterns) {
|
|
1248
|
+
if (message.includes(pattern)) {
|
|
1249
|
+
return false;
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
// Check retryable
|
|
1254
|
+
for (const pattern of retryablePatterns) {
|
|
1255
|
+
if (message.includes(pattern)) {
|
|
1256
|
+
return true;
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
// Default: if we're reconnecting, it's retryable
|
|
1261
|
+
if (this._reconnecting) {
|
|
1262
|
+
return true;
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
// Default: unknown errors are not retryable (fail fast)
|
|
1266
|
+
return false;
|
|
1267
|
+
}
|
|
1268
|
+
|
|
903
1269
|
/**
|
|
904
1270
|
* Starts consuming messages from the specified queue.
|
|
905
1271
|
* @param {string} queue - Queue name to consume from.
|
|
@@ -919,6 +1285,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
919
1285
|
const prefetch = options.prefetch !== undefined ? options.prefetch : this._config.prefetch;
|
|
920
1286
|
const noAck = options.noAck !== undefined ? options.noAck : this._config.noAck;
|
|
921
1287
|
|
|
1288
|
+
// Initialize queueOptions with default value
|
|
1289
|
+
let queueOptions = options.queueOptions || { durable };
|
|
1290
|
+
|
|
922
1291
|
try {
|
|
923
1292
|
// Skip assertQueue for reply queues (they're already created with specific settings)
|
|
924
1293
|
// Reply queues start with 'rpc.reply.' and are created as non-durable
|
|
@@ -938,8 +1307,6 @@ class RabbitMQClient extends EventEmitter {
|
|
|
938
1307
|
const isInfraQueue = queueConfig ? queueConfig.isInfrastructureQueue(queue) : false;
|
|
939
1308
|
const isBusinessQueue = queueConfig ? queueConfig.isBusinessQueue(queue) : false;
|
|
940
1309
|
|
|
941
|
-
let queueOptions = options.queueOptions || { durable };
|
|
942
|
-
|
|
943
1310
|
if (queueConfig) {
|
|
944
1311
|
if (isInfraQueue) {
|
|
945
1312
|
// Infrastructure queue - use central config
|
|
@@ -992,6 +1359,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
992
1359
|
|
|
993
1360
|
try {
|
|
994
1361
|
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] About to call assertQueue(${queue}, ${JSON.stringify(queueOptions)})`);
|
|
1362
|
+
|
|
1363
|
+
// Ensure queue channel is available
|
|
1364
|
+
await this._ensureQueueChannel();
|
|
1365
|
+
if (!this._queueChannel) {
|
|
1366
|
+
throw new Error('Queue channel is not available (connection may be closed)');
|
|
1367
|
+
}
|
|
1368
|
+
|
|
995
1369
|
this._trackChannelOperation(this._queueChannel, `assertQueue ${queue}`);
|
|
996
1370
|
await this._queueChannel.assertQueue(queue, queueOptions);
|
|
997
1371
|
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Queue ${queue} asserted successfully`);
|
|
@@ -1013,6 +1387,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1013
1387
|
// Set prefetch if provided (on consumer channel)
|
|
1014
1388
|
if (typeof prefetch === 'number') {
|
|
1015
1389
|
this._consumerChannel.prefetch(prefetch);
|
|
1390
|
+
|
|
1391
|
+
// Track prefetch for monitoring
|
|
1392
|
+
if (!this._prefetchTracking.has(queue)) {
|
|
1393
|
+
this._prefetchTracking.set(queue, {
|
|
1394
|
+
prefetchCount: prefetch,
|
|
1395
|
+
inFlight: 0,
|
|
1396
|
+
lastCheck: Date.now()
|
|
1397
|
+
});
|
|
1398
|
+
} else {
|
|
1399
|
+
const tracking = this._prefetchTracking.get(queue);
|
|
1400
|
+
tracking.prefetchCount = prefetch;
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
// Start prefetch monitoring if not already started
|
|
1404
|
+
if (!this._prefetchCheckTimer) {
|
|
1405
|
+
this._startPrefetchMonitoring();
|
|
1406
|
+
}
|
|
1016
1407
|
}
|
|
1017
1408
|
|
|
1018
1409
|
// CRITICAL: Log before calling amqplib's consume()
|
|
@@ -1024,18 +1415,47 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1024
1415
|
|
|
1025
1416
|
// Use dedicated consumer channel for consume operations
|
|
1026
1417
|
// This prevents conflicts with ConfirmChannel used for publish operations
|
|
1418
|
+
// Wrap handler to track message processing and prefetch utilization
|
|
1027
1419
|
const messageHandler = async (msg) => {
|
|
1028
1420
|
if (msg === null) {
|
|
1029
|
-
return;
|
|
1421
|
+
return; // Consumer cancellation
|
|
1030
1422
|
}
|
|
1423
|
+
|
|
1424
|
+
// Track in-flight message (increment)
|
|
1425
|
+
const tracking = this._prefetchTracking.get(queue);
|
|
1426
|
+
if (tracking) {
|
|
1427
|
+
tracking.inFlight++;
|
|
1428
|
+
tracking.lastCheck = Date.now();
|
|
1429
|
+
this._checkPrefetchUtilization(queue, tracking);
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1031
1432
|
try {
|
|
1032
1433
|
await onMessage(msg);
|
|
1434
|
+
// Acknowledge message after successful processing
|
|
1033
1435
|
if (!noAck) {
|
|
1034
1436
|
this._consumerChannel.ack(msg);
|
|
1437
|
+
// Track ack (decrement in-flight)
|
|
1438
|
+
if (tracking) {
|
|
1439
|
+
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
1440
|
+
tracking.lastCheck = Date.now();
|
|
1441
|
+
}
|
|
1035
1442
|
}
|
|
1036
1443
|
} catch (handlerErr) {
|
|
1037
1444
|
// Negative acknowledge and requeue by default
|
|
1038
|
-
|
|
1445
|
+
// Check if channel is still valid before nacking
|
|
1446
|
+
if (this._consumerChannel && !this._consumerChannel.closed) {
|
|
1447
|
+
try {
|
|
1448
|
+
this._consumerChannel.nack(msg, false, true);
|
|
1449
|
+
} catch (nackErr) {
|
|
1450
|
+
// Channel may have closed during nack - ignore
|
|
1451
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to nack message (channel may be closed): ${nackErr.message}`);
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
// Track nack (decrement in-flight)
|
|
1455
|
+
if (tracking) {
|
|
1456
|
+
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
1457
|
+
tracking.lastCheck = Date.now();
|
|
1458
|
+
}
|
|
1039
1459
|
}
|
|
1040
1460
|
};
|
|
1041
1461
|
|
|
@@ -1062,6 +1482,366 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1062
1482
|
}
|
|
1063
1483
|
}
|
|
1064
1484
|
|
|
1485
|
+
/**
|
|
1486
|
+
* Reconnect to RabbitMQ with exponential backoff
|
|
1487
|
+
* CONNECTION-LEVEL RECOVERY: Recreates connection and all channels, re-registers consumers
|
|
1488
|
+
* @private
|
|
1489
|
+
* @returns {Promise<void>}
|
|
1490
|
+
*/
|
|
1491
|
+
async _reconnectWithBackoff() {
|
|
1492
|
+
if (this._reconnecting) {
|
|
1493
|
+
console.log('[RabbitMQClient] Reconnection already in progress, skipping');
|
|
1494
|
+
return;
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
this._reconnecting = true;
|
|
1498
|
+
console.log(`[RabbitMQClient] Starting connection-level recovery (attempt ${this._reconnectAttempts + 1}/${this._maxReconnectAttempts})...`);
|
|
1499
|
+
|
|
1500
|
+
while (this._reconnectAttempts < this._maxReconnectAttempts) {
|
|
1501
|
+
try {
|
|
1502
|
+
// Calculate exponential backoff delay: baseDelay * 2^attempts, capped at maxDelay
|
|
1503
|
+
const delay = Math.min(
|
|
1504
|
+
this._reconnectBaseDelay * Math.pow(2, this._reconnectAttempts),
|
|
1505
|
+
this._reconnectMaxDelay
|
|
1506
|
+
);
|
|
1507
|
+
|
|
1508
|
+
console.log(`[RabbitMQClient] Waiting ${delay}ms before reconnection attempt ${this._reconnectAttempts + 1}...`);
|
|
1509
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
1510
|
+
|
|
1511
|
+
// Attempt to reconnect
|
|
1512
|
+
console.log(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts + 1}...`);
|
|
1513
|
+
|
|
1514
|
+
// Close old connection if exists
|
|
1515
|
+
if (this._connection) {
|
|
1516
|
+
try {
|
|
1517
|
+
await this._connection.close();
|
|
1518
|
+
} catch (_) {
|
|
1519
|
+
// Ignore errors when closing already-closed connection
|
|
1520
|
+
}
|
|
1521
|
+
this._connection = null;
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
// Reconnect
|
|
1525
|
+
const rawTarget = this._config.host || this._config.url;
|
|
1526
|
+
const heartbeat = this._config.heartbeat ?? 30;
|
|
1527
|
+
const connectionName =
|
|
1528
|
+
this._config.connectionName ||
|
|
1529
|
+
this._config.clientName ||
|
|
1530
|
+
`oa-client:${process.env.SERVICE_NAME || 'unknown'}:${process.pid}`;
|
|
1531
|
+
|
|
1532
|
+
const connectArgs = typeof rawTarget === 'string'
|
|
1533
|
+
? [rawTarget, { heartbeat, clientProperties: { connection_name: connectionName } }]
|
|
1534
|
+
: [{ ...rawTarget, heartbeat, clientProperties: { connection_name: connectionName } }];
|
|
1535
|
+
|
|
1536
|
+
const connectPromise = amqp.connect(...connectArgs);
|
|
1537
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
1538
|
+
setTimeout(() => reject(new Error('Connection timeout after 10 seconds')), 10000);
|
|
1539
|
+
});
|
|
1540
|
+
|
|
1541
|
+
this._connection = await Promise.race([connectPromise, timeoutPromise]);
|
|
1542
|
+
console.log('[RabbitMQClient] ✓ Connection re-established');
|
|
1543
|
+
|
|
1544
|
+
// Re-attach connection event handlers
|
|
1545
|
+
this._connection.on('error', (err) => {
|
|
1546
|
+
console.error('[RabbitMQClient] Connection error:', err.message);
|
|
1547
|
+
this.emit('error', err);
|
|
1548
|
+
});
|
|
1549
|
+
|
|
1550
|
+
this._connection.on('close', () => {
|
|
1551
|
+
console.warn('[RabbitMQClient] Connection closed unexpectedly');
|
|
1552
|
+
this._channel = null;
|
|
1553
|
+
this._queueChannel = null;
|
|
1554
|
+
this._consumerChannel = null;
|
|
1555
|
+
this.emit('error', new Error('RabbitMQ connection closed unexpectedly'));
|
|
1556
|
+
|
|
1557
|
+
// Attempt to reconnect again if enabled
|
|
1558
|
+
if (this._reconnectEnabled && !this._reconnecting) {
|
|
1559
|
+
this._reconnectWithBackoff().catch(err => {
|
|
1560
|
+
console.error('[RabbitMQClient] Reconnection failed after all attempts:', err.message);
|
|
1561
|
+
this.emit('error', err);
|
|
1562
|
+
});
|
|
1563
|
+
}
|
|
1564
|
+
});
|
|
1565
|
+
|
|
1566
|
+
// Recreate all channels
|
|
1567
|
+
console.log('[RabbitMQClient] Recreating channels...');
|
|
1568
|
+
|
|
1569
|
+
// Recreate publisher channel
|
|
1570
|
+
this._channel = await this._connection.createConfirmChannel();
|
|
1571
|
+
this._channel._createdAt = new Date().toISOString();
|
|
1572
|
+
this._channel._closeReason = null;
|
|
1573
|
+
this._channel._lastOperation = 'Recreated via _reconnectWithBackoff()';
|
|
1574
|
+
this._channel._type = 'publisher';
|
|
1575
|
+
this._attachPublisherChannelHandlers(this._channel);
|
|
1576
|
+
|
|
1577
|
+
// Recreate queue channel
|
|
1578
|
+
this._queueChannel = await this._connection.createChannel();
|
|
1579
|
+
this._queueChannel._createdAt = new Date().toISOString();
|
|
1580
|
+
this._queueChannel._closeReason = null;
|
|
1581
|
+
this._queueChannel._lastOperation = 'Recreated via _reconnectWithBackoff()';
|
|
1582
|
+
this._queueChannel._type = 'queue';
|
|
1583
|
+
this._attachQueueChannelHandlers(this._queueChannel);
|
|
1584
|
+
|
|
1585
|
+
// Recreate consumer channel and re-register all consumers
|
|
1586
|
+
this._consumerChannel = await this._connection.createChannel();
|
|
1587
|
+
this._consumerChannel._createdAt = new Date().toISOString();
|
|
1588
|
+
this._consumerChannel._closeReason = null;
|
|
1589
|
+
// IMPORTANT: Don't re-register consumers here - _ensureConsumerChannel() will do it
|
|
1590
|
+
// This prevents duplicate re-registration (once in _reconnectWithBackoff, once in _ensureConsumerChannel)
|
|
1591
|
+
// Just ensure consumer channel is created - it will automatically re-register all active consumers
|
|
1592
|
+
await this._ensureConsumerChannel();
|
|
1593
|
+
|
|
1594
|
+
// Reset reconnect state
|
|
1595
|
+
this._reconnectAttempts = 0;
|
|
1596
|
+
this._reconnecting = false;
|
|
1597
|
+
|
|
1598
|
+
// CRITICAL: Verify all channels are ready before emitting 'reconnected'
|
|
1599
|
+
// This ensures publish/consume operations can proceed immediately after reconnection
|
|
1600
|
+
if (!this._channel || this._channel.closed) {
|
|
1601
|
+
throw new Error('Publisher channel not ready after reconnection');
|
|
1602
|
+
}
|
|
1603
|
+
if (!this._queueChannel || this._queueChannel.closed) {
|
|
1604
|
+
throw new Error('Queue channel not ready after reconnection');
|
|
1605
|
+
}
|
|
1606
|
+
if (!this._consumerChannel || this._consumerChannel.closed) {
|
|
1607
|
+
throw new Error('Consumer channel not ready after reconnection');
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
console.log('[RabbitMQClient] ✓ Connection-level recovery completed successfully - all channels ready');
|
|
1611
|
+
this.emit('reconnected');
|
|
1612
|
+
|
|
1613
|
+
return; // Success - exit reconnection loop
|
|
1614
|
+
} catch (err) {
|
|
1615
|
+
this._reconnectAttempts++;
|
|
1616
|
+
console.error(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts} failed:`, err.message);
|
|
1617
|
+
|
|
1618
|
+
if (this._reconnectAttempts >= this._maxReconnectAttempts) {
|
|
1619
|
+
console.error(`[RabbitMQClient] ✗ Connection-level recovery failed after ${this._maxReconnectAttempts} attempts`);
|
|
1620
|
+
this._reconnecting = false;
|
|
1621
|
+
throw new Error(`Failed to reconnect after ${this._maxReconnectAttempts} attempts: ${err.message}`);
|
|
1622
|
+
}
|
|
1623
|
+
// Continue to next attempt
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
/**
|
|
1629
|
+
* Attach event handlers to publisher channel
|
|
1630
|
+
* @private
|
|
1631
|
+
*/
|
|
1632
|
+
_attachPublisherChannelHandlers(channel) {
|
|
1633
|
+
channel.on('error', async (err) => {
|
|
1634
|
+
const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
1635
|
+
channel._closeReason = reason;
|
|
1636
|
+
|
|
1637
|
+
// "unknown delivery tag" errors occur when channel closes during publish
|
|
1638
|
+
// This is expected - the publish callback will handle retry
|
|
1639
|
+
// We still log it but don't try to recreate channel (it's already closing)
|
|
1640
|
+
if (err.message && err.message.includes('unknown delivery tag')) {
|
|
1641
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Publisher channel error (channel closing during publish): ${err.message}`);
|
|
1642
|
+
// Don't try to recreate - channel is already closing
|
|
1643
|
+
// Don't emit error - publish callback will handle retry
|
|
1644
|
+
return;
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
console.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
1648
|
+
this._callChannelCloseHooks('publisher', channel, err, reason);
|
|
1649
|
+
|
|
1650
|
+
// Only try to recreate if connection is available
|
|
1651
|
+
// If connection is closed, reconnection logic will handle channel recreation
|
|
1652
|
+
if (this._connection && !this._connection.closed) {
|
|
1653
|
+
try {
|
|
1654
|
+
await this._ensurePublisherChannel();
|
|
1655
|
+
} catch (recreateErr) {
|
|
1656
|
+
// Ignore errors during channel recreation if connection is closing
|
|
1657
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${recreateErr.message}`);
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
this.emit('error', err);
|
|
1662
|
+
});
|
|
1663
|
+
|
|
1664
|
+
channel.on('close', async () => {
|
|
1665
|
+
const reason = channel._closeReason || 'Publisher channel closed unexpectedly';
|
|
1666
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
|
|
1667
|
+
this._callChannelCloseHooks('publisher', channel, null, reason);
|
|
1668
|
+
const closedChannel = this._channel;
|
|
1669
|
+
this._channel = null;
|
|
1670
|
+
|
|
1671
|
+
// Only try to recreate if connection is available
|
|
1672
|
+
// If connection is closed, reconnection logic will handle channel recreation
|
|
1673
|
+
if (this._connection && !this._connection.closed) {
|
|
1674
|
+
try {
|
|
1675
|
+
await this._ensurePublisherChannel();
|
|
1676
|
+
} catch (err) {
|
|
1677
|
+
// Ignore errors during channel recreation if connection is closing
|
|
1678
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${err.message}`);
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
});
|
|
1682
|
+
|
|
1683
|
+
channel.on('drain', () => {
|
|
1684
|
+
console.log('[RabbitMQClient] [mq-client-core] Publisher channel drained');
|
|
1685
|
+
});
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
/**
|
|
1689
|
+
* Attach event handlers to queue channel
|
|
1690
|
+
* @private
|
|
1691
|
+
*/
|
|
1692
|
+
_attachQueueChannelHandlers(channel) {
|
|
1693
|
+
channel.on('error', async (err) => {
|
|
1694
|
+
const reason = `Queue channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
1695
|
+
channel._closeReason = reason;
|
|
1696
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
1697
|
+
this._callChannelCloseHooks('queue', channel, err, reason);
|
|
1698
|
+
await this._ensureQueueChannel();
|
|
1699
|
+
});
|
|
1700
|
+
|
|
1701
|
+
channel.on('close', async () => {
|
|
1702
|
+
const reason = channel._closeReason || 'Queue channel closed unexpectedly';
|
|
1703
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
|
|
1704
|
+
this._callChannelCloseHooks('queue', channel, null, reason);
|
|
1705
|
+
const closedChannel = this._queueChannel;
|
|
1706
|
+
this._queueChannel = null;
|
|
1707
|
+
|
|
1708
|
+
// PREDICTABLE: Only recreate if connection is available and not closing
|
|
1709
|
+
// If connection is closed, reconnection logic will handle channel recreation
|
|
1710
|
+
if (this._connection && !this._connection.closed && !this._reconnecting) {
|
|
1711
|
+
try {
|
|
1712
|
+
await this._ensureQueueChannel();
|
|
1713
|
+
} catch (err) {
|
|
1714
|
+
// If connection is closing, this is expected - don't throw
|
|
1715
|
+
if (err.message && err.message.includes('Connection closed')) {
|
|
1716
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Queue channel close during connection closure (expected - reconnection will handle)`);
|
|
1717
|
+
} else {
|
|
1718
|
+
// Log but don't throw - channel will be recreated on next use or during reconnection
|
|
1719
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel: ${err.message} (will be recreated on next use)`);
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
} else {
|
|
1723
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Queue channel closed - will be recreated during reconnection or on next use`);
|
|
1724
|
+
}
|
|
1725
|
+
});
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
/**
|
|
1729
|
+
* Attach event handlers to consumer channel
|
|
1730
|
+
* @private
|
|
1731
|
+
*/
|
|
1732
|
+
_attachConsumerChannelHandlers(channel) {
|
|
1733
|
+
channel.on('error', async (err) => {
|
|
1734
|
+
const reason = `Consumer channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
1735
|
+
channel._closeReason = reason;
|
|
1736
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
1737
|
+
this._callChannelCloseHooks('consumer', channel, err, reason);
|
|
1738
|
+
await this._ensureConsumerChannel();
|
|
1739
|
+
this.emit('error', err);
|
|
1740
|
+
});
|
|
1741
|
+
|
|
1742
|
+
channel.on('close', async () => {
|
|
1743
|
+
const reason = channel._closeReason || 'Consumer channel closed unexpectedly';
|
|
1744
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
|
|
1745
|
+
this._callChannelCloseHooks('consumer', channel, null, reason);
|
|
1746
|
+
const closedChannel = this._consumerChannel;
|
|
1747
|
+
this._consumerChannel = null;
|
|
1748
|
+
|
|
1749
|
+
// Only try to recreate if connection is available
|
|
1750
|
+
// If connection is closed, reconnection logic will handle channel recreation
|
|
1751
|
+
if (this._connection && !this._connection.closed) {
|
|
1752
|
+
try {
|
|
1753
|
+
await this._ensureConsumerChannel();
|
|
1754
|
+
} catch (err) {
|
|
1755
|
+
// Ignore errors during channel recreation if connection is closing
|
|
1756
|
+
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel (connection may be closing): ${err.message}`);
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
});
|
|
1760
|
+
}
|
|
1761
|
+
|
|
1762
|
+
/**
|
|
1763
|
+
* Check prefetch utilization for a queue and alert if threshold exceeded
|
|
1764
|
+
* @private
|
|
1765
|
+
* @param {string} queue - Queue name
|
|
1766
|
+
* @param {Object} tracking - Prefetch tracking object
|
|
1767
|
+
*/
|
|
1768
|
+
_checkPrefetchUtilization(queue, tracking) {
|
|
1769
|
+
if (!tracking || tracking.prefetchCount === 0) {
|
|
1770
|
+
return; // No prefetch set or tracking not available
|
|
1771
|
+
}
|
|
1772
|
+
|
|
1773
|
+
const utilization = tracking.inFlight / tracking.prefetchCount;
|
|
1774
|
+
|
|
1775
|
+
if (utilization >= this._prefetchUtilizationThreshold) {
|
|
1776
|
+
const message = `Prefetch utilization high: queue '${queue}' has ${tracking.inFlight}/${tracking.prefetchCount} messages in-flight (${Math.round(utilization * 100)}%, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`;
|
|
1777
|
+
console.warn(`[RabbitMQClient] [mq-client-core] ⚠️ ${message}`);
|
|
1778
|
+
|
|
1779
|
+
// Call alert callback if provided
|
|
1780
|
+
if (this._prefetchAlertCallback) {
|
|
1781
|
+
try {
|
|
1782
|
+
this._prefetchAlertCallback(queue, utilization, tracking.inFlight, tracking.prefetchCount);
|
|
1783
|
+
} catch (alertErr) {
|
|
1784
|
+
console.error(`[RabbitMQClient] [mq-client-core] Prefetch alert callback error:`, alertErr.message);
|
|
1785
|
+
}
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
// Emit event for external listeners
|
|
1789
|
+
this.emit('prefetch:high-utilization', {
|
|
1790
|
+
queue,
|
|
1791
|
+
utilization,
|
|
1792
|
+
inFlight: tracking.inFlight,
|
|
1793
|
+
prefetchCount: tracking.prefetchCount,
|
|
1794
|
+
threshold: this._prefetchUtilizationThreshold,
|
|
1795
|
+
timestamp: new Date().toISOString()
|
|
1796
|
+
});
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
/**
|
|
1801
|
+
* Start periodic prefetch monitoring
|
|
1802
|
+
* @private
|
|
1803
|
+
*/
|
|
1804
|
+
_startPrefetchMonitoring() {
|
|
1805
|
+
if (this._prefetchCheckTimer) {
|
|
1806
|
+
return; // Already started
|
|
1807
|
+
}
|
|
1808
|
+
|
|
1809
|
+
this._prefetchCheckTimer = setInterval(() => {
|
|
1810
|
+
const now = Date.now();
|
|
1811
|
+
for (const [queue, tracking] of this._prefetchTracking.entries()) {
|
|
1812
|
+
// Only check if we have recent activity (within last 30s)
|
|
1813
|
+
if (now - tracking.lastCheck < 30000) {
|
|
1814
|
+
this._checkPrefetchUtilization(queue, tracking);
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
}, this._prefetchCheckInterval);
|
|
1818
|
+
|
|
1819
|
+
console.log(`[RabbitMQClient] [mq-client-core] Prefetch monitoring started (interval: ${this._prefetchCheckInterval}ms, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`);
|
|
1820
|
+
}
|
|
1821
|
+
|
|
1822
|
+
/**
|
|
1823
|
+
* Stop prefetch monitoring
|
|
1824
|
+
* @private
|
|
1825
|
+
*/
|
|
1826
|
+
_stopPrefetchMonitoring() {
|
|
1827
|
+
if (this._prefetchCheckTimer) {
|
|
1828
|
+
clearInterval(this._prefetchCheckTimer);
|
|
1829
|
+
this._prefetchCheckTimer = null;
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
/**
|
|
1834
|
+
* Track channel operation for debugging
|
|
1835
|
+
* @private
|
|
1836
|
+
* @param {Object} channel - Channel object
|
|
1837
|
+
* @param {string} operation - Operation description
|
|
1838
|
+
*/
|
|
1839
|
+
_trackChannelOperation(channel, operation) {
|
|
1840
|
+
if (channel && typeof channel === 'object') {
|
|
1841
|
+
channel._lastOperation = operation;
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1065
1845
|
/**
|
|
1066
1846
|
* Acknowledges a message.
|
|
1067
1847
|
* @param {Object} msg - RabbitMQ message object.
|