@onlineapps/mq-client-core 5.1.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +329 -0
- package/README.md +74 -12
- package/package.json +2 -2
- package/src/BaseClient.js +80 -66
- package/src/buffer/InMemoryBuffer.js +66 -3
- package/src/buffer/MessageBuffer.js +3 -0
- package/src/config/configSchema.js +3 -13
- package/src/config/defaultConfig.js +3 -5
- package/src/config/queueConfig.js +35 -1
- package/src/index.js +22 -0
- package/src/layers/PublishLayer.js +95 -18
- package/src/transports/rabbitmqClient.js +366 -76
- package/src/utils/errorHandler.js +25 -0
- package/src/utils/publishErrors.js +49 -0
|
@@ -16,6 +16,7 @@ const {
|
|
|
16
16
|
AMQP_NOT_FOUND,
|
|
17
17
|
AMQP_PRECONDITION_FAILED,
|
|
18
18
|
TransientPublishError,
|
|
19
|
+
RecoveryEndedPublishError,
|
|
19
20
|
PermanentPublishError,
|
|
20
21
|
QueueNotFoundError,
|
|
21
22
|
classifyPublishError,
|
|
@@ -32,7 +33,8 @@ const {
|
|
|
32
33
|
CONNECTION_CLOSED_UNEXPECTEDLY,
|
|
33
34
|
connectionClosedUnexpectedly,
|
|
34
35
|
CONSUMER_QUEUE_MISSING,
|
|
35
|
-
CONSUMER_DEAD_LETTER_ROUTE_MISSING
|
|
36
|
+
CONSUMER_DEAD_LETTER_ROUTE_MISSING,
|
|
37
|
+
carryVerdict
|
|
36
38
|
} = require('../utils/errorHandler');
|
|
37
39
|
const { assertLogger } = require('@onlineapps/logger-contract');
|
|
38
40
|
const PublishLayer = require('../layers/PublishLayer');
|
|
@@ -338,6 +340,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
338
340
|
// the cycle cap is spent. From that point this client will not try again and
|
|
339
341
|
// it must not report itself as connected.
|
|
340
342
|
this._connectionFatal = false;
|
|
343
|
+
// The verdict itself — the error `_failFatally()` built, handed to `onFatal` and
|
|
344
|
+
// `connection:fatal`. Kept so that a later use of this client can carry THE SAME object
|
|
345
|
+
// as its `cause` (d.1281): one error, however many readers; never a copy.
|
|
346
|
+
this._connectionFatalError = null;
|
|
341
347
|
// Injected by the owner of this client (service/wrapper). The LIBRARY never
|
|
342
348
|
// ends the process — that decision belongs to whoever owns the lifecycle.
|
|
343
349
|
this._onFatal = this._config.onFatal;
|
|
@@ -412,9 +418,12 @@ class RabbitMQClient extends EventEmitter {
|
|
|
412
418
|
// Health reporting callbacks
|
|
413
419
|
this._healthReportCallback = this._config.healthReportCallback; // (health) => Promise<void>
|
|
414
420
|
this._healthCriticalCallback = this._config.healthCriticalCallback; // (health) => Promise<void>
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
421
|
+
// When the current critical stretch began — read only by the two log lines that open and
|
|
422
|
+
// close it ("Critical health detected at", "Health recovered after"). Since d.1275 it arms
|
|
423
|
+
// nothing: `criticalHealthShutdown` and the `health:shutdown` event it armed are gone
|
|
424
|
+
// (owner 2026-10-06), because ending the process is the lifecycle owner's decision
|
|
425
|
+
// (`_failFatally()`), and nothing listened.
|
|
426
|
+
this._criticalHealthStartTime = null;
|
|
418
427
|
|
|
419
428
|
// Publish layer (retry + buffer)
|
|
420
429
|
this._publishLayer = new PublishLayer({
|
|
@@ -426,6 +435,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
426
435
|
ttlMs: this._config.publishBufferTtlMs,
|
|
427
436
|
},
|
|
428
437
|
},
|
|
438
|
+
// The buffer's expiry clock is THIS client's timer rail (d.1275): its handle sits in
|
|
439
|
+
// `_activeTimers` like every other timer the client arms, so `disconnect()` ends it and
|
|
440
|
+
// `timers-have-one-evidence` can see it (d.759e).
|
|
441
|
+
timers: {
|
|
442
|
+
setTimeout: (callback, ms) => this._setTimeout(callback, ms),
|
|
443
|
+
clearTimeout: (handle) => this._clearTimeout(handle),
|
|
444
|
+
},
|
|
429
445
|
});
|
|
430
446
|
|
|
431
447
|
// Publish monitor (metriky) — counts only, reports through return values,
|
|
@@ -565,11 +581,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
565
581
|
/**
|
|
566
582
|
* THE definition of "this client's consumer is attached to that queue".
|
|
567
583
|
*
|
|
568
|
-
*
|
|
569
|
-
* the
|
|
570
|
-
*
|
|
571
|
-
*
|
|
572
|
-
* the
|
|
584
|
+
* Two facts, both about THIS client, and both are needed:
|
|
585
|
+
* - the consumer channel is alive (`_isChannelAlive()`, d.260) — an attachment lives
|
|
586
|
+
* on that channel and dies with it, while the entry keeps its tag until the
|
|
587
|
+
* re-registration on the next channel replaces it;
|
|
588
|
+
* - the entry carries a tag: the broker gives it to the attachment it created, and
|
|
589
|
+
* since d.386b / d.605 the entry carries it only while that attachment exists — a
|
|
590
|
+
* refused re-registration and a cancel sent by the broker both drop it while the
|
|
591
|
+
* consumer stays TRACKED, because the next recovery must find it on the list.
|
|
592
|
+
*
|
|
593
|
+
* Nothing here asks the broker, and nothing here reads the QUEUE channel. Until d.1275
|
|
594
|
+
* the health check counted `active` only inside its `checkQueue` loop, so a dead queue
|
|
595
|
+
* channel or an unanswered probe reported 0 attached for a client whose consumer channel
|
|
596
|
+
* was alive and receiving (INFRA 531, W224 on the dev scheduler).
|
|
573
597
|
*
|
|
574
598
|
* It is a method rather than an expression repeated twice for the reason
|
|
575
599
|
* `change-discipline.md` § One rail per concern states: until d.613
|
|
@@ -584,7 +608,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
584
608
|
* @private
|
|
585
609
|
*/
|
|
586
610
|
_isConsumerAttached(consumerInfo) {
|
|
587
|
-
return
|
|
611
|
+
return this._isChannelAlive(this._consumerChannel)
|
|
612
|
+
&& typeof consumerInfo?.consumerTag === 'string'
|
|
613
|
+
&& consumerInfo.consumerTag !== '';
|
|
588
614
|
}
|
|
589
615
|
|
|
590
616
|
/**
|
|
@@ -1059,6 +1085,33 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1059
1085
|
return error;
|
|
1060
1086
|
}
|
|
1061
1087
|
|
|
1088
|
+
/**
|
|
1089
|
+
* Carry a verdict that is ALREADY written — on the error a caller is about to receive.
|
|
1090
|
+
*
|
|
1091
|
+
* `_markBrokerRefusal()` and `_failFatally()` write the verdict; this does not write
|
|
1092
|
+
* one, it copies `code` and `reason` from a verdict the client produced onto the error
|
|
1093
|
+
* built around it, so the receiver reads it where every receiver reads it: on the error
|
|
1094
|
+
* itself (`BaseClient._consumerRefusal()`, `@onlineapps/service-wrapper` FÁZE 0.3/0.7 read
|
|
1095
|
+
* `error.code`, never a `cause` chain — a reader walking the chain would be a fallback
|
|
1096
|
+
* over a buried marker, `architecture-principles.md` §3). Measured (d.1294, finding of
|
|
1097
|
+
* d.1278/d.1281): the `consume()` refusals built around the recovery verdict kept
|
|
1098
|
+
* `MQ_CONNECTION_FATAL` one `cause` down, so `_consumerRefusal()` wrapped them in a
|
|
1099
|
+
* `ConsumeError` and the lifecycle owner never saw the verdict it ends the process on.
|
|
1100
|
+
*
|
|
1101
|
+
* An error that is not a fatal verdict (a spent cycle carries no code, by contract)
|
|
1102
|
+
* leaves the error untouched.
|
|
1103
|
+
*
|
|
1104
|
+
* @param {Error} error - the error being built for the caller; `cause` already set
|
|
1105
|
+
* @param {Error|null|undefined} verdict - what the recovery produced
|
|
1106
|
+
* @returns {Error} the same error, with the verdict readable on top when there is one
|
|
1107
|
+
* @private
|
|
1108
|
+
*/
|
|
1109
|
+
_carryVerdict(error, verdict) {
|
|
1110
|
+
// One body, in `utils/errorHandler.js`: `PublishLayer` and `BaseClient.publish()` carry the
|
|
1111
|
+
// same verdict the same way (d.1300), so the rule is not written three times.
|
|
1112
|
+
return carryVerdict(error, verdict);
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1062
1115
|
/**
|
|
1063
1116
|
* A recovery cycle is over and the client is NOT connected, but the broker
|
|
1064
1117
|
* never refused it — so this is not the end.
|
|
@@ -1106,6 +1159,24 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1106
1159
|
return standby;
|
|
1107
1160
|
}
|
|
1108
1161
|
|
|
1162
|
+
/**
|
|
1163
|
+
* Whether a recovery cycle may start on this client without the caller's action — the
|
|
1164
|
+
* ONE statement of the three states that forbid it (d.1319): the connection declared lost
|
|
1165
|
+
* (`_connectionFatal`), the caller closed the client (`_closedByCaller`), recovery disabled
|
|
1166
|
+
* by configuration (`reconnectEnabled: false`). Read by `_resumeRecovery()` (may the next use
|
|
1167
|
+
* start a cycle), by `_recoveryEndedForPublish()` (`replayable` — may a `reconnected` still
|
|
1168
|
+
* come to replay a held message) and by every rail that meets a closed connection
|
|
1169
|
+
* (`_requireQueueChannel()`, `cancelConsumer()`, `ack()`/`nack()`: wait for the reconnect, or
|
|
1170
|
+
* say through `_refusalWithoutRecovery()` why none is coming). Until d.1319 the first two
|
|
1171
|
+
* carried their own copy of these three checks.
|
|
1172
|
+
*
|
|
1173
|
+
* @returns {boolean}
|
|
1174
|
+
* @private
|
|
1175
|
+
*/
|
|
1176
|
+
_recoveryMayStart() {
|
|
1177
|
+
return !this._connectionFatal && !this._closedByCaller && this._reconnectEnabled === true;
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1109
1180
|
/**
|
|
1110
1181
|
* Somebody wants to use a client that is not connected. Start the next
|
|
1111
1182
|
* recovery cycle if the contract allows one, and say whether a recovery is now
|
|
@@ -1122,9 +1193,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1122
1193
|
*/
|
|
1123
1194
|
_resumeRecovery() {
|
|
1124
1195
|
if (this._reconnecting) return true;
|
|
1125
|
-
if (this.
|
|
1126
|
-
if (this._closedByCaller) return false;
|
|
1127
|
-
if (!this._reconnectEnabled) return false;
|
|
1196
|
+
if (!this._recoveryMayStart()) return false;
|
|
1128
1197
|
|
|
1129
1198
|
this._logger.info(
|
|
1130
1199
|
`[RabbitMQClient] [mq-client-core] Client used while disconnected - starting recovery cycle `
|
|
@@ -1232,6 +1301,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1232
1301
|
fatal.attempts = this._maxReconnectAttempts;
|
|
1233
1302
|
fatal.cycles = this._reconnectCycles;
|
|
1234
1303
|
fatal.cause = lastError;
|
|
1304
|
+
this._connectionFatalError = fatal;
|
|
1235
1305
|
|
|
1236
1306
|
// The log line is the RECORD and is unconditional; the two lines below are
|
|
1237
1307
|
// the NOTIFICATION, and only a caller that is not awaiting needs it.
|
|
@@ -1845,6 +1915,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1845
1915
|
* Close the connection an abandoned handshake produces, whenever it completes.
|
|
1846
1916
|
* A handshake that fails after its timeout opened nothing; its error adds
|
|
1847
1917
|
* nothing to the timeout the caller already received.
|
|
1918
|
+
*
|
|
1919
|
+
* The close waits for the broker's `close-ok` no longer than `brokerAnswerTimeout` — the
|
|
1920
|
+
* one budget every wait on the broker has (conf `mq-broker-answer-budget` 001), on the one
|
|
1921
|
+
* rail the teardown already uses (`_closeWithinTeardownBudget()` → `_awaitBrokerAnswer()`),
|
|
1922
|
+
* and an unanswered close drops the socket (`_dropUnansweredConnection()`) exactly as
|
|
1923
|
+
* `connect()` does for a half-open connection. Until d.1322 this was a bare
|
|
1924
|
+
* `await late.close()`: a broker that never answered it left the late connection, its
|
|
1925
|
+
* socket and its heartbeat alive for the life of the process — the handshake leak of
|
|
1926
|
+
* INFRA wave 5.
|
|
1848
1927
|
* @private
|
|
1849
1928
|
* @param {Promise<Object>} connectPromise - the handshake the timeout abandoned
|
|
1850
1929
|
* @param {string} phase - 'connect' or 'reconnect'
|
|
@@ -1856,11 +1935,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1856
1935
|
`[RabbitMQClient] Handshake completed after the ${phase} timeout (${this._connectTimeout} ms) - `
|
|
1857
1936
|
+ 'closing that connection, nothing in this client owns it'
|
|
1858
1937
|
);
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1938
|
+
const closing = await this._closeWithinTeardownBudget(
|
|
1939
|
+
`abandoned ${phase} handshake`,
|
|
1940
|
+
late,
|
|
1941
|
+
this._brokerAnswerTimeout
|
|
1942
|
+
);
|
|
1943
|
+
if (closing.status === 'unanswered') {
|
|
1944
|
+
this._dropUnansweredConnection(late);
|
|
1945
|
+
} else if (closing.status === 'refused') {
|
|
1862
1946
|
this._logger.warn(
|
|
1863
|
-
`[RabbitMQClient] Error closing the connection whose handshake outlived the ${phase} timeout: ${
|
|
1947
|
+
`[RabbitMQClient] Error closing the connection whose handshake outlived the ${phase} timeout: ${closing.error.message}`
|
|
1864
1948
|
);
|
|
1865
1949
|
}
|
|
1866
1950
|
},
|
|
@@ -2209,32 +2293,59 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2209
2293
|
// process.
|
|
2210
2294
|
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot recreate publisher channel: connection is closed (waiting for reconnection)');
|
|
2211
2295
|
|
|
2296
|
+
// Every way this branch ends without a channel is ONE class, `RecoveryEndedPublishError`
|
|
2297
|
+
// (B1, owner's decision 2026-10-06, d.1300): the message is held in the buffer and
|
|
2298
|
+
// replayed after the next `reconnected` / `unblocked`, like every other transiently
|
|
2299
|
+
// failed publish — one rail with the retry that waited for a reconnect and saw it fail
|
|
2300
|
+
// (`PublishLayer`). Until d.1300 the wait's ending was thrown raw and the "no recovery"
|
|
2301
|
+
// cases as a `ConnectionError`, so the caller got a `PublishError` with no `buffered` at
|
|
2302
|
+
// all and the message was not held (INFRA-DOCS ID254 point 1). The verdict of a fatal
|
|
2303
|
+
// ending rides on top (`_carryVerdict()`, d.1297); a spent cycle carries no code.
|
|
2212
2304
|
if (this._resumeRecovery()) {
|
|
2213
|
-
|
|
2305
|
+
try {
|
|
2306
|
+
await this._waitForReconnection();
|
|
2307
|
+
} catch (err) {
|
|
2308
|
+
throw this._recoveryEndedForPublish(role, `the recovery it waited for did not arrive - ${err.message}`, err);
|
|
2309
|
+
}
|
|
2214
2310
|
|
|
2215
2311
|
// After reconnection, try again
|
|
2216
2312
|
if (this._isChannelAlive(this._sendChannelOf(role))) {
|
|
2217
2313
|
return; // Channel was recreated during reconnection
|
|
2218
2314
|
}
|
|
2219
|
-
//
|
|
2315
|
+
// The wait ended on `reconnected`, and the connection is already gone again (its
|
|
2316
|
+
// `close` fired before this continuation ran). Nothing holds that second death as an
|
|
2317
|
+
// object, so there is no cause.
|
|
2220
2318
|
if (!this._connectionAlive) {
|
|
2221
|
-
throw
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
+ 'RABBITMQ_MAX_RECONNECT_CYCLES cycles in all, then emits connection:fatal.'
|
|
2319
|
+
throw this._recoveryEndedForPublish(
|
|
2320
|
+
role,
|
|
2321
|
+
'the recovery it waited for completed, but the connection is not alive any more',
|
|
2322
|
+
null
|
|
2226
2323
|
);
|
|
2227
2324
|
}
|
|
2228
2325
|
} else {
|
|
2229
|
-
// No recovery can be started: the connection is permanently lost (the
|
|
2230
|
-
//
|
|
2231
|
-
//
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2326
|
+
// No recovery can be started: the connection is permanently lost (the broker refused
|
|
2327
|
+
// it, or every cycle is spent), the caller ended this client, or recovery is disarmed
|
|
2328
|
+
// by configuration — the same three states, in the same order, as the consumer path
|
|
2329
|
+
// (`_consumerChannelWithoutRecovery()`, d.1281).
|
|
2330
|
+
if (this._connectionFatal) {
|
|
2331
|
+
const fatal = this._connectionFatalError;
|
|
2332
|
+
throw this._recoveryEndedForPublish(
|
|
2333
|
+
role,
|
|
2334
|
+
`the connection is permanently lost and no recovery may start - ${fatal ? fatal.message : 'the verdict was not recorded'}`,
|
|
2335
|
+
fatal
|
|
2336
|
+
);
|
|
2337
|
+
}
|
|
2338
|
+
if (this._closedByCaller) {
|
|
2339
|
+
throw this._recoveryEndedForPublish(
|
|
2340
|
+
role,
|
|
2341
|
+
'the caller closed this client (disconnect()) and no recovery may start — only an explicit connect() re-arms it',
|
|
2342
|
+
null
|
|
2343
|
+
);
|
|
2344
|
+
}
|
|
2345
|
+
throw this._recoveryEndedForPublish(
|
|
2346
|
+
role,
|
|
2347
|
+
'the connection is closed and recovery is disabled by configuration (reconnectEnabled: false)',
|
|
2348
|
+
null
|
|
2238
2349
|
);
|
|
2239
2350
|
}
|
|
2240
2351
|
}
|
|
@@ -2386,6 +2497,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2386
2497
|
* d.1003d), never assumed: until d.1003d this branch answered
|
|
2387
2498
|
* `_activeConsumers.size` as re-registered, counting a consumer whose
|
|
2388
2499
|
* re-registration the broker had refused (tracked, no tag — d.262, d.386b).
|
|
2500
|
+
* @throws {ConnectionError} when the recovery this call waited for did not arrive —
|
|
2501
|
+
* `cause` is the error that ended the wait, the same object the cycle produced
|
|
2502
|
+
* (`code: 'MQ_CONNECTION_FATAL'` when it was the last cycle, and `onFatal` has already
|
|
2503
|
+
* received it — the thrown error then carries that `code` and `reason` on top too
|
|
2504
|
+
* (`_carryVerdict()`, d.1297); no `code` for a spent cycle — the contract keeps
|
|
2505
|
+
* `MQ_CONNECTION_STANDBY` inside the client); when the connection is closed and no recovery may start
|
|
2506
|
+
* (`_consumerChannelWithoutRecovery()`: permanently lost — `cause` is the fatal error
|
|
2507
|
+
* `onFatal` received; closed by the caller; recovery disabled); when the recovery it
|
|
2508
|
+
* waited for completed but the connection is not alive; or when the open connection
|
|
2509
|
+
* refuses the channel.
|
|
2389
2510
|
*/
|
|
2390
2511
|
async _ensureConsumerChannel() {
|
|
2391
2512
|
if (this._isChannelAlive(this._consumerChannel)) {
|
|
@@ -2399,16 +2520,39 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2399
2520
|
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot recreate consumer channel: connection is closed (reconnection in progress)');
|
|
2400
2521
|
|
|
2401
2522
|
if (!this._resumeRecovery()) {
|
|
2402
|
-
|
|
2523
|
+
// No recovery may start, and the caller is told WHY (d.1281, finding of d.1278):
|
|
2524
|
+
// until then this branch answered `{ failed }`, `consume()` went on without a
|
|
2525
|
+
// channel and the caller got the queue-channel rail's "queue channel is not
|
|
2526
|
+
// available … Fix: await client.connect()" — the right fix for one of the three
|
|
2527
|
+
// states and wrong for the other two. The three are the three `_resumeRecovery()`
|
|
2528
|
+
// refuses, asked in its order; the fatal one carries the verdict `onFatal` already
|
|
2529
|
+
// received as `cause` — the same object, so there is no second rail of reporting.
|
|
2530
|
+
throw this._consumerChannelWithoutRecovery();
|
|
2403
2531
|
}
|
|
2404
2532
|
|
|
2405
2533
|
try {
|
|
2406
2534
|
await this._waitForReconnection();
|
|
2407
2535
|
} catch (err) {
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
)
|
|
2411
|
-
|
|
2536
|
+
// The wait's ending is the caller's answer, the way it is on the publisher path
|
|
2537
|
+
// (`_ensurePublisherChannel()` lets it through). Until d.1278 this branch wrote a
|
|
2538
|
+
// WARN line and answered `{ failed }` as if that were an outcome; `consume()` then
|
|
2539
|
+
// went on without a channel and the caller got the queue-channel rail's refusal
|
|
2540
|
+
// ("queue channel is not available … Fix: await client.connect()") — no cause, no
|
|
2541
|
+
// cycle, and the verdict of a fatal cycle reached `onFatal` and nobody else
|
|
2542
|
+
// (INFRA-DOCS ID254). The cause is the SAME object the cycle produced, never a
|
|
2543
|
+
// copy: `onFatal` and this caller hold one error between them, so there is no
|
|
2544
|
+
// second channel of reporting to keep in step.
|
|
2545
|
+
// The verdict rides on top (d.1297): `code`/`reason` of a fatal ending are copied
|
|
2546
|
+
// onto this error, so `BaseClient._consumerRefusal()` lets it through unchanged and
|
|
2547
|
+
// the lifecycle owner reads `error.code` as everywhere else.
|
|
2548
|
+
throw this._carryVerdict(new ConnectionError(
|
|
2549
|
+
`[RabbitMQClient] Cannot recreate the consumer channel: the recovery it waited for did not arrive - ${err.message} `
|
|
2550
|
+
+ 'Expected: the recovery to re-establish the connection before a consumer attaches. '
|
|
2551
|
+
+ 'Fix: read error.code — MQ_CONNECTION_FATAL ends this client (onFatal has been called; error.cause is '
|
|
2552
|
+
+ 'that verdict, error.reason says broker-refused or cycles-spent); a spent cycle carries no code and is '
|
|
2553
|
+
+ 'an outage: the next consume(), publish() or performHealthCheck() starts the next cycle.',
|
|
2554
|
+
err
|
|
2555
|
+
), err);
|
|
2412
2556
|
}
|
|
2413
2557
|
|
|
2414
2558
|
if (this._isChannelAlive(this._consumerChannel)) {
|
|
@@ -2417,7 +2561,18 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2417
2561
|
return this._consumerAttachmentCounts();
|
|
2418
2562
|
}
|
|
2419
2563
|
if (!this._connectionAlive) {
|
|
2420
|
-
|
|
2564
|
+
// The wait ended on `reconnected`, and the connection is already gone again
|
|
2565
|
+
// (its `close` fired before this continuation ran). Nothing holds that second
|
|
2566
|
+
// death as an object here, so there is no `cause`; the sentence says what
|
|
2567
|
+
// happened and what starts the next cycle (d.1281).
|
|
2568
|
+
throw new ConnectionError(
|
|
2569
|
+
'[RabbitMQClient] Cannot recreate the consumer channel: the recovery it waited for completed, '
|
|
2570
|
+
+ 'but the connection is not alive any more - '
|
|
2571
|
+
+ 'Expected: a live connection at the end of the recovery. '
|
|
2572
|
+
+ "Fix: run consume() again — the connection's close handler starts the next recovery cycle; "
|
|
2573
|
+
+ 'if it keeps ending here, the broker is dropping the connection right after accepting it '
|
|
2574
|
+
+ '(read the broker log).'
|
|
2575
|
+
);
|
|
2421
2576
|
}
|
|
2422
2577
|
}
|
|
2423
2578
|
|
|
@@ -2465,6 +2620,104 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2465
2620
|
}
|
|
2466
2621
|
}
|
|
2467
2622
|
|
|
2623
|
+
/**
|
|
2624
|
+
* The refusal a consumer gets when the connection is closed and `_resumeRecovery()`
|
|
2625
|
+
* will not start a recovery — one sentence per state, in the order `_resumeRecovery()`
|
|
2626
|
+
* asks them (d.1281).
|
|
2627
|
+
*
|
|
2628
|
+
* - the connection is permanently lost (`_connectionFatal`): `cause` is THE fatal
|
|
2629
|
+
* error, the object `_failFatally()` handed to `onFatal` / `connection:fatal`;
|
|
2630
|
+
* - the caller closed this client (`_closedByCaller`): no cause — `disconnect()` is
|
|
2631
|
+
* not an error, and only `connect()` undoes it;
|
|
2632
|
+
* - recovery is disabled by configuration (`reconnectEnabled: false`): no cause.
|
|
2633
|
+
*
|
|
2634
|
+
* @returns {ConnectionError}
|
|
2635
|
+
* @private
|
|
2636
|
+
*/
|
|
2637
|
+
_consumerChannelWithoutRecovery() {
|
|
2638
|
+
return this._refusalWithoutRecovery('[RabbitMQClient] Cannot recreate the consumer channel', 'consume');
|
|
2639
|
+
}
|
|
2640
|
+
|
|
2641
|
+
/**
|
|
2642
|
+
* ONE builder of the refusal for "the connection is closed and no recovery may start" —
|
|
2643
|
+
* three states, three sentences, read in `_resumeRecovery()`'s order (d.1281, d.1314).
|
|
2644
|
+
* Every operation that meets a closed connection it cannot wait out says it through this
|
|
2645
|
+
* function: the consumer channel (`_consumerChannelWithoutRecovery()`), the queue-channel
|
|
2646
|
+
* rail (`_requireQueueChannel()`), `cancelConsumer()`, `ack()`/`nack()` — so the verdict of a
|
|
2647
|
+
* lost connection is written on the error once and the same way everywhere, and nothing
|
|
2648
|
+
* advises `connect()` to a client whose owner must end the process instead.
|
|
2649
|
+
*
|
|
2650
|
+
* @param {string} headline - `[RabbitMQClient] Cannot <what>` — what was asked
|
|
2651
|
+
* @param {string} verb - the public operation, named in the `disconnect()` Fix
|
|
2652
|
+
* @returns {ConnectionError} with the fatal verdict carried on top when that is the state
|
|
2653
|
+
* @private
|
|
2654
|
+
*/
|
|
2655
|
+
_refusalWithoutRecovery(headline, verb) {
|
|
2656
|
+
if (this._connectionFatal) {
|
|
2657
|
+
const fatal = this._connectionFatalError;
|
|
2658
|
+
return this._carryVerdict(new ConnectionError(
|
|
2659
|
+
`${headline}: the connection is permanently lost and no `
|
|
2660
|
+
+ `recovery may start - ${fatal ? fatal.message : 'the verdict was not recorded'} `
|
|
2661
|
+
+ 'Expected: a client whose connection was not declared lost. '
|
|
2662
|
+
+ 'Fix: this client is over — error.code is MQ_CONNECTION_FATAL and error.cause the verdict onFatal '
|
|
2663
|
+
+ '(and connection:fatal) already carried; the owner of the client stops the process so its restart '
|
|
2664
|
+
+ 'policy boots a new one against a verified broker.',
|
|
2665
|
+
fatal || undefined
|
|
2666
|
+
), fatal);
|
|
2667
|
+
}
|
|
2668
|
+
if (this._closedByCaller) {
|
|
2669
|
+
return new ConnectionError(
|
|
2670
|
+
`${headline}: the caller closed this client (disconnect()) `
|
|
2671
|
+
+ 'and no recovery may start - '
|
|
2672
|
+
+ 'Expected: a client that has not been disconnected. '
|
|
2673
|
+
+ `Fix: await client.connect() before ${verb}(); only an explicit connect() re-arms recovery.`
|
|
2674
|
+
);
|
|
2675
|
+
}
|
|
2676
|
+
return new ConnectionError(
|
|
2677
|
+
`${headline}: the connection is closed and recovery is `
|
|
2678
|
+
+ 'disabled by configuration (reconnectEnabled: false) - '
|
|
2679
|
+
+ 'Expected: either an open connection, or a client allowed to recover one. '
|
|
2680
|
+
+ 'Fix: await client.connect() to open a new connection, or run the client with reconnectEnabled: true '
|
|
2681
|
+
+ 'so the next consume(), publish() or performHealthCheck() starts a recovery cycle.'
|
|
2682
|
+
);
|
|
2683
|
+
}
|
|
2684
|
+
|
|
2685
|
+
/**
|
|
2686
|
+
* The refusal a publish gets when its send channel cannot be had because the connection
|
|
2687
|
+
* stayed down (B1, d.1300) — one class for every ending of that branch, with the reason in
|
|
2688
|
+
* the sentence and the verdict, when there is one, on top.
|
|
2689
|
+
*
|
|
2690
|
+
* @param {'publisher'|'telemetry'} role - whose channel it was
|
|
2691
|
+
* @param {string} why - what ended it, already including the wait's own sentence when one exists
|
|
2692
|
+
* @param {Error|null} ended - the error that ended the recovery (the fatal verdict is the very
|
|
2693
|
+
* object `onFatal` received), or null when nothing holds the state as an object
|
|
2694
|
+
* @returns {RecoveryEndedPublishError}
|
|
2695
|
+
* @private
|
|
2696
|
+
*/
|
|
2697
|
+
_recoveryEndedForPublish(role, why, ended) {
|
|
2698
|
+
// Whether a `reconnected` can still come on this client without the caller's action —
|
|
2699
|
+
// read from the state AFTER the ending, so a wait that ended fatal answers false. The
|
|
2700
|
+
// three states that refuse `_resumeRecovery()` are exactly the three where it never comes
|
|
2701
|
+
// (d.1301, lead's decision: a hold makes sense only where a reconnect can replay it).
|
|
2702
|
+
const replayable = this._recoveryMayStart();
|
|
2703
|
+
const fix = replayable
|
|
2704
|
+
? 'Fix: nothing to retry here — PublishLayer holds the message in the buffer (unless the call passed '
|
|
2705
|
+
+ 'bufferOnFailure: false) and replays it after the next reconnect or after the broker unblocks the '
|
|
2706
|
+
+ 'connection; the next publish(), consume() or performHealthCheck() starts the next recovery cycle. '
|
|
2707
|
+
: 'Fix: this client will not reconnect on its own, so PublishLayer holds nothing (error.buffered === false) — '
|
|
2708
|
+
+ 'the caller holds the only copy; the lifecycle owner decides what happens to the process (onFatal), '
|
|
2709
|
+
+ 'and only an explicit connect() would open a new connection. ';
|
|
2710
|
+
return this._carryVerdict(new RecoveryEndedPublishError(
|
|
2711
|
+
`[RabbitMQClient] Cannot recreate the ${role} channel: ${why} `
|
|
2712
|
+
+ 'Expected: a live connection before the send. '
|
|
2713
|
+
+ fix
|
|
2714
|
+
+ 'error.code MQ_CONNECTION_FATAL (with error.reason) means this client is over and onFatal has been '
|
|
2715
|
+
+ 'called; error.cause is what ended the recovery, when an object exists for it.',
|
|
2716
|
+
ended || undefined,
|
|
2717
|
+
{ replayable }
|
|
2718
|
+
), ended);
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2468
2721
|
/**
|
|
2469
2722
|
* Re-register every tracked consumer on the current consumer channel.
|
|
2470
2723
|
*
|
|
@@ -2793,7 +3046,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2793
3046
|
queues: {
|
|
2794
3047
|
checked: 0,
|
|
2795
3048
|
missing: [],
|
|
2796
|
-
noConsumer: []
|
|
3049
|
+
noConsumer: [],
|
|
3050
|
+
// The BROKER's count of consumers on each tracked queue (`checkQueue().consumerCount`
|
|
3051
|
+
// — everybody's, not only this client's), or 'unknown' where the probe did not run
|
|
3052
|
+
// or got no answer. Never 0 for "not asked" (d.1275).
|
|
3053
|
+
consumers: {}
|
|
2797
3054
|
},
|
|
2798
3055
|
healthy: true,
|
|
2799
3056
|
issues: []
|
|
@@ -2819,7 +3076,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2819
3076
|
health.issues.push('Consumer channel is closed or missing');
|
|
2820
3077
|
}
|
|
2821
3078
|
|
|
2822
|
-
//
|
|
3079
|
+
// Which of this client's consumers are attached is THIS client's fact, counted from its
|
|
3080
|
+
// own state — the consumer channel and the tags (`_isConsumerAttached()`, the definition
|
|
3081
|
+
// `getConsumerState()` reads too, d.613). Counted BEFORE the broker is asked and whatever
|
|
3082
|
+
// the broker answers: until d.1275 `active` was incremented only inside the probe loop
|
|
3083
|
+
// below, so a dead queue channel or an unanswered `checkQueue` left it at 0 and the
|
|
3084
|
+
// verdict further down told the operator a consuming client consumed nothing (INFRA 531).
|
|
3085
|
+
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
3086
|
+
health.queues.consumers[queue] = 'unknown';
|
|
3087
|
+
if (this._isConsumerAttached(consumerInfo)) {
|
|
3088
|
+
health.consumers.active++;
|
|
3089
|
+
} else {
|
|
3090
|
+
health.consumers.inactive.push(queue);
|
|
3091
|
+
}
|
|
3092
|
+
}
|
|
3093
|
+
|
|
3094
|
+
// The probe only ADDS what the broker knows: does each queue exist, and how many
|
|
3095
|
+
// consumers does it count on it. Where it cannot ask, the count stays 'unknown'.
|
|
2823
3096
|
if (this.isConnected() && this._isChannelAlive(this._queueChannel)) {
|
|
2824
3097
|
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
2825
3098
|
health.queues.checked++;
|
|
@@ -2832,25 +3105,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2832
3105
|
() => queueChannel.checkQueue(queue)
|
|
2833
3106
|
);
|
|
2834
3107
|
if (queueInfo) {
|
|
2835
|
-
// Queue exists
|
|
2836
|
-
//
|
|
2837
|
-
//
|
|
2838
|
-
//
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
// this client's and everybody else's alike, so it cannot say whether
|
|
2842
|
-
// the one asking is among them. The tag can: it is the name the
|
|
2843
|
-
// broker gave THIS client's attachment, and since d.386b it is set
|
|
2844
|
-
// only while that attachment exists. The question is asked through
|
|
2845
|
-
// `_isConsumerAttached()` — the one definition `getConsumerState()`
|
|
2846
|
-
// now reads too (d.613).
|
|
2847
|
-
if (this._isConsumerAttached(consumerInfo)) {
|
|
2848
|
-
health.consumers.active++;
|
|
2849
|
-
} else {
|
|
2850
|
-
health.consumers.inactive.push(queue);
|
|
3108
|
+
// Queue exists. The broker's `consumerCount` counts every consumer on the
|
|
3109
|
+
// queue, this client's and everybody else's alike, so it is REPORTED, never
|
|
3110
|
+
// used to decide whether the one asking is among them — that is
|
|
3111
|
+
// `_isConsumerAttached()`, counted above.
|
|
3112
|
+
health.queues.consumers[queue] = queueInfo.consumerCount;
|
|
3113
|
+
if (!this._isConsumerAttached(consumerInfo)) {
|
|
2851
3114
|
health.queues.noConsumer.push(queue);
|
|
2852
3115
|
health.healthy = false;
|
|
2853
|
-
health.issues.push(
|
|
3116
|
+
health.issues.push(
|
|
3117
|
+
this._isChannelAlive(this._consumerChannel)
|
|
3118
|
+
? `Queue ${queue} has no active consumer (consumerTag missing)`
|
|
3119
|
+
: `Queue ${queue} has no active consumer (consumer channel is not alive)`
|
|
3120
|
+
);
|
|
2854
3121
|
}
|
|
2855
3122
|
} else {
|
|
2856
3123
|
health.queues.missing.push(queue);
|
|
@@ -2874,12 +3141,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2874
3141
|
// that is its own verdict — not something a reader infers from two numbers
|
|
2875
3142
|
// sitting beside the channel findings. The per-queue issues above name WHICH
|
|
2876
3143
|
// queue lost its consumer, and they do not always exist: when the probe loop
|
|
2877
|
-
// does not run (no connection, or a dead queue channel)
|
|
2878
|
-
// consumer evidence is `tracked` and `active`, and every issue names a
|
|
3144
|
+
// does not run (no connection, or a dead queue channel) every issue names a
|
|
2879
3145
|
// channel instead. Measured 2026-09-08 (INFRA-monitoring): the whole account
|
|
2880
3146
|
// a reader got of a client taking no message off any queue was the line
|
|
2881
3147
|
// `Health check OK: 0/0 consumers active`.
|
|
2882
3148
|
//
|
|
3149
|
+
// `active` is this client's own count (above), so the verdict is never the
|
|
3150
|
+
// probe's absence restated: a dead queue channel beside a live consumer
|
|
3151
|
+
// channel is a channel finding, not "no consumer" (d.1275).
|
|
3152
|
+
//
|
|
2883
3153
|
// `tracked === 0` is a different client, not a lesser failure: a publisher
|
|
2884
3154
|
// registered no consumer and has none to lose. It stays healthy, and the OK
|
|
2885
3155
|
// line below says which of the two it is.
|
|
@@ -2898,7 +3168,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2898
3168
|
// Emit health check result
|
|
2899
3169
|
this.emit('health:check', health);
|
|
2900
3170
|
|
|
2901
|
-
// If unhealthy, log and
|
|
3171
|
+
// If unhealthy, log and report
|
|
2902
3172
|
if (!health.healthy) {
|
|
2903
3173
|
this._logger.error(`[RabbitMQClient] [mq-client-core] Health check FAILED:`, health.issues);
|
|
2904
3174
|
this._logger.error(`[RabbitMQClient] [mq-client-core] Health details:`, JSON.stringify(health, null, 2));
|
|
@@ -2927,21 +3197,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2927
3197
|
}
|
|
2928
3198
|
}
|
|
2929
3199
|
|
|
2930
|
-
// Emit critical health event
|
|
3200
|
+
// Emit critical health event. That is the whole of it: the client REPORTS critical
|
|
3201
|
+
// health and decides nothing about the process. `health:shutdown` and the
|
|
3202
|
+
// `criticalHealthShutdown*` keys that armed it were removed in d.1275 (owner
|
|
3203
|
+
// 2026-10-06) — the event reached no listener anywhere, and the line announcing it
|
|
3204
|
+
// promised a shutdown nobody performed. A lifecycle owner that wants to act on a
|
|
3205
|
+
// sustained critical state reads this event or `healthCriticalCallback`.
|
|
2931
3206
|
this.emit('health:critical', health);
|
|
2932
|
-
|
|
2933
|
-
// Check if we should shutdown after delay
|
|
2934
|
-
if (this._criticalHealthShutdown) {
|
|
2935
|
-
const timeSinceCritical = Date.now() - this._criticalHealthStartTime;
|
|
2936
|
-
if (timeSinceCritical >= this._criticalHealthShutdownDelay) {
|
|
2937
|
-
this._logger.error(`[RabbitMQClient] [mq-client-core] Critical health persisted for ${timeSinceCritical}ms (threshold: ${this._criticalHealthShutdownDelay}ms) - triggering shutdown`);
|
|
2938
|
-
this.emit('health:shutdown', health);
|
|
2939
|
-
// Note: Actual shutdown should be handled by the application using this event
|
|
2940
|
-
} else {
|
|
2941
|
-
const remaining = this._criticalHealthShutdownDelay - timeSinceCritical;
|
|
2942
|
-
this._logger.warn(`[RabbitMQClient] [mq-client-core] Critical health will trigger shutdown in ${remaining}ms if not resolved`);
|
|
2943
|
-
}
|
|
2944
|
-
}
|
|
2945
3207
|
} else {
|
|
2946
3208
|
// Health is OK - reset critical health tracking
|
|
2947
3209
|
if (this._criticalHealthStartTime) {
|
|
@@ -3286,6 +3548,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3286
3548
|
async _requireQueueChannel(verb, queue) {
|
|
3287
3549
|
await this._ensureQueueChannel();
|
|
3288
3550
|
if (!this._isChannelAlive(this._queueChannel)) {
|
|
3551
|
+
// A connection that is down and will not recover on its own is not "wait for the
|
|
3552
|
+
// reconnect": none is coming. The three states say so through the one builder every rail
|
|
3553
|
+
// uses — the verdict on top for a lost connection (d.1314), the state named for
|
|
3554
|
+
// `disconnect()` and `reconnectEnabled: false` (d.1319). A spent cycle (standby) keeps the
|
|
3555
|
+
// sentence below: the next publish, consume or health check starts the next one.
|
|
3556
|
+
if (!this._connectionAlive && !this._recoveryMayStart()) {
|
|
3557
|
+
throw this._refusalWithoutRecovery(`[RabbitMQClient] Cannot ${verb} ${queue}`, verb);
|
|
3558
|
+
}
|
|
3289
3559
|
throw new ConnectionError(
|
|
3290
3560
|
`[RabbitMQClient] Cannot ${verb} ${queue}: queue channel is not available - `
|
|
3291
3561
|
+ 'Expected: an open queue channel after _ensureQueueChannel(). '
|
|
@@ -3824,6 +4094,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3824
4094
|
// its tag named is gone (d.386b). There is nothing to cancel on the broker,
|
|
3825
4095
|
// and asking anyway is a channel error rather than a cancel, so the caller
|
|
3826
4096
|
// is told plainly instead of being handed one.
|
|
4097
|
+
// Asked before the tag: on a connection that is down and will not recover the tag is gone
|
|
4098
|
+
// too, and the sentence that matters is the state — the verdict for a lost connection
|
|
4099
|
+
// (d.1314), `disconnect()` or `reconnectEnabled: false` otherwise (d.1319) — not "tracked
|
|
4100
|
+
// but not attached".
|
|
4101
|
+
if (!this._connectionAlive && !this._recoveryMayStart()) {
|
|
4102
|
+
throw this._refusalWithoutRecovery(`[RabbitMQClient] Cannot cancelConsumer ${queue}`, 'cancelConsumer');
|
|
4103
|
+
}
|
|
4104
|
+
|
|
3827
4105
|
if (info.consumerTag === null || info.consumerTag === undefined) {
|
|
3828
4106
|
throw new ValidationError(
|
|
3829
4107
|
`[RabbitMQClient] Cannot cancelConsumer ${queue}: the consumer is tracked but not attached - `
|
|
@@ -5687,7 +5965,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
5687
5965
|
// catch polykal jako `warn`, takže konzumenti tiše přestali dostávat zprávy.
|
|
5688
5966
|
// `_ensureConsumerChannel()` odpovídá `ConnectionError` s chybou amqplibu
|
|
5689
5967
|
// v `cause` — jeden článek, vlastní příčina obalu (vzor obnovy frontového
|
|
5690
|
-
// kanálu níž, d.696).
|
|
5968
|
+
// kanálu níž, d.696). Od d.1278 i s chybou, kterou skončilo čekání na obnovu
|
|
5969
|
+
// (standby bez `code`, nebo fatální s `MQ_CONNECTION_FATAL`): spojení tehdy
|
|
5970
|
+
// neživé, takže `_connectionIsGoingAway()` odpoví `warn` — standby i fatal už
|
|
5971
|
+
// ohlásily samy sebe na vlastním kanálu, druhé hlášení na `error` sem nepatří.
|
|
5691
5972
|
if (this._connectionIsGoingAway(err.cause)) {
|
|
5692
5973
|
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel (connection may be closing): ${err.message}`);
|
|
5693
5974
|
} else {
|
|
@@ -6116,6 +6397,12 @@ class RabbitMQClient extends EventEmitter {
|
|
|
6116
6397
|
if (msg._mqProcessed) {
|
|
6117
6398
|
return; // Already acked/nacked — idempotent
|
|
6118
6399
|
}
|
|
6400
|
+
// On a connection that is down and will not recover on its own (lost, `disconnect()`,
|
|
6401
|
+
// `reconnectEnabled: false`) the delivery cannot be settled — the broker has requeued it
|
|
6402
|
+
// already; the caller learns the state, the verdict included (d.1314, d.1319).
|
|
6403
|
+
if (!this._connectionAlive && !this._recoveryMayStart()) {
|
|
6404
|
+
throw this._refusalWithoutRecovery('[RabbitMQClient] Cannot ack', 'ack');
|
|
6405
|
+
}
|
|
6119
6406
|
if (!this._consumerChannel) {
|
|
6120
6407
|
throw new ConnectionError(
|
|
6121
6408
|
'[RabbitMQClient] Cannot ack: consumer channel is not initialized - Expected: an open consumer channel. '
|
|
@@ -6158,6 +6445,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
6158
6445
|
if (msg._mqProcessed) {
|
|
6159
6446
|
return; // Already acked/nacked — idempotent
|
|
6160
6447
|
}
|
|
6448
|
+
if (!this._connectionAlive && !this._recoveryMayStart()) {
|
|
6449
|
+
throw this._refusalWithoutRecovery('[RabbitMQClient] Cannot nack', 'nack');
|
|
6450
|
+
}
|
|
6161
6451
|
if (!this._consumerChannel) {
|
|
6162
6452
|
throw new ConnectionError(
|
|
6163
6453
|
'[RabbitMQClient] Cannot nack: consumer channel is not initialized - Expected: an open consumer channel. '
|