@onlineapps/mq-client-core 5.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +474 -0
- package/README.md +117 -13
- package/package.json +2 -2
- package/src/BaseClient.js +101 -68
- package/src/buffer/InMemoryBuffer.js +161 -13
- package/src/buffer/MessageBuffer.js +8 -3
- package/src/config/configSchema.js +15 -19
- package/src/config/defaultConfig.js +10 -11
- package/src/config/queueConfig.js +35 -1
- package/src/index.js +22 -0
- package/src/layers/PublishLayer.js +241 -35
- package/src/monitoring-initiated.js +6 -9
- package/src/transports/rabbitmqClient.js +912 -151
- package/src/utils/errorHandler.js +25 -0
- package/src/utils/publishErrors.js +49 -0
|
@@ -16,6 +16,7 @@ const {
|
|
|
16
16
|
AMQP_NOT_FOUND,
|
|
17
17
|
AMQP_PRECONDITION_FAILED,
|
|
18
18
|
TransientPublishError,
|
|
19
|
+
RecoveryEndedPublishError,
|
|
19
20
|
PermanentPublishError,
|
|
20
21
|
QueueNotFoundError,
|
|
21
22
|
classifyPublishError,
|
|
@@ -32,7 +33,8 @@ const {
|
|
|
32
33
|
CONNECTION_CLOSED_UNEXPECTEDLY,
|
|
33
34
|
connectionClosedUnexpectedly,
|
|
34
35
|
CONSUMER_QUEUE_MISSING,
|
|
35
|
-
CONSUMER_DEAD_LETTER_ROUTE_MISSING
|
|
36
|
+
CONSUMER_DEAD_LETTER_ROUTE_MISSING,
|
|
37
|
+
carryVerdict
|
|
36
38
|
} = require('../utils/errorHandler');
|
|
37
39
|
const { assertLogger } = require('@onlineapps/logger-contract');
|
|
38
40
|
const PublishLayer = require('../layers/PublishLayer');
|
|
@@ -125,11 +127,10 @@ const REFUSAL_REPLY_CODES = [403, 530, 406];
|
|
|
125
127
|
* - `workflow_id` is the only spelling the envelope contract admits, and a
|
|
126
128
|
* reader that accepts the camelCase alias is forbidden as such, not merely a
|
|
127
129
|
* producer that writes it (`docs/biz/70-contracts/workflow-message.md` §2.2,
|
|
128
|
-
* owner decision `workflow-message-aliases` 001).
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
* a reader that
|
|
132
|
-
* retire, because nothing then shows which producers still write it.
|
|
130
|
+
* owner decision `workflow-message-aliases` 001). No producer in `shared/**`
|
|
131
|
+
* or `infra/**` writes the alias (re-measured d.1134: the gateway's headers
|
|
132
|
+
* carry `workflow_id` alone, `infra/api_gateway/shared/workflowMessageHeaders.js`),
|
|
133
|
+
* and a reader that honoured it would hide the next producer that did.
|
|
133
134
|
* - a missing id is reported as missing. `'unknown'` is a value, and a log line
|
|
134
135
|
* carrying it claims a workflow by that name — the same reason
|
|
135
136
|
* `BaseClient._emitDeadLetterEvent()` resolves an absent id to `null` and
|
|
@@ -282,6 +283,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
282
283
|
// socket whose channels are still being recreated is not yet usable (d.340).
|
|
283
284
|
// Channels do the same through `channel._alive` (see `_isChannelAlive()`).
|
|
284
285
|
this._connectionAlive = false;
|
|
286
|
+
// The broker's reason while it blocks this connection (`connection.blocked`), `null` otherwise
|
|
287
|
+
// (d.1145). Written by `_onConnectionBlocked()` / `_onConnectionUnblocked()`.
|
|
288
|
+
this._connectionBlockedBy = null;
|
|
285
289
|
this._channel = null; // ConfirmChannel for publish operations
|
|
286
290
|
this._queueChannel = null; // Regular channel for queue operations (assertQueue, checkQueue)
|
|
287
291
|
this._consumerChannel = null; // Dedicated channel for consume operations
|
|
@@ -336,6 +340,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
336
340
|
// the cycle cap is spent. From that point this client will not try again and
|
|
337
341
|
// it must not report itself as connected.
|
|
338
342
|
this._connectionFatal = false;
|
|
343
|
+
// The verdict itself — the error `_failFatally()` built, handed to `onFatal` and
|
|
344
|
+
// `connection:fatal`. Kept so that a later use of this client can carry THE SAME object
|
|
345
|
+
// as its `cause` (d.1281): one error, however many readers; never a copy.
|
|
346
|
+
this._connectionFatalError = null;
|
|
339
347
|
// Injected by the owner of this client (service/wrapper). The LIBRARY never
|
|
340
348
|
// ends the process — that decision belongs to whoever owns the lifecycle.
|
|
341
349
|
this._onFatal = this._config.onFatal;
|
|
@@ -366,12 +374,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
366
374
|
this._publishRetryBaseDelay = this._config.publishRetryBaseDelay;
|
|
367
375
|
this._publishRetryMaxDelay = this._config.publishRetryMaxDelay;
|
|
368
376
|
this._publishRetryBackoffMultiplier = this._config.publishRetryBackoffMultiplier;
|
|
369
|
-
// ONE number,
|
|
370
|
-
//
|
|
371
|
-
// publish confirm waits for it (`_publishOnce`),
|
|
372
|
-
//
|
|
373
|
-
//
|
|
374
|
-
//
|
|
377
|
+
// ONE number, and therefore a name that belongs to none of its consumers
|
|
378
|
+
// alone: how long this client waits for an ANSWER from the broker. The
|
|
379
|
+
// publish confirm waits for it (`_publishOnce`), so does the whole teardown
|
|
380
|
+
// of `disconnect()` (d.283 chose deliberately not to invent a second number
|
|
381
|
+
// for that; the name it inherited then said "publish confirmation", which was
|
|
382
|
+
// true of one consumer and false of the other), and so does every request a
|
|
383
|
+
// channel sends and waits on (`_askBroker()`, conf `mq-broker-answer-budget`
|
|
384
|
+
// 001, d.1135).
|
|
375
385
|
this._brokerAnswerTimeout = this._config.brokerAnswerTimeout;
|
|
376
386
|
// How long the connect handshake may take before the attempt is abandoned.
|
|
377
387
|
this._connectTimeout = this._config.connectTimeout;
|
|
@@ -408,9 +418,12 @@ class RabbitMQClient extends EventEmitter {
|
|
|
408
418
|
// Health reporting callbacks
|
|
409
419
|
this._healthReportCallback = this._config.healthReportCallback; // (health) => Promise<void>
|
|
410
420
|
this._healthCriticalCallback = this._config.healthCriticalCallback; // (health) => Promise<void>
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
421
|
+
// When the current critical stretch began — read only by the two log lines that open and
|
|
422
|
+
// close it ("Critical health detected at", "Health recovered after"). Since d.1275 it arms
|
|
423
|
+
// nothing: `criticalHealthShutdown` and the `health:shutdown` event it armed are gone
|
|
424
|
+
// (owner 2026-10-06), because ending the process is the lifecycle owner's decision
|
|
425
|
+
// (`_failFatally()`), and nothing listened.
|
|
426
|
+
this._criticalHealthStartTime = null;
|
|
414
427
|
|
|
415
428
|
// Publish layer (retry + buffer)
|
|
416
429
|
this._publishLayer = new PublishLayer({
|
|
@@ -422,6 +435,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
422
435
|
ttlMs: this._config.publishBufferTtlMs,
|
|
423
436
|
},
|
|
424
437
|
},
|
|
438
|
+
// The buffer's expiry clock is THIS client's timer rail (d.1275): its handle sits in
|
|
439
|
+
// `_activeTimers` like every other timer the client arms, so `disconnect()` ends it and
|
|
440
|
+
// `timers-have-one-evidence` can see it (d.759e).
|
|
441
|
+
timers: {
|
|
442
|
+
setTimeout: (callback, ms) => this._setTimeout(callback, ms),
|
|
443
|
+
clearTimeout: (handle) => this._clearTimeout(handle),
|
|
444
|
+
},
|
|
425
445
|
});
|
|
426
446
|
|
|
427
447
|
// Publish monitor (metriky) — counts only, reports through return values,
|
|
@@ -561,11 +581,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
561
581
|
/**
|
|
562
582
|
* THE definition of "this client's consumer is attached to that queue".
|
|
563
583
|
*
|
|
564
|
-
*
|
|
565
|
-
* the
|
|
566
|
-
*
|
|
567
|
-
*
|
|
568
|
-
* the
|
|
584
|
+
* Two facts, both about THIS client, and both are needed:
|
|
585
|
+
* - the consumer channel is alive (`_isChannelAlive()`, d.260) — an attachment lives
|
|
586
|
+
* on that channel and dies with it, while the entry keeps its tag until the
|
|
587
|
+
* re-registration on the next channel replaces it;
|
|
588
|
+
* - the entry carries a tag: the broker gives it to the attachment it created, and
|
|
589
|
+
* since d.386b / d.605 the entry carries it only while that attachment exists — a
|
|
590
|
+
* refused re-registration and a cancel sent by the broker both drop it while the
|
|
591
|
+
* consumer stays TRACKED, because the next recovery must find it on the list.
|
|
592
|
+
*
|
|
593
|
+
* Nothing here asks the broker, and nothing here reads the QUEUE channel. Until d.1275
|
|
594
|
+
* the health check counted `active` only inside its `checkQueue` loop, so a dead queue
|
|
595
|
+
* channel or an unanswered probe reported 0 attached for a client whose consumer channel
|
|
596
|
+
* was alive and receiving (INFRA 531, W224 on the dev scheduler).
|
|
569
597
|
*
|
|
570
598
|
* It is a method rather than an expression repeated twice for the reason
|
|
571
599
|
* `change-discipline.md` § One rail per concern states: until d.613
|
|
@@ -580,7 +608,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
580
608
|
* @private
|
|
581
609
|
*/
|
|
582
610
|
_isConsumerAttached(consumerInfo) {
|
|
583
|
-
return
|
|
611
|
+
return this._isChannelAlive(this._consumerChannel)
|
|
612
|
+
&& typeof consumerInfo?.consumerTag === 'string'
|
|
613
|
+
&& consumerInfo.consumerTag !== '';
|
|
584
614
|
}
|
|
585
615
|
|
|
586
616
|
/**
|
|
@@ -785,6 +815,71 @@ class RabbitMQClient extends EventEmitter {
|
|
|
785
815
|
this.emit('error', err);
|
|
786
816
|
});
|
|
787
817
|
connection.on('close', () => this._onConnectionDied());
|
|
818
|
+
// A broker in a memory or disk alarm blocks a connection that publishes and reads nothing
|
|
819
|
+
// more from it until the alarm clears; the connection stays alive, so no reconnect — and
|
|
820
|
+
// no flush — follows (measured d.1143 on the platform's image). amqplib announces the
|
|
821
|
+
// capability and re-emits both frames (`lib/connect.js`, `lib/channel_model.js`).
|
|
822
|
+
this._connectionBlockedBy = null;
|
|
823
|
+
connection.on('blocked', (reason) => this._onConnectionBlocked(connection, reason));
|
|
824
|
+
connection.on('unblocked', () => this._onConnectionUnblocked(connection));
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
/**
|
|
828
|
+
* The broker blocked this connection (`connection.blocked`, d.1145).
|
|
829
|
+
*
|
|
830
|
+
* Recorded as state — the publish path names it when it refuses (`_publishPathTransient()`)
|
|
831
|
+
* — and reported once as a warning. A frame of a stale connection (one this client already
|
|
832
|
+
* replaced) says nothing about the current one and is ignored.
|
|
833
|
+
*
|
|
834
|
+
* @param {Object} connection - the amqplib connection that received the frame
|
|
835
|
+
* @param {string} reason - the broker's reason (RabbitMQ: "low on memory", "low on disk")
|
|
836
|
+
* @private
|
|
837
|
+
*/
|
|
838
|
+
_onConnectionBlocked(connection, reason) {
|
|
839
|
+
if (connection !== this._connection) {
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
this._connectionBlockedBy = String(reason);
|
|
843
|
+
this._logger.warn(
|
|
844
|
+
`[RabbitMQClient] The broker blocked this connection (reason: ${this._connectionBlockedBy}) - `
|
|
845
|
+
+ 'Expected: an alarm-free broker. It reads nothing this client sends until the alarm clears: '
|
|
846
|
+
+ 'publishes and every request wait, and end on brokerAnswerTimeout. '
|
|
847
|
+
+ 'Fix: clear the broker alarm (management API /api/health/checks/alarms names it); the buffer '
|
|
848
|
+
+ 're-sends what never left once the broker unblocks the connection.'
|
|
849
|
+
);
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/**
|
|
853
|
+
* The broker unblocked this connection (`connection.unblocked`, d.1145).
|
|
854
|
+
*
|
|
855
|
+
* The flush after it is the one rail `PublishLayer.flushBuffered()` with the trigger
|
|
856
|
+
* `'unblocked'`: only the entries whose frame never left the client are re-sent; the broker
|
|
857
|
+
* delivers the rest by itself (conf `mq-publish-buffer` 001). A failed flush is reported the
|
|
858
|
+
* way the flush after a reconnect reports it.
|
|
859
|
+
*
|
|
860
|
+
* @param {Object} connection - the amqplib connection that received the frame
|
|
861
|
+
* @private
|
|
862
|
+
*/
|
|
863
|
+
_onConnectionUnblocked(connection) {
|
|
864
|
+
if (connection !== this._connection) {
|
|
865
|
+
return;
|
|
866
|
+
}
|
|
867
|
+
const wasBlockedBy = this._connectionBlockedBy;
|
|
868
|
+
this._connectionBlockedBy = null;
|
|
869
|
+
this._logger.warn(
|
|
870
|
+
`[RabbitMQClient] The broker unblocked this connection (it had been blocked: ${wasBlockedBy === null ? 'no blocked frame seen' : wasBlockedBy}) - `
|
|
871
|
+
+ 're-sending the buffered messages whose frame never left this client.'
|
|
872
|
+
);
|
|
873
|
+
this._publishLayer.flushBuffered('unblocked').then(
|
|
874
|
+
(flushed) => {
|
|
875
|
+
this._publishMonitor.trackFlushed(flushed);
|
|
876
|
+
this._logger.info(`[RabbitMQClient] ✓ Flushed ${flushed} buffered messages after the broker unblocked the connection`);
|
|
877
|
+
},
|
|
878
|
+
(flushErr) => {
|
|
879
|
+
this._logger.warn(`[RabbitMQClient] Failed to flush buffered messages after unblocked: ${flushErr.message}`);
|
|
880
|
+
this.emit('buffer:flush:failed', { error: flushErr.message, bufferSize: this._publishLayer._buffer.size() });
|
|
881
|
+
}
|
|
882
|
+
);
|
|
788
883
|
}
|
|
789
884
|
|
|
790
885
|
/**
|
|
@@ -990,6 +1085,33 @@ class RabbitMQClient extends EventEmitter {
|
|
|
990
1085
|
return error;
|
|
991
1086
|
}
|
|
992
1087
|
|
|
1088
|
+
/**
|
|
1089
|
+
* Carry a verdict that is ALREADY written — on the error a caller is about to receive.
|
|
1090
|
+
*
|
|
1091
|
+
* `_markBrokerRefusal()` and `_failFatally()` write the verdict; this does not write
|
|
1092
|
+
* one, it copies `code` and `reason` from a verdict the client produced onto the error
|
|
1093
|
+
* built around it, so the receiver reads it where every receiver reads it: on the error
|
|
1094
|
+
* itself (`BaseClient._consumerRefusal()`, `@onlineapps/service-wrapper` FÁZE 0.3/0.7 read
|
|
1095
|
+
* `error.code`, never a `cause` chain — a reader walking the chain would be a fallback
|
|
1096
|
+
* over a buried marker, `architecture-principles.md` §3). Measured (d.1294, finding of
|
|
1097
|
+
* d.1278/d.1281): the `consume()` refusals built around the recovery verdict kept
|
|
1098
|
+
* `MQ_CONNECTION_FATAL` one `cause` down, so `_consumerRefusal()` wrapped them in a
|
|
1099
|
+
* `ConsumeError` and the lifecycle owner never saw the verdict it ends the process on.
|
|
1100
|
+
*
|
|
1101
|
+
* An error that is not a fatal verdict (a spent cycle carries no code, by contract)
|
|
1102
|
+
* leaves the error untouched.
|
|
1103
|
+
*
|
|
1104
|
+
* @param {Error} error - the error being built for the caller; `cause` already set
|
|
1105
|
+
* @param {Error|null|undefined} verdict - what the recovery produced
|
|
1106
|
+
* @returns {Error} the same error, with the verdict readable on top when there is one
|
|
1107
|
+
* @private
|
|
1108
|
+
*/
|
|
1109
|
+
_carryVerdict(error, verdict) {
|
|
1110
|
+
// One body, in `utils/errorHandler.js`: `PublishLayer` and `BaseClient.publish()` carry the
|
|
1111
|
+
// same verdict the same way (d.1300), so the rule is not written three times.
|
|
1112
|
+
return carryVerdict(error, verdict);
|
|
1113
|
+
}
|
|
1114
|
+
|
|
993
1115
|
/**
|
|
994
1116
|
* A recovery cycle is over and the client is NOT connected, but the broker
|
|
995
1117
|
* never refused it — so this is not the end.
|
|
@@ -1037,6 +1159,24 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1037
1159
|
return standby;
|
|
1038
1160
|
}
|
|
1039
1161
|
|
|
1162
|
+
/**
|
|
1163
|
+
* Whether a recovery cycle may start on this client without the caller's action — the
|
|
1164
|
+
* ONE statement of the three states that forbid it (d.1319): the connection declared lost
|
|
1165
|
+
* (`_connectionFatal`), the caller closed the client (`_closedByCaller`), recovery disabled
|
|
1166
|
+
* by configuration (`reconnectEnabled: false`). Read by `_resumeRecovery()` (may the next use
|
|
1167
|
+
* start a cycle), by `_recoveryEndedForPublish()` (`replayable` — may a `reconnected` still
|
|
1168
|
+
* come to replay a held message) and by every rail that meets a closed connection
|
|
1169
|
+
* (`_requireQueueChannel()`, `cancelConsumer()`, `ack()`/`nack()`: wait for the reconnect, or
|
|
1170
|
+
* say through `_refusalWithoutRecovery()` why none is coming). Until d.1319 the first two
|
|
1171
|
+
* carried their own copy of these three checks.
|
|
1172
|
+
*
|
|
1173
|
+
* @returns {boolean}
|
|
1174
|
+
* @private
|
|
1175
|
+
*/
|
|
1176
|
+
_recoveryMayStart() {
|
|
1177
|
+
return !this._connectionFatal && !this._closedByCaller && this._reconnectEnabled === true;
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1040
1180
|
/**
|
|
1041
1181
|
* Somebody wants to use a client that is not connected. Start the next
|
|
1042
1182
|
* recovery cycle if the contract allows one, and say whether a recovery is now
|
|
@@ -1053,9 +1193,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1053
1193
|
*/
|
|
1054
1194
|
_resumeRecovery() {
|
|
1055
1195
|
if (this._reconnecting) return true;
|
|
1056
|
-
if (this.
|
|
1057
|
-
if (this._closedByCaller) return false;
|
|
1058
|
-
if (!this._reconnectEnabled) return false;
|
|
1196
|
+
if (!this._recoveryMayStart()) return false;
|
|
1059
1197
|
|
|
1060
1198
|
this._logger.info(
|
|
1061
1199
|
`[RabbitMQClient] [mq-client-core] Client used while disconnected - starting recovery cycle `
|
|
@@ -1163,6 +1301,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1163
1301
|
fatal.attempts = this._maxReconnectAttempts;
|
|
1164
1302
|
fatal.cycles = this._reconnectCycles;
|
|
1165
1303
|
fatal.cause = lastError;
|
|
1304
|
+
this._connectionFatalError = fatal;
|
|
1166
1305
|
|
|
1167
1306
|
// The log line is the RECORD and is unconditional; the two lines below are
|
|
1168
1307
|
// the NOTIFICATION, and only a caller that is not awaiting needs it.
|
|
@@ -1390,6 +1529,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1390
1529
|
*/
|
|
1391
1530
|
_onConnectionDied() {
|
|
1392
1531
|
this._connectionAlive = false;
|
|
1532
|
+
// A dead connection is no longer blocked; a new one starts unblocked (d.1145).
|
|
1533
|
+
this._connectionBlockedBy = null;
|
|
1393
1534
|
this._channel = null;
|
|
1394
1535
|
this._telemetryChannel = null;
|
|
1395
1536
|
this._queueChannel = null;
|
|
@@ -1437,12 +1578,56 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1437
1578
|
return { status: 'absent', error: null };
|
|
1438
1579
|
}
|
|
1439
1580
|
|
|
1440
|
-
const
|
|
1581
|
+
const outcome = await this._awaitBrokerAnswer(() => handle.close(), budgetMs);
|
|
1582
|
+
|
|
1583
|
+
if (outcome.status === 'unanswered') {
|
|
1584
|
+
this._logger.warn(
|
|
1585
|
+
`[RabbitMQClient] The broker did not answer the ${what} close within the teardown budget `
|
|
1586
|
+
+ `(${this._brokerAnswerTimeout}ms) - `
|
|
1587
|
+
+ 'Expected: a close-ok frame for the close this client sent. '
|
|
1588
|
+
+ 'Fix: nothing to do here — the teardown stops waiting and drops the socket, so '
|
|
1589
|
+
+ 'the teardown ends (disconnect(), or a connect() that failed half-way) and the process '
|
|
1590
|
+
+ 'can exit; a broker that never answers a close is a broker-side or network fault, and '
|
|
1591
|
+
+ 'the declared brokerAnswerTimeout sets how long this client gives it.'
|
|
1592
|
+
);
|
|
1593
|
+
return { status: 'unanswered', error: null };
|
|
1594
|
+
}
|
|
1595
|
+
if (outcome.status === 'refused') {
|
|
1596
|
+
return { status: 'refused', error: outcome.error };
|
|
1597
|
+
}
|
|
1598
|
+
return { status: 'closed', error: null };
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
/**
|
|
1602
|
+
* Wait for the broker's answer to ONE request, never longer than `budgetMs`.
|
|
1603
|
+
*
|
|
1604
|
+
* THE one clock for "how long does this client wait for the broker to answer" (conf
|
|
1605
|
+
* `mq-broker-answer-budget` 001, d.1135). Its two readers are `_askBroker()` — every
|
|
1606
|
+
* request a channel or the connection sends and waits on — and
|
|
1607
|
+
* `_closeWithinTeardownBudget()`, which asks it for the `close-ok` of a teardown with what
|
|
1608
|
+
* is left of that teardown's budget. The publish confirm keeps its own timer inside the
|
|
1609
|
+
* send callback (`_publishOnce()`), because it is not a promise to race: it is a callback
|
|
1610
|
+
* that also has to be disarmed, counted and attributed.
|
|
1611
|
+
*
|
|
1612
|
+
* `request` is called here, synchronously, so a request that throws before it is sent
|
|
1613
|
+
* reads as a refusal rather than escaping the budget as an exception.
|
|
1614
|
+
*
|
|
1615
|
+
* @param {() => Promise<*>} request - sends the request and answers its reply
|
|
1616
|
+
* @param {number} budgetMs - how long the answer may take
|
|
1617
|
+
* @returns {Promise<{status: 'answered', value: *, pending: Promise<*>}
|
|
1618
|
+
* |{status: 'refused', error: Error, pending: Promise<*>}
|
|
1619
|
+
* |{status: 'unanswered', pending: Promise<*>}>} `pending` is the request's own
|
|
1620
|
+
* promise, for a caller that must deal with an answer arriving after the budget
|
|
1621
|
+
* @private
|
|
1622
|
+
*/
|
|
1623
|
+
async _awaitBrokerAnswer(request, budgetMs) {
|
|
1624
|
+
const pending = new Promise((resolve) => resolve(request()));
|
|
1625
|
+
const UNANSWERED = Symbol('broker-answer-budget-spent');
|
|
1441
1626
|
let deadlineTimer = null;
|
|
1442
1627
|
const outcome = await Promise.race([
|
|
1443
|
-
|
|
1444
|
-
() => ({ status: '
|
|
1445
|
-
(
|
|
1628
|
+
pending.then(
|
|
1629
|
+
(value) => ({ status: 'answered', value }),
|
|
1630
|
+
(error) => ({ status: 'refused', error })
|
|
1446
1631
|
),
|
|
1447
1632
|
new Promise((resolve) => {
|
|
1448
1633
|
deadlineTimer = this._setTimeout(() => resolve(UNANSWERED), budgetMs);
|
|
@@ -1451,19 +1636,145 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1451
1636
|
this._clearTimeout(deadlineTimer);
|
|
1452
1637
|
|
|
1453
1638
|
if (outcome === UNANSWERED) {
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1639
|
+
return { status: 'unanswered', pending };
|
|
1640
|
+
}
|
|
1641
|
+
return { ...outcome, pending };
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
/**
|
|
1645
|
+
* Ask the broker one thing and wait for its answer within `brokerAnswerTimeout`.
|
|
1646
|
+
*
|
|
1647
|
+
* ONE rail for every request this client sends and then waits on (conf
|
|
1648
|
+
* `mq-broker-answer-budget` 001, verdict A; d.1135): queue and exchange declarations,
|
|
1649
|
+
* `checkQueue`, `bindQueue`, `purgeQueue`, `deleteQueue`, `basic.consume`, `basic.cancel`,
|
|
1650
|
+
* `basic.qos` and every channel open. Until d.1135 none of them had a limit. A broker in a
|
|
1651
|
+
* memory alarm keeps the connection open and its heartbeats flowing while it answers
|
|
1652
|
+
* nothing — measured by d.1126 on an isolated broker of the platform's image: every one of
|
|
1653
|
+
* these calls still pending after 300 s, settled only when the alarm was cleared; a paused
|
|
1654
|
+
* broker was ended by amqplib's heartbeat after ~90 s. The heartbeat is not part of this
|
|
1655
|
+
* rail: in an alarm it never fires, so only the budget ends the wait.
|
|
1656
|
+
*
|
|
1657
|
+
* The budget is `brokerAnswerTimeout` and never a second number — its declared meaning is
|
|
1658
|
+
* exactly this question (`config/configSchema.js`).
|
|
1659
|
+
*
|
|
1660
|
+
* WHEN THE BUDGET IS SPENT:
|
|
1661
|
+
* - the channel that carried the request is DISCARDED (`_discardUnansweredChannel()`):
|
|
1662
|
+
* amqplib queues the requests of one channel behind each other, so every later request
|
|
1663
|
+
* on it would wait behind the silent one. The next operation opens a fresh channel;
|
|
1664
|
+
* - a request that OPENS a channel has no channel yet; the one the broker may still open
|
|
1665
|
+
* later is closed as soon as it arrives, so an alarm does not leak channels;
|
|
1666
|
+
* - the caller gets `ConnectionError` naming the operation, its target and the budget.
|
|
1667
|
+
* The publish path translates it into `TransientPublishError`
|
|
1668
|
+
* (`_publishPathTransient()`), so retry, the buffer and `bufferOnFailure: false` apply
|
|
1669
|
+
* to it exactly as to any other transient failure.
|
|
1670
|
+
*
|
|
1671
|
+
* A bare `Error('Channel ended, no reply will be forthcoming')` — amqplib's
|
|
1672
|
+
* `_rejectPending()` answering a request whose channel or connection died first, no class,
|
|
1673
|
+
* no `code` — is the same fact ("no answer came") and becomes the same class. It is
|
|
1674
|
+
* recognised by the carrier's STATE, never by the sentence (d.683): no `code`, a plain
|
|
1675
|
+
* `Error`, and a carrier that is dead by the time the rejection arrives (amqplib emits
|
|
1676
|
+
* `close` synchronously before the rejection's handlers run). A numbered refusal is an
|
|
1677
|
+
* ANSWER and passes through untouched, as does anything already typed.
|
|
1678
|
+
*
|
|
1679
|
+
* @param {Object} request
|
|
1680
|
+
* @param {string} request.operation - the request's name, as the message states it
|
|
1681
|
+
* @param {string} request.target - what it is about (queue, exchange, channel)
|
|
1682
|
+
* @param {Object|null} request.channel - the channel carrying it; `null` for a channel
|
|
1683
|
+
* open, which the connection carries
|
|
1684
|
+
* @param {() => Promise<*>} send - sends the request and answers its reply
|
|
1685
|
+
* @returns {Promise<*>} the broker's answer
|
|
1686
|
+
* @throws {ConnectionError} when no answer came within the budget, or the carrier ended
|
|
1687
|
+
* before it could
|
|
1688
|
+
* @private
|
|
1689
|
+
*/
|
|
1690
|
+
async _askBroker({ operation, target, channel }, send) {
|
|
1691
|
+
const budgetMs = this._brokerAnswerTimeout;
|
|
1692
|
+
const outcome = await this._awaitBrokerAnswer(send, budgetMs);
|
|
1693
|
+
|
|
1694
|
+
if (outcome.status === 'answered') {
|
|
1695
|
+
return outcome.value;
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
if (outcome.status === 'unanswered') {
|
|
1699
|
+
if (channel) {
|
|
1700
|
+
this._discardUnansweredChannel(channel, `${operation} ${target}`);
|
|
1701
|
+
} else {
|
|
1702
|
+
outcome.pending.then(
|
|
1703
|
+
(late) => this._closeLateChannel(late, target),
|
|
1704
|
+
() => {}
|
|
1705
|
+
);
|
|
1706
|
+
}
|
|
1707
|
+
throw new ConnectionError(
|
|
1708
|
+
`[RabbitMQClient] The broker did not answer ${operation} ${target} within brokerAnswerTimeout `
|
|
1709
|
+
+ `(${budgetMs}ms) - Expected: an answer to the request this client sent; a broker that `
|
|
1710
|
+
+ 'blocks publishers (memory or disk alarm) or is frozen keeps the connection open and '
|
|
1711
|
+
+ 'answers nothing. Fix: check the broker for an alarm (management API '
|
|
1712
|
+
+ '/api/health/checks/alarms) or a paused node; the channel that carried the request '
|
|
1713
|
+
+ 'is discarded and the next operation opens a fresh one. brokerAnswerTimeout is the '
|
|
1714
|
+
+ 'one budget for every answer this client waits for.'
|
|
1462
1715
|
);
|
|
1463
|
-
return { status: 'unanswered', error: null };
|
|
1464
1716
|
}
|
|
1465
1717
|
|
|
1466
|
-
|
|
1718
|
+
const err = outcome.error;
|
|
1719
|
+
const carrierDead = channel ? !this._isChannelAlive(channel) : !this._connectionAlive;
|
|
1720
|
+
const bare = err instanceof Error && err.constructor === Error && err.code === undefined;
|
|
1721
|
+
if (bare && carrierDead) {
|
|
1722
|
+
throw new ConnectionError(
|
|
1723
|
+
`[RabbitMQClient] ${operation} ${target} got no answer: the ${channel ? 'channel' : 'connection'} `
|
|
1724
|
+
+ 'carrying it ended before the broker answered - Expected: an answer to the request '
|
|
1725
|
+
+ 'this client sent. Fix: nothing is refused — the request never got a reply; run the '
|
|
1726
|
+
+ 'operation again once the channel is back (error.cause carries amqplib\'s rejection).',
|
|
1727
|
+
err
|
|
1728
|
+
);
|
|
1729
|
+
}
|
|
1730
|
+
throw err;
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
/**
|
|
1734
|
+
* Take a channel whose request the broker did not answer out of service.
|
|
1735
|
+
*
|
|
1736
|
+
* It is marked dead first (`_isChannelAlive()`, the one liveness source — d.260), then
|
|
1737
|
+
* dropped from whichever slot still holds it — only if it still holds THIS channel, the
|
|
1738
|
+
* same obituary rule the close handlers keep — and then asked to close. The close is not
|
|
1739
|
+
* awaited: it is an answer from the same silent broker. amqplib sends it at once and
|
|
1740
|
+
* ignores every frame but `close-ok` from then on, so a broker that wakes up closes the
|
|
1741
|
+
* channel cleanly (its consumers cancelled, its unacked deliveries requeued) instead of
|
|
1742
|
+
* keeping it open with nobody holding it; its `close` event then reaches the handlers
|
|
1743
|
+
* attached to it, exactly as for any other channel that ends.
|
|
1744
|
+
*
|
|
1745
|
+
* @param {Object} channel - the amqplib channel that carried the unanswered request
|
|
1746
|
+
* @param {string} reason - which request went unanswered, for the post-mortem
|
|
1747
|
+
* @private
|
|
1748
|
+
*/
|
|
1749
|
+
_discardUnansweredChannel(channel, reason) {
|
|
1750
|
+
channel._alive = false;
|
|
1751
|
+
channel._closeReason = `No answer from the broker to ${reason} within brokerAnswerTimeout (${this._brokerAnswerTimeout}ms)`;
|
|
1752
|
+
if (this._channel === channel) this._channel = null;
|
|
1753
|
+
if (this._telemetryChannel === channel) this._telemetryChannel = null;
|
|
1754
|
+
if (this._queueChannel === channel) this._queueChannel = null;
|
|
1755
|
+
if (this._consumerChannel === channel) this._consumerChannel = null;
|
|
1756
|
+
channel.close().catch((err) => {
|
|
1757
|
+
this._logger.debug(`[RabbitMQClient] Discarded channel (${reason}) did not close: ${err.message}`);
|
|
1758
|
+
});
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
/**
|
|
1762
|
+
* Close a channel the broker opened after this client stopped waiting for it. Nobody
|
|
1763
|
+
* holds it, so left open it would be a channel on the broker with no owner.
|
|
1764
|
+
*
|
|
1765
|
+
* @param {Object} channel - the late amqplib channel
|
|
1766
|
+
* @param {string} what - which channel was being opened, for the log
|
|
1767
|
+
* @private
|
|
1768
|
+
*/
|
|
1769
|
+
_closeLateChannel(channel, what) {
|
|
1770
|
+
this._logger.warn(
|
|
1771
|
+
`[RabbitMQClient] The broker opened the ${what} after brokerAnswerTimeout (${this._brokerAnswerTimeout}ms) - `
|
|
1772
|
+
+ 'Expected: nothing to do; the open had already been reported as unanswered. '
|
|
1773
|
+
+ 'Fix: none — the late channel is closed now, so it is not left open with no owner.'
|
|
1774
|
+
);
|
|
1775
|
+
channel.close().catch((err) => {
|
|
1776
|
+
this._logger.debug(`[RabbitMQClient] Late ${what} did not close: ${err.message}`);
|
|
1777
|
+
});
|
|
1467
1778
|
}
|
|
1468
1779
|
|
|
1469
1780
|
/**
|
|
@@ -1604,6 +1915,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1604
1915
|
* Close the connection an abandoned handshake produces, whenever it completes.
|
|
1605
1916
|
* A handshake that fails after its timeout opened nothing; its error adds
|
|
1606
1917
|
* nothing to the timeout the caller already received.
|
|
1918
|
+
*
|
|
1919
|
+
* The close waits for the broker's `close-ok` no longer than `brokerAnswerTimeout` — the
|
|
1920
|
+
* one budget every wait on the broker has (conf `mq-broker-answer-budget` 001), on the one
|
|
1921
|
+
* rail the teardown already uses (`_closeWithinTeardownBudget()` → `_awaitBrokerAnswer()`),
|
|
1922
|
+
* and an unanswered close drops the socket (`_dropUnansweredConnection()`) exactly as
|
|
1923
|
+
* `connect()` does for a half-open connection. Until d.1322 this was a bare
|
|
1924
|
+
* `await late.close()`: a broker that never answered it left the late connection, its
|
|
1925
|
+
* socket and its heartbeat alive for the life of the process — the handshake leak of
|
|
1926
|
+
* INFRA wave 5.
|
|
1607
1927
|
* @private
|
|
1608
1928
|
* @param {Promise<Object>} connectPromise - the handshake the timeout abandoned
|
|
1609
1929
|
* @param {string} phase - 'connect' or 'reconnect'
|
|
@@ -1615,11 +1935,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1615
1935
|
`[RabbitMQClient] Handshake completed after the ${phase} timeout (${this._connectTimeout} ms) - `
|
|
1616
1936
|
+ 'closing that connection, nothing in this client owns it'
|
|
1617
1937
|
);
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1938
|
+
const closing = await this._closeWithinTeardownBudget(
|
|
1939
|
+
`abandoned ${phase} handshake`,
|
|
1940
|
+
late,
|
|
1941
|
+
this._brokerAnswerTimeout
|
|
1942
|
+
);
|
|
1943
|
+
if (closing.status === 'unanswered') {
|
|
1944
|
+
this._dropUnansweredConnection(late);
|
|
1945
|
+
} else if (closing.status === 'refused') {
|
|
1621
1946
|
this._logger.warn(
|
|
1622
|
-
`[RabbitMQClient] Error closing the connection whose handshake outlived the ${phase} timeout: ${
|
|
1947
|
+
`[RabbitMQClient] Error closing the connection whose handshake outlived the ${phase} timeout: ${closing.error.message}`
|
|
1623
1948
|
);
|
|
1624
1949
|
}
|
|
1625
1950
|
},
|
|
@@ -1698,7 +2023,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1698
2023
|
this._attachConnectionHandlers(this._connection);
|
|
1699
2024
|
|
|
1700
2025
|
// Use ConfirmChannel to enable publisher confirms for publish operations
|
|
1701
|
-
this._channel = await this.
|
|
2026
|
+
this._channel = await this._askBroker(
|
|
2027
|
+
{ operation: 'createConfirmChannel', target: 'publisher channel', channel: null },
|
|
2028
|
+
() => this._connection.createConfirmChannel()
|
|
2029
|
+
);
|
|
1702
2030
|
|
|
1703
2031
|
// Track channel metadata for debugging
|
|
1704
2032
|
this._channel._createdAt = new Date().toISOString();
|
|
@@ -1711,7 +2039,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1711
2039
|
|
|
1712
2040
|
// Create a separate regular channel for queue operations (assertQueue, checkQueue)
|
|
1713
2041
|
// This avoids RPC reply queue issues with ConfirmChannel.assertQueue()
|
|
1714
|
-
this._queueChannel = await this.
|
|
2042
|
+
this._queueChannel = await this._askBroker(
|
|
2043
|
+
{ operation: 'createChannel', target: 'queue channel', channel: null },
|
|
2044
|
+
() => this._connection.createChannel()
|
|
2045
|
+
);
|
|
1715
2046
|
|
|
1716
2047
|
// Track channel metadata for debugging
|
|
1717
2048
|
this._queueChannel._createdAt = new Date().toISOString();
|
|
@@ -1724,7 +2055,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1724
2055
|
|
|
1725
2056
|
// Create a dedicated channel for consume operations
|
|
1726
2057
|
// This prevents channel conflicts between publish (ConfirmChannel) and consume operations
|
|
1727
|
-
this._consumerChannel = await this.
|
|
2058
|
+
this._consumerChannel = await this._askBroker(
|
|
2059
|
+
{ operation: 'createChannel', target: 'consumer channel', channel: null },
|
|
2060
|
+
() => this._connection.createChannel()
|
|
2061
|
+
);
|
|
1728
2062
|
|
|
1729
2063
|
// Track channel metadata for debugging
|
|
1730
2064
|
this._consumerChannel._createdAt = new Date().toISOString();
|
|
@@ -1743,10 +2077,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1743
2077
|
// Cleanup partially created resources
|
|
1744
2078
|
this._connectionAlive = false;
|
|
1745
2079
|
if (this._connection) {
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
2080
|
+
// Within the one budget (d.1135): a connect that failed because the broker stopped
|
|
2081
|
+
// answering (a channel open above) would otherwise wait for a `close-ok` from that
|
|
2082
|
+
// same silent broker. What the close answers is not the caller's business here —
|
|
2083
|
+
// the failure that brought us into this catch is — so only a close that got no
|
|
2084
|
+
// answer at all is acted on: its socket is dropped, as the teardown does.
|
|
2085
|
+
const halfOpen = this._connection;
|
|
2086
|
+
const closing = await this._closeWithinTeardownBudget('connection', halfOpen, this._brokerAnswerTimeout);
|
|
2087
|
+
if (closing.status === 'unanswered') {
|
|
2088
|
+
this._dropUnansweredConnection(halfOpen);
|
|
1750
2089
|
}
|
|
1751
2090
|
this._connection = null;
|
|
1752
2091
|
}
|
|
@@ -1818,10 +2157,17 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1818
2157
|
|
|
1819
2158
|
const timeout = this._setTimeout(() => {
|
|
1820
2159
|
done();
|
|
2160
|
+
// The sentence names the lever that is READ (d.1138): the configuration key when
|
|
2161
|
+
// the caller wrote it, the derivation in the constructor otherwise. It named an
|
|
2162
|
+
// environment variable until d.1138 that nothing reads.
|
|
2163
|
+
const budgetSource = this._config.reconnectWaitTimeout === undefined
|
|
2164
|
+
? 'not set, so derived as max(60000, maxReconnectAttempts × reconnectMaxDelay)'
|
|
2165
|
+
: 'set in the client configuration';
|
|
1821
2166
|
reject(new ConnectionError(
|
|
1822
2167
|
`[RabbitMQClient] Reconnection timeout after ${this._reconnectWaitTimeout}ms - `
|
|
1823
|
-
+
|
|
1824
|
-
+ 'Fix: check the broker is up and reachable; raise
|
|
2168
|
+
+ `Expected: the broker to accept a new connection within reconnectWaitTimeout (${this._reconnectWaitTimeout}ms, ${budgetSource}). `
|
|
2169
|
+
+ 'Fix: check the broker is up and reachable; raise reconnectWaitTimeout in the configuration handed to '
|
|
2170
|
+
+ 'the client only if a longer outage must be survived in-process.'
|
|
1825
2171
|
));
|
|
1826
2172
|
}, this._reconnectWaitTimeout);
|
|
1827
2173
|
|
|
@@ -1947,39 +2293,69 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1947
2293
|
// process.
|
|
1948
2294
|
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot recreate publisher channel: connection is closed (waiting for reconnection)');
|
|
1949
2295
|
|
|
2296
|
+
// Every way this branch ends without a channel is ONE class, `RecoveryEndedPublishError`
|
|
2297
|
+
// (B1, owner's decision 2026-10-06, d.1300): the message is held in the buffer and
|
|
2298
|
+
// replayed after the next `reconnected` / `unblocked`, like every other transiently
|
|
2299
|
+
// failed publish — one rail with the retry that waited for a reconnect and saw it fail
|
|
2300
|
+
// (`PublishLayer`). Until d.1300 the wait's ending was thrown raw and the "no recovery"
|
|
2301
|
+
// cases as a `ConnectionError`, so the caller got a `PublishError` with no `buffered` at
|
|
2302
|
+
// all and the message was not held (INFRA-DOCS ID254 point 1). The verdict of a fatal
|
|
2303
|
+
// ending rides on top (`_carryVerdict()`, d.1297); a spent cycle carries no code.
|
|
1950
2304
|
if (this._resumeRecovery()) {
|
|
1951
|
-
|
|
2305
|
+
try {
|
|
2306
|
+
await this._waitForReconnection();
|
|
2307
|
+
} catch (err) {
|
|
2308
|
+
throw this._recoveryEndedForPublish(role, `the recovery it waited for did not arrive - ${err.message}`, err);
|
|
2309
|
+
}
|
|
1952
2310
|
|
|
1953
2311
|
// After reconnection, try again
|
|
1954
2312
|
if (this._isChannelAlive(this._sendChannelOf(role))) {
|
|
1955
2313
|
return; // Channel was recreated during reconnection
|
|
1956
2314
|
}
|
|
1957
|
-
//
|
|
2315
|
+
// The wait ended on `reconnected`, and the connection is already gone again (its
|
|
2316
|
+
// `close` fired before this continuation ran). Nothing holds that second death as an
|
|
2317
|
+
// object, so there is no cause.
|
|
1958
2318
|
if (!this._connectionAlive) {
|
|
1959
|
-
throw
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
+ 'RABBITMQ_MAX_RECONNECT_CYCLES cycles in all, then emits connection:fatal.'
|
|
2319
|
+
throw this._recoveryEndedForPublish(
|
|
2320
|
+
role,
|
|
2321
|
+
'the recovery it waited for completed, but the connection is not alive any more',
|
|
2322
|
+
null
|
|
1964
2323
|
);
|
|
1965
2324
|
}
|
|
1966
2325
|
} else {
|
|
1967
|
-
// No recovery can be started: the connection is permanently lost (the
|
|
1968
|
-
//
|
|
1969
|
-
//
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
2326
|
+
// No recovery can be started: the connection is permanently lost (the broker refused
|
|
2327
|
+
// it, or every cycle is spent), the caller ended this client, or recovery is disarmed
|
|
2328
|
+
// by configuration — the same three states, in the same order, as the consumer path
|
|
2329
|
+
// (`_consumerChannelWithoutRecovery()`, d.1281).
|
|
2330
|
+
if (this._connectionFatal) {
|
|
2331
|
+
const fatal = this._connectionFatalError;
|
|
2332
|
+
throw this._recoveryEndedForPublish(
|
|
2333
|
+
role,
|
|
2334
|
+
`the connection is permanently lost and no recovery may start - ${fatal ? fatal.message : 'the verdict was not recorded'}`,
|
|
2335
|
+
fatal
|
|
2336
|
+
);
|
|
2337
|
+
}
|
|
2338
|
+
if (this._closedByCaller) {
|
|
2339
|
+
throw this._recoveryEndedForPublish(
|
|
2340
|
+
role,
|
|
2341
|
+
'the caller closed this client (disconnect()) and no recovery may start — only an explicit connect() re-arms it',
|
|
2342
|
+
null
|
|
2343
|
+
);
|
|
2344
|
+
}
|
|
2345
|
+
throw this._recoveryEndedForPublish(
|
|
2346
|
+
role,
|
|
2347
|
+
'the connection is closed and recovery is disabled by configuration (reconnectEnabled: false)',
|
|
2348
|
+
null
|
|
1976
2349
|
);
|
|
1977
2350
|
}
|
|
1978
2351
|
}
|
|
1979
2352
|
|
|
1980
2353
|
try {
|
|
1981
2354
|
// Create new ConfirmChannel
|
|
1982
|
-
const channel = await this.
|
|
2355
|
+
const channel = await this._askBroker(
|
|
2356
|
+
{ operation: 'createConfirmChannel', target: `${role} channel`, channel: null },
|
|
2357
|
+
() => this._connection.createConfirmChannel()
|
|
2358
|
+
);
|
|
1983
2359
|
|
|
1984
2360
|
// Track channel metadata for debugging
|
|
1985
2361
|
channel._createdAt = new Date().toISOString();
|
|
@@ -2030,6 +2406,24 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2030
2406
|
err
|
|
2031
2407
|
);
|
|
2032
2408
|
}
|
|
2409
|
+
// Třetí nosič (d.1135): broker na otevření kanálu neodpověděl v `brokerAnswerTimeout`
|
|
2410
|
+
// (alarm paměti, zmrazený broker) — `_askBroker()` to hlásí `ConnectionError`, jinou
|
|
2411
|
+
// chybu z otevření kanálu tahle třída nenese. Je to týž stav „odpověď nepřišla“, ne
|
|
2412
|
+
// odmítnutí, takže přechodný jako oba nosiče výše. Stojí ZA nimi: spojení, které
|
|
2413
|
+
// odchází, má svou vlastní (přesnější) větu i tehdy, když amqplib odmítnutí složil
|
|
2414
|
+
// `_askBroker()` do `ConnectionError`.
|
|
2415
|
+
if (err instanceof ConnectionError) {
|
|
2416
|
+
this._logger.warn(
|
|
2417
|
+
`[RabbitMQClient] [mq-client-core] Failed to recreate ${role} channel (no answer from the broker): ${msg}`
|
|
2418
|
+
);
|
|
2419
|
+
throw new TransientPublishError(
|
|
2420
|
+
`[RabbitMQClient] ${role} channel cannot be recreated: the broker did not answer the open (transient condition) - `
|
|
2421
|
+
+ `Expected: an answer within brokerAnswerTimeout (${this._brokerAnswerTimeout}ms). `
|
|
2422
|
+
+ 'Fix: retry the publish; PublishLayer does this on its own and buffers the message meanwhile. '
|
|
2423
|
+
+ 'error.cause names the unanswered request.',
|
|
2424
|
+
err
|
|
2425
|
+
);
|
|
2426
|
+
}
|
|
2033
2427
|
|
|
2034
2428
|
// Ostatní chyby považujeme za permanentní problém na úrovni kanálu/spojení.
|
|
2035
2429
|
throw new PermanentPublishError(
|
|
@@ -2062,7 +2456,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2062
2456
|
|
|
2063
2457
|
try {
|
|
2064
2458
|
// Create new regular channel
|
|
2065
|
-
this._queueChannel = await this.
|
|
2459
|
+
this._queueChannel = await this._askBroker(
|
|
2460
|
+
{ operation: 'createChannel', target: 'queue channel', channel: null },
|
|
2461
|
+
() => this._connection.createChannel()
|
|
2462
|
+
);
|
|
2066
2463
|
|
|
2067
2464
|
// Track channel metadata for debugging
|
|
2068
2465
|
this._queueChannel._createdAt = new Date().toISOString();
|
|
@@ -2100,6 +2497,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2100
2497
|
* d.1003d), never assumed: until d.1003d this branch answered
|
|
2101
2498
|
* `_activeConsumers.size` as re-registered, counting a consumer whose
|
|
2102
2499
|
* re-registration the broker had refused (tracked, no tag — d.262, d.386b).
|
|
2500
|
+
* @throws {ConnectionError} when the recovery this call waited for did not arrive —
|
|
2501
|
+
* `cause` is the error that ended the wait, the same object the cycle produced
|
|
2502
|
+
* (`code: 'MQ_CONNECTION_FATAL'` when it was the last cycle, and `onFatal` has already
|
|
2503
|
+
* received it — the thrown error then carries that `code` and `reason` on top too
|
|
2504
|
+
* (`_carryVerdict()`, d.1297); no `code` for a spent cycle — the contract keeps
|
|
2505
|
+
* `MQ_CONNECTION_STANDBY` inside the client); when the connection is closed and no recovery may start
|
|
2506
|
+
* (`_consumerChannelWithoutRecovery()`: permanently lost — `cause` is the fatal error
|
|
2507
|
+
* `onFatal` received; closed by the caller; recovery disabled); when the recovery it
|
|
2508
|
+
* waited for completed but the connection is not alive; or when the open connection
|
|
2509
|
+
* refuses the channel.
|
|
2103
2510
|
*/
|
|
2104
2511
|
async _ensureConsumerChannel() {
|
|
2105
2512
|
if (this._isChannelAlive(this._consumerChannel)) {
|
|
@@ -2113,16 +2520,39 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2113
2520
|
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot recreate consumer channel: connection is closed (reconnection in progress)');
|
|
2114
2521
|
|
|
2115
2522
|
if (!this._resumeRecovery()) {
|
|
2116
|
-
|
|
2523
|
+
// No recovery may start, and the caller is told WHY (d.1281, finding of d.1278):
|
|
2524
|
+
// until then this branch answered `{ failed }`, `consume()` went on without a
|
|
2525
|
+
// channel and the caller got the queue-channel rail's "queue channel is not
|
|
2526
|
+
// available … Fix: await client.connect()" — the right fix for one of the three
|
|
2527
|
+
// states and wrong for the other two. The three are the three `_resumeRecovery()`
|
|
2528
|
+
// refuses, asked in its order; the fatal one carries the verdict `onFatal` already
|
|
2529
|
+
// received as `cause` — the same object, so there is no second rail of reporting.
|
|
2530
|
+
throw this._consumerChannelWithoutRecovery();
|
|
2117
2531
|
}
|
|
2118
2532
|
|
|
2119
2533
|
try {
|
|
2120
2534
|
await this._waitForReconnection();
|
|
2121
2535
|
} catch (err) {
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
)
|
|
2125
|
-
|
|
2536
|
+
// The wait's ending is the caller's answer, the way it is on the publisher path
|
|
2537
|
+
// (`_ensurePublisherChannel()` lets it through). Until d.1278 this branch wrote a
|
|
2538
|
+
// WARN line and answered `{ failed }` as if that were an outcome; `consume()` then
|
|
2539
|
+
// went on without a channel and the caller got the queue-channel rail's refusal
|
|
2540
|
+
// ("queue channel is not available … Fix: await client.connect()") — no cause, no
|
|
2541
|
+
// cycle, and the verdict of a fatal cycle reached `onFatal` and nobody else
|
|
2542
|
+
// (INFRA-DOCS ID254). The cause is the SAME object the cycle produced, never a
|
|
2543
|
+
// copy: `onFatal` and this caller hold one error between them, so there is no
|
|
2544
|
+
// second channel of reporting to keep in step.
|
|
2545
|
+
// The verdict rides on top (d.1297): `code`/`reason` of a fatal ending are copied
|
|
2546
|
+
// onto this error, so `BaseClient._consumerRefusal()` lets it through unchanged and
|
|
2547
|
+
// the lifecycle owner reads `error.code` as everywhere else.
|
|
2548
|
+
throw this._carryVerdict(new ConnectionError(
|
|
2549
|
+
`[RabbitMQClient] Cannot recreate the consumer channel: the recovery it waited for did not arrive - ${err.message} `
|
|
2550
|
+
+ 'Expected: the recovery to re-establish the connection before a consumer attaches. '
|
|
2551
|
+
+ 'Fix: read error.code — MQ_CONNECTION_FATAL ends this client (onFatal has been called; error.cause is '
|
|
2552
|
+
+ 'that verdict, error.reason says broker-refused or cycles-spent); a spent cycle carries no code and is '
|
|
2553
|
+
+ 'an outage: the next consume(), publish() or performHealthCheck() starts the next cycle.',
|
|
2554
|
+
err
|
|
2555
|
+
), err);
|
|
2126
2556
|
}
|
|
2127
2557
|
|
|
2128
2558
|
if (this._isChannelAlive(this._consumerChannel)) {
|
|
@@ -2131,13 +2561,27 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2131
2561
|
return this._consumerAttachmentCounts();
|
|
2132
2562
|
}
|
|
2133
2563
|
if (!this._connectionAlive) {
|
|
2134
|
-
|
|
2564
|
+
// The wait ended on `reconnected`, and the connection is already gone again
|
|
2565
|
+
// (its `close` fired before this continuation ran). Nothing holds that second
|
|
2566
|
+
// death as an object here, so there is no `cause`; the sentence says what
|
|
2567
|
+
// happened and what starts the next cycle (d.1281).
|
|
2568
|
+
throw new ConnectionError(
|
|
2569
|
+
'[RabbitMQClient] Cannot recreate the consumer channel: the recovery it waited for completed, '
|
|
2570
|
+
+ 'but the connection is not alive any more - '
|
|
2571
|
+
+ 'Expected: a live connection at the end of the recovery. '
|
|
2572
|
+
+ "Fix: run consume() again — the connection's close handler starts the next recovery cycle; "
|
|
2573
|
+
+ 'if it keeps ending here, the broker is dropping the connection right after accepting it '
|
|
2574
|
+
+ '(read the broker log).'
|
|
2575
|
+
);
|
|
2135
2576
|
}
|
|
2136
2577
|
}
|
|
2137
2578
|
|
|
2138
2579
|
try {
|
|
2139
2580
|
// Create new regular channel
|
|
2140
|
-
this._consumerChannel = await this.
|
|
2581
|
+
this._consumerChannel = await this._askBroker(
|
|
2582
|
+
{ operation: 'createChannel', target: 'consumer channel', channel: null },
|
|
2583
|
+
() => this._connection.createChannel()
|
|
2584
|
+
);
|
|
2141
2585
|
|
|
2142
2586
|
// Track channel metadata for debugging
|
|
2143
2587
|
this._consumerChannel._createdAt = new Date().toISOString();
|
|
@@ -2176,6 +2620,104 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2176
2620
|
}
|
|
2177
2621
|
}
|
|
2178
2622
|
|
|
2623
|
+
/**
|
|
2624
|
+
* The refusal a consumer gets when the connection is closed and `_resumeRecovery()`
|
|
2625
|
+
* will not start a recovery — one sentence per state, in the order `_resumeRecovery()`
|
|
2626
|
+
* asks them (d.1281).
|
|
2627
|
+
*
|
|
2628
|
+
* - the connection is permanently lost (`_connectionFatal`): `cause` is THE fatal
|
|
2629
|
+
* error, the object `_failFatally()` handed to `onFatal` / `connection:fatal`;
|
|
2630
|
+
* - the caller closed this client (`_closedByCaller`): no cause — `disconnect()` is
|
|
2631
|
+
* not an error, and only `connect()` undoes it;
|
|
2632
|
+
* - recovery is disabled by configuration (`reconnectEnabled: false`): no cause.
|
|
2633
|
+
*
|
|
2634
|
+
* @returns {ConnectionError}
|
|
2635
|
+
* @private
|
|
2636
|
+
*/
|
|
2637
|
+
_consumerChannelWithoutRecovery() {
|
|
2638
|
+
return this._refusalWithoutRecovery('[RabbitMQClient] Cannot recreate the consumer channel', 'consume');
|
|
2639
|
+
}
|
|
2640
|
+
|
|
2641
|
+
/**
|
|
2642
|
+
* ONE builder of the refusal for "the connection is closed and no recovery may start" —
|
|
2643
|
+
* three states, three sentences, read in `_resumeRecovery()`'s order (d.1281, d.1314).
|
|
2644
|
+
* Every operation that meets a closed connection it cannot wait out says it through this
|
|
2645
|
+
* function: the consumer channel (`_consumerChannelWithoutRecovery()`), the queue-channel
|
|
2646
|
+
* rail (`_requireQueueChannel()`), `cancelConsumer()`, `ack()`/`nack()` — so the verdict of a
|
|
2647
|
+
* lost connection is written on the error once and the same way everywhere, and nothing
|
|
2648
|
+
* advises `connect()` to a client whose owner must end the process instead.
|
|
2649
|
+
*
|
|
2650
|
+
* @param {string} headline - `[RabbitMQClient] Cannot <what>` — what was asked
|
|
2651
|
+
* @param {string} verb - the public operation, named in the `disconnect()` Fix
|
|
2652
|
+
* @returns {ConnectionError} with the fatal verdict carried on top when that is the state
|
|
2653
|
+
* @private
|
|
2654
|
+
*/
|
|
2655
|
+
_refusalWithoutRecovery(headline, verb) {
|
|
2656
|
+
if (this._connectionFatal) {
|
|
2657
|
+
const fatal = this._connectionFatalError;
|
|
2658
|
+
return this._carryVerdict(new ConnectionError(
|
|
2659
|
+
`${headline}: the connection is permanently lost and no `
|
|
2660
|
+
+ `recovery may start - ${fatal ? fatal.message : 'the verdict was not recorded'} `
|
|
2661
|
+
+ 'Expected: a client whose connection was not declared lost. '
|
|
2662
|
+
+ 'Fix: this client is over — error.code is MQ_CONNECTION_FATAL and error.cause the verdict onFatal '
|
|
2663
|
+
+ '(and connection:fatal) already carried; the owner of the client stops the process so its restart '
|
|
2664
|
+
+ 'policy boots a new one against a verified broker.',
|
|
2665
|
+
fatal || undefined
|
|
2666
|
+
), fatal);
|
|
2667
|
+
}
|
|
2668
|
+
if (this._closedByCaller) {
|
|
2669
|
+
return new ConnectionError(
|
|
2670
|
+
`${headline}: the caller closed this client (disconnect()) `
|
|
2671
|
+
+ 'and no recovery may start - '
|
|
2672
|
+
+ 'Expected: a client that has not been disconnected. '
|
|
2673
|
+
+ `Fix: await client.connect() before ${verb}(); only an explicit connect() re-arms recovery.`
|
|
2674
|
+
);
|
|
2675
|
+
}
|
|
2676
|
+
return new ConnectionError(
|
|
2677
|
+
`${headline}: the connection is closed and recovery is `
|
|
2678
|
+
+ 'disabled by configuration (reconnectEnabled: false) - '
|
|
2679
|
+
+ 'Expected: either an open connection, or a client allowed to recover one. '
|
|
2680
|
+
+ 'Fix: await client.connect() to open a new connection, or run the client with reconnectEnabled: true '
|
|
2681
|
+
+ 'so the next consume(), publish() or performHealthCheck() starts a recovery cycle.'
|
|
2682
|
+
);
|
|
2683
|
+
}
|
|
2684
|
+
|
|
2685
|
+
/**
|
|
2686
|
+
* The refusal a publish gets when its send channel cannot be had because the connection
|
|
2687
|
+
* stayed down (B1, d.1300) — one class for every ending of that branch, with the reason in
|
|
2688
|
+
* the sentence and the verdict, when there is one, on top.
|
|
2689
|
+
*
|
|
2690
|
+
* @param {'publisher'|'telemetry'} role - whose channel it was
|
|
2691
|
+
* @param {string} why - what ended it, already including the wait's own sentence when one exists
|
|
2692
|
+
* @param {Error|null} ended - the error that ended the recovery (the fatal verdict is the very
|
|
2693
|
+
* object `onFatal` received), or null when nothing holds the state as an object
|
|
2694
|
+
* @returns {RecoveryEndedPublishError}
|
|
2695
|
+
* @private
|
|
2696
|
+
*/
|
|
2697
|
+
_recoveryEndedForPublish(role, why, ended) {
|
|
2698
|
+
// Whether a `reconnected` can still come on this client without the caller's action —
|
|
2699
|
+
// read from the state AFTER the ending, so a wait that ended fatal answers false. The
|
|
2700
|
+
// three states that refuse `_resumeRecovery()` are exactly the three where it never comes
|
|
2701
|
+
// (d.1301, lead's decision: a hold makes sense only where a reconnect can replay it).
|
|
2702
|
+
const replayable = this._recoveryMayStart();
|
|
2703
|
+
const fix = replayable
|
|
2704
|
+
? 'Fix: nothing to retry here — PublishLayer holds the message in the buffer (unless the call passed '
|
|
2705
|
+
+ 'bufferOnFailure: false) and replays it after the next reconnect or after the broker unblocks the '
|
|
2706
|
+
+ 'connection; the next publish(), consume() or performHealthCheck() starts the next recovery cycle. '
|
|
2707
|
+
: 'Fix: this client will not reconnect on its own, so PublishLayer holds nothing (error.buffered === false) — '
|
|
2708
|
+
+ 'the caller holds the only copy; the lifecycle owner decides what happens to the process (onFatal), '
|
|
2709
|
+
+ 'and only an explicit connect() would open a new connection. ';
|
|
2710
|
+
return this._carryVerdict(new RecoveryEndedPublishError(
|
|
2711
|
+
`[RabbitMQClient] Cannot recreate the ${role} channel: ${why} `
|
|
2712
|
+
+ 'Expected: a live connection before the send. '
|
|
2713
|
+
+ fix
|
|
2714
|
+
+ 'error.code MQ_CONNECTION_FATAL (with error.reason) means this client is over and onFatal has been '
|
|
2715
|
+
+ 'called; error.cause is what ended the recovery, when an object exists for it.',
|
|
2716
|
+
ended || undefined,
|
|
2717
|
+
{ replayable }
|
|
2718
|
+
), ended);
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2179
2721
|
/**
|
|
2180
2722
|
* Re-register every tracked consumer on the current consumer channel.
|
|
2181
2723
|
*
|
|
@@ -2212,6 +2754,33 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2212
2754
|
return { reRegistered, failed };
|
|
2213
2755
|
}
|
|
2214
2756
|
|
|
2757
|
+
/**
|
|
2758
|
+
* The consumer channel, if it is alive — otherwise a refusal that names why.
|
|
2759
|
+
*
|
|
2760
|
+
* The one question every step that is about to use the consumer channel asks
|
|
2761
|
+
* (`_isChannelAlive()`, the one liveness source, d.260). A refusal here is not a broker
|
|
2762
|
+
* verdict: the consumer stays tracked and the next recovery re-attaches it.
|
|
2763
|
+
*
|
|
2764
|
+
* @param {string} action - what was about to happen, as the refusal states it
|
|
2765
|
+
* @param {string} queue - the queue it is about
|
|
2766
|
+
* @returns {Object} the live consumer channel
|
|
2767
|
+
* @throws {ConnectionError} when the channel is dead or gone
|
|
2768
|
+
* @private
|
|
2769
|
+
*/
|
|
2770
|
+
_liveConsumerChannel(action, queue) {
|
|
2771
|
+
const channel = this._consumerChannel;
|
|
2772
|
+
if (!this._isChannelAlive(channel)) {
|
|
2773
|
+
throw new ConnectionError(
|
|
2774
|
+
`[RabbitMQClient] Cannot ${action} ${queue}: consumer channel is not available - `
|
|
2775
|
+
+ 'Expected: a live consumer channel for the whole re-registration. '
|
|
2776
|
+
+ 'Fix: nothing to do by hand — the channel died during the recovery (closed by the broker, or '
|
|
2777
|
+
+ 'discarded after brokerAnswerTimeout); the consumer stays tracked and the next channel recreate '
|
|
2778
|
+
+ 're-attaches it. Listen for consumer:re-registration:failed to see which ones wait.'
|
|
2779
|
+
);
|
|
2780
|
+
}
|
|
2781
|
+
return channel;
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2215
2784
|
/**
|
|
2216
2785
|
* Re-attach ONE tracked consumer on the current consumer channel.
|
|
2217
2786
|
*
|
|
@@ -2235,19 +2804,29 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2235
2804
|
*/
|
|
2236
2805
|
async _reRegisterConsumer(queue, consumerInfo) {
|
|
2237
2806
|
try {
|
|
2807
|
+
// Asked before EACH re-registration, not once per loop (d.1138): a channel that
|
|
2808
|
+
// died after the previous consumer — closed by the broker, or discarded by the
|
|
2809
|
+
// broker-answer budget (d.1135) — leaves the slot empty, and reaching into it was a
|
|
2810
|
+
// TypeError naming a JavaScript accident instead of the channel's state. Asked
|
|
2811
|
+
// before the queue step, so a consumer that cannot be attached declares nothing.
|
|
2812
|
+
this._liveConsumerChannel('re-register the consumer of', queue);
|
|
2813
|
+
|
|
2238
2814
|
await this._prepareQueueForConsume(queue, {
|
|
2239
2815
|
durable: consumerInfo.options.durable,
|
|
2240
2816
|
queueOptions: consumerInfo.options.queueOptions
|
|
2241
2817
|
});
|
|
2242
2818
|
|
|
2819
|
+
// …and again after it: the queue step awaits the broker on another channel,
|
|
2820
|
+
// and the consumer channel can die meanwhile.
|
|
2821
|
+
const consumerChannel = this._liveConsumerChannel('re-register the consumer of', queue);
|
|
2822
|
+
|
|
2243
2823
|
// The consumer's prefetch is re-applied here for the same reason the
|
|
2244
2824
|
// queue step is: the channel is new, and QoS lives on the channel.
|
|
2245
|
-
await this._applyConsumerPrefetch(queue, consumerInfo.options.prefetch);
|
|
2825
|
+
await this._applyConsumerPrefetch(consumerChannel, queue, consumerInfo.options.prefetch);
|
|
2246
2826
|
|
|
2247
|
-
const consumeResult = await this.
|
|
2248
|
-
queue,
|
|
2249
|
-
consumerInfo.handler,
|
|
2250
|
-
{ noAck: consumerInfo.options.noAck }
|
|
2827
|
+
const consumeResult = await this._askBroker(
|
|
2828
|
+
{ operation: 'consume', target: queue, channel: consumerChannel },
|
|
2829
|
+
() => consumerChannel.consume(queue, consumerInfo.handler, { noAck: consumerInfo.options.noAck })
|
|
2251
2830
|
);
|
|
2252
2831
|
consumerInfo.consumerTag = consumeResult.consumerTag;
|
|
2253
2832
|
this._logger.info(`[RabbitMQClient] [mq-client-core] ✓ Re-registered consumer for queue: ${queue} (consumerTag: ${consumeResult.consumerTag})`);
|
|
@@ -2467,7 +3046,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2467
3046
|
queues: {
|
|
2468
3047
|
checked: 0,
|
|
2469
3048
|
missing: [],
|
|
2470
|
-
noConsumer: []
|
|
3049
|
+
noConsumer: [],
|
|
3050
|
+
// The BROKER's count of consumers on each tracked queue (`checkQueue().consumerCount`
|
|
3051
|
+
// — everybody's, not only this client's), or 'unknown' where the probe did not run
|
|
3052
|
+
// or got no answer. Never 0 for "not asked" (d.1275).
|
|
3053
|
+
consumers: {}
|
|
2471
3054
|
},
|
|
2472
3055
|
healthy: true,
|
|
2473
3056
|
issues: []
|
|
@@ -2493,34 +3076,48 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2493
3076
|
health.issues.push('Consumer channel is closed or missing');
|
|
2494
3077
|
}
|
|
2495
3078
|
|
|
2496
|
-
//
|
|
3079
|
+
// Which of this client's consumers are attached is THIS client's fact, counted from its
|
|
3080
|
+
// own state — the consumer channel and the tags (`_isConsumerAttached()`, the definition
|
|
3081
|
+
// `getConsumerState()` reads too, d.613). Counted BEFORE the broker is asked and whatever
|
|
3082
|
+
// the broker answers: until d.1275 `active` was incremented only inside the probe loop
|
|
3083
|
+
// below, so a dead queue channel or an unanswered `checkQueue` left it at 0 and the
|
|
3084
|
+
// verdict further down told the operator a consuming client consumed nothing (INFRA 531).
|
|
3085
|
+
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
3086
|
+
health.queues.consumers[queue] = 'unknown';
|
|
3087
|
+
if (this._isConsumerAttached(consumerInfo)) {
|
|
3088
|
+
health.consumers.active++;
|
|
3089
|
+
} else {
|
|
3090
|
+
health.consumers.inactive.push(queue);
|
|
3091
|
+
}
|
|
3092
|
+
}
|
|
3093
|
+
|
|
3094
|
+
// The probe only ADDS what the broker knows: does each queue exist, and how many
|
|
3095
|
+
// consumers does it count on it. Where it cannot ask, the count stays 'unknown'.
|
|
2497
3096
|
if (this.isConnected() && this._isChannelAlive(this._queueChannel)) {
|
|
2498
3097
|
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
2499
3098
|
health.queues.checked++;
|
|
2500
3099
|
try {
|
|
2501
3100
|
// Check if queue exists
|
|
2502
3101
|
this._trackChannelOperation(this._queueChannel, `checkQueue ${queue}`);
|
|
2503
|
-
const
|
|
3102
|
+
const queueChannel = this._queueChannel;
|
|
3103
|
+
const queueInfo = await this._askBroker(
|
|
3104
|
+
{ operation: 'checkQueue', target: queue, channel: queueChannel },
|
|
3105
|
+
() => queueChannel.checkQueue(queue)
|
|
3106
|
+
);
|
|
2504
3107
|
if (queueInfo) {
|
|
2505
|
-
// Queue exists
|
|
2506
|
-
//
|
|
2507
|
-
//
|
|
2508
|
-
//
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
// this client's and everybody else's alike, so it cannot say whether
|
|
2512
|
-
// the one asking is among them. The tag can: it is the name the
|
|
2513
|
-
// broker gave THIS client's attachment, and since d.386b it is set
|
|
2514
|
-
// only while that attachment exists. The question is asked through
|
|
2515
|
-
// `_isConsumerAttached()` — the one definition `getConsumerState()`
|
|
2516
|
-
// now reads too (d.613).
|
|
2517
|
-
if (this._isConsumerAttached(consumerInfo)) {
|
|
2518
|
-
health.consumers.active++;
|
|
2519
|
-
} else {
|
|
2520
|
-
health.consumers.inactive.push(queue);
|
|
3108
|
+
// Queue exists. The broker's `consumerCount` counts every consumer on the
|
|
3109
|
+
// queue, this client's and everybody else's alike, so it is REPORTED, never
|
|
3110
|
+
// used to decide whether the one asking is among them — that is
|
|
3111
|
+
// `_isConsumerAttached()`, counted above.
|
|
3112
|
+
health.queues.consumers[queue] = queueInfo.consumerCount;
|
|
3113
|
+
if (!this._isConsumerAttached(consumerInfo)) {
|
|
2521
3114
|
health.queues.noConsumer.push(queue);
|
|
2522
3115
|
health.healthy = false;
|
|
2523
|
-
health.issues.push(
|
|
3116
|
+
health.issues.push(
|
|
3117
|
+
this._isChannelAlive(this._consumerChannel)
|
|
3118
|
+
? `Queue ${queue} has no active consumer (consumerTag missing)`
|
|
3119
|
+
: `Queue ${queue} has no active consumer (consumer channel is not alive)`
|
|
3120
|
+
);
|
|
2524
3121
|
}
|
|
2525
3122
|
} else {
|
|
2526
3123
|
health.queues.missing.push(queue);
|
|
@@ -2544,12 +3141,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2544
3141
|
// that is its own verdict — not something a reader infers from two numbers
|
|
2545
3142
|
// sitting beside the channel findings. The per-queue issues above name WHICH
|
|
2546
3143
|
// queue lost its consumer, and they do not always exist: when the probe loop
|
|
2547
|
-
// does not run (no connection, or a dead queue channel)
|
|
2548
|
-
// consumer evidence is `tracked` and `active`, and every issue names a
|
|
3144
|
+
// does not run (no connection, or a dead queue channel) every issue names a
|
|
2549
3145
|
// channel instead. Measured 2026-09-08 (INFRA-monitoring): the whole account
|
|
2550
3146
|
// a reader got of a client taking no message off any queue was the line
|
|
2551
3147
|
// `Health check OK: 0/0 consumers active`.
|
|
2552
3148
|
//
|
|
3149
|
+
// `active` is this client's own count (above), so the verdict is never the
|
|
3150
|
+
// probe's absence restated: a dead queue channel beside a live consumer
|
|
3151
|
+
// channel is a channel finding, not "no consumer" (d.1275).
|
|
3152
|
+
//
|
|
2553
3153
|
// `tracked === 0` is a different client, not a lesser failure: a publisher
|
|
2554
3154
|
// registered no consumer and has none to lose. It stays healthy, and the OK
|
|
2555
3155
|
// line below says which of the two it is.
|
|
@@ -2568,7 +3168,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2568
3168
|
// Emit health check result
|
|
2569
3169
|
this.emit('health:check', health);
|
|
2570
3170
|
|
|
2571
|
-
// If unhealthy, log and
|
|
3171
|
+
// If unhealthy, log and report
|
|
2572
3172
|
if (!health.healthy) {
|
|
2573
3173
|
this._logger.error(`[RabbitMQClient] [mq-client-core] Health check FAILED:`, health.issues);
|
|
2574
3174
|
this._logger.error(`[RabbitMQClient] [mq-client-core] Health details:`, JSON.stringify(health, null, 2));
|
|
@@ -2597,21 +3197,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2597
3197
|
}
|
|
2598
3198
|
}
|
|
2599
3199
|
|
|
2600
|
-
// Emit critical health event
|
|
3200
|
+
// Emit critical health event. That is the whole of it: the client REPORTS critical
|
|
3201
|
+
// health and decides nothing about the process. `health:shutdown` and the
|
|
3202
|
+
// `criticalHealthShutdown*` keys that armed it were removed in d.1275 (owner
|
|
3203
|
+
// 2026-10-06) — the event reached no listener anywhere, and the line announcing it
|
|
3204
|
+
// promised a shutdown nobody performed. A lifecycle owner that wants to act on a
|
|
3205
|
+
// sustained critical state reads this event or `healthCriticalCallback`.
|
|
2601
3206
|
this.emit('health:critical', health);
|
|
2602
|
-
|
|
2603
|
-
// Check if we should shutdown after delay
|
|
2604
|
-
if (this._criticalHealthShutdown) {
|
|
2605
|
-
const timeSinceCritical = Date.now() - this._criticalHealthStartTime;
|
|
2606
|
-
if (timeSinceCritical >= this._criticalHealthShutdownDelay) {
|
|
2607
|
-
this._logger.error(`[RabbitMQClient] [mq-client-core] Critical health persisted for ${timeSinceCritical}ms (threshold: ${this._criticalHealthShutdownDelay}ms) - triggering shutdown`);
|
|
2608
|
-
this.emit('health:shutdown', health);
|
|
2609
|
-
// Note: Actual shutdown should be handled by the application using this event
|
|
2610
|
-
} else {
|
|
2611
|
-
const remaining = this._criticalHealthShutdownDelay - timeSinceCritical;
|
|
2612
|
-
this._logger.warn(`[RabbitMQClient] [mq-client-core] Critical health will trigger shutdown in ${remaining}ms if not resolved`);
|
|
2613
|
-
}
|
|
2614
|
-
}
|
|
2615
3207
|
} else {
|
|
2616
3208
|
// Health is OK - reset critical health tracking
|
|
2617
3209
|
if (this._criticalHealthStartTime) {
|
|
@@ -2854,7 +3446,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2854
3446
|
const channel = await this._requireQueueChannel('assertQueue', queue);
|
|
2855
3447
|
|
|
2856
3448
|
try {
|
|
2857
|
-
return await
|
|
3449
|
+
return await this._askBroker(
|
|
3450
|
+
{ operation: 'assertQueue', target: queue, channel },
|
|
3451
|
+
() => channel.assertQueue(queue, queueOptions)
|
|
3452
|
+
);
|
|
2858
3453
|
} catch (err) {
|
|
2859
3454
|
// The SAME verdict the consume rail states, through the SAME function
|
|
2860
3455
|
// (d.582). This is the rail FÁZE 0.8 walks: a service declaring its own
|
|
@@ -2953,6 +3548,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2953
3548
|
async _requireQueueChannel(verb, queue) {
|
|
2954
3549
|
await this._ensureQueueChannel();
|
|
2955
3550
|
if (!this._isChannelAlive(this._queueChannel)) {
|
|
3551
|
+
// A connection that is down and will not recover on its own is not "wait for the
|
|
3552
|
+
// reconnect": none is coming. The three states say so through the one builder every rail
|
|
3553
|
+
// uses — the verdict on top for a lost connection (d.1314), the state named for
|
|
3554
|
+
// `disconnect()` and `reconnectEnabled: false` (d.1319). A spent cycle (standby) keeps the
|
|
3555
|
+
// sentence below: the next publish, consume or health check starts the next one.
|
|
3556
|
+
if (!this._connectionAlive && !this._recoveryMayStart()) {
|
|
3557
|
+
throw this._refusalWithoutRecovery(`[RabbitMQClient] Cannot ${verb} ${queue}`, verb);
|
|
3558
|
+
}
|
|
2956
3559
|
throw new ConnectionError(
|
|
2957
3560
|
`[RabbitMQClient] Cannot ${verb} ${queue}: queue channel is not available - `
|
|
2958
3561
|
+ 'Expected: an open queue channel after _ensureQueueChannel(). '
|
|
@@ -2984,10 +3587,34 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2984
3587
|
* @private
|
|
2985
3588
|
*/
|
|
2986
3589
|
_publishPathTransient(verb, queue, cause) {
|
|
3590
|
+
// The sentence is chosen from this client's STATE, never from the cause's text (d.1145,
|
|
3591
|
+
// finding of d.1143): since d.1135 this refusal also ends a pre-check the broker never
|
|
3592
|
+
// answered, and on the platform's image the connection was then alive and BLOCKED by a
|
|
3593
|
+
// memory alarm — "connection may be closed" sent the reader to a reconnect that was not
|
|
3594
|
+
// coming. `sent` is false on every one of these (no frame was written), which is what the
|
|
3595
|
+
// flush after `unblocked` reads.
|
|
3596
|
+
let state;
|
|
3597
|
+
let fix;
|
|
3598
|
+
if (!this._connectionAlive) {
|
|
3599
|
+
state = 'the connection is closed';
|
|
3600
|
+
fix = 'Fix: nothing to do here — a reconnect restores it; PublishLayer retries and buffers the message meanwhile, '
|
|
3601
|
+
+ 'and the buffer re-sends it after the reconnect.';
|
|
3602
|
+
} else if (this._connectionBlockedBy !== null) {
|
|
3603
|
+
state = `the broker has blocked this connection (reason: ${this._connectionBlockedBy}) and answers nothing until it clears the alarm`;
|
|
3604
|
+
fix = 'Fix: clear the broker alarm (management API /api/health/checks/alarms names it); PublishLayer retries and '
|
|
3605
|
+
+ 'buffers the message meanwhile, and the buffer re-sends it once the broker unblocks the connection (its frame never left).';
|
|
3606
|
+
} else {
|
|
3607
|
+
// Neither closed nor blocked: the cause says what happened (a request unanswered
|
|
3608
|
+
// within brokerAnswerTimeout, or a channel open the broker refused) — this sentence
|
|
3609
|
+
// does not guess which.
|
|
3610
|
+
state = 'the queue channel is not available on a live, unblocked connection (error.cause names the request '
|
|
3611
|
+
+ 'that went unanswered or was refused)';
|
|
3612
|
+
fix = 'Fix: read error.cause; an unanswered request points at a paused or overloaded broker. PublishLayer retries '
|
|
3613
|
+
+ 'and buffers the message meanwhile.';
|
|
3614
|
+
}
|
|
2987
3615
|
return new TransientPublishError(
|
|
2988
|
-
`[RabbitMQClient] Cannot ${verb} ${queue}:
|
|
2989
|
-
+ `Expected: an open queue channel before ${verb} on the publish path.
|
|
2990
|
-
+ 'Fix: retry the publish after the reconnect; PublishLayer does this on its own and buffers the message meanwhile.',
|
|
3616
|
+
`[RabbitMQClient] Cannot ${verb} ${queue}: ${state} - `
|
|
3617
|
+
+ `Expected: an answering broker on an open queue channel before ${verb} on the publish path. ${fix}`,
|
|
2991
3618
|
cause
|
|
2992
3619
|
);
|
|
2993
3620
|
}
|
|
@@ -3354,7 +3981,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3354
3981
|
...options,
|
|
3355
3982
|
durable: options.durable !== undefined ? options.durable : this._config.durable
|
|
3356
3983
|
};
|
|
3357
|
-
return await
|
|
3984
|
+
return await this._askBroker(
|
|
3985
|
+
{ operation: 'assertExchange', target: exchange, channel },
|
|
3986
|
+
() => channel.assertExchange(exchange, type, exchangeOptions)
|
|
3987
|
+
);
|
|
3358
3988
|
}
|
|
3359
3989
|
|
|
3360
3990
|
/**
|
|
@@ -3398,7 +4028,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3398
4028
|
}
|
|
3399
4029
|
|
|
3400
4030
|
const channel = await this._requireQueueChannel('bindQueue', `${queue} -> ${exchange}`);
|
|
3401
|
-
return await
|
|
4031
|
+
return await this._askBroker(
|
|
4032
|
+
{ operation: 'bindQueue', target: `${queue} -> ${exchange}`, channel },
|
|
4033
|
+
() => channel.bindQueue(queue, exchange, pattern)
|
|
4034
|
+
);
|
|
3402
4035
|
}
|
|
3403
4036
|
|
|
3404
4037
|
/**
|
|
@@ -3461,6 +4094,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3461
4094
|
// its tag named is gone (d.386b). There is nothing to cancel on the broker,
|
|
3462
4095
|
// and asking anyway is a channel error rather than a cancel, so the caller
|
|
3463
4096
|
// is told plainly instead of being handed one.
|
|
4097
|
+
// Asked before the tag: on a connection that is down and will not recover the tag is gone
|
|
4098
|
+
// too, and the sentence that matters is the state — the verdict for a lost connection
|
|
4099
|
+
// (d.1314), `disconnect()` or `reconnectEnabled: false` otherwise (d.1319) — not "tracked
|
|
4100
|
+
// but not attached".
|
|
4101
|
+
if (!this._connectionAlive && !this._recoveryMayStart()) {
|
|
4102
|
+
throw this._refusalWithoutRecovery(`[RabbitMQClient] Cannot cancelConsumer ${queue}`, 'cancelConsumer');
|
|
4103
|
+
}
|
|
4104
|
+
|
|
3464
4105
|
if (info.consumerTag === null || info.consumerTag === undefined) {
|
|
3465
4106
|
throw new ValidationError(
|
|
3466
4107
|
`[RabbitMQClient] Cannot cancelConsumer ${queue}: the consumer is tracked but not attached - `
|
|
@@ -3481,7 +4122,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3481
4122
|
}
|
|
3482
4123
|
|
|
3483
4124
|
this._trackChannelOperation(this._consumerChannel, `cancelConsumer ${queue}`);
|
|
3484
|
-
const
|
|
4125
|
+
const consumerChannel = this._consumerChannel;
|
|
4126
|
+
const result = await this._askBroker(
|
|
4127
|
+
{ operation: 'cancel', target: queue, channel: consumerChannel },
|
|
4128
|
+
() => consumerChannel.cancel(info.consumerTag)
|
|
4129
|
+
);
|
|
3485
4130
|
// Dropped only after the broker accepted the cancel: a consumer that is
|
|
3486
4131
|
// still delivering must stay on the list recovery re-attaches.
|
|
3487
4132
|
this._activeConsumers.delete(queue);
|
|
@@ -3498,7 +4143,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3498
4143
|
*/
|
|
3499
4144
|
async checkQueue(queue) {
|
|
3500
4145
|
const channel = await this._requireQueueChannel('checkQueue', queue);
|
|
3501
|
-
return await
|
|
4146
|
+
return await this._askBroker(
|
|
4147
|
+
{ operation: 'checkQueue', target: queue, channel },
|
|
4148
|
+
() => channel.checkQueue(queue)
|
|
4149
|
+
);
|
|
3502
4150
|
}
|
|
3503
4151
|
|
|
3504
4152
|
/**
|
|
@@ -3507,7 +4155,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3507
4155
|
*/
|
|
3508
4156
|
async purgeQueue(queue) {
|
|
3509
4157
|
const channel = await this._requireQueueChannel('purgeQueue', queue);
|
|
3510
|
-
return await
|
|
4158
|
+
return await this._askBroker(
|
|
4159
|
+
{ operation: 'purgeQueue', target: queue, channel },
|
|
4160
|
+
() => channel.purgeQueue(queue)
|
|
4161
|
+
);
|
|
3511
4162
|
}
|
|
3512
4163
|
|
|
3513
4164
|
/**
|
|
@@ -3517,7 +4168,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3517
4168
|
*/
|
|
3518
4169
|
async deleteQueue(queue, options = {}) {
|
|
3519
4170
|
const channel = await this._requireQueueChannel('deleteQueue', queue);
|
|
3520
|
-
return await
|
|
4171
|
+
return await this._askBroker(
|
|
4172
|
+
{ operation: 'deleteQueue', target: queue, channel },
|
|
4173
|
+
() => channel.deleteQueue(queue, options)
|
|
4174
|
+
);
|
|
3521
4175
|
}
|
|
3522
4176
|
|
|
3523
4177
|
/**
|
|
@@ -3536,10 +4190,50 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3536
4190
|
}
|
|
3537
4191
|
|
|
3538
4192
|
/**
|
|
3539
|
-
*
|
|
4193
|
+
* One publish attempt, without retry — and, when it fails, whether its frame LEFT THE
|
|
4194
|
+
* CLIENT (d.1145, conf `mq-publish-buffer` 001).
|
|
4195
|
+
*
|
|
4196
|
+
* Every error this attempt throws carries `sent` (boolean): `true` once `sendToQueue()` /
|
|
4197
|
+
* `publish()` on the channel returned without throwing — the frame was written to the
|
|
4198
|
+
* channel — and `false` for everything that failed before that (the pre-check, the channel
|
|
4199
|
+
* ensure, a throwing send). It says nothing about whether the broker stored the frame; it is
|
|
4200
|
+
* what tells a message the broker may still deliver (held in a blocked socket, measured
|
|
4201
|
+
* d.1143) from one it never received. `PublishLayer` folds the attempts of one call into the
|
|
4202
|
+
* fact it writes on its own error and on the buffer entry.
|
|
4203
|
+
*
|
|
4204
|
+
* A nested republish from the confirm path (see `_publishOneAttempt`) may throw an error
|
|
4205
|
+
* that already says `sent: true`; that fact is kept, never lowered.
|
|
4206
|
+
*
|
|
4207
|
+
* @param {string} queue
|
|
4208
|
+
* @param {Buffer} buffer
|
|
4209
|
+
* @param {Object} [options]
|
|
4210
|
+
* @returns {Promise<void>}
|
|
4211
|
+
* @throws {Error} the attempt's failure, with `sent` written on it
|
|
3540
4212
|
* @private
|
|
3541
4213
|
*/
|
|
3542
4214
|
async _publishOnce(queue, buffer, options = {}) {
|
|
4215
|
+
const attempt = { sent: false };
|
|
4216
|
+
try {
|
|
4217
|
+
return await this._publishOneAttempt(queue, buffer, options, attempt);
|
|
4218
|
+
} catch (err) {
|
|
4219
|
+
if (err !== null && typeof err === 'object') {
|
|
4220
|
+
err.sent = attempt.sent || err.sent === true;
|
|
4221
|
+
}
|
|
4222
|
+
throw err;
|
|
4223
|
+
}
|
|
4224
|
+
}
|
|
4225
|
+
|
|
4226
|
+
/**
|
|
4227
|
+
* The body of one publish attempt. `attempt.sent` turns `true` the moment the frame is
|
|
4228
|
+
* written to the channel; `_publishOnce()` reads it.
|
|
4229
|
+
*
|
|
4230
|
+
* @param {string} queue
|
|
4231
|
+
* @param {Buffer} buffer
|
|
4232
|
+
* @param {Object} options
|
|
4233
|
+
* @param {{sent: boolean}} attempt - written here, read by `_publishOnce()`
|
|
4234
|
+
* @private
|
|
4235
|
+
*/
|
|
4236
|
+
async _publishOneAttempt(queue, buffer, options, attempt) {
|
|
3543
4237
|
// Whose channel this send goes out on (d.724). Read once, here: the role cannot
|
|
3544
4238
|
// change inside one attempt, and the channel it names can — so everything below asks
|
|
3545
4239
|
// `_sendChannelOf(sendRole)` rather than holding a member.
|
|
@@ -3620,7 +4314,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3620
4314
|
}
|
|
3621
4315
|
throw railErr;
|
|
3622
4316
|
}
|
|
3623
|
-
|
|
4317
|
+
const checkedOn = queueChannel;
|
|
4318
|
+
await this._askBroker(
|
|
4319
|
+
{ operation: 'checkQueue', target: queue, channel: checkedOn },
|
|
4320
|
+
() => checkedOn.checkQueue(queue)
|
|
4321
|
+
);
|
|
3624
4322
|
// Queue exists - proceed to publish
|
|
3625
4323
|
} catch (checkErr) {
|
|
3626
4324
|
// If queue doesn't exist (404), this je ERROR – ale rozlišujeme infra vs. non-infra:
|
|
@@ -3714,6 +4412,12 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3714
4412
|
// - not already typed — `_publishPathTransient()` above throws into
|
|
3715
4413
|
// this same `catch`, and wrapping a verdict in itself would bury the
|
|
3716
4414
|
// rail's own sentence one level deeper for no reader.
|
|
4415
|
+
//
|
|
4416
|
+
// Since d.1135 the check goes through `_askBroker()`, so what arrives here
|
|
4417
|
+
// is its `ConnectionError`: either the budget ran out (the channel is then
|
|
4418
|
+
// discarded — dead) or amqplib's bare rejection, folded where it arrived. Both
|
|
4419
|
+
// meet the three conditions unchanged; the bare error sits one level down, in
|
|
4420
|
+
// `cause.cause`.
|
|
3717
4421
|
throw this._publishPathTransient('checkQueue', queue, checkErr);
|
|
3718
4422
|
} else {
|
|
3719
4423
|
// Other error - rethrow
|
|
@@ -3844,12 +4548,20 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3844
4548
|
return this.publish(queue, buffer, options);
|
|
3845
4549
|
}).then(resolve).catch(reject);
|
|
3846
4550
|
} else {
|
|
3847
|
-
//
|
|
3848
|
-
//
|
|
4551
|
+
// No reconnect is in flight YET — which says nothing about whether one
|
|
4552
|
+
// follows. When the connection dies, amqplib closes its channels before
|
|
4553
|
+
// it announces the connection's own `close` (`lib/connection.js`,
|
|
4554
|
+
// `toClosed()`), and that announcement is what starts the reconnect, so
|
|
4555
|
+
// this callback runs before it (d.1134). The sentence therefore states
|
|
4556
|
+
// what is known here — the channel closed with the confirm outstanding —
|
|
4557
|
+
// and the classification makes it transient, so the publish path
|
|
4558
|
+
// retries and buffers it.
|
|
3849
4559
|
reject(new PublishError(
|
|
3850
|
-
`[RabbitMQClient] Channel closed during publish to queue "${queue}"
|
|
3851
|
-
+ '
|
|
3852
|
-
+ '
|
|
4560
|
+
`[RabbitMQClient] Channel closed during publish to queue "${queue}" before the broker confirmed the message - `
|
|
4561
|
+
+ 'message delivery not guaranteed. '
|
|
4562
|
+
+ 'Expected: the channel to stay open until the broker confirms. '
|
|
4563
|
+
+ 'Fix: republish the message — the confirm never arrived, so it may or may not have been stored; '
|
|
4564
|
+
+ 'error.buffered says whether the buffer holds it for the next reconnect.',
|
|
3853
4565
|
queue,
|
|
3854
4566
|
null
|
|
3855
4567
|
));
|
|
@@ -3890,6 +4602,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3890
4602
|
resolve();
|
|
3891
4603
|
}
|
|
3892
4604
|
});
|
|
4605
|
+
// The frame is written to the channel: `sendToQueue()` returned without throwing.
|
|
4606
|
+
attempt.sent = true;
|
|
3893
4607
|
|
|
3894
4608
|
// Set a safety timeout - if callback wasn't invoked and channel closed, retry.
|
|
3895
4609
|
// A broker confirm can land inside `sendToQueue()` itself, so the
|
|
@@ -4077,6 +4791,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4077
4791
|
resolve();
|
|
4078
4792
|
}
|
|
4079
4793
|
});
|
|
4794
|
+
// The frame is written to the channel: `publish()` returned without throwing.
|
|
4795
|
+
attempt.sent = true;
|
|
4080
4796
|
});
|
|
4081
4797
|
|
|
4082
4798
|
await confirmPromise;
|
|
@@ -4118,17 +4834,22 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4118
4834
|
* A non-numeric prefetch (the caller asked for none) sets none, and records
|
|
4119
4835
|
* none: there is nothing to measure utilisation against.
|
|
4120
4836
|
*
|
|
4837
|
+
* @param {Object} consumerChannel - The consumer channel the caller holds; the
|
|
4838
|
+
* re-registration hands over the one it has just asked is alive (d.1138)
|
|
4121
4839
|
* @param {string} queue - Queue name
|
|
4122
4840
|
* @param {number|null|undefined} prefetch - The consumer's prefetch
|
|
4123
4841
|
* @returns {Promise<void>}
|
|
4124
4842
|
* @private
|
|
4125
4843
|
*/
|
|
4126
|
-
async _applyConsumerPrefetch(queue, prefetch) {
|
|
4844
|
+
async _applyConsumerPrefetch(consumerChannel, queue, prefetch) {
|
|
4127
4845
|
if (typeof prefetch !== 'number') {
|
|
4128
4846
|
return;
|
|
4129
4847
|
}
|
|
4130
4848
|
|
|
4131
|
-
await this.
|
|
4849
|
+
await this._askBroker(
|
|
4850
|
+
{ operation: 'prefetch', target: queue, channel: consumerChannel },
|
|
4851
|
+
() => consumerChannel.prefetch(prefetch)
|
|
4852
|
+
);
|
|
4132
4853
|
|
|
4133
4854
|
const tracking = this._prefetchTracking.get(queue);
|
|
4134
4855
|
if (tracking) {
|
|
@@ -4326,7 +5047,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4326
5047
|
// If missing, fail-fast. The owning infra service must recreate on startup.
|
|
4327
5048
|
const queueChannel = await this._requireQueueChannel('checkQueue', queue);
|
|
4328
5049
|
try {
|
|
4329
|
-
await
|
|
5050
|
+
await this._askBroker(
|
|
5051
|
+
{ operation: 'checkQueue', target: queue, channel: queueChannel },
|
|
5052
|
+
() => queueChannel.checkQueue(queue)
|
|
5053
|
+
);
|
|
4330
5054
|
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Infrastructure queue ${queue} exists (consumer will proceed)`);
|
|
4331
5055
|
} catch (checkErr) {
|
|
4332
5056
|
if (checkErr.code === AMQP_NOT_FOUND) {
|
|
@@ -4451,7 +5175,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4451
5175
|
// Set prefetch if provided (on consumer channel). ONE rail, shared with
|
|
4452
5176
|
// `_reRegisterConsumers()` — prefetch belongs to the consumer, not to the
|
|
4453
5177
|
// channel that happened to be open when it first attached.
|
|
4454
|
-
await this._applyConsumerPrefetch(queue, prefetch);
|
|
5178
|
+
await this._applyConsumerPrefetch(this._consumerChannel, queue, prefetch);
|
|
4455
5179
|
|
|
4456
5180
|
// CRITICAL: Log before calling amqplib's consume()
|
|
4457
5181
|
// amqplib's consume() may internally call assertQueue() without parameters if queue doesn't exist
|
|
@@ -4666,10 +5390,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4666
5390
|
}
|
|
4667
5391
|
};
|
|
4668
5392
|
|
|
4669
|
-
const
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
{ noAck }
|
|
5393
|
+
const consumerChannel = this._consumerChannel;
|
|
5394
|
+
const consumeResult = await this._askBroker(
|
|
5395
|
+
{ operation: 'consume', target: queue, channel: consumerChannel },
|
|
5396
|
+
() => consumerChannel.consume(queue, messageHandler, { noAck })
|
|
4673
5397
|
);
|
|
4674
5398
|
|
|
4675
5399
|
// Track consumer for auto re-registration if channel closes
|
|
@@ -4778,13 +5502,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4778
5502
|
this._logger.info(
|
|
4779
5503
|
'[RabbitMQClient] Client was closed by its caller during recovery - closing the re-established connection'
|
|
4780
5504
|
);
|
|
4781
|
-
|
|
4782
|
-
|
|
4783
|
-
|
|
5505
|
+
// Within the one budget (d.1138, conf `mq-broker-answer-budget` 001): this close
|
|
5506
|
+
// waits for a `close-ok` like every teardown close, and a broker that stopped
|
|
5507
|
+
// answering right now would otherwise hold this round — and `_reconnecting` —
|
|
5508
|
+
// open forever. Unanswered, the socket is dropped the way the teardown drops it.
|
|
5509
|
+
const closing = await this._closeWithinTeardownBudget(
|
|
5510
|
+
'connection re-established after disconnect',
|
|
5511
|
+
revived,
|
|
5512
|
+
this._brokerAnswerTimeout
|
|
5513
|
+
);
|
|
5514
|
+
if (closing.status === 'refused') {
|
|
4784
5515
|
this._logger.warn(
|
|
4785
|
-
`[RabbitMQClient] Error closing the connection re-established after disconnect: ${
|
|
5516
|
+
`[RabbitMQClient] Error closing the connection re-established after disconnect: ${closing.error.message}`
|
|
4786
5517
|
);
|
|
4787
5518
|
}
|
|
5519
|
+
if (closing.status === 'unanswered') {
|
|
5520
|
+
this._dropUnansweredConnection(revived);
|
|
5521
|
+
}
|
|
4788
5522
|
this._connection = null;
|
|
4789
5523
|
this._connectionAlive = false;
|
|
4790
5524
|
this._reconnecting = false;
|
|
@@ -4831,7 +5565,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4831
5565
|
|
|
4832
5566
|
// Recreate consumer channel and re-register all consumers
|
|
4833
5567
|
try {
|
|
4834
|
-
this._consumerChannel = await this.
|
|
5568
|
+
this._consumerChannel = await this._askBroker(
|
|
5569
|
+
{ operation: 'createChannel', target: 'consumer channel', channel: null },
|
|
5570
|
+
() => this._connection.createChannel()
|
|
5571
|
+
);
|
|
4835
5572
|
this._consumerChannel._createdAt = new Date().toISOString();
|
|
4836
5573
|
this._consumerChannel._closeReason = null;
|
|
4837
5574
|
this._consumerChannel._lastOperation = 'Recreated via _reconnectWithBackoff()';
|
|
@@ -4882,7 +5619,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4882
5619
|
// Flush buffered messages po reconnectu
|
|
4883
5620
|
let bufferFlushResult = { inMemory: 0, flushed: false };
|
|
4884
5621
|
try {
|
|
4885
|
-
const flushResult = await this._publishLayer.flushBuffered();
|
|
5622
|
+
const flushResult = await this._publishLayer.flushBuffered('reconnected');
|
|
4886
5623
|
bufferFlushResult = { inMemory: flushResult, flushed: true };
|
|
4887
5624
|
this._publishMonitor.trackFlushed(bufferFlushResult.inMemory);
|
|
4888
5625
|
this._logger.info(`[RabbitMQClient] ✓ Flushed ${bufferFlushResult.inMemory} buffered messages after reconnection`);
|
|
@@ -5033,7 +5770,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
5033
5770
|
this.emit('error', err);
|
|
5034
5771
|
});
|
|
5035
5772
|
|
|
5036
|
-
|
|
5773
|
+
// PREPENDED, not appended (d.1134). amqplib registers its own `close` listener in the
|
|
5774
|
+
// channel's constructor — the one that answers every unconfirmed publish with the bare
|
|
5775
|
+
// `new Error('channel closed')` (`lib/channel.js`, the `Channel` constructor) — and a
|
|
5776
|
+
// constructor runs before anybody can attach anything, so an appended handler always
|
|
5777
|
+
// ran SECOND. The publish callbacks then read a channel that was still alive, took the
|
|
5778
|
+
// branch meant for a broker answer on a live channel, and the sentinel reached the
|
|
5779
|
+
// classification untyped: permanent, not buffered, not retried — for a message the
|
|
5780
|
+
// broker said nothing about (měřeno d.1126, `320 CONNECTION_FORCED` with a confirm in
|
|
5781
|
+
// flight). `connection.toClosed()` closes every channel before the connection's own
|
|
5782
|
+
// events (`lib/connection.js`), so this handler is the only one that can mark the
|
|
5783
|
+
// channel dead in time. Prepending makes it run before amqplib's own listener — an
|
|
5784
|
+
// order the emitter guarantees, with no timer and no race.
|
|
5785
|
+
channel.prependListener('close', async () => {
|
|
5037
5786
|
if (this._closedByCaller) {
|
|
5038
5787
|
return;
|
|
5039
5788
|
}
|
|
@@ -5216,7 +5965,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
5216
5965
|
// catch polykal jako `warn`, takže konzumenti tiše přestali dostávat zprávy.
|
|
5217
5966
|
// `_ensureConsumerChannel()` odpovídá `ConnectionError` s chybou amqplibu
|
|
5218
5967
|
// v `cause` — jeden článek, vlastní příčina obalu (vzor obnovy frontového
|
|
5219
|
-
// kanálu níž, d.696).
|
|
5968
|
+
// kanálu níž, d.696). Od d.1278 i s chybou, kterou skončilo čekání na obnovu
|
|
5969
|
+
// (standby bez `code`, nebo fatální s `MQ_CONNECTION_FATAL`): spojení tehdy
|
|
5970
|
+
// neživé, takže `_connectionIsGoingAway()` odpoví `warn` — standby i fatal už
|
|
5971
|
+
// ohlásily samy sebe na vlastním kanálu, druhé hlášení na `error` sem nepatří.
|
|
5220
5972
|
if (this._connectionIsGoingAway(err.cause)) {
|
|
5221
5973
|
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel (connection may be closing): ${err.message}`);
|
|
5222
5974
|
} else {
|
|
@@ -5645,6 +6397,12 @@ class RabbitMQClient extends EventEmitter {
|
|
|
5645
6397
|
if (msg._mqProcessed) {
|
|
5646
6398
|
return; // Already acked/nacked — idempotent
|
|
5647
6399
|
}
|
|
6400
|
+
// On a connection that is down and will not recover on its own (lost, `disconnect()`,
|
|
6401
|
+
// `reconnectEnabled: false`) the delivery cannot be settled — the broker has requeued it
|
|
6402
|
+
// already; the caller learns the state, the verdict included (d.1314, d.1319).
|
|
6403
|
+
if (!this._connectionAlive && !this._recoveryMayStart()) {
|
|
6404
|
+
throw this._refusalWithoutRecovery('[RabbitMQClient] Cannot ack', 'ack');
|
|
6405
|
+
}
|
|
5648
6406
|
if (!this._consumerChannel) {
|
|
5649
6407
|
throw new ConnectionError(
|
|
5650
6408
|
'[RabbitMQClient] Cannot ack: consumer channel is not initialized - Expected: an open consumer channel. '
|
|
@@ -5687,6 +6445,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
5687
6445
|
if (msg._mqProcessed) {
|
|
5688
6446
|
return; // Already acked/nacked — idempotent
|
|
5689
6447
|
}
|
|
6448
|
+
if (!this._connectionAlive && !this._recoveryMayStart()) {
|
|
6449
|
+
throw this._refusalWithoutRecovery('[RabbitMQClient] Cannot nack', 'nack');
|
|
6450
|
+
}
|
|
5690
6451
|
if (!this._consumerChannel) {
|
|
5691
6452
|
throw new ConnectionError(
|
|
5692
6453
|
'[RabbitMQClient] Cannot nack: consumer channel is not initialized - Expected: an open consumer channel. '
|