@onlineapps/mq-client-core 2.0.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +153 -0
- package/README.md +491 -8
- package/package.json +12 -8
- package/src/BaseClient.js +953 -80
- package/src/buffer/InMemoryBuffer.js +50 -9
- package/src/buffer/MessageBuffer.js +20 -52
- package/src/config/composeConfig.js +60 -0
- package/src/config/configSchema.js +398 -17
- package/src/config/defaultConfig.js +169 -15
- package/src/config/deliveryPolicy.js +165 -0
- package/src/config/queueConfig.js +738 -64
- package/src/config.js +29 -0
- package/src/defaults.js +43 -0
- package/src/index.js +91 -2
- package/src/layers/PublishLayer.js +83 -37
- package/src/monitoring/PublishMonitor.js +11 -4
- package/src/monitoring-publish.js +81 -54
- package/src/transports/rabbitmqClient.js +2698 -738
- package/src/transports/transportFactory.js +9 -2
- package/src/utils/errorHandler.js +83 -4
- package/src/utils/nearestKey.js +101 -0
- package/src/utils/publishErrors.js +95 -10
- package/src/utils/redactCredentials.js +106 -0
- package/src/utils/serializer.js +12 -2
- package/src/workers/RecoveryWorker.js +58 -81
- package/src/buffer/RedisBuffer.js +0 -57
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* RabbitMQClient: transport implementation for RabbitMQ using amqplib.
|
|
5
|
-
* Simplified version for infrastructure services - no queueConfig dependency.
|
|
6
5
|
* Implements connect, disconnect, publish, consume, ack, nack, and error propagation.
|
|
6
|
+
* Queue arguments come from the central `../config/queueConfig` — see the require below.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
const amqp = require('amqplib');
|
|
@@ -14,12 +14,66 @@ const {
|
|
|
14
14
|
QueueNotFoundError,
|
|
15
15
|
classifyPublishError,
|
|
16
16
|
missingInfrastructureQueueMessage,
|
|
17
|
+
missingQueueDefinitionMessage,
|
|
17
18
|
} = require('../utils/publishErrors');
|
|
18
|
-
const {
|
|
19
|
+
const {
|
|
20
|
+
ConnectionError,
|
|
21
|
+
PublishError,
|
|
22
|
+
ConsumeError,
|
|
23
|
+
ValidationError,
|
|
24
|
+
CONNECTION_CLOSED_UNEXPECTEDLY,
|
|
25
|
+
connectionClosedUnexpectedly,
|
|
26
|
+
CONSUMER_QUEUE_MISSING,
|
|
27
|
+
CONSUMER_DEAD_LETTER_ROUTE_MISSING
|
|
28
|
+
} = require('../utils/errorHandler');
|
|
29
|
+
const { assertLogger } = require('@onlineapps/logger-contract');
|
|
19
30
|
const PublishLayer = require('../layers/PublishLayer');
|
|
20
31
|
const RecoveryWorker = require('../workers/RecoveryWorker');
|
|
21
32
|
const PublishMonitor = require('../monitoring/PublishMonitor');
|
|
22
33
|
const runtimeCfg = require('../config');
|
|
34
|
+
const defaultConfig = require('../config/defaultConfig');
|
|
35
|
+
const composeConfig = require('../config/composeConfig');
|
|
36
|
+
const { redactConnectionTarget } = require('../utils/redactCredentials');
|
|
37
|
+
// Central queue classification and arguments. A sibling module of this file: it loads
|
|
38
|
+
// with the package or the package does not run, so there is nothing to guard against
|
|
39
|
+
// and nothing to stand in for it. It used to be required lazily inside three methods,
|
|
40
|
+
// one of them under a `catch` that carried on with "default queue options" — the exact
|
|
41
|
+
// 406 PRECONDITION-FAILED this config exists to prevent (`architecture-principles.md` §3).
|
|
42
|
+
const queueConfig = require('../config/queueConfig');
|
|
43
|
+
// The dead-letter policy of consume(): attempt counting, classification and the
|
|
44
|
+
// header that carries the count. A sibling module of this file, loaded with the
|
|
45
|
+
// package. @see ../config/deliveryPolicy.js
|
|
46
|
+
const deliveryPolicy = require('../config/deliveryPolicy');
|
|
47
|
+
// "Did you mean this one" for a key nothing declares — one implementation, shared
|
|
48
|
+
// with the config schema's own refusal. @see ../utils/nearestKey.js
|
|
49
|
+
const { nameUnknownKeys } = require('../utils/nearestKey');
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* How long a queue lives, said by the caller — the two keys `assertQueue()` gained
|
|
53
|
+
* in d.396c.
|
|
54
|
+
*
|
|
55
|
+
* `exclusive`: the broker deletes the queue when the connection that declared it
|
|
56
|
+
* goes away, and no other connection may use it in the meantime.
|
|
57
|
+
* `autoDelete`: the broker deletes it when its last consumer does.
|
|
58
|
+
*
|
|
59
|
+
* They are the ONLY rail on which a caller can say "this queue is temporary", and
|
|
60
|
+
* they are meaningful solely for a name the central `queueConfig` does not
|
|
61
|
+
* classify — an infrastructure or business queue outlives its declarer by design,
|
|
62
|
+
* so asking for either on one of those names is refused, not overruled in silence.
|
|
63
|
+
*/
|
|
64
|
+
const LIFETIME_QUEUE_OPTION_KEYS = Object.freeze(['exclusive', 'autoDelete']);
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Every option `assertQueue()` reads. A key outside this set is refused by name:
|
|
68
|
+
* an option nobody reads declares nothing, and the caller acts on a guarantee
|
|
69
|
+
* nobody gave (`architecture-principles.md` §4, §8). Same contract, and the same
|
|
70
|
+
* refusal shape, as the config schema's `additionalProperties: false` since d.291b.
|
|
71
|
+
*/
|
|
72
|
+
const ASSERT_QUEUE_OPTION_KEYS = Object.freeze([
|
|
73
|
+
'durable',
|
|
74
|
+
'arguments',
|
|
75
|
+
...LIFETIME_QUEUE_OPTION_KEYS,
|
|
76
|
+
]);
|
|
23
77
|
|
|
24
78
|
/** AMQP basic.publish fields forwarded from publish() options (amqplib Options.Publish). */
|
|
25
79
|
const AMQP_MESSAGE_PROPERTY_KEYS = [
|
|
@@ -36,6 +90,62 @@ const AMQP_MESSAGE_PROPERTY_KEYS = [
|
|
|
36
90
|
'appId',
|
|
37
91
|
];
|
|
38
92
|
|
|
93
|
+
/**
|
|
94
|
+
* The exchange types AMQP 0-9-1 defines. `assertExchange()` refuses anything
|
|
95
|
+
* else at the call site rather than letting the broker close the channel with a
|
|
96
|
+
* 503 the caller cannot read (`architecture-principles.md` §4, §5).
|
|
97
|
+
*/
|
|
98
|
+
const EXCHANGE_TYPES = ['direct', 'topic', 'fanout', 'headers'];
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The AMQP reply codes with which a broker REFUSES a connection or a channel —
|
|
102
|
+
* an answer, not a silence. `_isBrokerRefusal()` reads them; the note there
|
|
103
|
+
* carries the measurement and the reason the codes are matched in the message.
|
|
104
|
+
*
|
|
105
|
+
* 403 ACCESS-REFUSED the account may not log in / may not use the resource
|
|
106
|
+
* 530 NOT-ALLOWED the account may not use this vhost
|
|
107
|
+
* 406 PRECONDITION-FAILED the topology the broker holds is not the one asked for
|
|
108
|
+
*/
|
|
109
|
+
const REFUSAL_REPLY_CODES = [403, 530, 406];
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* The workflow id an AMQP message carries, or `null` when it carries none.
|
|
113
|
+
*
|
|
114
|
+
* ONE spelling, and no substitute for an absent value — the two halves of the
|
|
115
|
+
* expression this replaced (`headers.workflowId || headers.workflow_id ||
|
|
116
|
+
* 'unknown'`, three copies of it, d.416):
|
|
117
|
+
*
|
|
118
|
+
* - `workflow_id` is the only spelling the envelope contract admits, and a
|
|
119
|
+
* reader that accepts the camelCase alias is forbidden as such, not merely a
|
|
120
|
+
* producer that writes it (`docs/biz/70-contracts/workflow-message.md` §2.2,
|
|
121
|
+
* owner decision `workflow-message-aliases` 001). The alias is still written
|
|
122
|
+
* today by the gateway's webhook submit, whose removal is its owner's
|
|
123
|
+
* (`docs/standards/JSON_NAMING_CONVENTION.md` § alias read, `infra/TODO.md`);
|
|
124
|
+
* a reader that keeps honouring it is what makes the pair impossible to
|
|
125
|
+
* retire, because nothing then shows which producers still write it.
|
|
126
|
+
* - a missing id is reported as missing. `'unknown'` is a value, and a log line
|
|
127
|
+
* carrying it claims a workflow by that name — the same reason
|
|
128
|
+
* `BaseClient._emitDeadLetterEvent()` resolves an absent id to `null` and
|
|
129
|
+
* reports the absence rather than inventing a marker
|
|
130
|
+
* (`docs/governance/confirmations/dlq-purge.md` 002).
|
|
131
|
+
*
|
|
132
|
+
* This is a LOGGING read, and that is the whole of its job: the transport does
|
|
133
|
+
* not validate the envelope. A message whose envelope carries no `workflow_id`
|
|
134
|
+
* is refused where the envelope is read — the wrapper's `InvalidEnvelopeError`
|
|
135
|
+
* (d.383) — and a refusal here would be a second rail for one concern
|
|
136
|
+
* (`change-discipline.md` § One rail per concern).
|
|
137
|
+
*
|
|
138
|
+
* @param {Object|undefined} headers - AMQP headers as amqplib hands them over.
|
|
139
|
+
* @returns {string|null} the id, or `null` when the headers carry no non-empty one.
|
|
140
|
+
*/
|
|
141
|
+
function workflowIdOf(headers) {
|
|
142
|
+
if (!headers) {
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
const value = headers.workflow_id;
|
|
146
|
+
return typeof value === 'string' && value !== '' ? value : null;
|
|
147
|
+
}
|
|
148
|
+
|
|
39
149
|
/**
|
|
40
150
|
* Merge persistent + headers with whitelisted AMQP message properties for sendToQueue/publish.
|
|
41
151
|
* @param {Object} options
|
|
@@ -59,8 +169,8 @@ function buildAmqpMessageProperties(options, persistent, headers) {
|
|
|
59
169
|
class RabbitMQClient extends EventEmitter {
|
|
60
170
|
/**
|
|
61
171
|
* @param {Object} config
|
|
62
|
-
* @param {string} config.host - AMQP
|
|
63
|
-
*
|
|
172
|
+
* @param {string} config.host - AMQP URL (e.g., 'amqp://127.0.0.1:5672'). The ONE
|
|
173
|
+
* name and the ONE shape of the connection target; refused at construction otherwise.
|
|
64
174
|
* @param {string} [config.exchange] - Default exchange (default: '')
|
|
65
175
|
* @param {boolean} [config.durable] - Declare queues/exchanges as durable (default: true)
|
|
66
176
|
* @param {number} [config.prefetch] - Default prefetch count for consumers (default: 1)
|
|
@@ -68,17 +178,103 @@ class RabbitMQClient extends EventEmitter {
|
|
|
68
178
|
*/
|
|
69
179
|
constructor(config) {
|
|
70
180
|
super();
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
181
|
+
// The module defaults, then what the caller wrote — through the ONE owner of
|
|
182
|
+
// both the values (`../config/defaultConfig.js`) and the composition
|
|
183
|
+
// (`../config/composeConfig.js`). `BaseClient` composes the same two sources
|
|
184
|
+
// before it validates them, so a client built through it is composed of
|
|
185
|
+
// exactly this; the composition is repeated here because this class is
|
|
186
|
+
// exported (`../index.js`) and can be built on its own, and composing
|
|
187
|
+
// idempotently from the same owner is not a second rail — a literal of its
|
|
188
|
+
// own would be (`change-discipline.md`).
|
|
189
|
+
//
|
|
190
|
+
// Four of these keys used to be written out again right here (`exchange`,
|
|
191
|
+
// `durable`, `prefetch`, `noAck`), and twenty-two more lived below as
|
|
192
|
+
// `this._config.<key> || <literal>` — see the file header of defaultConfig.
|
|
193
|
+
this._config = composeConfig(defaultConfig, config);
|
|
194
|
+
|
|
195
|
+
// ONE channel: the logger the owner injected, validated here and used by
|
|
196
|
+
// this transport and by everything it builds (publish layer, recovery
|
|
197
|
+
// worker, publish monitor, buffers). Until 2026-09-07 this constructor
|
|
198
|
+
// built its own structured logger from the platform logger package,
|
|
199
|
+
// AND handed the global console to its three collaborators as their
|
|
200
|
+
// logger, AND wrote to that global directly 122 times — four channels
|
|
201
|
+
// for one concern, none of them the `config.logger` the service configures.
|
|
202
|
+
// Owner confirmation
|
|
203
|
+
// `docs/governance/confirmations/connector-logger-contract.md` 001–004.
|
|
204
|
+
this._logger = assertLogger(
|
|
205
|
+
'RabbitMQClient',
|
|
206
|
+
this._config.logger,
|
|
207
|
+
'the transport reports its connection lifecycle, channel recovery and every consumer it registers',
|
|
208
|
+
'pass config.logger (BaseClient forwards the one your service built)'
|
|
79
209
|
);
|
|
80
210
|
|
|
211
|
+
// THE connection target: one name, one shape. amqplib also accepts an options
|
|
212
|
+
// object (`{ hostname, port, username, … }`) and this constructor's `connect()`
|
|
213
|
+
// branched on `typeof rawTarget === 'string'` until 2026-09-14 — a branch no
|
|
214
|
+
// input could reach. The schema declares `host` as a string (`config/configSchema.js`),
|
|
215
|
+
// the platform supplies it as one URL in `RABBITMQ_URL` (`../config.js`), and
|
|
216
|
+
// nothing in the workspace passes an object (measured 2026-09-14: zero `host: {`
|
|
217
|
+
// in `shared/`, `infra/`, `api_biz/`). The second target name, `url`, is refused
|
|
218
|
+
// by `BaseClient` for the same reason — the value under it was discarded in
|
|
219
|
+
// silence. What is left is checked HERE too, because this class is exported
|
|
220
|
+
// (`src/index.js`) and can be built without `BaseClient`'s validation in front
|
|
221
|
+
// of it (`architecture-principles.md` §4).
|
|
222
|
+
//
|
|
223
|
+
// The REJECTED VALUE IS NOT PRINTED, only its type: a connection target
|
|
224
|
+
// carries the broker password (URL userinfo, or the `password` key of the
|
|
225
|
+
// options object this refuses), and an exception message is logged like any
|
|
226
|
+
// other line. The first version of this check did print it, and the suite
|
|
227
|
+
// that watches this client's logs for credentials caught it
|
|
228
|
+
// (`tests/unit/connect-credentials-not-logged.test.js`).
|
|
229
|
+
if (typeof this._config.host !== 'string' || this._config.host.length === 0) {
|
|
230
|
+
const got = typeof this._config.host === 'string'
|
|
231
|
+
? 'an empty string'
|
|
232
|
+
: `a value of type ${this._config.host === null ? 'null' : typeof this._config.host}`;
|
|
233
|
+
throw new Error(
|
|
234
|
+
`[RabbitMQClient] Invalid config value for "host" - Expected a non-empty AMQP URL string, got ${got}. `
|
|
235
|
+
+ 'Fix: pass config.host (BaseClient resolves it from the explicit value, else from RABBITMQ_URL); '
|
|
236
|
+
+ 'an options object and the name "url" are not accepted.'
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ONE declaration of the heartbeat. amqplib 0.10 reads it from the URL QUERY
|
|
241
|
+
// and from nowhere else (`lib/connect.js`: `intOrDefault(query.heartbeat, 0)`),
|
|
242
|
+
// so this class composes it into the target below — and a target that already
|
|
243
|
+
// carries one is REFUSED rather than overwritten: two declarations of one
|
|
244
|
+
// fact is the thing this library refuses everywhere else
|
|
245
|
+
// (`architecture-principles.md` §3, §8). The value is not printed; the target
|
|
246
|
+
// carries the broker password.
|
|
247
|
+
if (/[?&]heartbeat=/.test(this._config.host)) {
|
|
248
|
+
throw new Error(
|
|
249
|
+
'[RabbitMQClient] The connection target declares a heartbeat - Expected: the heartbeat to come '
|
|
250
|
+
+ 'from configuration (explicit value, else RABBITMQ_HEARTBEAT, else the module default), '
|
|
251
|
+
+ 'which this client composes into the connection URL. '
|
|
252
|
+
+ 'Fix: remove "heartbeat" from the query string of the AMQP URL and set config.heartbeat '
|
|
253
|
+
+ '(or RABBITMQ_HEARTBEAT) instead.'
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Resolved through the SAME three-leg rail as the delivery budget — explicit
|
|
258
|
+
// → RABBITMQ_HEARTBEAT → `../defaults.js heartbeatSeconds` — so the number
|
|
259
|
+
// has one owner and no literal stands in for it. Until d.299 the two connect
|
|
260
|
+
// paths each wrote `this._config.heartbeat ?? 30`, a copy of that default,
|
|
261
|
+
// and then handed the result to amqplib in `socketOptions`, where it is not
|
|
262
|
+
// read at all: every client on this platform ran with heartbeats DISABLED
|
|
263
|
+
// while the schema said the opposite.
|
|
264
|
+
this._heartbeatSeconds = runtimeCfg.get('heartbeat', this._config.heartbeat);
|
|
265
|
+
|
|
81
266
|
this._connection = null;
|
|
267
|
+
// THE source of truth about connection liveness. amqplib's `Connection`
|
|
268
|
+
// carries no `closed` property — measured on the live broker (0.10.9,
|
|
269
|
+
// 2026-09-12): `"closed" in conn === false` before, during and after death.
|
|
270
|
+
// So `!this._connection.closed` negated `undefined` and answered "alive" for
|
|
271
|
+
// a corpse. Liveness is recorded here instead, by the handlers that learn
|
|
272
|
+
// about it first-hand, and the connection close handler drops the reference
|
|
273
|
+
// in the same breath. The three `_ensure*Channel()` methods read this, and
|
|
274
|
+
// nothing else; `isConnected()` reads it plus `_reconnecting`, because a
|
|
275
|
+
// socket whose channels are still being recreated is not yet usable (d.340).
|
|
276
|
+
// Channels do the same through `channel._alive` (see `_isChannelAlive()`).
|
|
277
|
+
this._connectionAlive = false;
|
|
82
278
|
this._channel = null; // ConfirmChannel for publish operations
|
|
83
279
|
this._queueChannel = null; // Regular channel for queue operations (assertQueue, checkQueue)
|
|
84
280
|
this._consumerChannel = null; // Dedicated channel for consume operations
|
|
@@ -88,25 +284,89 @@ class RabbitMQClient extends EventEmitter {
|
|
|
88
284
|
|
|
89
285
|
// Connection-level recovery
|
|
90
286
|
this._reconnecting = false;
|
|
91
|
-
this
|
|
287
|
+
// Did the CALLER end this client? A connection can die two ways and they are
|
|
288
|
+
// not the same event: the broker or the network dropped it (a fault — that
|
|
289
|
+
// is what recovery is for), or `disconnect()` was called (an intention —
|
|
290
|
+
// there is nothing to recover, the caller wants this client gone).
|
|
291
|
+
// The flag is therefore TERMINAL: set by `disconnect()`, cleared only by an
|
|
292
|
+
// explicit `connect()`. Until d.283 the same member was called
|
|
293
|
+
// `_disconnecting` and lasted only for the duration of the teardown, so a
|
|
294
|
+
// recovery round already inside `amqp.connect()` — or any `close` frame
|
|
295
|
+
// arriving after `disconnect()` had returned — read `false` and revived a
|
|
296
|
+
// client its owner had already disposed of. Measured: three live sockets and
|
|
297
|
+
// three recreated channels after `disconnect()` resolved, which is why a
|
|
298
|
+
// test worker that closes its client never exits.
|
|
299
|
+
this._closedByCaller = false;
|
|
92
300
|
this._reconnectAttempts = 0;
|
|
93
301
|
this._activeTimers = new Set();
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
this.
|
|
302
|
+
// Recovery budget: explicit config → RABBITMQ_MAX_RECONNECT_ATTEMPTS → module
|
|
303
|
+
// default (see ../config.js and ../defaults.js). Not an inline `|| 10`: that
|
|
304
|
+
// hid the value from every operator and silently swallowed a configured 0.
|
|
305
|
+
this._maxReconnectAttempts = runtimeCfg.get(
|
|
306
|
+
'maxReconnectAttempts',
|
|
307
|
+
this._config.maxReconnectAttempts
|
|
308
|
+
);
|
|
309
|
+
// How many whole recovery CYCLES this client may spend before the connection
|
|
310
|
+
// is declared permanently lost. Same three legs as the attempt budget above
|
|
311
|
+
// (explicit config → RABBITMQ_MAX_RECONNECT_CYCLES → module default), for the
|
|
312
|
+
// same reason: one owner, one documented override path.
|
|
313
|
+
this._maxReconnectCycles = runtimeCfg.get(
|
|
314
|
+
'maxReconnectCycles',
|
|
315
|
+
this._config.maxReconnectCycles
|
|
316
|
+
);
|
|
317
|
+
// Cycles spent so far. Counted up when a cycle starts, back to 0 the moment
|
|
318
|
+
// one succeeds — a client that recovered owes nothing for the outage it
|
|
319
|
+
// survived.
|
|
320
|
+
this._reconnectCycles = 0;
|
|
321
|
+
// The reason the last cycle ended without a connection, kept for the refusal
|
|
322
|
+
// the next use gets and for the fatal error when the cap is reached.
|
|
323
|
+
this._lastConnectionError = null;
|
|
324
|
+
// Set once the connection is permanently lost — the broker REFUSED it, or
|
|
325
|
+
// the cycle cap is spent. From that point this client will not try again and
|
|
326
|
+
// it must not report itself as connected.
|
|
327
|
+
this._connectionFatal = false;
|
|
328
|
+
// Injected by the owner of this client (service/wrapper). The LIBRARY never
|
|
329
|
+
// ends the process — that decision belongs to whoever owns the lifecycle.
|
|
330
|
+
this._onFatal = this._config.onFatal;
|
|
331
|
+
if (this._onFatal !== null && typeof this._onFatal !== 'function') {
|
|
332
|
+
throw new Error(
|
|
333
|
+
'[RabbitMQClient] Invalid config value for "onFatal" - Expected a function, got ' +
|
|
334
|
+
`"${typeof this._config.onFatal}". Fix: pass a callback that stops the process, or omit the key.`
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
this._reconnectBaseDelay = this._config.reconnectBaseDelay;
|
|
338
|
+
this._reconnectMaxDelay = this._config.reconnectMaxDelay;
|
|
339
|
+
this._reconnectEnabled = this._config.reconnectEnabled;
|
|
98
340
|
this._reconnectTimer = null;
|
|
99
|
-
//
|
|
100
|
-
|
|
101
|
-
|
|
341
|
+
// The one default that is DERIVED rather than declared: how long a caller
|
|
342
|
+
// waiting for recovery waits, computed from the budget and the ceiling of the
|
|
343
|
+
// backoff, never shorter than a minute. A constant copy of it in
|
|
344
|
+
// `defaultConfig` would be free to disagree with the two values it comes from,
|
|
345
|
+
// so the derivation stays here — and `=== undefined` asks "did the caller
|
|
346
|
+
// write one?", which `||` could not: `reconnectWaitTimeout: 0` is a legal
|
|
347
|
+
// value of its own (`minimum: 0` in the schema).
|
|
348
|
+
this._reconnectWaitTimeout = this._config.reconnectWaitTimeout === undefined
|
|
349
|
+
? Math.max(60000, (this._maxReconnectAttempts * this._reconnectMaxDelay))
|
|
350
|
+
: this._config.reconnectWaitTimeout;
|
|
102
351
|
|
|
103
|
-
// Publisher retry configuration
|
|
104
|
-
this._publishRetryEnabled = this._config.publishRetryEnabled
|
|
105
|
-
this._publishMaxRetries = this._config.publishMaxRetries
|
|
106
|
-
this._publishRetryBaseDelay = this._config.publishRetryBaseDelay
|
|
107
|
-
this._publishRetryMaxDelay = this._config.publishRetryMaxDelay
|
|
108
|
-
this._publishRetryBackoffMultiplier = this._config.publishRetryBackoffMultiplier
|
|
109
|
-
|
|
352
|
+
// Publisher retry configuration (values: ../config/defaultConfig.js)
|
|
353
|
+
this._publishRetryEnabled = this._config.publishRetryEnabled;
|
|
354
|
+
this._publishMaxRetries = this._config.publishMaxRetries;
|
|
355
|
+
this._publishRetryBaseDelay = this._config.publishRetryBaseDelay;
|
|
356
|
+
this._publishRetryMaxDelay = this._config.publishRetryMaxDelay;
|
|
357
|
+
this._publishRetryBackoffMultiplier = this._config.publishRetryBackoffMultiplier;
|
|
358
|
+
// ONE number, two consumers, and therefore a name that belongs to neither of
|
|
359
|
+
// them alone: how long this client waits for an ANSWER from the broker. The
|
|
360
|
+
// publish confirm waits for it (`_publishOnce`), and so does the whole
|
|
361
|
+
// teardown of `disconnect()` (d.283 chose deliberately not to invent a second
|
|
362
|
+
// number for that; the name it inherited then said "publish confirmation",
|
|
363
|
+
// which was true of one consumer and false of the other).
|
|
364
|
+
this._brokerAnswerTimeout = this._config.brokerAnswerTimeout;
|
|
365
|
+
// How long the connect handshake may take before the attempt is abandoned.
|
|
366
|
+
this._connectTimeout = this._config.connectTimeout;
|
|
367
|
+
// How long after `sendToQueue()` the attempt looks to see whether the missing
|
|
368
|
+
// confirm is a dead channel rather than a slow broker.
|
|
369
|
+
this._publishConfirmWatchdogDelay = this._config.publishConfirmWatchdogDelay;
|
|
110
370
|
|
|
111
371
|
// Channel close hooks - called when channel closes with detailed information
|
|
112
372
|
this._channelCloseHooks = []; // Array of { type: 'publisher'|'queue'|'consumer', callback: (details) => void }
|
|
@@ -117,69 +377,54 @@ class RabbitMQClient extends EventEmitter {
|
|
|
117
377
|
queue: [],
|
|
118
378
|
consumer: []
|
|
119
379
|
};
|
|
120
|
-
this._thrashingThreshold = this._config.thrashingThreshold
|
|
121
|
-
this._thrashingWindowMs = this._config.thrashingWindowMs
|
|
122
|
-
this._thrashingAlertCallback = this._config.thrashingAlertCallback
|
|
380
|
+
this._thrashingThreshold = this._config.thrashingThreshold;
|
|
381
|
+
this._thrashingWindowMs = this._config.thrashingWindowMs;
|
|
382
|
+
this._thrashingAlertCallback = this._config.thrashingAlertCallback; // (type, count, windowMs) => void
|
|
123
383
|
this._thrashingAlertsSent = new Set(); // Track which alerts we've already sent to avoid spam
|
|
124
384
|
|
|
125
385
|
// Prefetch monitoring - track utilization per queue
|
|
126
386
|
this._prefetchTracking = new Map(); // queue -> { prefetchCount: number, inFlight: number, lastCheck: number }
|
|
127
|
-
this._prefetchUtilizationThreshold = this._config.prefetchUtilizationThreshold
|
|
128
|
-
this._prefetchCheckInterval = this._config.prefetchCheckInterval
|
|
129
|
-
this._prefetchAlertCallback = this._config.prefetchAlertCallback
|
|
387
|
+
this._prefetchUtilizationThreshold = this._config.prefetchUtilizationThreshold;
|
|
388
|
+
this._prefetchCheckInterval = this._config.prefetchCheckInterval;
|
|
389
|
+
this._prefetchAlertCallback = this._config.prefetchAlertCallback; // (queue, utilization, inFlight, prefetchCount) => void
|
|
130
390
|
this._prefetchCheckTimer = null;
|
|
131
391
|
|
|
132
392
|
// Health monitoring
|
|
133
393
|
this._healthCheckInterval = null;
|
|
134
|
-
this._healthCheckIntervalMs = this._config.healthCheckInterval
|
|
135
|
-
this._healthCheckEnabled = this._config.healthCheckEnabled
|
|
394
|
+
this._healthCheckIntervalMs = this._config.healthCheckInterval;
|
|
395
|
+
this._healthCheckEnabled = this._config.healthCheckEnabled;
|
|
136
396
|
|
|
137
397
|
// Health reporting callbacks
|
|
138
|
-
this._healthReportCallback = this._config.healthReportCallback
|
|
139
|
-
this._healthCriticalCallback = this._config.healthCriticalCallback
|
|
140
|
-
this._criticalHealthShutdown = this._config.criticalHealthShutdown
|
|
141
|
-
this._criticalHealthShutdownDelay = this._config.criticalHealthShutdownDelay
|
|
398
|
+
this._healthReportCallback = this._config.healthReportCallback; // (health) => Promise<void>
|
|
399
|
+
this._healthCriticalCallback = this._config.healthCriticalCallback; // (health) => Promise<void>
|
|
400
|
+
this._criticalHealthShutdown = this._config.criticalHealthShutdown;
|
|
401
|
+
this._criticalHealthShutdownDelay = this._config.criticalHealthShutdownDelay;
|
|
142
402
|
this._criticalHealthStartTime = null; // Track when critical health started
|
|
143
403
|
|
|
144
|
-
// Create structured logger for infrastructure logging (module-owned)
|
|
145
|
-
// ONE implementation of the infrastructure logger, in
|
|
146
|
-
// @onlineapps/infra-logger. A private copy lived here until 2026-08-29,
|
|
147
|
-
// justified as avoiding a dependency cycle — a cycle that is not there:
|
|
148
|
-
// the comment named infrastructure-tools, but the logger is infra-logger,
|
|
149
|
-
// which declares no @onlineapps dependency at all. The copy meanwhile
|
|
150
|
-
// missed the identity fail-fast that 2.0.0 added (change-discipline.md:
|
|
151
|
-
// one fact, one owner).
|
|
152
|
-
this._log = createLogger('mq-client-core', 'transport');
|
|
153
|
-
|
|
154
404
|
// Publish layer (retry + buffer)
|
|
155
405
|
this._publishLayer = new PublishLayer({
|
|
156
406
|
client: this,
|
|
157
|
-
logger:
|
|
407
|
+
logger: this._logger,
|
|
158
408
|
bufferConfig: {
|
|
159
409
|
inMemory: {
|
|
160
|
-
maxSize: this._config.publishBufferMaxSize
|
|
161
|
-
ttlMs: this._config.publishBufferTtlMs
|
|
162
|
-
},
|
|
163
|
-
persistent: {
|
|
164
|
-
enabled: !!this._config.persistentBufferEnabled,
|
|
165
|
-
redisClient: this._config.persistentRedisClient || null,
|
|
410
|
+
maxSize: this._config.publishBufferMaxSize,
|
|
411
|
+
ttlMs: this._config.publishBufferTtlMs,
|
|
166
412
|
},
|
|
167
413
|
},
|
|
168
414
|
});
|
|
169
415
|
|
|
170
|
-
// Recovery worker
|
|
416
|
+
// Recovery worker — connection recovery, and the report of a queue that does
|
|
417
|
+
// not exist. It creates none: queue creation left it with the two callbacks
|
|
418
|
+
// that fed it (d.419).
|
|
171
419
|
this._recoveryWorker = new RecoveryWorker({
|
|
172
420
|
client: this,
|
|
173
|
-
scope: this._config.recoveryScope
|
|
174
|
-
|
|
175
|
-
queueCreationCallback: this._config.queueCreationCallback || null, // Delegates to QueueManager if provided
|
|
176
|
-
logger: console,
|
|
421
|
+
scope: this._config.recoveryScope,
|
|
422
|
+
logger: this._logger,
|
|
177
423
|
});
|
|
178
424
|
|
|
179
|
-
// Publish monitor (metriky)
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
});
|
|
425
|
+
// Publish monitor (metriky) — counts only, reports through return values,
|
|
426
|
+
// takes no logger (see PublishMonitor's header).
|
|
427
|
+
this._publishMonitor = new PublishMonitor();
|
|
183
428
|
|
|
184
429
|
// Event listeners pro monitoring
|
|
185
430
|
this._setupPublishMonitoring();
|
|
@@ -244,22 +489,40 @@ class RabbitMQClient extends EventEmitter {
|
|
|
244
489
|
return {
|
|
245
490
|
publisher: {
|
|
246
491
|
exists: !!this._channel,
|
|
247
|
-
closed: this.
|
|
248
|
-
ready: this.
|
|
492
|
+
closed: !this._isChannelAlive(this._channel),
|
|
493
|
+
ready: this._isChannelAlive(this._channel)
|
|
249
494
|
},
|
|
250
495
|
queue: {
|
|
251
496
|
exists: !!this._queueChannel,
|
|
252
|
-
closed: this.
|
|
253
|
-
ready: this.
|
|
497
|
+
closed: !this._isChannelAlive(this._queueChannel),
|
|
498
|
+
ready: this._isChannelAlive(this._queueChannel)
|
|
254
499
|
},
|
|
255
500
|
consumer: {
|
|
256
501
|
exists: !!this._consumerChannel,
|
|
257
|
-
closed: this.
|
|
258
|
-
ready: this.
|
|
502
|
+
closed: !this._isChannelAlive(this._consumerChannel),
|
|
503
|
+
ready: this._isChannelAlive(this._consumerChannel)
|
|
259
504
|
}
|
|
260
505
|
};
|
|
261
506
|
}
|
|
262
507
|
|
|
508
|
+
/**
|
|
509
|
+
* THE source of truth about one channel's liveness.
|
|
510
|
+
*
|
|
511
|
+
* `_attach*ChannelHandlers()` marks the channel alive when it attaches, and
|
|
512
|
+
* its `error` / `close` handlers mark it dead — the two events that know. The
|
|
513
|
+
* reading this replaced, `channel && !channel.closed`, asked amqplib for a
|
|
514
|
+
* property it does not define (measured, 0.10.9), so it answered "alive" for
|
|
515
|
+
* every channel it was handed, dead or not, and returned the channel object
|
|
516
|
+
* rather than a boolean.
|
|
517
|
+
*
|
|
518
|
+
* @param {Object|null} channel
|
|
519
|
+
* @returns {boolean}
|
|
520
|
+
* @private
|
|
521
|
+
*/
|
|
522
|
+
_isChannelAlive(channel) {
|
|
523
|
+
return !!channel && channel._alive === true;
|
|
524
|
+
}
|
|
525
|
+
|
|
263
526
|
/**
|
|
264
527
|
* Get current consumer state
|
|
265
528
|
* @returns {Object} Consumer state information
|
|
@@ -278,17 +541,334 @@ class RabbitMQClient extends EventEmitter {
|
|
|
278
541
|
getBufferState() {
|
|
279
542
|
return {
|
|
280
543
|
size: this._publishLayer._buffer.size(),
|
|
281
|
-
|
|
282
|
-
|
|
544
|
+
// No `persistent` count. There is one buffer, so a second number could
|
|
545
|
+
// only ever be 0 — a field that reports a mechanism that does not exist
|
|
546
|
+
// (d.343, `automation-gates.md` §5).
|
|
547
|
+
inMemory: this._publishLayer._buffer._inMemory?.size() || 0
|
|
283
548
|
};
|
|
284
549
|
}
|
|
285
550
|
|
|
286
551
|
/**
|
|
287
|
-
*
|
|
288
|
-
*
|
|
552
|
+
* Can this client be used right now?
|
|
553
|
+
*
|
|
554
|
+
* Three things have to be true, and they are three different facts:
|
|
555
|
+
* - the recovery budget is not spent (`_connectionFatal`);
|
|
556
|
+
* - the socket is up (`_connectionAlive`, THE liveness source — d.260);
|
|
557
|
+
* - the recovery is FINISHED (`_reconnecting`), because a re-established
|
|
558
|
+
* socket with no channels on it cannot publish or consume.
|
|
559
|
+
*
|
|
560
|
+
* The third was missing until d.340. `_reconnectWithBackoff()` marks the
|
|
561
|
+
* connection alive the instant `amqp.connect()` resolves and recreates the
|
|
562
|
+
* publisher, queue and consumer channels after that, so for the width of that
|
|
563
|
+
* restore `isConnected()` answered TRUE while `getChannelState()` reported all
|
|
564
|
+
* three channels missing — the same split answer d.260 removed from the
|
|
565
|
+
* outage, reappearing at the other end of it. Measured as a race in
|
|
566
|
+
* `tests/integration/connection-liveness-single-source.integration.test.js`
|
|
567
|
+
* ("when the broker is reachable again"): the poll on `isConnected()` returned
|
|
568
|
+
* before the channels were back and the channel-state assertion that followed
|
|
569
|
+
* it failed.
|
|
570
|
+
*
|
|
571
|
+
* `_reconnecting` is the recovery's own flag: set when a round starts, cleared
|
|
572
|
+
* when the channels are back (success), when the budget is spent
|
|
573
|
+
* (`_failFatally`) or when the caller ended the client mid-round. So "recovery
|
|
574
|
+
* in progress" and "not usable" are the same state, read from the one member
|
|
575
|
+
* that already records it — no second liveness flag.
|
|
576
|
+
*
|
|
577
|
+
* @returns {boolean} True if the connection AND its channels are in place
|
|
289
578
|
*/
|
|
290
579
|
isConnected() {
|
|
291
|
-
|
|
580
|
+
if (this._connectionFatal) return false;
|
|
581
|
+
if (this._reconnecting) return false;
|
|
582
|
+
return this._connectionAlive === true;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* THE connection target, composed: the configured host plus the resolved
|
|
587
|
+
* heartbeat, in the one place amqplib reads it — the URL query.
|
|
588
|
+
*
|
|
589
|
+
* Both connect paths (the first handshake and every reconnect) call this, so
|
|
590
|
+
* a revived connection is negotiated with exactly the parameters the first one
|
|
591
|
+
* was. Until d.299 they each built their own options object and put the
|
|
592
|
+
* heartbeat where amqplib does not look.
|
|
593
|
+
*
|
|
594
|
+
* @returns {string} the AMQP URL, credentials untouched
|
|
595
|
+
* @private
|
|
596
|
+
*/
|
|
597
|
+
_connectionTarget() {
|
|
598
|
+
const target = new URL(this._config.host);
|
|
599
|
+
target.searchParams.set('heartbeat', String(this._heartbeatSeconds));
|
|
600
|
+
return target.toString();
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* THE client properties of a connection — what the broker shows an operator in
|
|
605
|
+
* its connection list.
|
|
606
|
+
*
|
|
607
|
+
* @returns {Object} amqplib socket options
|
|
608
|
+
* @private
|
|
609
|
+
*/
|
|
610
|
+
_connectionSocketOptions() {
|
|
611
|
+
const connectionName =
|
|
612
|
+
this._config.connectionName ||
|
|
613
|
+
this._config.clientName ||
|
|
614
|
+
`oa-client:${runtimeCfg.get('serviceName')}:${process.pid}`;
|
|
615
|
+
return { clientProperties: { connection_name: connectionName } };
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* The two things this client must know about ANY connection it holds: that it
|
|
620
|
+
* errored, and that it died.
|
|
621
|
+
*
|
|
622
|
+
* One rail since d.299. Both were registered twice, once in `connect()` and
|
|
623
|
+
* once in the recovery path, as two copies of the same two handlers — and a
|
|
624
|
+
* handler maintained twice diverges (`change-discipline.md` § One rail per
|
|
625
|
+
* concern). amqplib always follows a connection `error` with `close`, so the
|
|
626
|
+
* error handler is not the rail that recovers: it is one of the two moments
|
|
627
|
+
* that KNOW the connection is gone, and liveness is recorded at both.
|
|
628
|
+
*
|
|
629
|
+
* @param {Object} connection - the amqplib connection
|
|
630
|
+
* @private
|
|
631
|
+
*/
|
|
632
|
+
_attachConnectionHandlers(connection) {
|
|
633
|
+
connection.on('error', (err) => {
|
|
634
|
+
this._connectionAlive = false;
|
|
635
|
+
this._logger.error('[RabbitMQClient] Connection error:', err.message);
|
|
636
|
+
this.emit('error', err);
|
|
637
|
+
});
|
|
638
|
+
connection.on('close', () => this._onConnectionDied());
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* True once the connection is permanently lost — the broker refused it, or the
|
|
643
|
+
* whole cycle cap is spent. From that point the client makes no further
|
|
644
|
+
* attempts. A single spent ATTEMPT budget is not this state; see
|
|
645
|
+
* `_standDown()`.
|
|
646
|
+
* @returns {boolean}
|
|
647
|
+
*/
|
|
648
|
+
isConnectionFatal() {
|
|
649
|
+
return this._connectionFatal === true;
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
/**
|
|
653
|
+
* Did the BROKER refuse this connection, or did nobody answer?
|
|
654
|
+
*
|
|
655
|
+
* The two are not the same failure and must not get the same treatment. A
|
|
656
|
+
* refusal is an answer: the broker read the handshake and said no — wrong
|
|
657
|
+
* credentials, a vhost this account may not use, a topology it will not
|
|
658
|
+
* accept. Nothing about that answer changes by asking again, so retrying is
|
|
659
|
+
* dishonest work. Nobody answering is a condition that ends on its own, and
|
|
660
|
+
* it is the one the owner forbade turning into a permanent death
|
|
661
|
+
* (`docs/governance/confirmations/mq-client-lifecycle-contract.md` 001 point 3).
|
|
662
|
+
*
|
|
663
|
+
* WHAT IT READS, and why the message rather than a code. Measured against
|
|
664
|
+
* amqplib 0.10.9 on the live dev broker, 2026-09-14:
|
|
665
|
+
*
|
|
666
|
+
* amqp://nobody:wrong@… Error, code=undefined, message=
|
|
667
|
+
* 'Handshake terminated by server: 403 (ACCESS-REFUSED) with message
|
|
668
|
+
* "ACCESS_REFUSED - Login was refused using authentication mechanism
|
|
669
|
+
* PLAIN. For details see the broker logfile."'
|
|
670
|
+
* amqp://…@127.0.0.1:39999 Error, code='ECONNREFUSED', errno=-61,
|
|
671
|
+
* syscall='connect'
|
|
672
|
+
*
|
|
673
|
+
* amqplib renders the broker's `connection.close` frame into the message and
|
|
674
|
+
* leaves `err.code` undefined, so the reply code lives in the text. What is
|
|
675
|
+
* matched is therefore the AMQP reply code the BROKER sent — 403, 530, 406 —
|
|
676
|
+
* a protocol constant, never wording invented here. A channel-level refusal
|
|
677
|
+
* arrives the other way round, as a numeric `err.code`, and that is read too.
|
|
678
|
+
*
|
|
679
|
+
* NOT recognised, and said rather than implied (`automation-gates.md` §5): a
|
|
680
|
+
* wrong vhost reaches amqplib as `Expected ConnectionOpenOk; got
|
|
681
|
+
* <ConnectionClose channel:0>` (measured, same run) — the broker's 530 is
|
|
682
|
+
* dropped before the error is built, so this classifier cannot see it and
|
|
683
|
+
* treats it as transient. It ends in the cycle cap instead of at once.
|
|
684
|
+
*
|
|
685
|
+
* Anything unrecognised is TRANSIENT on purpose: the confirmation forbids
|
|
686
|
+
* giving up forever, and the cycle cap already bounds the other end.
|
|
687
|
+
*
|
|
688
|
+
* @param {Error|null} err
|
|
689
|
+
* @returns {boolean} true if the broker answered and refused
|
|
690
|
+
* @private
|
|
691
|
+
*/
|
|
692
|
+
_isBrokerRefusal(err) {
|
|
693
|
+
if (!err) return false;
|
|
694
|
+
if (REFUSAL_REPLY_CODES.includes(err.code)) return true;
|
|
695
|
+
const message = typeof err.message === 'string' ? err.message : '';
|
|
696
|
+
return REFUSAL_REPLY_CODES.some((code) => message.includes(`server: ${code} (`));
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/**
|
|
700
|
+
* A recovery cycle is over and the client is NOT connected, but the broker
|
|
701
|
+
* never refused it — so this is not the end.
|
|
702
|
+
*
|
|
703
|
+
* The client stands down: no timer, no socket, `isConnected()` false, and the
|
|
704
|
+
* attempt counter back to zero so the next cycle gets its full budget. Nothing
|
|
705
|
+
* polls; the next USE of this client (publish, consume, `performHealthCheck()`)
|
|
706
|
+
* starts the next cycle through `_resumeRecovery()`. That is the "lazy retry on
|
|
707
|
+
* next use" the owner's decision requires, and it is why "dead until the
|
|
708
|
+
* process restarts" — what this package did until d.339, and what its README
|
|
709
|
+
* taught — is gone.
|
|
710
|
+
*
|
|
711
|
+
* @param {Error|null} lastError - the error from the final failed attempt
|
|
712
|
+
* @returns {Error} the stand-down error, so callers can throw it
|
|
713
|
+
* @private
|
|
714
|
+
*/
|
|
715
|
+
_standDown(lastError) {
|
|
716
|
+
this._reconnecting = false;
|
|
717
|
+
this._reconnectAttempts = 0;
|
|
718
|
+
this._lastConnectionError = lastError || null;
|
|
719
|
+
|
|
720
|
+
const standby = new ConnectionError(
|
|
721
|
+
`[RabbitMQClient] Connection not re-established - recovery cycle ${this._reconnectCycles}`
|
|
722
|
+
+ `/${this._maxReconnectCycles} spent its ${this._maxReconnectAttempts} attempts and the broker `
|
|
723
|
+
+ `did not answer (last error: ${lastError ? lastError.message : 'unknown'}). `
|
|
724
|
+
+ 'Expected: the broker to become reachable again; this is an outage, not a refusal. '
|
|
725
|
+
+ 'Fix: nothing to do here — the next publish, consume or performHealthCheck() starts the next '
|
|
726
|
+
+ 'cycle; raise RABBITMQ_MAX_RECONNECT_CYCLES if more of them must be survived in-process.'
|
|
727
|
+
);
|
|
728
|
+
standby.code = 'MQ_CONNECTION_STANDBY';
|
|
729
|
+
standby.cycle = this._reconnectCycles;
|
|
730
|
+
standby.cyclesMax = this._maxReconnectCycles;
|
|
731
|
+
standby.attempts = this._maxReconnectAttempts;
|
|
732
|
+
standby.cause = lastError;
|
|
733
|
+
|
|
734
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] ${standby.message}`);
|
|
735
|
+
this.emit('connection:standby', {
|
|
736
|
+
cycle: this._reconnectCycles,
|
|
737
|
+
cyclesMax: this._maxReconnectCycles,
|
|
738
|
+
attempts: this._maxReconnectAttempts,
|
|
739
|
+
lastError: lastError ? lastError.message : null,
|
|
740
|
+
timestamp: new Date().toISOString()
|
|
741
|
+
});
|
|
742
|
+
|
|
743
|
+
return standby;
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* Somebody wants to use a client that is not connected. Start the next
|
|
748
|
+
* recovery cycle if the contract allows one, and say whether a recovery is now
|
|
749
|
+
* in flight.
|
|
750
|
+
*
|
|
751
|
+
* This is the lazy half of the retry contract, and it deliberately starts NO
|
|
752
|
+
* second loop: it re-enters `_reconnectWithBackoff()`, the one recovery rail
|
|
753
|
+
* this transport has (`change-discipline.md` § One rail per concern). What it
|
|
754
|
+
* adds is the decision to enter it again, which the automatic close handler
|
|
755
|
+
* makes only once.
|
|
756
|
+
*
|
|
757
|
+
* @returns {boolean} true if a recovery is in flight (already, or from here)
|
|
758
|
+
* @private
|
|
759
|
+
*/
|
|
760
|
+
_resumeRecovery() {
|
|
761
|
+
if (this._reconnecting) return true;
|
|
762
|
+
if (this._connectionFatal) return false;
|
|
763
|
+
if (this._closedByCaller) return false;
|
|
764
|
+
if (!this._reconnectEnabled) return false;
|
|
765
|
+
|
|
766
|
+
this._logger.info(
|
|
767
|
+
`[RabbitMQClient] [mq-client-core] Client used while disconnected - starting recovery cycle `
|
|
768
|
+
+ `${this._reconnectCycles + 1}/${this._maxReconnectCycles}`
|
|
769
|
+
);
|
|
770
|
+
this._reconnectWithBackoff().catch((err) => this._reportRecoveryFailure(err));
|
|
771
|
+
|
|
772
|
+
return this._reconnecting === true;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
/**
|
|
776
|
+
* A recovery cycle rejected. Say so — ONCE, on the channel that belongs to the
|
|
777
|
+
* thing that happened.
|
|
778
|
+
*
|
|
779
|
+
* A cycle that stood down or died fatally has already announced itself on
|
|
780
|
+
* `connection:standby` / `connection:fatal`, synchronously, at the moment it
|
|
781
|
+
* happened. Re-emitting the same error on the generic `error` channel is a
|
|
782
|
+
* second notification of one event, and the harmful kind: it arrives a turn
|
|
783
|
+
* later, which is long enough to land inside the NEXT cycle's
|
|
784
|
+
* `_waitForReconnection()` and reject it with the previous cycle's obituary
|
|
785
|
+
* (measured while writing d.339 — a publish that started a healthy cycle was
|
|
786
|
+
* rejected by the stand-down of the cycle before it).
|
|
787
|
+
*
|
|
788
|
+
* Anything else that comes out of a recovery round is unannounced, so it goes
|
|
789
|
+
* to `error`.
|
|
790
|
+
*
|
|
791
|
+
* @param {Error} err
|
|
792
|
+
* @private
|
|
793
|
+
*/
|
|
794
|
+
_reportRecoveryFailure(err) {
|
|
795
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Recovery cycle ended: ${err.message}`);
|
|
796
|
+
if (err.code === 'MQ_CONNECTION_FATAL' || err.code === 'MQ_CONNECTION_STANDBY') {
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
this.emit('error', err);
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/**
|
|
803
|
+
* Declare the connection permanently lost: stop retrying, stop reporting
|
|
804
|
+
* health, tell the owner.
|
|
805
|
+
*
|
|
806
|
+
* There are exactly TWO ways to get here, and they are different facts, so
|
|
807
|
+
* they carry different sentences (`architecture-principles.md` §5 — a message
|
|
808
|
+
* that cannot tell an operator which of the two happened sends them to the
|
|
809
|
+
* wrong place):
|
|
810
|
+
*
|
|
811
|
+
* - `broker-refused` — the broker answered and said no (403/530/406). No
|
|
812
|
+
* number of retries changes that answer, so not one attempt is spent on
|
|
813
|
+
* it; the fix is credentials, vhost or topology.
|
|
814
|
+
* - `cycles-spent` — every recovery cycle the cap allows has been spent
|
|
815
|
+
* against a broker that never answered. This is the bounded end of the
|
|
816
|
+
* lazy-retry contract, not the first spent attempt budget: that one stands
|
|
817
|
+
* the client down (`_standDown()`) and waits to be used again.
|
|
818
|
+
*
|
|
819
|
+
* The library deliberately does NOT call `process.exit()`. Ending the process
|
|
820
|
+
* is the lifecycle owner's decision (service / wrapper), and a library that
|
|
821
|
+
* takes it removes the owner's chance to drain, flush or log first. What the
|
|
822
|
+
* library owes the owner is the truth, once, on a channel that cannot be
|
|
823
|
+
* confused with an ordinary transient error: the `connection:fatal` event and
|
|
824
|
+
* the injected `onFatal` callback.
|
|
825
|
+
*
|
|
826
|
+
* @param {Error} lastError - The error from the final failed attempt.
|
|
827
|
+
* @param {'broker-refused'|'cycles-spent'} reason - Which of the two it is.
|
|
828
|
+
* @returns {Error} The fatal error, so callers can throw it.
|
|
829
|
+
* @private
|
|
830
|
+
*/
|
|
831
|
+
_failFatally(lastError, reason) {
|
|
832
|
+
this._connectionFatal = true;
|
|
833
|
+
this._reconnecting = false;
|
|
834
|
+
this._lastConnectionError = lastError || null;
|
|
835
|
+
|
|
836
|
+
const because = lastError ? lastError.message : 'unknown';
|
|
837
|
+
const fatal = new Error(
|
|
838
|
+
reason === 'broker-refused'
|
|
839
|
+
? '[RabbitMQClient] Connection permanently lost - the broker REFUSED the connection, '
|
|
840
|
+
+ `so retrying cannot change the answer (reason: ${because}). `
|
|
841
|
+
+ 'Expected: credentials, vhost and topology this broker accepts. '
|
|
842
|
+
+ 'Fix: correct RABBITMQ_URL (user, password, vhost) or the topology the broker refused, '
|
|
843
|
+
+ 'then restart the process.'
|
|
844
|
+
: `[RabbitMQClient] Connection permanently lost - all ${this._maxReconnectCycles} recovery cycles `
|
|
845
|
+
+ `of ${this._maxReconnectAttempts} attempts each failed against a broker that did not answer `
|
|
846
|
+
+ `(last error: ${because}). `
|
|
847
|
+
+ 'Expected: the owner of this client stops the process so its restart policy can boot it against a verified broker. '
|
|
848
|
+
+ 'Fix: register an onFatal handler (or a "connection:fatal" listener) that exits non-zero; '
|
|
849
|
+
+ 'raise RABBITMQ_MAX_RECONNECT_CYCLES (or RABBITMQ_MAX_RECONNECT_ATTEMPTS) only if a longer outage '
|
|
850
|
+
+ 'must be survived in-process.'
|
|
851
|
+
);
|
|
852
|
+
fatal.code = 'MQ_CONNECTION_FATAL';
|
|
853
|
+
fatal.reason = reason;
|
|
854
|
+
fatal.attempts = this._maxReconnectAttempts;
|
|
855
|
+
fatal.cycles = this._reconnectCycles;
|
|
856
|
+
fatal.cause = lastError;
|
|
857
|
+
|
|
858
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] ${fatal.message}`);
|
|
859
|
+
this.emit('connection:fatal', fatal);
|
|
860
|
+
|
|
861
|
+
if (this._onFatal) {
|
|
862
|
+
try {
|
|
863
|
+
this._onFatal(fatal);
|
|
864
|
+
} catch (err) {
|
|
865
|
+
this._logger.error(
|
|
866
|
+
`[RabbitMQClient] [mq-client-core] onFatal handler threw: ${err.message}`
|
|
867
|
+
);
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
return fatal;
|
|
292
872
|
}
|
|
293
873
|
|
|
294
874
|
/**
|
|
@@ -307,11 +887,40 @@ class RabbitMQClient extends EventEmitter {
|
|
|
307
887
|
}
|
|
308
888
|
|
|
309
889
|
/**
|
|
310
|
-
*
|
|
311
|
-
*
|
|
890
|
+
* The channel queue operations run on — `assertQueue`, `checkQueue`,
|
|
891
|
+
* `purgeQueue`, `deleteQueue`.
|
|
892
|
+
*
|
|
893
|
+
* It answers with the queue channel or with an error, never with another
|
|
894
|
+
* channel. Until d.281 it read `this._queueChannel || this._channel`, so
|
|
895
|
+
* whenever the queue channel was absent — the state every channel death
|
|
896
|
+
* leaves behind, because the close handler nulls the reference — a caller
|
|
897
|
+
* reaching queue operations through this getter ran them on the publisher's
|
|
898
|
+
* ConfirmChannel. A queue operation is EXPECTED to fail (404 for a missing
|
|
899
|
+
* queue, 406 for a divergent declaration) and an AMQP channel-level failure
|
|
900
|
+
* closes the channel, which on the ConfirmChannel takes every pending
|
|
901
|
+
* publisher confirm with it. That is the very outcome the three-channel
|
|
902
|
+
* separation exists to prevent, so the fallback contradicted both its own
|
|
903
|
+
* premise and `architecture-principles.md` §3.
|
|
904
|
+
*
|
|
905
|
+
* This getter is a state reading and cannot open anything: the asynchronous
|
|
906
|
+
* way to a live queue channel is a queue operation on this client, each of
|
|
907
|
+
* which ensures the channel first.
|
|
908
|
+
*
|
|
909
|
+
* @see /docs/architecture/rabbitmq-channel-lifecycle.md
|
|
910
|
+
* @returns {Object} the live queue channel
|
|
911
|
+
* @throws {ConnectionError} when the queue channel is absent or not alive
|
|
312
912
|
*/
|
|
313
913
|
get queueChannel() {
|
|
314
|
-
|
|
914
|
+
if (!this._isChannelAlive(this._queueChannel)) {
|
|
915
|
+
throw new ConnectionError(
|
|
916
|
+
'[RabbitMQClient] Queue channel is not available - '
|
|
917
|
+
+ 'Expected: an open queue channel, opened by connect() and recreated after a channel death. '
|
|
918
|
+
+ 'Fix: run the queue operation through the client (assertQueue/checkQueue/purgeQueue/deleteQueue) — '
|
|
919
|
+
+ 'those ensure the channel first; the publisher channel is never a substitute, because a queue '
|
|
920
|
+
+ "operation's 404/406 would close it and take the pending publisher confirms with it."
|
|
921
|
+
);
|
|
922
|
+
}
|
|
923
|
+
return this._queueChannel;
|
|
315
924
|
}
|
|
316
925
|
|
|
317
926
|
/**
|
|
@@ -334,7 +943,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
334
943
|
const history = this._channelCloseHistory[channelType];
|
|
335
944
|
|
|
336
945
|
if (!history) {
|
|
337
|
-
|
|
946
|
+
this._logger.warn(`[RabbitMQClient] Unknown channel type for thrashing tracking: ${channelType}`);
|
|
338
947
|
return;
|
|
339
948
|
}
|
|
340
949
|
|
|
@@ -365,7 +974,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
365
974
|
}
|
|
366
975
|
|
|
367
976
|
const message = `Channel thrashing detected: ${channelType} channel closed ${history.length} times in ${Math.round(this._thrashingWindowMs / 1000)}s (threshold: ${this._thrashingThreshold})`;
|
|
368
|
-
|
|
977
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] 🚨 ${message}`);
|
|
369
978
|
|
|
370
979
|
// Call alert callback if provided
|
|
371
980
|
if (this._thrashingAlertCallback) {
|
|
@@ -376,7 +985,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
376
985
|
windowMs: this._thrashingWindowMs
|
|
377
986
|
});
|
|
378
987
|
} catch (alertErr) {
|
|
379
|
-
|
|
988
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Thrashing alert callback error:`, alertErr.message);
|
|
380
989
|
}
|
|
381
990
|
}
|
|
382
991
|
|
|
@@ -412,7 +1021,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
412
1021
|
reason: reason,
|
|
413
1022
|
timestamp: new Date().toISOString(),
|
|
414
1023
|
stack: error?.stack || new Error().stack,
|
|
415
|
-
connectionState: this._connection ? (this.
|
|
1024
|
+
connectionState: this._connection ? (this._connectionAlive ? 'open' : 'closed') : 'null',
|
|
416
1025
|
channelCreatedAt: channel?._createdAt || 'unknown',
|
|
417
1026
|
lastOperation: channel?._lastOperation || 'unknown'
|
|
418
1027
|
};
|
|
@@ -422,7 +1031,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
422
1031
|
try {
|
|
423
1032
|
hook.callback(details);
|
|
424
1033
|
} catch (hookErr) {
|
|
425
|
-
|
|
1034
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Channel close hook error:`, hookErr.message);
|
|
426
1035
|
}
|
|
427
1036
|
}
|
|
428
1037
|
}
|
|
@@ -436,6 +1045,157 @@ class RabbitMQClient extends EventEmitter {
|
|
|
436
1045
|
* @returns {Promise<void>}
|
|
437
1046
|
* @throws {Error} If connection or channel creation fails.
|
|
438
1047
|
*/
|
|
1048
|
+
/**
|
|
1049
|
+
* The connection died — the ONE thing this client does about it.
|
|
1050
|
+
*
|
|
1051
|
+
* A connection can die twice over the life of one client: the one `connect()`
|
|
1052
|
+
* opened, and the one a recovery round re-established. Both registrations of
|
|
1053
|
+
* `connection.on('close')` come here, because this is one concern and a
|
|
1054
|
+
* concern with two implementations is a defect by default
|
|
1055
|
+
* (`change-discipline.md` § One rail per concern). The cost of the two copies
|
|
1056
|
+
* is on record: d.283 had to fix one defect in two places, and the copy on the
|
|
1057
|
+
* recovery side had never asked about `_closedByCaller` at all — which is how
|
|
1058
|
+
* a client its owner had already disposed of came back to life.
|
|
1059
|
+
*
|
|
1060
|
+
* The steps, in this order:
|
|
1061
|
+
*
|
|
1062
|
+
* 1. record the death — `_connectionAlive` is the ONE liveness source
|
|
1063
|
+
* (d.260), and it is set before anything decides what to do about it;
|
|
1064
|
+
* 2. drop the corpse: the three channels and the connection itself. Until
|
|
1065
|
+
* d.260 the reference survived until the recovery loop's first backoff had
|
|
1066
|
+
* elapsed, and `!this._connection.closed` read `undefined` throughout — so
|
|
1067
|
+
* for that whole window `isConnected()` answered TRUE and
|
|
1068
|
+
* `_ensure*Channel()` opened channels on a corpse;
|
|
1069
|
+
* 3. ask whose doing it was. A close the CALLER ordered is not a fault:
|
|
1070
|
+
* there is nothing to recover and nothing to report (d.283);
|
|
1071
|
+
* 4. start recovery BEFORE reporting the close, so an `error` listener that
|
|
1072
|
+
* throws cannot prevent the reconnect;
|
|
1073
|
+
* 5. report the close.
|
|
1074
|
+
*
|
|
1075
|
+
* @private
|
|
1076
|
+
*/
|
|
1077
|
+
_onConnectionDied() {
|
|
1078
|
+
this._connectionAlive = false;
|
|
1079
|
+
this._channel = null;
|
|
1080
|
+
this._queueChannel = null;
|
|
1081
|
+
this._consumerChannel = null;
|
|
1082
|
+
this._connection = null;
|
|
1083
|
+
|
|
1084
|
+
if (this._closedByCaller) {
|
|
1085
|
+
this._logger.info('[RabbitMQClient] Connection closed during disconnect (expected)');
|
|
1086
|
+
return;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
this._logger.warn('[RabbitMQClient] Connection closed unexpectedly - close handler STARTED');
|
|
1090
|
+
this._logger.debug(`[RabbitMQClient] Close handler: reconnectEnabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
|
|
1091
|
+
|
|
1092
|
+
if (this._reconnectEnabled && !this._reconnecting) {
|
|
1093
|
+
this._logger.info('[RabbitMQClient] \u2713 Conditions met - starting connection-level recovery');
|
|
1094
|
+
this._reconnectWithBackoff().catch((err) => this._reportRecoveryFailure(err));
|
|
1095
|
+
} else {
|
|
1096
|
+
this._logger.warn(`[RabbitMQClient] \u2717 Reconnect NOT starting: enabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
this.emit('error', connectionClosedUnexpectedly());
|
|
1100
|
+
|
|
1101
|
+
this._logger.debug('[RabbitMQClient] Close handler FINISHED');
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
/**
|
|
1105
|
+
* Close one amqplib handle during teardown, and never wait longer than the
|
|
1106
|
+
* budget left for the whole teardown.
|
|
1107
|
+
*
|
|
1108
|
+
* The budget is NOT a new number: `brokerAnswerTimeout` is this client's
|
|
1109
|
+
* declared answer to exactly this question — how long it waits for the broker
|
|
1110
|
+
* to answer a frame it sent (`config/configSchema.js`) — and a `close-ok` is
|
|
1111
|
+
* that same kind of answer. The key carried the name of only one of its two
|
|
1112
|
+
* consumers (`publishConfirmationTimeout`) until d.299.
|
|
1113
|
+
*
|
|
1114
|
+
* @param {string} what - the handle's name, for the log
|
|
1115
|
+
* @param {Object|null} handle - an amqplib channel or connection, or nothing
|
|
1116
|
+
* @param {number} budgetMs - what is left of the teardown budget
|
|
1117
|
+
* @returns {Promise<{status: 'absent'|'closed'|'refused'|'unanswered', error: Error|null}>}
|
|
1118
|
+
* @private
|
|
1119
|
+
*/
|
|
1120
|
+
async _closeWithinTeardownBudget(what, handle, budgetMs) {
|
|
1121
|
+
if (!handle) {
|
|
1122
|
+
return { status: 'absent', error: null };
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
const UNANSWERED = Symbol('teardown-budget-spent');
|
|
1126
|
+
let deadlineTimer = null;
|
|
1127
|
+
const outcome = await Promise.race([
|
|
1128
|
+
handle.close().then(
|
|
1129
|
+
() => ({ status: 'closed', error: null }),
|
|
1130
|
+
(err) => ({ status: 'refused', error: err })
|
|
1131
|
+
),
|
|
1132
|
+
new Promise((resolve) => {
|
|
1133
|
+
deadlineTimer = this._setTimeout(() => resolve(UNANSWERED), budgetMs);
|
|
1134
|
+
})
|
|
1135
|
+
]);
|
|
1136
|
+
this._clearTimeout(deadlineTimer);
|
|
1137
|
+
|
|
1138
|
+
if (outcome === UNANSWERED) {
|
|
1139
|
+
this._logger.warn(
|
|
1140
|
+
`[RabbitMQClient] The broker did not answer the ${what} close within the teardown budget `
|
|
1141
|
+
+ `(${this._brokerAnswerTimeout}ms) - `
|
|
1142
|
+
+ 'Expected: a close-ok frame for the close this client sent. '
|
|
1143
|
+
+ 'Fix: nothing to do here — the teardown stops waiting and drops the socket, so '
|
|
1144
|
+
+ 'disconnect() ends and the process can exit; a broker that never answers a close is '
|
|
1145
|
+
+ 'a broker-side or network fault, and the declared brokerAnswerTimeout sets how '
|
|
1146
|
+
+ 'long this client gives it.'
|
|
1147
|
+
);
|
|
1148
|
+
return { status: 'unanswered', error: null };
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
return outcome;
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
/**
|
|
1155
|
+
* Drop everything a connection the broker never answered on still holds.
|
|
1156
|
+
*
|
|
1157
|
+
* amqplib lets go of both of these itself when a close COMPLETES — its
|
|
1158
|
+
* `toClosed()` clears the heartbeater and ends the stream. A close that is
|
|
1159
|
+
* never answered never reaches it, so the teardown does it: the socket,
|
|
1160
|
+
* because an open socket keeps the process alive, and the heartbeat interval,
|
|
1161
|
+
* because an armed repeating timer does exactly the same (measured: the
|
|
1162
|
+
* integration tier reported one open handle, `setInterval(30000)` from
|
|
1163
|
+
* `amqplib/lib/heartbeat.js`, after the socket alone was dropped).
|
|
1164
|
+
*
|
|
1165
|
+
* Both live on amqplib's inner `Connection` (`connection.connection`):
|
|
1166
|
+
* `.stream` is the socket (0.10.9 — `wrapStream()` returns a `net.Socket`
|
|
1167
|
+
* unchanged, because a socket already is a Duplex) and `.heartbeater` is the
|
|
1168
|
+
* `Heart`. There is no public way to abort a connection whose `close()` will
|
|
1169
|
+
* never settle, so the shape is checked before it is used and a change in it
|
|
1170
|
+
* is reported rather than silently survived.
|
|
1171
|
+
*
|
|
1172
|
+
* @param {Object|null} connection - the amqplib ChannelModel being torn down
|
|
1173
|
+
* @private
|
|
1174
|
+
*/
|
|
1175
|
+
_dropUnansweredConnection(connection) {
|
|
1176
|
+
const inner = connection && connection.connection ? connection.connection : null;
|
|
1177
|
+
const socket = inner && inner.stream ? inner.stream : null;
|
|
1178
|
+
|
|
1179
|
+
if (!socket || typeof socket.destroy !== 'function') {
|
|
1180
|
+
this._logger.warn(
|
|
1181
|
+
'[RabbitMQClient] Teardown could not reach the socket of an unanswered connection - '
|
|
1182
|
+
+ 'Expected: amqplib to expose it at connection.connection.stream (0.10.x). '
|
|
1183
|
+
+ 'Fix: the socket may outlive disconnect() and keep the process alive; check whether '
|
|
1184
|
+
+ 'the installed amqplib still has that shape.'
|
|
1185
|
+
);
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
// A connection that never finished opening has no heartbeater; one that did
|
|
1190
|
+
// has an interval nobody else will clear now.
|
|
1191
|
+
if (inner.heartbeater && typeof inner.heartbeater.clear === 'function') {
|
|
1192
|
+
inner.heartbeater.clear();
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
socket.destroy();
|
|
1196
|
+
this._logger.warn('[RabbitMQClient] Teardown dropped the socket and heartbeat of a connection the broker never answered on');
|
|
1197
|
+
}
|
|
1198
|
+
|
|
439
1199
|
/**
|
|
440
1200
|
* Helper to set a trackable timeout
|
|
441
1201
|
* @private
|
|
@@ -473,72 +1233,46 @@ class RabbitMQClient extends EventEmitter {
|
|
|
473
1233
|
|
|
474
1234
|
async connect() {
|
|
475
1235
|
let connectTimeoutTimer = null;
|
|
1236
|
+
// An explicit `connect()` is the one thing that undoes an explicit
|
|
1237
|
+
// `disconnect()`: the caller wants this client alive again, so recovery is
|
|
1238
|
+
// armed again with it. Nothing else clears the flag.
|
|
1239
|
+
this._closedByCaller = false;
|
|
1240
|
+
// …and the recovery budget starts over with it, for the same reason: the
|
|
1241
|
+
// cycles were spent against the outage the caller has just decided is over
|
|
1242
|
+
// (d.339).
|
|
1243
|
+
this._reconnectCycles = 0;
|
|
1244
|
+
this._reconnectAttempts = 0;
|
|
1245
|
+
this._lastConnectionError = null;
|
|
476
1246
|
try {
|
|
477
|
-
const rawTarget = this.
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
const connectArgs = typeof rawTarget === 'string'
|
|
486
|
-
? [rawTarget, { heartbeat, clientProperties: { connection_name: connectionName } }]
|
|
487
|
-
: [{ ...rawTarget, heartbeat, clientProperties: { connection_name: connectionName } }];
|
|
488
|
-
|
|
489
|
-
const connectPromise = amqp.connect(...connectArgs);
|
|
1247
|
+
const rawTarget = this._connectionTarget();
|
|
1248
|
+
|
|
1249
|
+
// The target carries the broker password in its URL userinfo. Only the
|
|
1250
|
+
// redacted form reaches the log; the real value goes to amqplib untouched.
|
|
1251
|
+
const safeTarget = redactConnectionTarget(rawTarget);
|
|
1252
|
+
this._logger.info(`[RabbitMQClient] Attempting to connect to: ${safeTarget}`);
|
|
1253
|
+
|
|
1254
|
+
const connectPromise = amqp.connect(rawTarget, this._connectionSocketOptions());
|
|
490
1255
|
const timeoutPromise = new Promise((_, reject) => {
|
|
491
|
-
connectTimeoutTimer = this._setTimeout(() => reject(new
|
|
1256
|
+
connectTimeoutTimer = this._setTimeout(() => reject(new ConnectionError(
|
|
1257
|
+
`[RabbitMQClient] Connection timeout after ${this._connectTimeout} ms - `
|
|
1258
|
+
+ `Expected: the broker to accept a TCP+AMQP handshake within connectTimeout (${this._connectTimeout} ms). `
|
|
1259
|
+
+ 'Fix: check the broker is up and reachable from this container, and that the configured host/port and credentials are the ones it serves.'
|
|
1260
|
+
)), this._connectTimeout);
|
|
492
1261
|
if (connectTimeoutTimer && typeof connectTimeoutTimer.unref === 'function') {
|
|
493
1262
|
connectTimeoutTimer.unref();
|
|
494
1263
|
}
|
|
495
1264
|
});
|
|
496
|
-
|
|
1265
|
+
this._logger.debug('[RabbitMQClient] Starting connection race...');
|
|
497
1266
|
this._connection = await Promise.race([connectPromise, timeoutPromise]);
|
|
498
1267
|
if (connectTimeoutTimer) {
|
|
499
1268
|
this._clearTimeout(connectTimeoutTimer);
|
|
500
1269
|
connectTimeoutTimer = null;
|
|
501
1270
|
}
|
|
502
|
-
|
|
1271
|
+
this._logger.info('[RabbitMQClient] Connection established');
|
|
1272
|
+
this._connectionAlive = true;
|
|
503
1273
|
this._reconnectAttempts = 0; // Reset reconnect attempts on successful connection
|
|
504
|
-
|
|
505
|
-
this._connection
|
|
506
|
-
console.error('[RabbitMQClient] Connection error:', err.message);
|
|
507
|
-
this.emit('error', err);
|
|
508
|
-
// Connection errors may lead to close event - let close handler handle reconnection
|
|
509
|
-
});
|
|
510
|
-
|
|
511
|
-
this._connection.on('close', () => {
|
|
512
|
-
if (this._disconnecting) {
|
|
513
|
-
console.log('[RabbitMQClient] Connection closed during disconnect (expected)');
|
|
514
|
-
return;
|
|
515
|
-
}
|
|
516
|
-
console.warn('[RabbitMQClient] Connection closed unexpectedly - close handler STARTED');
|
|
517
|
-
console.log(`[RabbitMQClient] Close handler: reconnectEnabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
|
|
518
|
-
|
|
519
|
-
// Mark all channels as closed
|
|
520
|
-
this._channel = null;
|
|
521
|
-
this._queueChannel = null;
|
|
522
|
-
this._consumerChannel = null;
|
|
523
|
-
|
|
524
|
-
// CONNECTION-LEVEL RECOVERY: Attempt to reconnect with exponential backoff
|
|
525
|
-
// PREDICTABLE: Check conditions and log why reconnect is/isn't starting
|
|
526
|
-
// IMPORTANT: Do this BEFORE emitting error, so reconnect can start even if error handler throws
|
|
527
|
-
if (this._reconnectEnabled && !this._reconnecting) {
|
|
528
|
-
console.log('[RabbitMQClient] ✓ Conditions met - starting connection-level recovery from initial connect() close handler');
|
|
529
|
-
this._reconnectWithBackoff().catch(err => {
|
|
530
|
-
console.error('[RabbitMQClient] Reconnection failed after all attempts:', err.message);
|
|
531
|
-
this.emit('error', err);
|
|
532
|
-
});
|
|
533
|
-
} else {
|
|
534
|
-
console.warn(`[RabbitMQClient] ✗ Reconnect NOT starting: enabled=${this._reconnectEnabled}, reconnecting=${this._reconnecting}`);
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
// Emit connection close event AFTER starting reconnect (so reconnect isn't blocked by error handler)
|
|
538
|
-
this.emit('error', new Error('RabbitMQ connection closed unexpectedly'));
|
|
539
|
-
|
|
540
|
-
console.log('[RabbitMQClient] Close handler FINISHED');
|
|
541
|
-
});
|
|
1274
|
+
|
|
1275
|
+
this._attachConnectionHandlers(this._connection);
|
|
542
1276
|
|
|
543
1277
|
// Use ConfirmChannel to enable publisher confirms for publish operations
|
|
544
1278
|
this._channel = await this._connection.createConfirmChannel();
|
|
@@ -584,6 +1318,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
584
1318
|
}
|
|
585
1319
|
} catch (err) {
|
|
586
1320
|
// Cleanup partially created resources
|
|
1321
|
+
this._connectionAlive = false;
|
|
587
1322
|
if (this._connection) {
|
|
588
1323
|
try {
|
|
589
1324
|
await this._connection.close();
|
|
@@ -610,37 +1345,104 @@ class RabbitMQClient extends EventEmitter {
|
|
|
610
1345
|
*/
|
|
611
1346
|
async _waitForReconnection() {
|
|
612
1347
|
if (!this._reconnecting) {
|
|
613
|
-
throw new
|
|
1348
|
+
throw new ConnectionError(
|
|
1349
|
+
'[RabbitMQClient] Cannot wait for reconnection: not currently reconnecting - '
|
|
1350
|
+
+ 'Expected: _waitForReconnection() to be called only while a reconnect is in flight. '
|
|
1351
|
+
+ 'Fix: check this._reconnecting before calling it; a closed connection that is not reconnecting is fatal, not transient.'
|
|
1352
|
+
);
|
|
614
1353
|
}
|
|
615
1354
|
|
|
616
|
-
//
|
|
1355
|
+
// A cycle has exactly three endings, and this wait ends on each of them:
|
|
1356
|
+
// `reconnected` (it worked), `connection:standby` (the cycle is spent, the
|
|
1357
|
+
// client waits to be used again — d.339) and `connection:fatal` (it is over).
|
|
1358
|
+
// The generic `error` channel stays as the fourth listener for everything
|
|
1359
|
+
// else, but the two terminal outcomes are NOT read from it: they are
|
|
1360
|
+
// announced on their own channels the moment they happen, whereas the copy
|
|
1361
|
+
// that reaches `error` is re-emitted a turn later by the recovery's catch —
|
|
1362
|
+
// late enough to land inside the NEXT cycle's wait and reject it with the
|
|
1363
|
+
// previous cycle's obituary (measured while writing d.339).
|
|
617
1364
|
return new Promise((resolve, reject) => {
|
|
618
|
-
const
|
|
1365
|
+
const done = () => {
|
|
1366
|
+
this._clearTimeout(timeout);
|
|
619
1367
|
this.removeListener('reconnected', onReconnected);
|
|
620
1368
|
this.removeListener('error', onError);
|
|
621
|
-
|
|
1369
|
+
this.removeListener('connection:standby', onStandby);
|
|
1370
|
+
this.removeListener('connection:fatal', onFatal);
|
|
1371
|
+
};
|
|
1372
|
+
|
|
1373
|
+
const timeout = this._setTimeout(() => {
|
|
1374
|
+
done();
|
|
1375
|
+
reject(new ConnectionError(
|
|
1376
|
+
`[RabbitMQClient] Reconnection timeout after ${this._reconnectWaitTimeout}ms - `
|
|
1377
|
+
+ 'Expected: the broker to accept a new connection within RABBITMQ_RECONNECT_WAIT_TIMEOUT. '
|
|
1378
|
+
+ 'Fix: check the broker is up and reachable; raise that timeout only if a longer outage must be survived in-process.'
|
|
1379
|
+
));
|
|
622
1380
|
}, this._reconnectWaitTimeout);
|
|
623
1381
|
|
|
624
1382
|
const onReconnected = () => {
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
//
|
|
629
|
-
this._setTimeout(resolve, 500)
|
|
1383
|
+
done();
|
|
1384
|
+
// Resolved AT the event, not half a second after it. `reconnected` is
|
|
1385
|
+
// emitted once the connection is back and its three channels have been
|
|
1386
|
+
// recreated (d.340), so the guarantee this used to buy with
|
|
1387
|
+
// `this._setTimeout(resolve, 500)` — "wait a bit for channels to be fully
|
|
1388
|
+
// recreated" — is now in the event's own meaning.
|
|
1389
|
+
//
|
|
1390
|
+
// The sleep was also a hang: `_setTimeout` registers in `_activeTimers`
|
|
1391
|
+
// and `disconnect()` clears every one of them, so a disconnect inside
|
|
1392
|
+
// that window cancelled the pending `resolve` and left the awaiting
|
|
1393
|
+
// publish unsettled for ever (measured, d.369).
|
|
1394
|
+
resolve();
|
|
1395
|
+
};
|
|
1396
|
+
|
|
1397
|
+
const onStandby = (state) => {
|
|
1398
|
+
done();
|
|
1399
|
+
reject(new ConnectionError(
|
|
1400
|
+
`[RabbitMQClient] Waiting for reconnection ended: recovery cycle ${state.cycle}/${state.cyclesMax} `
|
|
1401
|
+
+ `spent its ${state.attempts} attempts without reaching the broker (last error: ${state.lastError}). `
|
|
1402
|
+
+ 'Expected: the broker to answer within one recovery cycle. '
|
|
1403
|
+
+ 'Fix: run the operation again once the broker is back — that starts the next cycle; '
|
|
1404
|
+
+ 'RABBITMQ_MAX_RECONNECT_CYCLES bounds how many of them this client will run.'
|
|
1405
|
+
));
|
|
1406
|
+
};
|
|
1407
|
+
|
|
1408
|
+
const onFatal = (fatal) => {
|
|
1409
|
+
done();
|
|
1410
|
+
reject(fatal);
|
|
630
1411
|
};
|
|
631
1412
|
|
|
632
1413
|
const onError = (error) => {
|
|
633
|
-
//
|
|
634
|
-
|
|
1414
|
+
// The transport's own close notification is the normal state of the thing being
|
|
1415
|
+
// waited for, so it never ends this wait. Recognised by its code, never by its
|
|
1416
|
+
// wording: the text comparison this replaced asked for a capital `C` the emitted
|
|
1417
|
+
// message never had, so it ignored nothing at all (d.153a → d.164).
|
|
1418
|
+
if (error.code === CONNECTION_CLOSED_UNEXPECTEDLY) {
|
|
635
1419
|
return; // Expected during reconnection
|
|
636
1420
|
}
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
this
|
|
1421
|
+
// A PUBLISH failure is the outcome of a MESSAGE, not of the connection, so it
|
|
1422
|
+
// never ends a wait for the connection. `_publishOnce()` puts every classified
|
|
1423
|
+
// publish failure on this same `error` channel, so until d.435 one concurrent
|
|
1424
|
+
// publish that failed while the link was coming back rejected EVERY waiter:
|
|
1425
|
+
// `PublishLayer` buffered their messages and reported `reconnectFailed: true`
|
|
1426
|
+
// while the recovery it declared failed was still running.
|
|
1427
|
+
//
|
|
1428
|
+
// The test is the PRESENCE of a publish classification, not its verdict: d.435
|
|
1429
|
+
// asked `retryable === true`, which still let a `PermanentPublishError` or a
|
|
1430
|
+
// `QueueNotFoundError` from a concurrent publish end the wait — and neither is
|
|
1431
|
+
// a statement about this connection either; "no retry will help this message"
|
|
1432
|
+
// says nothing about whether the link is coming back (d.435b). An error that
|
|
1433
|
+
// never went through the ONE classification rail (`utils/publishErrors.js`)
|
|
1434
|
+
// carries no `retryable` at all, and that absence is the whole condition —
|
|
1435
|
+
// never a second list of codes kept here.
|
|
1436
|
+
if (error.retryable !== undefined) {
|
|
1437
|
+
return; // Somebody else's message failed; the connection has said nothing
|
|
1438
|
+
}
|
|
1439
|
+
done();
|
|
640
1440
|
reject(error);
|
|
641
1441
|
};
|
|
642
1442
|
|
|
643
1443
|
this.once('reconnected', onReconnected);
|
|
1444
|
+
this.once('connection:standby', onStandby);
|
|
1445
|
+
this.once('connection:fatal', onFatal);
|
|
644
1446
|
this.on('error', onError);
|
|
645
1447
|
});
|
|
646
1448
|
}
|
|
@@ -652,42 +1454,48 @@ class RabbitMQClient extends EventEmitter {
|
|
|
652
1454
|
* @returns {Promise<void>}
|
|
653
1455
|
*/
|
|
654
1456
|
async _ensurePublisherChannel() {
|
|
655
|
-
if (this.
|
|
1457
|
+
if (this._isChannelAlive(this._channel)) {
|
|
656
1458
|
return; // Channel is good
|
|
657
1459
|
}
|
|
658
|
-
|
|
659
|
-
if (!this.
|
|
660
|
-
// Connection is closed
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
//
|
|
664
|
-
|
|
1460
|
+
|
|
1461
|
+
if (!this._connectionAlive) {
|
|
1462
|
+
// Connection is closed. THIS is the lazy half of the retry contract: a
|
|
1463
|
+
// publish through a client that stood down after a spent cycle starts the
|
|
1464
|
+
// next one here (d.339), rather than being refused for the life of the
|
|
1465
|
+
// process.
|
|
1466
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot recreate publisher channel: connection is closed (waiting for reconnection)');
|
|
1467
|
+
|
|
1468
|
+
if (this._resumeRecovery()) {
|
|
665
1469
|
await this._waitForReconnection();
|
|
666
|
-
|
|
1470
|
+
|
|
667
1471
|
// After reconnection, try again
|
|
668
|
-
if (this.
|
|
1472
|
+
if (this._isChannelAlive(this._channel)) {
|
|
669
1473
|
return; // Channel was recreated during reconnection
|
|
670
1474
|
}
|
|
671
1475
|
// If still no channel, connection might have failed - check again
|
|
672
|
-
if (!this.
|
|
673
|
-
throw new
|
|
1476
|
+
if (!this._connectionAlive) {
|
|
1477
|
+
throw new ConnectionError(
|
|
1478
|
+
'[RabbitMQClient] Connection reconnection failed - cannot create publisher channel. '
|
|
1479
|
+
+ 'Expected: an open connection once the reconnect completed. '
|
|
1480
|
+
+ 'Fix: check the broker is reachable; the client spends RABBITMQ_MAX_RECONNECT_ATTEMPTS per cycle and '
|
|
1481
|
+
+ 'RABBITMQ_MAX_RECONNECT_CYCLES cycles in all, then emits connection:fatal.'
|
|
1482
|
+
);
|
|
674
1483
|
}
|
|
675
1484
|
} else {
|
|
676
|
-
//
|
|
677
|
-
|
|
1485
|
+
// No recovery can be started: the connection is permanently lost (the
|
|
1486
|
+
// broker refused it, or every cycle is spent), the caller ended this
|
|
1487
|
+
// client, or recovery is disarmed by configuration.
|
|
1488
|
+
throw new ConnectionError(
|
|
1489
|
+
'[RabbitMQClient] Connection is closed and no recovery can be started - cannot create publisher channel. '
|
|
1490
|
+
+ 'Expected: either an open connection, or a recovery this client is still allowed to run. '
|
|
1491
|
+
+ 'Fix: read connection:fatal for which of the two ended it — a broker refusal (credentials, vhost, '
|
|
1492
|
+
+ 'topology) or RABBITMQ_MAX_RECONNECT_CYCLES spent; reconnectEnabled: false and disconnect() also '
|
|
1493
|
+
+ 'leave the client here, on purpose.'
|
|
1494
|
+
);
|
|
678
1495
|
}
|
|
679
1496
|
}
|
|
680
1497
|
|
|
681
1498
|
try {
|
|
682
|
-
// Close old channel if exists
|
|
683
|
-
if (this._channel) {
|
|
684
|
-
try {
|
|
685
|
-
await this._channel.close();
|
|
686
|
-
} catch (_) {
|
|
687
|
-
// Ignore errors when closing already-closed channel
|
|
688
|
-
}
|
|
689
|
-
}
|
|
690
|
-
|
|
691
1499
|
// Create new ConfirmChannel
|
|
692
1500
|
this._channel = await this._connection.createConfirmChannel();
|
|
693
1501
|
|
|
@@ -700,7 +1508,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
700
1508
|
// Attach event handlers
|
|
701
1509
|
this._attachPublisherChannelHandlers(this._channel);
|
|
702
1510
|
|
|
703
|
-
|
|
1511
|
+
this._logger.info('[RabbitMQClient] [mq-client-core] ✓ Publisher channel recreated');
|
|
704
1512
|
} catch (err) {
|
|
705
1513
|
this._channel = null;
|
|
706
1514
|
|
|
@@ -709,17 +1517,24 @@ class RabbitMQClient extends EventEmitter {
|
|
|
709
1517
|
// Pokud se connection právě zavírá, považujeme to za přechodný stav – explicitně vyhodíme
|
|
710
1518
|
// TransientPublishError, aby vyšší vrstva (retry / worker) věděla, že má počkat na reconnect.
|
|
711
1519
|
if (msg.includes('Connection closing') || msg.includes('Connection closed')) {
|
|
712
|
-
|
|
1520
|
+
this._logger.warn(
|
|
713
1521
|
`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection closing): ${msg}`
|
|
714
1522
|
);
|
|
715
1523
|
throw new TransientPublishError(
|
|
716
|
-
'Publisher channel cannot be recreated because connection is closing/closed (transient condition)'
|
|
1524
|
+
'[RabbitMQClient] Publisher channel cannot be recreated because connection is closing/closed (transient condition) - '
|
|
1525
|
+
+ 'Expected: the reconnect to restore the connection shortly. '
|
|
1526
|
+
+ 'Fix: retry the publish after the reconnect; PublishLayer does this on its own and buffers the message meanwhile.',
|
|
717
1527
|
err
|
|
718
1528
|
);
|
|
719
1529
|
}
|
|
720
1530
|
|
|
721
1531
|
// Ostatní chyby považujeme za permanentní problém na úrovni kanálu/spojení.
|
|
722
|
-
throw new PermanentPublishError(
|
|
1532
|
+
throw new PermanentPublishError(
|
|
1533
|
+
`[RabbitMQClient] Failed to recreate publisher channel: ${msg} - `
|
|
1534
|
+
+ 'Expected: createConfirmChannel() to succeed on an open connection. '
|
|
1535
|
+
+ 'Fix: read error.cause for the broker reason; a channel-level refusal here is not retried, so the connection must be re-established.',
|
|
1536
|
+
err
|
|
1537
|
+
);
|
|
723
1538
|
}
|
|
724
1539
|
}
|
|
725
1540
|
|
|
@@ -730,28 +1545,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
730
1545
|
* @returns {Promise<void>}
|
|
731
1546
|
*/
|
|
732
1547
|
async _ensureQueueChannel() {
|
|
733
|
-
if (this.
|
|
1548
|
+
if (this._isChannelAlive(this._queueChannel)) {
|
|
734
1549
|
return; // Channel is good
|
|
735
1550
|
}
|
|
736
|
-
|
|
737
|
-
if (!this.
|
|
1551
|
+
|
|
1552
|
+
if (!this._connectionAlive) {
|
|
738
1553
|
// Connection is closed - cannot recreate channel
|
|
739
1554
|
// Reconnection logic will handle this - don't throw, just return
|
|
740
1555
|
// The channel will be recreated after reconnection completes
|
|
741
|
-
|
|
1556
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot recreate queue channel: connection is closed (reconnection in progress)');
|
|
742
1557
|
return;
|
|
743
1558
|
}
|
|
744
1559
|
|
|
745
1560
|
try {
|
|
746
|
-
// Close old channel if exists
|
|
747
|
-
if (this._queueChannel) {
|
|
748
|
-
try {
|
|
749
|
-
await this._queueChannel.close();
|
|
750
|
-
} catch (_) {
|
|
751
|
-
// Ignore errors when closing already-closed channel
|
|
752
|
-
}
|
|
753
|
-
}
|
|
754
|
-
|
|
755
1561
|
// Create new regular channel
|
|
756
1562
|
this._queueChannel = await this._connection.createChannel();
|
|
757
1563
|
|
|
@@ -764,15 +1570,20 @@ class RabbitMQClient extends EventEmitter {
|
|
|
764
1570
|
// Attach event handlers
|
|
765
1571
|
this._attachQueueChannelHandlers(this._queueChannel);
|
|
766
1572
|
|
|
767
|
-
|
|
1573
|
+
this._logger.info('[RabbitMQClient] [mq-client-core] ✓ Queue channel recreated');
|
|
768
1574
|
} catch (err) {
|
|
769
1575
|
this._queueChannel = null;
|
|
770
1576
|
// If connection is closed, don't throw - reconnection will handle it
|
|
771
|
-
if (this.
|
|
772
|
-
throw new
|
|
1577
|
+
if (this._connectionAlive) {
|
|
1578
|
+
throw new ConnectionError(
|
|
1579
|
+
`[RabbitMQClient] Failed to recreate queue channel: ${err.message} - `
|
|
1580
|
+
+ 'Expected: createChannel() to succeed while the connection is open. '
|
|
1581
|
+
+ 'Fix: read error.cause for the broker reason; if the connection itself dropped, the reconnect recreates the channel.',
|
|
1582
|
+
err
|
|
1583
|
+
);
|
|
773
1584
|
}
|
|
774
1585
|
// Connection is closed - reconnection will recreate channel
|
|
775
|
-
|
|
1586
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel (connection closed): ${err.message}`);
|
|
776
1587
|
}
|
|
777
1588
|
}
|
|
778
1589
|
|
|
@@ -783,29 +1594,40 @@ class RabbitMQClient extends EventEmitter {
|
|
|
783
1594
|
* @returns {Promise<{reRegistered: number, failed: number}>} - Number of consumers re-registered and failed
|
|
784
1595
|
*/
|
|
785
1596
|
async _ensureConsumerChannel() {
|
|
786
|
-
if (this.
|
|
1597
|
+
if (this._isChannelAlive(this._consumerChannel)) {
|
|
787
1598
|
// Channel is good - return current consumer state
|
|
788
1599
|
return { reRegistered: this._activeConsumers.size, failed: 0 };
|
|
789
1600
|
}
|
|
790
|
-
|
|
791
|
-
if (!this.
|
|
792
|
-
//
|
|
793
|
-
//
|
|
794
|
-
//
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
try {
|
|
800
|
-
// Close old channel if exists
|
|
801
|
-
if (this._consumerChannel) {
|
|
802
|
-
try {
|
|
803
|
-
await this._consumerChannel.close();
|
|
804
|
-
} catch (_) {
|
|
805
|
-
// Ignore errors when closing already-closed channel
|
|
806
|
-
}
|
|
1601
|
+
|
|
1602
|
+
if (!this._connectionAlive) {
|
|
1603
|
+
// Same lazy half as the publisher path: `consume()` is one of the three
|
|
1604
|
+
// uses the contract names, so it starts the next recovery cycle instead of
|
|
1605
|
+
// reporting every consumer as failed and leaving the client down (d.339).
|
|
1606
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot recreate consumer channel: connection is closed (reconnection in progress)');
|
|
1607
|
+
|
|
1608
|
+
if (!this._resumeRecovery()) {
|
|
1609
|
+
return { reRegistered: 0, failed: this._activeConsumers.size };
|
|
807
1610
|
}
|
|
808
|
-
|
|
1611
|
+
|
|
1612
|
+
try {
|
|
1613
|
+
await this._waitForReconnection();
|
|
1614
|
+
} catch (err) {
|
|
1615
|
+
this._logger.warn(
|
|
1616
|
+
`[RabbitMQClient] [mq-client-core] Consumer channel waited for a recovery that did not arrive: ${err.message}`
|
|
1617
|
+
);
|
|
1618
|
+
return { reRegistered: 0, failed: this._activeConsumers.size };
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
if (this._isChannelAlive(this._consumerChannel)) {
|
|
1622
|
+
// The recovery recreated it and re-registered what was tracked.
|
|
1623
|
+
return { reRegistered: this._activeConsumers.size, failed: 0 };
|
|
1624
|
+
}
|
|
1625
|
+
if (!this._connectionAlive) {
|
|
1626
|
+
return { reRegistered: 0, failed: this._activeConsumers.size };
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1630
|
+
try {
|
|
809
1631
|
// Create new regular channel
|
|
810
1632
|
this._consumerChannel = await this._connection.createChannel();
|
|
811
1633
|
|
|
@@ -821,16 +1643,31 @@ class RabbitMQClient extends EventEmitter {
|
|
|
821
1643
|
// Re-register all active consumers
|
|
822
1644
|
const { reRegistered, failed } = await this._reRegisterConsumers();
|
|
823
1645
|
|
|
824
|
-
|
|
1646
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] ✓ Consumer channel recreated (re-registered: ${reRegistered}, failed: ${failed})`);
|
|
825
1647
|
return { reRegistered, failed };
|
|
826
1648
|
} catch (err) {
|
|
827
1649
|
this._consumerChannel = null;
|
|
828
|
-
throw new
|
|
1650
|
+
throw new ConnectionError(
|
|
1651
|
+
`[RabbitMQClient] Failed to recreate consumer channel: ${err.message} - `
|
|
1652
|
+
+ 'Expected: createChannel() and the consumer re-registration to succeed while the connection is open. '
|
|
1653
|
+
+ 'Fix: read error.cause for the broker reason; consumers are re-registered automatically on the next reconnect.',
|
|
1654
|
+
err
|
|
1655
|
+
);
|
|
829
1656
|
}
|
|
830
1657
|
}
|
|
831
1658
|
|
|
832
1659
|
/**
|
|
833
|
-
*
|
|
1660
|
+
* Re-register every tracked consumer on the current consumer channel.
|
|
1661
|
+
*
|
|
1662
|
+
* The queue step is NOT repeated here: it is `_prepareQueueForConsume()`, the
|
|
1663
|
+
* same rail the first `consume()` runs, so a re-registration classifies,
|
|
1664
|
+
* declares and refuses exactly what the first attachment did. The copy this
|
|
1665
|
+
* replaced asserted every tracked queue unconditionally — declaring queues the
|
|
1666
|
+
* consumer does not own — and answered a 406 by DELETING the consumer from
|
|
1667
|
+
* tracking, which made the eviction permanent: nothing re-registered it before
|
|
1668
|
+
* the process restarted. A failure now leaves the consumer tracked, so the
|
|
1669
|
+
* next channel recreate tries again (d.262).
|
|
1670
|
+
*
|
|
834
1671
|
* @private
|
|
835
1672
|
* @returns {Promise<{reRegistered: number, failed: number}>}
|
|
836
1673
|
*/
|
|
@@ -838,43 +1675,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
838
1675
|
let reRegistered = 0;
|
|
839
1676
|
let failed = 0;
|
|
840
1677
|
|
|
841
|
-
if (!this.
|
|
1678
|
+
if (!this._isChannelAlive(this._consumerChannel)) {
|
|
842
1679
|
return { reRegistered: 0, failed: this._activeConsumers.size };
|
|
843
1680
|
}
|
|
844
1681
|
|
|
845
1682
|
if (this._activeConsumers.size > 0) {
|
|
846
|
-
|
|
1683
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] Re-registering ${this._activeConsumers.size} consumers...`);
|
|
847
1684
|
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
848
1685
|
try {
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
} catch (assertErr) {
|
|
859
|
-
if (assertErr.code === 404) {
|
|
860
|
-
console.warn(`[RabbitMQClient] [mq-client-core] Queue ${queue} does not exist, skipping consumer re-registration`);
|
|
861
|
-
this._activeConsumers.delete(queue);
|
|
862
|
-
failed++;
|
|
863
|
-
this.emit('consumer:re-registration:failed', { queue, error: 'Queue does not exist', code: 404 });
|
|
864
|
-
continue;
|
|
865
|
-
}
|
|
866
|
-
// 406 error means queue exists with different args - skip this consumer
|
|
867
|
-
if (assertErr.code === 406) {
|
|
868
|
-
console.warn(`[RabbitMQClient] [mq-client-core] Queue ${queue} exists with different args, skipping consumer re-registration`);
|
|
869
|
-
this._activeConsumers.delete(queue);
|
|
870
|
-
failed++;
|
|
871
|
-
this.emit('consumer:re-registration:failed', { queue, error: 'Queue exists with different arguments', code: 406 });
|
|
872
|
-
continue;
|
|
873
|
-
}
|
|
874
|
-
throw assertErr;
|
|
875
|
-
}
|
|
876
|
-
}
|
|
877
|
-
|
|
1686
|
+
await this._prepareQueueForConsume(queue, {
|
|
1687
|
+
durable: consumerInfo.options.durable,
|
|
1688
|
+
queueOptions: consumerInfo.options.queueOptions
|
|
1689
|
+
});
|
|
1690
|
+
|
|
1691
|
+
// The consumer's prefetch is re-applied here for the same reason the
|
|
1692
|
+
// queue step is: the channel is new, and QoS lives on the channel.
|
|
1693
|
+
await this._applyConsumerPrefetch(queue, consumerInfo.options.prefetch);
|
|
1694
|
+
|
|
878
1695
|
const consumeResult = await this._consumerChannel.consume(
|
|
879
1696
|
queue,
|
|
880
1697
|
consumerInfo.handler,
|
|
@@ -882,14 +1699,31 @@ class RabbitMQClient extends EventEmitter {
|
|
|
882
1699
|
);
|
|
883
1700
|
consumerInfo.consumerTag = consumeResult.consumerTag;
|
|
884
1701
|
reRegistered++;
|
|
885
|
-
|
|
1702
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] \u2713 Re-registered consumer for queue: ${queue} (consumerTag: ${consumeResult.consumerTag})`);
|
|
886
1703
|
} catch (err) {
|
|
887
1704
|
failed++;
|
|
888
|
-
|
|
889
|
-
//
|
|
890
|
-
|
|
891
|
-
//
|
|
892
|
-
|
|
1705
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Failed to re-register consumer for queue ${queue}:`, err.message);
|
|
1706
|
+
// The consumer STAYS tracked. Whatever refused it — a 406 against the
|
|
1707
|
+
// owner's declaration, a missing infrastructure queue, a channel that
|
|
1708
|
+
// went away mid-flight — is a condition somebody fixes at its source,
|
|
1709
|
+
// and the next recreate must find the consumer still on the list.
|
|
1710
|
+
//
|
|
1711
|
+
// Its TAG does not stay, and the two are not the same fact (d.386b). A
|
|
1712
|
+
// consumer tag names an attachment on ONE channel; the channel that
|
|
1713
|
+
// issued this one is gone and the new one refused to re-attach, so the
|
|
1714
|
+
// string the entry carried names nothing on the broker. Two readers
|
|
1715
|
+
// took it for an attachment that exists: `_performHealthCheck()` counts
|
|
1716
|
+
// `active` by "is the tag set", so a client whose only consumer had
|
|
1717
|
+
// just been refused reported `healthy: true` and `1/1 consumers active`
|
|
1718
|
+
// — the false OK d.386 exists to end, one layer below where d.386 could
|
|
1719
|
+
// reach it; and `cancelConsumer()` sent `basic.cancel` with it, asking
|
|
1720
|
+
// the broker to stop a consumer it does not hold.
|
|
1721
|
+
consumerInfo.consumerTag = null;
|
|
1722
|
+
// The broker's own code is what a listener acts on (406 = the queue
|
|
1723
|
+
// exists with different arguments), and the rail wraps it in a typed
|
|
1724
|
+
// error, so it is read from the cause when the wrapper carries none.
|
|
1725
|
+
const code = err.code !== undefined ? err.code : (err.cause ? err.cause.code : undefined);
|
|
1726
|
+
this.emit('consumer:re-registration:failed', { queue, error: err.message, code });
|
|
893
1727
|
}
|
|
894
1728
|
}
|
|
895
1729
|
}
|
|
@@ -913,13 +1747,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
913
1747
|
try {
|
|
914
1748
|
await this._performHealthCheck();
|
|
915
1749
|
} catch (err) {
|
|
916
|
-
|
|
1750
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Health check error:`, err.message);
|
|
917
1751
|
// Emit event for external monitoring
|
|
918
1752
|
this.emit('health:check:error', err);
|
|
919
1753
|
}
|
|
920
1754
|
}, this._healthCheckIntervalMs);
|
|
921
1755
|
|
|
922
|
-
|
|
1756
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] Health monitoring started (interval: ${this._healthCheckIntervalMs}ms)`);
|
|
923
1757
|
}
|
|
924
1758
|
|
|
925
1759
|
/**
|
|
@@ -930,7 +1764,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
930
1764
|
if (this._healthCheckInterval) {
|
|
931
1765
|
clearInterval(this._healthCheckInterval);
|
|
932
1766
|
this._healthCheckInterval = null;
|
|
933
|
-
|
|
1767
|
+
this._logger.info('[RabbitMQClient] [mq-client-core] Health monitoring stopped');
|
|
934
1768
|
}
|
|
935
1769
|
}
|
|
936
1770
|
|
|
@@ -941,6 +1775,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
941
1775
|
* @returns {Promise<Object>} Health check results
|
|
942
1776
|
*/
|
|
943
1777
|
async performHealthCheck() {
|
|
1778
|
+
// A health check by the SERVICE is a use, and the third the contract names:
|
|
1779
|
+
// it starts the next recovery cycle if this client stood down (d.339). It
|
|
1780
|
+
// does not WAIT for it — a health check owes its caller the state now, not
|
|
1781
|
+
// after a backoff — so the answer it returns is "not connected, recovering",
|
|
1782
|
+
// and the next one tells whether the cycle worked.
|
|
1783
|
+
//
|
|
1784
|
+
// The internal monitor calls `_performHealthCheck()` instead, deliberately:
|
|
1785
|
+
// a timer resuming recovery would make the retry periodic, which is the
|
|
1786
|
+
// opposite of the lazy contract.
|
|
1787
|
+
this._resumeRecovery();
|
|
944
1788
|
return this._performHealthCheck();
|
|
945
1789
|
}
|
|
946
1790
|
|
|
@@ -952,22 +1796,26 @@ class RabbitMQClient extends EventEmitter {
|
|
|
952
1796
|
async _performHealthCheck() {
|
|
953
1797
|
const health = {
|
|
954
1798
|
timestamp: new Date().toISOString(),
|
|
1799
|
+
// Liveness comes from the ONE source the handlers set first-hand
|
|
1800
|
+
// (`isConnected()` / `_isChannelAlive()`, d.260) — never from `closed`, a
|
|
1801
|
+
// property amqplib does not define, which read `undefined` and made this
|
|
1802
|
+
// check report `healthy: true` for a channel the broker had killed.
|
|
955
1803
|
connection: {
|
|
956
1804
|
exists: !!this._connection,
|
|
957
|
-
closed: this.
|
|
1805
|
+
closed: !this.isConnected()
|
|
958
1806
|
},
|
|
959
1807
|
channels: {
|
|
960
1808
|
publisher: {
|
|
961
1809
|
exists: !!this._channel,
|
|
962
|
-
closed: this.
|
|
1810
|
+
closed: !this._isChannelAlive(this._channel)
|
|
963
1811
|
},
|
|
964
1812
|
queue: {
|
|
965
1813
|
exists: !!this._queueChannel,
|
|
966
|
-
closed: this.
|
|
1814
|
+
closed: !this._isChannelAlive(this._queueChannel)
|
|
967
1815
|
},
|
|
968
1816
|
consumer: {
|
|
969
1817
|
exists: !!this._consumerChannel,
|
|
970
|
-
closed: this.
|
|
1818
|
+
closed: !this._isChannelAlive(this._consumerChannel)
|
|
971
1819
|
}
|
|
972
1820
|
},
|
|
973
1821
|
consumers: {
|
|
@@ -1005,7 +1853,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1005
1853
|
}
|
|
1006
1854
|
|
|
1007
1855
|
// Check tracked consumers - verify queues exist and have active consumers
|
|
1008
|
-
if (this.
|
|
1856
|
+
if (this.isConnected() && this._isChannelAlive(this._queueChannel)) {
|
|
1009
1857
|
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
1010
1858
|
health.queues.checked++;
|
|
1011
1859
|
try {
|
|
@@ -1013,9 +1861,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1013
1861
|
this._trackChannelOperation(this._queueChannel, `checkQueue ${queue}`);
|
|
1014
1862
|
const queueInfo = await this._queueChannel.checkQueue(queue);
|
|
1015
1863
|
if (queueInfo) {
|
|
1016
|
-
// Queue exists -
|
|
1017
|
-
//
|
|
1018
|
-
//
|
|
1864
|
+
// Queue exists - is THIS client's consumer attached to it?
|
|
1865
|
+
//
|
|
1866
|
+
// The tag is what answers, because it is the only fact that is about
|
|
1867
|
+
// this client. `checkQueue()` does return `consumerCount` — the
|
|
1868
|
+
// comment that stood here said RabbitMQ offers no way to ask, which
|
|
1869
|
+
// is not true — but that number counts every consumer on the queue,
|
|
1870
|
+
// this client's and everybody else's alike, so it cannot say whether
|
|
1871
|
+
// the one asking is among them. The tag can: it is the name the
|
|
1872
|
+
// broker gave THIS client's attachment, and since d.386b it is set
|
|
1873
|
+
// only while that attachment exists.
|
|
1019
1874
|
if (consumerInfo.consumerTag) {
|
|
1020
1875
|
health.consumers.active++;
|
|
1021
1876
|
} else {
|
|
@@ -1042,18 +1897,43 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1042
1897
|
}
|
|
1043
1898
|
}
|
|
1044
1899
|
|
|
1900
|
+
// A client that TRACKS consumers and has NONE attached consumes nothing, and
|
|
1901
|
+
// that is its own verdict — not something a reader infers from two numbers
|
|
1902
|
+
// sitting beside the channel findings. The per-queue issues above name WHICH
|
|
1903
|
+
// queue lost its consumer, and they do not always exist: when the probe loop
|
|
1904
|
+
// does not run (no connection, or a dead queue channel) the snapshot's only
|
|
1905
|
+
// consumer evidence is `tracked` and `active`, and every issue names a
|
|
1906
|
+
// channel instead. Measured 2026-09-08 (INFRA-monitoring): the whole account
|
|
1907
|
+
// a reader got of a client taking no message off any queue was the line
|
|
1908
|
+
// `Health check OK: 0/0 consumers active`.
|
|
1909
|
+
//
|
|
1910
|
+
// `tracked === 0` is a different client, not a lesser failure: a publisher
|
|
1911
|
+
// registered no consumer and has none to lose. It stays healthy, and the OK
|
|
1912
|
+
// line below says which of the two it is.
|
|
1913
|
+
if (health.consumers.tracked > 0 && health.consumers.active === 0) {
|
|
1914
|
+
health.healthy = false;
|
|
1915
|
+
health.issues.push(
|
|
1916
|
+
'[RabbitMQClient] No consumer of this client is attached - '
|
|
1917
|
+
+ `${health.consumers.tracked} consumer(s) tracked, 0 attached. `
|
|
1918
|
+
+ 'Expected: every consumer this client registered attached to the consumer channel. '
|
|
1919
|
+
+ 'Fix: none here - the issues beside this one name what stopped them (a dead channel, a '
|
|
1920
|
+
+ 'missing queue, a re-registration the broker refused); until one of those is resolved '
|
|
1921
|
+
+ 'this client takes no message off any queue.'
|
|
1922
|
+
);
|
|
1923
|
+
}
|
|
1924
|
+
|
|
1045
1925
|
// Emit health check result
|
|
1046
1926
|
this.emit('health:check', health);
|
|
1047
1927
|
|
|
1048
1928
|
// If unhealthy, log and potentially trigger shutdown
|
|
1049
1929
|
if (!health.healthy) {
|
|
1050
|
-
|
|
1051
|
-
|
|
1930
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Health check FAILED:`, health.issues);
|
|
1931
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Health details:`, JSON.stringify(health, null, 2));
|
|
1052
1932
|
|
|
1053
1933
|
// Track critical health start time
|
|
1054
1934
|
if (!this._criticalHealthStartTime) {
|
|
1055
1935
|
this._criticalHealthStartTime = Date.now();
|
|
1056
|
-
|
|
1936
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Critical health detected at ${new Date(this._criticalHealthStartTime).toISOString()}`);
|
|
1057
1937
|
}
|
|
1058
1938
|
|
|
1059
1939
|
// Report to monitoring if callback provided
|
|
@@ -1061,7 +1941,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1061
1941
|
try {
|
|
1062
1942
|
await this._healthReportCallback(health);
|
|
1063
1943
|
} catch (reportErr) {
|
|
1064
|
-
|
|
1944
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Health report callback failed:`, reportErr.message);
|
|
1065
1945
|
}
|
|
1066
1946
|
}
|
|
1067
1947
|
|
|
@@ -1070,7 +1950,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1070
1950
|
try {
|
|
1071
1951
|
await this._healthCriticalCallback(health);
|
|
1072
1952
|
} catch (criticalErr) {
|
|
1073
|
-
|
|
1953
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Critical health callback failed:`, criticalErr.message);
|
|
1074
1954
|
}
|
|
1075
1955
|
}
|
|
1076
1956
|
|
|
@@ -1081,31 +1961,57 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1081
1961
|
if (this._criticalHealthShutdown) {
|
|
1082
1962
|
const timeSinceCritical = Date.now() - this._criticalHealthStartTime;
|
|
1083
1963
|
if (timeSinceCritical >= this._criticalHealthShutdownDelay) {
|
|
1084
|
-
|
|
1964
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Critical health persisted for ${timeSinceCritical}ms (threshold: ${this._criticalHealthShutdownDelay}ms) - triggering shutdown`);
|
|
1085
1965
|
this.emit('health:shutdown', health);
|
|
1086
1966
|
// Note: Actual shutdown should be handled by the application using this event
|
|
1087
1967
|
} else {
|
|
1088
1968
|
const remaining = this._criticalHealthShutdownDelay - timeSinceCritical;
|
|
1089
|
-
|
|
1969
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Critical health will trigger shutdown in ${remaining}ms if not resolved`);
|
|
1090
1970
|
}
|
|
1091
1971
|
}
|
|
1092
1972
|
} else {
|
|
1093
1973
|
// Health is OK - reset critical health tracking
|
|
1094
1974
|
if (this._criticalHealthStartTime) {
|
|
1095
1975
|
const duration = Date.now() - this._criticalHealthStartTime;
|
|
1096
|
-
|
|
1976
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] Health recovered after ${duration}ms of critical state`);
|
|
1097
1977
|
this._criticalHealthStartTime = null;
|
|
1098
1978
|
}
|
|
1099
1979
|
|
|
1100
|
-
|
|
1980
|
+
// `0/0` said two different things and distinguished neither: a publisher
|
|
1981
|
+
// that never registered a consumer, and a client whose consumers are all
|
|
1982
|
+
// gone. The second one no longer reaches this branch at all (it is a
|
|
1983
|
+
// failure now), so the line only has to name the first one plainly.
|
|
1984
|
+
const consumers = health.consumers.tracked === 0
|
|
1985
|
+
? 'no consumers registered - this client only publishes'
|
|
1986
|
+
: `${health.consumers.active}/${health.consumers.tracked} consumers active`;
|
|
1987
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] Health check OK: ${consumers}, ${health.queues.checked} queues checked`);
|
|
1101
1988
|
}
|
|
1102
1989
|
|
|
1103
1990
|
return health;
|
|
1104
1991
|
}
|
|
1105
1992
|
|
|
1993
|
+
/**
|
|
1994
|
+
* End this client: stop recovery, stop monitoring, close the three channels
|
|
1995
|
+
* and the connection, in reverse creation order.
|
|
1996
|
+
*
|
|
1997
|
+
* CONTRACT — an intentional close is not a fault. From the first line of this
|
|
1998
|
+
* method the client is closed BY ITS CALLER, and that state is terminal: no
|
|
1999
|
+
* `close` frame, no `error` frame and no recovery round already in flight
|
|
2000
|
+
* brings the connection back. When this method resolves, the client holds no
|
|
2001
|
+
* timer, no socket and no channel, `isConnected()` answers `false`, and the
|
|
2002
|
+
* process is free to exit. Only an explicit `connect()` re-arms recovery.
|
|
2003
|
+
*
|
|
2004
|
+
* Idempotent: a second call finds nothing left to close and does nothing.
|
|
2005
|
+
*
|
|
2006
|
+
* @returns {Promise<void>}
|
|
2007
|
+
*/
|
|
1106
2008
|
async disconnect() {
|
|
1107
|
-
this.
|
|
1108
|
-
|
|
2009
|
+
this._closedByCaller = true;
|
|
2010
|
+
// From this point the client is not connected, whatever the sockets are still
|
|
2011
|
+
// doing: the state says so before the first `close()` is awaited, so nothing
|
|
2012
|
+
// reads "alive" out of a teardown in progress.
|
|
2013
|
+
this._connectionAlive = false;
|
|
2014
|
+
|
|
1109
2015
|
// Clear all active timers (reconnection, waits, etc.)
|
|
1110
2016
|
this._clearAllTimers();
|
|
1111
2017
|
|
|
@@ -1121,95 +2027,476 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1121
2027
|
// Clear prefetch tracking
|
|
1122
2028
|
this._prefetchTracking.clear();
|
|
1123
2029
|
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
2030
|
+
// ONE budget for the whole teardown. Every close below waits for an answer
|
|
2031
|
+
// from the broker — a `close-ok` frame — and a socket that is OPEN but
|
|
2032
|
+
// carries nothing never delivers one; amqplib clears its heartbeat watchdog
|
|
2033
|
+
// the moment a close begins, so nothing else ends the wait either. Measured
|
|
2034
|
+
// through a TCP proxy that keeps the socket open and forwards nothing:
|
|
2035
|
+
// `disconnect()` was still pending after 335 s (d.287). The budget is one
|
|
2036
|
+
// deadline for the teardown, not one per handle — the contract is that
|
|
2037
|
+
// `disconnect()` ends, not that each of four closes ends.
|
|
2038
|
+
const teardownDeadline = Date.now() + this._brokerAnswerTimeout;
|
|
2039
|
+
const remaining = () => Math.max(0, teardownDeadline - Date.now());
|
|
2040
|
+
let socketUnanswered = false;
|
|
2041
|
+
const connection = this._connection;
|
|
2042
|
+
|
|
2043
|
+
const consumerOutcome = await this._closeWithinTeardownBudget(
|
|
2044
|
+
'consumer channel',
|
|
2045
|
+
this._consumerChannel,
|
|
2046
|
+
remaining()
|
|
2047
|
+
);
|
|
2048
|
+
this._consumerChannel = null;
|
|
2049
|
+
if (consumerOutcome.status === 'refused') {
|
|
2050
|
+
this._logger.warn('[RabbitMQClient] Error closing consumer channel:', consumerOutcome.error.message);
|
|
1131
2051
|
}
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
2052
|
+
socketUnanswered = socketUnanswered || consumerOutcome.status === 'unanswered';
|
|
2053
|
+
|
|
2054
|
+
const queueOutcome = await this._closeWithinTeardownBudget(
|
|
2055
|
+
'queue channel',
|
|
2056
|
+
this._queueChannel,
|
|
2057
|
+
remaining()
|
|
2058
|
+
);
|
|
2059
|
+
this._queueChannel = null;
|
|
2060
|
+
if (queueOutcome.status === 'refused') {
|
|
1138
2061
|
// Log but don't emit - queue channel errors are less critical
|
|
1139
|
-
|
|
2062
|
+
this._logger.warn('[RabbitMQClient] Error closing queue channel:', queueOutcome.error.message);
|
|
1140
2063
|
}
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
2064
|
+
socketUnanswered = socketUnanswered || queueOutcome.status === 'unanswered';
|
|
2065
|
+
|
|
2066
|
+
const publisherOutcome = await this._closeWithinTeardownBudget(
|
|
2067
|
+
'publisher channel',
|
|
2068
|
+
this._channel,
|
|
2069
|
+
remaining()
|
|
2070
|
+
);
|
|
2071
|
+
this._channel = null;
|
|
2072
|
+
if (publisherOutcome.status === 'refused') {
|
|
2073
|
+
this.emit('error', publisherOutcome.error);
|
|
1148
2074
|
}
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
2075
|
+
socketUnanswered = socketUnanswered || publisherOutcome.status === 'unanswered';
|
|
2076
|
+
|
|
2077
|
+
const connectionOutcome = await this._closeWithinTeardownBudget(
|
|
2078
|
+
'connection',
|
|
2079
|
+
this._connection,
|
|
2080
|
+
remaining()
|
|
2081
|
+
);
|
|
2082
|
+
this._connection = null;
|
|
2083
|
+
if (connectionOutcome.status === 'refused') {
|
|
1155
2084
|
// Don't emit error for expected connection closure during disconnect
|
|
1156
2085
|
// This is a predictable, intentional closure
|
|
1157
|
-
if (
|
|
1158
|
-
|
|
2086
|
+
if (connectionOutcome.error.message && connectionOutcome.error.message.includes('Connection closed (by client)')) {
|
|
2087
|
+
this._logger.info('[RabbitMQClient] Connection closed during disconnect (expected)');
|
|
1159
2088
|
} else {
|
|
1160
|
-
this.emit('error',
|
|
2089
|
+
this.emit('error', connectionOutcome.error);
|
|
1161
2090
|
}
|
|
1162
2091
|
}
|
|
1163
|
-
|
|
2092
|
+
socketUnanswered = socketUnanswered || connectionOutcome.status === 'unanswered';
|
|
2093
|
+
|
|
2094
|
+
// A teardown that returns while its socket is still open keeps the process
|
|
2095
|
+
// alive — the very defect d.283 and d.285 were both about. So when the
|
|
2096
|
+
// broker never answered, the socket is not left behind: amqplib would have
|
|
2097
|
+
// ended it itself on a graceful close, and only then.
|
|
2098
|
+
if (socketUnanswered) {
|
|
2099
|
+
this._dropUnansweredConnection(connection);
|
|
2100
|
+
}
|
|
2101
|
+
// Second sweep, and the one that matters: the first happened before the
|
|
2102
|
+
// awaits above, and anything the transport scheduled while they ran — a
|
|
2103
|
+
// backoff wait, a connect timeout — would otherwise outlive the teardown
|
|
2104
|
+
// that was supposed to end it.
|
|
2105
|
+
this._clearAllTimers();
|
|
1164
2106
|
}
|
|
1165
2107
|
|
|
1166
2108
|
/**
|
|
1167
2109
|
* Ensure a queue exists with correct parameters (TTL/DLQ/etc).
|
|
1168
|
-
*
|
|
2110
|
+
*
|
|
2111
|
+
* PUBLIC: this is the one operation a collaborator needs in order to declare a
|
|
2112
|
+
* queue on this client — higher-level routing libraries (cookbook-router's
|
|
2113
|
+
* QueueManager) and `workers/RecoveryWorker.js` alike. It owns the whole
|
|
2114
|
+
* sequence, so nobody has to assemble it out of private members: ensure the
|
|
2115
|
+
* queue channel, refuse a channel that is not alive, resolve the arguments
|
|
2116
|
+
* from the central `queueConfig`, declare.
|
|
2117
|
+
*
|
|
2118
|
+
* OPTIONS ARE A DECLARED SET, and everything outside it is refused by name.
|
|
2119
|
+
* Until d.396c this method rebuilt `queueOptions` from `durable` and
|
|
2120
|
+
* `arguments` and dropped every other key the caller wrote, without a word:
|
|
2121
|
+
* a suite asking for `{ durable: false, exclusive: true }` got a queue the
|
|
2122
|
+
* broker reported as `"exclusive": false`, believed it had declared one that
|
|
2123
|
+
* dies with its connection, and left it standing with 0 consumers when the run
|
|
2124
|
+
* was interrupted. A silently ignored option is worse than a rejected one —
|
|
2125
|
+
* the caller acts on a guarantee nobody gave (`architecture-principles.md` §4
|
|
2126
|
+
* fail-fast, §8 explicit over implicit). The refusal is the shape the config
|
|
2127
|
+
* schema has used since d.291b: the key by name, and the declared key it is
|
|
2128
|
+
* closest to (`utils/nearestKey.js`, shared with `BaseClient`).
|
|
2129
|
+
*
|
|
2130
|
+
* `exclusive` and `autoDelete` joined `durable` and `arguments` in that set
|
|
2131
|
+
* because they are how a caller states a queue's LIFETIME, and no other rail
|
|
2132
|
+
* offers it: a temporary queue that does not die with the thing that made it
|
|
2133
|
+
* is residue on a shared broker.
|
|
2134
|
+
*
|
|
2135
|
+
* The central `queueConfig` keeps its ownership undiminished. For an
|
|
2136
|
+
* infrastructure or business name it decides `durable` and `arguments` as
|
|
2137
|
+
* before — and a LIFETIME asked for on such a name is refused rather than
|
|
2138
|
+
* quietly overruled, because those queues outlive the connection that declares
|
|
2139
|
+
* them by design.
|
|
1169
2140
|
*
|
|
1170
2141
|
* @param {string} queue - Queue name
|
|
1171
|
-
* @param {Object} [options]
|
|
2142
|
+
* @param {Object} [options] - `durable`, `arguments`, `exclusive`, `autoDelete`
|
|
2143
|
+
* for a queue the central config does not classify; for an infrastructure or
|
|
2144
|
+
* business queue the config's own declaration wins over `durable` and
|
|
2145
|
+
* `arguments`, and the two lifetime keys are refused. Any other key is refused.
|
|
1172
2146
|
* @returns {Promise<Object>} amqplib assertQueue result
|
|
2147
|
+
* @throws {Error} If `options` carries a key this operation does not read.
|
|
1173
2148
|
*/
|
|
1174
2149
|
async assertQueue(queue, options = {}) {
|
|
2150
|
+
// Both refusals come BEFORE the channel is touched: an option this operation
|
|
2151
|
+
// cannot honour is a caller mistake, and the broker has nothing to do with it.
|
|
2152
|
+
this._refuseUndeclaredQueueOptions(options);
|
|
2153
|
+
|
|
2154
|
+
// Central queueConfig decides the arguments (fail-fast queue argument consistency).
|
|
2155
|
+
// The try/catch that used to wrap this block existed only for the lazy require —
|
|
2156
|
+
// its own comment said so ("If queueConfig cannot be loaded") — and with the module
|
|
2157
|
+
// loaded at the top there is nothing left for it to catch except a genuine error
|
|
2158
|
+
// from the config itself, which must not be swallowed.
|
|
2159
|
+
// ONE function answers "what is this queue declared with", for every path that
|
|
2160
|
+
// declares a queue — this rail and the consumer's `_prepareQueueForConsume()`
|
|
2161
|
+
// (d.410). Until then each rebuilt `{ durable: cfg.durable !== false,
|
|
2162
|
+
// arguments: { ...cfg.arguments } }` from its own lookup, which is one concern
|
|
2163
|
+
// on two rails (`change-discipline.md` § One rail per concern): edit one copy
|
|
2164
|
+
// and the broker answers whichever declarer it hears second with
|
|
2165
|
+
// `406 PRECONDITION_FAILED`.
|
|
2166
|
+
const declared = queueConfig.declarationOptions(queue);
|
|
2167
|
+
|
|
2168
|
+
let queueOptions;
|
|
2169
|
+
|
|
2170
|
+
if (declared !== null) {
|
|
2171
|
+
this._refuseCallerLifetime(queue, options);
|
|
2172
|
+
queueOptions = declared;
|
|
2173
|
+
} else {
|
|
2174
|
+
queueOptions = {
|
|
2175
|
+
durable: options.durable !== false,
|
|
2176
|
+
arguments: { ...(options.arguments || {}) }
|
|
2177
|
+
};
|
|
2178
|
+
|
|
2179
|
+
// Omitted, never sent as `false`: amqplib leaves out what it is not given,
|
|
2180
|
+
// and an explicit `false` would make every re-declaration of an existing
|
|
2181
|
+
// queue carry a value its first declaration never had.
|
|
2182
|
+
for (const key of LIFETIME_QUEUE_OPTION_KEYS) {
|
|
2183
|
+
if (options[key] !== undefined) {
|
|
2184
|
+
queueOptions[key] = options[key];
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
const channel = await this._requireQueueChannel('assertQueue', queue);
|
|
2190
|
+
|
|
2191
|
+
return await channel.assertQueue(queue, queueOptions);
|
|
2192
|
+
}
|
|
2193
|
+
|
|
2194
|
+
/**
|
|
2195
|
+
* Refuse an option `assertQueue()` does not read, by name.
|
|
2196
|
+
*
|
|
2197
|
+
* @param {Object} options - what the caller passed.
|
|
2198
|
+
* @throws {Error} naming every undeclared key and the declared key it is closest to.
|
|
2199
|
+
* @private
|
|
2200
|
+
*/
|
|
2201
|
+
_refuseUndeclaredQueueOptions(options) {
|
|
2202
|
+
const unknown = Object.keys(options).filter(
|
|
2203
|
+
(key) => !ASSERT_QUEUE_OPTION_KEYS.includes(key)
|
|
2204
|
+
);
|
|
2205
|
+
|
|
2206
|
+
if (unknown.length === 0) {
|
|
2207
|
+
return;
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
throw new Error(
|
|
2211
|
+
`[RabbitMQClient] Queue option not declared by assertQueue(): `
|
|
2212
|
+
+ `${nameUnknownKeys(unknown, ASSERT_QUEUE_OPTION_KEYS)} - `
|
|
2213
|
+
+ `Expected: only ${ASSERT_QUEUE_OPTION_KEYS.join(', ')}. `
|
|
2214
|
+
+ 'Fix: correct the spelling, or remove the key - an option this operation does not read '
|
|
2215
|
+
+ 'declares nothing, and the queue the broker creates is not the queue the caller asked for.'
|
|
2216
|
+
);
|
|
2217
|
+
}
|
|
2218
|
+
|
|
2219
|
+
/**
|
|
2220
|
+
* Refuse a lifetime asked for on a queue whose declaration the central config
|
|
2221
|
+
* owns. Overruling it in silence is the defect this whole method was fixed for,
|
|
2222
|
+
* one name further along.
|
|
2223
|
+
*
|
|
2224
|
+
* @param {string} queue - the classified queue name.
|
|
2225
|
+
* @param {Object} options - what the caller passed.
|
|
2226
|
+
* @throws {Error} naming the lifetime keys the caller wrote.
|
|
2227
|
+
* @private
|
|
2228
|
+
*/
|
|
2229
|
+
_refuseCallerLifetime(queue, options) {
|
|
2230
|
+
const asked = LIFETIME_QUEUE_OPTION_KEYS.filter((key) => options[key] !== undefined);
|
|
2231
|
+
|
|
2232
|
+
if (asked.length === 0) {
|
|
2233
|
+
return;
|
|
2234
|
+
}
|
|
2235
|
+
|
|
2236
|
+
throw new Error(
|
|
2237
|
+
`[RabbitMQClient] Queue lifetime is not the caller's to set for "${queue}": `
|
|
2238
|
+
+ `${asked.join(', ')} - Expected: the central queueConfig owns the declaration of every `
|
|
2239
|
+
+ 'infrastructure and business queue, including how long it lives. '
|
|
2240
|
+
+ 'Fix: drop the option - this queue outlives the connection that declares it, by design; '
|
|
2241
|
+
+ "a queue whose lifetime is the caller's is one queueConfig does not classify."
|
|
2242
|
+
);
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
/**
|
|
2246
|
+
* The one rail to the queue channel: ensure it, refuse it unless it is alive,
|
|
2247
|
+
* record the operation for the post-mortem, hand it back.
|
|
2248
|
+
*
|
|
2249
|
+
* All four public queue operations (`assertQueue`, `checkQueue`, `purgeQueue`,
|
|
2250
|
+
* `deleteQueue`) share one precondition, so they share one implementation of
|
|
2251
|
+
* it. `_ensureQueueChannel()` alone is not that precondition: it returns
|
|
2252
|
+
* without recreating anything while the connection is down, leaving the dead
|
|
2253
|
+
* reference in place — so EXISTENCE of the reference is not the question,
|
|
2254
|
+
* liveness is, read from the one source (`_isChannelAlive`, d.260).
|
|
2255
|
+
*
|
|
2256
|
+
* Until d.282 only `assertQueue()` asked that question (d.277b). The other
|
|
2257
|
+
* three asked `if (!this._queueChannel)`, which a corpse answers exactly like
|
|
2258
|
+
* a live channel, and then ran the operation on it — the caller got amqplib's
|
|
2259
|
+
* bare "Channel closed" instead of a sentence naming the fix
|
|
2260
|
+
* (`architecture-principles.md` §5). `checkQueue()` is also the operation that
|
|
2261
|
+
* CAUSES those deaths: a 404 on a missing queue is a channel-level failure and
|
|
2262
|
+
* closes the channel by AMQP rule.
|
|
2263
|
+
*
|
|
2264
|
+
* `_trackChannelOperation()` belongs here for the same reason: the channels
|
|
2265
|
+
* most likely to die are the ones a queue operation is running on, and until
|
|
2266
|
+
* this rail existed three of the four left `channel._lastOperation` unwritten,
|
|
2267
|
+
* so the post-mortem in `getChannelState()` had nothing to report.
|
|
2268
|
+
*
|
|
2269
|
+
* @param {string} verb - the public operation asking, used in the refusal and
|
|
2270
|
+
* in `channel._lastOperation`
|
|
2271
|
+
* @param {string} queue - the queue it is about
|
|
2272
|
+
* @returns {Promise<Object>} the live queue channel
|
|
2273
|
+
* @throws {ConnectionError} when no live queue channel can be had
|
|
2274
|
+
* @private
|
|
2275
|
+
*/
|
|
2276
|
+
async _requireQueueChannel(verb, queue) {
|
|
1175
2277
|
await this._ensureQueueChannel();
|
|
1176
|
-
if (!this._queueChannel) {
|
|
1177
|
-
throw new
|
|
2278
|
+
if (!this._isChannelAlive(this._queueChannel)) {
|
|
2279
|
+
throw new ConnectionError(
|
|
2280
|
+
`[RabbitMQClient] Cannot ${verb} ${queue}: queue channel is not available - `
|
|
2281
|
+
+ 'Expected: an open queue channel after _ensureQueueChannel(). '
|
|
2282
|
+
+ `Fix: await client.connect() before ${verb}(); if the connection dropped, wait for `
|
|
2283
|
+
+ 'the reconnect to recreate the channel and run the operation again.'
|
|
2284
|
+
);
|
|
2285
|
+
}
|
|
2286
|
+
this._trackChannelOperation(this._queueChannel, `${verb} ${queue}`);
|
|
2287
|
+
return this._queueChannel;
|
|
2288
|
+
}
|
|
2289
|
+
|
|
2290
|
+
/**
|
|
2291
|
+
* The publish path's dialect of the queue-channel rail's refusal.
|
|
2292
|
+
*
|
|
2293
|
+
* `_requireQueueChannel()` answers `ConnectionError`, which on every other
|
|
2294
|
+
* path means "this is over". On the publish path it means the opposite: the
|
|
2295
|
+
* channel is missing because the connection is being restored, so the message
|
|
2296
|
+
* is not lost — PublishLayer buffers it and retries after the reconnect. That
|
|
2297
|
+
* is why the publish path translates rather than propagates.
|
|
2298
|
+
*
|
|
2299
|
+
* ONE place builds that translation, because there are two sites needing it
|
|
2300
|
+
* (the pre-publish existence check and the declaration of a missing ownerless
|
|
2301
|
+
* queue) and a sentence maintained twice diverges.
|
|
2302
|
+
*
|
|
2303
|
+
* @param {string} verb - the operation the publish path was about to run
|
|
2304
|
+
* @param {string} queue
|
|
2305
|
+
* @param {ConnectionError} cause - the rail's refusal
|
|
2306
|
+
* @returns {TransientPublishError}
|
|
2307
|
+
* @private
|
|
2308
|
+
*/
|
|
2309
|
+
_publishPathTransient(verb, queue, cause) {
|
|
2310
|
+
return new TransientPublishError(
|
|
2311
|
+
`[RabbitMQClient] Cannot ${verb} ${queue}: queue channel is not available (connection may be closed) - `
|
|
2312
|
+
+ `Expected: an open queue channel before ${verb} on the publish path. `
|
|
2313
|
+
+ 'Fix: retry the publish after the reconnect; PublishLayer does this on its own and buffers the message meanwhile.',
|
|
2314
|
+
cause
|
|
2315
|
+
);
|
|
2316
|
+
}
|
|
2317
|
+
|
|
2318
|
+
/**
|
|
2319
|
+
* Declare an exchange.
|
|
2320
|
+
*
|
|
2321
|
+
* Public since d.302: confirmation `mq-client-lifecycle-contract` 001 point 4
|
|
2322
|
+
* ("L8") names it as one of the four operations a service must be able to
|
|
2323
|
+
* reach through this library. Until then the only `assertExchange` in the
|
|
2324
|
+
* package was a private line on the publish path, so a service that had to
|
|
2325
|
+
* declare an exchange reached around the library for a raw amqplib channel —
|
|
2326
|
+
* the drift `mq-consumer-contract` 001 records as DL-007.
|
|
2327
|
+
*
|
|
2328
|
+
* It runs on the QUEUE channel, like `assertQueue()` and the other three queue
|
|
2329
|
+
* operations: declaring topology is what that channel is for, and a rejected
|
|
2330
|
+
* declaration (406 against an existing exchange of another type) must close
|
|
2331
|
+
* that channel rather than the publisher's ConfirmChannel, which carries
|
|
2332
|
+
* unconfirmed messages.
|
|
2333
|
+
*
|
|
2334
|
+
* @param {string} exchange - Exchange name; a non-empty string.
|
|
2335
|
+
* @param {string} type - `direct`, `topic`, `fanout` or `headers`. Required:
|
|
2336
|
+
* an exchange's type is a topology decision the caller makes, and guessing
|
|
2337
|
+
* one here would declare a different exchange from the one they meant.
|
|
2338
|
+
* @param {Object} [options] - amqplib exchange options. `durable` defaults to
|
|
2339
|
+
* the client's configured `durable`, never to a literal.
|
|
2340
|
+
* @returns {Promise<Object>} amqplib's `assertExchange` result.
|
|
2341
|
+
* @throws {ValidationError} If the name or the type cannot declare an exchange.
|
|
2342
|
+
* @throws {ConnectionError} If no live queue channel can be had.
|
|
2343
|
+
*/
|
|
2344
|
+
async assertExchange(exchange, type, options = {}) {
|
|
2345
|
+
if (typeof exchange !== 'string' || exchange.trim() === '') {
|
|
2346
|
+
throw new ValidationError(
|
|
2347
|
+
'[RabbitMQClient] Invalid exchange name - Expected: a non-empty string, got '
|
|
2348
|
+
+ `${JSON.stringify(exchange)}. Fix: pass assertExchange('<exchange>', '<type>').`
|
|
2349
|
+
);
|
|
2350
|
+
}
|
|
2351
|
+
if (!EXCHANGE_TYPES.includes(type)) {
|
|
2352
|
+
throw new ValidationError(
|
|
2353
|
+
`[RabbitMQClient] Invalid exchange type - Expected one of ${EXCHANGE_TYPES.join(', ')}, got `
|
|
2354
|
+
+ `${JSON.stringify(type)}. Fix: pass assertExchange('${exchange}', '<type>'); an exchange's `
|
|
2355
|
+
+ 'type is a topology decision, so this library does not choose one for you.'
|
|
2356
|
+
);
|
|
1178
2357
|
}
|
|
1179
2358
|
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
2359
|
+
const channel = await this._requireQueueChannel('assertExchange', exchange);
|
|
2360
|
+
const exchangeOptions = {
|
|
2361
|
+
...options,
|
|
2362
|
+
durable: options.durable !== undefined ? options.durable : this._config.durable
|
|
1183
2363
|
};
|
|
2364
|
+
return await channel.assertExchange(exchange, type, exchangeOptions);
|
|
2365
|
+
}
|
|
1184
2366
|
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
2367
|
+
/**
|
|
2368
|
+
* Bind a queue to an exchange under a routing pattern.
|
|
2369
|
+
*
|
|
2370
|
+
* Public since d.302, for the same reason and on the same channel as
|
|
2371
|
+
* `assertExchange()` above. The binding is what makes a fanout or topic
|
|
2372
|
+
* exchange reach a queue at all, so a library that declares exchanges and
|
|
2373
|
+
* queues but cannot bind them leaves every consumer of an exchange-routed
|
|
2374
|
+
* queue reaching for a raw channel.
|
|
2375
|
+
*
|
|
2376
|
+
* @param {string} queue - Queue to bind; a non-empty string.
|
|
2377
|
+
* @param {string} exchange - Exchange to bind it to; a non-empty string.
|
|
2378
|
+
* @param {string} pattern - Routing pattern / key. REQUIRED, and an empty
|
|
2379
|
+
* string is a legal value (a fanout exchange ignores the key): which key a
|
|
2380
|
+
* binding carries is a decision, and defaulting it would bind a queue to
|
|
2381
|
+
* traffic nobody asked for.
|
|
2382
|
+
* @returns {Promise<Object>} amqplib's `bindQueue` result.
|
|
2383
|
+
* @throws {ValidationError} If a name or the pattern cannot form a binding.
|
|
2384
|
+
* @throws {ConnectionError} If no live queue channel can be had.
|
|
2385
|
+
*/
|
|
2386
|
+
async bindQueue(queue, exchange, pattern) {
|
|
2387
|
+
if (typeof queue !== 'string' || queue.trim() === '') {
|
|
2388
|
+
throw new ValidationError(
|
|
2389
|
+
'[RabbitMQClient] Invalid queue name for binding - Expected: a non-empty string, got '
|
|
2390
|
+
+ `${JSON.stringify(queue)}. Fix: pass bindQueue('<queue>', '<exchange>', '<pattern>').`
|
|
2391
|
+
);
|
|
2392
|
+
}
|
|
2393
|
+
if (typeof exchange !== 'string' || exchange.trim() === '') {
|
|
2394
|
+
throw new ValidationError(
|
|
2395
|
+
'[RabbitMQClient] Invalid exchange name for binding - Expected: a non-empty string, got '
|
|
2396
|
+
+ `${JSON.stringify(exchange)}. Fix: pass bindQueue('${queue}', '<exchange>', '<pattern>').`
|
|
2397
|
+
);
|
|
2398
|
+
}
|
|
2399
|
+
if (typeof pattern !== 'string') {
|
|
2400
|
+
throw new ValidationError(
|
|
2401
|
+
'[RabbitMQClient] Invalid binding pattern - Expected: a string (the empty string is legal, '
|
|
2402
|
+
+ `a fanout exchange ignores the key), got ${JSON.stringify(pattern)}. Fix: pass `
|
|
2403
|
+
+ `bindQueue('${queue}', '${exchange}', '<pattern>') — this library does not guess a routing key.`
|
|
2404
|
+
);
|
|
2405
|
+
}
|
|
2406
|
+
|
|
2407
|
+
const channel = await this._requireQueueChannel('bindQueue', `${queue} -> ${exchange}`);
|
|
2408
|
+
return await channel.bindQueue(queue, exchange, pattern);
|
|
2409
|
+
}
|
|
2410
|
+
|
|
2411
|
+
/**
|
|
2412
|
+
* Stop one consumer this client registered, by its consumer tag or by the
|
|
2413
|
+
* queue it consumes.
|
|
2414
|
+
*
|
|
2415
|
+
* Public since d.302 ("L8"). The tag is the broker's name for the consumer and
|
|
2416
|
+
* is what `consume()` returns; the queue name is what the caller usually still
|
|
2417
|
+
* has at hand, so both are accepted and resolved against the ONE registry of
|
|
2418
|
+
* live consumers this client keeps (`_activeConsumers`).
|
|
2419
|
+
*
|
|
2420
|
+
* The consumer is also DROPPED from that registry, not merely cancelled on the
|
|
2421
|
+
* broker. The registry is what connection-level recovery re-attaches after a
|
|
2422
|
+
* reconnect (`_reRegisterConsumers()`), so a cancelled consumer left in it
|
|
2423
|
+
* would come back on the next outage — the caller asked for it to stop, not to
|
|
2424
|
+
* pause.
|
|
2425
|
+
*
|
|
2426
|
+
* @param {string} consumerTagOrQueue - The tag `consume()` returned, or the queue name.
|
|
2427
|
+
* @returns {Promise<Object>} amqplib's `cancel` result.
|
|
2428
|
+
* @throws {ValidationError} If this client has no such consumer, or holds it
|
|
2429
|
+
* tracked but not attached (its re-registration was refused — d.386b).
|
|
2430
|
+
* @throws {ConnectionError} If the consumer channel is not alive.
|
|
2431
|
+
*/
|
|
2432
|
+
async cancelConsumer(consumerTagOrQueue) {
|
|
2433
|
+
if (typeof consumerTagOrQueue !== 'string' || consumerTagOrQueue.trim() === '') {
|
|
2434
|
+
throw new ValidationError(
|
|
2435
|
+
'[RabbitMQClient] Invalid consumer reference - Expected: the consumer tag consume() returned, '
|
|
2436
|
+
+ `or the queue name, got ${JSON.stringify(consumerTagOrQueue)}. `
|
|
2437
|
+
+ "Fix: pass cancelConsumer('<consumerTag>') or cancelConsumer('<queue>')."
|
|
2438
|
+
);
|
|
2439
|
+
}
|
|
2440
|
+
|
|
2441
|
+
let queue = null;
|
|
2442
|
+
if (this._activeConsumers.has(consumerTagOrQueue)) {
|
|
2443
|
+
queue = consumerTagOrQueue;
|
|
2444
|
+
} else {
|
|
2445
|
+
for (const [registeredQueue, info] of this._activeConsumers.entries()) {
|
|
2446
|
+
if (info.consumerTag === consumerTagOrQueue) {
|
|
2447
|
+
queue = registeredQueue;
|
|
2448
|
+
break;
|
|
1205
2449
|
}
|
|
1206
2450
|
}
|
|
1207
|
-
} catch (err) {
|
|
1208
|
-
// If queueConfig cannot be loaded, keep caller-provided options (explicit).
|
|
1209
|
-
// No fallbacks to "random defaults" here.
|
|
1210
2451
|
}
|
|
1211
2452
|
|
|
1212
|
-
|
|
2453
|
+
if (queue === null) {
|
|
2454
|
+
const known = Array.from(this._activeConsumers.entries())
|
|
2455
|
+
.map(([registeredQueue, info]) => `${registeredQueue} (${info.consumerTag})`);
|
|
2456
|
+
throw new ValidationError(
|
|
2457
|
+
`[RabbitMQClient] No consumer to cancel - this client holds no consumer for ${JSON.stringify(consumerTagOrQueue)}. `
|
|
2458
|
+
+ `Expected: a queue name or consumer tag from ${known.length > 0 ? known.join(', ') : 'this client (it holds none)'}. `
|
|
2459
|
+
+ 'Fix: cancel on the client that registered the consumer, or read the tag consume() returned.'
|
|
2460
|
+
);
|
|
2461
|
+
}
|
|
2462
|
+
|
|
2463
|
+
const info = this._activeConsumers.get(queue);
|
|
2464
|
+
|
|
2465
|
+
// Tracked is not attached. A consumer whose re-registration the broker
|
|
2466
|
+
// refused keeps its place on the list — that is how the next channel
|
|
2467
|
+
// recreate finds it (d.262) — but it holds no tag, because the attachment
|
|
2468
|
+
// its tag named is gone (d.386b). There is nothing to cancel on the broker,
|
|
2469
|
+
// and asking anyway is a channel error rather than a cancel, so the caller
|
|
2470
|
+
// is told plainly instead of being handed one.
|
|
2471
|
+
if (info.consumerTag === null || info.consumerTag === undefined) {
|
|
2472
|
+
throw new ValidationError(
|
|
2473
|
+
`[RabbitMQClient] Cannot cancelConsumer ${queue}: the consumer is tracked but not attached - `
|
|
2474
|
+
+ 'Expected: a consumer holding the tag its last basic.consume returned. '
|
|
2475
|
+
+ 'Fix: nothing to cancel on the broker - the attachment is already gone and the client keeps '
|
|
2476
|
+
+ 'the consumer only so the next channel recreate can re-attach it. Listen for '
|
|
2477
|
+
+ 'consumer:re-registration:failed to learn why it could not.'
|
|
2478
|
+
);
|
|
2479
|
+
}
|
|
2480
|
+
|
|
2481
|
+
if (!this._isChannelAlive(this._consumerChannel)) {
|
|
2482
|
+
throw new ConnectionError(
|
|
2483
|
+
`[RabbitMQClient] Cannot cancelConsumer ${queue}: consumer channel is not available - `
|
|
2484
|
+
+ 'Expected: an open consumer channel to send the cancel on. '
|
|
2485
|
+
+ 'Fix: a dead channel has already stopped the consumer; it is re-registered only if it '
|
|
2486
|
+
+ 'is still tracked, so call cancelConsumer() again once the connection is back.'
|
|
2487
|
+
);
|
|
2488
|
+
}
|
|
2489
|
+
|
|
2490
|
+
this._trackChannelOperation(this._consumerChannel, `cancelConsumer ${queue}`);
|
|
2491
|
+
const result = await this._consumerChannel.cancel(info.consumerTag);
|
|
2492
|
+
// Dropped only after the broker accepted the cancel: a consumer that is
|
|
2493
|
+
// still delivering must stay on the list recovery re-attaches.
|
|
2494
|
+
this._activeConsumers.delete(queue);
|
|
2495
|
+
this._prefetchTracking.delete(queue);
|
|
2496
|
+
this._logger.info(
|
|
2497
|
+
`[RabbitMQClient] [mq-client-core] [CONSUMER] Cancelled consumer for queue "${queue}" (consumerTag: ${info.consumerTag})`
|
|
2498
|
+
);
|
|
2499
|
+
return result;
|
|
1213
2500
|
}
|
|
1214
2501
|
|
|
1215
2502
|
/**
|
|
@@ -1217,11 +2504,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1217
2504
|
* @returns {Promise<Object>} amqplib checkQueue result
|
|
1218
2505
|
*/
|
|
1219
2506
|
async checkQueue(queue) {
|
|
1220
|
-
await this.
|
|
1221
|
-
|
|
1222
|
-
throw new Error('Cannot checkQueue: queue channel is not initialized');
|
|
1223
|
-
}
|
|
1224
|
-
return await this._queueChannel.checkQueue(queue);
|
|
2507
|
+
const channel = await this._requireQueueChannel('checkQueue', queue);
|
|
2508
|
+
return await channel.checkQueue(queue);
|
|
1225
2509
|
}
|
|
1226
2510
|
|
|
1227
2511
|
/**
|
|
@@ -1229,11 +2513,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1229
2513
|
* @returns {Promise<Object>} amqplib purgeQueue result
|
|
1230
2514
|
*/
|
|
1231
2515
|
async purgeQueue(queue) {
|
|
1232
|
-
await this.
|
|
1233
|
-
|
|
1234
|
-
throw new Error('Cannot purgeQueue: queue channel is not initialized');
|
|
1235
|
-
}
|
|
1236
|
-
return await this._queueChannel.purgeQueue(queue);
|
|
2516
|
+
const channel = await this._requireQueueChannel('purgeQueue', queue);
|
|
2517
|
+
return await channel.purgeQueue(queue);
|
|
1237
2518
|
}
|
|
1238
2519
|
|
|
1239
2520
|
/**
|
|
@@ -1242,11 +2523,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1242
2523
|
* @returns {Promise<Object>} amqplib deleteQueue result
|
|
1243
2524
|
*/
|
|
1244
2525
|
async deleteQueue(queue, options = {}) {
|
|
1245
|
-
await this.
|
|
1246
|
-
|
|
1247
|
-
throw new Error('Cannot deleteQueue: queue channel is not initialized');
|
|
1248
|
-
}
|
|
1249
|
-
return await this._queueChannel.deleteQueue(queue, options);
|
|
2526
|
+
const channel = await this._requireQueueChannel('deleteQueue', queue);
|
|
2527
|
+
return await channel.deleteQueue(queue, options);
|
|
1250
2528
|
}
|
|
1251
2529
|
|
|
1252
2530
|
/**
|
|
@@ -1275,16 +2553,32 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1275
2553
|
// Track operation for debugging
|
|
1276
2554
|
this._trackChannelOperation(this._channel, `publish to ${queue}`);
|
|
1277
2555
|
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
2556
|
+
// Asked with `=== undefined`, not with `||`: the empty string IS a value here
|
|
2557
|
+
// — it names the default exchange — and the third leg was a third copy of a
|
|
2558
|
+
// default `config/defaultConfig.js` already owns (d.311).
|
|
2559
|
+
const exchange = options.exchange !== undefined ? options.exchange : this._config.exchange;
|
|
2560
|
+
// NO `exchangeType` here. It existed for the per-publish `assertExchange()`
|
|
2561
|
+
// this path no longer makes, so it has no reader and is not left lying
|
|
2562
|
+
// around (`change-discipline.md` § Removing something removes its
|
|
2563
|
+
// declaration). An exchange's type is declared by its owner, in
|
|
2564
|
+
// `config/queueConfig.js`, and applied through the public `assertExchange()`.
|
|
2565
|
+
// Asked with `=== undefined`, like the exchange above and for the same
|
|
2566
|
+
// reason: `''` is a VALUE here — it is the routing key a fanout publish
|
|
2567
|
+
// writes, and what `monitoring-publish.js` passes — not "nothing written".
|
|
2568
|
+
// `||` replaced it with the queue name, which a topic or direct exchange
|
|
2569
|
+
// would then route by (d.369; same family as d.311, one leg it missed).
|
|
2570
|
+
const routingKey = options.routingKey !== undefined ? options.routingKey : queue;
|
|
1281
2571
|
const persistent = options.persistent !== undefined ? options.persistent : this._config.durable;
|
|
1282
2572
|
const headers = options.headers || {};
|
|
1283
2573
|
|
|
1284
|
-
// Structured log:
|
|
1285
|
-
|
|
2574
|
+
// Structured log: the workflow id this message carries, read on the one rail
|
|
2575
|
+
// that reads it (`workflowIdOf()` above) — one spelling, and `null` when the
|
|
2576
|
+
// message carries none.
|
|
2577
|
+
const workflowId = workflowIdOf(headers);
|
|
1286
2578
|
const msgSize = buffer ? buffer.length : 0;
|
|
1287
|
-
this.
|
|
2579
|
+
this._logger.debug('[RabbitMQClient] PUBLISH_START', {
|
|
2580
|
+
workflow_id: workflowId,
|
|
2581
|
+
action: 'PUBLISH_START',
|
|
1288
2582
|
handler: 'publish',
|
|
1289
2583
|
function: 'RabbitMQClient._publishOnce',
|
|
1290
2584
|
input: {
|
|
@@ -1311,17 +2605,21 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1311
2605
|
// This prevents creating queues with wrong arguments (no TTL) which causes 406 errors later
|
|
1312
2606
|
if (!skipQueueExistencePrecheck) {
|
|
1313
2607
|
try {
|
|
1314
|
-
//
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
);
|
|
2608
|
+
// The queue channel comes from the ONE rail (d.282): ensure it, refuse
|
|
2609
|
+
// it unless it is ALIVE, record the operation. Until d.284 this site
|
|
2610
|
+
// assembled that sequence itself — `_ensureQueueChannel()` plus
|
|
2611
|
+
// `_isChannelAlive()` by hand — differing from the rail in nothing but
|
|
2612
|
+
// the error type, which is `_publishPathTransient()`'s job below.
|
|
2613
|
+
let queueChannel;
|
|
2614
|
+
try {
|
|
2615
|
+
queueChannel = await this._requireQueueChannel('checkQueue', queue);
|
|
2616
|
+
} catch (railErr) {
|
|
2617
|
+
if (railErr instanceof ConnectionError) {
|
|
2618
|
+
throw this._publishPathTransient('checkQueue', queue, railErr);
|
|
2619
|
+
}
|
|
2620
|
+
throw railErr;
|
|
1322
2621
|
}
|
|
1323
|
-
|
|
1324
|
-
await this._queueChannel.checkQueue(queue);
|
|
2622
|
+
await queueChannel.checkQueue(queue);
|
|
1325
2623
|
// Queue exists - proceed to publish
|
|
1326
2624
|
} catch (checkErr) {
|
|
1327
2625
|
// If queue doesn't exist (404), this je ERROR – ale rozlišujeme infra vs. non-infra:
|
|
@@ -1331,19 +2629,17 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1331
2629
|
// - Business fronty (`queueConfig.isBusinessQueue`, tj. `{service}.{workflow|queue|dlq}`)
|
|
1332
2630
|
// zakládá vlastnící služba přes setupServiceQueues() po registraci
|
|
1333
2631
|
// → QueueNotFoundError(kind='business'). Publisher je nikdy nevytváří.
|
|
1334
|
-
// - Ostatní jména (ani infra, ani business vzor)
|
|
1335
|
-
//
|
|
2632
|
+
// - Ostatní jména (ani infra, ani business vzor) nemá kdo vlastnit, takže
|
|
2633
|
+
// je nezakládá nikdo → QueueNotFoundError(kind='unowned'), d.419.
|
|
1336
2634
|
if (checkErr.code === 404) {
|
|
1337
|
-
//
|
|
1338
|
-
//
|
|
2635
|
+
// `queueConfig` is loaded once at the top of this module. It used to be
|
|
2636
|
+
// required here, inside a try/catch whose `catch` re-implemented
|
|
1339
2637
|
// `isInfrastructureQueue` as an inline prefix list — a fallback
|
|
1340
2638
|
// (`architecture-principles.md` §3) that had also gone stale: it was
|
|
1341
2639
|
// missing `telemetry.` and `delivery.`, both of which the real
|
|
1342
2640
|
// queueConfig classifies as infrastructure. A fallback that answers
|
|
1343
2641
|
// differently from the thing it stands in for is a second, wrong source
|
|
1344
2642
|
// of truth, so it is gone.
|
|
1345
|
-
const queueConfig = require('../config/queueConfig');
|
|
1346
|
-
|
|
1347
2643
|
if (queueConfig.isInfrastructureQueue(queue)) {
|
|
1348
2644
|
// Jasný, hlasitý signál pro infra služby – fronta chybí, je to programátorská chyba
|
|
1349
2645
|
throw new QueueNotFoundError(queue, true, checkErr, 'infrastructure');
|
|
@@ -1364,35 +2660,32 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1364
2660
|
if (queueConfig.isBusinessQueue(queue)) {
|
|
1365
2661
|
throw new QueueNotFoundError(queue, false, checkErr, 'business');
|
|
1366
2662
|
}
|
|
1367
|
-
//
|
|
1368
|
-
//
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
//
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
//
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
//
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
}
|
|
1394
|
-
this._trackChannelOperation(this._queueChannel, `assertQueue ${queue}`);
|
|
1395
|
-
await this._queueChannel.assertQueue(queue, queueOptions);
|
|
2663
|
+
// Nobody owns this name. Until d.419 that opened a THIRD case beside
|
|
2664
|
+
// the two above: the publish path asked `RecoveryWorker` whether this
|
|
2665
|
+
// client might declare it and, when the answer was yes, declared it —
|
|
2666
|
+
// `durable=true, arguments={}`, no TTL, no dead-letter route (measured
|
|
2667
|
+
// live, d.410). The concept had already ruled that out: "queues come
|
|
2668
|
+
// into being only from declarations (`queueConfig`,
|
|
2669
|
+
// `setupServiceQueues()` from the business templates)", and a new family
|
|
2670
|
+
// of queues gets "a `queueConfig` template first, never a generic
|
|
2671
|
+
// create-anything method"
|
|
2672
|
+
// (`docs/governance/confirmations/mq-consumer-contract.md` 006). And the
|
|
2673
|
+
// queue so created was unusable at the other end anyway: `consume()`
|
|
2674
|
+
// refuses a queue the config declares no dead-letter route for (d.259),
|
|
2675
|
+
// so the door produced queues that could be published into and never
|
|
2676
|
+
// consumed from.
|
|
2677
|
+
//
|
|
2678
|
+
// The name reaching this line is exactly the one the ONE owner of "what
|
|
2679
|
+
// is this queue declared with" — `queueConfig.declarationOptions()`
|
|
2680
|
+
// (d.410) — answers `null` for: that function returns a declaration for
|
|
2681
|
+
// an infrastructure name and for a business name, and both were refused
|
|
2682
|
+
// above, so the remainder is the set with no declaration at all. It is
|
|
2683
|
+
// asked this way round rather than called here because it REFUSES a name
|
|
2684
|
+
// belonging to a section that holds no entry for it (a configuration
|
|
2685
|
+
// defect, reported by the lookup that owns it) — a different failure from
|
|
2686
|
+
// "nothing declares this name", and the publish path must not flatten the
|
|
2687
|
+
// two into one message.
|
|
2688
|
+
throw new QueueNotFoundError(queue, false, checkErr, 'unowned');
|
|
1396
2689
|
} else {
|
|
1397
2690
|
// Other error - rethrow
|
|
1398
2691
|
throw checkErr;
|
|
@@ -1401,25 +2694,57 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1401
2694
|
}
|
|
1402
2695
|
// Publish to queue using ConfirmChannel (for publisher confirms)
|
|
1403
2696
|
// Channel is guaranteed to be open (ensured above)
|
|
1404
|
-
|
|
2697
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [PUBLISH] Sending message to queue "${queue}" (size: ${buffer.length} bytes)`);
|
|
1405
2698
|
|
|
1406
2699
|
// Use callback-based confirmation - kanály jsou spolehlivé, takže callback vždy dorazí
|
|
1407
2700
|
const confirmPromise = new Promise((resolve, reject) => {
|
|
2701
|
+
// ONE disarm for this attempt. Both timers below belong to this single
|
|
2702
|
+
// publish, so every settle path — confirmed, refused, timed out, thrown —
|
|
2703
|
+
// clears them by one name instead of naming whichever handle it happens
|
|
2704
|
+
// to remember. The safety timer was the one nobody remembered: its handle
|
|
2705
|
+
// was discarded at the call site, so a publish the broker confirmed in a
|
|
2706
|
+
// millisecond left a referenced timer armed for a further second, holding
|
|
2707
|
+
// the event loop and the closure's `buffer`/`options`/`originalChannel`
|
|
2708
|
+
// with it (d.285).
|
|
2709
|
+
let timeout = null;
|
|
2710
|
+
let safetyTimer = null;
|
|
2711
|
+
let settled = false;
|
|
2712
|
+
const disarm = () => {
|
|
2713
|
+
settled = true;
|
|
2714
|
+
this._clearTimeout(timeout);
|
|
2715
|
+
this._clearTimeout(safetyTimer);
|
|
2716
|
+
timeout = null;
|
|
2717
|
+
safetyTimer = null;
|
|
2718
|
+
};
|
|
2719
|
+
|
|
1408
2720
|
// Set timeout for publish confirmation (configurable)
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
2721
|
+
timeout = this._setTimeout(() => {
|
|
2722
|
+
disarm();
|
|
2723
|
+
reject(new PublishError(
|
|
2724
|
+
`[RabbitMQClient] Publish confirmation timeout for queue "${queue}" after ${this._brokerAnswerTimeout}ms - `
|
|
2725
|
+
+ 'Expected: the broker to confirm the message within brokerAnswerTimeout. '
|
|
2726
|
+
+ 'Fix: delivery is NOT guaranteed — treat the message as unconfirmed and republish it; check broker load and flow control.',
|
|
2727
|
+
queue,
|
|
2728
|
+
null
|
|
2729
|
+
));
|
|
2730
|
+
}, this._brokerAnswerTimeout);
|
|
1412
2731
|
|
|
1413
|
-
// Check if channel is still valid before sending
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
2732
|
+
// Check if channel is still valid before sending. The channel can die
|
|
2733
|
+
// during the awaits above (queue existence pre-check), so this is not
|
|
2734
|
+
// a repeat of `_ensurePublisherChannel()` — it is the last reading
|
|
2735
|
+
// before the payload leaves, and it reads the one liveness source.
|
|
2736
|
+
if (!this._isChannelAlive(this._channel)) {
|
|
2737
|
+
disarm();
|
|
2738
|
+
reject(new ConnectionError(
|
|
2739
|
+
`[RabbitMQClient] Cannot publish: channel is closed for queue "${queue}" - `
|
|
2740
|
+
+ 'Expected: an open publisher channel at send time. '
|
|
2741
|
+
+ 'Fix: retry the publish after the reconnect; PublishLayer does this on its own and buffers the message meanwhile.'
|
|
2742
|
+
));
|
|
1417
2743
|
return;
|
|
1418
2744
|
}
|
|
1419
2745
|
|
|
1420
2746
|
// Track original channel to detect if it was closed/recreated during publish
|
|
1421
2747
|
const originalChannel = this._channel;
|
|
1422
|
-
const channelId = originalChannel ? originalChannel._createdAt : null;
|
|
1423
2748
|
let callbackInvoked = false;
|
|
1424
2749
|
|
|
1425
2750
|
// Send message with callback
|
|
@@ -1430,14 +2755,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1430
2755
|
buildAmqpMessageProperties(options, persistent, headers),
|
|
1431
2756
|
(err, ok) => {
|
|
1432
2757
|
callbackInvoked = true;
|
|
1433
|
-
|
|
2758
|
+
disarm();
|
|
1434
2759
|
|
|
1435
|
-
// CRITICAL:
|
|
1436
|
-
//
|
|
1437
|
-
|
|
1438
|
-
|
|
2760
|
+
// CRITICAL: the delivery tag belongs to the channel that SENT the
|
|
2761
|
+
// message, so that channel's liveness is the whole question. One
|
|
2762
|
+
// reading answers it: a channel that was replaced is a channel that
|
|
2763
|
+
// died first (`_ensurePublisherChannel()` recreates only a dead one),
|
|
2764
|
+
// so identity and `_createdAt` comparisons said nothing this does not.
|
|
2765
|
+
if (!this._isChannelAlive(originalChannel)) {
|
|
1439
2766
|
// Channel was closed or recreated - delivery tag is invalid
|
|
1440
|
-
|
|
2767
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed/recreated during publish to queue "${queue}", ignoring callback`);
|
|
1441
2768
|
// If we're reconnecting, wait and retry
|
|
1442
2769
|
if (this._reconnecting) {
|
|
1443
2770
|
this._waitForReconnection().then(() => {
|
|
@@ -1447,7 +2774,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1447
2774
|
} else {
|
|
1448
2775
|
// Not reconnecting - this is a real error
|
|
1449
2776
|
// We cannot guarantee the message was delivered - must fail
|
|
1450
|
-
reject(new
|
|
2777
|
+
reject(new PublishError(
|
|
2778
|
+
`[RabbitMQClient] Channel closed during publish to queue "${queue}" and not reconnecting - message delivery not guaranteed. `
|
|
2779
|
+
+ 'Expected: the channel to stay open until the broker confirms, or a reconnect to be in flight. '
|
|
2780
|
+
+ 'Fix: republish the message — the confirm never arrived, so it may or may not have been stored.',
|
|
2781
|
+
queue,
|
|
2782
|
+
null
|
|
2783
|
+
));
|
|
1451
2784
|
}
|
|
1452
2785
|
return;
|
|
1453
2786
|
}
|
|
@@ -1455,7 +2788,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1455
2788
|
if (err) {
|
|
1456
2789
|
// Check if error is about invalid delivery tag (channel was closed)
|
|
1457
2790
|
if (err.message && (err.message.includes('unknown delivery tag') || err.message.includes('PRECONDITION_FAILED'))) {
|
|
1458
|
-
|
|
2791
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Delivery tag invalid (channel may have been closed) for queue "${queue}"`);
|
|
1459
2792
|
// If we're reconnecting, wait and retry
|
|
1460
2793
|
if (this._reconnecting) {
|
|
1461
2794
|
this._waitForReconnection().then(() => {
|
|
@@ -1464,16 +2797,24 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1464
2797
|
}).then(resolve).catch(reject);
|
|
1465
2798
|
} else {
|
|
1466
2799
|
// Not reconnecting - delivery tag is invalid, message delivery not guaranteed
|
|
1467
|
-
reject(new
|
|
2800
|
+
reject(new PublishError(
|
|
2801
|
+
`[RabbitMQClient] Delivery tag invalid for queue "${queue}" (channel closed) and not reconnecting - message delivery not guaranteed. `
|
|
2802
|
+
+ 'Expected: the delivery tag to stay valid until the broker confirms, or a reconnect to be in flight. '
|
|
2803
|
+
+ 'Fix: republish the message — the confirm never arrived, so it may or may not have been stored.',
|
|
2804
|
+
queue,
|
|
2805
|
+
null
|
|
2806
|
+
));
|
|
1468
2807
|
}
|
|
1469
2808
|
} else {
|
|
1470
|
-
|
|
2809
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
|
|
1471
2810
|
reject(err);
|
|
1472
2811
|
}
|
|
1473
2812
|
} else {
|
|
1474
2813
|
// Structured log: publish confirmed
|
|
1475
|
-
const wfId = headers
|
|
1476
|
-
this.
|
|
2814
|
+
const wfId = workflowIdOf(headers);
|
|
2815
|
+
this._logger.debug('[RabbitMQClient] PUBLISH_CONFIRMED', {
|
|
2816
|
+
workflow_id: wfId,
|
|
2817
|
+
action: 'PUBLISH_CONFIRMED',
|
|
1477
2818
|
handler: 'publish',
|
|
1478
2819
|
function: 'RabbitMQClient._publishOnce',
|
|
1479
2820
|
input: { queue, size: buffer ? buffer.length : 0 },
|
|
@@ -1483,23 +2824,28 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1483
2824
|
}
|
|
1484
2825
|
});
|
|
1485
2826
|
|
|
1486
|
-
// Set a safety timeout - if callback wasn't invoked and channel closed, retry
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
2827
|
+
// Set a safety timeout - if callback wasn't invoked and channel closed, retry.
|
|
2828
|
+
// A broker confirm can land inside `sendToQueue()` itself, so the
|
|
2829
|
+
// attempt may already be settled here; arming a safety net for an
|
|
2830
|
+
// attempt that has ended is the leak, not a safety net.
|
|
2831
|
+
if (!settled) {
|
|
2832
|
+
safetyTimer = this._setTimeout(() => {
|
|
2833
|
+
if (!callbackInvoked && !this._isChannelAlive(originalChannel)) {
|
|
2834
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Callback timeout and channel closed for queue "${queue}", will retry after reconnection`);
|
|
2835
|
+
if (this._reconnecting) {
|
|
2836
|
+
disarm();
|
|
2837
|
+
this._waitForReconnection().then(() => {
|
|
2838
|
+
return this.publish(queue, buffer, options);
|
|
2839
|
+
}).then(resolve).catch(reject);
|
|
2840
|
+
}
|
|
1495
2841
|
}
|
|
1496
|
-
}
|
|
1497
|
-
}
|
|
2842
|
+
}, this._publishConfirmWatchdogDelay);
|
|
2843
|
+
}
|
|
1498
2844
|
} catch (sendErr) {
|
|
1499
|
-
|
|
2845
|
+
disarm();
|
|
1500
2846
|
// If channel is closed, try to wait for reconnection and retry
|
|
1501
2847
|
if (sendErr.message && (sendErr.message.includes('Channel ended') || sendErr.message.includes('Channel closed'))) {
|
|
1502
|
-
|
|
2848
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during sendToQueue, will retry after reconnection`);
|
|
1503
2849
|
// Wait for reconnection and retry
|
|
1504
2850
|
if (this._reconnecting) {
|
|
1505
2851
|
this._waitForReconnection().then(() => {
|
|
@@ -1517,16 +2863,41 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1517
2863
|
|
|
1518
2864
|
await confirmPromise;
|
|
1519
2865
|
} else {
|
|
1520
|
-
//
|
|
1521
|
-
//
|
|
1522
|
-
|
|
2866
|
+
// An exchange is NAMED here, never declared. Its type and durability are
|
|
2867
|
+
// topology, and topology belongs to the owner: `config/queueConfig.js`
|
|
2868
|
+
// declares each platform exchange and the owning service asserts exactly
|
|
2869
|
+
// that (`infra/api_monitoring/src/consumer/index.js`,
|
|
2870
|
+
// `infra/api_services_registry/src/services/infrastructureEventPublisher.js`,
|
|
2871
|
+
// `infra/api_delivery_endpoint/src/mq/MQConnection.js`).
|
|
2872
|
+
//
|
|
2873
|
+
// Until d.342 this line asserted the exchange before EVERY publish, with
|
|
2874
|
+
// arguments made up on the spot — the type from `options.exchangeType ||
|
|
2875
|
+
// 'direct'` and the durability from the PUBLISHING client's own config.
|
|
2876
|
+
// That is a second declaration beside the owner's, and the owner's
|
|
2877
|
+
// decision names this exact defect one object down: "the endpoint's own
|
|
2878
|
+
// `{ durable: true }` assert is a second declaration and already yields
|
|
2879
|
+
// 406 against the library's"
|
|
2880
|
+
// (`docs/governance/confirmations/mq-consumer-contract.md` 003, point 4).
|
|
2881
|
+
// A client configured `durable: false` therefore got
|
|
2882
|
+
// 406 PRECONDITION-FAILED from a publish that was itself faultless.
|
|
2883
|
+
//
|
|
2884
|
+
// Same rule as for queues, one object up (`README.md` § Queue ownership):
|
|
2885
|
+
// a publisher does not create what somebody else owns. An exchange that
|
|
2886
|
+
// nobody declared is a broker-side 404 on this channel — fail-fast, not
|
|
2887
|
+
// an exchange invented by whoever published first.
|
|
1523
2888
|
|
|
1524
2889
|
// Use callback-based confirmation - kanály jsou spolehlivé
|
|
1525
2890
|
const confirmPromise = new Promise((resolve, reject) => {
|
|
1526
2891
|
// Set timeout for exchange publish confirmation
|
|
1527
2892
|
const timeout = this._setTimeout(() => {
|
|
1528
|
-
reject(new
|
|
1529
|
-
|
|
2893
|
+
reject(new PublishError(
|
|
2894
|
+
`[RabbitMQClient] Exchange publish confirmation timeout for exchange "${exchange}" after ${this._brokerAnswerTimeout}ms - `
|
|
2895
|
+
+ 'Expected: the broker to confirm the message within brokerAnswerTimeout. '
|
|
2896
|
+
+ 'Fix: delivery is NOT guaranteed — treat the message as unconfirmed and republish it; check broker load and flow control.',
|
|
2897
|
+
exchange,
|
|
2898
|
+
null
|
|
2899
|
+
));
|
|
2900
|
+
}, this._brokerAnswerTimeout);
|
|
1530
2901
|
|
|
1531
2902
|
this._channel.publish(
|
|
1532
2903
|
exchange,
|
|
@@ -1536,10 +2907,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1536
2907
|
(err, ok) => {
|
|
1537
2908
|
this._clearTimeout(timeout);
|
|
1538
2909
|
if (err) {
|
|
1539
|
-
|
|
2910
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Exchange publish callback error:`, err.message);
|
|
1540
2911
|
reject(err);
|
|
1541
2912
|
} else {
|
|
1542
|
-
|
|
2913
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [PUBLISH] ✓ Exchange publish confirmed`);
|
|
1543
2914
|
resolve();
|
|
1544
2915
|
}
|
|
1545
2916
|
});
|
|
@@ -1553,13 +2924,265 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1553
2924
|
// Pokud channel closed, vyhodíme TransientPublishError a necháme PublishLayer řešit retry
|
|
1554
2925
|
|
|
1555
2926
|
// Všechny chyby projdou jednotnou klasifikací do Transient/Permanent/QueueNotFound
|
|
1556
|
-
const classifiedErr = classifyPublishError(err);
|
|
2927
|
+
const classifiedErr = classifyPublishError(err, queue);
|
|
1557
2928
|
this.emit('error', classifiedErr);
|
|
1558
2929
|
throw classifiedErr;
|
|
1559
2930
|
}
|
|
1560
2931
|
}
|
|
1561
2932
|
|
|
1562
2933
|
|
|
2934
|
+
/**
|
|
2935
|
+
* Apply one consumer's prefetch to the CURRENT consumer channel, and record
|
|
2936
|
+
* what the broker was told.
|
|
2937
|
+
*
|
|
2938
|
+
* `.prefetch()` is per-channel QoS, and a consumer channel is recreated on
|
|
2939
|
+
* every channel death and every reconnect. Until d.263 it was applied in
|
|
2940
|
+
* exactly one place — the first `consume()` — so every later channel carried
|
|
2941
|
+
* no bound at all, while `_prefetchTracking` went on measuring in-flight
|
|
2942
|
+
* messages against the number the consumer had asked for. The utilisation
|
|
2943
|
+
* alarm was therefore reading a limit nobody enforced.
|
|
2944
|
+
*
|
|
2945
|
+
* A non-numeric prefetch (the caller asked for none) sets none, and records
|
|
2946
|
+
* none: there is nothing to measure utilisation against.
|
|
2947
|
+
*
|
|
2948
|
+
* @param {string} queue - Queue name
|
|
2949
|
+
* @param {number|null|undefined} prefetch - The consumer's prefetch
|
|
2950
|
+
* @returns {Promise<void>}
|
|
2951
|
+
* @private
|
|
2952
|
+
*/
|
|
2953
|
+
async _applyConsumerPrefetch(queue, prefetch) {
|
|
2954
|
+
if (typeof prefetch !== 'number') {
|
|
2955
|
+
return;
|
|
2956
|
+
}
|
|
2957
|
+
|
|
2958
|
+
await this._consumerChannel.prefetch(prefetch);
|
|
2959
|
+
|
|
2960
|
+
const tracking = this._prefetchTracking.get(queue);
|
|
2961
|
+
if (tracking) {
|
|
2962
|
+
tracking.prefetchCount = prefetch;
|
|
2963
|
+
} else {
|
|
2964
|
+
this._prefetchTracking.set(queue, {
|
|
2965
|
+
prefetchCount: prefetch,
|
|
2966
|
+
inFlight: 0,
|
|
2967
|
+
lastCheck: Date.now()
|
|
2968
|
+
});
|
|
2969
|
+
}
|
|
2970
|
+
|
|
2971
|
+
if (!this._prefetchCheckTimer) {
|
|
2972
|
+
this._startPrefetchMonitoring();
|
|
2973
|
+
}
|
|
2974
|
+
}
|
|
2975
|
+
|
|
2976
|
+
/**
|
|
2977
|
+
* Prepare one queue for a consumer, and answer with the arguments it is
|
|
2978
|
+
* declared with.
|
|
2979
|
+
*
|
|
2980
|
+
* ONE rail for the first `consume()` and for every re-registration after a
|
|
2981
|
+
* channel recreate. It classifies the queue (reply / infrastructure /
|
|
2982
|
+
* business), resolves the arguments from the central config, refuses a queue
|
|
2983
|
+
* whose config declares no dead-letter route, and then CHECKS an
|
|
2984
|
+
* infrastructure queue (consumers never create those) or ASSERTS a business
|
|
2985
|
+
* one.
|
|
2986
|
+
*
|
|
2987
|
+
* Until d.262 `_reRegisterConsumers()` carried a second, simpler copy:
|
|
2988
|
+
* `assertQueue(queue, options.queueOptions)` for every tracked queue, with no
|
|
2989
|
+
* classification and no dead-letter gate — so a recreate could DECLARE a queue
|
|
2990
|
+
* the consumer does not own, and its `404` branch was unreachable, because
|
|
2991
|
+
* `assertQueue()` creates what is missing instead of answering 404.
|
|
2992
|
+
*
|
|
2993
|
+
* @param {string} queue - Queue name
|
|
2994
|
+
* @param {Object} options - `durable` and the caller's `queueOptions`, if any
|
|
2995
|
+
* @returns {Promise<Object>} the queueOptions this queue is declared with
|
|
2996
|
+
* @private
|
|
2997
|
+
*/
|
|
2998
|
+
async _prepareQueueForConsume(queue, options = {}) {
|
|
2999
|
+
const durable = options.durable !== undefined ? options.durable : this._config.durable;
|
|
3000
|
+
let queueOptions = options.queueOptions || { durable };
|
|
3001
|
+
|
|
3002
|
+
// Skip assertQueue for reply queues (they're already created with specific settings)
|
|
3003
|
+
// Reply queues start with 'rpc.reply.' and are created as non-durable
|
|
3004
|
+
const isReplyQueue = queue.startsWith('rpc.reply.');
|
|
3005
|
+
// Classification is read once and used by both blocks below — the one that
|
|
3006
|
+
// resolves the central arguments, and the one that checks or asserts the queue.
|
|
3007
|
+
const isInfraQueue = !isReplyQueue && queueConfig.isInfrastructureQueue(queue);
|
|
3008
|
+
const isBusinessQueue = !isReplyQueue && queueConfig.isBusinessQueue(queue);
|
|
3009
|
+
|
|
3010
|
+
if (!isReplyQueue) {
|
|
3011
|
+
// CRITICAL: Use queueConfig.js to get correct parameters (TTL, max-length, etc.)
|
|
3012
|
+
// This prevents 406 PRECONDITION-FAILED errors from TTL mismatches.
|
|
3013
|
+
// queueConfig is loaded once at the top of this module; it used to be required
|
|
3014
|
+
// here, with a `catch` that announced a fall back to the caller's defaults and
|
|
3015
|
+
// then asserted the queue with exactly the arguments this config exists to override.
|
|
3016
|
+
|
|
3017
|
+
if (isInfraQueue || isBusinessQueue) {
|
|
3018
|
+
// An infrastructure queue is CHECKED here and never created — ownership rule:
|
|
3019
|
+
// infra queues are created by their owning infra service. A business queue is
|
|
3020
|
+
// asserted, on the public rail below.
|
|
3021
|
+
//
|
|
3022
|
+
// Either way the arguments come from the ONE function that owns the mapping
|
|
3023
|
+
// name → declaration (`queueConfig.declarationOptions()`, d.410), the same one
|
|
3024
|
+
// the public `assertQueue()` reads. Until then this method rebuilt
|
|
3025
|
+
// `{ durable: cfg.durable !== false, arguments: { ...cfg.arguments } }` from
|
|
3026
|
+
// its own lookup, once per class of queue — a second implementation of one
|
|
3027
|
+
// concern (`change-discipline.md` § One rail per concern), and the way a queue
|
|
3028
|
+
// ends up declared differently by two paths that both believe they follow the
|
|
3029
|
+
// central config.
|
|
3030
|
+
//
|
|
3031
|
+
// A failed lookup is a CONFIGURATION DEFECT, not a condition to log and
|
|
3032
|
+
// survive. Until 2026-09-07 both branches below caught it, warned, and
|
|
3033
|
+
// carried on with `queueOptions` as the caller left them — the bare
|
|
3034
|
+
// `{ durable }` default — which is precisely the substitution the
|
|
3035
|
+
// central config exists to prevent (406 PRECONDITION-FAILED against the
|
|
3036
|
+
// owner's declaration, or a queue created without TTL and DLQ). Same
|
|
3037
|
+
// fallback the lazy `require` had one floor up in `assertQueue()`, and
|
|
3038
|
+
// removed there for the same reason.
|
|
3039
|
+
try {
|
|
3040
|
+
queueOptions = queueConfig.declarationOptions(queue);
|
|
3041
|
+
} catch (configErr) {
|
|
3042
|
+
throw new ConsumeError(
|
|
3043
|
+
missingQueueDefinitionMessage(queue, isInfraQueue),
|
|
3044
|
+
queue,
|
|
3045
|
+
configErr
|
|
3046
|
+
);
|
|
3047
|
+
}
|
|
3048
|
+
this._logger.debug(
|
|
3049
|
+
isInfraQueue
|
|
3050
|
+
? `[RabbitMQClient] [mq-client-core] [CONSUMER] Checking infrastructure queue ${queue} exists (no auto-create)`
|
|
3051
|
+
: `[RabbitMQClient] [mq-client-core] [CONSUMER] Asserting business queue ${queue} with the declaration queueConfig owns`
|
|
3052
|
+
);
|
|
3053
|
+
}
|
|
3054
|
+
// End of the queue-classification block. What follows applies to EVERY
|
|
3055
|
+
// queue this consumer may attach to, reply queues included.
|
|
3056
|
+
}
|
|
3057
|
+
|
|
3058
|
+
// The dead-letter policy below rejects a spent message with
|
|
3059
|
+
// `nack(requeue=false)`, and the broker then moves it ONLY if the queue
|
|
3060
|
+
// declares where to: `x-dead-letter-exchange` + `x-dead-letter-routing-key`.
|
|
3061
|
+
// Without them the broker DROPS it — while `_rejectDelivery()` publishes a
|
|
3062
|
+
// `message_dlq` event saying it reached `.dlq`. So the policy refuses to run
|
|
3063
|
+
// at all over a queue with no route, at REGISTRATION rather than at the first
|
|
3064
|
+
// failure (`architecture-principles.md` §4): a consumer that cannot
|
|
3065
|
+
// dead-letter is never attached, so no message is taken off that queue and
|
|
3066
|
+
// lost. There is no opt-out and no fallback to requeue-for-ever — one rail
|
|
3067
|
+
// for business and infrastructure consumers alike
|
|
3068
|
+
// (`docs/governance/confirmations/mq-consumer-contract.md` 002;
|
|
3069
|
+
// `automation-gates.md` §1 requirement 5).
|
|
3070
|
+
//
|
|
3071
|
+
// The source of truth is the declaration in queueConfig, never a question to
|
|
3072
|
+
// the broker: `checkQueue()` returns `queue.declare-ok`, which carries no
|
|
3073
|
+
// arguments (measured, 2026-09-12 — see queueConfig.getDeadLetterRoute()).
|
|
3074
|
+
// The caller's own `queueOptions` do not count either; a route supplied at
|
|
3075
|
+
// the call site would be a second declaration of one queue's topology.
|
|
3076
|
+
if (queueConfig.getDeadLetterRoute(queue) === null) {
|
|
3077
|
+
throw new ConsumeError(
|
|
3078
|
+
`[RabbitMQClient] consume(${queue}): the dead-letter policy needs a dead-letter route and `
|
|
3079
|
+
+ 'queueConfig declares none for this queue - '
|
|
3080
|
+
+ "Expected: x-dead-letter-exchange/-routing-key in the queue's queueConfig section "
|
|
3081
|
+
+ '(docs/governance/confirmations/mq-consumer-contract.md 002, d.198a). '
|
|
3082
|
+
+ 'Fix: declare the route in src/config/queueConfig.js and bind the queue that receives it, '
|
|
3083
|
+
+ 'or consume a queue that already declares one — there is no consumer without a dead-letter '
|
|
3084
|
+
+ 'route, because nack(requeue=false) on a queue with no route makes the broker drop the message.',
|
|
3085
|
+
queue,
|
|
3086
|
+
undefined,
|
|
3087
|
+
CONSUMER_DEAD_LETTER_ROUTE_MISSING
|
|
3088
|
+
);
|
|
3089
|
+
}
|
|
3090
|
+
|
|
3091
|
+
if (!isReplyQueue) {
|
|
3092
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] Asserting queue ${queue} before consume() at ${new Date().toISOString()}`);
|
|
3093
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] Queue options:`, JSON.stringify(queueOptions, null, 2));
|
|
3094
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] _queueChannel state: exists=${!!this._queueChannel}, closed=${!this._isChannelAlive(this._queueChannel)}`);
|
|
3095
|
+
|
|
3096
|
+
// Every queue operation below goes to the queue channel through the ONE rail
|
|
3097
|
+
// (`_requireQueueChannel`, d.282): ensure it, refuse it unless it is ALIVE,
|
|
3098
|
+
// record the operation. Until d.284 this method was the sixth place with its
|
|
3099
|
+
// own copy of that sequence, and the copy asked the wrong question:
|
|
3100
|
+
// `if (!this._queueChannel)` is about the REFERENCE, which a corpse answers
|
|
3101
|
+
// exactly like a live channel. A consumer over a dead-but-referenced channel
|
|
3102
|
+
// therefore ran checkQueue/assertQueue on the corpse and the caller got
|
|
3103
|
+
// amqplib's bare "Channel closed" — the defect d.282 removed from the four
|
|
3104
|
+
// public queue operations, one floor down.
|
|
3105
|
+
//
|
|
3106
|
+
// There is exactly ONE question to the broker per queue, and it is the one
|
|
3107
|
+
// that decides: `checkQueue` for an infrastructure queue (a consumer never
|
|
3108
|
+
// creates one, so a 404 is the verdict), `assertQueue` for a business queue
|
|
3109
|
+
// (the declaration itself, and a 406 is the verdict on argument drift).
|
|
3110
|
+
// A `checkQueue` used to run ahead of both, its result reaching nothing but
|
|
3111
|
+
// a `debug` line. It was written to catch a queue "already created with
|
|
3112
|
+
// different arguments" — a question `checkQueue` cannot answer, because
|
|
3113
|
+
// `queue.declare-ok` carries the name and two counters and no arguments at
|
|
3114
|
+
// all, which its author noted in the next breath and then asked anyway. It
|
|
3115
|
+
// was not free: a 404 closes the channel by AMQP rule, and a queue that does
|
|
3116
|
+
// not exist yet is the ordinary case for every new business queue, so each
|
|
3117
|
+
// first `consume()` cost one queue channel, killed and recreated (measured
|
|
3118
|
+
// against the live broker: closes=1, channel identity changed). On the
|
|
3119
|
+
// infrastructure path it was a literal duplicate of the call below (d.287).
|
|
3120
|
+
|
|
3121
|
+
if (isInfraQueue) {
|
|
3122
|
+
// IMPORTANT: Do NOT auto-create infrastructure queues in consumers.
|
|
3123
|
+
// If missing, fail-fast. The owning infra service must recreate on startup.
|
|
3124
|
+
const queueChannel = await this._requireQueueChannel('checkQueue', queue);
|
|
3125
|
+
try {
|
|
3126
|
+
await queueChannel.checkQueue(queue);
|
|
3127
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Infrastructure queue ${queue} exists (consumer will proceed)`);
|
|
3128
|
+
} catch (checkErr) {
|
|
3129
|
+
if (checkErr.code === 404) {
|
|
3130
|
+
// Typed and coded since d.297: a bare `Error` lost the queue AND the
|
|
3131
|
+
// class, so the layer above could not tell "this queue does not exist"
|
|
3132
|
+
// from "the policy refused this queue" and told the reader the wrong
|
|
3133
|
+
// one. The sentence is unchanged — it was already the true one.
|
|
3134
|
+
throw new ConsumeError(
|
|
3135
|
+
missingInfrastructureQueueMessage(queue),
|
|
3136
|
+
queue,
|
|
3137
|
+
checkErr,
|
|
3138
|
+
CONSUMER_QUEUE_MISSING
|
|
3139
|
+
);
|
|
3140
|
+
}
|
|
3141
|
+
throw checkErr;
|
|
3142
|
+
}
|
|
3143
|
+
} else {
|
|
3144
|
+
// Business queue (or unknown) — declared on the ONE public rail, the same
|
|
3145
|
+
// `assertQueue()` the publish path's 404 branch (d.281) and
|
|
3146
|
+
// `RecoveryWorker.createQueue()` (d.277b) go through. It ensures the queue
|
|
3147
|
+
// channel, refuses one that is not alive (d.282) and resolves the arguments
|
|
3148
|
+
// from `queueConfig.declarationOptions()` — so a declaration made here and a
|
|
3149
|
+
// declaration made anywhere else are the same declaration by construction,
|
|
3150
|
+
// not by two copies of the same three lines happening to agree (d.410).
|
|
3151
|
+
//
|
|
3152
|
+
// A name the central config declares carries NO caller options: the
|
|
3153
|
+
// declaration is the owner's, and `assertQueue()` refuses a caller lifetime
|
|
3154
|
+
// over it by name. For an unclassified name the caller's `queueOptions` are
|
|
3155
|
+
// the only statement of the queue's lifetime there is, so they travel on.
|
|
3156
|
+
try {
|
|
3157
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] About to call assertQueue(${queue}, ${JSON.stringify(queueOptions)})`);
|
|
3158
|
+
await this.assertQueue(queue, isBusinessQueue ? {} : (options.queueOptions || {}));
|
|
3159
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Queue ${queue} asserted successfully`);
|
|
3160
|
+
} catch (assertErr) {
|
|
3161
|
+
// If queue exists with different arguments (406), this is a CRITICAL ERROR
|
|
3162
|
+
// We should NOT proceed - the root cause must be fixed
|
|
3163
|
+
if (assertErr.code === 406) {
|
|
3164
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✗ CRITICAL: Queue ${queue} exists with different arguments!`);
|
|
3165
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] Error:`, assertErr.message);
|
|
3166
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] Expected options:`, JSON.stringify(queueOptions, null, 2));
|
|
3167
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] This means assertQueue() was called without parameters somewhere else. Root cause must be fixed!`);
|
|
3168
|
+
throw new ConsumeError(
|
|
3169
|
+
`[RabbitMQClient] Cannot assertQueue ${queue}: queue exists with different arguments (406 PRECONDITION-FAILED). `
|
|
3170
|
+
+ 'Expected: every declaration of this queue to use the arguments queueConfig prescribes (TTL, DLQ, max-length). '
|
|
3171
|
+
+ 'Fix: find the assertQueue() call that declared it without parameters and fix that call; '
|
|
3172
|
+
+ 'the queue is NOT redeclared here, because doing so would hide the drift instead of ending it.',
|
|
3173
|
+
queue,
|
|
3174
|
+
assertErr
|
|
3175
|
+
);
|
|
3176
|
+
}
|
|
3177
|
+
// Other error - rethrow
|
|
3178
|
+
throw assertErr;
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
}
|
|
3182
|
+
|
|
3183
|
+
return queueOptions;
|
|
3184
|
+
}
|
|
3185
|
+
|
|
1563
3186
|
/**
|
|
1564
3187
|
* Starts consuming messages from the specified queue.
|
|
1565
3188
|
* @param {string} queue - Queue name to consume from.
|
|
@@ -1569,6 +3192,77 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1569
3192
|
* @throws {Error} If consume setup fails or channel is not available.
|
|
1570
3193
|
*/
|
|
1571
3194
|
async consume(queue, onMessage, options = {}) {
|
|
3195
|
+
// Dead-letter policy FIRST, before the broker is touched: a budget that
|
|
3196
|
+
// cannot bound anything must not reach a registered consumer
|
|
3197
|
+
// (`architecture-principles.md` §4). The default comes from ../config.js
|
|
3198
|
+
// (explicit → RABBITMQ_MAX_DELIVERY_ATTEMPTS → defaults.js), never from a
|
|
3199
|
+
// literal here; only the env form is coerced, an explicit value is taken
|
|
3200
|
+
// exactly as written. @see ../config/deliveryPolicy.js
|
|
3201
|
+
// `requeueOnError` is the SAME policy said in one word, never a second one
|
|
3202
|
+
// (`mq-client-lifecycle-contract` 001 point 4 "L8" names the option;
|
|
3203
|
+
// `mq-consumer-contract` 002 point 3 owns the policy it must fold into).
|
|
3204
|
+
// `false` = the budget is one attempt, so the first failure rejects into
|
|
3205
|
+
// `<svc>.dlq` — exactly what a `permanent` classification already does.
|
|
3206
|
+
// `true` or absent = the budget configuration decides. There is no third
|
|
3207
|
+
// meaning, and the combination that would need one is refused below rather
|
|
3208
|
+
// than resolved by silently preferring one of the two.
|
|
3209
|
+
if (options.requeueOnError !== undefined && typeof options.requeueOnError !== 'boolean') {
|
|
3210
|
+
throw new ValidationError(
|
|
3211
|
+
'[RabbitMQClient] Invalid requeueOnError - Expected: true or false, got '
|
|
3212
|
+
+ `${JSON.stringify(options.requeueOnError)}. Fix: pass consume(queue, handler, `
|
|
3213
|
+
+ '{ requeueOnError: false }) to dead-letter on the first failure, or leave it out and '
|
|
3214
|
+
+ 'let the delivery budget decide.'
|
|
3215
|
+
);
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3218
|
+
const noRequeue = options.requeueOnError === false;
|
|
3219
|
+
|
|
3220
|
+
if (noRequeue && options.maxAttempts !== undefined && options.maxAttempts !== 1) {
|
|
3221
|
+
throw new ValidationError(
|
|
3222
|
+
'[RabbitMQClient] Contradictory delivery policy - requeueOnError: false means one attempt, '
|
|
3223
|
+
+ `but maxAttempts says ${JSON.stringify(options.maxAttempts)}. Expected: one statement about `
|
|
3224
|
+
+ 'how many times a handler may run. Fix: drop requeueOnError to keep the budget, or drop '
|
|
3225
|
+
+ 'maxAttempts to dead-letter on the first failure.'
|
|
3226
|
+
);
|
|
3227
|
+
}
|
|
3228
|
+
|
|
3229
|
+
const maxAttempts = noRequeue
|
|
3230
|
+
? 1
|
|
3231
|
+
: options.maxAttempts === undefined
|
|
3232
|
+
? runtimeCfg.get('maxDeliveryAttempts', this._config.maxDeliveryAttempts)
|
|
3233
|
+
: options.maxAttempts;
|
|
3234
|
+
|
|
3235
|
+
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
|
|
3236
|
+
throw new ValidationError(
|
|
3237
|
+
'[RabbitMQClient] Invalid delivery budget - maxAttempts must be an integer >= 1, got '
|
|
3238
|
+
+ `${JSON.stringify(maxAttempts)}. Fix: pass consume(queue, handler, { maxAttempts: <integer >= 1> }), `
|
|
3239
|
+
+ 'or leave it out and let the module default (defaults.js maxDeliveryAttempts, '
|
|
3240
|
+
+ 'env RABBITMQ_MAX_DELIVERY_ATTEMPTS) decide.'
|
|
3241
|
+
);
|
|
3242
|
+
}
|
|
3243
|
+
|
|
3244
|
+
const classify = options.classify === undefined
|
|
3245
|
+
? deliveryPolicy.defaultClassification
|
|
3246
|
+
: options.classify;
|
|
3247
|
+
|
|
3248
|
+
if (typeof classify !== 'function') {
|
|
3249
|
+
throw new ValidationError(
|
|
3250
|
+
'[RabbitMQClient] Invalid error classifier - classify must be a function returning '
|
|
3251
|
+
+ `'transient' or 'permanent', got ${typeof options.classify}. Fix: pass consume(queue, handler, `
|
|
3252
|
+
+ '{ classify: (error) => … }), or leave it out and every error is treated as transient.'
|
|
3253
|
+
);
|
|
3254
|
+
}
|
|
3255
|
+
|
|
3256
|
+
const policy = { maxAttempts, classify };
|
|
3257
|
+
const onDeadLetter = options.onDeadLetter;
|
|
3258
|
+
if (onDeadLetter !== undefined && typeof onDeadLetter !== 'function') {
|
|
3259
|
+
throw new ValidationError(
|
|
3260
|
+
'[RabbitMQClient] Invalid dead-letter hook - onDeadLetter must be a function, got '
|
|
3261
|
+
+ `${typeof onDeadLetter}. Fix: consume through BaseClient, which injects the hook that `
|
|
3262
|
+
+ 'publishes the message_dlq event, or omit the option when driving this transport directly.'
|
|
3263
|
+
);
|
|
3264
|
+
}
|
|
3265
|
+
|
|
1572
3266
|
// Ensure consumer channel exists and is open (auto-recreates if closed and re-registers consumers)
|
|
1573
3267
|
await this._ensureConsumerChannel();
|
|
1574
3268
|
|
|
@@ -1579,148 +3273,27 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1579
3273
|
const prefetch = options.prefetch !== undefined ? options.prefetch : this._config.prefetch;
|
|
1580
3274
|
const noAck = options.noAck !== undefined ? options.noAck : this._config.noAck;
|
|
1581
3275
|
|
|
1582
|
-
|
|
1583
|
-
let queueOptions = options.queueOptions || { durable };
|
|
1584
|
-
|
|
1585
|
-
try {
|
|
1586
|
-
// Skip assertQueue for reply queues (they're already created with specific settings)
|
|
1587
|
-
// Reply queues start with 'rpc.reply.' and are created as non-durable
|
|
1588
|
-
if (!queue.startsWith('rpc.reply.')) {
|
|
1589
|
-
// CRITICAL: Use queueConfig.js to get correct parameters (TTL, max-length, etc.)
|
|
1590
|
-
// This prevents 406 PRECONDITION-FAILED errors from TTL mismatches
|
|
1591
|
-
// Use local queueConfig from mq-client-core (it has infrastructure queue configs)
|
|
1592
|
-
let queueConfig = null;
|
|
1593
|
-
try {
|
|
1594
|
-
queueConfig = require('../config/queueConfig');
|
|
1595
|
-
} catch (requireErr) {
|
|
1596
|
-
console.warn(`[RabbitMQClient] [mq-client-core] [CONSUMER] Cannot load queueConfig:`, requireErr.message);
|
|
1597
|
-
console.warn(`[RabbitMQClient] [mq-client-core] [CONSUMER] Using default queue options (this may cause 406 errors if queue exists with different args)`);
|
|
1598
|
-
}
|
|
1599
|
-
|
|
1600
|
-
// Only check queue types if queueConfig is available
|
|
1601
|
-
const isInfraQueue = queueConfig ? queueConfig.isInfrastructureQueue(queue) : false;
|
|
1602
|
-
const isBusinessQueue = queueConfig ? queueConfig.isBusinessQueue(queue) : false;
|
|
1603
|
-
|
|
1604
|
-
if (queueConfig) {
|
|
1605
|
-
if (isInfraQueue) {
|
|
1606
|
-
// Infrastructure queue - use central config for expected arguments,
|
|
1607
|
-
// but DO NOT create it here. Ownership rule: infra queues are created by their owning infra service.
|
|
1608
|
-
try {
|
|
1609
|
-
const infraConfig = queueConfig.getInfrastructureQueueConfig(queue);
|
|
1610
|
-
queueOptions = {
|
|
1611
|
-
durable: infraConfig.durable !== false,
|
|
1612
|
-
arguments: { ...infraConfig.arguments }
|
|
1613
|
-
};
|
|
1614
|
-
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] Checking infrastructure queue ${queue} exists (no auto-create)`);
|
|
1615
|
-
} catch (configErr) {
|
|
1616
|
-
console.warn(`[RabbitMQClient] [mq-client-core] [CONSUMER] Infrastructure queue config not found for ${queue}, will still require it to exist:`, configErr.message);
|
|
1617
|
-
}
|
|
1618
|
-
} else if (isBusinessQueue) {
|
|
1619
|
-
// Business queue - use central config
|
|
1620
|
-
try {
|
|
1621
|
-
const parsed = queueConfig.parseBusinessQueue(queue);
|
|
1622
|
-
if (parsed) {
|
|
1623
|
-
const businessConfig = queueConfig.getBusinessQueueConfig(parsed.queueType, parsed.serviceName);
|
|
1624
|
-
queueOptions = {
|
|
1625
|
-
durable: businessConfig.durable !== false,
|
|
1626
|
-
arguments: { ...businessConfig.arguments }
|
|
1627
|
-
};
|
|
1628
|
-
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] Asserting business queue ${queue} with config from queueConfig`);
|
|
1629
|
-
}
|
|
1630
|
-
} catch (configErr) {
|
|
1631
|
-
console.warn(`[RabbitMQClient] [mq-client-core] [CONSUMER] Business queue config not found for ${queue}, using default:`, configErr.message);
|
|
1632
|
-
}
|
|
1633
|
-
}
|
|
1634
|
-
}
|
|
1635
|
-
|
|
1636
|
-
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] Asserting queue ${queue} before consume() at ${new Date().toISOString()}`);
|
|
1637
|
-
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] Queue options:`, JSON.stringify(queueOptions, null, 2));
|
|
1638
|
-
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] _queueChannel state: exists=${!!this._queueChannel}, closed=${this._queueChannel ? this._queueChannel.closed : 'N/A'}`);
|
|
1639
|
-
|
|
1640
|
-
// CRITICAL: Check if queue already exists with different arguments
|
|
1641
|
-
// This can happen if sendToQueue() or amqplib's consume() auto-created it without TTL
|
|
1642
|
-
try {
|
|
1643
|
-
const queueInfo = await this._queueChannel.checkQueue(queue);
|
|
1644
|
-
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] Queue ${queue} already exists with arguments:`, JSON.stringify(queueInfo.messageCount !== undefined ? { messageCount: queueInfo.messageCount, consumerCount: queueInfo.consumerCount } : queueInfo));
|
|
1645
|
-
// If queue exists, we need to check if arguments match
|
|
1646
|
-
// Note: checkQueue() doesn't return arguments, so we'll try assertQueue() and catch 406
|
|
1647
|
-
} catch (checkErr) {
|
|
1648
|
-
if (checkErr.code === 404) {
|
|
1649
|
-
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] Queue ${queue} does not exist (404), will be created with options`);
|
|
1650
|
-
} else {
|
|
1651
|
-
console.warn(`[RabbitMQClient] [mq-client-core] [CONSUMER] checkQueue() failed for ${queue}:`, checkErr.message);
|
|
1652
|
-
}
|
|
1653
|
-
}
|
|
1654
|
-
|
|
1655
|
-
// Ensure queue channel is available
|
|
1656
|
-
await this._ensureQueueChannel();
|
|
1657
|
-
if (!this._queueChannel) {
|
|
1658
|
-
throw new Error('Queue channel is not available (connection may be closed)');
|
|
1659
|
-
}
|
|
3276
|
+
let queueOptions;
|
|
1660
3277
|
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
}
|
|
1674
|
-
} else {
|
|
1675
|
-
// Business queue (or unknown) - assert with canonical parameters to prevent 406 drift.
|
|
1676
|
-
try {
|
|
1677
|
-
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] About to call assertQueue(${queue}, ${JSON.stringify(queueOptions)})`);
|
|
1678
|
-
this._trackChannelOperation(this._queueChannel, `assertQueue ${queue}`);
|
|
1679
|
-
await this._queueChannel.assertQueue(queue, queueOptions);
|
|
1680
|
-
console.log(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Queue ${queue} asserted successfully`);
|
|
1681
|
-
} catch (assertErr) {
|
|
1682
|
-
// If queue exists with different arguments (406), this is a CRITICAL ERROR
|
|
1683
|
-
// We should NOT proceed - the root cause must be fixed
|
|
1684
|
-
if (assertErr.code === 406) {
|
|
1685
|
-
console.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✗ CRITICAL: Queue ${queue} exists with different arguments!`);
|
|
1686
|
-
console.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] Error:`, assertErr.message);
|
|
1687
|
-
console.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] Expected options:`, JSON.stringify(queueOptions, null, 2));
|
|
1688
|
-
console.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] This means assertQueue() was called without parameters somewhere else. Root cause must be fixed!`);
|
|
1689
|
-
throw new Error(`Cannot assertQueue ${queue}: queue exists with different arguments. Root cause: assertQueue() was called without parameters. Fix the root cause instead of proceeding.`);
|
|
1690
|
-
}
|
|
1691
|
-
// Other error - rethrow
|
|
1692
|
-
throw assertErr;
|
|
1693
|
-
}
|
|
1694
|
-
}
|
|
1695
|
-
}
|
|
1696
|
-
// Set prefetch if provided (on consumer channel)
|
|
1697
|
-
if (typeof prefetch === 'number') {
|
|
1698
|
-
this._consumerChannel.prefetch(prefetch);
|
|
1699
|
-
|
|
1700
|
-
// Track prefetch for monitoring
|
|
1701
|
-
if (!this._prefetchTracking.has(queue)) {
|
|
1702
|
-
this._prefetchTracking.set(queue, {
|
|
1703
|
-
prefetchCount: prefetch,
|
|
1704
|
-
inFlight: 0,
|
|
1705
|
-
lastCheck: Date.now()
|
|
1706
|
-
});
|
|
1707
|
-
} else {
|
|
1708
|
-
const tracking = this._prefetchTracking.get(queue);
|
|
1709
|
-
tracking.prefetchCount = prefetch;
|
|
1710
|
-
}
|
|
1711
|
-
|
|
1712
|
-
// Start prefetch monitoring if not already started
|
|
1713
|
-
if (!this._prefetchCheckTimer) {
|
|
1714
|
-
this._startPrefetchMonitoring();
|
|
1715
|
-
}
|
|
1716
|
-
}
|
|
3278
|
+
try {
|
|
3279
|
+
// Classify the queue, resolve its declared arguments, refuse it if the
|
|
3280
|
+
// config declares no dead-letter route, check or assert it. ONE rail,
|
|
3281
|
+
// shared with `_reRegisterConsumers()`.
|
|
3282
|
+
queueOptions = await this._prepareQueueForConsume(queue, {
|
|
3283
|
+
durable,
|
|
3284
|
+
queueOptions: options.queueOptions
|
|
3285
|
+
});
|
|
3286
|
+
// Set prefetch if provided (on consumer channel). ONE rail, shared with
|
|
3287
|
+
// `_reRegisterConsumers()` — prefetch belongs to the consumer, not to the
|
|
3288
|
+
// channel that happened to be open when it first attached.
|
|
3289
|
+
await this._applyConsumerPrefetch(queue, prefetch);
|
|
1717
3290
|
|
|
1718
3291
|
// CRITICAL: Log before calling amqplib's consume()
|
|
1719
3292
|
// amqplib's consume() may internally call assertQueue() without parameters if queue doesn't exist
|
|
1720
3293
|
// This would create queue with default arguments (no TTL), causing 406 errors later
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
3294
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] About to call amqplib's consumerChannel.consume(${queue})`);
|
|
3295
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] ⚠ WARNING: amqplib's consume() may internally call assertQueue() WITHOUT parameters if queue doesn't exist`);
|
|
3296
|
+
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] ⚠ WARNING: We already asserted queue with correct parameters above - this should prevent auto-creation`);
|
|
1724
3297
|
|
|
1725
3298
|
// Use dedicated consumer channel for consume operations
|
|
1726
3299
|
// This prevents conflicts with ConfirmChannel used for publish operations
|
|
@@ -1740,9 +3313,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1740
3313
|
|
|
1741
3314
|
// Structured log: message received
|
|
1742
3315
|
const msgHeaders = msg.properties?.headers || {};
|
|
1743
|
-
const wfId = msgHeaders
|
|
3316
|
+
const wfId = workflowIdOf(msgHeaders);
|
|
1744
3317
|
const msgSize = msg.content ? msg.content.length : 0;
|
|
1745
|
-
this.
|
|
3318
|
+
this._logger.debug('[RabbitMQClient] MSG_RECEIVED', {
|
|
3319
|
+
workflow_id: wfId,
|
|
3320
|
+
action: 'MSG_RECEIVED',
|
|
1746
3321
|
handler: 'consume',
|
|
1747
3322
|
function: 'RabbitMQClient.consume',
|
|
1748
3323
|
input: {
|
|
@@ -1760,12 +3335,63 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1760
3335
|
this._checkPrefetchUtilization(queue, tracking);
|
|
1761
3336
|
}
|
|
1762
3337
|
|
|
3338
|
+
// Which attempt this delivery is, read from the header this library
|
|
3339
|
+
// writes (`../config/deliveryPolicy.js`). `null` = the header carries a
|
|
3340
|
+
// value this library never wrote, so the message cannot be bounded and
|
|
3341
|
+
// the handler is NOT run for it; it is rejected below.
|
|
3342
|
+
const attemptsMade = deliveryPolicy.readAttemptsMade(msgHeaders);
|
|
3343
|
+
const attempt = attemptsMade === null ? null : attemptsMade + 1;
|
|
3344
|
+
|
|
3345
|
+
if (attempt === null) {
|
|
3346
|
+
this._logger.error(
|
|
3347
|
+
'[RabbitMQClient] Unreadable delivery counter - the '
|
|
3348
|
+
+ `${deliveryPolicy.ATTEMPTS_HEADER} header of a message on "${queue}" holds `
|
|
3349
|
+
+ `${JSON.stringify(msgHeaders[deliveryPolicy.ATTEMPTS_HEADER])}, which is not a `
|
|
3350
|
+
+ 'non-negative integer. Expected: only this library writes that header. '
|
|
3351
|
+
+ 'Fix: the message is rejected into the dead-letter queue unread — reading a '
|
|
3352
|
+
+ 'malformed counter as zero would make the redelivery loop unbounded again.',
|
|
3353
|
+
{ queue }
|
|
3354
|
+
);
|
|
3355
|
+
await this._rejectDelivery({
|
|
3356
|
+
msg,
|
|
3357
|
+
queue,
|
|
3358
|
+
channelForMsg,
|
|
3359
|
+
noAck,
|
|
3360
|
+
error: new Error(
|
|
3361
|
+
`[RabbitMQClient] Message on "${queue}" carries an unreadable `
|
|
3362
|
+
+ `${deliveryPolicy.ATTEMPTS_HEADER} header`
|
|
3363
|
+
),
|
|
3364
|
+
classification: deliveryPolicy.UNCLASSIFIABLE,
|
|
3365
|
+
attempts: null,
|
|
3366
|
+
maxAttempts: policy.maxAttempts,
|
|
3367
|
+
onDeadLetter
|
|
3368
|
+
});
|
|
3369
|
+
if (tracking) {
|
|
3370
|
+
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
3371
|
+
tracking.lastCheck = Date.now();
|
|
3372
|
+
}
|
|
3373
|
+
return;
|
|
3374
|
+
}
|
|
3375
|
+
|
|
1763
3376
|
try {
|
|
1764
|
-
|
|
3377
|
+
// The handler is told WHICH attempt it is on, so a side effect that
|
|
3378
|
+
// must happen once — the RPC error reply above all — happens on the
|
|
3379
|
+
// last attempt only, instead of once per redelivery. A handler that
|
|
3380
|
+
// takes one argument ignores the second.
|
|
3381
|
+
await onMessage(msg, {
|
|
3382
|
+
attempt,
|
|
3383
|
+
maxAttempts: policy.maxAttempts,
|
|
3384
|
+
isFinalAttempt: attempt >= policy.maxAttempts
|
|
3385
|
+
});
|
|
1765
3386
|
// Acknowledge message after successful processing
|
|
1766
3387
|
if (!noAck && !msg._mqProcessed) {
|
|
1767
|
-
|
|
1768
|
-
|
|
3388
|
+
// Liveness comes from the ONE source the channel handlers set
|
|
3389
|
+
// first-hand (`_isChannelAlive()`, d.260). The reading this
|
|
3390
|
+
// replaced, `channelForMsg.closed`, asked amqplib for a property it
|
|
3391
|
+
// does not define, so it answered "alive" for the channel the
|
|
3392
|
+
// broker had just killed and handed the corpse an `ack`.
|
|
3393
|
+
if (!this._isChannelAlive(channelForMsg)) {
|
|
3394
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] [CONSUMER] Cannot ack - consumer channel is closed/recreated (message will be requeued by broker)');
|
|
1769
3395
|
return;
|
|
1770
3396
|
}
|
|
1771
3397
|
try {
|
|
@@ -1775,12 +3401,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1775
3401
|
const m = ackErr && ackErr.message ? ackErr.message : '';
|
|
1776
3402
|
if (m.includes('unknown delivery tag') || m.includes('PRECONDITION_FAILED') || m.includes('Channel closed')) {
|
|
1777
3403
|
msg._mqProcessed = true;
|
|
1778
|
-
|
|
3404
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] [CONSUMER] Ack failed due to invalid delivery tag / closed channel (message delivery will be handled by broker)', { error: m });
|
|
1779
3405
|
return;
|
|
1780
3406
|
}
|
|
1781
3407
|
throw ackErr;
|
|
1782
3408
|
}
|
|
1783
|
-
this.
|
|
3409
|
+
this._logger.debug('[RabbitMQClient] MSG_ACKED', {
|
|
3410
|
+
workflow_id: wfId,
|
|
3411
|
+
action: 'MSG_ACKED',
|
|
1784
3412
|
handler: 'consume',
|
|
1785
3413
|
function: 'RabbitMQClient.consume',
|
|
1786
3414
|
input: { queue },
|
|
@@ -1793,17 +3421,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1793
3421
|
}
|
|
1794
3422
|
}
|
|
1795
3423
|
} catch (handlerErr) {
|
|
1796
|
-
//
|
|
1797
|
-
//
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
3424
|
+
// The dead-letter policy decides what a failed handler costs: one more
|
|
3425
|
+
// attempt (a copy carrying the advanced counter, then an ack of the
|
|
3426
|
+
// original) or a rejection into `<svc>.dlq`. Until 2026-09-11 this
|
|
3427
|
+
// branch was an unconditional `nack(msg, false, true)` with no counter,
|
|
3428
|
+
// so a permanently failing message was redelivered for ever and the DLQ
|
|
3429
|
+
// the topology declares was never reached from the client side
|
|
3430
|
+
// (`docs/governance/confirmations/mq-consumer-contract.md` 002 point 3).
|
|
3431
|
+
await this._applyDeliveryPolicy({
|
|
3432
|
+
msg,
|
|
3433
|
+
queue,
|
|
3434
|
+
channelForMsg,
|
|
3435
|
+
noAck,
|
|
3436
|
+
error: handlerErr,
|
|
3437
|
+
attempt,
|
|
3438
|
+
policy,
|
|
3439
|
+
onDeadLetter
|
|
3440
|
+
});
|
|
1807
3441
|
if (tracking) {
|
|
1808
3442
|
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
1809
3443
|
tracking.lastCheck = Date.now();
|
|
@@ -1824,7 +3458,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1824
3458
|
consumerTag: consumeResult.consumerTag
|
|
1825
3459
|
});
|
|
1826
3460
|
|
|
1827
|
-
|
|
3461
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Consumer registered for queue "${queue}" (consumerTag: ${consumeResult.consumerTag})`);
|
|
1828
3462
|
|
|
1829
3463
|
// Return consumer tag for cancellation
|
|
1830
3464
|
return consumeResult.consumerTag;
|
|
@@ -1841,15 +3475,37 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1841
3475
|
* @returns {Promise<void>}
|
|
1842
3476
|
*/
|
|
1843
3477
|
async _reconnectWithBackoff() {
|
|
3478
|
+
if (this._closedByCaller) {
|
|
3479
|
+
this._logger.debug(
|
|
3480
|
+
'[RabbitMQClient] Client was closed by its caller - no connection-level recovery'
|
|
3481
|
+
);
|
|
3482
|
+
return;
|
|
3483
|
+
}
|
|
3484
|
+
if (this._connectionFatal) {
|
|
3485
|
+
this._logger.warn(
|
|
3486
|
+
'[RabbitMQClient] Connection already declared permanently lost - no further recovery attempts'
|
|
3487
|
+
);
|
|
3488
|
+
return;
|
|
3489
|
+
}
|
|
1844
3490
|
if (this._reconnecting) {
|
|
1845
|
-
|
|
3491
|
+
this._logger.debug('[RabbitMQClient] Reconnection already in progress, skipping');
|
|
1846
3492
|
return;
|
|
1847
3493
|
}
|
|
1848
3494
|
|
|
1849
3495
|
this._reconnecting = true;
|
|
1850
|
-
|
|
3496
|
+
// A CYCLE is one whole attempt budget. The first is started by the death of
|
|
3497
|
+
// the connection, every further one by `_resumeRecovery()` when somebody uses
|
|
3498
|
+
// the client. The counter is what makes the lazy retry finite, and it goes
|
|
3499
|
+
// back to 0 the moment a cycle succeeds.
|
|
3500
|
+
this._reconnectCycles += 1;
|
|
3501
|
+
this._logger.info(
|
|
3502
|
+
`[RabbitMQClient] Starting connection-level recovery, cycle ${this._reconnectCycles}/${this._maxReconnectCycles} `
|
|
3503
|
+
+ `(attempt ${this._reconnectAttempts + 1}/${this._maxReconnectAttempts})...`
|
|
3504
|
+
);
|
|
3505
|
+
|
|
3506
|
+
let lastError = null;
|
|
1851
3507
|
|
|
1852
|
-
while (this._reconnectAttempts < this._maxReconnectAttempts && !this.
|
|
3508
|
+
while (this._reconnectAttempts < this._maxReconnectAttempts && !this._closedByCaller) {
|
|
1853
3509
|
try {
|
|
1854
3510
|
// Calculate exponential backoff delay: baseDelay * 2^attempts, capped at maxDelay
|
|
1855
3511
|
const delay = Math.min(
|
|
@@ -1857,72 +3513,70 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1857
3513
|
this._reconnectMaxDelay
|
|
1858
3514
|
);
|
|
1859
3515
|
|
|
1860
|
-
|
|
3516
|
+
this._logger.debug(`[RabbitMQClient] Waiting ${delay}ms before reconnection attempt ${this._reconnectAttempts + 1}...`);
|
|
1861
3517
|
await new Promise(resolve => this._setTimeout(resolve, delay));
|
|
1862
3518
|
|
|
1863
3519
|
// Check if we started disconnecting during the wait
|
|
1864
|
-
if (this.
|
|
1865
|
-
|
|
3520
|
+
if (this._closedByCaller) {
|
|
3521
|
+
this._logger.info('[RabbitMQClient] Disconnecting during reconnection wait, aborting');
|
|
3522
|
+
this._reconnecting = false;
|
|
1866
3523
|
return;
|
|
1867
3524
|
}
|
|
1868
3525
|
|
|
1869
3526
|
// Attempt to reconnect
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
// Close old connection if exists
|
|
1873
|
-
if (this._connection) {
|
|
1874
|
-
try {
|
|
1875
|
-
await this._connection.close();
|
|
1876
|
-
} catch (_) {
|
|
1877
|
-
// Ignore errors when closing already-closed connection
|
|
1878
|
-
}
|
|
1879
|
-
this._connection = null;
|
|
1880
|
-
}
|
|
3527
|
+
this._logger.info(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts + 1}...`);
|
|
1881
3528
|
|
|
1882
3529
|
// Reconnect
|
|
1883
|
-
const
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
this._config.clientName ||
|
|
1888
|
-
`oa-client:${runtimeCfg.get('serviceName')}:${process.pid}`;
|
|
1889
|
-
|
|
1890
|
-
const connectArgs = typeof rawTarget === 'string'
|
|
1891
|
-
? [rawTarget, { heartbeat, clientProperties: { connection_name: connectionName } }]
|
|
1892
|
-
: [{ ...rawTarget, heartbeat, clientProperties: { connection_name: connectionName } }];
|
|
1893
|
-
|
|
1894
|
-
const connectPromise = amqp.connect(...connectArgs);
|
|
3530
|
+
const connectPromise = amqp.connect(
|
|
3531
|
+
this._connectionTarget(),
|
|
3532
|
+
this._connectionSocketOptions()
|
|
3533
|
+
);
|
|
1895
3534
|
const timeoutPromise = new Promise((_, reject) => {
|
|
1896
|
-
this._setTimeout(() => reject(new
|
|
3535
|
+
this._setTimeout(() => reject(new ConnectionError(
|
|
3536
|
+
`[RabbitMQClient] Connection timeout after ${this._connectTimeout} ms while reconnecting - `
|
|
3537
|
+
+ `Expected: the broker to accept a TCP+AMQP handshake within connectTimeout (${this._connectTimeout} ms). `
|
|
3538
|
+
+ 'Fix: check the broker is up and reachable from this container; the reconnect retries until the attempt budget is spent.'
|
|
3539
|
+
)), this._connectTimeout);
|
|
1897
3540
|
});
|
|
1898
3541
|
|
|
1899
|
-
|
|
1900
|
-
console.log('[RabbitMQClient] ✓ Connection re-established');
|
|
1901
|
-
|
|
1902
|
-
// Re-attach connection event handlers
|
|
1903
|
-
this._connection.on('error', (err) => {
|
|
1904
|
-
console.error('[RabbitMQClient] Connection error:', err.message);
|
|
1905
|
-
this.emit('error', err);
|
|
1906
|
-
});
|
|
3542
|
+
const revived = await Promise.race([connectPromise, timeoutPromise]);
|
|
1907
3543
|
|
|
1908
|
-
this
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
3544
|
+
// The caller may have disposed of this client while that handshake was
|
|
3545
|
+
// in flight. The socket is up, and it belongs to nobody: keeping it
|
|
3546
|
+
// would revive a client its owner has already ended, and dropping the
|
|
3547
|
+
// reference without closing it would leak an open handle that keeps the
|
|
3548
|
+
// process alive. Close it, and stop.
|
|
3549
|
+
if (this._closedByCaller) {
|
|
3550
|
+
this._logger.info(
|
|
3551
|
+
'[RabbitMQClient] Client was closed by its caller during recovery - closing the re-established connection'
|
|
3552
|
+
);
|
|
3553
|
+
try {
|
|
3554
|
+
await revived.close();
|
|
3555
|
+
} catch (closeErr) {
|
|
3556
|
+
this._logger.warn(
|
|
3557
|
+
`[RabbitMQClient] Error closing the connection re-established after disconnect: ${closeErr.message}`
|
|
3558
|
+
);
|
|
1921
3559
|
}
|
|
1922
|
-
|
|
3560
|
+
this._connection = null;
|
|
3561
|
+
this._connectionAlive = false;
|
|
3562
|
+
this._reconnecting = false;
|
|
3563
|
+
this._clearAllTimers();
|
|
3564
|
+
return;
|
|
3565
|
+
}
|
|
3566
|
+
|
|
3567
|
+
this._connection = revived;
|
|
3568
|
+
// The socket is up. The CLIENT is not yet: `_reconnecting` stays true
|
|
3569
|
+
// until the channels below are back, and `isConnected()` reads it, so
|
|
3570
|
+
// nobody is told "connected" while there is nothing to publish on
|
|
3571
|
+
// (d.340).
|
|
3572
|
+
this._connectionAlive = true;
|
|
3573
|
+
this._logger.info('[RabbitMQClient] ✓ Connection re-established');
|
|
3574
|
+
|
|
3575
|
+
// Re-attach connection event handlers — the same rail as the first connect.
|
|
3576
|
+
this._attachConnectionHandlers(this._connection);
|
|
1923
3577
|
|
|
1924
3578
|
// Recreate all channels with best effort - partial success is better than total failure
|
|
1925
|
-
|
|
3579
|
+
this._logger.debug('[RabbitMQClient] Recreating channels...');
|
|
1926
3580
|
|
|
1927
3581
|
const channelState = { publisher: false, queue: false, consumer: false };
|
|
1928
3582
|
let consumerReRegResult = { reRegistered: 0, failed: 0 };
|
|
@@ -1931,9 +3585,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1931
3585
|
try {
|
|
1932
3586
|
await this._ensurePublisherChannel();
|
|
1933
3587
|
channelState.publisher = true;
|
|
1934
|
-
|
|
3588
|
+
this._logger.info('[RabbitMQClient] ✓ Publisher channel recreated');
|
|
1935
3589
|
} catch (err) {
|
|
1936
|
-
|
|
3590
|
+
this._logger.error('[RabbitMQClient] Failed to recreate publisher channel:', err.message);
|
|
1937
3591
|
// Continue - partial success is better than total failure
|
|
1938
3592
|
}
|
|
1939
3593
|
|
|
@@ -1941,9 +3595,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1941
3595
|
try {
|
|
1942
3596
|
await this._ensureQueueChannel();
|
|
1943
3597
|
channelState.queue = true;
|
|
1944
|
-
|
|
3598
|
+
this._logger.info('[RabbitMQClient] ✓ Queue channel recreated');
|
|
1945
3599
|
} catch (err) {
|
|
1946
|
-
|
|
3600
|
+
this._logger.error('[RabbitMQClient] Failed to recreate queue channel:', err.message);
|
|
1947
3601
|
// Continue - partial success is better than total failure
|
|
1948
3602
|
}
|
|
1949
3603
|
|
|
@@ -1959,15 +3613,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1959
3613
|
// Re-register all consumers - use internal helper to avoid the "channel already open" check
|
|
1960
3614
|
consumerReRegResult = await this._reRegisterConsumers();
|
|
1961
3615
|
channelState.consumer = true;
|
|
1962
|
-
|
|
3616
|
+
this._logger.info('[RabbitMQClient] ✓ Consumer channel recreated');
|
|
1963
3617
|
} catch (err) {
|
|
1964
|
-
|
|
3618
|
+
this._logger.error('[RabbitMQClient] Failed to recreate consumer channel:', err.message);
|
|
1965
3619
|
this._consumerChannel = null;
|
|
1966
3620
|
// Continue - partial success is better than total failure
|
|
1967
3621
|
}
|
|
1968
3622
|
|
|
1969
|
-
// Reset reconnect state
|
|
3623
|
+
// Reset reconnect state. The cycle counter goes with it: a client that
|
|
3624
|
+
// recovered owes nothing for the outage it survived, so the next one
|
|
3625
|
+
// starts again at cycle 1 (d.339).
|
|
1970
3626
|
this._reconnectAttempts = 0;
|
|
3627
|
+
this._reconnectCycles = 0;
|
|
3628
|
+
this._lastConnectionError = null;
|
|
1971
3629
|
this._reconnecting = false;
|
|
1972
3630
|
|
|
1973
3631
|
// Emit reconnected event with state information
|
|
@@ -1978,49 +3636,85 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1978
3636
|
timestamp: new Date().toISOString()
|
|
1979
3637
|
};
|
|
1980
3638
|
|
|
1981
|
-
|
|
3639
|
+
this._logger.info('[RabbitMQClient] ✓ Connection-level recovery completed', JSON.stringify(reconnectState, null, 2));
|
|
1982
3640
|
this.emit('reconnected', reconnectState);
|
|
1983
3641
|
|
|
1984
3642
|
// Flush buffered messages po reconnectu
|
|
1985
|
-
let bufferFlushResult = { inMemory: 0,
|
|
3643
|
+
let bufferFlushResult = { inMemory: 0, flushed: false };
|
|
1986
3644
|
try {
|
|
1987
3645
|
const flushResult = await this._publishLayer.flushBuffered();
|
|
1988
|
-
bufferFlushResult = {
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
flushed: true
|
|
1992
|
-
};
|
|
1993
|
-
this._publishMonitor.trackFlushed(bufferFlushResult.inMemory + bufferFlushResult.persistent);
|
|
1994
|
-
console.log(`[RabbitMQClient] ✓ Flushed ${bufferFlushResult.inMemory + bufferFlushResult.persistent} buffered messages after reconnection`);
|
|
3646
|
+
bufferFlushResult = { inMemory: flushResult, flushed: true };
|
|
3647
|
+
this._publishMonitor.trackFlushed(bufferFlushResult.inMemory);
|
|
3648
|
+
this._logger.info(`[RabbitMQClient] ✓ Flushed ${bufferFlushResult.inMemory} buffered messages after reconnection`);
|
|
1995
3649
|
} catch (flushErr) {
|
|
1996
|
-
|
|
3650
|
+
this._logger.warn(`[RabbitMQClient] Failed to flush buffered messages after reconnection: ${flushErr.message}`);
|
|
1997
3651
|
this.emit('buffer:flush:failed', { error: flushErr.message, bufferSize: this._publishLayer._buffer.size() });
|
|
1998
3652
|
}
|
|
1999
3653
|
|
|
2000
3654
|
// If critical channels failed, log warning but don't throw - connection is re-established
|
|
2001
3655
|
if (!channelState.publisher) {
|
|
2002
|
-
|
|
3656
|
+
this._logger.warn('[RabbitMQClient] ⚠ Publisher channel not ready after reconnection - publish operations may fail');
|
|
2003
3657
|
}
|
|
2004
3658
|
if (!channelState.queue) {
|
|
2005
|
-
|
|
3659
|
+
this._logger.warn('[RabbitMQClient] ⚠ Queue channel not ready after reconnection - queue operations may fail');
|
|
2006
3660
|
}
|
|
2007
3661
|
if (!channelState.consumer) {
|
|
2008
|
-
|
|
3662
|
+
this._logger.warn('[RabbitMQClient] ⚠ Consumer channel not ready after reconnection - consume operations may fail');
|
|
2009
3663
|
}
|
|
2010
3664
|
|
|
2011
3665
|
return; // Success - exit reconnection loop
|
|
2012
3666
|
} catch (err) {
|
|
3667
|
+
lastError = err;
|
|
2013
3668
|
this._reconnectAttempts++;
|
|
2014
|
-
|
|
2015
|
-
|
|
3669
|
+
this._logger.error(`[RabbitMQClient] Reconnection attempt ${this._reconnectAttempts} failed:`, err.message);
|
|
3670
|
+
|
|
3671
|
+
// The broker ANSWERED and refused. Spending the rest of the budget on it
|
|
3672
|
+
// would be retrying a decision, not an outage (d.339).
|
|
3673
|
+
if (this._isBrokerRefusal(err)) {
|
|
3674
|
+
this._logger.error('[RabbitMQClient] ✗ The broker refused the connection - no further attempts');
|
|
3675
|
+
throw this._failFatally(err, 'broker-refused');
|
|
3676
|
+
}
|
|
3677
|
+
|
|
2016
3678
|
if (this._reconnectAttempts >= this._maxReconnectAttempts) {
|
|
2017
|
-
|
|
2018
|
-
this.
|
|
2019
|
-
throw new Error(`Failed to reconnect after ${this._maxReconnectAttempts} attempts: ${err.message}`);
|
|
3679
|
+
this._logger.error(`[RabbitMQClient] ✗ Connection-level recovery failed after ${this._maxReconnectAttempts} attempts`);
|
|
3680
|
+
throw this._endOfCycle(err);
|
|
2020
3681
|
}
|
|
2021
3682
|
// Continue to next attempt
|
|
2022
3683
|
}
|
|
2023
3684
|
}
|
|
3685
|
+
|
|
3686
|
+
// The loop can also fall through without ever running its body: the budget
|
|
3687
|
+
// was already spent by an earlier recovery round. Before 2026-09-07 that
|
|
3688
|
+
// path returned silently with `_reconnecting` left at true, which made every
|
|
3689
|
+
// later recovery a no-op ("already in progress") and hung
|
|
3690
|
+
// `_waitForReconnection()` for its full timeout. The state is the same as an
|
|
3691
|
+
// exhausted round, so it is reported the same way.
|
|
3692
|
+
if (this._closedByCaller) {
|
|
3693
|
+
this._reconnecting = false;
|
|
3694
|
+
return;
|
|
3695
|
+
}
|
|
3696
|
+
throw this._endOfCycle(lastError);
|
|
3697
|
+
}
|
|
3698
|
+
|
|
3699
|
+
/**
|
|
3700
|
+
* A cycle ended without a connection and the broker never refused one. Which
|
|
3701
|
+
* of the two endings is it — stand down and wait to be used, or the bounded
|
|
3702
|
+
* end of the lazy retry?
|
|
3703
|
+
*
|
|
3704
|
+
* ONE place decides, because there are two sites that reach this state (the
|
|
3705
|
+
* attempt budget running out inside the loop, and the loop falling through on
|
|
3706
|
+
* a budget an earlier round already spent) and a decision written twice
|
|
3707
|
+
* diverges.
|
|
3708
|
+
*
|
|
3709
|
+
* @param {Error|null} lastError
|
|
3710
|
+
* @returns {Error} the error to throw
|
|
3711
|
+
* @private
|
|
3712
|
+
*/
|
|
3713
|
+
_endOfCycle(lastError) {
|
|
3714
|
+
if (this._reconnectCycles >= this._maxReconnectCycles) {
|
|
3715
|
+
return this._failFatally(lastError, 'cycles-spent');
|
|
3716
|
+
}
|
|
3717
|
+
return this._standDown(lastError);
|
|
2024
3718
|
}
|
|
2025
3719
|
|
|
2026
3720
|
/**
|
|
@@ -2028,64 +3722,75 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2028
3722
|
* @private
|
|
2029
3723
|
*/
|
|
2030
3724
|
_attachPublisherChannelHandlers(channel) {
|
|
3725
|
+
// One closure, one notification. amqplib announces a broker-side channel
|
|
3726
|
+
// death TWICE — `error` first, then `close` (measured against amqplib 0.10.9
|
|
3727
|
+
// on the live broker, 2026-09-12: a `checkQueue()` on a missing queue yields
|
|
3728
|
+
// `["error:404","close","rpcReject:404"]`). Until d.264 both handlers called
|
|
3729
|
+
// `_callChannelCloseHooks()`, so every hook and every `channel:close`
|
|
3730
|
+
// listener heard one death twice and `_trackChannelClose()` recorded two
|
|
3731
|
+
// entries for it — which made channel thrashing fire at HALF of
|
|
3732
|
+
// `thrashingThreshold`, an alarm at the wrong number.
|
|
3733
|
+
// The rail is the `close` handler, because `close` always arrives, with or
|
|
3734
|
+
// without a preceding `error`. The `error` handler only records WHY the
|
|
3735
|
+
// channel is dying, here, for the one notification to carry.
|
|
3736
|
+
channel._closeError = null;
|
|
3737
|
+
channel._alive = true;
|
|
2031
3738
|
channel.on('error', async (err) => {
|
|
2032
|
-
|
|
3739
|
+
channel._alive = false;
|
|
3740
|
+
if (this._closedByCaller) {
|
|
2033
3741
|
return;
|
|
2034
3742
|
}
|
|
2035
3743
|
const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
2036
3744
|
channel._closeReason = reason;
|
|
2037
|
-
|
|
3745
|
+
|
|
2038
3746
|
// "unknown delivery tag" errors occur when channel closes during publish
|
|
2039
3747
|
// This is expected - the publish callback will handle retry
|
|
2040
3748
|
// We still log it but don't try to recreate channel (it's already closing)
|
|
2041
3749
|
if (err.message && err.message.includes('unknown delivery tag')) {
|
|
2042
|
-
|
|
3750
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Publisher channel error (channel closing during publish): ${err.message}`);
|
|
2043
3751
|
// Don't try to recreate - channel is already closing
|
|
2044
3752
|
// Don't emit error - publish callback will handle retry
|
|
2045
3753
|
return;
|
|
2046
3754
|
}
|
|
2047
3755
|
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
// Only try to recreate if connection is available
|
|
2052
|
-
// If connection is closed, reconnection logic will handle channel recreation
|
|
2053
|
-
if (this._connection && !this._connection.closed) {
|
|
2054
|
-
try {
|
|
2055
|
-
await this._ensurePublisherChannel();
|
|
2056
|
-
} catch (recreateErr) {
|
|
2057
|
-
// Ignore errors during channel recreation if connection is closing
|
|
2058
|
-
console.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${recreateErr.message}`);
|
|
2059
|
-
}
|
|
2060
|
-
}
|
|
2061
|
-
|
|
3756
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
3757
|
+
channel._closeError = err;
|
|
3758
|
+
// Recovery is NOT started here — one rail, the `close` handler (d.261).
|
|
2062
3759
|
this.emit('error', err);
|
|
2063
3760
|
});
|
|
2064
3761
|
|
|
2065
3762
|
channel.on('close', async () => {
|
|
2066
|
-
if (this.
|
|
3763
|
+
if (this._closedByCaller) {
|
|
2067
3764
|
return;
|
|
2068
3765
|
}
|
|
3766
|
+
channel._alive = false;
|
|
2069
3767
|
const reason = channel._closeReason || 'Publisher channel closed unexpectedly';
|
|
2070
|
-
|
|
2071
|
-
this._callChannelCloseHooks('publisher', channel, null, reason);
|
|
2072
|
-
|
|
2073
|
-
|
|
3768
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next publish`);
|
|
3769
|
+
this._callChannelCloseHooks('publisher', channel, channel._closeError || null, reason);
|
|
3770
|
+
// Clear the reference ONLY if it still points at the channel that died.
|
|
3771
|
+
// A stale channel's obituary must not erase a live successor: measured
|
|
3772
|
+
// 2026-09-12 on the live integration tier, an unconditional `= null` here
|
|
3773
|
+
// wiped a channel that had just been opened, and `consume()` then refused
|
|
3774
|
+
// with "Queue channel is not available" against a perfectly open
|
|
3775
|
+
// connection.
|
|
3776
|
+
if (this._channel === channel) {
|
|
3777
|
+
this._channel = null;
|
|
3778
|
+
}
|
|
2074
3779
|
|
|
2075
3780
|
// Only try to recreate if connection is available
|
|
2076
3781
|
// If connection is closed, reconnection logic will handle channel recreation
|
|
2077
|
-
if (this.
|
|
3782
|
+
if (this._connectionAlive) {
|
|
2078
3783
|
try {
|
|
2079
3784
|
await this._ensurePublisherChannel();
|
|
2080
3785
|
} catch (err) {
|
|
2081
3786
|
// Ignore errors during channel recreation if connection is closing
|
|
2082
|
-
|
|
3787
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection may be closing): ${err.message}`);
|
|
2083
3788
|
}
|
|
2084
3789
|
}
|
|
2085
3790
|
});
|
|
2086
3791
|
|
|
2087
3792
|
channel.on('drain', () => {
|
|
2088
|
-
|
|
3793
|
+
this._logger.debug('[RabbitMQClient] [mq-client-core] Publisher channel drained');
|
|
2089
3794
|
});
|
|
2090
3795
|
}
|
|
2091
3796
|
|
|
@@ -2094,43 +3799,56 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2094
3799
|
* @private
|
|
2095
3800
|
*/
|
|
2096
3801
|
_attachQueueChannelHandlers(channel) {
|
|
3802
|
+
// One closure, one notification — see the note in
|
|
3803
|
+
// `_attachPublisherChannelHandlers()` for the measurement behind it.
|
|
3804
|
+
channel._closeError = null;
|
|
3805
|
+
channel._alive = true;
|
|
2097
3806
|
channel.on('error', async (err) => {
|
|
2098
|
-
|
|
3807
|
+
channel._alive = false;
|
|
3808
|
+
if (this._closedByCaller) {
|
|
2099
3809
|
return;
|
|
2100
3810
|
}
|
|
2101
3811
|
const reason = `Queue channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
2102
3812
|
channel._closeReason = reason;
|
|
2103
|
-
|
|
2104
|
-
this.
|
|
2105
|
-
|
|
3813
|
+
channel._closeError = err;
|
|
3814
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
3815
|
+
// Recovery is NOT started here — one rail, the `close` handler (d.261).
|
|
2106
3816
|
});
|
|
2107
3817
|
|
|
2108
3818
|
channel.on('close', async () => {
|
|
2109
|
-
if (this.
|
|
3819
|
+
if (this._closedByCaller) {
|
|
2110
3820
|
return;
|
|
2111
3821
|
}
|
|
3822
|
+
channel._alive = false;
|
|
2112
3823
|
const reason = channel._closeReason || 'Queue channel closed unexpectedly';
|
|
2113
|
-
|
|
2114
|
-
this._callChannelCloseHooks('queue', channel, null, reason);
|
|
2115
|
-
|
|
2116
|
-
|
|
3824
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate on next operation`);
|
|
3825
|
+
this._callChannelCloseHooks('queue', channel, channel._closeError || null, reason);
|
|
3826
|
+
// Clear the reference ONLY if it still points at the channel that died.
|
|
3827
|
+
// A stale channel's obituary must not erase a live successor: measured
|
|
3828
|
+
// 2026-09-12 on the live integration tier, an unconditional `= null` here
|
|
3829
|
+
// wiped a channel that had just been opened, and `consume()` then refused
|
|
3830
|
+
// with "Queue channel is not available" against a perfectly open
|
|
3831
|
+
// connection.
|
|
3832
|
+
if (this._queueChannel === channel) {
|
|
3833
|
+
this._queueChannel = null;
|
|
3834
|
+
}
|
|
2117
3835
|
|
|
2118
3836
|
// PREDICTABLE: Only recreate if connection is available and not closing
|
|
2119
3837
|
// If connection is closed, reconnection logic will handle channel recreation
|
|
2120
|
-
if (this.
|
|
3838
|
+
if (this._connectionAlive && !this._reconnecting) {
|
|
2121
3839
|
try {
|
|
2122
3840
|
await this._ensureQueueChannel();
|
|
2123
3841
|
} catch (err) {
|
|
2124
3842
|
// If connection is closing, this is expected - don't throw
|
|
2125
3843
|
if (err.message && err.message.includes('Connection closed')) {
|
|
2126
|
-
|
|
3844
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Queue channel close during connection closure (expected - reconnection will handle)`);
|
|
2127
3845
|
} else {
|
|
2128
3846
|
// Log but don't throw - channel will be recreated on next use or during reconnection
|
|
2129
|
-
|
|
3847
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel: ${err.message} (will be recreated on next use)`);
|
|
2130
3848
|
}
|
|
2131
3849
|
}
|
|
2132
3850
|
} else {
|
|
2133
|
-
|
|
3851
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Queue channel closed - will be recreated during reconnection or on next use`);
|
|
2134
3852
|
}
|
|
2135
3853
|
});
|
|
2136
3854
|
}
|
|
@@ -2140,36 +3858,49 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2140
3858
|
* @private
|
|
2141
3859
|
*/
|
|
2142
3860
|
_attachConsumerChannelHandlers(channel) {
|
|
3861
|
+
// One closure, one notification — see the note in
|
|
3862
|
+
// `_attachPublisherChannelHandlers()` for the measurement behind it.
|
|
3863
|
+
channel._closeError = null;
|
|
3864
|
+
channel._alive = true;
|
|
2143
3865
|
channel.on('error', async (err) => {
|
|
2144
|
-
|
|
3866
|
+
channel._alive = false;
|
|
3867
|
+
if (this._closedByCaller) {
|
|
2145
3868
|
return;
|
|
2146
3869
|
}
|
|
2147
3870
|
const reason = `Consumer channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
2148
3871
|
channel._closeReason = reason;
|
|
2149
|
-
|
|
2150
|
-
this.
|
|
2151
|
-
|
|
3872
|
+
channel._closeError = err;
|
|
3873
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
3874
|
+
// Recovery is NOT started here — one rail, the `close` handler (d.261).
|
|
2152
3875
|
this.emit('error', err);
|
|
2153
3876
|
});
|
|
2154
3877
|
|
|
2155
3878
|
channel.on('close', async () => {
|
|
2156
|
-
if (this.
|
|
3879
|
+
if (this._closedByCaller) {
|
|
2157
3880
|
return;
|
|
2158
3881
|
}
|
|
3882
|
+
channel._alive = false;
|
|
2159
3883
|
const reason = channel._closeReason || 'Consumer channel closed unexpectedly';
|
|
2160
|
-
|
|
2161
|
-
this._callChannelCloseHooks('consumer', channel, null, reason);
|
|
2162
|
-
|
|
2163
|
-
|
|
3884
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] ${reason} - will auto-recreate and re-register consumers`);
|
|
3885
|
+
this._callChannelCloseHooks('consumer', channel, channel._closeError || null, reason);
|
|
3886
|
+
// Clear the reference ONLY if it still points at the channel that died.
|
|
3887
|
+
// A stale channel's obituary must not erase a live successor: measured
|
|
3888
|
+
// 2026-09-12 on the live integration tier, an unconditional `= null` here
|
|
3889
|
+
// wiped a channel that had just been opened, and `consume()` then refused
|
|
3890
|
+
// with "Queue channel is not available" against a perfectly open
|
|
3891
|
+
// connection.
|
|
3892
|
+
if (this._consumerChannel === channel) {
|
|
3893
|
+
this._consumerChannel = null;
|
|
3894
|
+
}
|
|
2164
3895
|
|
|
2165
3896
|
// Only try to recreate if connection is available
|
|
2166
3897
|
// If connection is closed, reconnection logic will handle channel recreation
|
|
2167
|
-
if (this.
|
|
3898
|
+
if (this._connectionAlive) {
|
|
2168
3899
|
try {
|
|
2169
3900
|
await this._ensureConsumerChannel();
|
|
2170
3901
|
} catch (err) {
|
|
2171
3902
|
// Ignore errors during channel recreation if connection is closing
|
|
2172
|
-
|
|
3903
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel (connection may be closing): ${err.message}`);
|
|
2173
3904
|
}
|
|
2174
3905
|
}
|
|
2175
3906
|
});
|
|
@@ -2190,14 +3921,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2190
3921
|
|
|
2191
3922
|
if (utilization >= this._prefetchUtilizationThreshold) {
|
|
2192
3923
|
const message = `Prefetch utilization high: queue '${queue}' has ${tracking.inFlight}/${tracking.prefetchCount} messages in-flight (${Math.round(utilization * 100)}%, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`;
|
|
2193
|
-
|
|
3924
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] ⚠️ ${message}`);
|
|
2194
3925
|
|
|
2195
3926
|
// Call alert callback if provided
|
|
2196
3927
|
if (this._prefetchAlertCallback) {
|
|
2197
3928
|
try {
|
|
2198
3929
|
this._prefetchAlertCallback(queue, utilization, tracking.inFlight, tracking.prefetchCount);
|
|
2199
3930
|
} catch (alertErr) {
|
|
2200
|
-
|
|
3931
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Prefetch alert callback error:`, alertErr.message);
|
|
2201
3932
|
}
|
|
2202
3933
|
}
|
|
2203
3934
|
|
|
@@ -2232,7 +3963,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2232
3963
|
}
|
|
2233
3964
|
}, this._prefetchCheckInterval);
|
|
2234
3965
|
|
|
2235
|
-
|
|
3966
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] Prefetch monitoring started (interval: ${this._prefetchCheckInterval}ms, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`);
|
|
2236
3967
|
}
|
|
2237
3968
|
|
|
2238
3969
|
/**
|
|
@@ -2258,6 +3989,219 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2258
3989
|
}
|
|
2259
3990
|
}
|
|
2260
3991
|
|
|
3992
|
+
/**
|
|
3993
|
+
* Answer one failed delivery, per the dead-letter policy.
|
|
3994
|
+
*
|
|
3995
|
+
* Transient and budget left → publish a copy carrying the advanced counter to
|
|
3996
|
+
* the SAME queue, then ack the original. That order is the whole safety
|
|
3997
|
+
* argument: a copy that reaches the broker before the original leaves the
|
|
3998
|
+
* queue can at worst duplicate the message, while an ack before the publish
|
|
3999
|
+
* can lose it. A requeue cannot be used for this — it returns the message
|
|
4000
|
+
* byte-for-byte, so it can carry no counter (measured; see
|
|
4001
|
+
* ../config/deliveryPolicy.js).
|
|
4002
|
+
*
|
|
4003
|
+
* Anything else → reject into the dead-letter queue the broker routes for
|
|
4004
|
+
* this queue (`x-dead-letter-routing-key`, `<svc>.dlq`).
|
|
4005
|
+
*
|
|
4006
|
+
* @param {Object} args
|
|
4007
|
+
* @private
|
|
4008
|
+
*/
|
|
4009
|
+
async _applyDeliveryPolicy({ msg, queue, channelForMsg, noAck, error, attempt, policy, onDeadLetter }) {
|
|
4010
|
+
// The handler settled the message itself, or there is nothing to settle:
|
|
4011
|
+
// both are the pre-existing contract of this branch, unchanged.
|
|
4012
|
+
if (noAck || msg._mqProcessed) {
|
|
4013
|
+
return;
|
|
4014
|
+
}
|
|
4015
|
+
|
|
4016
|
+
// A dead channel is a THIRD case, and it is not silent. The policy spends
|
|
4017
|
+
// an attempt by publishing a copy and acking the original; on a channel the
|
|
4018
|
+
// broker has killed the ack cannot land, so the broker returns the original
|
|
4019
|
+
// to the queue and the copy duplicates it. So nothing is settled and
|
|
4020
|
+
// nothing is published — the delivery goes back to the queue by itself —
|
|
4021
|
+
// and it is said out loud, because an attempt that never ran must not look
|
|
4022
|
+
// like one that was spent. Liveness from the one source (`_isChannelAlive`,
|
|
4023
|
+
// d.260); the reading this replaced (`channelForMsg.closed`) answered
|
|
4024
|
+
// "alive" for every channel, amqplib defining no such property.
|
|
4025
|
+
if (!this._isChannelAlive(channelForMsg)) {
|
|
4026
|
+
this._logger.warn(
|
|
4027
|
+
`[RabbitMQClient] [mq-client-core] [CONSUMER] Failed delivery on "${queue}" left unsettled - `
|
|
4028
|
+
+ 'the consumer channel died while the handler ran, so neither the retry copy nor the '
|
|
4029
|
+
+ 'rejection may be issued on it. Expected: the broker returns an unacked delivery to the '
|
|
4030
|
+
+ 'queue when the channel goes. Fix: none needed here — the message is redelivered and this '
|
|
4031
|
+
+ 'attempt is not counted against its budget.',
|
|
4032
|
+
{ queue, attempt, error: error && error.message ? error.message : null }
|
|
4033
|
+
);
|
|
4034
|
+
return;
|
|
4035
|
+
}
|
|
4036
|
+
|
|
4037
|
+
const { classification, failure } = deliveryPolicy.classifyError(policy.classify, error);
|
|
4038
|
+
|
|
4039
|
+
if (failure !== null) {
|
|
4040
|
+
this._logger.error(
|
|
4041
|
+
'[RabbitMQClient] Error classifier refused - classify() must return '
|
|
4042
|
+
+ `'transient' or 'permanent'; it ${failure instanceof Error ? `threw "${failure.message}"` : failure}. `
|
|
4043
|
+
+ 'Expected: a total function over the errors this handler can throw. '
|
|
4044
|
+
+ 'Fix: the message is rejected into the dead-letter queue, because a message whose error '
|
|
4045
|
+
+ 'cannot be classified must not be retried on a guess.',
|
|
4046
|
+
{ queue, attempt }
|
|
4047
|
+
);
|
|
4048
|
+
}
|
|
4049
|
+
|
|
4050
|
+
const hasBudgetLeft = classification === deliveryPolicy.TRANSIENT && attempt < policy.maxAttempts;
|
|
4051
|
+
|
|
4052
|
+
if (!hasBudgetLeft) {
|
|
4053
|
+
await this._rejectDelivery({
|
|
4054
|
+
msg,
|
|
4055
|
+
queue,
|
|
4056
|
+
channelForMsg,
|
|
4057
|
+
noAck,
|
|
4058
|
+
error,
|
|
4059
|
+
classification,
|
|
4060
|
+
attempts: attempt,
|
|
4061
|
+
maxAttempts: policy.maxAttempts,
|
|
4062
|
+
onDeadLetter
|
|
4063
|
+
});
|
|
4064
|
+
return;
|
|
4065
|
+
}
|
|
4066
|
+
|
|
4067
|
+
// One more attempt: the copy carries every property of the original — the
|
|
4068
|
+
// RPC addressing included, so the reply still reaches the caller — and the
|
|
4069
|
+
// counter advanced to the attempt just spent.
|
|
4070
|
+
const headers = { ...(msg.properties.headers || {}) };
|
|
4071
|
+
headers[deliveryPolicy.ATTEMPTS_HEADER] = attempt;
|
|
4072
|
+
|
|
4073
|
+
const copyOptions = {
|
|
4074
|
+
headers,
|
|
4075
|
+
// Preserved, never assumed: a message published transient stays transient.
|
|
4076
|
+
persistent: msg.properties.deliveryMode === 2
|
|
4077
|
+
};
|
|
4078
|
+
for (const key of AMQP_MESSAGE_PROPERTY_KEYS) {
|
|
4079
|
+
if (msg.properties[key] !== undefined && msg.properties[key] !== null && msg.properties[key] !== '') {
|
|
4080
|
+
copyOptions[key] = msg.properties[key];
|
|
4081
|
+
}
|
|
4082
|
+
}
|
|
4083
|
+
|
|
4084
|
+
try {
|
|
4085
|
+
await this.publish(queue, msg.content, copyOptions);
|
|
4086
|
+
} catch (publishErr) {
|
|
4087
|
+
// The copy did not reach the broker, so the original must NOT be acked.
|
|
4088
|
+
// Requeue instead: the attempt number does not advance (the header on the
|
|
4089
|
+
// requeued message is unchanged), so this delivery is repeated rather than
|
|
4090
|
+
// consumed from the budget — the alternative, dead-lettering, would throw
|
|
4091
|
+
// away a message that never spent its attempts because of a broker hiccup.
|
|
4092
|
+
this._logger.error(
|
|
4093
|
+
`[RabbitMQClient] Re-publish of a failed delivery did not reach the broker - queue "${queue}", `
|
|
4094
|
+
+ `attempt ${attempt} of ${policy.maxAttempts}, reason: ${publishErr.message}. `
|
|
4095
|
+
+ 'Expected: the copy carrying the advanced attempt counter to be accepted before the original is acked. '
|
|
4096
|
+
+ 'Fix: the original is requeued unchanged, so this attempt is retried, not spent; '
|
|
4097
|
+
+ 'check the broker and this queue for flow control.',
|
|
4098
|
+
{ queue, attempt }
|
|
4099
|
+
);
|
|
4100
|
+
try {
|
|
4101
|
+
channelForMsg.nack(msg, false, true);
|
|
4102
|
+
msg._mqProcessed = true;
|
|
4103
|
+
} catch (nackErr) {
|
|
4104
|
+
msg._mqProcessed = true;
|
|
4105
|
+
this._logger.warn(
|
|
4106
|
+
`[RabbitMQClient] [mq-client-core] Failed to nack message (channel may be closed): ${nackErr.message}`
|
|
4107
|
+
);
|
|
4108
|
+
}
|
|
4109
|
+
return;
|
|
4110
|
+
}
|
|
4111
|
+
|
|
4112
|
+
try {
|
|
4113
|
+
channelForMsg.ack(msg);
|
|
4114
|
+
msg._mqProcessed = true;
|
|
4115
|
+
} catch (ackErr) {
|
|
4116
|
+
// The copy is already in the queue. A failed ack means the broker will
|
|
4117
|
+
// redeliver the original too, so the message is processed twice — said
|
|
4118
|
+
// out loud rather than hidden, because the consumer's idempotency is the
|
|
4119
|
+
// only thing that covers it.
|
|
4120
|
+
msg._mqProcessed = true;
|
|
4121
|
+
this._logger.warn(
|
|
4122
|
+
'[RabbitMQClient] [mq-client-core] Ack of a re-published delivery failed (channel may be closed); '
|
|
4123
|
+
+ 'the copy is already queued, so the original may be delivered a second time',
|
|
4124
|
+
{ queue, attempt, error: ackErr.message }
|
|
4125
|
+
);
|
|
4126
|
+
}
|
|
4127
|
+
}
|
|
4128
|
+
|
|
4129
|
+
/**
|
|
4130
|
+
* Reject one message into the dead-letter queue the broker routes for this
|
|
4131
|
+
* queue, and report it exactly once.
|
|
4132
|
+
*
|
|
4133
|
+
* `nack(requeue=false)` is what makes the broker dead-letter: it then stamps
|
|
4134
|
+
* `x-death[0].reason = 'rejected'` on the copy it moves (measured on the live
|
|
4135
|
+
* broker, 2026-09-11), which is the word the published event carries as its
|
|
4136
|
+
* `status`.
|
|
4137
|
+
*
|
|
4138
|
+
* The `onDeadLetter` hook — and therefore the `message_dlq` event it publishes —
|
|
4139
|
+
* is only ever reached for a message that CAN be dead-lettered: `consume()`
|
|
4140
|
+
* refuses to register a consumer on a queue for which `queueConfig` declares no
|
|
4141
|
+
* `x-dead-letter-exchange`/`-routing-key` (d.259), so by the time a delivery
|
|
4142
|
+
* arrives here the route exists by declaration. Without that gate the same
|
|
4143
|
+
* `nack(requeue=false)` made the broker DROP the message while this event
|
|
4144
|
+
* announced it had reached `.dlq` — a report of a destination nothing routed to.
|
|
4145
|
+
*
|
|
4146
|
+
* @param {Object} args
|
|
4147
|
+
* @private
|
|
4148
|
+
*/
|
|
4149
|
+
async _rejectDelivery({ msg, queue, channelForMsg, noAck, error, classification, attempts, maxAttempts, onDeadLetter }) {
|
|
4150
|
+
if (noAck || msg._mqProcessed) {
|
|
4151
|
+
return;
|
|
4152
|
+
}
|
|
4153
|
+
|
|
4154
|
+
// Same third case as in `_applyDeliveryPolicy()`, and here it also decides
|
|
4155
|
+
// what monitoring is told: the `message_dlq` event reports a rejection the
|
|
4156
|
+
// BROKER carried out, so a rejection that could not be issued must not
|
|
4157
|
+
// publish one. `nack()` on a dead channel is an AMQP error, never a
|
|
4158
|
+
// rejection. Liveness from the one source (`_isChannelAlive`, d.260).
|
|
4159
|
+
if (!this._isChannelAlive(channelForMsg)) {
|
|
4160
|
+
this._logger.warn(
|
|
4161
|
+
`[RabbitMQClient] [mq-client-core] [CONSUMER] Delivery on "${queue}" not rejected - the `
|
|
4162
|
+
+ 'consumer channel died before the rejection could be issued, so no dead-letter event is '
|
|
4163
|
+
+ 'published for it. Expected: the broker returns the unacked delivery to the queue when '
|
|
4164
|
+
+ 'the channel goes. Fix: none needed here — the message is redelivered and rejected then.',
|
|
4165
|
+
{ queue, attempts, classification }
|
|
4166
|
+
);
|
|
4167
|
+
return;
|
|
4168
|
+
}
|
|
4169
|
+
|
|
4170
|
+
try {
|
|
4171
|
+
channelForMsg.nack(msg, false, false);
|
|
4172
|
+
msg._mqProcessed = true;
|
|
4173
|
+
} catch (nackErr) {
|
|
4174
|
+
msg._mqProcessed = true;
|
|
4175
|
+
this._logger.warn(
|
|
4176
|
+
`[RabbitMQClient] [mq-client-core] Failed to nack message (channel may be closed): ${nackErr.message}`
|
|
4177
|
+
);
|
|
4178
|
+
return;
|
|
4179
|
+
}
|
|
4180
|
+
|
|
4181
|
+
if (onDeadLetter === undefined) {
|
|
4182
|
+
// Driving this transport directly is allowed (it is an export of this
|
|
4183
|
+
// package), but then nothing owns the monitoring rail, and a message
|
|
4184
|
+
// leaving for the DLQ with nobody told is the silence the contract exists
|
|
4185
|
+
// to end. Said once, at the moment it happens.
|
|
4186
|
+
this._logger.warn(
|
|
4187
|
+
`[RabbitMQClient] Message rejected into the dead-letter queue of "${queue}" with no monitoring hook - `
|
|
4188
|
+
+ 'Expected: consume() through BaseClient, which injects the hook publishing the message_dlq event. '
|
|
4189
|
+
+ 'Fix: consume through BaseClient, or pass consume(queue, handler, { onDeadLetter }) yourself.',
|
|
4190
|
+
{ queue, attempts, classification }
|
|
4191
|
+
);
|
|
4192
|
+
return;
|
|
4193
|
+
}
|
|
4194
|
+
|
|
4195
|
+
await onDeadLetter({
|
|
4196
|
+
queue,
|
|
4197
|
+
content: msg.content,
|
|
4198
|
+
attempts,
|
|
4199
|
+
maxAttempts,
|
|
4200
|
+
classification,
|
|
4201
|
+
error
|
|
4202
|
+
});
|
|
4203
|
+
}
|
|
4204
|
+
|
|
2261
4205
|
/**
|
|
2262
4206
|
* Acknowledges a message.
|
|
2263
4207
|
* @param {Object} msg - RabbitMQ message object.
|
|
@@ -2267,7 +4211,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2267
4211
|
return; // Already acked/nacked — idempotent
|
|
2268
4212
|
}
|
|
2269
4213
|
if (!this._consumerChannel) {
|
|
2270
|
-
throw new
|
|
4214
|
+
throw new ConnectionError(
|
|
4215
|
+
'[RabbitMQClient] Cannot ack: consumer channel is not initialized - Expected: an open consumer channel. '
|
|
4216
|
+
+ 'Fix: await client.connect() and start the consumer before acking; a message received before a reconnect can no longer be acked.'
|
|
4217
|
+
);
|
|
2271
4218
|
}
|
|
2272
4219
|
try {
|
|
2273
4220
|
this._consumerChannel.ack(msg);
|
|
@@ -2276,7 +4223,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2276
4223
|
const m = err && err.message ? err.message : '';
|
|
2277
4224
|
if (m.includes('unknown delivery tag') || m.includes('PRECONDITION_FAILED') || m.includes('Channel closed')) {
|
|
2278
4225
|
msg._mqProcessed = true;
|
|
2279
|
-
|
|
4226
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot ack - consumer channel is closed/recreated (delivery tag invalid)', { error: m });
|
|
2280
4227
|
return;
|
|
2281
4228
|
}
|
|
2282
4229
|
this.emit('error', err);
|
|
@@ -2286,15 +4233,28 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2286
4233
|
|
|
2287
4234
|
/**
|
|
2288
4235
|
* Negative-acknowledges a message.
|
|
4236
|
+
*
|
|
4237
|
+
* `options` is an OBJECT, not amqplib's positional `(msg, allUpTo, requeue)`.
|
|
4238
|
+
* A caller that passes the positional form hands `false` where the object is
|
|
4239
|
+
* expected, so `options.requeue` reads `undefined` and the default below
|
|
4240
|
+
* turns it into requeue=TRUE — the opposite of the intent, and an infinite
|
|
4241
|
+
* redelivery loop. Pinned by tests/unit/nack-options-signature.test.js.
|
|
4242
|
+
*
|
|
2289
4243
|
* @param {Object} msg - RabbitMQ message object.
|
|
2290
|
-
* @param {Object} [options] - { requeue: boolean }
|
|
4244
|
+
* @param {Object} [options] - { requeue: boolean }; `requeue` defaults to
|
|
4245
|
+
* `true` (an ordinary handler failure puts the message back on the queue),
|
|
4246
|
+
* so a caller that wants the message discarded must pass
|
|
4247
|
+
* `{ requeue: false }` explicitly.
|
|
2291
4248
|
*/
|
|
2292
4249
|
async nack(msg, options = {}) {
|
|
2293
4250
|
if (msg._mqProcessed) {
|
|
2294
4251
|
return; // Already acked/nacked — idempotent
|
|
2295
4252
|
}
|
|
2296
4253
|
if (!this._consumerChannel) {
|
|
2297
|
-
throw new
|
|
4254
|
+
throw new ConnectionError(
|
|
4255
|
+
'[RabbitMQClient] Cannot nack: consumer channel is not initialized - Expected: an open consumer channel. '
|
|
4256
|
+
+ 'Fix: await client.connect() and start the consumer before nacking; a message received before a reconnect can no longer be nacked.'
|
|
4257
|
+
);
|
|
2298
4258
|
}
|
|
2299
4259
|
const requeue = options.requeue !== undefined ? options.requeue : true;
|
|
2300
4260
|
try {
|
|
@@ -2304,7 +4264,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2304
4264
|
const m = err && err.message ? err.message : '';
|
|
2305
4265
|
if (m.includes('unknown delivery tag') || m.includes('PRECONDITION_FAILED') || m.includes('Channel closed')) {
|
|
2306
4266
|
msg._mqProcessed = true;
|
|
2307
|
-
|
|
4267
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot nack - consumer channel is closed/recreated (delivery tag invalid)', { error: m });
|
|
2308
4268
|
return;
|
|
2309
4269
|
}
|
|
2310
4270
|
this.emit('error', err);
|