@onlineapps/mq-client-core 4.2.0 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +768 -0
- package/README.md +149 -19
- package/package.json +1 -1
- package/src/BaseClient.js +126 -31
- package/src/buffer/InMemoryBuffer.js +98 -13
- package/src/buffer/MessageBuffer.js +5 -3
- package/src/config/configSchema.js +33 -10
- package/src/config/defaultConfig.js +21 -10
- package/src/config/deliveryPolicy.js +86 -6
- package/src/config/queueConfig.js +68 -39
- package/src/config.js +5 -1
- package/src/defaults.js +7 -1
- package/src/index.js +41 -4
- package/src/layers/PublishLayer.js +264 -27
- package/src/monitoring-initiated.js +6 -9
- package/src/monitoring-publish.js +89 -40
- package/src/scheduler-request.js +32 -11
- package/src/transports/rabbitmqClient.js +1782 -390
- package/src/utils/decodeMessage.js +123 -0
- package/src/utils/publishErrors.js +256 -17
- package/jest.config.js +0 -15
- package/jest.integration.config.js +0 -16
- package/src/workers/RecoveryWorker.js +0 -130
|
@@ -7,14 +7,22 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
const amqp = require('amqplib');
|
|
10
|
+
// amqplib's own class for "this handle is in a terminal state". It is what
|
|
11
|
+
// `close()` on a closed connection and `createChannel()` on one answer with, and it
|
|
12
|
+
// is read instead of the sentence those answers carry (d.696).
|
|
13
|
+
const { IllegalOperationError } = require('amqplib');
|
|
10
14
|
const EventEmitter = require('events');
|
|
11
15
|
const {
|
|
16
|
+
AMQP_NOT_FOUND,
|
|
17
|
+
AMQP_PRECONDITION_FAILED,
|
|
12
18
|
TransientPublishError,
|
|
13
19
|
PermanentPublishError,
|
|
14
20
|
QueueNotFoundError,
|
|
15
21
|
classifyPublishError,
|
|
22
|
+
collateralCloseMessage,
|
|
16
23
|
missingInfrastructureQueueMessage,
|
|
17
24
|
missingQueueDefinitionMessage,
|
|
25
|
+
refusedSendMessage,
|
|
18
26
|
} = require('../utils/publishErrors');
|
|
19
27
|
const {
|
|
20
28
|
ConnectionError,
|
|
@@ -28,7 +36,6 @@ const {
|
|
|
28
36
|
} = require('../utils/errorHandler');
|
|
29
37
|
const { assertLogger } = require('@onlineapps/logger-contract');
|
|
30
38
|
const PublishLayer = require('../layers/PublishLayer');
|
|
31
|
-
const RecoveryWorker = require('../workers/RecoveryWorker');
|
|
32
39
|
const PublishMonitor = require('../monitoring/PublishMonitor');
|
|
33
40
|
const runtimeCfg = require('../config');
|
|
34
41
|
const defaultConfig = require('../config/defaultConfig');
|
|
@@ -118,11 +125,10 @@ const REFUSAL_REPLY_CODES = [403, 530, 406];
|
|
|
118
125
|
* - `workflow_id` is the only spelling the envelope contract admits, and a
|
|
119
126
|
* reader that accepts the camelCase alias is forbidden as such, not merely a
|
|
120
127
|
* producer that writes it (`docs/biz/70-contracts/workflow-message.md` §2.2,
|
|
121
|
-
* owner decision `workflow-message-aliases` 001).
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
* a reader that
|
|
125
|
-
* retire, because nothing then shows which producers still write it.
|
|
128
|
+
* owner decision `workflow-message-aliases` 001). No producer in `shared/**`
|
|
129
|
+
* or `infra/**` writes the alias (re-measured d.1134: the gateway's headers
|
|
130
|
+
* carry `workflow_id` alone, `infra/api_gateway/shared/workflowMessageHeaders.js`),
|
|
131
|
+
* and a reader that honoured it would hide the next producer that did.
|
|
126
132
|
* - a missing id is reported as missing. `'unknown'` is a value, and a log line
|
|
127
133
|
* carrying it claims a workflow by that name — the same reason
|
|
128
134
|
* `BaseClient._emitDeadLetterEvent()` resolves an absent id to `null` and
|
|
@@ -275,9 +281,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
275
281
|
// socket whose channels are still being recreated is not yet usable (d.340).
|
|
276
282
|
// Channels do the same through `channel._alive` (see `_isChannelAlive()`).
|
|
277
283
|
this._connectionAlive = false;
|
|
284
|
+
// The broker's reason while it blocks this connection (`connection.blocked`), `null` otherwise
|
|
285
|
+
// (d.1145). Written by `_onConnectionBlocked()` / `_onConnectionUnblocked()`.
|
|
286
|
+
this._connectionBlockedBy = null;
|
|
278
287
|
this._channel = null; // ConfirmChannel for publish operations
|
|
279
288
|
this._queueChannel = null; // Regular channel for queue operations (assertQueue, checkQueue)
|
|
280
289
|
this._consumerChannel = null; // Dedicated channel for consume operations
|
|
290
|
+
// A SECOND ConfirmChannel, for the client's own telemetry (d.724). It is created on
|
|
291
|
+
// the first telemetry publish and never before: a client that publishes no telemetry
|
|
292
|
+
// must not hold a channel for it. See `_ensurePublisherChannel()` § roles.
|
|
293
|
+
this._telemetryChannel = null;
|
|
281
294
|
|
|
282
295
|
// Track active consumers for re-registration after channel recreation
|
|
283
296
|
this._activeConsumers = new Map(); // queue -> { handler, options, consumerTag }
|
|
@@ -355,12 +368,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
355
368
|
this._publishRetryBaseDelay = this._config.publishRetryBaseDelay;
|
|
356
369
|
this._publishRetryMaxDelay = this._config.publishRetryMaxDelay;
|
|
357
370
|
this._publishRetryBackoffMultiplier = this._config.publishRetryBackoffMultiplier;
|
|
358
|
-
// ONE number,
|
|
359
|
-
//
|
|
360
|
-
// publish confirm waits for it (`_publishOnce`),
|
|
361
|
-
//
|
|
362
|
-
//
|
|
363
|
-
//
|
|
371
|
+
// ONE number, and therefore a name that belongs to none of its consumers
|
|
372
|
+
// alone: how long this client waits for an ANSWER from the broker. The
|
|
373
|
+
// publish confirm waits for it (`_publishOnce`), so does the whole teardown
|
|
374
|
+
// of `disconnect()` (d.283 chose deliberately not to invent a second number
|
|
375
|
+
// for that; the name it inherited then said "publish confirmation", which was
|
|
376
|
+
// true of one consumer and false of the other), and so does every request a
|
|
377
|
+
// channel sends and waits on (`_askBroker()`, conf `mq-broker-answer-budget`
|
|
378
|
+
// 001, d.1135).
|
|
364
379
|
this._brokerAnswerTimeout = this._config.brokerAnswerTimeout;
|
|
365
380
|
// How long the connect handshake may take before the attempt is abandoned.
|
|
366
381
|
this._connectTimeout = this._config.connectTimeout;
|
|
@@ -413,16 +428,6 @@ class RabbitMQClient extends EventEmitter {
|
|
|
413
428
|
},
|
|
414
429
|
});
|
|
415
430
|
|
|
416
|
-
// Recovery worker — connection recovery, and the report of a queue that does
|
|
417
|
-
// not exist. It creates none: queue creation left it with the two callbacks
|
|
418
|
-
// that fed it (d.419) and with `recoveryScope`, the permission they were
|
|
419
|
-
// filtered by (d.514). Which rule the report names is decided by the
|
|
420
|
-
// ownership class in the error, not by who built this client.
|
|
421
|
-
this._recoveryWorker = new RecoveryWorker({
|
|
422
|
-
client: this,
|
|
423
|
-
logger: this._logger,
|
|
424
|
-
});
|
|
425
|
-
|
|
426
431
|
// Publish monitor (metriky) — counts only, reports through return values,
|
|
427
432
|
// takes no logger (see PublishMonitor's header).
|
|
428
433
|
this._publishMonitor = new PublishMonitor();
|
|
@@ -456,16 +461,12 @@ class RabbitMQClient extends EventEmitter {
|
|
|
456
461
|
this._publishMonitor.trackBuffered();
|
|
457
462
|
});
|
|
458
463
|
|
|
459
|
-
//
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
// Recovery failed - error already thrown
|
|
466
|
-
}
|
|
467
|
-
}
|
|
468
|
-
});
|
|
464
|
+
// A missing queue is answered ONCE, by the refusal of the publish (d.971): the
|
|
465
|
+
// caller gets `PublishError` with the `QueueNotFoundError` as its `cause`, which
|
|
466
|
+
// names the queue, its ownership class and the Fix. Until d.971 a second worker
|
|
467
|
+
// listened on this client's `error` channel for the same verdict and repeated it
|
|
468
|
+
// into the log; d.419 had taken its only other job (creating the queue) and
|
|
469
|
+
// d.811b its input.
|
|
469
470
|
}
|
|
470
471
|
|
|
471
472
|
/**
|
|
@@ -483,8 +484,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
483
484
|
}
|
|
484
485
|
|
|
485
486
|
/**
|
|
486
|
-
*
|
|
487
|
-
*
|
|
487
|
+
* The state of every channel this client holds — the keys ARE the list:
|
|
488
|
+
* `publisher`, `queue`, `consumer` and `telemetry` (the client's own events,
|
|
489
|
+
* d.724), each as `{ exists, closed, ready }`. The telemetry channel opens on the
|
|
490
|
+
* first telemetry publish and not before; until then it reads the way any channel
|
|
491
|
+
* not yet opened reads, `{ exists: false, closed: true, ready: false }`. It was
|
|
492
|
+
* missing from this report until d.1007.
|
|
493
|
+
*
|
|
494
|
+
* @returns {{publisher: Object, queue: Object, consumer: Object, telemetry: Object}}
|
|
488
495
|
*/
|
|
489
496
|
getChannelState() {
|
|
490
497
|
return {
|
|
@@ -502,6 +509,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
502
509
|
exists: !!this._consumerChannel,
|
|
503
510
|
closed: !this._isChannelAlive(this._consumerChannel),
|
|
504
511
|
ready: this._isChannelAlive(this._consumerChannel)
|
|
512
|
+
},
|
|
513
|
+
telemetry: {
|
|
514
|
+
exists: !!this._telemetryChannel,
|
|
515
|
+
closed: !this._isChannelAlive(this._telemetryChannel),
|
|
516
|
+
ready: this._isChannelAlive(this._telemetryChannel)
|
|
505
517
|
}
|
|
506
518
|
};
|
|
507
519
|
}
|
|
@@ -524,6 +536,32 @@ class RabbitMQClient extends EventEmitter {
|
|
|
524
536
|
return !!channel && channel._alive === true;
|
|
525
537
|
}
|
|
526
538
|
|
|
539
|
+
/**
|
|
540
|
+
* Did this failure to open a channel only report the connection going away?
|
|
541
|
+
*
|
|
542
|
+
* THE rule for "a refused channel open is transient" (d.1000, d.1003, d.1003b, d.1003c)
|
|
543
|
+
* — one body, read by `_ensurePublisherChannel()` and by the consumer and queue
|
|
544
|
+
* channels' close handlers. Two carriers, neither of them a sentence:
|
|
545
|
+
*
|
|
546
|
+
* - amqplib's CLASS: on a closing or closed connection `createChannel()` /
|
|
547
|
+
* `createConfirmChannel()` answer `IllegalOperationError` (`'Connection closing'`,
|
|
548
|
+
* `'Connection closed (…)'`), no `code` — measured on amqplib 0.10.9 over a live
|
|
549
|
+
* broker (d.1000);
|
|
550
|
+
* - this client's STATE: a connection that dies WHILE the channel opens rejects the
|
|
551
|
+
* pending RPC with a bare `Error('Channel ended, no reply will be forthcoming')`,
|
|
552
|
+
* no class, no code — but by then the connection has already emitted `'close'`
|
|
553
|
+
* and `_onConnectionDied()` has dropped `_connectionAlive`, the one liveness
|
|
554
|
+
* source (measured over a live broker, d.1003: 8 of 12 broker-side closes during
|
|
555
|
+
* the open).
|
|
556
|
+
*
|
|
557
|
+
* @param {*} amqpError - the error amqplib answered the open with
|
|
558
|
+
* @returns {boolean}
|
|
559
|
+
* @private
|
|
560
|
+
*/
|
|
561
|
+
_connectionIsGoingAway(amqpError) {
|
|
562
|
+
return amqpError instanceof IllegalOperationError || !this._connectionAlive;
|
|
563
|
+
}
|
|
564
|
+
|
|
527
565
|
/**
|
|
528
566
|
* THE definition of "this client's consumer is attached to that queue".
|
|
529
567
|
*
|
|
@@ -549,6 +587,37 @@ class RabbitMQClient extends EventEmitter {
|
|
|
549
587
|
return typeof consumerInfo?.consumerTag === 'string' && consumerInfo.consumerTag !== '';
|
|
550
588
|
}
|
|
551
589
|
|
|
590
|
+
/**
|
|
591
|
+
* The tracked queues whose consumer is NOT attached (`_isConsumerAttached()`), in
|
|
592
|
+
* tracking order — what a recreate summary names (d.1003d).
|
|
593
|
+
*
|
|
594
|
+
* @returns {string[]}
|
|
595
|
+
* @private
|
|
596
|
+
*/
|
|
597
|
+
_unattachedConsumerQueues() {
|
|
598
|
+
const unattached = [];
|
|
599
|
+
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
600
|
+
if (!this._isConsumerAttached(consumerInfo)) {
|
|
601
|
+
unattached.push(queue);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
return unattached;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* `_ensureConsumerChannel()`'s answer for a channel that is already alive: every
|
|
609
|
+
* tracked consumer is counted by whether it is ATTACHED, not assumed re-registered
|
|
610
|
+
* (d.1003d). `failed` is a tracked consumer without an attachment — the state a
|
|
611
|
+
* refused re-registration leaves (d.262 keeps it tracked, d.386b drops its tag).
|
|
612
|
+
*
|
|
613
|
+
* @returns {{reRegistered: number, failed: number}}
|
|
614
|
+
* @private
|
|
615
|
+
*/
|
|
616
|
+
_consumerAttachmentCounts() {
|
|
617
|
+
const failed = this._unattachedConsumerQueues().length;
|
|
618
|
+
return { reRegistered: this._activeConsumers.size - failed, failed };
|
|
619
|
+
}
|
|
620
|
+
|
|
552
621
|
/**
|
|
553
622
|
* Get current consumer state.
|
|
554
623
|
*
|
|
@@ -575,17 +644,32 @@ class RabbitMQClient extends EventEmitter {
|
|
|
575
644
|
}
|
|
576
645
|
|
|
577
646
|
/**
|
|
578
|
-
* Get current buffer state
|
|
579
|
-
*
|
|
647
|
+
* Get current buffer state.
|
|
648
|
+
*
|
|
649
|
+
* ONE buffer, read ONCE. `MessageBuffer.size()` IS `_inMemory.size()`
|
|
650
|
+
* (`buffer/MessageBuffer.js`), so both numbers describe the same messages and
|
|
651
|
+
* both come from the same call.
|
|
652
|
+
*
|
|
653
|
+
* Until d.645 the second one came off `_buffer._inMemory?.size() || 0`, and
|
|
654
|
+
* neither operator could do anything but hide: the `?.` cannot fire, because
|
|
655
|
+
* `MessageBuffer`'s constructor always builds `_inMemory` and the line above
|
|
656
|
+
* already calls `_buffer.size()` without a guard — a buffer that is not there
|
|
657
|
+
* has thrown before this expression runs. What the pair DID produce, for a
|
|
658
|
+
* buffer whose internals were not the assumed ones, was `inMemory: 0` beside a
|
|
659
|
+
* non-zero `size`: "held, but nowhere", a state no real buffer can be in,
|
|
660
|
+
* reported without a word and indistinguishable from an empty buffer for
|
|
661
|
+
* whoever reads it (`architecture-principles.md` §3 No Fallbacks, §8 Explicit
|
|
662
|
+
* Over Implicit; `automation-gates.md` §5).
|
|
663
|
+
*
|
|
664
|
+
* @returns {{size: number, inMemory: number}} How many messages are held
|
|
580
665
|
*/
|
|
581
666
|
getBufferState() {
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
};
|
|
667
|
+
// No `persistent` count. There is one buffer, so a second number could
|
|
668
|
+
// only ever be 0 — a field that reports a mechanism that does not exist
|
|
669
|
+
// (d.343, `automation-gates.md` §5).
|
|
670
|
+
const held = this._publishLayer._buffer.size();
|
|
671
|
+
|
|
672
|
+
return { size: held, inMemory: held };
|
|
589
673
|
}
|
|
590
674
|
|
|
591
675
|
/**
|
|
@@ -648,11 +732,31 @@ class RabbitMQClient extends EventEmitter {
|
|
|
648
732
|
* @private
|
|
649
733
|
*/
|
|
650
734
|
_connectionSocketOptions() {
|
|
651
|
-
const connectionName
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
735
|
+
const explicit = this._config.connectionName || this._config.clientName;
|
|
736
|
+
|
|
737
|
+
if (explicit) {
|
|
738
|
+
return { clientProperties: { connection_name: explicit } };
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// Composed from THIS CLIENT'S OWN config, not from the runtime config (d.770). The
|
|
742
|
+
// name is the caller's fact and the constructor already resolved it — asking the
|
|
743
|
+
// runtime config here was a second reader of it, and the one that answered the
|
|
744
|
+
// fabricated module default `oa-service` when the caller had named itself properly.
|
|
745
|
+
if (typeof this._config.serviceName !== 'string' || this._config.serviceName === '') {
|
|
746
|
+
throw new ValidationError(
|
|
747
|
+
'[RabbitMQClient] Missing configuration - the connection has no name and no '
|
|
748
|
+
+ 'serviceName to compose one from. '
|
|
749
|
+
+ 'Expected: connectionName, clientName or serviceName in this client\'s config '
|
|
750
|
+
+ '(BaseClient resolves serviceName from the explicit key or ENV SERVICE_NAME). '
|
|
751
|
+
+ 'Fix: pass serviceName to the client, or set SERVICE_NAME in env-active/*.env.'
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
return {
|
|
756
|
+
clientProperties: {
|
|
757
|
+
connection_name: `oa-client:${this._config.serviceName}:${process.pid}`
|
|
758
|
+
}
|
|
759
|
+
};
|
|
656
760
|
}
|
|
657
761
|
|
|
658
762
|
/**
|
|
@@ -670,12 +774,86 @@ class RabbitMQClient extends EventEmitter {
|
|
|
670
774
|
* @private
|
|
671
775
|
*/
|
|
672
776
|
_attachConnectionHandlers(connection) {
|
|
777
|
+
// A fault is RECORDED on the connection, the way `_attach*ChannelHandlers()` has
|
|
778
|
+
// recorded a channel's since d.264 — and for the same reason (d.696). Once a
|
|
779
|
+
// connection is closed, amqplib answers every operation on it with the same
|
|
780
|
+
// `IllegalOperationError`, whether it got there by our `close()` or by a broker
|
|
781
|
+
// fault; the only difference is a sentence it composes (`Connection.toClosed()`, `lib/connection.js`).
|
|
782
|
+
// Which of the two happened is this client's own knowledge, so it is kept as state
|
|
783
|
+
// rather than read back out of a message (`error-handling.md` §6).
|
|
784
|
+
connection._closeError = null;
|
|
673
785
|
connection.on('error', (err) => {
|
|
674
786
|
this._connectionAlive = false;
|
|
787
|
+
connection._closeError = err;
|
|
675
788
|
this._logger.error('[RabbitMQClient] Connection error:', err.message);
|
|
676
789
|
this.emit('error', err);
|
|
677
790
|
});
|
|
678
791
|
connection.on('close', () => this._onConnectionDied());
|
|
792
|
+
// A broker in a memory or disk alarm blocks a connection that publishes and reads nothing
|
|
793
|
+
// more from it until the alarm clears; the connection stays alive, so no reconnect — and
|
|
794
|
+
// no flush — follows (measured d.1143 on the platform's image). amqplib announces the
|
|
795
|
+
// capability and re-emits both frames (`lib/connect.js`, `lib/channel_model.js`).
|
|
796
|
+
this._connectionBlockedBy = null;
|
|
797
|
+
connection.on('blocked', (reason) => this._onConnectionBlocked(connection, reason));
|
|
798
|
+
connection.on('unblocked', () => this._onConnectionUnblocked(connection));
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/**
|
|
802
|
+
* The broker blocked this connection (`connection.blocked`, d.1145).
|
|
803
|
+
*
|
|
804
|
+
* Recorded as state — the publish path names it when it refuses (`_publishPathTransient()`)
|
|
805
|
+
* — and reported once as a warning. A frame of a stale connection (one this client already
|
|
806
|
+
* replaced) says nothing about the current one and is ignored.
|
|
807
|
+
*
|
|
808
|
+
* @param {Object} connection - the amqplib connection that received the frame
|
|
809
|
+
* @param {string} reason - the broker's reason (RabbitMQ: "low on memory", "low on disk")
|
|
810
|
+
* @private
|
|
811
|
+
*/
|
|
812
|
+
_onConnectionBlocked(connection, reason) {
|
|
813
|
+
if (connection !== this._connection) {
|
|
814
|
+
return;
|
|
815
|
+
}
|
|
816
|
+
this._connectionBlockedBy = String(reason);
|
|
817
|
+
this._logger.warn(
|
|
818
|
+
`[RabbitMQClient] The broker blocked this connection (reason: ${this._connectionBlockedBy}) - `
|
|
819
|
+
+ 'Expected: an alarm-free broker. It reads nothing this client sends until the alarm clears: '
|
|
820
|
+
+ 'publishes and every request wait, and end on brokerAnswerTimeout. '
|
|
821
|
+
+ 'Fix: clear the broker alarm (management API /api/health/checks/alarms names it); the buffer '
|
|
822
|
+
+ 're-sends what never left once the broker unblocks the connection.'
|
|
823
|
+
);
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
/**
|
|
827
|
+
* The broker unblocked this connection (`connection.unblocked`, d.1145).
|
|
828
|
+
*
|
|
829
|
+
* The flush after it is the one rail `PublishLayer.flushBuffered()` with the trigger
|
|
830
|
+
* `'unblocked'`: only the entries whose frame never left the client are re-sent; the broker
|
|
831
|
+
* delivers the rest by itself (conf `mq-publish-buffer` 001). A failed flush is reported the
|
|
832
|
+
* way the flush after a reconnect reports it.
|
|
833
|
+
*
|
|
834
|
+
* @param {Object} connection - the amqplib connection that received the frame
|
|
835
|
+
* @private
|
|
836
|
+
*/
|
|
837
|
+
_onConnectionUnblocked(connection) {
|
|
838
|
+
if (connection !== this._connection) {
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
const wasBlockedBy = this._connectionBlockedBy;
|
|
842
|
+
this._connectionBlockedBy = null;
|
|
843
|
+
this._logger.warn(
|
|
844
|
+
`[RabbitMQClient] The broker unblocked this connection (it had been blocked: ${wasBlockedBy === null ? 'no blocked frame seen' : wasBlockedBy}) - `
|
|
845
|
+
+ 're-sending the buffered messages whose frame never left this client.'
|
|
846
|
+
);
|
|
847
|
+
this._publishLayer.flushBuffered('unblocked').then(
|
|
848
|
+
(flushed) => {
|
|
849
|
+
this._publishMonitor.trackFlushed(flushed);
|
|
850
|
+
this._logger.info(`[RabbitMQClient] ✓ Flushed ${flushed} buffered messages after the broker unblocked the connection`);
|
|
851
|
+
},
|
|
852
|
+
(flushErr) => {
|
|
853
|
+
this._logger.warn(`[RabbitMQClient] Failed to flush buffered messages after unblocked: ${flushErr.message}`);
|
|
854
|
+
this.emit('buffer:flush:failed', { error: flushErr.message, bufferSize: this._publishLayer._buffer.size() });
|
|
855
|
+
}
|
|
856
|
+
);
|
|
679
857
|
}
|
|
680
858
|
|
|
681
859
|
/**
|
|
@@ -732,6 +910,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
732
910
|
_isBrokerRefusal(err) {
|
|
733
911
|
if (!err) return false;
|
|
734
912
|
if (REFUSAL_REPLY_CODES.includes(err.code)) return true;
|
|
913
|
+
// THE ONE TEXT READING THIS FILE KEEPS, and it is kept because amqplib leaves nothing
|
|
914
|
+
// else. A refused HANDSHAKE is the carrier: measured 2026-09-20 against the dev broker
|
|
915
|
+
// with a wrong password, amqplib answers `connect()` with a plain `Error` whose own
|
|
916
|
+
// properties are `stack` and `message` — NO `code`, no class of its own — and whose
|
|
917
|
+
// sentence is `'Handshake terminated by server: 403 (ACCESS-REFUSED) with message …'`.
|
|
918
|
+
// The branch above cannot see it, so without this one a refused login would read as a
|
|
919
|
+
// transient connection fault and the client would retry credentials that can never
|
|
920
|
+
// work. Do not "finish the d.677 work" by deleting it: measure first, and if amqplib
|
|
921
|
+
// ever puts a code on that error, the branch above already covers it (d.696).
|
|
735
922
|
const message = typeof err.message === 'string' ? err.message : '';
|
|
736
923
|
return REFUSAL_REPLY_CODES.some((code) => message.includes(`server: ${code} (`));
|
|
737
924
|
}
|
|
@@ -1272,7 +1459,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1272
1459
|
*/
|
|
1273
1460
|
_onConnectionDied() {
|
|
1274
1461
|
this._connectionAlive = false;
|
|
1462
|
+
// A dead connection is no longer blocked; a new one starts unblocked (d.1145).
|
|
1463
|
+
this._connectionBlockedBy = null;
|
|
1275
1464
|
this._channel = null;
|
|
1465
|
+
this._telemetryChannel = null;
|
|
1276
1466
|
this._queueChannel = null;
|
|
1277
1467
|
this._consumerChannel = null;
|
|
1278
1468
|
this._connection = null;
|
|
@@ -1318,12 +1508,56 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1318
1508
|
return { status: 'absent', error: null };
|
|
1319
1509
|
}
|
|
1320
1510
|
|
|
1321
|
-
const
|
|
1511
|
+
const outcome = await this._awaitBrokerAnswer(() => handle.close(), budgetMs);
|
|
1512
|
+
|
|
1513
|
+
if (outcome.status === 'unanswered') {
|
|
1514
|
+
this._logger.warn(
|
|
1515
|
+
`[RabbitMQClient] The broker did not answer the ${what} close within the teardown budget `
|
|
1516
|
+
+ `(${this._brokerAnswerTimeout}ms) - `
|
|
1517
|
+
+ 'Expected: a close-ok frame for the close this client sent. '
|
|
1518
|
+
+ 'Fix: nothing to do here — the teardown stops waiting and drops the socket, so '
|
|
1519
|
+
+ 'the teardown ends (disconnect(), or a connect() that failed half-way) and the process '
|
|
1520
|
+
+ 'can exit; a broker that never answers a close is a broker-side or network fault, and '
|
|
1521
|
+
+ 'the declared brokerAnswerTimeout sets how long this client gives it.'
|
|
1522
|
+
);
|
|
1523
|
+
return { status: 'unanswered', error: null };
|
|
1524
|
+
}
|
|
1525
|
+
if (outcome.status === 'refused') {
|
|
1526
|
+
return { status: 'refused', error: outcome.error };
|
|
1527
|
+
}
|
|
1528
|
+
return { status: 'closed', error: null };
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
/**
|
|
1532
|
+
* Wait for the broker's answer to ONE request, never longer than `budgetMs`.
|
|
1533
|
+
*
|
|
1534
|
+
* THE one clock for "how long does this client wait for the broker to answer" (conf
|
|
1535
|
+
* `mq-broker-answer-budget` 001, d.1135). Its two readers are `_askBroker()` — every
|
|
1536
|
+
* request a channel or the connection sends and waits on — and
|
|
1537
|
+
* `_closeWithinTeardownBudget()`, which asks it for the `close-ok` of a teardown with what
|
|
1538
|
+
* is left of that teardown's budget. The publish confirm keeps its own timer inside the
|
|
1539
|
+
* send callback (`_publishOnce()`), because it is not a promise to race: it is a callback
|
|
1540
|
+
* that also has to be disarmed, counted and attributed.
|
|
1541
|
+
*
|
|
1542
|
+
* `request` is called here, synchronously, so a request that throws before it is sent
|
|
1543
|
+
* reads as a refusal rather than escaping the budget as an exception.
|
|
1544
|
+
*
|
|
1545
|
+
* @param {() => Promise<*>} request - sends the request and answers its reply
|
|
1546
|
+
* @param {number} budgetMs - how long the answer may take
|
|
1547
|
+
* @returns {Promise<{status: 'answered', value: *, pending: Promise<*>}
|
|
1548
|
+
* |{status: 'refused', error: Error, pending: Promise<*>}
|
|
1549
|
+
* |{status: 'unanswered', pending: Promise<*>}>} `pending` is the request's own
|
|
1550
|
+
* promise, for a caller that must deal with an answer arriving after the budget
|
|
1551
|
+
* @private
|
|
1552
|
+
*/
|
|
1553
|
+
async _awaitBrokerAnswer(request, budgetMs) {
|
|
1554
|
+
const pending = new Promise((resolve) => resolve(request()));
|
|
1555
|
+
const UNANSWERED = Symbol('broker-answer-budget-spent');
|
|
1322
1556
|
let deadlineTimer = null;
|
|
1323
1557
|
const outcome = await Promise.race([
|
|
1324
|
-
|
|
1325
|
-
() => ({ status: '
|
|
1326
|
-
(
|
|
1558
|
+
pending.then(
|
|
1559
|
+
(value) => ({ status: 'answered', value }),
|
|
1560
|
+
(error) => ({ status: 'refused', error })
|
|
1327
1561
|
),
|
|
1328
1562
|
new Promise((resolve) => {
|
|
1329
1563
|
deadlineTimer = this._setTimeout(() => resolve(UNANSWERED), budgetMs);
|
|
@@ -1332,19 +1566,145 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1332
1566
|
this._clearTimeout(deadlineTimer);
|
|
1333
1567
|
|
|
1334
1568
|
if (outcome === UNANSWERED) {
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1569
|
+
return { status: 'unanswered', pending };
|
|
1570
|
+
}
|
|
1571
|
+
return { ...outcome, pending };
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
/**
|
|
1575
|
+
* Ask the broker one thing and wait for its answer within `brokerAnswerTimeout`.
|
|
1576
|
+
*
|
|
1577
|
+
* ONE rail for every request this client sends and then waits on (conf
|
|
1578
|
+
* `mq-broker-answer-budget` 001, verdict A; d.1135): queue and exchange declarations,
|
|
1579
|
+
* `checkQueue`, `bindQueue`, `purgeQueue`, `deleteQueue`, `basic.consume`, `basic.cancel`,
|
|
1580
|
+
* `basic.qos` and every channel open. Until d.1135 none of them had a limit. A broker in a
|
|
1581
|
+
* memory alarm keeps the connection open and its heartbeats flowing while it answers
|
|
1582
|
+
* nothing — measured by d.1126 on an isolated broker of the platform's image: every one of
|
|
1583
|
+
* these calls still pending after 300 s, settled only when the alarm was cleared; a paused
|
|
1584
|
+
* broker was ended by amqplib's heartbeat after ~90 s. The heartbeat is not part of this
|
|
1585
|
+
* rail: in an alarm it never fires, so only the budget ends the wait.
|
|
1586
|
+
*
|
|
1587
|
+
* The budget is `brokerAnswerTimeout` and never a second number — its declared meaning is
|
|
1588
|
+
* exactly this question (`config/configSchema.js`).
|
|
1589
|
+
*
|
|
1590
|
+
* WHEN THE BUDGET IS SPENT:
|
|
1591
|
+
* - the channel that carried the request is DISCARDED (`_discardUnansweredChannel()`):
|
|
1592
|
+
* amqplib queues the requests of one channel behind each other, so every later request
|
|
1593
|
+
* on it would wait behind the silent one. The next operation opens a fresh channel;
|
|
1594
|
+
* - a request that OPENS a channel has no channel yet; the one the broker may still open
|
|
1595
|
+
* later is closed as soon as it arrives, so an alarm does not leak channels;
|
|
1596
|
+
* - the caller gets `ConnectionError` naming the operation, its target and the budget.
|
|
1597
|
+
* The publish path translates it into `TransientPublishError`
|
|
1598
|
+
* (`_publishPathTransient()`), so retry, the buffer and `bufferOnFailure: false` apply
|
|
1599
|
+
* to it exactly as to any other transient failure.
|
|
1600
|
+
*
|
|
1601
|
+
* A bare `Error('Channel ended, no reply will be forthcoming')` — amqplib's
|
|
1602
|
+
* `_rejectPending()` answering a request whose channel or connection died first, no class,
|
|
1603
|
+
* no `code` — is the same fact ("no answer came") and becomes the same class. It is
|
|
1604
|
+
* recognised by the carrier's STATE, never by the sentence (d.683): no `code`, a plain
|
|
1605
|
+
* `Error`, and a carrier that is dead by the time the rejection arrives (amqplib emits
|
|
1606
|
+
* `close` synchronously before the rejection's handlers run). A numbered refusal is an
|
|
1607
|
+
* ANSWER and passes through untouched, as does anything already typed.
|
|
1608
|
+
*
|
|
1609
|
+
* @param {Object} request
|
|
1610
|
+
* @param {string} request.operation - the request's name, as the message states it
|
|
1611
|
+
* @param {string} request.target - what it is about (queue, exchange, channel)
|
|
1612
|
+
* @param {Object|null} request.channel - the channel carrying it; `null` for a channel
|
|
1613
|
+
* open, which the connection carries
|
|
1614
|
+
* @param {() => Promise<*>} send - sends the request and answers its reply
|
|
1615
|
+
* @returns {Promise<*>} the broker's answer
|
|
1616
|
+
* @throws {ConnectionError} when no answer came within the budget, or the carrier ended
|
|
1617
|
+
* before it could
|
|
1618
|
+
* @private
|
|
1619
|
+
*/
|
|
1620
|
+
async _askBroker({ operation, target, channel }, send) {
|
|
1621
|
+
const budgetMs = this._brokerAnswerTimeout;
|
|
1622
|
+
const outcome = await this._awaitBrokerAnswer(send, budgetMs);
|
|
1623
|
+
|
|
1624
|
+
if (outcome.status === 'answered') {
|
|
1625
|
+
return outcome.value;
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
if (outcome.status === 'unanswered') {
|
|
1629
|
+
if (channel) {
|
|
1630
|
+
this._discardUnansweredChannel(channel, `${operation} ${target}`);
|
|
1631
|
+
} else {
|
|
1632
|
+
outcome.pending.then(
|
|
1633
|
+
(late) => this._closeLateChannel(late, target),
|
|
1634
|
+
() => {}
|
|
1635
|
+
);
|
|
1636
|
+
}
|
|
1637
|
+
throw new ConnectionError(
|
|
1638
|
+
`[RabbitMQClient] The broker did not answer ${operation} ${target} within brokerAnswerTimeout `
|
|
1639
|
+
+ `(${budgetMs}ms) - Expected: an answer to the request this client sent; a broker that `
|
|
1640
|
+
+ 'blocks publishers (memory or disk alarm) or is frozen keeps the connection open and '
|
|
1641
|
+
+ 'answers nothing. Fix: check the broker for an alarm (management API '
|
|
1642
|
+
+ '/api/health/checks/alarms) or a paused node; the channel that carried the request '
|
|
1643
|
+
+ 'is discarded and the next operation opens a fresh one. brokerAnswerTimeout is the '
|
|
1644
|
+
+ 'one budget for every answer this client waits for.'
|
|
1343
1645
|
);
|
|
1344
|
-
return { status: 'unanswered', error: null };
|
|
1345
1646
|
}
|
|
1346
1647
|
|
|
1347
|
-
|
|
1648
|
+
const err = outcome.error;
|
|
1649
|
+
const carrierDead = channel ? !this._isChannelAlive(channel) : !this._connectionAlive;
|
|
1650
|
+
const bare = err instanceof Error && err.constructor === Error && err.code === undefined;
|
|
1651
|
+
if (bare && carrierDead) {
|
|
1652
|
+
throw new ConnectionError(
|
|
1653
|
+
`[RabbitMQClient] ${operation} ${target} got no answer: the ${channel ? 'channel' : 'connection'} `
|
|
1654
|
+
+ 'carrying it ended before the broker answered - Expected: an answer to the request '
|
|
1655
|
+
+ 'this client sent. Fix: nothing is refused — the request never got a reply; run the '
|
|
1656
|
+
+ 'operation again once the channel is back (error.cause carries amqplib\'s rejection).',
|
|
1657
|
+
err
|
|
1658
|
+
);
|
|
1659
|
+
}
|
|
1660
|
+
throw err;
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
/**
|
|
1664
|
+
* Take a channel whose request the broker did not answer out of service.
|
|
1665
|
+
*
|
|
1666
|
+
* It is marked dead first (`_isChannelAlive()`, the one liveness source — d.260), then
|
|
1667
|
+
* dropped from whichever slot still holds it — only if it still holds THIS channel, the
|
|
1668
|
+
* same obituary rule the close handlers keep — and then asked to close. The close is not
|
|
1669
|
+
* awaited: it is an answer from the same silent broker. amqplib sends it at once and
|
|
1670
|
+
* ignores every frame but `close-ok` from then on, so a broker that wakes up closes the
|
|
1671
|
+
* channel cleanly (its consumers cancelled, its unacked deliveries requeued) instead of
|
|
1672
|
+
* keeping it open with nobody holding it; its `close` event then reaches the handlers
|
|
1673
|
+
* attached to it, exactly as for any other channel that ends.
|
|
1674
|
+
*
|
|
1675
|
+
* @param {Object} channel - the amqplib channel that carried the unanswered request
|
|
1676
|
+
* @param {string} reason - which request went unanswered, for the post-mortem
|
|
1677
|
+
* @private
|
|
1678
|
+
*/
|
|
1679
|
+
_discardUnansweredChannel(channel, reason) {
|
|
1680
|
+
channel._alive = false;
|
|
1681
|
+
channel._closeReason = `No answer from the broker to ${reason} within brokerAnswerTimeout (${this._brokerAnswerTimeout}ms)`;
|
|
1682
|
+
if (this._channel === channel) this._channel = null;
|
|
1683
|
+
if (this._telemetryChannel === channel) this._telemetryChannel = null;
|
|
1684
|
+
if (this._queueChannel === channel) this._queueChannel = null;
|
|
1685
|
+
if (this._consumerChannel === channel) this._consumerChannel = null;
|
|
1686
|
+
channel.close().catch((err) => {
|
|
1687
|
+
this._logger.debug(`[RabbitMQClient] Discarded channel (${reason}) did not close: ${err.message}`);
|
|
1688
|
+
});
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
/**
|
|
1692
|
+
* Close a channel the broker opened after this client stopped waiting for it. Nobody
|
|
1693
|
+
* holds it, so left open it would be a channel on the broker with no owner.
|
|
1694
|
+
*
|
|
1695
|
+
* @param {Object} channel - the late amqplib channel
|
|
1696
|
+
* @param {string} what - which channel was being opened, for the log
|
|
1697
|
+
* @private
|
|
1698
|
+
*/
|
|
1699
|
+
_closeLateChannel(channel, what) {
|
|
1700
|
+
this._logger.warn(
|
|
1701
|
+
`[RabbitMQClient] The broker opened the ${what} after brokerAnswerTimeout (${this._brokerAnswerTimeout}ms) - `
|
|
1702
|
+
+ 'Expected: nothing to do; the open had already been reported as unanswered. '
|
|
1703
|
+
+ 'Fix: none — the late channel is closed now, so it is not left open with no owner.'
|
|
1704
|
+
);
|
|
1705
|
+
channel.close().catch((err) => {
|
|
1706
|
+
this._logger.debug(`[RabbitMQClient] Late ${what} did not close: ${err.message}`);
|
|
1707
|
+
});
|
|
1348
1708
|
}
|
|
1349
1709
|
|
|
1350
1710
|
/**
|
|
@@ -1416,6 +1776,38 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1416
1776
|
}
|
|
1417
1777
|
}
|
|
1418
1778
|
|
|
1779
|
+
/**
|
|
1780
|
+
* Helper to set a trackable interval — the SAME evidence, the other kind of timer.
|
|
1781
|
+
*
|
|
1782
|
+
* A repeating timer is the one that most needs to be in it: `_setTimeout()` deletes its
|
|
1783
|
+
* handle when it fires, an interval never fires last, so nothing ever removes it by
|
|
1784
|
+
* itself. Until d.759e the two repeating timers this client arms — the health monitor
|
|
1785
|
+
* and the prefetch monitor — were armed with the bare global, so `_activeTimers` could
|
|
1786
|
+
* not see them and `_clearAllTimers()` could not end them. Measured (d.759d): a client
|
|
1787
|
+
* that had consumed held a live `setInterval(10000)` which only `disconnect()` could
|
|
1788
|
+
* end; a unit suite that left four behind never let its runner exit, while
|
|
1789
|
+
* `expect(client._activeTimers.size).toBe(0)` — the assertion five suites of this
|
|
1790
|
+
* package read — stayed green throughout.
|
|
1791
|
+
*
|
|
1792
|
+
* @private
|
|
1793
|
+
*/
|
|
1794
|
+
_setInterval(callback, ms) {
|
|
1795
|
+
const timer = setInterval(callback, ms);
|
|
1796
|
+
this._activeTimers.add(timer);
|
|
1797
|
+
return timer;
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
/**
|
|
1801
|
+
* Helper to clear a trackable interval
|
|
1802
|
+
* @private
|
|
1803
|
+
*/
|
|
1804
|
+
_clearInterval(timer) {
|
|
1805
|
+
if (timer) {
|
|
1806
|
+
clearInterval(timer);
|
|
1807
|
+
this._activeTimers.delete(timer);
|
|
1808
|
+
}
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1419
1811
|
/**
|
|
1420
1812
|
* Race an `amqp.connect()` handshake against its `connectTimeout`, and own the
|
|
1421
1813
|
* handshake the timeout abandons.
|
|
@@ -1482,9 +1874,21 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1482
1874
|
|
|
1483
1875
|
/**
|
|
1484
1876
|
* Clear all active timers
|
|
1877
|
+
*
|
|
1878
|
+
* The repeating ones go through their own owners first, because each of them holds a
|
|
1879
|
+
* REFERENCE that has to be dropped together with the handle: a sweep that ended the
|
|
1880
|
+
* timer and left `_healthCheckInterval` / `_prefetchCheckTimer` pointing at the dead
|
|
1881
|
+
* handle would make the next `_start*Monitoring()` return early ("already started") and
|
|
1882
|
+
* monitoring would be off for the rest of the client's life, silently. The rest is a
|
|
1883
|
+
* plain sweep — `clearTimeout()` ends an interval handle as well (measured on this
|
|
1884
|
+
* runtime: a `setInterval` cleared with `clearTimeout` never ticks and reports
|
|
1885
|
+
* `_destroyed`), so one loop covers both kinds.
|
|
1886
|
+
*
|
|
1485
1887
|
* @private
|
|
1486
1888
|
*/
|
|
1487
1889
|
_clearAllTimers() {
|
|
1890
|
+
this._stopHealthMonitoring();
|
|
1891
|
+
this._stopPrefetchMonitoring();
|
|
1488
1892
|
for (const timer of this._activeTimers) {
|
|
1489
1893
|
clearTimeout(timer);
|
|
1490
1894
|
}
|
|
@@ -1535,7 +1939,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1535
1939
|
this._attachConnectionHandlers(this._connection);
|
|
1536
1940
|
|
|
1537
1941
|
// Use ConfirmChannel to enable publisher confirms for publish operations
|
|
1538
|
-
this._channel = await this.
|
|
1942
|
+
this._channel = await this._askBroker(
|
|
1943
|
+
{ operation: 'createConfirmChannel', target: 'publisher channel', channel: null },
|
|
1944
|
+
() => this._connection.createConfirmChannel()
|
|
1945
|
+
);
|
|
1539
1946
|
|
|
1540
1947
|
// Track channel metadata for debugging
|
|
1541
1948
|
this._channel._createdAt = new Date().toISOString();
|
|
@@ -1544,11 +1951,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1544
1951
|
this._channel._type = 'publisher';
|
|
1545
1952
|
|
|
1546
1953
|
// Attach event handlers
|
|
1547
|
-
this._attachPublisherChannelHandlers(this._channel);
|
|
1954
|
+
this._attachPublisherChannelHandlers(this._channel, 'publisher');
|
|
1548
1955
|
|
|
1549
1956
|
// Create a separate regular channel for queue operations (assertQueue, checkQueue)
|
|
1550
1957
|
// This avoids RPC reply queue issues with ConfirmChannel.assertQueue()
|
|
1551
|
-
this._queueChannel = await this.
|
|
1958
|
+
this._queueChannel = await this._askBroker(
|
|
1959
|
+
{ operation: 'createChannel', target: 'queue channel', channel: null },
|
|
1960
|
+
() => this._connection.createChannel()
|
|
1961
|
+
);
|
|
1552
1962
|
|
|
1553
1963
|
// Track channel metadata for debugging
|
|
1554
1964
|
this._queueChannel._createdAt = new Date().toISOString();
|
|
@@ -1561,7 +1971,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1561
1971
|
|
|
1562
1972
|
// Create a dedicated channel for consume operations
|
|
1563
1973
|
// This prevents channel conflicts between publish (ConfirmChannel) and consume operations
|
|
1564
|
-
this._consumerChannel = await this.
|
|
1974
|
+
this._consumerChannel = await this._askBroker(
|
|
1975
|
+
{ operation: 'createChannel', target: 'consumer channel', channel: null },
|
|
1976
|
+
() => this._connection.createChannel()
|
|
1977
|
+
);
|
|
1565
1978
|
|
|
1566
1979
|
// Track channel metadata for debugging
|
|
1567
1980
|
this._consumerChannel._createdAt = new Date().toISOString();
|
|
@@ -1580,10 +1993,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1580
1993
|
// Cleanup partially created resources
|
|
1581
1994
|
this._connectionAlive = false;
|
|
1582
1995
|
if (this._connection) {
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1996
|
+
// Within the one budget (d.1135): a connect that failed because the broker stopped
|
|
1997
|
+
// answering (a channel open above) would otherwise wait for a `close-ok` from that
|
|
1998
|
+
// same silent broker. What the close answers is not the caller's business here —
|
|
1999
|
+
// the failure that brought us into this catch is — so only a close that got no
|
|
2000
|
+
// answer at all is acted on: its socket is dropped, as the teardown does.
|
|
2001
|
+
const halfOpen = this._connection;
|
|
2002
|
+
const closing = await this._closeWithinTeardownBudget('connection', halfOpen, this._brokerAnswerTimeout);
|
|
2003
|
+
if (closing.status === 'unanswered') {
|
|
2004
|
+
this._dropUnansweredConnection(halfOpen);
|
|
1587
2005
|
}
|
|
1588
2006
|
this._connection = null;
|
|
1589
2007
|
}
|
|
@@ -1609,7 +2027,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1609
2027
|
throw err;
|
|
1610
2028
|
} finally {
|
|
1611
2029
|
if (connectTimeoutTimer) {
|
|
1612
|
-
|
|
2030
|
+
// Through the evidence, like every other ending of this timer: a bare
|
|
2031
|
+
// `clearTimeout()` here left the handle in `_activeTimers`, so after a FAILED
|
|
2032
|
+
// connect the client reported one armed timer it no longer had (měřeno d.759e:
|
|
2033
|
+
// size 1 with nothing armed).
|
|
2034
|
+
this._clearTimeout(connectTimeoutTimer);
|
|
1613
2035
|
connectTimeoutTimer = null;
|
|
1614
2036
|
}
|
|
1615
2037
|
}
|
|
@@ -1651,10 +2073,17 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1651
2073
|
|
|
1652
2074
|
const timeout = this._setTimeout(() => {
|
|
1653
2075
|
done();
|
|
2076
|
+
// The sentence names the lever that is READ (d.1138): the configuration key when
|
|
2077
|
+
// the caller wrote it, the derivation in the constructor otherwise. It named an
|
|
2078
|
+
// environment variable until d.1138 that nothing reads.
|
|
2079
|
+
const budgetSource = this._config.reconnectWaitTimeout === undefined
|
|
2080
|
+
? 'not set, so derived as max(60000, maxReconnectAttempts × reconnectMaxDelay)'
|
|
2081
|
+
: 'set in the client configuration';
|
|
1654
2082
|
reject(new ConnectionError(
|
|
1655
2083
|
`[RabbitMQClient] Reconnection timeout after ${this._reconnectWaitTimeout}ms - `
|
|
1656
|
-
+
|
|
1657
|
-
+ 'Fix: check the broker is up and reachable; raise
|
|
2084
|
+
+ `Expected: the broker to accept a new connection within reconnectWaitTimeout (${this._reconnectWaitTimeout}ms, ${budgetSource}). `
|
|
2085
|
+
+ 'Fix: check the broker is up and reachable; raise reconnectWaitTimeout in the configuration handed to '
|
|
2086
|
+
+ 'the client only if a longer outage must be survived in-process.'
|
|
1658
2087
|
));
|
|
1659
2088
|
}, this._reconnectWaitTimeout);
|
|
1660
2089
|
|
|
@@ -1727,13 +2156,49 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1727
2156
|
}
|
|
1728
2157
|
|
|
1729
2158
|
/**
|
|
1730
|
-
*
|
|
1731
|
-
*
|
|
2159
|
+
* The ConfirmChannel of one ROLE, open — recreated when the last one died.
|
|
2160
|
+
*
|
|
2161
|
+
* TWO ROLES, TWO CHANNELS (d.724):
|
|
2162
|
+
*
|
|
2163
|
+
* - `'publisher'` — the caller's messages. `this._channel`, created at connect time
|
|
2164
|
+
* with everything else.
|
|
2165
|
+
* - `'telemetry'` — the client's OWN events, the ones nobody asked it to send
|
|
2166
|
+
* (`src/monitoring-publish.js`, reached through `BaseClient.publishTelemetry()`).
|
|
2167
|
+
* `this._telemetryChannel`, created on the first such publish and not before.
|
|
2168
|
+
*
|
|
2169
|
+
* WHY THEY MAY NOT SHARE ONE (measured d.711, 3 of 7 runs of an unrelated suite red):
|
|
2170
|
+
* a channel dies of the FIRST refusal it meets, and takes every unconfirmed publish on
|
|
2171
|
+
* it along. The client's dead-letter telemetry publishes through
|
|
2172
|
+
* `monitoring.workflow.fanout`; in a vhost where that exchange is not declared the
|
|
2173
|
+
* broker answers `404` and closes the channel — and the caller's business message,
|
|
2174
|
+
* still waiting for its confirm on the same channel, died with it. d.723 stopped that
|
|
2175
|
+
* from being reported as the business message's own refusal; this separation stops it
|
|
2176
|
+
* from happening at all. The module that sends the telemetry states the rule it could
|
|
2177
|
+
* not keep on its own: "Fail-safe about PUBLISHING" (`src/monitoring-publish.js`
|
|
2178
|
+
* § Principles) — fail-safe for the event is worth nothing if it is not also fail-safe
|
|
2179
|
+
* for everything beside it.
|
|
2180
|
+
*
|
|
2181
|
+
* IT IS A CONFIRM CHANNEL, NOT A PLAIN ONE, because the telemetry's answer to its
|
|
2182
|
+
* caller is `true`/`false`, and only a confirm can tell those apart: on a plain channel
|
|
2183
|
+
* `publish()` returns as soon as the bytes are written, so every event would be
|
|
2184
|
+
* reported as sent — including the ones the broker refuses, which is exactly the case
|
|
2185
|
+
* this separation exists for.
|
|
2186
|
+
*
|
|
2187
|
+
* @param {'publisher'|'telemetry'} [role] - Whose channel to ensure.
|
|
1732
2188
|
* @private
|
|
1733
2189
|
* @returns {Promise<void>}
|
|
1734
2190
|
*/
|
|
1735
|
-
async _ensurePublisherChannel() {
|
|
1736
|
-
if (
|
|
2191
|
+
async _ensurePublisherChannel(role = 'publisher') {
|
|
2192
|
+
if (role !== 'publisher' && role !== 'telemetry') {
|
|
2193
|
+
throw new Error(
|
|
2194
|
+
'[RabbitMQClient] Unknown send channel role - '
|
|
2195
|
+
+ `Expected: 'publisher' (the caller's messages) or 'telemetry' (the client's own events), `
|
|
2196
|
+
+ `got ${JSON.stringify(role)}. `
|
|
2197
|
+
+ 'Fix: publish through publish() or publishTelemetry(); nothing else picks a channel.'
|
|
2198
|
+
);
|
|
2199
|
+
}
|
|
2200
|
+
|
|
2201
|
+
if (this._isChannelAlive(this._sendChannelOf(role))) {
|
|
1737
2202
|
return; // Channel is good
|
|
1738
2203
|
}
|
|
1739
2204
|
|
|
@@ -1748,7 +2213,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1748
2213
|
await this._waitForReconnection();
|
|
1749
2214
|
|
|
1750
2215
|
// After reconnection, try again
|
|
1751
|
-
if (this._isChannelAlive(this.
|
|
2216
|
+
if (this._isChannelAlive(this._sendChannelOf(role))) {
|
|
1752
2217
|
return; // Channel was recreated during reconnection
|
|
1753
2218
|
}
|
|
1754
2219
|
// If still no channel, connection might have failed - check again
|
|
@@ -1776,26 +2241,50 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1776
2241
|
|
|
1777
2242
|
try {
|
|
1778
2243
|
// Create new ConfirmChannel
|
|
1779
|
-
|
|
1780
|
-
|
|
2244
|
+
const channel = await this._askBroker(
|
|
2245
|
+
{ operation: 'createConfirmChannel', target: `${role} channel`, channel: null },
|
|
2246
|
+
() => this._connection.createConfirmChannel()
|
|
2247
|
+
);
|
|
2248
|
+
|
|
1781
2249
|
// Track channel metadata for debugging
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
2250
|
+
channel._createdAt = new Date().toISOString();
|
|
2251
|
+
channel._closeReason = null;
|
|
2252
|
+
channel._lastOperation = `Recreated via _ensurePublisherChannel('${role}')`;
|
|
2253
|
+
channel._type = role;
|
|
2254
|
+
|
|
2255
|
+
this._setSendChannelOf(role, channel);
|
|
2256
|
+
|
|
1787
2257
|
// Attach event handlers
|
|
1788
|
-
this._attachPublisherChannelHandlers(
|
|
1789
|
-
|
|
1790
|
-
this._logger.info(
|
|
2258
|
+
this._attachPublisherChannelHandlers(channel, role);
|
|
2259
|
+
|
|
2260
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] ✓ ${role} channel recreated`);
|
|
1791
2261
|
} catch (err) {
|
|
1792
|
-
this.
|
|
2262
|
+
this._setSendChannelOf(role, null);
|
|
1793
2263
|
|
|
1794
2264
|
const msg = err && err.message ? err.message : String(err);
|
|
1795
2265
|
|
|
1796
|
-
//
|
|
1797
|
-
//
|
|
1798
|
-
|
|
2266
|
+
// Spojení odchází — rozhoduje TŘÍDA amqplibu, ne věta (d.1000, poslední textová
|
|
2267
|
+
// větev po d.676–d.711). Změřeno nad amqplib 0.10.9 a živým brokerem:
|
|
2268
|
+
// `createConfirmChannel()` nad spojením, jehož `close()` začalo, odpoví
|
|
2269
|
+
// `IllegalOperationError('Connection closing')`, nad zavřeným
|
|
2270
|
+
// `IllegalOperationError('Connection closed (…)')` — bez `code`, jiný nosič
|
|
2271
|
+
// nemá (`lib/connection.js` `invalidateSend()` / `toClosed()`). Uvnitř
|
|
2272
|
+
// `createConfirmChannel()` tu třídu nevydá nic jiného než odcházející spojení
|
|
2273
|
+
// (`'Channel closed'` nového kanálu jen přes spojení, které zavírá své kanály).
|
|
2274
|
+
//
|
|
2275
|
+
// Druhý nosič je STAV (d.1003): spadne-li spojení UPROSTŘED otevírání,
|
|
2276
|
+
// amqplib odmítne čekající RPC holým `Error('Channel ended, no reply will be
|
|
2277
|
+
// forthcoming')` (`_rejectPending()`, `lib/channel.js`) — bez třídy, bez `code`.
|
|
2278
|
+
// Změřeno nad živým brokerem: zavření brokerem (management API) během
|
|
2279
|
+
// `createConfirmChannel()` 8× z 12, `close()` o `setImmediate` později 3× ze 3;
|
|
2280
|
+
// pokaždé spojení UŽ ohlásilo `'close'`, takže `_onConnectionDied()` shodil
|
|
2281
|
+
// `_connectionAlive` — jediný zdroj živosti (d.260) — dřív, než sem odmítnutí
|
|
2282
|
+
// doběhlo. Věta nerozhoduje: táž věta nad živým spojením zůstává trvalá.
|
|
2283
|
+
//
|
|
2284
|
+
// Stav je přechodný: PublishLayer publikaci podrží a zopakuje po reconnectu.
|
|
2285
|
+
// Pravidlo má jedno tělo, `_connectionIsGoingAway()` — čte ho i obnova
|
|
2286
|
+
// consumer kanálu (d.1003b).
|
|
2287
|
+
if (this._connectionIsGoingAway(err)) {
|
|
1799
2288
|
this._logger.warn(
|
|
1800
2289
|
`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection closing): ${msg}`
|
|
1801
2290
|
);
|
|
@@ -1806,10 +2295,28 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1806
2295
|
err
|
|
1807
2296
|
);
|
|
1808
2297
|
}
|
|
2298
|
+
// Třetí nosič (d.1135): broker na otevření kanálu neodpověděl v `brokerAnswerTimeout`
|
|
2299
|
+
// (alarm paměti, zmrazený broker) — `_askBroker()` to hlásí `ConnectionError`, jinou
|
|
2300
|
+
// chybu z otevření kanálu tahle třída nenese. Je to týž stav „odpověď nepřišla“, ne
|
|
2301
|
+
// odmítnutí, takže přechodný jako oba nosiče výše. Stojí ZA nimi: spojení, které
|
|
2302
|
+
// odchází, má svou vlastní (přesnější) větu i tehdy, když amqplib odmítnutí složil
|
|
2303
|
+
// `_askBroker()` do `ConnectionError`.
|
|
2304
|
+
if (err instanceof ConnectionError) {
|
|
2305
|
+
this._logger.warn(
|
|
2306
|
+
`[RabbitMQClient] [mq-client-core] Failed to recreate ${role} channel (no answer from the broker): ${msg}`
|
|
2307
|
+
);
|
|
2308
|
+
throw new TransientPublishError(
|
|
2309
|
+
`[RabbitMQClient] ${role} channel cannot be recreated: the broker did not answer the open (transient condition) - `
|
|
2310
|
+
+ `Expected: an answer within brokerAnswerTimeout (${this._brokerAnswerTimeout}ms). `
|
|
2311
|
+
+ 'Fix: retry the publish; PublishLayer does this on its own and buffers the message meanwhile. '
|
|
2312
|
+
+ 'error.cause names the unanswered request.',
|
|
2313
|
+
err
|
|
2314
|
+
);
|
|
2315
|
+
}
|
|
1809
2316
|
|
|
1810
2317
|
// Ostatní chyby považujeme za permanentní problém na úrovni kanálu/spojení.
|
|
1811
2318
|
throw new PermanentPublishError(
|
|
1812
|
-
`[RabbitMQClient] Failed to recreate
|
|
2319
|
+
`[RabbitMQClient] Failed to recreate ${role} channel: ${msg} - `
|
|
1813
2320
|
+ 'Expected: createConfirmChannel() to succeed on an open connection. '
|
|
1814
2321
|
+ 'Fix: read error.cause for the broker reason; a channel-level refusal here is not retried, so the connection must be re-established.',
|
|
1815
2322
|
err
|
|
@@ -1838,7 +2345,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1838
2345
|
|
|
1839
2346
|
try {
|
|
1840
2347
|
// Create new regular channel
|
|
1841
|
-
this._queueChannel = await this.
|
|
2348
|
+
this._queueChannel = await this._askBroker(
|
|
2349
|
+
{ operation: 'createChannel', target: 'queue channel', channel: null },
|
|
2350
|
+
() => this._connection.createChannel()
|
|
2351
|
+
);
|
|
1842
2352
|
|
|
1843
2353
|
// Track channel metadata for debugging
|
|
1844
2354
|
this._queueChannel._createdAt = new Date().toISOString();
|
|
@@ -1870,12 +2380,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1870
2380
|
* Ensure consumer channel exists and is open
|
|
1871
2381
|
* Auto-recreates and re-registers all consumers if closed
|
|
1872
2382
|
* @private
|
|
1873
|
-
* @returns {Promise<{reRegistered: number, failed: number}>} -
|
|
2383
|
+
* @returns {Promise<{reRegistered: number, failed: number}>} - how many tracked
|
|
2384
|
+
* consumers are attached, and how many are tracked without being attached. On a
|
|
2385
|
+
* channel that was already alive both are MEASURED (`_consumerAttachmentCounts()`,
|
|
2386
|
+
* d.1003d), never assumed: until d.1003d this branch answered
|
|
2387
|
+
* `_activeConsumers.size` as re-registered, counting a consumer whose
|
|
2388
|
+
* re-registration the broker had refused (tracked, no tag — d.262, d.386b).
|
|
1874
2389
|
*/
|
|
1875
2390
|
async _ensureConsumerChannel() {
|
|
1876
2391
|
if (this._isChannelAlive(this._consumerChannel)) {
|
|
1877
|
-
|
|
1878
|
-
return { reRegistered: this._activeConsumers.size, failed: 0 };
|
|
2392
|
+
return this._consumerAttachmentCounts();
|
|
1879
2393
|
}
|
|
1880
2394
|
|
|
1881
2395
|
if (!this._connectionAlive) {
|
|
@@ -1898,8 +2412,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1898
2412
|
}
|
|
1899
2413
|
|
|
1900
2414
|
if (this._isChannelAlive(this._consumerChannel)) {
|
|
1901
|
-
// The recovery recreated it and re-
|
|
1902
|
-
|
|
2415
|
+
// The recovery recreated it and ran the re-registration; what it attached is
|
|
2416
|
+
// measured, not assumed (d.1003d).
|
|
2417
|
+
return this._consumerAttachmentCounts();
|
|
1903
2418
|
}
|
|
1904
2419
|
if (!this._connectionAlive) {
|
|
1905
2420
|
return { reRegistered: 0, failed: this._activeConsumers.size };
|
|
@@ -1908,7 +2423,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1908
2423
|
|
|
1909
2424
|
try {
|
|
1910
2425
|
// Create new regular channel
|
|
1911
|
-
this._consumerChannel = await this.
|
|
2426
|
+
this._consumerChannel = await this._askBroker(
|
|
2427
|
+
{ operation: 'createChannel', target: 'consumer channel', channel: null },
|
|
2428
|
+
() => this._connection.createChannel()
|
|
2429
|
+
);
|
|
1912
2430
|
|
|
1913
2431
|
// Track channel metadata for debugging
|
|
1914
2432
|
this._consumerChannel._createdAt = new Date().toISOString();
|
|
@@ -1921,8 +2439,20 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1921
2439
|
|
|
1922
2440
|
// Re-register all active consumers
|
|
1923
2441
|
const { reRegistered, failed } = await this._reRegisterConsumers();
|
|
1924
|
-
|
|
1925
|
-
|
|
2442
|
+
|
|
2443
|
+
// A recreate that left a consumer unattached is not a success line (d.1003d):
|
|
2444
|
+
// the ✓ at INFO read as "all good" while a queue had nobody reading it. The
|
|
2445
|
+
// refusal itself is already an ERROR per queue (`_reRegisterConsumer()`); this
|
|
2446
|
+
// summary names which queues are left without a consumer.
|
|
2447
|
+
if (failed > 0) {
|
|
2448
|
+
const unattached = this._unattachedConsumerQueues();
|
|
2449
|
+
this._logger.warn(
|
|
2450
|
+
`[RabbitMQClient] [mq-client-core] Consumer channel recreated, but ${unattached.length} consumer(s) are not attached: `
|
|
2451
|
+
+ `${unattached.join(', ')} (re-registered: ${reRegistered}, failed: ${failed})`
|
|
2452
|
+
);
|
|
2453
|
+
} else {
|
|
2454
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] ✓ Consumer channel recreated (re-registered: ${reRegistered}, failed: ${failed})`);
|
|
2455
|
+
}
|
|
1926
2456
|
return { reRegistered, failed };
|
|
1927
2457
|
} catch (err) {
|
|
1928
2458
|
this._consumerChannel = null;
|
|
@@ -1971,6 +2501,33 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1971
2501
|
return { reRegistered, failed };
|
|
1972
2502
|
}
|
|
1973
2503
|
|
|
2504
|
+
/**
|
|
2505
|
+
* The consumer channel, if it is alive — otherwise a refusal that names why.
|
|
2506
|
+
*
|
|
2507
|
+
* The one question every step that is about to use the consumer channel asks
|
|
2508
|
+
* (`_isChannelAlive()`, the one liveness source, d.260). A refusal here is not a broker
|
|
2509
|
+
* verdict: the consumer stays tracked and the next recovery re-attaches it.
|
|
2510
|
+
*
|
|
2511
|
+
* @param {string} action - what was about to happen, as the refusal states it
|
|
2512
|
+
* @param {string} queue - the queue it is about
|
|
2513
|
+
* @returns {Object} the live consumer channel
|
|
2514
|
+
* @throws {ConnectionError} when the channel is dead or gone
|
|
2515
|
+
* @private
|
|
2516
|
+
*/
|
|
2517
|
+
_liveConsumerChannel(action, queue) {
|
|
2518
|
+
const channel = this._consumerChannel;
|
|
2519
|
+
if (!this._isChannelAlive(channel)) {
|
|
2520
|
+
throw new ConnectionError(
|
|
2521
|
+
`[RabbitMQClient] Cannot ${action} ${queue}: consumer channel is not available - `
|
|
2522
|
+
+ 'Expected: a live consumer channel for the whole re-registration. '
|
|
2523
|
+
+ 'Fix: nothing to do by hand — the channel died during the recovery (closed by the broker, or '
|
|
2524
|
+
+ 'discarded after brokerAnswerTimeout); the consumer stays tracked and the next channel recreate '
|
|
2525
|
+
+ 're-attaches it. Listen for consumer:re-registration:failed to see which ones wait.'
|
|
2526
|
+
);
|
|
2527
|
+
}
|
|
2528
|
+
return channel;
|
|
2529
|
+
}
|
|
2530
|
+
|
|
1974
2531
|
/**
|
|
1975
2532
|
* Re-attach ONE tracked consumer on the current consumer channel.
|
|
1976
2533
|
*
|
|
@@ -1994,19 +2551,29 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1994
2551
|
*/
|
|
1995
2552
|
async _reRegisterConsumer(queue, consumerInfo) {
|
|
1996
2553
|
try {
|
|
2554
|
+
// Asked before EACH re-registration, not once per loop (d.1138): a channel that
|
|
2555
|
+
// died after the previous consumer — closed by the broker, or discarded by the
|
|
2556
|
+
// broker-answer budget (d.1135) — leaves the slot empty, and reaching into it was a
|
|
2557
|
+
// TypeError naming a JavaScript accident instead of the channel's state. Asked
|
|
2558
|
+
// before the queue step, so a consumer that cannot be attached declares nothing.
|
|
2559
|
+
this._liveConsumerChannel('re-register the consumer of', queue);
|
|
2560
|
+
|
|
1997
2561
|
await this._prepareQueueForConsume(queue, {
|
|
1998
2562
|
durable: consumerInfo.options.durable,
|
|
1999
2563
|
queueOptions: consumerInfo.options.queueOptions
|
|
2000
2564
|
});
|
|
2001
2565
|
|
|
2566
|
+
// …and again after it: the queue step awaits the broker on another channel,
|
|
2567
|
+
// and the consumer channel can die meanwhile.
|
|
2568
|
+
const consumerChannel = this._liveConsumerChannel('re-register the consumer of', queue);
|
|
2569
|
+
|
|
2002
2570
|
// The consumer's prefetch is re-applied here for the same reason the
|
|
2003
2571
|
// queue step is: the channel is new, and QoS lives on the channel.
|
|
2004
|
-
await this._applyConsumerPrefetch(queue, consumerInfo.options.prefetch);
|
|
2572
|
+
await this._applyConsumerPrefetch(consumerChannel, queue, consumerInfo.options.prefetch);
|
|
2005
2573
|
|
|
2006
|
-
const consumeResult = await this.
|
|
2007
|
-
queue,
|
|
2008
|
-
consumerInfo.handler,
|
|
2009
|
-
{ noAck: consumerInfo.options.noAck }
|
|
2574
|
+
const consumeResult = await this._askBroker(
|
|
2575
|
+
{ operation: 'consume', target: queue, channel: consumerChannel },
|
|
2576
|
+
() => consumerChannel.consume(queue, consumerInfo.handler, { noAck: consumerInfo.options.noAck })
|
|
2010
2577
|
);
|
|
2011
2578
|
consumerInfo.consumerTag = consumeResult.consumerTag;
|
|
2012
2579
|
this._logger.info(`[RabbitMQClient] [mq-client-core] ✓ Re-registered consumer for queue: ${queue} (consumerTag: ${consumeResult.consumerTag})`);
|
|
@@ -2128,7 +2695,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2128
2695
|
return; // Already started
|
|
2129
2696
|
}
|
|
2130
2697
|
|
|
2131
|
-
this._healthCheckInterval =
|
|
2698
|
+
this._healthCheckInterval = this._setInterval(async () => {
|
|
2132
2699
|
try {
|
|
2133
2700
|
await this._performHealthCheck();
|
|
2134
2701
|
} catch (err) {
|
|
@@ -2147,7 +2714,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2147
2714
|
*/
|
|
2148
2715
|
_stopHealthMonitoring() {
|
|
2149
2716
|
if (this._healthCheckInterval) {
|
|
2150
|
-
|
|
2717
|
+
this._clearInterval(this._healthCheckInterval);
|
|
2151
2718
|
this._healthCheckInterval = null;
|
|
2152
2719
|
this._logger.info('[RabbitMQClient] [mq-client-core] Health monitoring stopped');
|
|
2153
2720
|
}
|
|
@@ -2259,7 +2826,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2259
2826
|
try {
|
|
2260
2827
|
// Check if queue exists
|
|
2261
2828
|
this._trackChannelOperation(this._queueChannel, `checkQueue ${queue}`);
|
|
2262
|
-
const
|
|
2829
|
+
const queueChannel = this._queueChannel;
|
|
2830
|
+
const queueInfo = await this._askBroker(
|
|
2831
|
+
{ operation: 'checkQueue', target: queue, channel: queueChannel },
|
|
2832
|
+
() => queueChannel.checkQueue(queue)
|
|
2833
|
+
);
|
|
2263
2834
|
if (queueInfo) {
|
|
2264
2835
|
// Queue exists - is THIS client's consumer attached to it?
|
|
2265
2836
|
//
|
|
@@ -2287,7 +2858,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2287
2858
|
health.issues.push(`Queue ${queue} does not exist`);
|
|
2288
2859
|
}
|
|
2289
2860
|
} catch (err) {
|
|
2290
|
-
if (err.code ===
|
|
2861
|
+
if (err.code === AMQP_NOT_FOUND) {
|
|
2291
2862
|
health.queues.missing.push(queue);
|
|
2292
2863
|
health.healthy = false;
|
|
2293
2864
|
health.issues.push(`Queue ${queue} does not exist (404)`);
|
|
@@ -2465,6 +3036,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2465
3036
|
}
|
|
2466
3037
|
socketUnanswered = socketUnanswered || queueOutcome.status === 'unanswered';
|
|
2467
3038
|
|
|
3039
|
+
// The telemetry channel exists only when this client published telemetry (d.724);
|
|
3040
|
+
// `_closeWithinTeardownBudget()` takes `null` as "nothing to close", so the client
|
|
3041
|
+
// that never sent an event pays nothing for this line.
|
|
3042
|
+
const telemetryOutcome = await this._closeWithinTeardownBudget(
|
|
3043
|
+
'telemetry channel',
|
|
3044
|
+
this._telemetryChannel,
|
|
3045
|
+
remaining()
|
|
3046
|
+
);
|
|
3047
|
+
this._telemetryChannel = null;
|
|
3048
|
+
if (telemetryOutcome.status === 'refused') {
|
|
3049
|
+
// Reported, never raised: the client's own events are not the caller's business,
|
|
3050
|
+
// and a telemetry channel that would not close is not a reason to fail a disconnect
|
|
3051
|
+
// the caller asked for.
|
|
3052
|
+
this._logger.warn('[RabbitMQClient] Error closing telemetry channel:', telemetryOutcome.error.message);
|
|
3053
|
+
}
|
|
3054
|
+
socketUnanswered = socketUnanswered || telemetryOutcome.status === 'unanswered';
|
|
3055
|
+
|
|
2468
3056
|
const publisherOutcome = await this._closeWithinTeardownBudget(
|
|
2469
3057
|
'publisher channel',
|
|
2470
3058
|
this._channel,
|
|
@@ -2483,9 +3071,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2483
3071
|
);
|
|
2484
3072
|
this._connection = null;
|
|
2485
3073
|
if (connectionOutcome.status === 'refused') {
|
|
2486
|
-
//
|
|
2487
|
-
//
|
|
2488
|
-
|
|
3074
|
+
// An EXPECTED refusal is two facts, and both are first-hand (d.696): amqplib
|
|
3075
|
+
// answered with its terminal-state class — the connection was already closed — and
|
|
3076
|
+
// this client recorded no fault on it, so what closed it was a `close()`. A fault
|
|
3077
|
+
// recorded here means the connection died on its own and the refusal is worth
|
|
3078
|
+
// raising, exactly as it was while this read the sentence `'Connection closed (by
|
|
3079
|
+
// client)'` (amqplib composes that in `toClosed()`, `lib/connection.js`, and a
|
|
3080
|
+
// fault gets `'Connection closed (<the error>)'` instead).
|
|
3081
|
+
if (connectionOutcome.error instanceof IllegalOperationError && !connection._closeError) {
|
|
2489
3082
|
this._logger.info('[RabbitMQClient] Connection closed during disconnect (expected)');
|
|
2490
3083
|
} else {
|
|
2491
3084
|
this.emit('error', connectionOutcome.error);
|
|
@@ -2512,7 +3105,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2512
3105
|
*
|
|
2513
3106
|
* PUBLIC: this is the one operation a collaborator needs in order to declare a
|
|
2514
3107
|
* queue on this client — higher-level routing libraries (cookbook-router's
|
|
2515
|
-
* QueueManager)
|
|
3108
|
+
* QueueManager) alike. It owns the whole
|
|
2516
3109
|
* sequence, so nobody has to assemble it out of private members: ensure the
|
|
2517
3110
|
* queue channel, refuse a channel that is not alive, resolve the arguments
|
|
2518
3111
|
* from the central `queueConfig`, declare.
|
|
@@ -2591,7 +3184,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2591
3184
|
const channel = await this._requireQueueChannel('assertQueue', queue);
|
|
2592
3185
|
|
|
2593
3186
|
try {
|
|
2594
|
-
return await
|
|
3187
|
+
return await this._askBroker(
|
|
3188
|
+
{ operation: 'assertQueue', target: queue, channel },
|
|
3189
|
+
() => channel.assertQueue(queue, queueOptions)
|
|
3190
|
+
);
|
|
2595
3191
|
} catch (err) {
|
|
2596
3192
|
// The SAME verdict the consume rail states, through the SAME function
|
|
2597
3193
|
// (d.582). This is the rail FÁZE 0.8 walks: a service declaring its own
|
|
@@ -2721,14 +3317,354 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2721
3317
|
* @private
|
|
2722
3318
|
*/
|
|
2723
3319
|
_publishPathTransient(verb, queue, cause) {
|
|
3320
|
+
// The sentence is chosen from this client's STATE, never from the cause's text (d.1145,
|
|
3321
|
+
// finding of d.1143): since d.1135 this refusal also ends a pre-check the broker never
|
|
3322
|
+
// answered, and on the platform's image the connection was then alive and BLOCKED by a
|
|
3323
|
+
// memory alarm — "connection may be closed" sent the reader to a reconnect that was not
|
|
3324
|
+
// coming. `sent` is false on every one of these (no frame was written), which is what the
|
|
3325
|
+
// flush after `unblocked` reads.
|
|
3326
|
+
let state;
|
|
3327
|
+
let fix;
|
|
3328
|
+
if (!this._connectionAlive) {
|
|
3329
|
+
state = 'the connection is closed';
|
|
3330
|
+
fix = 'Fix: nothing to do here — a reconnect restores it; PublishLayer retries and buffers the message meanwhile, '
|
|
3331
|
+
+ 'and the buffer re-sends it after the reconnect.';
|
|
3332
|
+
} else if (this._connectionBlockedBy !== null) {
|
|
3333
|
+
state = `the broker has blocked this connection (reason: ${this._connectionBlockedBy}) and answers nothing until it clears the alarm`;
|
|
3334
|
+
fix = 'Fix: clear the broker alarm (management API /api/health/checks/alarms names it); PublishLayer retries and '
|
|
3335
|
+
+ 'buffers the message meanwhile, and the buffer re-sends it once the broker unblocks the connection (its frame never left).';
|
|
3336
|
+
} else {
|
|
3337
|
+
// Neither closed nor blocked: the cause says what happened (a request unanswered
|
|
3338
|
+
// within brokerAnswerTimeout, or a channel open the broker refused) — this sentence
|
|
3339
|
+
// does not guess which.
|
|
3340
|
+
state = 'the queue channel is not available on a live, unblocked connection (error.cause names the request '
|
|
3341
|
+
+ 'that went unanswered or was refused)';
|
|
3342
|
+
fix = 'Fix: read error.cause; an unanswered request points at a paused or overloaded broker. PublishLayer retries '
|
|
3343
|
+
+ 'and buffers the message meanwhile.';
|
|
3344
|
+
}
|
|
2724
3345
|
return new TransientPublishError(
|
|
2725
|
-
`[RabbitMQClient] Cannot ${verb} ${queue}:
|
|
2726
|
-
+ `Expected: an open queue channel before ${verb} on the publish path.
|
|
2727
|
-
+ 'Fix: retry the publish after the reconnect; PublishLayer does this on its own and buffers the message meanwhile.',
|
|
3346
|
+
`[RabbitMQClient] Cannot ${verb} ${queue}: ${state} - `
|
|
3347
|
+
+ `Expected: an answering broker on an open queue channel before ${verb} on the publish path. ${fix}`,
|
|
2728
3348
|
cause
|
|
2729
3349
|
);
|
|
2730
3350
|
}
|
|
2731
3351
|
|
|
3352
|
+
/**
|
|
3353
|
+
* The broker's REFUSAL of this send, when that is what killed the channel — or `null`
|
|
3354
|
+
* when the channel died without the broker saying anything about this message.
|
|
3355
|
+
*
|
|
3356
|
+
* A send the broker refuses closes the channel with a numbered close frame. amqplib
|
|
3357
|
+
* puts that object on the channel's `error` event (`lib/channel.js`, `Channel.accept()`, case `ChannelClose`) and
|
|
3358
|
+
* answers the publish callback with the bare `new Error('channel closed')`
|
|
3359
|
+
* (the `Channel` constructor's `'close'` listener, `lib/channel.js`) — so the callback cannot see the reason, but the channel can:
|
|
3360
|
+
* `_attachPublisherChannelHandlers()` records it as `channel._closeError`, and does so
|
|
3361
|
+
* BEFORE the callback runs, because amqplib emits `error` first and only then `close`,
|
|
3362
|
+
* which is what rejects the unconfirmed publishes.
|
|
3363
|
+
*
|
|
3364
|
+
* WHOSE ANSWER IT IS, THOUGH, IS A SECOND QUESTION (d.723). A ConfirmChannel carries
|
|
3365
|
+
* more than one unconfirmed publish, and a numbered frame is an answer about exactly
|
|
3366
|
+
* ONE of them; amqplib answers all of them with that same object. Until d.723 this
|
|
3367
|
+
* method's own JSDoc called the reading "a fact about THIS send, first-hand" — it is
|
|
3368
|
+
* that only while this send is the only unconfirmed one on the channel. Measured
|
|
3369
|
+
* 2026-09-20, two publishes on one channel in a vhost of its own: one to an exchange
|
|
3370
|
+
* nobody declared, one to a `good.fanout` the same client had just declared. Both came
|
|
3371
|
+
* back `PermanentPublishError … Cannot publish to exchange "good.fanout": 404 NOT-FOUND
|
|
3372
|
+
* - nothing declares it`, while the frame underneath named `no exchange
|
|
3373
|
+
* 'no.such.exchange'`. A message the broker never refused was dropped, and its
|
|
3374
|
+
* publisher was sent to declare an exchange that was already there.
|
|
3375
|
+
*
|
|
3376
|
+
* So `_closeFrameNamesThisSend()` decides attribution before the class is decided at
|
|
3377
|
+
* all, and the sends it cannot claim get the collateral verdict
|
|
3378
|
+
* (`_collateralSendTransient()`), which is transient: they were never answered, so they
|
|
3379
|
+
* were never refused, and republishing them is right.
|
|
3380
|
+
*
|
|
3381
|
+
* So the reading is a fact about the send the frame is about:
|
|
3382
|
+
* - a NUMBER is an answer about this request. Measured against the dev broker,
|
|
3383
|
+
* `classId 60 / methodId 40` — `basic.publish` itself: `403 ACCESS-REFUSED` (an
|
|
3384
|
+
* account without `write` in the vhost) and `406 PRECONDITION-FAILED` (a `user_id`
|
|
3385
|
+
* property that is not the authenticated user). Repeating the request repeats the
|
|
3386
|
+
* answer, so it is PERMANENT — the same reading the queue-existence pre-check has
|
|
3387
|
+
* used for `405 RESOURCE_LOCKED` since d.683;
|
|
3388
|
+
* - `404 NOT-FOUND` is one of them, since d.693 (carrier B1: a publish through an
|
|
3389
|
+
* exchange nobody declared, measured `classId 60` on the send). It says something
|
|
3390
|
+
* the other codes do not — nothing DECLARED what was published to — but the
|
|
3391
|
+
* consequence is the same, because a publisher declares nothing and no reconnect
|
|
3392
|
+
* creates an exchange: `config/queueConfig.js` declares it and its owning service
|
|
3393
|
+
* asserts it. Retrying and buffering only held the message until its TTL dropped it.
|
|
3394
|
+
* It is NOT a `QueueNotFoundError`: that class belongs to the queue-existence
|
|
3395
|
+
* pre-check, and it carries a `queueName` and an ownership `kind` that
|
|
3396
|
+
* the caller reads from the refusal's `cause` — an exchange is neither;
|
|
3397
|
+
* - anything that is not a number is not a broker reply code. A socket error's
|
|
3398
|
+
* `'ECONNRESET'` says nothing about whether the message was stored, and reading it
|
|
3399
|
+
* as a refusal would drop messages a reconnect would have delivered.
|
|
3400
|
+
*
|
|
3401
|
+
* Until d.685a neither send leg read it. Both answered with their own `PublishError`,
|
|
3402
|
+
* which the classification reads as transient (`utils/publishErrors.js`), so a message
|
|
3403
|
+
* the broker refuses every single time was retried `publishMaxRetries` times and then
|
|
3404
|
+
* buffered — and `InMemoryBuffer.flush()` returns an unsuccessful flush to the buffer,
|
|
3405
|
+
* so it was re-offered on every reconnect until its TTL dropped it.
|
|
3406
|
+
*
|
|
3407
|
+
* @param {Object|null} channel - The channel that CARRIED the send, read before it was
|
|
3408
|
+
* replaced: `this._channel` may already be the successor by callback time.
|
|
3409
|
+
* @param {'queue'|'exchange'} kind - Which leg is asking, so the message names what the
|
|
3410
|
+
* publish was addressed to (`architecture-principles.md` §5).
|
|
3411
|
+
* @param {string} target - That queue or exchange.
|
|
3412
|
+
* @returns {PermanentPublishError|null}
|
|
3413
|
+
* @private
|
|
3414
|
+
*/
|
|
3415
|
+
_refusedSendPermanent(channel, kind, target) {
|
|
3416
|
+
const closeError = channel ? channel._closeError : null;
|
|
3417
|
+
const code = closeError ? closeError.code : undefined;
|
|
3418
|
+
|
|
3419
|
+
if (typeof code !== 'number') {
|
|
3420
|
+
return null;
|
|
3421
|
+
}
|
|
3422
|
+
|
|
3423
|
+
if (!this._closeFrameNamesThisSend(channel, target)) {
|
|
3424
|
+
return null;
|
|
3425
|
+
}
|
|
3426
|
+
|
|
3427
|
+
return new PermanentPublishError(refusedSendMessage(kind, target, code), closeError);
|
|
3428
|
+
}
|
|
3429
|
+
|
|
3430
|
+
/**
|
|
3431
|
+
* The verdict for a send that was on the channel when SOMEBODY ELSE'S refusal closed
|
|
3432
|
+
* it — or `null` when there was no numbered frame, or the frame was this send's own.
|
|
3433
|
+
*
|
|
3434
|
+
* Transient, and that is the whole point: the broker answered one message and never
|
|
3435
|
+
* answered this one, so nothing about this message was decided. `PublishLayer` retries
|
|
3436
|
+
* a `TransientPublishError` and buffers it meanwhile, which republishes it over the
|
|
3437
|
+
* channel's successor; the permanent verdict would have dropped it on the first attempt
|
|
3438
|
+
* (`src/layers/PublishLayer.js`).
|
|
3439
|
+
*
|
|
3440
|
+
* @param {Object|null} channel - The channel that CARRIED the send.
|
|
3441
|
+
* @param {'queue'|'exchange'} kind - Which leg is asking.
|
|
3442
|
+
* @param {string} target - That queue or exchange.
|
|
3443
|
+
* @returns {TransientPublishError|null}
|
|
3444
|
+
* @private
|
|
3445
|
+
*/
|
|
3446
|
+
/**
|
|
3447
|
+
* Say out loud what the broker answered about a send whose promise had already given up.
|
|
3448
|
+
*
|
|
3449
|
+
* The publish callback is not guarded by the attempt's `settled` flag, so when the
|
|
3450
|
+
* confirmation budget has already rejected, the callback still runs and still builds the
|
|
3451
|
+
* right verdict — and then hands it to a settled promise, where `reject()` is a no-op.
|
|
3452
|
+
* The frame is not lost (it stays on `channel._closeError`) and the verdict is not
|
|
3453
|
+
* overwritten; it simply reached nobody, while `PublishLayer` went on retrying and
|
|
3454
|
+
* buffering a message the broker had refused for good.
|
|
3455
|
+
*
|
|
3456
|
+
* Silence is a defect of the same severity as a wrong result
|
|
3457
|
+
* (`automation-gates.md` §5), so the verdict is REPORTED — and only reported. Nothing
|
|
3458
|
+
* here changes the promise's outcome, the retry policy or the budget: making a late
|
|
3459
|
+
* verdict DECIDE is a change of what happens to a message, which goes with the major
|
|
3460
|
+
* wave (d.801).
|
|
3461
|
+
*
|
|
3462
|
+
* Measured (d.801, dev broker): a refused send is answered in tens of milliseconds —
|
|
3463
|
+
* `404` median 13 ms idle and 31 ms (max 126 ms) under load 32 — so this path is not the
|
|
3464
|
+
* ordinary one. It is what a budget set below that latency produces.
|
|
3465
|
+
*
|
|
3466
|
+
* @param {PermanentPublishError|TransientPublishError} verdict - What the frame decided.
|
|
3467
|
+
* @param {'queue'|'exchange'} kind - Which leg is reporting.
|
|
3468
|
+
* @param {string} target - The queue or exchange the send was addressed to.
|
|
3469
|
+
* @param {string} queue - The queue the caller named (the exchange leg routes by it).
|
|
3470
|
+
* @param {Object} options - The publish options, which carry the retry layer's `attempt`.
|
|
3471
|
+
* @private
|
|
3472
|
+
*/
|
|
3473
|
+
_reportLateSendVerdict(verdict, kind, target, queue, options) {
|
|
3474
|
+
const code = verdict && verdict.cause ? verdict.cause.code : undefined;
|
|
3475
|
+
const attempt = options && typeof options.attempt === 'number' ? options.attempt : null;
|
|
3476
|
+
|
|
3477
|
+
this._logger.warn(
|
|
3478
|
+
`[RabbitMQClient] [mq-client-core] [PUBLISH] The broker refused this send and said so AFTER `
|
|
3479
|
+
+ `the confirmation budget of ${this._brokerAnswerTimeout}ms had already answered the caller - `
|
|
3480
|
+
+ `${kind} "${target}", broker code ${code}, verdict ${verdict.constructor.name}. `
|
|
3481
|
+
+ 'Expected: an answer within brokerAnswerTimeout, so the caller hears the broker rather '
|
|
3482
|
+
+ 'than the timeout. '
|
|
3483
|
+
+ 'Fix: the publish layer has already treated this message as unconfirmed and may retry it; '
|
|
3484
|
+
+ `raise brokerAnswerTimeout above the broker's answer latency for this route, or read `
|
|
3485
|
+
+ 'the publish:refused-late event to see how often it happens.'
|
|
3486
|
+
);
|
|
3487
|
+
|
|
3488
|
+
this.emit('publish:refused-late', {
|
|
3489
|
+
queue,
|
|
3490
|
+
exchange: kind === 'exchange' ? target : undefined,
|
|
3491
|
+
target,
|
|
3492
|
+
kind,
|
|
3493
|
+
code,
|
|
3494
|
+
verdict: verdict.constructor.name,
|
|
3495
|
+
// The verdict OBJECT, not only its name: since d.801c the retry layer acts on this
|
|
3496
|
+
// event — it ends its loop with this very error instead of publishing again — and a
|
|
3497
|
+
// listener that must act needs the thing, not a description of it.
|
|
3498
|
+
error: verdict,
|
|
3499
|
+
attempt,
|
|
3500
|
+
budgetMs: this._brokerAnswerTimeout
|
|
3501
|
+
});
|
|
3502
|
+
}
|
|
3503
|
+
|
|
3504
|
+
_collateralSendTransient(channel, kind, target) {
|
|
3505
|
+
const closeError = channel ? channel._closeError : null;
|
|
3506
|
+
const code = closeError ? closeError.code : undefined;
|
|
3507
|
+
|
|
3508
|
+
if (typeof code !== 'number') {
|
|
3509
|
+
return null;
|
|
3510
|
+
}
|
|
3511
|
+
|
|
3512
|
+
if (this._closeFrameNamesThisSend(channel, target)) {
|
|
3513
|
+
return null;
|
|
3514
|
+
}
|
|
3515
|
+
|
|
3516
|
+
return new TransientPublishError(collateralCloseMessage(kind, target, code), closeError);
|
|
3517
|
+
}
|
|
3518
|
+
|
|
3519
|
+
/**
|
|
3520
|
+
* Is the broker's close frame on `channel` an answer about the send addressed to
|
|
3521
|
+
* `target`?
|
|
3522
|
+
*
|
|
3523
|
+
* TWO READINGS, IN THIS ORDER, and the second only when the first cannot answer:
|
|
3524
|
+
*
|
|
3525
|
+
* 1. **How many sends the frame could be about.** When exactly one publish was
|
|
3526
|
+
* unconfirmed at the moment the frame arrived, the frame is about that one and
|
|
3527
|
+
* nothing else is possible. This is the case every earlier batch measured — d.685a
|
|
3528
|
+
* (403, 406), d.693 (404 on an undeclared exchange) — and it is why those verdicts
|
|
3529
|
+
* do not move. The count is the client's own (`_countSendInFlight()`), snapshotted
|
|
3530
|
+
* by `_attachPublisherChannelHandlers()` when it records the frame, because amqplib
|
|
3531
|
+
* emits `error` BEFORE `close` and it is `close` that rejects the callbacks: by
|
|
3532
|
+
* callback time the channel's own bookkeeping is already unwound.
|
|
3533
|
+
* 2. **Whether the frame NAMES this target.** The broker quotes the subject of its
|
|
3534
|
+
* refusal in the reply text, and amqplib keeps that text inside the close error's
|
|
3535
|
+
* `message` — measured live 2026-09-20, the close error's own properties are
|
|
3536
|
+
* `["code","classId","methodId"]` and there is no `replyText` member, so the
|
|
3537
|
+
* sentence is where the name is:
|
|
3538
|
+
* `Channel closed by server: 404 (NOT-FOUND) with message "NOT_FOUND - no exchange
|
|
3539
|
+
* 'no.such.exchange' in vhost 'mq-client-core-probe-…'"`.
|
|
3540
|
+
*
|
|
3541
|
+
* READING A SENTENCE IS NOT HOW THE CLASS IS DECIDED, and this does not change that
|
|
3542
|
+
* (`docs/biz/70-contracts/error-handling.md` §6): the CODE still says what the answer
|
|
3543
|
+
* means. The name only says WHOSE answer it is, and it is read in the one direction
|
|
3544
|
+
* that is safe to get wrong — a name this cannot find, or a rewording that hides it,
|
|
3545
|
+
* means "not claimed", so the send falls to the transient collateral verdict and the
|
|
3546
|
+
* message is republished. The failure mode of a broker that rewords its text is a
|
|
3547
|
+
* message sent twice, never a message dropped on a refusal that was not about it.
|
|
3548
|
+
*
|
|
3549
|
+
* @param {Object} channel - The channel carrying the recorded frame.
|
|
3550
|
+
* @param {string} target - The queue or exchange THIS send was addressed to.
|
|
3551
|
+
* @returns {boolean}
|
|
3552
|
+
* @private
|
|
3553
|
+
*/
|
|
3554
|
+
_closeFrameNamesThisSend(channel, target) {
|
|
3555
|
+
const unconfirmed = channel._unconfirmedAtCloseFrame;
|
|
3556
|
+
|
|
3557
|
+
// The frame was recorded by `_attachPublisherChannelHandlers()`, which takes the
|
|
3558
|
+
// snapshot in the same statement — so a missing number means the channel was never
|
|
3559
|
+
// prepared by this client, and an unprepared channel is not one whose sends this may
|
|
3560
|
+
// speak for. Not claimed, therefore transient: the safe direction.
|
|
3561
|
+
if (typeof unconfirmed !== 'number') {
|
|
3562
|
+
return false;
|
|
3563
|
+
}
|
|
3564
|
+
|
|
3565
|
+
if (unconfirmed <= 1) {
|
|
3566
|
+
return true;
|
|
3567
|
+
}
|
|
3568
|
+
|
|
3569
|
+
const named = this._nameInCloseFrame(channel._closeError);
|
|
3570
|
+
return named !== null && named === target;
|
|
3571
|
+
}
|
|
3572
|
+
|
|
3573
|
+
/**
|
|
3574
|
+
* The name the broker quoted in its close frame, or `null` when it quoted none.
|
|
3575
|
+
*
|
|
3576
|
+
* The first single-quoted token of the reply text is its subject — `no exchange 'X' in
|
|
3577
|
+
* vhost 'Y'`, `no queue 'X' in vhost 'Y'`, `cannot obtain exclusive access to locked
|
|
3578
|
+
* queue 'X' in vhost 'Y'`. A frame that quotes nothing (`PRECONDITION_FAILED - unknown
|
|
3579
|
+
* delivery tag 999999`, measured d.698) answers with `null`, which is a real answer
|
|
3580
|
+
* here: "this frame names nobody".
|
|
3581
|
+
*
|
|
3582
|
+
* @param {Error} closeError - The recorded close frame.
|
|
3583
|
+
* @returns {string|null}
|
|
3584
|
+
* @private
|
|
3585
|
+
*/
|
|
3586
|
+
_nameInCloseFrame(closeError) {
|
|
3587
|
+
const text = closeError && typeof closeError.message === 'string' ? closeError.message : '';
|
|
3588
|
+
const quoted = /'([^']*)'/.exec(text);
|
|
3589
|
+
return quoted ? quoted[1] : null;
|
|
3590
|
+
}
|
|
3591
|
+
|
|
3592
|
+
/**
|
|
3593
|
+
* Count one send as unconfirmed on `channel`, and hand back the ONE way to stop
|
|
3594
|
+
* counting it.
|
|
3595
|
+
*
|
|
3596
|
+
* The count exists for `_closeFrameNamesThisSend()`: it is what says whether a numbered
|
|
3597
|
+
* close frame can only be about this send. It is the client's own rather than amqplib's
|
|
3598
|
+
* `unconfirmed` array, because that array is emptied by the very event this has to
|
|
3599
|
+
* measure — `toClosed()` rejects every pending publish, and only then do the callbacks
|
|
3600
|
+
* that ask this question run.
|
|
3601
|
+
*
|
|
3602
|
+
* A closure rather than a bare decrement at each site, because a send ends in three
|
|
3603
|
+
* places (the callback, a synchronous throw, and nowhere at all when neither happens),
|
|
3604
|
+
* and a count that is decremented twice or not at all makes the reading above answer
|
|
3605
|
+
* for the wrong number of messages.
|
|
3606
|
+
*
|
|
3607
|
+
* @param {Object} channel - The channel the send is going out on.
|
|
3608
|
+
* @returns {function(): void} Idempotent; call it when the send is no longer unconfirmed.
|
|
3609
|
+
* @private
|
|
3610
|
+
*/
|
|
3611
|
+
/**
|
|
3612
|
+
* The channel of one send role, as it stands NOW.
|
|
3613
|
+
*
|
|
3614
|
+
* A getter rather than a captured value, because the member is replaced when the
|
|
3615
|
+
* channel dies (`_attachPublisherChannelHandlers()` clears it, the next publish
|
|
3616
|
+
* recreates it), and a caller holding the previous one would be sending on a corpse.
|
|
3617
|
+
*
|
|
3618
|
+
* @param {'publisher'|'telemetry'} role
|
|
3619
|
+
* @returns {Object|null}
|
|
3620
|
+
* @private
|
|
3621
|
+
*/
|
|
3622
|
+
_sendChannelOf(role) {
|
|
3623
|
+
return role === 'telemetry' ? this._telemetryChannel : this._channel;
|
|
3624
|
+
}
|
|
3625
|
+
|
|
3626
|
+
/**
|
|
3627
|
+
* The one place a send channel is assigned — so "which member does this role live in"
|
|
3628
|
+
* is written once rather than at every site that creates, clears or tears one down.
|
|
3629
|
+
*
|
|
3630
|
+
* @param {'publisher'|'telemetry'} role
|
|
3631
|
+
* @param {Object|null} channel
|
|
3632
|
+
* @private
|
|
3633
|
+
*/
|
|
3634
|
+
_setSendChannelOf(role, channel) {
|
|
3635
|
+
if (role === 'telemetry') {
|
|
3636
|
+
this._telemetryChannel = channel;
|
|
3637
|
+
} else {
|
|
3638
|
+
this._channel = channel;
|
|
3639
|
+
}
|
|
3640
|
+
}
|
|
3641
|
+
|
|
3642
|
+
/**
|
|
3643
|
+
* Which channel a publish goes out on, from the options it was given.
|
|
3644
|
+
*
|
|
3645
|
+
* Explicit, never derived from the queue name (`architecture-principles.md` §8, §9): a
|
|
3646
|
+
* telemetry publish says so because `BaseClient.publishTelemetry()` says so, and a name
|
|
3647
|
+
* that merely looks like a monitoring queue does not move a caller's message onto
|
|
3648
|
+
* another channel.
|
|
3649
|
+
*
|
|
3650
|
+
* @param {Object} options - The publish options as the caller handed them over.
|
|
3651
|
+
* @returns {'publisher'|'telemetry'}
|
|
3652
|
+
* @private
|
|
3653
|
+
*/
|
|
3654
|
+
_sendRoleFor(options) {
|
|
3655
|
+
return options && options.channelRole === 'telemetry' ? 'telemetry' : 'publisher';
|
|
3656
|
+
}
|
|
3657
|
+
|
|
3658
|
+
_countSendInFlight(channel) {
|
|
3659
|
+
channel._unconfirmedSends = (channel._unconfirmedSends || 0) + 1;
|
|
3660
|
+
let counted = true;
|
|
3661
|
+
return () => {
|
|
3662
|
+
if (!counted) return;
|
|
3663
|
+
counted = false;
|
|
3664
|
+
channel._unconfirmedSends -= 1;
|
|
3665
|
+
};
|
|
3666
|
+
}
|
|
3667
|
+
|
|
2732
3668
|
/**
|
|
2733
3669
|
* Declare an exchange.
|
|
2734
3670
|
*
|
|
@@ -2775,7 +3711,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2775
3711
|
...options,
|
|
2776
3712
|
durable: options.durable !== undefined ? options.durable : this._config.durable
|
|
2777
3713
|
};
|
|
2778
|
-
return await
|
|
3714
|
+
return await this._askBroker(
|
|
3715
|
+
{ operation: 'assertExchange', target: exchange, channel },
|
|
3716
|
+
() => channel.assertExchange(exchange, type, exchangeOptions)
|
|
3717
|
+
);
|
|
2779
3718
|
}
|
|
2780
3719
|
|
|
2781
3720
|
/**
|
|
@@ -2819,7 +3758,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2819
3758
|
}
|
|
2820
3759
|
|
|
2821
3760
|
const channel = await this._requireQueueChannel('bindQueue', `${queue} -> ${exchange}`);
|
|
2822
|
-
return await
|
|
3761
|
+
return await this._askBroker(
|
|
3762
|
+
{ operation: 'bindQueue', target: `${queue} -> ${exchange}`, channel },
|
|
3763
|
+
() => channel.bindQueue(queue, exchange, pattern)
|
|
3764
|
+
);
|
|
2823
3765
|
}
|
|
2824
3766
|
|
|
2825
3767
|
/**
|
|
@@ -2902,7 +3844,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2902
3844
|
}
|
|
2903
3845
|
|
|
2904
3846
|
this._trackChannelOperation(this._consumerChannel, `cancelConsumer ${queue}`);
|
|
2905
|
-
const
|
|
3847
|
+
const consumerChannel = this._consumerChannel;
|
|
3848
|
+
const result = await this._askBroker(
|
|
3849
|
+
{ operation: 'cancel', target: queue, channel: consumerChannel },
|
|
3850
|
+
() => consumerChannel.cancel(info.consumerTag)
|
|
3851
|
+
);
|
|
2906
3852
|
// Dropped only after the broker accepted the cancel: a consumer that is
|
|
2907
3853
|
// still delivering must stay on the list recovery re-attaches.
|
|
2908
3854
|
this._activeConsumers.delete(queue);
|
|
@@ -2919,7 +3865,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2919
3865
|
*/
|
|
2920
3866
|
async checkQueue(queue) {
|
|
2921
3867
|
const channel = await this._requireQueueChannel('checkQueue', queue);
|
|
2922
|
-
return await
|
|
3868
|
+
return await this._askBroker(
|
|
3869
|
+
{ operation: 'checkQueue', target: queue, channel },
|
|
3870
|
+
() => channel.checkQueue(queue)
|
|
3871
|
+
);
|
|
2923
3872
|
}
|
|
2924
3873
|
|
|
2925
3874
|
/**
|
|
@@ -2928,7 +3877,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2928
3877
|
*/
|
|
2929
3878
|
async purgeQueue(queue) {
|
|
2930
3879
|
const channel = await this._requireQueueChannel('purgeQueue', queue);
|
|
2931
|
-
return await
|
|
3880
|
+
return await this._askBroker(
|
|
3881
|
+
{ operation: 'purgeQueue', target: queue, channel },
|
|
3882
|
+
() => channel.purgeQueue(queue)
|
|
3883
|
+
);
|
|
2932
3884
|
}
|
|
2933
3885
|
|
|
2934
3886
|
/**
|
|
@@ -2938,7 +3890,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2938
3890
|
*/
|
|
2939
3891
|
async deleteQueue(queue, options = {}) {
|
|
2940
3892
|
const channel = await this._requireQueueChannel('deleteQueue', queue);
|
|
2941
|
-
return await
|
|
3893
|
+
return await this._askBroker(
|
|
3894
|
+
{ operation: 'deleteQueue', target: queue, channel },
|
|
3895
|
+
() => channel.deleteQueue(queue, options)
|
|
3896
|
+
);
|
|
2942
3897
|
}
|
|
2943
3898
|
|
|
2944
3899
|
/**
|
|
@@ -2957,15 +3912,60 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2957
3912
|
}
|
|
2958
3913
|
|
|
2959
3914
|
/**
|
|
2960
|
-
*
|
|
3915
|
+
* One publish attempt, without retry — and, when it fails, whether its frame LEFT THE
|
|
3916
|
+
* CLIENT (d.1145, conf `mq-publish-buffer` 001).
|
|
3917
|
+
*
|
|
3918
|
+
* Every error this attempt throws carries `sent` (boolean): `true` once `sendToQueue()` /
|
|
3919
|
+
* `publish()` on the channel returned without throwing — the frame was written to the
|
|
3920
|
+
* channel — and `false` for everything that failed before that (the pre-check, the channel
|
|
3921
|
+
* ensure, a throwing send). It says nothing about whether the broker stored the frame; it is
|
|
3922
|
+
* what tells a message the broker may still deliver (held in a blocked socket, measured
|
|
3923
|
+
* d.1143) from one it never received. `PublishLayer` folds the attempts of one call into the
|
|
3924
|
+
* fact it writes on its own error and on the buffer entry.
|
|
3925
|
+
*
|
|
3926
|
+
* A nested republish from the confirm path (see `_publishOneAttempt`) may throw an error
|
|
3927
|
+
* that already says `sent: true`; that fact is kept, never lowered.
|
|
3928
|
+
*
|
|
3929
|
+
* @param {string} queue
|
|
3930
|
+
* @param {Buffer} buffer
|
|
3931
|
+
* @param {Object} [options]
|
|
3932
|
+
* @returns {Promise<void>}
|
|
3933
|
+
* @throws {Error} the attempt's failure, with `sent` written on it
|
|
2961
3934
|
* @private
|
|
2962
3935
|
*/
|
|
2963
3936
|
async _publishOnce(queue, buffer, options = {}) {
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
3937
|
+
const attempt = { sent: false };
|
|
3938
|
+
try {
|
|
3939
|
+
return await this._publishOneAttempt(queue, buffer, options, attempt);
|
|
3940
|
+
} catch (err) {
|
|
3941
|
+
if (err !== null && typeof err === 'object') {
|
|
3942
|
+
err.sent = attempt.sent || err.sent === true;
|
|
3943
|
+
}
|
|
3944
|
+
throw err;
|
|
3945
|
+
}
|
|
3946
|
+
}
|
|
3947
|
+
|
|
3948
|
+
/**
|
|
3949
|
+
* The body of one publish attempt. `attempt.sent` turns `true` the moment the frame is
|
|
3950
|
+
* written to the channel; `_publishOnce()` reads it.
|
|
3951
|
+
*
|
|
3952
|
+
* @param {string} queue
|
|
3953
|
+
* @param {Buffer} buffer
|
|
3954
|
+
* @param {Object} options
|
|
3955
|
+
* @param {{sent: boolean}} attempt - written here, read by `_publishOnce()`
|
|
3956
|
+
* @private
|
|
3957
|
+
*/
|
|
3958
|
+
async _publishOneAttempt(queue, buffer, options, attempt) {
|
|
3959
|
+
// Whose channel this send goes out on (d.724). Read once, here: the role cannot
|
|
3960
|
+
// change inside one attempt, and the channel it names can — so everything below asks
|
|
3961
|
+
// `_sendChannelOf(sendRole)` rather than holding a member.
|
|
3962
|
+
const sendRole = this._sendRoleFor(options);
|
|
3963
|
+
|
|
3964
|
+
// Ensure the send channel exists and is open (auto-recreates if closed)
|
|
3965
|
+
await this._ensurePublisherChannel(sendRole);
|
|
3966
|
+
|
|
2967
3967
|
// Track operation for debugging
|
|
2968
|
-
this._trackChannelOperation(this.
|
|
3968
|
+
this._trackChannelOperation(this._sendChannelOf(sendRole), `publish to ${queue}`);
|
|
2969
3969
|
|
|
2970
3970
|
// Asked with `=== undefined`, not with `||`: the empty string IS a value here
|
|
2971
3971
|
// — it names the default exchange — and the third leg was a third copy of a
|
|
@@ -3018,13 +4018,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3018
4018
|
// If queue doesn't exist (404), we should NOT auto-create it - infrastructure queues must be created explicitly
|
|
3019
4019
|
// This prevents creating queues with wrong arguments (no TTL) which causes 406 errors later
|
|
3020
4020
|
if (!skipQueueExistencePrecheck) {
|
|
4021
|
+
// Declared OUTSIDE the `try`, because the `catch` below has to ask whether
|
|
4022
|
+
// the channel that CARRIED the RPC is still alive — that reading is what
|
|
4023
|
+
// tells "the broker answered" from "the answer never came" (d.683).
|
|
4024
|
+
let queueChannel = null;
|
|
3021
4025
|
try {
|
|
3022
4026
|
// The queue channel comes from the ONE rail (d.282): ensure it, refuse
|
|
3023
4027
|
// it unless it is ALIVE, record the operation. Until d.284 this site
|
|
3024
4028
|
// assembled that sequence itself — `_ensureQueueChannel()` plus
|
|
3025
4029
|
// `_isChannelAlive()` by hand — differing from the rail in nothing but
|
|
3026
4030
|
// the error type, which is `_publishPathTransient()`'s job below.
|
|
3027
|
-
let queueChannel;
|
|
3028
4031
|
try {
|
|
3029
4032
|
queueChannel = await this._requireQueueChannel('checkQueue', queue);
|
|
3030
4033
|
} catch (railErr) {
|
|
@@ -3033,7 +4036,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3033
4036
|
}
|
|
3034
4037
|
throw railErr;
|
|
3035
4038
|
}
|
|
3036
|
-
|
|
4039
|
+
const checkedOn = queueChannel;
|
|
4040
|
+
await this._askBroker(
|
|
4041
|
+
{ operation: 'checkQueue', target: queue, channel: checkedOn },
|
|
4042
|
+
() => checkedOn.checkQueue(queue)
|
|
4043
|
+
);
|
|
3037
4044
|
// Queue exists - proceed to publish
|
|
3038
4045
|
} catch (checkErr) {
|
|
3039
4046
|
// If queue doesn't exist (404), this je ERROR – ale rozlišujeme infra vs. non-infra:
|
|
@@ -3045,7 +4052,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3045
4052
|
// → QueueNotFoundError(kind='business'). Publisher je nikdy nevytváří.
|
|
3046
4053
|
// - Ostatní jména (ani infra, ani business vzor) nemá kdo vlastnit, takže
|
|
3047
4054
|
// je nezakládá nikdo → QueueNotFoundError(kind='unowned'), d.419.
|
|
3048
|
-
if (checkErr.code ===
|
|
4055
|
+
if (checkErr.code === AMQP_NOT_FOUND) {
|
|
3049
4056
|
// `queueConfig` is loaded once at the top of this module. It used to be
|
|
3050
4057
|
// required here, inside a try/catch whose `catch` re-implemented
|
|
3051
4058
|
// `isInfrastructureQueue` as an inline prefix list — a fallback
|
|
@@ -3067,7 +4074,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3067
4074
|
// registration; a publisher that creates it is a programming error.
|
|
3068
4075
|
//
|
|
3069
4076
|
// The rule and this exact wording came from
|
|
3070
|
-
// `conn-infra-mq/src/transports/rabbitmqClient.js
|
|
4077
|
+
// `conn-infra-mq/src/transports/rabbitmqClient.js` — a transport
|
|
3071
4078
|
// that nothing in `src/` instantiated, so the guard protected nobody
|
|
3072
4079
|
// while this file, the one every client actually runs, auto-created the
|
|
3073
4080
|
// queue and only printed a warning.
|
|
@@ -3075,7 +4082,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3075
4082
|
throw new QueueNotFoundError(queue, false, checkErr, 'business');
|
|
3076
4083
|
}
|
|
3077
4084
|
// Nobody owns this name. Until d.419 that opened a THIRD case beside
|
|
3078
|
-
// the two above: the publish path asked
|
|
4085
|
+
// the two above: the publish path asked the recovery worker whether this
|
|
3079
4086
|
// client might declare it and, when the answer was yes, declared it —
|
|
3080
4087
|
// `durable=true, arguments={}`, no TTL, no dead-letter route (measured
|
|
3081
4088
|
// live, d.410). The concept had already ruled that out: "queues come
|
|
@@ -3100,6 +4107,40 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3100
4107
|
// "nothing declares this name", and the publish path must not flatten the
|
|
3101
4108
|
// two into one message.
|
|
3102
4109
|
throw new QueueNotFoundError(queue, false, checkErr, 'unowned');
|
|
4110
|
+
} else if (
|
|
4111
|
+
!(checkErr instanceof TransientPublishError)
|
|
4112
|
+
&& checkErr.code === undefined
|
|
4113
|
+
&& !this._isChannelAlive(queueChannel)
|
|
4114
|
+
) {
|
|
4115
|
+
// THE RPC NEVER GOT AN ANSWER. amqplib answers the request that was in
|
|
4116
|
+
// flight when a channel closed with the broker's own close frame — a
|
|
4117
|
+
// NUMBER in `code` — and rejects whatever was queued behind it with a
|
|
4118
|
+
// bare `new Error('Channel ended, no reply will be forthcoming')`
|
|
4119
|
+
// (`_rejectPending`, `node_modules/amqplib/lib/channel.js`), whose
|
|
4120
|
+
// only own properties are `stack` and `message`. Until d.683 that object
|
|
4121
|
+
// travelled untouched to `classifyPublishError()`, which had to keep its
|
|
4122
|
+
// verdict by matching the sentence — the one text branch d.677 could not
|
|
4123
|
+
// remove from where it stood, because the sentence was all there was.
|
|
4124
|
+
//
|
|
4125
|
+
// This site does not have to read it: it knows which channel carried the
|
|
4126
|
+
// RPC and can see that the channel is gone, so it answers in the dialect
|
|
4127
|
+
// every other leg of this path already speaks. The three conditions are
|
|
4128
|
+
// each doing work:
|
|
4129
|
+
// - `code === undefined` — a numbered refusal is an ANSWER about this
|
|
4130
|
+
// request (405 RESOURCE_LOCKED, measured), and repeating the request
|
|
4131
|
+
// repeats it. It stays permanent;
|
|
4132
|
+
// - the channel is dead — a failure on a LIVE channel did get through
|
|
4133
|
+
// to the broker and back, whatever it says;
|
|
4134
|
+
// - not already typed — `_publishPathTransient()` above throws into
|
|
4135
|
+
// this same `catch`, and wrapping a verdict in itself would bury the
|
|
4136
|
+
// rail's own sentence one level deeper for no reader.
|
|
4137
|
+
//
|
|
4138
|
+
// Since d.1135 the check goes through `_askBroker()`, so what arrives here
|
|
4139
|
+
// is its `ConnectionError`: either the budget ran out (the channel is then
|
|
4140
|
+
// discarded — dead) or amqplib's bare rejection, folded where it arrived. Both
|
|
4141
|
+
// meet the three conditions unchanged; the bare error sits one level down, in
|
|
4142
|
+
// `cause.cause`.
|
|
4143
|
+
throw this._publishPathTransient('checkQueue', queue, checkErr);
|
|
3103
4144
|
} else {
|
|
3104
4145
|
// Other error - rethrow
|
|
3105
4146
|
throw checkErr;
|
|
@@ -3123,8 +4164,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3123
4164
|
let timeout = null;
|
|
3124
4165
|
let safetyTimer = null;
|
|
3125
4166
|
let settled = false;
|
|
4167
|
+
// Assigned once the send is on its way (below), and released here with the
|
|
4168
|
+
// timers: the attempt ends in five places — the callback, a synchronous throw,
|
|
4169
|
+
// the confirmation timeout, the liveness check before the send, and the
|
|
4170
|
+
// watchdog — and a send still counted after its attempt has ended would make
|
|
4171
|
+
// the next close frame on this channel look as if it could be about two
|
|
4172
|
+
// messages (`_closeFrameNamesThisSend()`, d.723).
|
|
4173
|
+
let sendSettled = () => {};
|
|
3126
4174
|
const disarm = () => {
|
|
3127
4175
|
settled = true;
|
|
4176
|
+
sendSettled();
|
|
3128
4177
|
this._clearTimeout(timeout);
|
|
3129
4178
|
this._clearTimeout(safetyTimer);
|
|
3130
4179
|
timeout = null;
|
|
@@ -3147,7 +4196,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3147
4196
|
// during the awaits above (queue existence pre-check), so this is not
|
|
3148
4197
|
// a repeat of `_ensurePublisherChannel()` — it is the last reading
|
|
3149
4198
|
// before the payload leaves, and it reads the one liveness source.
|
|
3150
|
-
if (!this._isChannelAlive(this.
|
|
4199
|
+
if (!this._isChannelAlive(this._sendChannelOf(sendRole))) {
|
|
3151
4200
|
disarm();
|
|
3152
4201
|
reject(new ConnectionError(
|
|
3153
4202
|
`[RabbitMQClient] Cannot publish: channel is closed for queue "${queue}" - `
|
|
@@ -3158,9 +4207,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3158
4207
|
}
|
|
3159
4208
|
|
|
3160
4209
|
// Track original channel to detect if it was closed/recreated during publish
|
|
3161
|
-
const originalChannel = this.
|
|
4210
|
+
const originalChannel = this._sendChannelOf(sendRole);
|
|
3162
4211
|
let callbackInvoked = false;
|
|
3163
|
-
|
|
4212
|
+
// This send is unconfirmed from the moment it is handed over until the broker
|
|
4213
|
+
// answers it, the channel dies under it, or amqplib refuses it outright — and
|
|
4214
|
+
// that number is what decides whose answer a close frame is (d.723).
|
|
4215
|
+
sendSettled = this._countSendInFlight(originalChannel);
|
|
4216
|
+
|
|
3164
4217
|
// Send message with callback
|
|
3165
4218
|
try {
|
|
3166
4219
|
originalChannel.sendToQueue(
|
|
@@ -3169,14 +4222,45 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3169
4222
|
buildAmqpMessageProperties(options, persistent, headers),
|
|
3170
4223
|
(err, ok) => {
|
|
3171
4224
|
callbackInvoked = true;
|
|
4225
|
+
// Read BEFORE `disarm()`, which sets it: the question is whether the budget
|
|
4226
|
+
// had already answered the caller, and after `disarm()` nobody can tell.
|
|
4227
|
+
const answeredAlready = settled;
|
|
3172
4228
|
disarm();
|
|
3173
|
-
|
|
4229
|
+
|
|
3174
4230
|
// CRITICAL: the delivery tag belongs to the channel that SENT the
|
|
3175
4231
|
// message, so that channel's liveness is the whole question. One
|
|
3176
4232
|
// reading answers it: a channel that was replaced is a channel that
|
|
3177
4233
|
// died first (`_ensurePublisherChannel()` recreates only a dead one),
|
|
3178
4234
|
// so identity and `_createdAt` comparisons said nothing this does not.
|
|
3179
4235
|
if (!this._isChannelAlive(originalChannel)) {
|
|
4236
|
+
// WHY the channel died decides first. A numbered close frame is the
|
|
4237
|
+
// broker's answer about THIS message, and no reconnect changes it
|
|
4238
|
+
// (`_refusedSendPermanent()`, d.685a) — so it is asked before the
|
|
4239
|
+
// reconnect wait below, which would otherwise re-offer a message the
|
|
4240
|
+
// broker will refuse again.
|
|
4241
|
+
const refused = this._refusedSendPermanent(originalChannel, 'queue', queue);
|
|
4242
|
+
if (refused) {
|
|
4243
|
+
if (answeredAlready) {
|
|
4244
|
+
this._reportLateSendVerdict(refused, 'queue', queue, queue, options);
|
|
4245
|
+
} else {
|
|
4246
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] ${refused.message}`);
|
|
4247
|
+
}
|
|
4248
|
+
reject(refused);
|
|
4249
|
+
return;
|
|
4250
|
+
}
|
|
4251
|
+
// The frame was there but it was about a DIFFERENT send on this channel
|
|
4252
|
+
// (d.723): this message was never answered, so it is republished rather
|
|
4253
|
+
// than dropped on somebody else's refusal.
|
|
4254
|
+
const collateral = this._collateralSendTransient(originalChannel, 'queue', queue);
|
|
4255
|
+
if (collateral) {
|
|
4256
|
+
if (answeredAlready) {
|
|
4257
|
+
this._reportLateSendVerdict(collateral, 'queue', queue, queue, options);
|
|
4258
|
+
} else {
|
|
4259
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] ${collateral.message}`);
|
|
4260
|
+
}
|
|
4261
|
+
reject(collateral);
|
|
4262
|
+
return;
|
|
4263
|
+
}
|
|
3180
4264
|
// Channel was closed or recreated - delivery tag is invalid
|
|
3181
4265
|
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed/recreated during publish to queue "${queue}", ignoring callback`);
|
|
3182
4266
|
// If we're reconnecting, wait and retry
|
|
@@ -3186,12 +4270,20 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3186
4270
|
return this.publish(queue, buffer, options);
|
|
3187
4271
|
}).then(resolve).catch(reject);
|
|
3188
4272
|
} else {
|
|
3189
|
-
//
|
|
3190
|
-
//
|
|
4273
|
+
// No reconnect is in flight YET — which says nothing about whether one
|
|
4274
|
+
// follows. When the connection dies, amqplib closes its channels before
|
|
4275
|
+
// it announces the connection's own `close` (`lib/connection.js`,
|
|
4276
|
+
// `toClosed()`), and that announcement is what starts the reconnect, so
|
|
4277
|
+
// this callback runs before it (d.1134). The sentence therefore states
|
|
4278
|
+
// what is known here — the channel closed with the confirm outstanding —
|
|
4279
|
+
// and the classification makes it transient, so the publish path
|
|
4280
|
+
// retries and buffers it.
|
|
3191
4281
|
reject(new PublishError(
|
|
3192
|
-
`[RabbitMQClient] Channel closed during publish to queue "${queue}"
|
|
3193
|
-
+ '
|
|
3194
|
-
+ '
|
|
4282
|
+
`[RabbitMQClient] Channel closed during publish to queue "${queue}" before the broker confirmed the message - `
|
|
4283
|
+
+ 'message delivery not guaranteed. '
|
|
4284
|
+
+ 'Expected: the channel to stay open until the broker confirms. '
|
|
4285
|
+
+ 'Fix: republish the message — the confirm never arrived, so it may or may not have been stored; '
|
|
4286
|
+
+ 'error.buffered says whether the buffer holds it for the next reconnect.',
|
|
3195
4287
|
queue,
|
|
3196
4288
|
null
|
|
3197
4289
|
));
|
|
@@ -3200,29 +4292,24 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3200
4292
|
}
|
|
3201
4293
|
|
|
3202
4294
|
if (err) {
|
|
3203
|
-
//
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
}
|
|
3222
|
-
} else {
|
|
3223
|
-
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
|
|
3224
|
-
reject(err);
|
|
3225
|
-
}
|
|
4295
|
+
// A confirm callback is handed ONE of three things by amqplib, and this is
|
|
4296
|
+
// the whole list: `null` (the broker confirmed), `new Error('message
|
|
4297
|
+
// nacked')` and `new Error('channel closed')` — composed in
|
|
4298
|
+
// `lib/channel.js` (the `Channel` constructor's confirm listeners) and
|
|
4299
|
+
// `lib/channel_model.js` (`ConfirmChannel.waitForConfirms()`), with no `code` on
|
|
4300
|
+
// either. The callback cannot see WHY the channel died; the channel can,
|
|
4301
|
+
// and the branch above reads it (`_refusedSendPermanent()`, d.685a).
|
|
4302
|
+
//
|
|
4303
|
+
// So there is no delivery-tag case to catch here, and until d.698 one was
|
|
4304
|
+
// written anyway: a substring test for the words a 406 reply text carries.
|
|
4305
|
+
// It matched none of the three, and what it did was not neutral: it
|
|
4306
|
+
// answered with a `PublishError`, which the
|
|
4307
|
+
// classification reads as TRANSIENT, so a message would have been retried
|
|
4308
|
+
// `publishMaxRetries` times and then buffered on the strength of a wording.
|
|
4309
|
+
// The live carrier of that sentence is a 406 close frame on the channel's
|
|
4310
|
+
// `error` event (measured d.693/d.698), never an argument to this callback.
|
|
4311
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
|
|
4312
|
+
reject(err);
|
|
3226
4313
|
} else {
|
|
3227
4314
|
// Structured log: publish confirmed
|
|
3228
4315
|
const wfId = workflowIdOf(headers);
|
|
@@ -3237,6 +4324,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3237
4324
|
resolve();
|
|
3238
4325
|
}
|
|
3239
4326
|
});
|
|
4327
|
+
// The frame is written to the channel: `sendToQueue()` returned without throwing.
|
|
4328
|
+
attempt.sent = true;
|
|
3240
4329
|
|
|
3241
4330
|
// Set a safety timeout - if callback wasn't invoked and channel closed, retry.
|
|
3242
4331
|
// A broker confirm can land inside `sendToQueue()` itself, so the
|
|
@@ -3257,8 +4346,27 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3257
4346
|
}
|
|
3258
4347
|
} catch (sendErr) {
|
|
3259
4348
|
disarm();
|
|
3260
|
-
//
|
|
3261
|
-
|
|
4349
|
+
// THE CHANNEL ANSWERS, NOT THE SENTENCE (d.685c). A synchronous throw from
|
|
4350
|
+
// `sendToQueue()` means amqplib refused to send at all, and the one question
|
|
4351
|
+
// that decides what to do with the message is whether the channel it was
|
|
4352
|
+
// handed is still alive: a dead one will be replaced by the reconnect, so the
|
|
4353
|
+
// message is republished over its successor; a live one did accept the call
|
|
4354
|
+
// and failed for its own reason, which no reconnect changes.
|
|
4355
|
+
//
|
|
4356
|
+
// Until this batch the question was asked of the message text
|
|
4357
|
+
// (`includes('Channel ended') || includes('Channel closed')`), which is the
|
|
4358
|
+
// defect d.677 removed from the classification: a wording decided whether a
|
|
4359
|
+
// message was re-offered to the broker. It was also matching a sentence that
|
|
4360
|
+
// cannot arrive here — `'Channel ended, no reply will be forthcoming'` is
|
|
4361
|
+
// built for an RPC awaiting a REPLY (`_rejectPending()`, amqplib
|
|
4362
|
+
// `lib/channel.js`), and a send is not an RPC.
|
|
4363
|
+
//
|
|
4364
|
+
// Measured carrier, live against the dev broker: amqplib invalidates a dead
|
|
4365
|
+
// channel's send methods (`invalidateSend()`, `lib/channel.js`), so
|
|
4366
|
+
// what arrives is its `IllegalOperationError` with `'Channel closed'` — no
|
|
4367
|
+
// code, and `channel._closeError` null. Liveness from the one source
|
|
4368
|
+
// (`_isChannelAlive`, d.260), like the two callbacks above.
|
|
4369
|
+
if (!this._isChannelAlive(originalChannel)) {
|
|
3262
4370
|
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during sendToQueue, will retry after reconnection`);
|
|
3263
4371
|
// Wait for reconnection and retry
|
|
3264
4372
|
if (this._reconnecting) {
|
|
@@ -3302,8 +4410,20 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3302
4410
|
|
|
3303
4411
|
// Use callback-based confirmation - kanály jsou spolehlivé
|
|
3304
4412
|
const confirmPromise = new Promise((resolve, reject) => {
|
|
4413
|
+
// Assigned once the send is on its way (below). It is declared HERE because the
|
|
4414
|
+
// count has to be released on EVERY ending of this attempt, and the earliest of
|
|
4415
|
+
// them — the confirmation timeout — is armed before the send exists. A send
|
|
4416
|
+
// still counted after its attempt has ended would make the next close frame on
|
|
4417
|
+
// this channel look as if it could be about two messages (d.723).
|
|
4418
|
+
let sendSettled = () => {};
|
|
4419
|
+
// Whether the budget has already answered the caller — read by the callback, so
|
|
4420
|
+
// a verdict that arrives afterwards can be REPORTED instead of handed to a
|
|
4421
|
+
// settled promise (d.801b). The queue leg keeps the same fact in `settled`.
|
|
4422
|
+
let answeredAlready = false;
|
|
3305
4423
|
// Set timeout for exchange publish confirmation
|
|
3306
4424
|
const timeout = this._setTimeout(() => {
|
|
4425
|
+
answeredAlready = true;
|
|
4426
|
+
sendSettled();
|
|
3307
4427
|
reject(new PublishError(
|
|
3308
4428
|
`[RabbitMQClient] Exchange publish confirmation timeout for exchange "${exchange}" after ${this._brokerAnswerTimeout}ms - `
|
|
3309
4429
|
+ 'Expected: the broker to confirm the message within brokerAnswerTimeout. '
|
|
@@ -3313,14 +4433,79 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3313
4433
|
));
|
|
3314
4434
|
}, this._brokerAnswerTimeout);
|
|
3315
4435
|
|
|
3316
|
-
|
|
4436
|
+
// The channel that SENDS is the channel whose liveness answers the
|
|
4437
|
+
// callback below — read once here, like the queue leg does, because
|
|
4438
|
+
// the member may already hold the replacement by the time the
|
|
4439
|
+
// callback runs.
|
|
4440
|
+
const sendingChannel = this._sendChannelOf(sendRole);
|
|
4441
|
+
// Unconfirmed until answered — the same count the queue leg keeps, and for the
|
|
4442
|
+
// same reason: it is what decides whose answer a close frame is (d.723).
|
|
4443
|
+
sendSettled = this._countSendInFlight(sendingChannel);
|
|
4444
|
+
|
|
4445
|
+
sendingChannel.publish(
|
|
3317
4446
|
exchange,
|
|
3318
4447
|
routingKey,
|
|
3319
4448
|
buffer,
|
|
3320
4449
|
buildAmqpMessageProperties(options, persistent, headers),
|
|
3321
4450
|
(err, ok) => {
|
|
4451
|
+
sendSettled();
|
|
3322
4452
|
this._clearTimeout(timeout);
|
|
3323
4453
|
if (err) {
|
|
4454
|
+
// THE CONFIRM NEVER CAME. amqplib answers every unconfirmed publish of
|
|
4455
|
+
// a channel that has just closed with a bare `new Error('channel
|
|
4456
|
+
// closed')` (the `Channel` constructor's `'close'` listener, `node_modules/amqplib/lib/channel.js`) — class `Error`,
|
|
4457
|
+
// no code, own properties `stack` and `message`. The broker's own
|
|
4458
|
+
// numbered refusal is NOT what arrives here: it goes to the channel's
|
|
4459
|
+
// `error` event (measured d.677, carrier B1 — a publish through an
|
|
4460
|
+
// exchange nobody declared).
|
|
4461
|
+
//
|
|
4462
|
+
// The queue leg above has read the sending channel's liveness since
|
|
4463
|
+
// long before this batch, which is why a refused send already reached
|
|
4464
|
+
// the classification as this package's own `PublishError` (carrier E1).
|
|
4465
|
+
// This leg had no such reading, so its bare sentinel travelled untouched
|
|
4466
|
+
// and `classifyPublishError()` had to keep its verdict by the sentence.
|
|
4467
|
+
// Now both legs answer the same way, and the sentinel is kept in `cause`
|
|
4468
|
+
// rather than read (d.683).
|
|
4469
|
+
//
|
|
4470
|
+
// A live channel means the broker DID answer — a nack is the measured
|
|
4471
|
+
// case (carrier G, `'message nacked'`, amqplib `lib/channel.js`, the `Channel` constructor's `'nack'` listener) —
|
|
4472
|
+
// and that answer is about this message, so it stays what it was.
|
|
4473
|
+
if (!this._isChannelAlive(sendingChannel)) {
|
|
4474
|
+
// The same question this leg's sibling asks one branch up: did the broker
|
|
4475
|
+
// REFUSE this send, or did the channel merely die under it (d.685a)?
|
|
4476
|
+
const refused = this._refusedSendPermanent(sendingChannel, 'exchange', exchange);
|
|
4477
|
+
if (refused) {
|
|
4478
|
+
if (answeredAlready) {
|
|
4479
|
+
this._reportLateSendVerdict(refused, 'exchange', exchange, queue, options);
|
|
4480
|
+
} else {
|
|
4481
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] ${refused.message}`);
|
|
4482
|
+
}
|
|
4483
|
+
reject(refused);
|
|
4484
|
+
return;
|
|
4485
|
+
}
|
|
4486
|
+
// Somebody else's refusal closed the channel under this send (d.723) —
|
|
4487
|
+
// never answered, therefore never refused, therefore republished.
|
|
4488
|
+
const collateral = this._collateralSendTransient(sendingChannel, 'exchange', exchange);
|
|
4489
|
+
if (collateral) {
|
|
4490
|
+
if (answeredAlready) {
|
|
4491
|
+
this._reportLateSendVerdict(collateral, 'exchange', exchange, queue, options);
|
|
4492
|
+
} else {
|
|
4493
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] ${collateral.message}`);
|
|
4494
|
+
}
|
|
4495
|
+
reject(collateral);
|
|
4496
|
+
return;
|
|
4497
|
+
}
|
|
4498
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during publish to exchange "${exchange}", classifying as transient`);
|
|
4499
|
+
reject(new PublishError(
|
|
4500
|
+
`[RabbitMQClient] Channel closed during publish to exchange "${exchange}" - message delivery not guaranteed. `
|
|
4501
|
+
+ 'Expected: the channel to stay open until the broker confirms. '
|
|
4502
|
+
+ 'Fix: republish the message — the confirm never arrived, so it may or may not have been routed; '
|
|
4503
|
+
+ 'read error.cause for what amqplib said.',
|
|
4504
|
+
exchange,
|
|
4505
|
+
err
|
|
4506
|
+
));
|
|
4507
|
+
return;
|
|
4508
|
+
}
|
|
3324
4509
|
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Exchange publish callback error:`, err.message);
|
|
3325
4510
|
reject(err);
|
|
3326
4511
|
} else {
|
|
@@ -3328,6 +4513,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3328
4513
|
resolve();
|
|
3329
4514
|
}
|
|
3330
4515
|
});
|
|
4516
|
+
// The frame is written to the channel: `publish()` returned without throwing.
|
|
4517
|
+
attempt.sent = true;
|
|
3331
4518
|
});
|
|
3332
4519
|
|
|
3333
4520
|
await confirmPromise;
|
|
@@ -3338,9 +4525,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3338
4525
|
// Pokud channel closed, vyhodíme TransientPublishError a necháme PublishLayer řešit retry
|
|
3339
4526
|
|
|
3340
4527
|
// Všechny chyby projdou jednotnou klasifikací do Transient/Permanent/QueueNotFound
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
4528
|
+
//
|
|
4529
|
+
// A VERDICT ABOUT A MESSAGE GOES ONE WAY: to whoever sent it. Until d.811b this line
|
|
4530
|
+
// also put the classified failure on the client's `error` channel, which is the
|
|
4531
|
+
// channel the rest of this client uses for the CONNECTION. Measured (d.811): it
|
|
4532
|
+
// fired for every classification, transient as well as permanent; nobody read it
|
|
4533
|
+
// (no `.on('error')` on this client anywhere in `shared/**` or `infra/**`, and zero
|
|
4534
|
+
// callers of `onError()` in production code); the concept never asked for it
|
|
4535
|
+
// (`docs/architecture/mq-publish-reliability.md` does not mention the channel); and
|
|
4536
|
+
// the only rail that ever met it had to learn to IGNORE it, because one concurrent
|
|
4537
|
+
// publish failing while the link came back rejected every reconnect waiter
|
|
4538
|
+
// (`_waitForReconnection()`, d.435/d.435b). It made work for another rail instead of
|
|
4539
|
+
// serving a reader.
|
|
4540
|
+
throw classifyPublishError(err, queue);
|
|
3344
4541
|
}
|
|
3345
4542
|
}
|
|
3346
4543
|
|
|
@@ -3359,17 +4556,22 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3359
4556
|
* A non-numeric prefetch (the caller asked for none) sets none, and records
|
|
3360
4557
|
* none: there is nothing to measure utilisation against.
|
|
3361
4558
|
*
|
|
4559
|
+
* @param {Object} consumerChannel - The consumer channel the caller holds; the
|
|
4560
|
+
* re-registration hands over the one it has just asked is alive (d.1138)
|
|
3362
4561
|
* @param {string} queue - Queue name
|
|
3363
4562
|
* @param {number|null|undefined} prefetch - The consumer's prefetch
|
|
3364
4563
|
* @returns {Promise<void>}
|
|
3365
4564
|
* @private
|
|
3366
4565
|
*/
|
|
3367
|
-
async _applyConsumerPrefetch(queue, prefetch) {
|
|
4566
|
+
async _applyConsumerPrefetch(consumerChannel, queue, prefetch) {
|
|
3368
4567
|
if (typeof prefetch !== 'number') {
|
|
3369
4568
|
return;
|
|
3370
4569
|
}
|
|
3371
4570
|
|
|
3372
|
-
await this.
|
|
4571
|
+
await this._askBroker(
|
|
4572
|
+
{ operation: 'prefetch', target: queue, channel: consumerChannel },
|
|
4573
|
+
() => consumerChannel.prefetch(prefetch)
|
|
4574
|
+
);
|
|
3373
4575
|
|
|
3374
4576
|
const tracking = this._prefetchTracking.get(queue);
|
|
3375
4577
|
if (tracking) {
|
|
@@ -3378,7 +4580,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3378
4580
|
this._prefetchTracking.set(queue, {
|
|
3379
4581
|
prefetchCount: prefetch,
|
|
3380
4582
|
inFlight: 0,
|
|
3381
|
-
lastCheck: Date.now()
|
|
4583
|
+
lastCheck: Date.now(),
|
|
4584
|
+
// Which side of the threshold this queue was on when the verdict was last
|
|
4585
|
+
// taken. It is what makes the alarm an EPISODE rather than a line per tick
|
|
4586
|
+
// — see `_checkPrefetchUtilization()`.
|
|
4587
|
+
utilizationHigh: false
|
|
3382
4588
|
});
|
|
3383
4589
|
}
|
|
3384
4590
|
|
|
@@ -3496,7 +4702,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3496
4702
|
//
|
|
3497
4703
|
// The ONE queue this demand cannot be made of is the one at the END of a
|
|
3498
4704
|
// chain: a queue something else dead-letters TO. `workflow.failed` is named
|
|
3499
|
-
// in `x-dead-letter-routing-key` by `workflow.init
|
|
4705
|
+
// in `x-dead-letter-routing-key` by `workflow.init`,
|
|
3500
4706
|
// `<service>.dlq` by every business template — and the confirmed topology
|
|
3501
4707
|
// leaves exactly those without a route of their own, because a dead-letter
|
|
3502
4708
|
// route on a dead-letter queue is a loop or a chain with no end
|
|
@@ -3563,10 +4769,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3563
4769
|
// If missing, fail-fast. The owning infra service must recreate on startup.
|
|
3564
4770
|
const queueChannel = await this._requireQueueChannel('checkQueue', queue);
|
|
3565
4771
|
try {
|
|
3566
|
-
await
|
|
4772
|
+
await this._askBroker(
|
|
4773
|
+
{ operation: 'checkQueue', target: queue, channel: queueChannel },
|
|
4774
|
+
() => queueChannel.checkQueue(queue)
|
|
4775
|
+
);
|
|
3567
4776
|
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Infrastructure queue ${queue} exists (consumer will proceed)`);
|
|
3568
4777
|
} catch (checkErr) {
|
|
3569
|
-
if (checkErr.code ===
|
|
4778
|
+
if (checkErr.code === AMQP_NOT_FOUND) {
|
|
3570
4779
|
// Typed and coded since d.297: a bare `Error` lost the queue AND the
|
|
3571
4780
|
// class, so the layer above could not tell "this queue does not exist"
|
|
3572
4781
|
// from "the policy refused this queue" and told the reader the wrong
|
|
@@ -3582,8 +4791,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3582
4791
|
}
|
|
3583
4792
|
} else {
|
|
3584
4793
|
// Business queue (or unknown) — declared on the ONE public rail, the same
|
|
3585
|
-
// `assertQueue()`
|
|
3586
|
-
// `RecoveryWorker.createQueue()` (d.277b) go through. It ensures the queue
|
|
4794
|
+
// `assertQueue()` every collaborator that declares a queue goes through. It ensures the queue
|
|
3587
4795
|
// channel, refuses one that is not alive (d.282) and resolves the arguments
|
|
3588
4796
|
// from `queueConfig.declarationOptions()` — so a declaration made here and a
|
|
3589
4797
|
// declaration made anywhere else are the same declaration by construction,
|
|
@@ -3649,56 +4857,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3649
4857
|
// `false` = the budget is one attempt, so the first failure rejects into
|
|
3650
4858
|
// `<svc>.dlq` — exactly what a `permanent` classification already does.
|
|
3651
4859
|
// `true` or absent = the budget configuration decides. There is no third
|
|
3652
|
-
// meaning, and the combination that would need one is refused
|
|
4860
|
+
// meaning, and the combination that would need one is refused rather
|
|
3653
4861
|
// than resolved by silently preferring one of the two.
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
);
|
|
3661
|
-
}
|
|
3662
|
-
|
|
3663
|
-
const noRequeue = options.requeueOnError === false;
|
|
3664
|
-
|
|
3665
|
-
if (noRequeue && options.maxAttempts !== undefined && options.maxAttempts !== 1) {
|
|
3666
|
-
throw new ValidationError(
|
|
3667
|
-
'[RabbitMQClient] Contradictory delivery policy - requeueOnError: false means one attempt, '
|
|
3668
|
-
+ `but maxAttempts says ${JSON.stringify(options.maxAttempts)}. Expected: one statement about `
|
|
3669
|
-
+ 'how many times a handler may run. Fix: drop requeueOnError to keep the budget, or drop '
|
|
3670
|
-
+ 'maxAttempts to dead-letter on the first failure.'
|
|
3671
|
-
);
|
|
3672
|
-
}
|
|
3673
|
-
|
|
3674
|
-
const maxAttempts = noRequeue
|
|
3675
|
-
? 1
|
|
3676
|
-
: options.maxAttempts === undefined
|
|
3677
|
-
? runtimeCfg.get('maxDeliveryAttempts', this._config.maxDeliveryAttempts)
|
|
3678
|
-
: options.maxAttempts;
|
|
3679
|
-
|
|
3680
|
-
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
|
|
3681
|
-
throw new ValidationError(
|
|
3682
|
-
'[RabbitMQClient] Invalid delivery budget - maxAttempts must be an integer >= 1, got '
|
|
3683
|
-
+ `${JSON.stringify(maxAttempts)}. Fix: pass consume(queue, handler, { maxAttempts: <integer >= 1> }), `
|
|
3684
|
-
+ 'or leave it out and let the module default (defaults.js maxDeliveryAttempts, '
|
|
3685
|
-
+ 'env RABBITMQ_MAX_DELIVERY_ATTEMPTS) decide.'
|
|
3686
|
-
);
|
|
3687
|
-
}
|
|
3688
|
-
|
|
3689
|
-
const classify = options.classify === undefined
|
|
3690
|
-
? deliveryPolicy.defaultClassification
|
|
3691
|
-
: options.classify;
|
|
3692
|
-
|
|
3693
|
-
if (typeof classify !== 'function') {
|
|
3694
|
-
throw new ValidationError(
|
|
3695
|
-
'[RabbitMQClient] Invalid error classifier - classify must be a function returning '
|
|
3696
|
-
+ `'transient' or 'permanent', got ${typeof options.classify}. Fix: pass consume(queue, handler, `
|
|
3697
|
-
+ '{ classify: (error) => … }), or leave it out and every error is treated as transient.'
|
|
3698
|
-
);
|
|
3699
|
-
}
|
|
3700
|
-
|
|
3701
|
-
const policy = { maxAttempts, classify };
|
|
4862
|
+
// The rules have ONE owner, exported for every consumer's test double
|
|
4863
|
+
// (d.1005): `deliveryPolicy.validateConsumeOptions`.
|
|
4864
|
+
const policy = deliveryPolicy.validateConsumeOptions(
|
|
4865
|
+
options,
|
|
4866
|
+
() => runtimeCfg.get('maxDeliveryAttempts', this._config.maxDeliveryAttempts)
|
|
4867
|
+
);
|
|
3702
4868
|
const onDeadLetter = options.onDeadLetter;
|
|
3703
4869
|
if (onDeadLetter !== undefined && typeof onDeadLetter !== 'function') {
|
|
3704
4870
|
throw new ValidationError(
|
|
@@ -3731,7 +4897,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3731
4897
|
// Set prefetch if provided (on consumer channel). ONE rail, shared with
|
|
3732
4898
|
// `_reRegisterConsumers()` — prefetch belongs to the consumer, not to the
|
|
3733
4899
|
// channel that happened to be open when it first attached.
|
|
3734
|
-
await this._applyConsumerPrefetch(queue, prefetch);
|
|
4900
|
+
await this._applyConsumerPrefetch(this._consumerChannel, queue, prefetch);
|
|
3735
4901
|
|
|
3736
4902
|
// CRITICAL: Log before calling amqplib's consume()
|
|
3737
4903
|
// amqplib's consume() may internally call assertQueue() without parameters if queue doesn't exist
|
|
@@ -3786,128 +4952,170 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3786
4952
|
}
|
|
3787
4953
|
});
|
|
3788
4954
|
|
|
3789
|
-
//
|
|
4955
|
+
// ONE delivery raises this counter once and lowers it once. The
|
|
4956
|
+
// `finally` at the end of this callback is that second half, and since
|
|
4957
|
+
// d.680 it is the ONLY place the counter goes down
|
|
4958
|
+
// (`change-discipline.md` § One rail per concern).
|
|
4959
|
+
//
|
|
4960
|
+
// MEASURED on production (d.680, 2026-09-20, the dispatcher):
|
|
4961
|
+
//
|
|
4962
|
+
// Prefetch utilization high: queue 'workflow.completed' has 43/5
|
|
4963
|
+
// messages in-flight (860%)
|
|
4964
|
+
//
|
|
4965
|
+
// — and the number grew with every workflow, never falling. Until d.680
|
|
4966
|
+
// the decrement lived in the branches that settle a message HERE: the
|
|
4967
|
+
// auto-ack, the delivery policy, and the rejection of an unreadable
|
|
4968
|
+
// counter. Every OTHER way a delivery can end left the increment
|
|
4969
|
+
// standing — a handler that settles the message itself (the public
|
|
4970
|
+
// `ack()`/`nack()` set `msg._mqProcessed`, which is exactly what makes
|
|
4971
|
+
// the auto-ack branch stand down), a `noAck` consumer, and the two early
|
|
4972
|
+
// returns taken when the channel died under a running handler.
|
|
4973
|
+
//
|
|
4974
|
+
// Nothing stalled, because the window is enforced by the BROKER and not
|
|
4975
|
+
// by this number. The only casualty was the alarm, which from then on
|
|
4976
|
+
// fired for every delivery whatever the load — the warning that cannot
|
|
4977
|
+
// fail to happen that d.550 removed for prefetch 1, and that
|
|
4978
|
+
// `automation-gates.md` §5 counts as a defect of its own.
|
|
4979
|
+
//
|
|
4980
|
+
// The counter is therefore bound to the DELIVERY, not to the branch that
|
|
4981
|
+
// settles it: it measures what this consumer is holding, and it stops
|
|
4982
|
+
// holding a delivery when this callback returns — whoever settled it,
|
|
4983
|
+
// and however many times.
|
|
3790
4984
|
const tracking = this._prefetchTracking.get(queue);
|
|
3791
4985
|
if (tracking) {
|
|
3792
4986
|
tracking.inFlight++;
|
|
3793
4987
|
tracking.lastCheck = Date.now();
|
|
3794
4988
|
this._checkPrefetchUtilization(queue, tracking);
|
|
3795
4989
|
}
|
|
3796
|
-
|
|
3797
|
-
// Which attempt this delivery is, read from the header this library
|
|
3798
|
-
// writes (`../config/deliveryPolicy.js`). `null` = the header carries a
|
|
3799
|
-
// value this library never wrote, so the message cannot be bounded and
|
|
3800
|
-
// the handler is NOT run for it; it is rejected below.
|
|
3801
|
-
const attemptsMade = deliveryPolicy.readAttemptsMade(msgHeaders);
|
|
3802
|
-
const attempt = attemptsMade === null ? null : attemptsMade + 1;
|
|
3803
|
-
|
|
3804
|
-
if (attempt === null) {
|
|
3805
|
-
this._logger.error(
|
|
3806
|
-
'[RabbitMQClient] Unreadable delivery counter - the '
|
|
3807
|
-
+ `${deliveryPolicy.ATTEMPTS_HEADER} header of a message on "${queue}" holds `
|
|
3808
|
-
+ `${JSON.stringify(msgHeaders[deliveryPolicy.ATTEMPTS_HEADER])}, which is not a `
|
|
3809
|
-
+ 'non-negative integer. Expected: only this library writes that header. '
|
|
3810
|
-
+ 'Fix: the message is rejected into the dead-letter queue unread — reading a '
|
|
3811
|
-
+ 'malformed counter as zero would make the redelivery loop unbounded again.',
|
|
3812
|
-
{ queue }
|
|
3813
|
-
);
|
|
3814
|
-
await this._rejectDelivery({
|
|
3815
|
-
msg,
|
|
3816
|
-
queue,
|
|
3817
|
-
channelForMsg,
|
|
3818
|
-
noAck,
|
|
3819
|
-
error: new Error(
|
|
3820
|
-
`[RabbitMQClient] Message on "${queue}" carries an unreadable `
|
|
3821
|
-
+ `${deliveryPolicy.ATTEMPTS_HEADER} header`
|
|
3822
|
-
),
|
|
3823
|
-
classification: deliveryPolicy.UNCLASSIFIABLE,
|
|
3824
|
-
attempts: null,
|
|
3825
|
-
maxAttempts: policy.maxAttempts,
|
|
3826
|
-
onDeadLetter
|
|
3827
|
-
});
|
|
3828
|
-
if (tracking) {
|
|
3829
|
-
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
3830
|
-
tracking.lastCheck = Date.now();
|
|
3831
|
-
}
|
|
3832
|
-
return;
|
|
3833
|
-
}
|
|
3834
4990
|
|
|
3835
4991
|
try {
|
|
3836
|
-
//
|
|
3837
|
-
//
|
|
3838
|
-
//
|
|
3839
|
-
//
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
4992
|
+
// Which attempt this delivery is, read from the header this library
|
|
4993
|
+
// writes (`../config/deliveryPolicy.js`). `null` = the header carries a
|
|
4994
|
+
// value this library never wrote, so the message cannot be bounded and
|
|
4995
|
+
// the handler is NOT run for it; it is rejected below.
|
|
4996
|
+
const attemptsMade = deliveryPolicy.readAttemptsMade(msgHeaders);
|
|
4997
|
+
const attempt = attemptsMade === null ? null : attemptsMade + 1;
|
|
4998
|
+
|
|
4999
|
+
if (attempt === null) {
|
|
5000
|
+
this._logger.error(
|
|
5001
|
+
'[RabbitMQClient] Unreadable delivery counter - the '
|
|
5002
|
+
+ `${deliveryPolicy.ATTEMPTS_HEADER} header of a message on "${queue}" holds `
|
|
5003
|
+
+ `${JSON.stringify(msgHeaders[deliveryPolicy.ATTEMPTS_HEADER])}, which is not a `
|
|
5004
|
+
+ 'non-negative integer. Expected: only this library writes that header. '
|
|
5005
|
+
+ 'Fix: the message is rejected into the dead-letter queue unread — reading a '
|
|
5006
|
+
+ 'malformed counter as zero would make the redelivery loop unbounded again.',
|
|
5007
|
+
{ queue }
|
|
5008
|
+
);
|
|
5009
|
+
await this._rejectDelivery({
|
|
5010
|
+
msg,
|
|
5011
|
+
queue,
|
|
5012
|
+
channelForMsg,
|
|
5013
|
+
noAck,
|
|
5014
|
+
error: new Error(
|
|
5015
|
+
`[RabbitMQClient] Message on "${queue}" carries an unreadable `
|
|
5016
|
+
+ `${deliveryPolicy.ATTEMPTS_HEADER} header`
|
|
5017
|
+
),
|
|
5018
|
+
classification: deliveryPolicy.UNCLASSIFIABLE,
|
|
5019
|
+
attempts: null,
|
|
5020
|
+
maxAttempts: policy.maxAttempts,
|
|
5021
|
+
onDeadLetter
|
|
5022
|
+
});
|
|
5023
|
+
return;
|
|
5024
|
+
}
|
|
5025
|
+
|
|
5026
|
+
try {
|
|
5027
|
+
// The handler is told WHICH attempt it is on, so a side effect that
|
|
5028
|
+
// must happen once — the RPC error reply above all — happens on the
|
|
5029
|
+
// last attempt only, instead of once per redelivery. A handler that
|
|
5030
|
+
// takes one argument ignores the second.
|
|
5031
|
+
await onMessage(msg, {
|
|
5032
|
+
attempt,
|
|
5033
|
+
maxAttempts: policy.maxAttempts,
|
|
5034
|
+
isFinalAttempt: attempt >= policy.maxAttempts
|
|
5035
|
+
});
|
|
5036
|
+
// Acknowledge message after successful processing
|
|
5037
|
+
if (!noAck && !msg._mqProcessed) {
|
|
5038
|
+
// Liveness comes from the ONE source the channel handlers set
|
|
5039
|
+
// first-hand (`_isChannelAlive()`, d.260). The reading this
|
|
5040
|
+
// replaced, `channelForMsg.closed`, asked amqplib for a property it
|
|
5041
|
+
// does not define, so it answered "alive" for the channel the
|
|
5042
|
+
// broker had just killed and handed the corpse an `ack`.
|
|
5043
|
+
if (!this._isChannelAlive(channelForMsg)) {
|
|
5044
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] [CONSUMER] Cannot ack - consumer channel is closed/recreated (message will be requeued by broker)');
|
|
3864
5045
|
return;
|
|
3865
5046
|
}
|
|
3866
|
-
|
|
5047
|
+
try {
|
|
5048
|
+
channelForMsg.ack(msg);
|
|
5049
|
+
msg._mqProcessed = true;
|
|
5050
|
+
} catch (ackErr) {
|
|
5051
|
+
// A REFUSED CALL IS A CLASS, NOT A SENTENCE (d.698). amqplib replaces a
|
|
5052
|
+
// dead channel's send methods with one that throws its own
|
|
5053
|
+
// `IllegalOperationError` (`invalidateSend()`/`invalidOp()`,
|
|
5054
|
+
// `lib/channel.js`), so that class IS "the channel would not take
|
|
5055
|
+
// this ack" — and it is the only thing `ack()` ever throws.
|
|
5056
|
+
//
|
|
5057
|
+
// The sentence branch this replaced listed `'unknown delivery tag'`,
|
|
5058
|
+
// `'PRECONDITION_FAILED'` and `'Channel closed'`. The first two cannot
|
|
5059
|
+
// arrive: an ack with a tag the broker does not hold does not throw at all
|
|
5060
|
+
// — the broker answers by closing the channel with a 406 frame on its
|
|
5061
|
+
// `error` event (measured live, d.698). The third is only ONE of amqplib's
|
|
5062
|
+
// two wordings, so the other, `'Channel closing'` (`toClosing()`, `:167`),
|
|
5063
|
+
// was rethrown — into the handler's own catch, which republished a RETRY
|
|
5064
|
+
// COPY of a message whose handler had already succeeded.
|
|
5065
|
+
if (ackErr instanceof IllegalOperationError) {
|
|
5066
|
+
msg._mqProcessed = true;
|
|
5067
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] [CONSUMER] Ack refused by a closed/closing channel (message delivery will be handled by broker)', { error: ackErr.message });
|
|
5068
|
+
return;
|
|
5069
|
+
}
|
|
5070
|
+
throw ackErr;
|
|
5071
|
+
}
|
|
5072
|
+
this._logger.debug('[RabbitMQClient] MSG_ACKED', {
|
|
5073
|
+
workflow_id: wfId,
|
|
5074
|
+
action: 'MSG_ACKED',
|
|
5075
|
+
handler: 'consume',
|
|
5076
|
+
function: 'RabbitMQClient.consume',
|
|
5077
|
+
input: { queue },
|
|
5078
|
+
output: { status: 'acknowledged', queue }
|
|
5079
|
+
});
|
|
3867
5080
|
}
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
5081
|
+
} catch (handlerErr) {
|
|
5082
|
+
// The dead-letter policy decides what a failed handler costs: one more
|
|
5083
|
+
// attempt (a copy carrying the advanced counter, then an ack of the
|
|
5084
|
+
// original) or a rejection into `<svc>.dlq`. Until 2026-09-11 this
|
|
5085
|
+
// branch was an unconditional `nack(msg, false, true)` with no counter,
|
|
5086
|
+
// so a permanently failing message was redelivered for ever and the DLQ
|
|
5087
|
+
// the topology declares was never reached from the client side
|
|
5088
|
+
// (`docs/governance/confirmations/mq-consumer-contract.md` 002 point 3).
|
|
5089
|
+
await this._applyDeliveryPolicy({
|
|
5090
|
+
msg,
|
|
5091
|
+
queue,
|
|
5092
|
+
channelForMsg,
|
|
5093
|
+
noAck,
|
|
5094
|
+
error: handlerErr,
|
|
5095
|
+
attempt,
|
|
5096
|
+
policy,
|
|
5097
|
+
onDeadLetter
|
|
3875
5098
|
});
|
|
3876
|
-
|
|
3877
|
-
if (tracking) {
|
|
3878
|
-
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
3879
|
-
tracking.lastCheck = Date.now();
|
|
3880
|
-
}
|
|
3881
5099
|
}
|
|
3882
|
-
}
|
|
3883
|
-
// The dead-letter policy decides what a failed handler costs: one more
|
|
3884
|
-
// attempt (a copy carrying the advanced counter, then an ack of the
|
|
3885
|
-
// original) or a rejection into `<svc>.dlq`. Until 2026-09-11 this
|
|
3886
|
-
// branch was an unconditional `nack(msg, false, true)` with no counter,
|
|
3887
|
-
// so a permanently failing message was redelivered for ever and the DLQ
|
|
3888
|
-
// the topology declares was never reached from the client side
|
|
3889
|
-
// (`docs/governance/confirmations/mq-consumer-contract.md` 002 point 3).
|
|
3890
|
-
await this._applyDeliveryPolicy({
|
|
3891
|
-
msg,
|
|
3892
|
-
queue,
|
|
3893
|
-
channelForMsg,
|
|
3894
|
-
noAck,
|
|
3895
|
-
error: handlerErr,
|
|
3896
|
-
attempt,
|
|
3897
|
-
policy,
|
|
3898
|
-
onDeadLetter
|
|
3899
|
-
});
|
|
5100
|
+
} finally {
|
|
3900
5101
|
if (tracking) {
|
|
3901
5102
|
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
3902
5103
|
tracking.lastCheck = Date.now();
|
|
5104
|
+
// The verdict is taken on BOTH edges (d.736a). Until this call the
|
|
5105
|
+
// window was measured only while it FILLED, so "the queue came back
|
|
5106
|
+
// under the threshold" was a fact nothing ever stated — measured in
|
|
5107
|
+
// `api_delivery_endpoint`'s log on 2026-09-18: three warnings about
|
|
5108
|
+
// one standing condition in 68 ms, and no line of any kind about its
|
|
5109
|
+
// end. See `_checkPrefetchUtilization()`.
|
|
5110
|
+
this._checkPrefetchUtilization(queue, tracking);
|
|
3903
5111
|
}
|
|
3904
5112
|
}
|
|
3905
5113
|
};
|
|
3906
5114
|
|
|
3907
|
-
const
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
{ noAck }
|
|
5115
|
+
const consumerChannel = this._consumerChannel;
|
|
5116
|
+
const consumeResult = await this._askBroker(
|
|
5117
|
+
{ operation: 'consume', target: queue, channel: consumerChannel },
|
|
5118
|
+
() => consumerChannel.consume(queue, messageHandler, { noAck })
|
|
3911
5119
|
);
|
|
3912
5120
|
|
|
3913
5121
|
// Track consumer for auto re-registration if channel closes
|
|
@@ -4016,13 +5224,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4016
5224
|
this._logger.info(
|
|
4017
5225
|
'[RabbitMQClient] Client was closed by its caller during recovery - closing the re-established connection'
|
|
4018
5226
|
);
|
|
4019
|
-
|
|
4020
|
-
|
|
4021
|
-
|
|
5227
|
+
// Within the one budget (d.1138, conf `mq-broker-answer-budget` 001): this close
|
|
5228
|
+
// waits for a `close-ok` like every teardown close, and a broker that stopped
|
|
5229
|
+
// answering right now would otherwise hold this round — and `_reconnecting` —
|
|
5230
|
+
// open forever. Unanswered, the socket is dropped the way the teardown drops it.
|
|
5231
|
+
const closing = await this._closeWithinTeardownBudget(
|
|
5232
|
+
'connection re-established after disconnect',
|
|
5233
|
+
revived,
|
|
5234
|
+
this._brokerAnswerTimeout
|
|
5235
|
+
);
|
|
5236
|
+
if (closing.status === 'refused') {
|
|
4022
5237
|
this._logger.warn(
|
|
4023
|
-
`[RabbitMQClient] Error closing the connection re-established after disconnect: ${
|
|
5238
|
+
`[RabbitMQClient] Error closing the connection re-established after disconnect: ${closing.error.message}`
|
|
4024
5239
|
);
|
|
4025
5240
|
}
|
|
5241
|
+
if (closing.status === 'unanswered') {
|
|
5242
|
+
this._dropUnansweredConnection(revived);
|
|
5243
|
+
}
|
|
4026
5244
|
this._connection = null;
|
|
4027
5245
|
this._connectionAlive = false;
|
|
4028
5246
|
this._reconnecting = false;
|
|
@@ -4069,7 +5287,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4069
5287
|
|
|
4070
5288
|
// Recreate consumer channel and re-register all consumers
|
|
4071
5289
|
try {
|
|
4072
|
-
this._consumerChannel = await this.
|
|
5290
|
+
this._consumerChannel = await this._askBroker(
|
|
5291
|
+
{ operation: 'createChannel', target: 'consumer channel', channel: null },
|
|
5292
|
+
() => this._connection.createChannel()
|
|
5293
|
+
);
|
|
4073
5294
|
this._consumerChannel._createdAt = new Date().toISOString();
|
|
4074
5295
|
this._consumerChannel._closeReason = null;
|
|
4075
5296
|
this._consumerChannel._lastOperation = 'Recreated via _reconnectWithBackoff()';
|
|
@@ -4079,7 +5300,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4079
5300
|
// Re-register all consumers - use internal helper to avoid the "channel already open" check
|
|
4080
5301
|
consumerReRegResult = await this._reRegisterConsumers();
|
|
4081
5302
|
channelState.consumer = true;
|
|
4082
|
-
|
|
5303
|
+
// The same summary rule as `_ensureConsumerChannel()` (d.1003d, d.1003e): a
|
|
5304
|
+
// recovery that left a consumer unattached is not a ✓ line.
|
|
5305
|
+
if (consumerReRegResult.failed > 0) {
|
|
5306
|
+
const unattached = this._unattachedConsumerQueues();
|
|
5307
|
+
this._logger.warn(
|
|
5308
|
+
`[RabbitMQClient] Consumer channel recreated, but ${unattached.length} consumer(s) are not attached: `
|
|
5309
|
+
+ `${unattached.join(', ')} (re-registered: ${consumerReRegResult.reRegistered}, failed: ${consumerReRegResult.failed})`
|
|
5310
|
+
);
|
|
5311
|
+
} else {
|
|
5312
|
+
this._logger.info(
|
|
5313
|
+
`[RabbitMQClient] ✓ Consumer channel recreated (re-registered: ${consumerReRegResult.reRegistered}, failed: ${consumerReRegResult.failed})`
|
|
5314
|
+
);
|
|
5315
|
+
}
|
|
4083
5316
|
} catch (err) {
|
|
4084
5317
|
this._logger.error('[RabbitMQClient] Failed to recreate consumer channel:', err.message);
|
|
4085
5318
|
this._consumerChannel = null;
|
|
@@ -4108,7 +5341,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4108
5341
|
// Flush buffered messages po reconnectu
|
|
4109
5342
|
let bufferFlushResult = { inMemory: 0, flushed: false };
|
|
4110
5343
|
try {
|
|
4111
|
-
const flushResult = await this._publishLayer.flushBuffered();
|
|
5344
|
+
const flushResult = await this._publishLayer.flushBuffered('reconnected');
|
|
4112
5345
|
bufferFlushResult = { inMemory: flushResult, flushed: true };
|
|
4113
5346
|
this._publishMonitor.trackFlushed(bufferFlushResult.inMemory);
|
|
4114
5347
|
this._logger.info(`[RabbitMQClient] ✓ Flushed ${bufferFlushResult.inMemory} buffered messages after reconnection`);
|
|
@@ -4187,7 +5420,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4187
5420
|
* Attach event handlers to publisher channel
|
|
4188
5421
|
* @private
|
|
4189
5422
|
*/
|
|
4190
|
-
_attachPublisherChannelHandlers(channel) {
|
|
5423
|
+
_attachPublisherChannelHandlers(channel, role) {
|
|
4191
5424
|
// One closure, one notification. amqplib announces a broker-side channel
|
|
4192
5425
|
// death TWICE — `error` first, then `close` (measured against amqplib 0.10.9
|
|
4193
5426
|
// on the live broker, 2026-09-12: a `checkQueue()` on a missing queue yields
|
|
@@ -4200,6 +5433,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4200
5433
|
// without a preceding `error`. The `error` handler only records WHY the
|
|
4201
5434
|
// channel is dying, here, for the one notification to carry.
|
|
4202
5435
|
channel._closeError = null;
|
|
5436
|
+
// The publishes this channel is still carrying, and the snapshot of that number taken
|
|
5437
|
+
// when a close frame arrives — `_closeFrameNamesThisSend()` is the one reader of both
|
|
5438
|
+
// (d.723). Declared here because this is the one place a PUBLISHER channel is
|
|
5439
|
+
// prepared, and a member nobody declared would make that reading answer for
|
|
5440
|
+
// `undefined` instead of for a number.
|
|
5441
|
+
channel._unconfirmedSends = 0;
|
|
5442
|
+
channel._unconfirmedAtCloseFrame = null;
|
|
4203
5443
|
channel._alive = true;
|
|
4204
5444
|
channel.on('error', async (err) => {
|
|
4205
5445
|
channel._alive = false;
|
|
@@ -4208,24 +5448,63 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4208
5448
|
}
|
|
4209
5449
|
const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
4210
5450
|
channel._closeReason = reason;
|
|
5451
|
+
// How many publishes this frame could possibly be an answer about (d.723). Taken
|
|
5452
|
+
// HERE and nowhere else: amqplib emits `error` before `close`, and it is `close`
|
|
5453
|
+
// that rejects the unconfirmed publishes — by the time their callbacks ask, the
|
|
5454
|
+
// count has already been unwound. `_closeFrameNamesThisSend()` reads it.
|
|
5455
|
+
channel._unconfirmedAtCloseFrame = channel._unconfirmedSends;
|
|
5456
|
+
// WHY the channel is dying is recorded FIRST, for every reason there is.
|
|
5457
|
+
//
|
|
5458
|
+
// Since d.685a the publish path answers a refused send from this member
|
|
5459
|
+
// (`_refusedSendPermanent()`), so a branch that returns before writing it leaves the
|
|
5460
|
+
// publish path blind: the send reads a channel that died for no stated reason and
|
|
5461
|
+
// calls it transient — retried `publishMaxRetries` times and then buffered, for a
|
|
5462
|
+
// message the broker refused. The `'unknown delivery tag'` branch below did exactly
|
|
5463
|
+
// that until d.696, and the frame it catches is a NUMBERED refusal like any other
|
|
5464
|
+
// (measured live, d.693: `Error` with `code: 406`, arriving on this very event).
|
|
5465
|
+
// Recording first and branching afterwards changes what the branch DOES in nothing.
|
|
5466
|
+
channel._closeError = err;
|
|
4211
5467
|
|
|
4212
|
-
//
|
|
4213
|
-
//
|
|
4214
|
-
//
|
|
4215
|
-
|
|
4216
|
-
|
|
5468
|
+
// A 406 ON THE PUBLISHER CHANNEL IS THE PUBLISH PATH'S BUSINESS, and the CODE is
|
|
5469
|
+
// what says so (d.698).
|
|
5470
|
+
//
|
|
5471
|
+
// The publisher channel is a ConfirmChannel; this client sends no `basic.ack` on it,
|
|
5472
|
+
// so the sentence this branch read until d.698 — `'unknown delivery tag'` — never
|
|
5473
|
+
// reached it. What does reach it is the broker refusing the SEND with 406: a
|
|
5474
|
+
// `user_id` property that is not the authenticated user (measured d.685a, `classId
|
|
5475
|
+
// 60 / methodId 40`). That refusal is already answered where it belongs — the send
|
|
5476
|
+
// reads this very `_closeError` and rejects the caller with a `PermanentPublishError`
|
|
5477
|
+
// (`_refusedSendPermanent()`) — so raising it a second time as a client fault would
|
|
5478
|
+
// report one event twice, once to the publisher and once to everybody.
|
|
5479
|
+
//
|
|
5480
|
+
// Only 406. A `404 NOT-FOUND` says nothing declares the exchange, which no publisher
|
|
5481
|
+
// can fix and no caller of THIS message is the right audience for, and a codeless
|
|
5482
|
+
// death is not an answer about any request at all: both stay raised, below.
|
|
5483
|
+
if (err.code === AMQP_PRECONDITION_FAILED) {
|
|
5484
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Publisher channel refused the send with ${AMQP_PRECONDITION_FAILED} (answered to the publisher): ${err.message}`);
|
|
4217
5485
|
// Don't try to recreate - channel is already closing
|
|
4218
|
-
// Don't emit error - publish
|
|
5486
|
+
// Don't emit error - the publish path answers the caller from `_closeError`
|
|
4219
5487
|
return;
|
|
4220
5488
|
}
|
|
4221
|
-
|
|
5489
|
+
|
|
4222
5490
|
this._logger.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
4223
|
-
channel._closeError = err;
|
|
4224
5491
|
// Recovery is NOT started here — one rail, the `close` handler (d.261).
|
|
4225
5492
|
this.emit('error', err);
|
|
4226
5493
|
});
|
|
4227
5494
|
|
|
4228
|
-
|
|
5495
|
+
// PREPENDED, not appended (d.1134). amqplib registers its own `close` listener in the
|
|
5496
|
+
// channel's constructor — the one that answers every unconfirmed publish with the bare
|
|
5497
|
+
// `new Error('channel closed')` (`lib/channel.js`, the `Channel` constructor) — and a
|
|
5498
|
+
// constructor runs before anybody can attach anything, so an appended handler always
|
|
5499
|
+
// ran SECOND. The publish callbacks then read a channel that was still alive, took the
|
|
5500
|
+
// branch meant for a broker answer on a live channel, and the sentinel reached the
|
|
5501
|
+
// classification untyped: permanent, not buffered, not retried — for a message the
|
|
5502
|
+
// broker said nothing about (měřeno d.1126, `320 CONNECTION_FORCED` with a confirm in
|
|
5503
|
+
// flight). `connection.toClosed()` closes every channel before the connection's own
|
|
5504
|
+
// events (`lib/connection.js`), so this handler is the only one that can mark the
|
|
5505
|
+
// channel dead in time. Prepending makes it run before amqplib's own listener — an
|
|
5506
|
+
// order the emitter guarantees, with no timer and no race.
|
|
5507
|
+
channel.prependListener('close', async () => {
|
|
4229
5508
|
if (this._closedByCaller) {
|
|
4230
5509
|
return;
|
|
4231
5510
|
}
|
|
@@ -4242,15 +5521,37 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4242
5521
|
if (this._channel === channel) {
|
|
4243
5522
|
this._channel = null;
|
|
4244
5523
|
}
|
|
5524
|
+
// The same obituary for the other send role (d.724). Without this line a dead
|
|
5525
|
+
// telemetry channel would stay in its member and every later telemetry publish
|
|
5526
|
+
// would be handed a corpse — `_ensurePublisherChannel('telemetry')` recreates only
|
|
5527
|
+
// what it finds cleared.
|
|
5528
|
+
if (this._telemetryChannel === channel) {
|
|
5529
|
+
this._telemetryChannel = null;
|
|
5530
|
+
}
|
|
4245
5531
|
|
|
4246
5532
|
// Only try to recreate if connection is available
|
|
4247
5533
|
// If connection is closed, reconnection logic will handle channel recreation
|
|
5534
|
+
//
|
|
5535
|
+
// Obnovuje se ROLE, která zemřela (d.1003b): tenhle handler visí na obou send
|
|
5536
|
+
// kanálech a do d.1003b volal `_ensurePublisherChannel()` bez role — po smrti
|
|
5537
|
+
// telemetry kanálu tedy „obnovil" živý publisher (no-op) a telemetry zůstal
|
|
5538
|
+
// prázdný až do příští události.
|
|
4248
5539
|
if (this._connectionAlive) {
|
|
4249
5540
|
try {
|
|
4250
|
-
await this._ensurePublisherChannel();
|
|
5541
|
+
await this._ensurePublisherChannel(role);
|
|
4251
5542
|
} catch (err) {
|
|
4252
|
-
//
|
|
4253
|
-
|
|
5543
|
+
// Přechodné selhání obnovy (spojení odchází — d.1000/d.1003) je jen `warn`:
|
|
5544
|
+
// kanál zůstane prázdný a příští publish ho po reconnectu obnoví sám
|
|
5545
|
+
// (`_ensurePublisherChannel()`). Cokoli jiného je nehlášené selhání, a to
|
|
5546
|
+
// má v tomhle klientovi kolej `error` (viz `_reportRecoveryFailure()`) —
|
|
5547
|
+
// do d.1003 ho tenhle catch polykal jako `warn`, takže se o trvale
|
|
5548
|
+
// nepoužitelném kanálu dozvěděl až první, kdo zkusil publikovat.
|
|
5549
|
+
if (err instanceof TransientPublishError) {
|
|
5550
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate ${role} channel (connection may be closing): ${err.message}`);
|
|
5551
|
+
} else {
|
|
5552
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Failed to recreate ${role} channel after it closed: ${err.message}`);
|
|
5553
|
+
this.emit('error', err);
|
|
5554
|
+
}
|
|
4254
5555
|
}
|
|
4255
5556
|
}
|
|
4256
5557
|
});
|
|
@@ -4305,12 +5606,27 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4305
5606
|
try {
|
|
4306
5607
|
await this._ensureQueueChannel();
|
|
4307
5608
|
} catch (err) {
|
|
4308
|
-
//
|
|
4309
|
-
|
|
5609
|
+
// The connection is going away — amqplib's terminal-state class, measured
|
|
5610
|
+
// live (d.693 step b): `createChannel()` on a closed connection answers
|
|
5611
|
+
// `IllegalOperationError('Connection closed (by client)')`. The class is the
|
|
5612
|
+
// fact; the sentence was only carrying it (d.696).
|
|
5613
|
+
//
|
|
5614
|
+
// It is read from `cause`, because the rail this failure travels wraps it:
|
|
5615
|
+
// `_ensureQueueChannel()` answers with a `ConnectionError` naming the fix and
|
|
5616
|
+
// keeps amqplib's object underneath. One level, and first-hand — that IS the
|
|
5617
|
+
// wrapper's own cause, not a chain walked in hope. The rule itself has one
|
|
5618
|
+
// body, `_connectionIsGoingAway()` (d.1003c), shared with the send and the
|
|
5619
|
+
// consumer channel.
|
|
5620
|
+
//
|
|
5621
|
+
// Anything else is an unannounced failure and goes to `error` (d.1003c) —
|
|
5622
|
+
// until then it was a `warn`, like the transient case. Recovery is NOT added
|
|
5623
|
+
// here: the channel stays cleared and the next queue operation recreates it
|
|
5624
|
+
// (`_requireQueueChannel()`), exactly as before.
|
|
5625
|
+
if (this._connectionIsGoingAway(err.cause)) {
|
|
4310
5626
|
this._logger.warn(`[RabbitMQClient] [mq-client-core] Queue channel close during connection closure (expected - reconnection will handle)`);
|
|
4311
5627
|
} else {
|
|
4312
|
-
|
|
4313
|
-
this.
|
|
5628
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel after it closed: ${err.message} (the next queue operation tries again)`);
|
|
5629
|
+
this.emit('error', err);
|
|
4314
5630
|
}
|
|
4315
5631
|
}
|
|
4316
5632
|
} else {
|
|
@@ -4365,8 +5681,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4365
5681
|
try {
|
|
4366
5682
|
await this._ensureConsumerChannel();
|
|
4367
5683
|
} catch (err) {
|
|
4368
|
-
//
|
|
4369
|
-
|
|
5684
|
+
// Táž kolej jako u send kanálu (d.1003b): selhání, které jen hlásí odcházející
|
|
5685
|
+
// spojení, je `warn` — reconnect kanál obnoví a konzumenty zaregistruje znovu.
|
|
5686
|
+
// Cokoli jiného je nehlášené selhání a jde na `error`; do d.1003b ho tenhle
|
|
5687
|
+
// catch polykal jako `warn`, takže konzumenti tiše přestali dostávat zprávy.
|
|
5688
|
+
// `_ensureConsumerChannel()` odpovídá `ConnectionError` s chybou amqplibu
|
|
5689
|
+
// v `cause` — jeden článek, vlastní příčina obalu (vzor obnovy frontového
|
|
5690
|
+
// kanálu níž, d.696).
|
|
5691
|
+
if (this._connectionIsGoingAway(err.cause)) {
|
|
5692
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel (connection may be closing): ${err.message}`);
|
|
5693
|
+
} else {
|
|
5694
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel after it closed: ${err.message}`);
|
|
5695
|
+
this.emit('error', err);
|
|
5696
|
+
}
|
|
4370
5697
|
}
|
|
4371
5698
|
}
|
|
4372
5699
|
});
|
|
@@ -4389,6 +5716,42 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4389
5716
|
* The threshold itself is unchanged and no new one is introduced: a window
|
|
4390
5717
|
* larger than one message is measured exactly as before.
|
|
4391
5718
|
*
|
|
5719
|
+
* THE VERDICT IS TAKEN ON THE TRANSITION, BOTH WAYS (d.736a). Until this batch
|
|
5720
|
+
* the check ran only while the window FILLED, and it spoke every time it found
|
|
5721
|
+
* the condition standing. MEASURED (`api_delivery_endpoint`, 2026-09-18,
|
|
5722
|
+
* one consumer):
|
|
5723
|
+
*
|
|
5724
|
+
* 08:00:10.928 WARN … queue 'delivery.health.events' has 8/10 … (80%)
|
|
5725
|
+
* 08:00:10.995 WARN … queue 'delivery.health.events' has 8/10 … (80%)
|
|
5726
|
+
* 08:00:10.996 WARN … queue 'delivery.health.events' has 9/10 … (90%)
|
|
5727
|
+
*
|
|
5728
|
+
* — three lines in 68 ms about ONE standing condition, and never a line saying
|
|
5729
|
+
* the queue came back under the threshold. The operator reading that log can
|
|
5730
|
+
* see an alarm start and can never see it end, which makes the line unusable
|
|
5731
|
+
* for the only question it exists to answer.
|
|
5732
|
+
*
|
|
5733
|
+
* The shape is the one the monitoring concept settled the same day: a rail that
|
|
5734
|
+
* says a standing condition ONCE — OPEN states it, everything after it is the
|
|
5735
|
+
* same episode, and the end is a RECORD rather than a second alarm
|
|
5736
|
+
* (`infra/api_monitoring`, `c996215c` telemetry-intake rejections, `1b1c3912`
|
|
5737
|
+
* registry health events; `watchers/alertLifecycle.js`
|
|
5738
|
+
* § closeRecoveredInfrastructureAlerts: "each episode's one message went out
|
|
5739
|
+
* when it opened"). This library keeps no episode store, so its record is the
|
|
5740
|
+
* log: the onset stays a `warn`, the return is one `info`.
|
|
5741
|
+
*
|
|
5742
|
+
* WHAT DOES NOT CHANGE: the callback and the `prefetch:high-utilization` event
|
|
5743
|
+
* still announce the onset, with the same payload — they simply fire once per
|
|
5744
|
+
* episode now. The return gets no callback and no event of its own: both would
|
|
5745
|
+
* be a declaration with no consumer anywhere in this workspace
|
|
5746
|
+
* (`change-discipline.md` § Removing something removes its declaration), while
|
|
5747
|
+
* the log line has a measured reader.
|
|
5748
|
+
*
|
|
5749
|
+
* NO HYSTERESIS is introduced. Monitoring damps a flapping subject with a floor
|
|
5750
|
+
* taken from configuration, and it needs one because a notification stands
|
|
5751
|
+
* behind each of its episodes; here the episode is a log line, and a floor
|
|
5752
|
+
* written into this file would be a number with no owner
|
|
5753
|
+
* (`architecture-principles.md` §2). The edge alone removes the repetition.
|
|
5754
|
+
*
|
|
4392
5755
|
* @private
|
|
4393
5756
|
* @param {string} queue - Queue name
|
|
4394
5757
|
* @param {Object} tracking - Prefetch tracking object
|
|
@@ -4399,38 +5762,65 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4399
5762
|
}
|
|
4400
5763
|
|
|
4401
5764
|
if (tracking.prefetchCount <= 1) {
|
|
4402
|
-
|
|
5765
|
+
// One-message window: 0 % or 100 %, and 100 % is the working state. There is
|
|
5766
|
+
// no condition to stand, so an episode carried over from a larger window
|
|
5767
|
+
// (the prefetch of a re-registered consumer can change) ends here — silently,
|
|
5768
|
+
// because what it was measured against no longer exists.
|
|
5769
|
+
tracking.utilizationHigh = false;
|
|
5770
|
+
return;
|
|
4403
5771
|
}
|
|
4404
5772
|
|
|
4405
5773
|
const utilization = tracking.inFlight / tracking.prefetchCount;
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
5774
|
+
const high = utilization >= this._prefetchUtilizationThreshold;
|
|
5775
|
+
|
|
5776
|
+
if (high === tracking.utilizationHigh) {
|
|
5777
|
+
return; // The same condition as at the last verdict — it has already been said.
|
|
5778
|
+
}
|
|
5779
|
+
|
|
5780
|
+
tracking.utilizationHigh = high;
|
|
5781
|
+
|
|
5782
|
+
const state = `queue '${queue}' has ${tracking.inFlight}/${tracking.prefetchCount} messages in-flight (${Math.round(utilization * 100)}%, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`;
|
|
5783
|
+
|
|
5784
|
+
if (!high) {
|
|
5785
|
+
// The end of the episode: a record, not an alarm.
|
|
5786
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] Prefetch utilization back below threshold: ${state}`);
|
|
5787
|
+
return;
|
|
5788
|
+
}
|
|
5789
|
+
|
|
5790
|
+
const message = `Prefetch utilization high: ${state}`;
|
|
5791
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] ⚠️ ${message}`);
|
|
5792
|
+
|
|
5793
|
+
// Call alert callback if provided
|
|
5794
|
+
if (this._prefetchAlertCallback) {
|
|
5795
|
+
try {
|
|
5796
|
+
this._prefetchAlertCallback(queue, utilization, tracking.inFlight, tracking.prefetchCount);
|
|
5797
|
+
} catch (alertErr) {
|
|
5798
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Prefetch alert callback error:`, alertErr.message);
|
|
4418
5799
|
}
|
|
4419
|
-
|
|
4420
|
-
// Emit event for external listeners
|
|
4421
|
-
this.emit('prefetch:high-utilization', {
|
|
4422
|
-
queue,
|
|
4423
|
-
utilization,
|
|
4424
|
-
inFlight: tracking.inFlight,
|
|
4425
|
-
prefetchCount: tracking.prefetchCount,
|
|
4426
|
-
threshold: this._prefetchUtilizationThreshold,
|
|
4427
|
-
timestamp: new Date().toISOString()
|
|
4428
|
-
});
|
|
4429
5800
|
}
|
|
5801
|
+
|
|
5802
|
+
// Emit event for external listeners
|
|
5803
|
+
this.emit('prefetch:high-utilization', {
|
|
5804
|
+
queue,
|
|
5805
|
+
utilization,
|
|
5806
|
+
inFlight: tracking.inFlight,
|
|
5807
|
+
prefetchCount: tracking.prefetchCount,
|
|
5808
|
+
threshold: this._prefetchUtilizationThreshold,
|
|
5809
|
+
timestamp: new Date().toISOString()
|
|
5810
|
+
});
|
|
4430
5811
|
}
|
|
4431
5812
|
|
|
4432
5813
|
/**
|
|
4433
5814
|
* Start periodic prefetch monitoring
|
|
5815
|
+
*
|
|
5816
|
+
* WHAT IS LEFT FOR THIS TICK TO SAY since d.736a: the verdict is taken on both
|
|
5817
|
+
* edges of every delivery (`consume()`), and `_checkPrefetchUtilization()` returns
|
|
5818
|
+
* in silence when the condition is the one already announced — so a tick that sees
|
|
5819
|
+
* the same `inFlight` and the same `prefetchCount` says nothing. It speaks only
|
|
5820
|
+
* where the verdict changed without a delivery in between, and the one quantity
|
|
5821
|
+
* that can move that way is `prefetchCount`: a consumer re-registered with a
|
|
5822
|
+
* different window (`_applyConsumerPrefetch()`).
|
|
5823
|
+
*
|
|
4434
5824
|
* @private
|
|
4435
5825
|
*/
|
|
4436
5826
|
_startPrefetchMonitoring() {
|
|
@@ -4438,7 +5828,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4438
5828
|
return; // Already started
|
|
4439
5829
|
}
|
|
4440
5830
|
|
|
4441
|
-
this._prefetchCheckTimer =
|
|
5831
|
+
this._prefetchCheckTimer = this._setInterval(() => {
|
|
4442
5832
|
const now = Date.now();
|
|
4443
5833
|
for (const [queue, tracking] of this._prefetchTracking.entries()) {
|
|
4444
5834
|
// Only check if we have recent activity (within last 30s)
|
|
@@ -4457,7 +5847,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4457
5847
|
*/
|
|
4458
5848
|
_stopPrefetchMonitoring() {
|
|
4459
5849
|
if (this._prefetchCheckTimer) {
|
|
4460
|
-
|
|
5850
|
+
this._clearInterval(this._prefetchCheckTimer);
|
|
4461
5851
|
this._prefetchCheckTimer = null;
|
|
4462
5852
|
}
|
|
4463
5853
|
}
|
|
@@ -4736,10 +6126,12 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4736
6126
|
this._consumerChannel.ack(msg);
|
|
4737
6127
|
msg._mqProcessed = true;
|
|
4738
6128
|
} catch (err) {
|
|
4739
|
-
|
|
4740
|
-
|
|
6129
|
+
// The same reading as the auto-ack inside `consume()` — see the note there (d.698).
|
|
6130
|
+
// amqplib's terminal-state class is the only thing `ack()` throws; anything else
|
|
6131
|
+
// reaching this catch came from outside amqplib and is a defect that must be heard.
|
|
6132
|
+
if (err instanceof IllegalOperationError) {
|
|
4741
6133
|
msg._mqProcessed = true;
|
|
4742
|
-
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot ack - consumer channel is closed/
|
|
6134
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot ack - consumer channel is closed/closing (the broker returns the unacked delivery)', { error: err.message });
|
|
4743
6135
|
return;
|
|
4744
6136
|
}
|
|
4745
6137
|
this.emit('error', err);
|
|
@@ -4777,10 +6169,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4777
6169
|
this._consumerChannel.nack(msg, false, requeue);
|
|
4778
6170
|
msg._mqProcessed = true;
|
|
4779
6171
|
} catch (err) {
|
|
4780
|
-
|
|
4781
|
-
if (
|
|
6172
|
+
// The same reading as `ack()` above (d.698): the class amqplib refuses a call with.
|
|
6173
|
+
if (err instanceof IllegalOperationError) {
|
|
4782
6174
|
msg._mqProcessed = true;
|
|
4783
|
-
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot nack - consumer channel is closed/
|
|
6175
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot nack - consumer channel is closed/closing (the broker returns the unacked delivery)', { error: err.message });
|
|
4784
6176
|
return;
|
|
4785
6177
|
}
|
|
4786
6178
|
this.emit('error', err);
|