@onlineapps/mq-client-core 3.2.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +368 -0
- package/README.md +77 -6
- package/package.json +2 -2
- package/src/BaseClient.js +65 -4
- package/src/config/configSchema.js +7 -14
- package/src/config/defaultConfig.js +3 -7
- package/src/config/queueConfig.js +22 -2
- package/src/transports/rabbitmqClient.js +462 -77
- package/src/workers/RecoveryWorker.js +10 -6
|
@@ -415,10 +415,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
415
415
|
|
|
416
416
|
// Recovery worker — connection recovery, and the report of a queue that does
|
|
417
417
|
// not exist. It creates none: queue creation left it with the two callbacks
|
|
418
|
-
// that fed it (d.419)
|
|
418
|
+
// that fed it (d.419) and with `recoveryScope`, the permission they were
|
|
419
|
+
// filtered by (d.514). Which rule the report names is decided by the
|
|
420
|
+
// ownership class in the error, not by who built this client.
|
|
419
421
|
this._recoveryWorker = new RecoveryWorker({
|
|
420
422
|
client: this,
|
|
421
|
-
scope: this._config.recoveryScope,
|
|
422
423
|
logger: this._logger,
|
|
423
424
|
});
|
|
424
425
|
|
|
@@ -524,13 +525,52 @@ class RabbitMQClient extends EventEmitter {
|
|
|
524
525
|
}
|
|
525
526
|
|
|
526
527
|
/**
|
|
527
|
-
*
|
|
528
|
-
*
|
|
528
|
+
* THE definition of "this client's consumer is attached to that queue".
|
|
529
|
+
*
|
|
530
|
+
* The tag is the one fact that is about THIS client: the broker gives it to
|
|
531
|
+
* the attachment it created, and since d.386b / d.605 the entry carries it
|
|
532
|
+
* only while that attachment exists — a refused re-registration and a cancel
|
|
533
|
+
* sent by the broker both drop it while the consumer stays TRACKED, because
|
|
534
|
+
* the next recovery must find it on the list.
|
|
535
|
+
*
|
|
536
|
+
* It is a method rather than an expression repeated twice for the reason
|
|
537
|
+
* `change-discipline.md` § One rail per concern states: until d.613
|
|
538
|
+
* `getConsumerState()` answered from `_activeConsumers.keys()` and
|
|
539
|
+
* `_performHealthCheck()` from the tag, so one file held two definitions of
|
|
540
|
+
* "active" — and the cheap synchronous reader, the one a service and an
|
|
541
|
+
* operator actually read, was the one that answered about a consumer the
|
|
542
|
+
* broker had already cancelled.
|
|
543
|
+
*
|
|
544
|
+
* @param {Object|undefined} consumerInfo - The entry `_activeConsumers` holds
|
|
545
|
+
* @returns {boolean}
|
|
546
|
+
* @private
|
|
547
|
+
*/
|
|
548
|
+
_isConsumerAttached(consumerInfo) {
|
|
549
|
+
return typeof consumerInfo?.consumerTag === 'string' && consumerInfo.consumerTag !== '';
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/**
|
|
553
|
+
* Get current consumer state.
|
|
554
|
+
*
|
|
555
|
+
* `tracked` counts the consumers this client would restore; `active` names the
|
|
556
|
+
* ones that are ATTACHED right now (`_isConsumerAttached()`, the same question
|
|
557
|
+
* `_performHealthCheck()` asks). `tracked > active.length` is therefore the
|
|
558
|
+
* cheap, synchronous statement that this client takes no message off some
|
|
559
|
+
* queue it registered — the state
|
|
560
|
+
* `docs/operations/rabbitmq-connection-recovery-troubleshooting.md` sends an
|
|
561
|
+
* operator here to read, and which the queue-name list could not carry.
|
|
562
|
+
*
|
|
563
|
+
* @returns {{tracked: number, active: string[]}} Consumer state information
|
|
529
564
|
*/
|
|
530
565
|
getConsumerState() {
|
|
566
|
+
const active = [];
|
|
567
|
+
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
568
|
+
if (this._isConsumerAttached(consumerInfo)) active.push(queue);
|
|
569
|
+
}
|
|
570
|
+
|
|
531
571
|
return {
|
|
532
572
|
tracked: this._activeConsumers.size,
|
|
533
|
-
active
|
|
573
|
+
active
|
|
534
574
|
};
|
|
535
575
|
}
|
|
536
576
|
|
|
@@ -696,6 +736,142 @@ class RabbitMQClient extends EventEmitter {
|
|
|
696
736
|
return REFUSAL_REPLY_CODES.some((code) => message.includes(`server: ${code} (`));
|
|
697
737
|
}
|
|
698
738
|
|
|
739
|
+
/**
|
|
740
|
+
* State the broker's refusal of a QUEUE the same way `_failFatally()` states
|
|
741
|
+
* its refusal of a CONNECTION — as a marker on the error, read by the caller
|
|
742
|
+
* that must decide what to do about it.
|
|
743
|
+
*
|
|
744
|
+
* Measured defect (d.577, sonda W532 against the live broker): a queue that
|
|
745
|
+
* already exists with other arguments answers `assertQueue()` with `406
|
|
746
|
+
* PRECONDITION-FAILED`, and `_prepareQueueForConsume()` built a `ConsumeError`
|
|
747
|
+
* for it with no `code` and no `reason`. The reply code survived in `cause`
|
|
748
|
+
* alone — and one wrapping further up (`@onlineapps/conn-orch-registry`) in
|
|
749
|
+
* `cause.cause`, where nothing looks. `@onlineapps/service-wrapper` ends FÁZE
|
|
750
|
+
* 0.7 permanently only on `code: 'MQ_CONNECTION_FATAL'`, so a 406 on a queue
|
|
751
|
+
* was classified TRANSIENT and the service went on booting against a queue it
|
|
752
|
+
* will never be allowed to declare. Every attempt gets the same answer.
|
|
753
|
+
*
|
|
754
|
+
* ONE vocabulary for one fact, therefore: the marker is the one `_failFatally()`
|
|
755
|
+
* already writes for `broker-refused` (d.295/d.490), not a second spelling of
|
|
756
|
+
* it. A refusal of the topology and a refusal of the connection ask the reader
|
|
757
|
+
* for the same thing — stop retrying, fix what the broker named — and the
|
|
758
|
+
* lifecycle owner reads one marker, not a list of them.
|
|
759
|
+
*
|
|
760
|
+
* What it deliberately does NOT do is `_failFatally()`: that declares the
|
|
761
|
+
* CONNECTION permanently lost and stands the client down. The connection here
|
|
762
|
+
* is healthy; it is this queue the broker refuses, and the publish path over
|
|
763
|
+
* the same connection has nothing to answer for.
|
|
764
|
+
*
|
|
765
|
+
* The decision is taken on the REPLY CODE (`_isBrokerRefusal()`,
|
|
766
|
+
* `REFUSAL_REPLY_CODES`), never on the wording of a sentence somebody may
|
|
767
|
+
* reword — the defect d.164 measured. The broker's own error is kept as
|
|
768
|
+
* `cause`, so its reply code stays readable.
|
|
769
|
+
*
|
|
770
|
+
* BOTH rails that talk to the broker about a queue go through this one
|
|
771
|
+
* function (d.582). `consume()` was the first; `assertQueue()` is the other
|
|
772
|
+
* half of the same fact, and it is the one FÁZE 0.8 walks — a service that
|
|
773
|
+
* declares its business queues gets the 406 there, not on the consume path.
|
|
774
|
+
* Two copies of one classification would be free to drift
|
|
775
|
+
* (`change-discipline.md` § One rail per concern), so `kind` decides only the
|
|
776
|
+
* SENTENCE and the class of the wrapper; the verdict is written once.
|
|
777
|
+
*
|
|
778
|
+
* @param {'consume'|'declare'} kind - which rail is answering.
|
|
779
|
+
* @param {string} queue - the queue the broker was asked about.
|
|
780
|
+
* @param {Error} err - what that rail threw.
|
|
781
|
+
* @returns {Error} the same error carrying the verdict, a typed error carrying
|
|
782
|
+
* it when the throw was the broker's raw one, or `err` untouched when the
|
|
783
|
+
* broker refused nothing.
|
|
784
|
+
* @private
|
|
785
|
+
*/
|
|
786
|
+
_asRefusal(kind, queue, err) {
|
|
787
|
+
const brokerError = this._brokerRefusalCause(err);
|
|
788
|
+
|
|
789
|
+
if (!brokerError) return err;
|
|
790
|
+
|
|
791
|
+
// Already stated, and in the shape this rail hands out: the verdict is
|
|
792
|
+
// written once, not once per layer that passes it on.
|
|
793
|
+
const alreadyStated = err.code === 'MQ_CONNECTION_FATAL'
|
|
794
|
+
&& (kind !== 'consume' || err instanceof ConsumeError);
|
|
795
|
+
if (alreadyStated) return err;
|
|
796
|
+
|
|
797
|
+
// An error this rail built itself — the 406 sentence of
|
|
798
|
+
// `_prepareQueueForConsume()` — is marked in place: it names the fix more
|
|
799
|
+
// precisely than anything this function could write.
|
|
800
|
+
if (kind === 'consume' && err instanceof ConsumeError) {
|
|
801
|
+
return this._markBrokerRefusal(err);
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
const refusal = kind === 'consume'
|
|
805
|
+
? new ConsumeError(
|
|
806
|
+
`[RabbitMQClient] Cannot consume from queue "${queue}": the broker REFUSED the operation, `
|
|
807
|
+
+ `so retrying cannot change the answer (reason: ${brokerError.message}). `
|
|
808
|
+
+ 'Expected: an account allowed to use this queue, and a queue whose declaration is the one '
|
|
809
|
+
+ 'queueConfig prescribes. '
|
|
810
|
+
+ 'Fix: correct the permission or the topology the broker named, then restart the process; '
|
|
811
|
+
+ 'error.cause carries the broker reply code.',
|
|
812
|
+
queue,
|
|
813
|
+
brokerError
|
|
814
|
+
)
|
|
815
|
+
: new Error(
|
|
816
|
+
`[RabbitMQClient] Cannot declare queue "${queue}": the broker REFUSED the declaration, `
|
|
817
|
+
+ `so retrying cannot change the answer (reason: ${brokerError.message}). `
|
|
818
|
+
+ 'Expected: the queue to exist with the arguments queueConfig prescribes, and an account '
|
|
819
|
+
+ 'allowed to declare it. '
|
|
820
|
+
+ 'Fix: align the declaration the broker named (406 means the queue exists with other '
|
|
821
|
+
+ 'arguments) or correct the permission, then restart the process; '
|
|
822
|
+
+ 'error.cause carries the broker reply code.',
|
|
823
|
+
{ cause: brokerError }
|
|
824
|
+
);
|
|
825
|
+
|
|
826
|
+
if (kind !== 'consume') refusal.queue = queue;
|
|
827
|
+
return this._markBrokerRefusal(refusal);
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
/**
|
|
831
|
+
* The broker error behind a failure, or `null` when the broker refused nothing.
|
|
832
|
+
*
|
|
833
|
+
* One `cause` deep, because that is how far the rails here wrap: the
|
|
834
|
+
* declaration step throws the broker's error, and the sentence built around it
|
|
835
|
+
* keeps it as `cause`. The decision is taken on the REPLY CODE
|
|
836
|
+
* (`_isBrokerRefusal()`, `REFUSAL_REPLY_CODES`), never on the wording of a
|
|
837
|
+
* sentence somebody may reword — the defect d.164 measured.
|
|
838
|
+
*
|
|
839
|
+
* @param {Error|null} err
|
|
840
|
+
* @returns {Error|null} the error carrying the broker's reply code
|
|
841
|
+
* @private
|
|
842
|
+
*/
|
|
843
|
+
_brokerRefusalCause(err) {
|
|
844
|
+
if (!err) return null;
|
|
845
|
+
|
|
846
|
+
// An error already carrying the verdict is NOT the broker's own — the
|
|
847
|
+
// broker's is one `cause` below it. Asked of the marked error, the text half
|
|
848
|
+
// of `_isBrokerRefusal()` would answer yes about the wrapper, because the
|
|
849
|
+
// sentence built here quotes the broker's message verbatim (measured while
|
|
850
|
+
// writing d.582: `error.cause.code` came back `'MQ_CONNECTION_FATAL'`
|
|
851
|
+
// instead of `406`). The code is asked first, so the quote cannot be
|
|
852
|
+
// mistaken for the answer.
|
|
853
|
+
if (err.code === 'MQ_CONNECTION_FATAL') {
|
|
854
|
+
return this._isBrokerRefusal(err.cause) ? err.cause : null;
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
if (this._isBrokerRefusal(err)) return err;
|
|
858
|
+
if (this._isBrokerRefusal(err.cause)) return err.cause;
|
|
859
|
+
return null;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
/**
|
|
863
|
+
* Write the verdict. ONE spelling, the one `_failFatally()` uses.
|
|
864
|
+
*
|
|
865
|
+
* @param {Error} error
|
|
866
|
+
* @returns {Error} the same error, marked
|
|
867
|
+
* @private
|
|
868
|
+
*/
|
|
869
|
+
_markBrokerRefusal(error) {
|
|
870
|
+
error.code = 'MQ_CONNECTION_FATAL';
|
|
871
|
+
error.reason = 'broker-refused';
|
|
872
|
+
return error;
|
|
873
|
+
}
|
|
874
|
+
|
|
699
875
|
/**
|
|
700
876
|
* A recovery cycle is over and the client is NOT connected, but the broker
|
|
701
877
|
* never refused it — so this is not the end.
|
|
@@ -823,12 +999,27 @@ class RabbitMQClient extends EventEmitter {
|
|
|
823
999
|
* confused with an ordinary transient error: the `connection:fatal` event and
|
|
824
1000
|
* the injected `onFatal` callback.
|
|
825
1001
|
*
|
|
1002
|
+
* WHO IS TOLD, and why that is not always the same. The `connection:fatal`
|
|
1003
|
+
* event and the injected `onFatal` callback exist for a loss that happens
|
|
1004
|
+
* asynchronously, with nobody awaiting anything — the owner would otherwise
|
|
1005
|
+
* never hear of it. On the BOOT path a caller IS awaiting `connect()`, so it
|
|
1006
|
+
* learns by the throw, and announcing as well would be a second notification
|
|
1007
|
+
* of one event: the harm `_reportRecoveryFailure()` documents a few methods
|
|
1008
|
+
* up, and a crossing of the line the boot caller draws itself (a boot failure
|
|
1009
|
+
* is the boot's outcome, not a lifecycle event — see the `BOOT budget` note in
|
|
1010
|
+
* `@onlineapps/service-wrapper`'s `_initializeMQ()`). Hence `announce`. The
|
|
1011
|
+
* verdict, the state and the sentence are identical either way; only the
|
|
1012
|
+
* channel differs (d.295).
|
|
1013
|
+
*
|
|
826
1014
|
* @param {Error} lastError - The error from the final failed attempt.
|
|
827
1015
|
* @param {'broker-refused'|'cycles-spent'} reason - Which of the two it is.
|
|
1016
|
+
* @param {Object} [options] - Reporting options.
|
|
1017
|
+
* @param {boolean} [options.announce=true] - Emit `connection:fatal` and call
|
|
1018
|
+
* `onFatal`. False on a path whose caller is awaiting the throw.
|
|
828
1019
|
* @returns {Error} The fatal error, so callers can throw it.
|
|
829
1020
|
* @private
|
|
830
1021
|
*/
|
|
831
|
-
_failFatally(lastError, reason) {
|
|
1022
|
+
_failFatally(lastError, reason, { announce = true } = {}) {
|
|
832
1023
|
this._connectionFatal = true;
|
|
833
1024
|
this._reconnecting = false;
|
|
834
1025
|
this._lastConnectionError = lastError || null;
|
|
@@ -855,16 +1046,21 @@ class RabbitMQClient extends EventEmitter {
|
|
|
855
1046
|
fatal.cycles = this._reconnectCycles;
|
|
856
1047
|
fatal.cause = lastError;
|
|
857
1048
|
|
|
1049
|
+
// The log line is the RECORD and is unconditional; the two lines below are
|
|
1050
|
+
// the NOTIFICATION, and only a caller that is not awaiting needs it.
|
|
858
1051
|
this._logger.error(`[RabbitMQClient] [mq-client-core] ${fatal.message}`);
|
|
859
|
-
this.emit('connection:fatal', fatal);
|
|
860
1052
|
|
|
861
|
-
if (
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
)
|
|
1053
|
+
if (announce) {
|
|
1054
|
+
this.emit('connection:fatal', fatal);
|
|
1055
|
+
|
|
1056
|
+
if (this._onFatal) {
|
|
1057
|
+
try {
|
|
1058
|
+
this._onFatal(fatal);
|
|
1059
|
+
} catch (err) {
|
|
1060
|
+
this._logger.error(
|
|
1061
|
+
`[RabbitMQClient] [mq-client-core] onFatal handler threw: ${err.message}`
|
|
1062
|
+
);
|
|
1063
|
+
}
|
|
868
1064
|
}
|
|
869
1065
|
}
|
|
870
1066
|
|
|
@@ -1327,6 +1523,25 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1327
1523
|
}
|
|
1328
1524
|
this._connection = null;
|
|
1329
1525
|
}
|
|
1526
|
+
|
|
1527
|
+
// The broker ANSWERED and refused — 403, 530, 406. The same verdict
|
|
1528
|
+
// `_reconnectWithBackoff()` reaches at runtime, on the same rail
|
|
1529
|
+
// (`_isBrokerRefusal`), because it is the same question: did anybody
|
|
1530
|
+
// answer? Until d.295 this catch rethrew whatever amqplib raised, with no
|
|
1531
|
+
// verdict on it, and the boot caller could only guess — `ServiceWrapper`
|
|
1532
|
+
// guessed "transient" and spent its whole connection budget, backoff and
|
|
1533
|
+
// all, on an answer no further attempt can change
|
|
1534
|
+
// (`architecture-principles.md` §4).
|
|
1535
|
+
//
|
|
1536
|
+
// Anything unrecognised still leaves here as it arrived, and that is the
|
|
1537
|
+
// owner's decision, not an omission: giving up forever on a broker that
|
|
1538
|
+
// never answered is forbidden
|
|
1539
|
+
// (`docs/governance/confirmations/mq-client-lifecycle-contract.md` 001
|
|
1540
|
+
// point 3), so only an ANSWERED refusal ends the attempts.
|
|
1541
|
+
if (this._isBrokerRefusal(err)) {
|
|
1542
|
+
throw this._failFatally(err, 'broker-refused', { announce: false });
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1330
1545
|
throw err;
|
|
1331
1546
|
} finally {
|
|
1332
1547
|
if (connectTimeoutTimer) {
|
|
@@ -1682,54 +1897,160 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1682
1897
|
if (this._activeConsumers.size > 0) {
|
|
1683
1898
|
this._logger.info(`[RabbitMQClient] [mq-client-core] Re-registering ${this._activeConsumers.size} consumers...`);
|
|
1684
1899
|
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
1685
|
-
|
|
1686
|
-
await this._prepareQueueForConsume(queue, {
|
|
1687
|
-
durable: consumerInfo.options.durable,
|
|
1688
|
-
queueOptions: consumerInfo.options.queueOptions
|
|
1689
|
-
});
|
|
1690
|
-
|
|
1691
|
-
// The consumer's prefetch is re-applied here for the same reason the
|
|
1692
|
-
// queue step is: the channel is new, and QoS lives on the channel.
|
|
1693
|
-
await this._applyConsumerPrefetch(queue, consumerInfo.options.prefetch);
|
|
1694
|
-
|
|
1695
|
-
const consumeResult = await this._consumerChannel.consume(
|
|
1696
|
-
queue,
|
|
1697
|
-
consumerInfo.handler,
|
|
1698
|
-
{ noAck: consumerInfo.options.noAck }
|
|
1699
|
-
);
|
|
1700
|
-
consumerInfo.consumerTag = consumeResult.consumerTag;
|
|
1900
|
+
if (await this._reRegisterConsumer(queue, consumerInfo)) {
|
|
1701
1901
|
reRegistered++;
|
|
1702
|
-
|
|
1703
|
-
} catch (err) {
|
|
1902
|
+
} else {
|
|
1704
1903
|
failed++;
|
|
1705
|
-
this._logger.error(`[RabbitMQClient] [mq-client-core] Failed to re-register consumer for queue ${queue}:`, err.message);
|
|
1706
|
-
// The consumer STAYS tracked. Whatever refused it — a 406 against the
|
|
1707
|
-
// owner's declaration, a missing infrastructure queue, a channel that
|
|
1708
|
-
// went away mid-flight — is a condition somebody fixes at its source,
|
|
1709
|
-
// and the next recreate must find the consumer still on the list.
|
|
1710
|
-
//
|
|
1711
|
-
// Its TAG does not stay, and the two are not the same fact (d.386b). A
|
|
1712
|
-
// consumer tag names an attachment on ONE channel; the channel that
|
|
1713
|
-
// issued this one is gone and the new one refused to re-attach, so the
|
|
1714
|
-
// string the entry carried names nothing on the broker. Two readers
|
|
1715
|
-
// took it for an attachment that exists: `_performHealthCheck()` counts
|
|
1716
|
-
// `active` by "is the tag set", so a client whose only consumer had
|
|
1717
|
-
// just been refused reported `healthy: true` and `1/1 consumers active`
|
|
1718
|
-
// — the false OK d.386 exists to end, one layer below where d.386 could
|
|
1719
|
-
// reach it; and `cancelConsumer()` sent `basic.cancel` with it, asking
|
|
1720
|
-
// the broker to stop a consumer it does not hold.
|
|
1721
|
-
consumerInfo.consumerTag = null;
|
|
1722
|
-
// The broker's own code is what a listener acts on (406 = the queue
|
|
1723
|
-
// exists with different arguments), and the rail wraps it in a typed
|
|
1724
|
-
// error, so it is read from the cause when the wrapper carries none.
|
|
1725
|
-
const code = err.code !== undefined ? err.code : (err.cause ? err.cause.code : undefined);
|
|
1726
|
-
this.emit('consumer:re-registration:failed', { queue, error: err.message, code });
|
|
1727
1904
|
}
|
|
1728
1905
|
}
|
|
1729
1906
|
}
|
|
1730
1907
|
return { reRegistered, failed };
|
|
1731
1908
|
}
|
|
1732
1909
|
|
|
1910
|
+
/**
|
|
1911
|
+
* Re-attach ONE tracked consumer on the current consumer channel.
|
|
1912
|
+
*
|
|
1913
|
+
* This is the body of the loop above, given a name in d.605 because a second
|
|
1914
|
+
* event needs exactly it: `basic.cancel` from the broker takes ONE consumer
|
|
1915
|
+
* away while every other one stays attached, so re-running the whole loop
|
|
1916
|
+
* would re-attach consumers that never lost their place — a second
|
|
1917
|
+
* `basic.consume` for each of them on the broker. One rail, one set of
|
|
1918
|
+
* consequences (`change-discipline.md` § One rail per concern).
|
|
1919
|
+
*
|
|
1920
|
+
* Everything a first `consume()` does is done here: the queue is classified,
|
|
1921
|
+
* declared or checked by `_prepareQueueForConsume()` — so an owned business
|
|
1922
|
+
* queue is re-declared and an unowned infrastructure queue is only checked,
|
|
1923
|
+
* its absence answered with `CONSUMER_QUEUE_MISSING` rather than a silent
|
|
1924
|
+
* creation — and the prefetch is re-applied, because QoS lives on the channel.
|
|
1925
|
+
*
|
|
1926
|
+
* @param {string} queue - Queue name
|
|
1927
|
+
* @param {Object} consumerInfo - The entry `_activeConsumers` holds for it
|
|
1928
|
+
* @returns {Promise<boolean>} `true` when the consumer is attached again
|
|
1929
|
+
* @private
|
|
1930
|
+
*/
|
|
1931
|
+
async _reRegisterConsumer(queue, consumerInfo) {
|
|
1932
|
+
try {
|
|
1933
|
+
await this._prepareQueueForConsume(queue, {
|
|
1934
|
+
durable: consumerInfo.options.durable,
|
|
1935
|
+
queueOptions: consumerInfo.options.queueOptions
|
|
1936
|
+
});
|
|
1937
|
+
|
|
1938
|
+
// The consumer's prefetch is re-applied here for the same reason the
|
|
1939
|
+
// queue step is: the channel is new, and QoS lives on the channel.
|
|
1940
|
+
await this._applyConsumerPrefetch(queue, consumerInfo.options.prefetch);
|
|
1941
|
+
|
|
1942
|
+
const consumeResult = await this._consumerChannel.consume(
|
|
1943
|
+
queue,
|
|
1944
|
+
consumerInfo.handler,
|
|
1945
|
+
{ noAck: consumerInfo.options.noAck }
|
|
1946
|
+
);
|
|
1947
|
+
consumerInfo.consumerTag = consumeResult.consumerTag;
|
|
1948
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] ✓ Re-registered consumer for queue: ${queue} (consumerTag: ${consumeResult.consumerTag})`);
|
|
1949
|
+
return true;
|
|
1950
|
+
} catch (err) {
|
|
1951
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Failed to re-register consumer for queue ${queue}:`, err.message);
|
|
1952
|
+
// The consumer STAYS tracked. Whatever refused it — a 406 against the
|
|
1953
|
+
// owner's declaration, a missing infrastructure queue, a channel that
|
|
1954
|
+
// went away mid-flight — is a condition somebody fixes at its source,
|
|
1955
|
+
// and the next recreate must find the consumer still on the list.
|
|
1956
|
+
//
|
|
1957
|
+
// Its TAG does not stay, and the two are not the same fact (d.386b). A
|
|
1958
|
+
// consumer tag names an attachment on ONE channel; the channel that
|
|
1959
|
+
// issued this one is gone and the new one refused to re-attach, so the
|
|
1960
|
+
// string the entry carried names nothing on the broker. Two readers
|
|
1961
|
+
// took it for an attachment that exists: `_performHealthCheck()` counts
|
|
1962
|
+
// `active` by "is the tag set", so a client whose only consumer had
|
|
1963
|
+
// just been refused reported `healthy: true` and `1/1 consumers active`
|
|
1964
|
+
// — the false OK d.386 exists to end, one layer below where d.386 could
|
|
1965
|
+
// reach it; and `cancelConsumer()` sent `basic.cancel` with it, asking
|
|
1966
|
+
// the broker to stop a consumer it does not hold.
|
|
1967
|
+
consumerInfo.consumerTag = null;
|
|
1968
|
+
// The broker's own code is what a listener acts on (406 = the queue
|
|
1969
|
+
// exists with different arguments), and the rail wraps it in a typed
|
|
1970
|
+
// error, so it is read from the cause when the wrapper carries none.
|
|
1971
|
+
const code = err.code !== undefined ? err.code : (err.cause ? err.cause.code : undefined);
|
|
1972
|
+
this.emit('consumer:re-registration:failed', { queue, error: err.message, code });
|
|
1973
|
+
return false;
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
/**
|
|
1978
|
+
* The broker cancelled one of this client's consumers.
|
|
1979
|
+
*
|
|
1980
|
+
* `basic.cancel` sent BY THE BROKER is not an error frame and does not touch
|
|
1981
|
+
* the channel or the connection: the queue was deleted, or the broker moved it
|
|
1982
|
+
* (`x-cancel-on-ha-failover`). Nothing else in this client hears it — the
|
|
1983
|
+
* channel handlers listen for `error` and `close`, and both stay silent — so
|
|
1984
|
+
* this method is the whole of the client's answer, and until d.605 that answer
|
|
1985
|
+
* was nothing at all.
|
|
1986
|
+
*
|
|
1987
|
+
* Three things happen, in this order:
|
|
1988
|
+
*
|
|
1989
|
+
* 1. the TAG is dropped, first, because it is the one fact
|
|
1990
|
+
* `_performHealthCheck()` reads to answer "is THIS client's consumer
|
|
1991
|
+
* attached" (d.386b). Dropping it before anything else leaves no window in
|
|
1992
|
+
* which the check reports an attachment the broker has already cancelled;
|
|
1993
|
+
* 2. the loss is stated at ERROR level, naming the queue and the cancelled
|
|
1994
|
+
* tag. A consumer that stops reading a queue is not a debug-level event,
|
|
1995
|
+
* and the record is what tells an operator which queue went quiet;
|
|
1996
|
+
* 3. the consumer is re-attached through `_reRegisterConsumer()`, the rail a
|
|
1997
|
+
* channel recreate already uses. A refusal there keeps the consumer
|
|
1998
|
+
* tracked with no tag and travels on `consumer:re-registration:failed` —
|
|
1999
|
+
* which is why this method introduces no event and no public method of its
|
|
2000
|
+
* own: every state it can reach is a state the package already reports.
|
|
2001
|
+
*
|
|
2002
|
+
* @param {string} queue - The queue whose consumer the broker cancelled
|
|
2003
|
+
* @returns {Promise<void>}
|
|
2004
|
+
* @private
|
|
2005
|
+
*/
|
|
2006
|
+
async _onBrokerCancel(queue) {
|
|
2007
|
+
if (this._closedByCaller) {
|
|
2008
|
+
// An intentional close is not a fault — the doctrine `disconnect()` states
|
|
2009
|
+
// and both channel handlers already follow.
|
|
2010
|
+
this._logger.debug(
|
|
2011
|
+
`[RabbitMQClient] [mq-client-core] [CONSUMER] Broker cancelled the consumer for queue "${queue}" while the client was being closed by its caller - no alarm, no re-attachment`
|
|
2012
|
+
);
|
|
2013
|
+
return;
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
const consumerInfo = this._activeConsumers.get(queue);
|
|
2017
|
+
const cancelledTag = consumerInfo ? consumerInfo.consumerTag : null;
|
|
2018
|
+
|
|
2019
|
+
if (consumerInfo) {
|
|
2020
|
+
consumerInfo.consumerTag = null;
|
|
2021
|
+
}
|
|
2022
|
+
|
|
2023
|
+
this._logger.error(
|
|
2024
|
+
`[RabbitMQClient] Consumer cancelled by the broker - the consumer on queue "${queue}" `
|
|
2025
|
+
+ `(consumerTag: ${cancelledTag}) was stopped by the BROKER, not by this client: the queue `
|
|
2026
|
+
+ 'was deleted, or the broker moved it (x-cancel-on-ha-failover). '
|
|
2027
|
+
+ 'Expected: a consumer stays attached until cancelConsumer() or disconnect(). '
|
|
2028
|
+
+ 'Fix: none here - the client re-attaches it on the rail a channel recreate uses; if that '
|
|
2029
|
+
+ 'is refused, the consumer stays tracked with no tag, performHealthCheck() reports this '
|
|
2030
|
+
+ 'client unhealthy and names the queue, and consumer:re-registration:failed carries the '
|
|
2031
|
+
+ 'broker reason.',
|
|
2032
|
+
{ queue, consumerTag: cancelledTag }
|
|
2033
|
+
);
|
|
2034
|
+
|
|
2035
|
+
if (!consumerInfo) {
|
|
2036
|
+
// The client no longer holds this consumer, so there is nothing to
|
|
2037
|
+
// re-attach and no state left to correct. The loss is still stated above.
|
|
2038
|
+
return;
|
|
2039
|
+
}
|
|
2040
|
+
|
|
2041
|
+
if (!this._isChannelAlive(this._consumerChannel)) {
|
|
2042
|
+
// Re-attaching on a dead channel is not this method's job: the channel
|
|
2043
|
+
// recovery re-registers every tracked consumer when it recreates the
|
|
2044
|
+
// channel (d.262), and the consumer stays tracked for exactly that.
|
|
2045
|
+
this._logger.warn(
|
|
2046
|
+
`[RabbitMQClient] [mq-client-core] [CONSUMER] Consumer for queue "${queue}" stays tracked and unattached - the consumer channel is not alive, so the channel recovery re-registers it`
|
|
2047
|
+
);
|
|
2048
|
+
return;
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
await this._reRegisterConsumer(queue, consumerInfo);
|
|
2052
|
+
}
|
|
2053
|
+
|
|
1733
2054
|
/**
|
|
1734
2055
|
* Disconnects: closes channel and connection.
|
|
1735
2056
|
* @returns {Promise<void>}
|
|
@@ -1885,8 +2206,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1885
2206
|
// this client's and everybody else's alike, so it cannot say whether
|
|
1886
2207
|
// the one asking is among them. The tag can: it is the name the
|
|
1887
2208
|
// broker gave THIS client's attachment, and since d.386b it is set
|
|
1888
|
-
// only while that attachment exists.
|
|
1889
|
-
|
|
2209
|
+
// only while that attachment exists. The question is asked through
|
|
2210
|
+
// `_isConsumerAttached()` — the one definition `getConsumerState()`
|
|
2211
|
+
// now reads too (d.613).
|
|
2212
|
+
if (this._isConsumerAttached(consumerInfo)) {
|
|
1890
2213
|
health.consumers.active++;
|
|
1891
2214
|
} else {
|
|
1892
2215
|
health.consumers.inactive.push(queue);
|
|
@@ -2203,7 +2526,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2203
2526
|
|
|
2204
2527
|
const channel = await this._requireQueueChannel('assertQueue', queue);
|
|
2205
2528
|
|
|
2206
|
-
|
|
2529
|
+
try {
|
|
2530
|
+
return await channel.assertQueue(queue, queueOptions);
|
|
2531
|
+
} catch (err) {
|
|
2532
|
+
// The SAME verdict the consume rail states, through the SAME function
|
|
2533
|
+
// (d.582). This is the rail FÁZE 0.8 walks: a service declaring its own
|
|
2534
|
+
// business queues meets the 406 here, and until this batch the raw amqplib
|
|
2535
|
+
// error left with the reply code as a NUMBER and no classification at all,
|
|
2536
|
+
// so `@onlineapps/conn-orch-registry` had nothing to carry and
|
|
2537
|
+
// `@onlineapps/service-wrapper` — which reads `error.code` — called a
|
|
2538
|
+
// refusal transient. `_failFatally()` is deliberately not called: it is
|
|
2539
|
+
// this QUEUE the broker refuses, not the connection.
|
|
2540
|
+
throw this._asRefusal('declare', queue, err);
|
|
2541
|
+
}
|
|
2207
2542
|
}
|
|
2208
2543
|
|
|
2209
2544
|
/**
|
|
@@ -2993,8 +3328,8 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2993
3328
|
* declared with.
|
|
2994
3329
|
*
|
|
2995
3330
|
* ONE rail for the first `consume()` and for every re-registration after a
|
|
2996
|
-
* channel recreate. It classifies the queue (
|
|
2997
|
-
*
|
|
3331
|
+
* channel recreate. It classifies the queue (infrastructure / business /
|
|
3332
|
+
* neither), resolves the arguments from the central config, refuses a queue
|
|
2998
3333
|
* whose config declares no dead-letter route, and then CHECKS an
|
|
2999
3334
|
* infrastructure queue (consumers never create those) or ASSERTS a business
|
|
3000
3335
|
* one.
|
|
@@ -3014,15 +3349,24 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3014
3349
|
const durable = options.durable !== undefined ? options.durable : this._config.durable;
|
|
3015
3350
|
let queueOptions = options.queueOptions || { durable };
|
|
3016
3351
|
|
|
3017
|
-
// Skip assertQueue for reply queues (they're already created with specific settings)
|
|
3018
|
-
// Reply queues start with 'rpc.reply.' and are created as non-durable
|
|
3019
|
-
const isReplyQueue = queue.startsWith('rpc.reply.');
|
|
3020
3352
|
// Classification is read once and used by both blocks below — the one that
|
|
3021
3353
|
// resolves the central arguments, and the one that checks or asserts the queue.
|
|
3022
|
-
|
|
3023
|
-
|
|
3354
|
+
//
|
|
3355
|
+
// There is no third class. Until d.514 a name starting `rpc.reply.` was
|
|
3356
|
+
// exempted here and again below: classification, declaration and the broker
|
|
3357
|
+
// question were all skipped, because the retired RPC reply path declared such
|
|
3358
|
+
// a queue itself, with its own non-durable lifetime. Both exemptions had been
|
|
3359
|
+
// unreachable since d.259: `queueConfig` classifies that name as nothing, so
|
|
3360
|
+
// it declares no dead-letter route, so the gate between the two blocks throws
|
|
3361
|
+
// first — measured against the live broker on a reply queue that really
|
|
3362
|
+
// existed (`tests/integration/dead-letter-route-required.integration.test.js`).
|
|
3363
|
+
// The four questions are answered in the CHANGELOG of that batch; the path the
|
|
3364
|
+
// exemption served is gone (`docs/governance/confirmations/mq-consumer-contract.md`
|
|
3365
|
+
// 003 point 3 — "`rpc.reply.*` is not a topology question … it is a removal
|
|
3366
|
+
// question", and `conn-infra-mq` removed `createTemporaryQueue()` in d.275b).
|
|
3367
|
+
const isInfraQueue = queueConfig.isInfrastructureQueue(queue);
|
|
3368
|
+
const isBusinessQueue = queueConfig.isBusinessQueue(queue);
|
|
3024
3369
|
|
|
3025
|
-
if (!isReplyQueue) {
|
|
3026
3370
|
// CRITICAL: Use queueConfig.js to get correct parameters (TTL, max-length, etc.)
|
|
3027
3371
|
// This prevents 406 PRECONDITION-FAILED errors from TTL mismatches.
|
|
3028
3372
|
// queueConfig is loaded once at the top of this module; it used to be required
|
|
@@ -3066,9 +3410,6 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3066
3410
|
: `[RabbitMQClient] [mq-client-core] [CONSUMER] Asserting business queue ${queue} with the declaration queueConfig owns`
|
|
3067
3411
|
);
|
|
3068
3412
|
}
|
|
3069
|
-
// End of the queue-classification block. What follows applies to EVERY
|
|
3070
|
-
// queue this consumer may attach to, reply queues included.
|
|
3071
|
-
}
|
|
3072
3413
|
|
|
3073
3414
|
// The dead-letter policy below rejects a spent message with
|
|
3074
3415
|
// `nack(requeue=false)`, and the broker then moves it ONLY if the queue
|
|
@@ -3122,7 +3463,6 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3122
3463
|
);
|
|
3123
3464
|
}
|
|
3124
3465
|
|
|
3125
|
-
if (!isReplyQueue) {
|
|
3126
3466
|
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] Asserting queue ${queue} before consume() at ${new Date().toISOString()}`);
|
|
3127
3467
|
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] Queue options:`, JSON.stringify(queueOptions, null, 2));
|
|
3128
3468
|
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] _queueChannel state: exists=${!!this._queueChannel}, closed=${!this._isChannelAlive(this._queueChannel)}`);
|
|
@@ -3194,7 +3534,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3194
3534
|
} catch (assertErr) {
|
|
3195
3535
|
// If queue exists with different arguments (406), this is a CRITICAL ERROR
|
|
3196
3536
|
// We should NOT proceed - the root cause must be fixed
|
|
3197
|
-
|
|
3537
|
+
// Read off the BROKER's reply code, which since d.582 travels as the
|
|
3538
|
+
// `cause` of the declaration rail's own refusal — never off the marker,
|
|
3539
|
+
// which says "refused" without saying which of the three codes it was.
|
|
3540
|
+
const brokerError = this._brokerRefusalCause(assertErr);
|
|
3541
|
+
if (brokerError && brokerError.code === 406) {
|
|
3198
3542
|
this._logger.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✗ CRITICAL: Queue ${queue} exists with different arguments!`);
|
|
3199
3543
|
this._logger.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] Error:`, assertErr.message);
|
|
3200
3544
|
this._logger.error(`[RabbitMQClient] [mq-client-core] [CONSUMER] Expected options:`, JSON.stringify(queueOptions, null, 2));
|
|
@@ -3205,14 +3549,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3205
3549
|
+ 'Fix: find the assertQueue() call that declared it without parameters and fix that call; '
|
|
3206
3550
|
+ 'the queue is NOT redeclared here, because doing so would hide the drift instead of ending it.',
|
|
3207
3551
|
queue,
|
|
3208
|
-
|
|
3552
|
+
// The BROKER's error, not the declaration rail's wrapper around it:
|
|
3553
|
+
// one `cause` hop, and `error.cause.code` is the reply code (d.582).
|
|
3554
|
+
brokerError
|
|
3209
3555
|
);
|
|
3210
3556
|
}
|
|
3211
3557
|
// Other error - rethrow
|
|
3212
3558
|
throw assertErr;
|
|
3213
3559
|
}
|
|
3214
3560
|
}
|
|
3215
|
-
}
|
|
3216
3561
|
|
|
3217
3562
|
return queueOptions;
|
|
3218
3563
|
}
|
|
@@ -3334,7 +3679,21 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3334
3679
|
// Wrap handler to track message processing and prefetch utilization
|
|
3335
3680
|
const messageHandler = async (msg) => {
|
|
3336
3681
|
if (msg === null) {
|
|
3337
|
-
|
|
3682
|
+
// `null` is not a message. It is `basic.cancel` FROM THE BROKER,
|
|
3683
|
+
// handed to this callback by amqplib (`handleCancel()` →
|
|
3684
|
+
// `dispatchMessage(fields, null)`), and it is the only way a consumer
|
|
3685
|
+
// learns that its attachment is gone while the channel and the
|
|
3686
|
+
// connection stay up: the queue was deleted, or the broker moved it
|
|
3687
|
+
// (`x-cancel-on-ha-failover`).
|
|
3688
|
+
//
|
|
3689
|
+
// Until d.605 the answer was this bare `return`. Nothing was logged,
|
|
3690
|
+
// the tag stayed set — so `_performHealthCheck()` went on counting the
|
|
3691
|
+
// consumer as attached — and nothing re-attached it. The service
|
|
3692
|
+
// reported itself healthy while no message was taken off the queue,
|
|
3693
|
+
// which is the failure `automation-gates.md` §5 calls a defect of the
|
|
3694
|
+
// same severity as a wrong result.
|
|
3695
|
+
await this._onBrokerCancel(queue);
|
|
3696
|
+
return;
|
|
3338
3697
|
}
|
|
3339
3698
|
|
|
3340
3699
|
// IMPORTANT:
|
|
@@ -3497,8 +3856,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3497
3856
|
// Return consumer tag for cancellation
|
|
3498
3857
|
return consumeResult.consumerTag;
|
|
3499
3858
|
} catch (err) {
|
|
3500
|
-
|
|
3501
|
-
|
|
3859
|
+
// ONE place states the broker's verdict for this whole rail — the queue
|
|
3860
|
+
// classification, the declaration and the consumer registration alike —
|
|
3861
|
+
// because it is ONE question ("did the broker answer and refuse?") and a
|
|
3862
|
+
// copy of it per branch is free to drift (`change-discipline.md` § One rail
|
|
3863
|
+
// per concern). Anything the broker did not refuse leaves exactly as it
|
|
3864
|
+
// arrived (d.577).
|
|
3865
|
+
const refusal = this._asRefusal('consume', queue, err);
|
|
3866
|
+
this.emit('error', refusal);
|
|
3867
|
+
throw refusal;
|
|
3502
3868
|
}
|
|
3503
3869
|
}
|
|
3504
3870
|
|
|
@@ -3941,7 +4307,22 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3941
4307
|
}
|
|
3942
4308
|
|
|
3943
4309
|
/**
|
|
3944
|
-
* Check prefetch utilization for a queue and alert if threshold exceeded
|
|
4310
|
+
* Check prefetch utilization for a queue and alert if threshold exceeded.
|
|
4311
|
+
*
|
|
4312
|
+
* A window of ONE message has no utilisation to watch (d.550). Its only two
|
|
4313
|
+
* states are 0 % (idle) and 100 % (the consumer is doing the one piece of work
|
|
4314
|
+
* it is allowed), and 100 % is the normal state of a healthy consumer, not a
|
|
4315
|
+
* finding anybody can act on. The alarm therefore fired on EVERY delivery to
|
|
4316
|
+
* such a queue, whatever the threshold - `1/1 >= 0.8` holds exactly as
|
|
4317
|
+
* `1/1 >= 0.99` does. MEASURED (BIZ-pdfgen, BIZ-converter, twice per boot):
|
|
4318
|
+
* `Prefetch utilization high: queue 'biz-pdfgen.registry' has 1/1 messages
|
|
4319
|
+
* in-flight (100%, threshold: 80%)` - a warning that cannot fail to happen
|
|
4320
|
+
* says nothing, and it teaches the reader to skip the line where a real
|
|
4321
|
+
* saturation will one day stand.
|
|
4322
|
+
*
|
|
4323
|
+
* The threshold itself is unchanged and no new one is introduced: a window
|
|
4324
|
+
* larger than one message is measured exactly as before.
|
|
4325
|
+
*
|
|
3945
4326
|
* @private
|
|
3946
4327
|
* @param {string} queue - Queue name
|
|
3947
4328
|
* @param {Object} tracking - Prefetch tracking object
|
|
@@ -3950,7 +4331,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3950
4331
|
if (!tracking || tracking.prefetchCount === 0) {
|
|
3951
4332
|
return; // No prefetch set or tracking not available
|
|
3952
4333
|
}
|
|
3953
|
-
|
|
4334
|
+
|
|
4335
|
+
if (tracking.prefetchCount <= 1) {
|
|
4336
|
+
return; // One-message window: 0 % or 100 %, and 100 % is the working state.
|
|
4337
|
+
}
|
|
4338
|
+
|
|
3954
4339
|
const utilization = tracking.inFlight / tracking.prefetchCount;
|
|
3955
4340
|
|
|
3956
4341
|
if (utilization >= this._prefetchUtilizationThreshold) {
|