@onlineapps/mq-client-core 4.2.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +623 -0
- package/README.md +99 -11
- package/package.json +1 -1
- package/src/BaseClient.js +105 -29
- package/src/config/configSchema.js +21 -4
- package/src/config/defaultConfig.js +14 -4
- package/src/config/deliveryPolicy.js +86 -6
- package/src/config/queueConfig.js +68 -39
- package/src/config.js +5 -1
- package/src/defaults.js +7 -1
- package/src/index.js +41 -4
- package/src/layers/PublishLayer.js +113 -5
- package/src/monitoring-publish.js +89 -40
- package/src/scheduler-request.js +32 -11
- package/src/transports/rabbitmqClient.js +1237 -316
- package/src/utils/decodeMessage.js +123 -0
- package/src/utils/publishErrors.js +256 -17
- package/jest.config.js +0 -15
- package/jest.integration.config.js +0 -16
- package/src/workers/RecoveryWorker.js +0 -130
|
@@ -7,14 +7,22 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
const amqp = require('amqplib');
|
|
10
|
+
// amqplib's own class for "this handle is in a terminal state". It is what
|
|
11
|
+
// `close()` on a closed connection and `createChannel()` on one answer with, and it
|
|
12
|
+
// is read instead of the sentence those answers carry (d.696).
|
|
13
|
+
const { IllegalOperationError } = require('amqplib');
|
|
10
14
|
const EventEmitter = require('events');
|
|
11
15
|
const {
|
|
16
|
+
AMQP_NOT_FOUND,
|
|
17
|
+
AMQP_PRECONDITION_FAILED,
|
|
12
18
|
TransientPublishError,
|
|
13
19
|
PermanentPublishError,
|
|
14
20
|
QueueNotFoundError,
|
|
15
21
|
classifyPublishError,
|
|
22
|
+
collateralCloseMessage,
|
|
16
23
|
missingInfrastructureQueueMessage,
|
|
17
24
|
missingQueueDefinitionMessage,
|
|
25
|
+
refusedSendMessage,
|
|
18
26
|
} = require('../utils/publishErrors');
|
|
19
27
|
const {
|
|
20
28
|
ConnectionError,
|
|
@@ -28,7 +36,6 @@ const {
|
|
|
28
36
|
} = require('../utils/errorHandler');
|
|
29
37
|
const { assertLogger } = require('@onlineapps/logger-contract');
|
|
30
38
|
const PublishLayer = require('../layers/PublishLayer');
|
|
31
|
-
const RecoveryWorker = require('../workers/RecoveryWorker');
|
|
32
39
|
const PublishMonitor = require('../monitoring/PublishMonitor');
|
|
33
40
|
const runtimeCfg = require('../config');
|
|
34
41
|
const defaultConfig = require('../config/defaultConfig');
|
|
@@ -278,6 +285,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
278
285
|
this._channel = null; // ConfirmChannel for publish operations
|
|
279
286
|
this._queueChannel = null; // Regular channel for queue operations (assertQueue, checkQueue)
|
|
280
287
|
this._consumerChannel = null; // Dedicated channel for consume operations
|
|
288
|
+
// A SECOND ConfirmChannel, for the client's own telemetry (d.724). It is created on
|
|
289
|
+
// the first telemetry publish and never before: a client that publishes no telemetry
|
|
290
|
+
// must not hold a channel for it. See `_ensurePublisherChannel()` § roles.
|
|
291
|
+
this._telemetryChannel = null;
|
|
281
292
|
|
|
282
293
|
// Track active consumers for re-registration after channel recreation
|
|
283
294
|
this._activeConsumers = new Map(); // queue -> { handler, options, consumerTag }
|
|
@@ -413,16 +424,6 @@ class RabbitMQClient extends EventEmitter {
|
|
|
413
424
|
},
|
|
414
425
|
});
|
|
415
426
|
|
|
416
|
-
// Recovery worker — connection recovery, and the report of a queue that does
|
|
417
|
-
// not exist. It creates none: queue creation left it with the two callbacks
|
|
418
|
-
// that fed it (d.419) and with `recoveryScope`, the permission they were
|
|
419
|
-
// filtered by (d.514). Which rule the report names is decided by the
|
|
420
|
-
// ownership class in the error, not by who built this client.
|
|
421
|
-
this._recoveryWorker = new RecoveryWorker({
|
|
422
|
-
client: this,
|
|
423
|
-
logger: this._logger,
|
|
424
|
-
});
|
|
425
|
-
|
|
426
427
|
// Publish monitor (metriky) — counts only, reports through return values,
|
|
427
428
|
// takes no logger (see PublishMonitor's header).
|
|
428
429
|
this._publishMonitor = new PublishMonitor();
|
|
@@ -456,16 +457,12 @@ class RabbitMQClient extends EventEmitter {
|
|
|
456
457
|
this._publishMonitor.trackBuffered();
|
|
457
458
|
});
|
|
458
459
|
|
|
459
|
-
//
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
// Recovery failed - error already thrown
|
|
466
|
-
}
|
|
467
|
-
}
|
|
468
|
-
});
|
|
460
|
+
// A missing queue is answered ONCE, by the refusal of the publish (d.971): the
|
|
461
|
+
// caller gets `PublishError` with the `QueueNotFoundError` as its `cause`, which
|
|
462
|
+
// names the queue, its ownership class and the Fix. Until d.971 a second worker
|
|
463
|
+
// listened on this client's `error` channel for the same verdict and repeated it
|
|
464
|
+
// into the log; d.419 had taken its only other job (creating the queue) and
|
|
465
|
+
// d.811b its input.
|
|
469
466
|
}
|
|
470
467
|
|
|
471
468
|
/**
|
|
@@ -483,8 +480,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
483
480
|
}
|
|
484
481
|
|
|
485
482
|
/**
|
|
486
|
-
*
|
|
487
|
-
*
|
|
483
|
+
* The state of every channel this client holds — the keys ARE the list:
|
|
484
|
+
* `publisher`, `queue`, `consumer` and `telemetry` (the client's own events,
|
|
485
|
+
* d.724), each as `{ exists, closed, ready }`. The telemetry channel opens on the
|
|
486
|
+
* first telemetry publish and not before; until then it reads the way any channel
|
|
487
|
+
* not yet opened reads, `{ exists: false, closed: true, ready: false }`. It was
|
|
488
|
+
* missing from this report until d.1007.
|
|
489
|
+
*
|
|
490
|
+
* @returns {{publisher: Object, queue: Object, consumer: Object, telemetry: Object}}
|
|
488
491
|
*/
|
|
489
492
|
getChannelState() {
|
|
490
493
|
return {
|
|
@@ -502,6 +505,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
502
505
|
exists: !!this._consumerChannel,
|
|
503
506
|
closed: !this._isChannelAlive(this._consumerChannel),
|
|
504
507
|
ready: this._isChannelAlive(this._consumerChannel)
|
|
508
|
+
},
|
|
509
|
+
telemetry: {
|
|
510
|
+
exists: !!this._telemetryChannel,
|
|
511
|
+
closed: !this._isChannelAlive(this._telemetryChannel),
|
|
512
|
+
ready: this._isChannelAlive(this._telemetryChannel)
|
|
505
513
|
}
|
|
506
514
|
};
|
|
507
515
|
}
|
|
@@ -524,6 +532,32 @@ class RabbitMQClient extends EventEmitter {
|
|
|
524
532
|
return !!channel && channel._alive === true;
|
|
525
533
|
}
|
|
526
534
|
|
|
535
|
+
/**
|
|
536
|
+
* Did this failure to open a channel only report the connection going away?
|
|
537
|
+
*
|
|
538
|
+
* THE rule for "a refused channel open is transient" (d.1000, d.1003, d.1003b, d.1003c)
|
|
539
|
+
* — one body, read by `_ensurePublisherChannel()` and by the consumer and queue
|
|
540
|
+
* channels' close handlers. Two carriers, neither of them a sentence:
|
|
541
|
+
*
|
|
542
|
+
* - amqplib's CLASS: on a closing or closed connection `createChannel()` /
|
|
543
|
+
* `createConfirmChannel()` answer `IllegalOperationError` (`'Connection closing'`,
|
|
544
|
+
* `'Connection closed (…)'`), no `code` — measured on amqplib 0.10.9 over a live
|
|
545
|
+
* broker (d.1000);
|
|
546
|
+
* - this client's STATE: a connection that dies WHILE the channel opens rejects the
|
|
547
|
+
* pending RPC with a bare `Error('Channel ended, no reply will be forthcoming')`,
|
|
548
|
+
* no class, no code — but by then the connection has already emitted `'close'`
|
|
549
|
+
* and `_onConnectionDied()` has dropped `_connectionAlive`, the one liveness
|
|
550
|
+
* source (measured over a live broker, d.1003: 8 of 12 broker-side closes during
|
|
551
|
+
* the open).
|
|
552
|
+
*
|
|
553
|
+
* @param {*} amqpError - the error amqplib answered the open with
|
|
554
|
+
* @returns {boolean}
|
|
555
|
+
* @private
|
|
556
|
+
*/
|
|
557
|
+
_connectionIsGoingAway(amqpError) {
|
|
558
|
+
return amqpError instanceof IllegalOperationError || !this._connectionAlive;
|
|
559
|
+
}
|
|
560
|
+
|
|
527
561
|
/**
|
|
528
562
|
* THE definition of "this client's consumer is attached to that queue".
|
|
529
563
|
*
|
|
@@ -549,6 +583,37 @@ class RabbitMQClient extends EventEmitter {
|
|
|
549
583
|
return typeof consumerInfo?.consumerTag === 'string' && consumerInfo.consumerTag !== '';
|
|
550
584
|
}
|
|
551
585
|
|
|
586
|
+
/**
|
|
587
|
+
* The tracked queues whose consumer is NOT attached (`_isConsumerAttached()`), in
|
|
588
|
+
* tracking order — what a recreate summary names (d.1003d).
|
|
589
|
+
*
|
|
590
|
+
* @returns {string[]}
|
|
591
|
+
* @private
|
|
592
|
+
*/
|
|
593
|
+
_unattachedConsumerQueues() {
|
|
594
|
+
const unattached = [];
|
|
595
|
+
for (const [queue, consumerInfo] of this._activeConsumers.entries()) {
|
|
596
|
+
if (!this._isConsumerAttached(consumerInfo)) {
|
|
597
|
+
unattached.push(queue);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
return unattached;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* `_ensureConsumerChannel()`'s answer for a channel that is already alive: every
|
|
605
|
+
* tracked consumer is counted by whether it is ATTACHED, not assumed re-registered
|
|
606
|
+
* (d.1003d). `failed` is a tracked consumer without an attachment — the state a
|
|
607
|
+
* refused re-registration leaves (d.262 keeps it tracked, d.386b drops its tag).
|
|
608
|
+
*
|
|
609
|
+
* @returns {{reRegistered: number, failed: number}}
|
|
610
|
+
* @private
|
|
611
|
+
*/
|
|
612
|
+
_consumerAttachmentCounts() {
|
|
613
|
+
const failed = this._unattachedConsumerQueues().length;
|
|
614
|
+
return { reRegistered: this._activeConsumers.size - failed, failed };
|
|
615
|
+
}
|
|
616
|
+
|
|
552
617
|
/**
|
|
553
618
|
* Get current consumer state.
|
|
554
619
|
*
|
|
@@ -575,17 +640,32 @@ class RabbitMQClient extends EventEmitter {
|
|
|
575
640
|
}
|
|
576
641
|
|
|
577
642
|
/**
|
|
578
|
-
* Get current buffer state
|
|
579
|
-
*
|
|
643
|
+
* Get current buffer state.
|
|
644
|
+
*
|
|
645
|
+
* ONE buffer, read ONCE. `MessageBuffer.size()` IS `_inMemory.size()`
|
|
646
|
+
* (`buffer/MessageBuffer.js`), so both numbers describe the same messages and
|
|
647
|
+
* both come from the same call.
|
|
648
|
+
*
|
|
649
|
+
* Until d.645 the second one came off `_buffer._inMemory?.size() || 0`, and
|
|
650
|
+
* neither operator could do anything but hide: the `?.` cannot fire, because
|
|
651
|
+
* `MessageBuffer`'s constructor always builds `_inMemory` and the line above
|
|
652
|
+
* already calls `_buffer.size()` without a guard — a buffer that is not there
|
|
653
|
+
* has thrown before this expression runs. What the pair DID produce, for a
|
|
654
|
+
* buffer whose internals were not the assumed ones, was `inMemory: 0` beside a
|
|
655
|
+
* non-zero `size`: "held, but nowhere", a state no real buffer can be in,
|
|
656
|
+
* reported without a word and indistinguishable from an empty buffer for
|
|
657
|
+
* whoever reads it (`architecture-principles.md` §3 No Fallbacks, §8 Explicit
|
|
658
|
+
* Over Implicit; `automation-gates.md` §5).
|
|
659
|
+
*
|
|
660
|
+
* @returns {{size: number, inMemory: number}} How many messages are held
|
|
580
661
|
*/
|
|
581
662
|
getBufferState() {
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
};
|
|
663
|
+
// No `persistent` count. There is one buffer, so a second number could
|
|
664
|
+
// only ever be 0 — a field that reports a mechanism that does not exist
|
|
665
|
+
// (d.343, `automation-gates.md` §5).
|
|
666
|
+
const held = this._publishLayer._buffer.size();
|
|
667
|
+
|
|
668
|
+
return { size: held, inMemory: held };
|
|
589
669
|
}
|
|
590
670
|
|
|
591
671
|
/**
|
|
@@ -648,11 +728,31 @@ class RabbitMQClient extends EventEmitter {
|
|
|
648
728
|
* @private
|
|
649
729
|
*/
|
|
650
730
|
_connectionSocketOptions() {
|
|
651
|
-
const connectionName
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
731
|
+
const explicit = this._config.connectionName || this._config.clientName;
|
|
732
|
+
|
|
733
|
+
if (explicit) {
|
|
734
|
+
return { clientProperties: { connection_name: explicit } };
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
// Composed from THIS CLIENT'S OWN config, not from the runtime config (d.770). The
|
|
738
|
+
// name is the caller's fact and the constructor already resolved it — asking the
|
|
739
|
+
// runtime config here was a second reader of it, and the one that answered the
|
|
740
|
+
// fabricated module default `oa-service` when the caller had named itself properly.
|
|
741
|
+
if (typeof this._config.serviceName !== 'string' || this._config.serviceName === '') {
|
|
742
|
+
throw new ValidationError(
|
|
743
|
+
'[RabbitMQClient] Missing configuration - the connection has no name and no '
|
|
744
|
+
+ 'serviceName to compose one from. '
|
|
745
|
+
+ 'Expected: connectionName, clientName or serviceName in this client\'s config '
|
|
746
|
+
+ '(BaseClient resolves serviceName from the explicit key or ENV SERVICE_NAME). '
|
|
747
|
+
+ 'Fix: pass serviceName to the client, or set SERVICE_NAME in env-active/*.env.'
|
|
748
|
+
);
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
return {
|
|
752
|
+
clientProperties: {
|
|
753
|
+
connection_name: `oa-client:${this._config.serviceName}:${process.pid}`
|
|
754
|
+
}
|
|
755
|
+
};
|
|
656
756
|
}
|
|
657
757
|
|
|
658
758
|
/**
|
|
@@ -670,8 +770,17 @@ class RabbitMQClient extends EventEmitter {
|
|
|
670
770
|
* @private
|
|
671
771
|
*/
|
|
672
772
|
_attachConnectionHandlers(connection) {
|
|
773
|
+
// A fault is RECORDED on the connection, the way `_attach*ChannelHandlers()` has
|
|
774
|
+
// recorded a channel's since d.264 — and for the same reason (d.696). Once a
|
|
775
|
+
// connection is closed, amqplib answers every operation on it with the same
|
|
776
|
+
// `IllegalOperationError`, whether it got there by our `close()` or by a broker
|
|
777
|
+
// fault; the only difference is a sentence it composes (`Connection.toClosed()`, `lib/connection.js`).
|
|
778
|
+
// Which of the two happened is this client's own knowledge, so it is kept as state
|
|
779
|
+
// rather than read back out of a message (`error-handling.md` §6).
|
|
780
|
+
connection._closeError = null;
|
|
673
781
|
connection.on('error', (err) => {
|
|
674
782
|
this._connectionAlive = false;
|
|
783
|
+
connection._closeError = err;
|
|
675
784
|
this._logger.error('[RabbitMQClient] Connection error:', err.message);
|
|
676
785
|
this.emit('error', err);
|
|
677
786
|
});
|
|
@@ -732,6 +841,15 @@ class RabbitMQClient extends EventEmitter {
|
|
|
732
841
|
_isBrokerRefusal(err) {
|
|
733
842
|
if (!err) return false;
|
|
734
843
|
if (REFUSAL_REPLY_CODES.includes(err.code)) return true;
|
|
844
|
+
// THE ONE TEXT READING THIS FILE KEEPS, and it is kept because amqplib leaves nothing
|
|
845
|
+
// else. A refused HANDSHAKE is the carrier: measured 2026-09-20 against the dev broker
|
|
846
|
+
// with a wrong password, amqplib answers `connect()` with a plain `Error` whose own
|
|
847
|
+
// properties are `stack` and `message` — NO `code`, no class of its own — and whose
|
|
848
|
+
// sentence is `'Handshake terminated by server: 403 (ACCESS-REFUSED) with message …'`.
|
|
849
|
+
// The branch above cannot see it, so without this one a refused login would read as a
|
|
850
|
+
// transient connection fault and the client would retry credentials that can never
|
|
851
|
+
// work. Do not "finish the d.677 work" by deleting it: measure first, and if amqplib
|
|
852
|
+
// ever puts a code on that error, the branch above already covers it (d.696).
|
|
735
853
|
const message = typeof err.message === 'string' ? err.message : '';
|
|
736
854
|
return REFUSAL_REPLY_CODES.some((code) => message.includes(`server: ${code} (`));
|
|
737
855
|
}
|
|
@@ -1273,6 +1391,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1273
1391
|
_onConnectionDied() {
|
|
1274
1392
|
this._connectionAlive = false;
|
|
1275
1393
|
this._channel = null;
|
|
1394
|
+
this._telemetryChannel = null;
|
|
1276
1395
|
this._queueChannel = null;
|
|
1277
1396
|
this._consumerChannel = null;
|
|
1278
1397
|
this._connection = null;
|
|
@@ -1416,6 +1535,38 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1416
1535
|
}
|
|
1417
1536
|
}
|
|
1418
1537
|
|
|
1538
|
+
/**
|
|
1539
|
+
* Helper to set a trackable interval — the SAME evidence, the other kind of timer.
|
|
1540
|
+
*
|
|
1541
|
+
* A repeating timer is the one that most needs to be in it: `_setTimeout()` deletes its
|
|
1542
|
+
* handle when it fires, an interval never fires last, so nothing ever removes it by
|
|
1543
|
+
* itself. Until d.759e the two repeating timers this client arms — the health monitor
|
|
1544
|
+
* and the prefetch monitor — were armed with the bare global, so `_activeTimers` could
|
|
1545
|
+
* not see them and `_clearAllTimers()` could not end them. Measured (d.759d): a client
|
|
1546
|
+
* that had consumed held a live `setInterval(10000)` which only `disconnect()` could
|
|
1547
|
+
* end; a unit suite that left four behind never let its runner exit, while
|
|
1548
|
+
* `expect(client._activeTimers.size).toBe(0)` — the assertion five suites of this
|
|
1549
|
+
* package read — stayed green throughout.
|
|
1550
|
+
*
|
|
1551
|
+
* @private
|
|
1552
|
+
*/
|
|
1553
|
+
_setInterval(callback, ms) {
|
|
1554
|
+
const timer = setInterval(callback, ms);
|
|
1555
|
+
this._activeTimers.add(timer);
|
|
1556
|
+
return timer;
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
/**
|
|
1560
|
+
* Helper to clear a trackable interval
|
|
1561
|
+
* @private
|
|
1562
|
+
*/
|
|
1563
|
+
_clearInterval(timer) {
|
|
1564
|
+
if (timer) {
|
|
1565
|
+
clearInterval(timer);
|
|
1566
|
+
this._activeTimers.delete(timer);
|
|
1567
|
+
}
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1419
1570
|
/**
|
|
1420
1571
|
* Race an `amqp.connect()` handshake against its `connectTimeout`, and own the
|
|
1421
1572
|
* handshake the timeout abandons.
|
|
@@ -1482,9 +1633,21 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1482
1633
|
|
|
1483
1634
|
/**
|
|
1484
1635
|
* Clear all active timers
|
|
1636
|
+
*
|
|
1637
|
+
* The repeating ones go through their own owners first, because each of them holds a
|
|
1638
|
+
* REFERENCE that has to be dropped together with the handle: a sweep that ended the
|
|
1639
|
+
* timer and left `_healthCheckInterval` / `_prefetchCheckTimer` pointing at the dead
|
|
1640
|
+
* handle would make the next `_start*Monitoring()` return early ("already started") and
|
|
1641
|
+
* monitoring would be off for the rest of the client's life, silently. The rest is a
|
|
1642
|
+
* plain sweep — `clearTimeout()` ends an interval handle as well (measured on this
|
|
1643
|
+
* runtime: a `setInterval` cleared with `clearTimeout` never ticks and reports
|
|
1644
|
+
* `_destroyed`), so one loop covers both kinds.
|
|
1645
|
+
*
|
|
1485
1646
|
* @private
|
|
1486
1647
|
*/
|
|
1487
1648
|
_clearAllTimers() {
|
|
1649
|
+
this._stopHealthMonitoring();
|
|
1650
|
+
this._stopPrefetchMonitoring();
|
|
1488
1651
|
for (const timer of this._activeTimers) {
|
|
1489
1652
|
clearTimeout(timer);
|
|
1490
1653
|
}
|
|
@@ -1544,7 +1707,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1544
1707
|
this._channel._type = 'publisher';
|
|
1545
1708
|
|
|
1546
1709
|
// Attach event handlers
|
|
1547
|
-
this._attachPublisherChannelHandlers(this._channel);
|
|
1710
|
+
this._attachPublisherChannelHandlers(this._channel, 'publisher');
|
|
1548
1711
|
|
|
1549
1712
|
// Create a separate regular channel for queue operations (assertQueue, checkQueue)
|
|
1550
1713
|
// This avoids RPC reply queue issues with ConfirmChannel.assertQueue()
|
|
@@ -1609,7 +1772,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1609
1772
|
throw err;
|
|
1610
1773
|
} finally {
|
|
1611
1774
|
if (connectTimeoutTimer) {
|
|
1612
|
-
|
|
1775
|
+
// Through the evidence, like every other ending of this timer: a bare
|
|
1776
|
+
// `clearTimeout()` here left the handle in `_activeTimers`, so after a FAILED
|
|
1777
|
+
// connect the client reported one armed timer it no longer had (měřeno d.759e:
|
|
1778
|
+
// size 1 with nothing armed).
|
|
1779
|
+
this._clearTimeout(connectTimeoutTimer);
|
|
1613
1780
|
connectTimeoutTimer = null;
|
|
1614
1781
|
}
|
|
1615
1782
|
}
|
|
@@ -1727,13 +1894,49 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1727
1894
|
}
|
|
1728
1895
|
|
|
1729
1896
|
/**
|
|
1730
|
-
*
|
|
1731
|
-
*
|
|
1897
|
+
* The ConfirmChannel of one ROLE, open — recreated when the last one died.
|
|
1898
|
+
*
|
|
1899
|
+
* TWO ROLES, TWO CHANNELS (d.724):
|
|
1900
|
+
*
|
|
1901
|
+
* - `'publisher'` — the caller's messages. `this._channel`, created at connect time
|
|
1902
|
+
* with everything else.
|
|
1903
|
+
* - `'telemetry'` — the client's OWN events, the ones nobody asked it to send
|
|
1904
|
+
* (`src/monitoring-publish.js`, reached through `BaseClient.publishTelemetry()`).
|
|
1905
|
+
* `this._telemetryChannel`, created on the first such publish and not before.
|
|
1906
|
+
*
|
|
1907
|
+
* WHY THEY MAY NOT SHARE ONE (measured d.711, 3 of 7 runs of an unrelated suite red):
|
|
1908
|
+
* a channel dies of the FIRST refusal it meets, and takes every unconfirmed publish on
|
|
1909
|
+
* it along. The client's dead-letter telemetry publishes through
|
|
1910
|
+
* `monitoring.workflow.fanout`; in a vhost where that exchange is not declared the
|
|
1911
|
+
* broker answers `404` and closes the channel — and the caller's business message,
|
|
1912
|
+
* still waiting for its confirm on the same channel, died with it. d.723 stopped that
|
|
1913
|
+
* from being reported as the business message's own refusal; this separation stops it
|
|
1914
|
+
* from happening at all. The module that sends the telemetry states the rule it could
|
|
1915
|
+
* not keep on its own: "Fail-safe about PUBLISHING" (`src/monitoring-publish.js`
|
|
1916
|
+
* § Principles) — fail-safe for the event is worth nothing if it is not also fail-safe
|
|
1917
|
+
* for everything beside it.
|
|
1918
|
+
*
|
|
1919
|
+
* IT IS A CONFIRM CHANNEL, NOT A PLAIN ONE, because the telemetry's answer to its
|
|
1920
|
+
* caller is `true`/`false`, and only a confirm can tell those apart: on a plain channel
|
|
1921
|
+
* `publish()` returns as soon as the bytes are written, so every event would be
|
|
1922
|
+
* reported as sent — including the ones the broker refuses, which is exactly the case
|
|
1923
|
+
* this separation exists for.
|
|
1924
|
+
*
|
|
1925
|
+
* @param {'publisher'|'telemetry'} [role] - Whose channel to ensure.
|
|
1732
1926
|
* @private
|
|
1733
1927
|
* @returns {Promise<void>}
|
|
1734
1928
|
*/
|
|
1735
|
-
async _ensurePublisherChannel() {
|
|
1736
|
-
if (
|
|
1929
|
+
async _ensurePublisherChannel(role = 'publisher') {
|
|
1930
|
+
if (role !== 'publisher' && role !== 'telemetry') {
|
|
1931
|
+
throw new Error(
|
|
1932
|
+
'[RabbitMQClient] Unknown send channel role - '
|
|
1933
|
+
+ `Expected: 'publisher' (the caller's messages) or 'telemetry' (the client's own events), `
|
|
1934
|
+
+ `got ${JSON.stringify(role)}. `
|
|
1935
|
+
+ 'Fix: publish through publish() or publishTelemetry(); nothing else picks a channel.'
|
|
1936
|
+
);
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
if (this._isChannelAlive(this._sendChannelOf(role))) {
|
|
1737
1940
|
return; // Channel is good
|
|
1738
1941
|
}
|
|
1739
1942
|
|
|
@@ -1748,7 +1951,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1748
1951
|
await this._waitForReconnection();
|
|
1749
1952
|
|
|
1750
1953
|
// After reconnection, try again
|
|
1751
|
-
if (this._isChannelAlive(this.
|
|
1954
|
+
if (this._isChannelAlive(this._sendChannelOf(role))) {
|
|
1752
1955
|
return; // Channel was recreated during reconnection
|
|
1753
1956
|
}
|
|
1754
1957
|
// If still no channel, connection might have failed - check again
|
|
@@ -1776,26 +1979,47 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1776
1979
|
|
|
1777
1980
|
try {
|
|
1778
1981
|
// Create new ConfirmChannel
|
|
1779
|
-
|
|
1780
|
-
|
|
1982
|
+
const channel = await this._connection.createConfirmChannel();
|
|
1983
|
+
|
|
1781
1984
|
// Track channel metadata for debugging
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1985
|
+
channel._createdAt = new Date().toISOString();
|
|
1986
|
+
channel._closeReason = null;
|
|
1987
|
+
channel._lastOperation = `Recreated via _ensurePublisherChannel('${role}')`;
|
|
1988
|
+
channel._type = role;
|
|
1989
|
+
|
|
1990
|
+
this._setSendChannelOf(role, channel);
|
|
1991
|
+
|
|
1787
1992
|
// Attach event handlers
|
|
1788
|
-
this._attachPublisherChannelHandlers(
|
|
1789
|
-
|
|
1790
|
-
this._logger.info(
|
|
1993
|
+
this._attachPublisherChannelHandlers(channel, role);
|
|
1994
|
+
|
|
1995
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] ✓ ${role} channel recreated`);
|
|
1791
1996
|
} catch (err) {
|
|
1792
|
-
this.
|
|
1997
|
+
this._setSendChannelOf(role, null);
|
|
1793
1998
|
|
|
1794
1999
|
const msg = err && err.message ? err.message : String(err);
|
|
1795
2000
|
|
|
1796
|
-
//
|
|
1797
|
-
//
|
|
1798
|
-
|
|
2001
|
+
// Spojení odchází — rozhoduje TŘÍDA amqplibu, ne věta (d.1000, poslední textová
|
|
2002
|
+
// větev po d.676–d.711). Změřeno nad amqplib 0.10.9 a živým brokerem:
|
|
2003
|
+
// `createConfirmChannel()` nad spojením, jehož `close()` začalo, odpoví
|
|
2004
|
+
// `IllegalOperationError('Connection closing')`, nad zavřeným
|
|
2005
|
+
// `IllegalOperationError('Connection closed (…)')` — bez `code`, jiný nosič
|
|
2006
|
+
// nemá (`lib/connection.js` `invalidateSend()` / `toClosed()`). Uvnitř
|
|
2007
|
+
// `createConfirmChannel()` tu třídu nevydá nic jiného než odcházející spojení
|
|
2008
|
+
// (`'Channel closed'` nového kanálu jen přes spojení, které zavírá své kanály).
|
|
2009
|
+
//
|
|
2010
|
+
// Druhý nosič je STAV (d.1003): spadne-li spojení UPROSTŘED otevírání,
|
|
2011
|
+
// amqplib odmítne čekající RPC holým `Error('Channel ended, no reply will be
|
|
2012
|
+
// forthcoming')` (`_rejectPending()`, `lib/channel.js`) — bez třídy, bez `code`.
|
|
2013
|
+
// Změřeno nad živým brokerem: zavření brokerem (management API) během
|
|
2014
|
+
// `createConfirmChannel()` 8× z 12, `close()` o `setImmediate` později 3× ze 3;
|
|
2015
|
+
// pokaždé spojení UŽ ohlásilo `'close'`, takže `_onConnectionDied()` shodil
|
|
2016
|
+
// `_connectionAlive` — jediný zdroj živosti (d.260) — dřív, než sem odmítnutí
|
|
2017
|
+
// doběhlo. Věta nerozhoduje: táž věta nad živým spojením zůstává trvalá.
|
|
2018
|
+
//
|
|
2019
|
+
// Stav je přechodný: PublishLayer publikaci podrží a zopakuje po reconnectu.
|
|
2020
|
+
// Pravidlo má jedno tělo, `_connectionIsGoingAway()` — čte ho i obnova
|
|
2021
|
+
// consumer kanálu (d.1003b).
|
|
2022
|
+
if (this._connectionIsGoingAway(err)) {
|
|
1799
2023
|
this._logger.warn(
|
|
1800
2024
|
`[RabbitMQClient] [mq-client-core] Failed to recreate publisher channel (connection closing): ${msg}`
|
|
1801
2025
|
);
|
|
@@ -1809,7 +2033,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1809
2033
|
|
|
1810
2034
|
// Ostatní chyby považujeme za permanentní problém na úrovni kanálu/spojení.
|
|
1811
2035
|
throw new PermanentPublishError(
|
|
1812
|
-
`[RabbitMQClient] Failed to recreate
|
|
2036
|
+
`[RabbitMQClient] Failed to recreate ${role} channel: ${msg} - `
|
|
1813
2037
|
+ 'Expected: createConfirmChannel() to succeed on an open connection. '
|
|
1814
2038
|
+ 'Fix: read error.cause for the broker reason; a channel-level refusal here is not retried, so the connection must be re-established.',
|
|
1815
2039
|
err
|
|
@@ -1870,12 +2094,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1870
2094
|
* Ensure consumer channel exists and is open
|
|
1871
2095
|
* Auto-recreates and re-registers all consumers if closed
|
|
1872
2096
|
* @private
|
|
1873
|
-
* @returns {Promise<{reRegistered: number, failed: number}>} -
|
|
2097
|
+
* @returns {Promise<{reRegistered: number, failed: number}>} - how many tracked
|
|
2098
|
+
* consumers are attached, and how many are tracked without being attached. On a
|
|
2099
|
+
* channel that was already alive both are MEASURED (`_consumerAttachmentCounts()`,
|
|
2100
|
+
* d.1003d), never assumed: until d.1003d this branch answered
|
|
2101
|
+
* `_activeConsumers.size` as re-registered, counting a consumer whose
|
|
2102
|
+
* re-registration the broker had refused (tracked, no tag — d.262, d.386b).
|
|
1874
2103
|
*/
|
|
1875
2104
|
async _ensureConsumerChannel() {
|
|
1876
2105
|
if (this._isChannelAlive(this._consumerChannel)) {
|
|
1877
|
-
|
|
1878
|
-
return { reRegistered: this._activeConsumers.size, failed: 0 };
|
|
2106
|
+
return this._consumerAttachmentCounts();
|
|
1879
2107
|
}
|
|
1880
2108
|
|
|
1881
2109
|
if (!this._connectionAlive) {
|
|
@@ -1898,8 +2126,9 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1898
2126
|
}
|
|
1899
2127
|
|
|
1900
2128
|
if (this._isChannelAlive(this._consumerChannel)) {
|
|
1901
|
-
// The recovery recreated it and re-
|
|
1902
|
-
|
|
2129
|
+
// The recovery recreated it and ran the re-registration; what it attached is
|
|
2130
|
+
// measured, not assumed (d.1003d).
|
|
2131
|
+
return this._consumerAttachmentCounts();
|
|
1903
2132
|
}
|
|
1904
2133
|
if (!this._connectionAlive) {
|
|
1905
2134
|
return { reRegistered: 0, failed: this._activeConsumers.size };
|
|
@@ -1921,8 +2150,20 @@ class RabbitMQClient extends EventEmitter {
|
|
|
1921
2150
|
|
|
1922
2151
|
// Re-register all active consumers
|
|
1923
2152
|
const { reRegistered, failed } = await this._reRegisterConsumers();
|
|
1924
|
-
|
|
1925
|
-
|
|
2153
|
+
|
|
2154
|
+
// A recreate that left a consumer unattached is not a success line (d.1003d):
|
|
2155
|
+
// the ✓ at INFO read as "all good" while a queue had nobody reading it. The
|
|
2156
|
+
// refusal itself is already an ERROR per queue (`_reRegisterConsumer()`); this
|
|
2157
|
+
// summary names which queues are left without a consumer.
|
|
2158
|
+
if (failed > 0) {
|
|
2159
|
+
const unattached = this._unattachedConsumerQueues();
|
|
2160
|
+
this._logger.warn(
|
|
2161
|
+
`[RabbitMQClient] [mq-client-core] Consumer channel recreated, but ${unattached.length} consumer(s) are not attached: `
|
|
2162
|
+
+ `${unattached.join(', ')} (re-registered: ${reRegistered}, failed: ${failed})`
|
|
2163
|
+
);
|
|
2164
|
+
} else {
|
|
2165
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] ✓ Consumer channel recreated (re-registered: ${reRegistered}, failed: ${failed})`);
|
|
2166
|
+
}
|
|
1926
2167
|
return { reRegistered, failed };
|
|
1927
2168
|
} catch (err) {
|
|
1928
2169
|
this._consumerChannel = null;
|
|
@@ -2128,7 +2369,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2128
2369
|
return; // Already started
|
|
2129
2370
|
}
|
|
2130
2371
|
|
|
2131
|
-
this._healthCheckInterval =
|
|
2372
|
+
this._healthCheckInterval = this._setInterval(async () => {
|
|
2132
2373
|
try {
|
|
2133
2374
|
await this._performHealthCheck();
|
|
2134
2375
|
} catch (err) {
|
|
@@ -2147,7 +2388,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2147
2388
|
*/
|
|
2148
2389
|
_stopHealthMonitoring() {
|
|
2149
2390
|
if (this._healthCheckInterval) {
|
|
2150
|
-
|
|
2391
|
+
this._clearInterval(this._healthCheckInterval);
|
|
2151
2392
|
this._healthCheckInterval = null;
|
|
2152
2393
|
this._logger.info('[RabbitMQClient] [mq-client-core] Health monitoring stopped');
|
|
2153
2394
|
}
|
|
@@ -2287,7 +2528,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2287
2528
|
health.issues.push(`Queue ${queue} does not exist`);
|
|
2288
2529
|
}
|
|
2289
2530
|
} catch (err) {
|
|
2290
|
-
if (err.code ===
|
|
2531
|
+
if (err.code === AMQP_NOT_FOUND) {
|
|
2291
2532
|
health.queues.missing.push(queue);
|
|
2292
2533
|
health.healthy = false;
|
|
2293
2534
|
health.issues.push(`Queue ${queue} does not exist (404)`);
|
|
@@ -2465,6 +2706,23 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2465
2706
|
}
|
|
2466
2707
|
socketUnanswered = socketUnanswered || queueOutcome.status === 'unanswered';
|
|
2467
2708
|
|
|
2709
|
+
// The telemetry channel exists only when this client published telemetry (d.724);
|
|
2710
|
+
// `_closeWithinTeardownBudget()` takes `null` as "nothing to close", so the client
|
|
2711
|
+
// that never sent an event pays nothing for this line.
|
|
2712
|
+
const telemetryOutcome = await this._closeWithinTeardownBudget(
|
|
2713
|
+
'telemetry channel',
|
|
2714
|
+
this._telemetryChannel,
|
|
2715
|
+
remaining()
|
|
2716
|
+
);
|
|
2717
|
+
this._telemetryChannel = null;
|
|
2718
|
+
if (telemetryOutcome.status === 'refused') {
|
|
2719
|
+
// Reported, never raised: the client's own events are not the caller's business,
|
|
2720
|
+
// and a telemetry channel that would not close is not a reason to fail a disconnect
|
|
2721
|
+
// the caller asked for.
|
|
2722
|
+
this._logger.warn('[RabbitMQClient] Error closing telemetry channel:', telemetryOutcome.error.message);
|
|
2723
|
+
}
|
|
2724
|
+
socketUnanswered = socketUnanswered || telemetryOutcome.status === 'unanswered';
|
|
2725
|
+
|
|
2468
2726
|
const publisherOutcome = await this._closeWithinTeardownBudget(
|
|
2469
2727
|
'publisher channel',
|
|
2470
2728
|
this._channel,
|
|
@@ -2483,9 +2741,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2483
2741
|
);
|
|
2484
2742
|
this._connection = null;
|
|
2485
2743
|
if (connectionOutcome.status === 'refused') {
|
|
2486
|
-
//
|
|
2487
|
-
//
|
|
2488
|
-
|
|
2744
|
+
// An EXPECTED refusal is two facts, and both are first-hand (d.696): amqplib
|
|
2745
|
+
// answered with its terminal-state class — the connection was already closed — and
|
|
2746
|
+
// this client recorded no fault on it, so what closed it was a `close()`. A fault
|
|
2747
|
+
// recorded here means the connection died on its own and the refusal is worth
|
|
2748
|
+
// raising, exactly as it was while this read the sentence `'Connection closed (by
|
|
2749
|
+
// client)'` (amqplib composes that in `toClosed()`, `lib/connection.js`, and a
|
|
2750
|
+
// fault gets `'Connection closed (<the error>)'` instead).
|
|
2751
|
+
if (connectionOutcome.error instanceof IllegalOperationError && !connection._closeError) {
|
|
2489
2752
|
this._logger.info('[RabbitMQClient] Connection closed during disconnect (expected)');
|
|
2490
2753
|
} else {
|
|
2491
2754
|
this.emit('error', connectionOutcome.error);
|
|
@@ -2512,7 +2775,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2512
2775
|
*
|
|
2513
2776
|
* PUBLIC: this is the one operation a collaborator needs in order to declare a
|
|
2514
2777
|
* queue on this client — higher-level routing libraries (cookbook-router's
|
|
2515
|
-
* QueueManager)
|
|
2778
|
+
* QueueManager) alike. It owns the whole
|
|
2516
2779
|
* sequence, so nobody has to assemble it out of private members: ensure the
|
|
2517
2780
|
* queue channel, refuse a channel that is not alive, resolve the arguments
|
|
2518
2781
|
* from the central `queueConfig`, declare.
|
|
@@ -2729,6 +2992,322 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2729
2992
|
);
|
|
2730
2993
|
}
|
|
2731
2994
|
|
|
2995
|
+
/**
|
|
2996
|
+
* The broker's REFUSAL of this send, when that is what killed the channel — or `null`
|
|
2997
|
+
* when the channel died without the broker saying anything about this message.
|
|
2998
|
+
*
|
|
2999
|
+
* A send the broker refuses closes the channel with a numbered close frame. amqplib
|
|
3000
|
+
* puts that object on the channel's `error` event (`lib/channel.js`, `Channel.accept()`, case `ChannelClose`) and
|
|
3001
|
+
* answers the publish callback with the bare `new Error('channel closed')`
|
|
3002
|
+
* (the `Channel` constructor's `'close'` listener, `lib/channel.js`) — so the callback cannot see the reason, but the channel can:
|
|
3003
|
+
* `_attachPublisherChannelHandlers()` records it as `channel._closeError`, and does so
|
|
3004
|
+
* BEFORE the callback runs, because amqplib emits `error` first and only then `close`,
|
|
3005
|
+
* which is what rejects the unconfirmed publishes.
|
|
3006
|
+
*
|
|
3007
|
+
* WHOSE ANSWER IT IS, THOUGH, IS A SECOND QUESTION (d.723). A ConfirmChannel carries
|
|
3008
|
+
* more than one unconfirmed publish, and a numbered frame is an answer about exactly
|
|
3009
|
+
* ONE of them; amqplib answers all of them with that same object. Until d.723 this
|
|
3010
|
+
* method's own JSDoc called the reading "a fact about THIS send, first-hand" — it is
|
|
3011
|
+
* that only while this send is the only unconfirmed one on the channel. Measured
|
|
3012
|
+
* 2026-09-20, two publishes on one channel in a vhost of its own: one to an exchange
|
|
3013
|
+
* nobody declared, one to a `good.fanout` the same client had just declared. Both came
|
|
3014
|
+
* back `PermanentPublishError … Cannot publish to exchange "good.fanout": 404 NOT-FOUND
|
|
3015
|
+
* - nothing declares it`, while the frame underneath named `no exchange
|
|
3016
|
+
* 'no.such.exchange'`. A message the broker never refused was dropped, and its
|
|
3017
|
+
* publisher was sent to declare an exchange that was already there.
|
|
3018
|
+
*
|
|
3019
|
+
* So `_closeFrameNamesThisSend()` decides attribution before the class is decided at
|
|
3020
|
+
* all, and the sends it cannot claim get the collateral verdict
|
|
3021
|
+
* (`_collateralSendTransient()`), which is transient: they were never answered, so they
|
|
3022
|
+
* were never refused, and republishing them is right.
|
|
3023
|
+
*
|
|
3024
|
+
* So the reading is a fact about the send the frame is about:
|
|
3025
|
+
* - a NUMBER is an answer about this request. Measured against the dev broker,
|
|
3026
|
+
* `classId 60 / methodId 40` — `basic.publish` itself: `403 ACCESS-REFUSED` (an
|
|
3027
|
+
* account without `write` in the vhost) and `406 PRECONDITION-FAILED` (a `user_id`
|
|
3028
|
+
* property that is not the authenticated user). Repeating the request repeats the
|
|
3029
|
+
* answer, so it is PERMANENT — the same reading the queue-existence pre-check has
|
|
3030
|
+
* used for `405 RESOURCE_LOCKED` since d.683;
|
|
3031
|
+
* - `404 NOT-FOUND` is one of them, since d.693 (carrier B1: a publish through an
|
|
3032
|
+
* exchange nobody declared, measured `classId 60` on the send). It says something
|
|
3033
|
+
* the other codes do not — nothing DECLARED what was published to — but the
|
|
3034
|
+
* consequence is the same, because a publisher declares nothing and no reconnect
|
|
3035
|
+
* creates an exchange: `config/queueConfig.js` declares it and its owning service
|
|
3036
|
+
* asserts it. Retrying and buffering only held the message until its TTL dropped it.
|
|
3037
|
+
* It is NOT a `QueueNotFoundError`: that class belongs to the queue-existence
|
|
3038
|
+
* pre-check, and it carries a `queueName` and an ownership `kind` that
|
|
3039
|
+
* the caller reads from the refusal's `cause` — an exchange is neither;
|
|
3040
|
+
* - anything that is not a number is not a broker reply code. A socket error's
|
|
3041
|
+
* `'ECONNRESET'` says nothing about whether the message was stored, and reading it
|
|
3042
|
+
* as a refusal would drop messages a reconnect would have delivered.
|
|
3043
|
+
*
|
|
3044
|
+
* Until d.685a neither send leg read it. Both answered with their own `PublishError`,
|
|
3045
|
+
* which the classification reads as transient (`utils/publishErrors.js`), so a message
|
|
3046
|
+
* the broker refuses every single time was retried `publishMaxRetries` times and then
|
|
3047
|
+
* buffered — and `InMemoryBuffer.flush()` returns an unsuccessful flush to the buffer,
|
|
3048
|
+
* so it was re-offered on every reconnect until its TTL dropped it.
|
|
3049
|
+
*
|
|
3050
|
+
* @param {Object|null} channel - The channel that CARRIED the send, read before it was
|
|
3051
|
+
* replaced: `this._channel` may already be the successor by callback time.
|
|
3052
|
+
* @param {'queue'|'exchange'} kind - Which leg is asking, so the message names what the
|
|
3053
|
+
* publish was addressed to (`architecture-principles.md` §5).
|
|
3054
|
+
* @param {string} target - That queue or exchange.
|
|
3055
|
+
* @returns {PermanentPublishError|null}
|
|
3056
|
+
* @private
|
|
3057
|
+
*/
|
|
3058
|
+
_refusedSendPermanent(channel, kind, target) {
|
|
3059
|
+
const closeError = channel ? channel._closeError : null;
|
|
3060
|
+
const code = closeError ? closeError.code : undefined;
|
|
3061
|
+
|
|
3062
|
+
if (typeof code !== 'number') {
|
|
3063
|
+
return null;
|
|
3064
|
+
}
|
|
3065
|
+
|
|
3066
|
+
if (!this._closeFrameNamesThisSend(channel, target)) {
|
|
3067
|
+
return null;
|
|
3068
|
+
}
|
|
3069
|
+
|
|
3070
|
+
return new PermanentPublishError(refusedSendMessage(kind, target, code), closeError);
|
|
3071
|
+
}
|
|
3072
|
+
|
|
3073
|
+
/**
|
|
3074
|
+
* The verdict for a send that was on the channel when SOMEBODY ELSE'S refusal closed
|
|
3075
|
+
* it — or `null` when there was no numbered frame, or the frame was this send's own.
|
|
3076
|
+
*
|
|
3077
|
+
* Transient, and that is the whole point: the broker answered one message and never
|
|
3078
|
+
* answered this one, so nothing about this message was decided. `PublishLayer` retries
|
|
3079
|
+
* a `TransientPublishError` and buffers it meanwhile, which republishes it over the
|
|
3080
|
+
* channel's successor; the permanent verdict would have dropped it on the first attempt
|
|
3081
|
+
* (`src/layers/PublishLayer.js`).
|
|
3082
|
+
*
|
|
3083
|
+
* @param {Object|null} channel - The channel that CARRIED the send.
|
|
3084
|
+
* @param {'queue'|'exchange'} kind - Which leg is asking.
|
|
3085
|
+
* @param {string} target - That queue or exchange.
|
|
3086
|
+
* @returns {TransientPublishError|null}
|
|
3087
|
+
* @private
|
|
3088
|
+
*/
|
|
3089
|
+
/**
|
|
3090
|
+
* Say out loud what the broker answered about a send whose promise had already given up.
|
|
3091
|
+
*
|
|
3092
|
+
* The publish callback is not guarded by the attempt's `settled` flag, so when the
|
|
3093
|
+
* confirmation budget has already rejected, the callback still runs and still builds the
|
|
3094
|
+
* right verdict — and then hands it to a settled promise, where `reject()` is a no-op.
|
|
3095
|
+
* The frame is not lost (it stays on `channel._closeError`) and the verdict is not
|
|
3096
|
+
* overwritten; it simply reached nobody, while `PublishLayer` went on retrying and
|
|
3097
|
+
* buffering a message the broker had refused for good.
|
|
3098
|
+
*
|
|
3099
|
+
* Silence is a defect of the same severity as a wrong result
|
|
3100
|
+
* (`automation-gates.md` §5), so the verdict is REPORTED — and only reported. Nothing
|
|
3101
|
+
* here changes the promise's outcome, the retry policy or the budget: making a late
|
|
3102
|
+
* verdict DECIDE is a change of what happens to a message, which goes with the major
|
|
3103
|
+
* wave (d.801).
|
|
3104
|
+
*
|
|
3105
|
+
* Measured (d.801, dev broker): a refused send is answered in tens of milliseconds —
|
|
3106
|
+
* `404` median 13 ms idle and 31 ms (max 126 ms) under load 32 — so this path is not the
|
|
3107
|
+
* ordinary one. It is what a budget set below that latency produces.
|
|
3108
|
+
*
|
|
3109
|
+
* @param {PermanentPublishError|TransientPublishError} verdict - What the frame decided.
|
|
3110
|
+
* @param {'queue'|'exchange'} kind - Which leg is reporting.
|
|
3111
|
+
* @param {string} target - The queue or exchange the send was addressed to.
|
|
3112
|
+
* @param {string} queue - The queue the caller named (the exchange leg routes by it).
|
|
3113
|
+
* @param {Object} options - The publish options, which carry the retry layer's `attempt`.
|
|
3114
|
+
* @private
|
|
3115
|
+
*/
|
|
3116
|
+
_reportLateSendVerdict(verdict, kind, target, queue, options) {
|
|
3117
|
+
const code = verdict && verdict.cause ? verdict.cause.code : undefined;
|
|
3118
|
+
const attempt = options && typeof options.attempt === 'number' ? options.attempt : null;
|
|
3119
|
+
|
|
3120
|
+
this._logger.warn(
|
|
3121
|
+
`[RabbitMQClient] [mq-client-core] [PUBLISH] The broker refused this send and said so AFTER `
|
|
3122
|
+
+ `the confirmation budget of ${this._brokerAnswerTimeout}ms had already answered the caller - `
|
|
3123
|
+
+ `${kind} "${target}", broker code ${code}, verdict ${verdict.constructor.name}. `
|
|
3124
|
+
+ 'Expected: an answer within brokerAnswerTimeout, so the caller hears the broker rather '
|
|
3125
|
+
+ 'than the timeout. '
|
|
3126
|
+
+ 'Fix: the publish layer has already treated this message as unconfirmed and may retry it; '
|
|
3127
|
+
+ `raise brokerAnswerTimeout above the broker's answer latency for this route, or read `
|
|
3128
|
+
+ 'the publish:refused-late event to see how often it happens.'
|
|
3129
|
+
);
|
|
3130
|
+
|
|
3131
|
+
this.emit('publish:refused-late', {
|
|
3132
|
+
queue,
|
|
3133
|
+
exchange: kind === 'exchange' ? target : undefined,
|
|
3134
|
+
target,
|
|
3135
|
+
kind,
|
|
3136
|
+
code,
|
|
3137
|
+
verdict: verdict.constructor.name,
|
|
3138
|
+
// The verdict OBJECT, not only its name: since d.801c the retry layer acts on this
|
|
3139
|
+
// event — it ends its loop with this very error instead of publishing again — and a
|
|
3140
|
+
// listener that must act needs the thing, not a description of it.
|
|
3141
|
+
error: verdict,
|
|
3142
|
+
attempt,
|
|
3143
|
+
budgetMs: this._brokerAnswerTimeout
|
|
3144
|
+
});
|
|
3145
|
+
}
|
|
3146
|
+
|
|
3147
|
+
_collateralSendTransient(channel, kind, target) {
|
|
3148
|
+
const closeError = channel ? channel._closeError : null;
|
|
3149
|
+
const code = closeError ? closeError.code : undefined;
|
|
3150
|
+
|
|
3151
|
+
if (typeof code !== 'number') {
|
|
3152
|
+
return null;
|
|
3153
|
+
}
|
|
3154
|
+
|
|
3155
|
+
if (this._closeFrameNamesThisSend(channel, target)) {
|
|
3156
|
+
return null;
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
return new TransientPublishError(collateralCloseMessage(kind, target, code), closeError);
|
|
3160
|
+
}
|
|
3161
|
+
|
|
3162
|
+
/**
|
|
3163
|
+
* Is the broker's close frame on `channel` an answer about the send addressed to
|
|
3164
|
+
* `target`?
|
|
3165
|
+
*
|
|
3166
|
+
* TWO READINGS, IN THIS ORDER, and the second only when the first cannot answer:
|
|
3167
|
+
*
|
|
3168
|
+
* 1. **How many sends the frame could be about.** When exactly one publish was
|
|
3169
|
+
* unconfirmed at the moment the frame arrived, the frame is about that one and
|
|
3170
|
+
* nothing else is possible. This is the case every earlier batch measured — d.685a
|
|
3171
|
+
* (403, 406), d.693 (404 on an undeclared exchange) — and it is why those verdicts
|
|
3172
|
+
* do not move. The count is the client's own (`_countSendInFlight()`), snapshotted
|
|
3173
|
+
* by `_attachPublisherChannelHandlers()` when it records the frame, because amqplib
|
|
3174
|
+
* emits `error` BEFORE `close` and it is `close` that rejects the callbacks: by
|
|
3175
|
+
* callback time the channel's own bookkeeping is already unwound.
|
|
3176
|
+
* 2. **Whether the frame NAMES this target.** The broker quotes the subject of its
|
|
3177
|
+
* refusal in the reply text, and amqplib keeps that text inside the close error's
|
|
3178
|
+
* `message` — measured live 2026-09-20, the close error's own properties are
|
|
3179
|
+
* `["code","classId","methodId"]` and there is no `replyText` member, so the
|
|
3180
|
+
* sentence is where the name is:
|
|
3181
|
+
* `Channel closed by server: 404 (NOT-FOUND) with message "NOT_FOUND - no exchange
|
|
3182
|
+
* 'no.such.exchange' in vhost 'mq-client-core-probe-…'"`.
|
|
3183
|
+
*
|
|
3184
|
+
* READING A SENTENCE IS NOT HOW THE CLASS IS DECIDED, and this does not change that
|
|
3185
|
+
* (`docs/biz/70-contracts/error-handling.md` §6): the CODE still says what the answer
|
|
3186
|
+
* means. The name only says WHOSE answer it is, and it is read in the one direction
|
|
3187
|
+
* that is safe to get wrong — a name this cannot find, or a rewording that hides it,
|
|
3188
|
+
* means "not claimed", so the send falls to the transient collateral verdict and the
|
|
3189
|
+
* message is republished. The failure mode of a broker that rewords its text is a
|
|
3190
|
+
* message sent twice, never a message dropped on a refusal that was not about it.
|
|
3191
|
+
*
|
|
3192
|
+
* @param {Object} channel - The channel carrying the recorded frame.
|
|
3193
|
+
* @param {string} target - The queue or exchange THIS send was addressed to.
|
|
3194
|
+
* @returns {boolean}
|
|
3195
|
+
* @private
|
|
3196
|
+
*/
|
|
3197
|
+
_closeFrameNamesThisSend(channel, target) {
|
|
3198
|
+
const unconfirmed = channel._unconfirmedAtCloseFrame;
|
|
3199
|
+
|
|
3200
|
+
// The frame was recorded by `_attachPublisherChannelHandlers()`, which takes the
|
|
3201
|
+
// snapshot in the same statement — so a missing number means the channel was never
|
|
3202
|
+
// prepared by this client, and an unprepared channel is not one whose sends this may
|
|
3203
|
+
// speak for. Not claimed, therefore transient: the safe direction.
|
|
3204
|
+
if (typeof unconfirmed !== 'number') {
|
|
3205
|
+
return false;
|
|
3206
|
+
}
|
|
3207
|
+
|
|
3208
|
+
if (unconfirmed <= 1) {
|
|
3209
|
+
return true;
|
|
3210
|
+
}
|
|
3211
|
+
|
|
3212
|
+
const named = this._nameInCloseFrame(channel._closeError);
|
|
3213
|
+
return named !== null && named === target;
|
|
3214
|
+
}
|
|
3215
|
+
|
|
3216
|
+
/**
|
|
3217
|
+
* The name the broker quoted in its close frame, or `null` when it quoted none.
|
|
3218
|
+
*
|
|
3219
|
+
* The first single-quoted token of the reply text is its subject — `no exchange 'X' in
|
|
3220
|
+
* vhost 'Y'`, `no queue 'X' in vhost 'Y'`, `cannot obtain exclusive access to locked
|
|
3221
|
+
* queue 'X' in vhost 'Y'`. A frame that quotes nothing (`PRECONDITION_FAILED - unknown
|
|
3222
|
+
* delivery tag 999999`, measured d.698) answers with `null`, which is a real answer
|
|
3223
|
+
* here: "this frame names nobody".
|
|
3224
|
+
*
|
|
3225
|
+
* @param {Error} closeError - The recorded close frame.
|
|
3226
|
+
* @returns {string|null}
|
|
3227
|
+
* @private
|
|
3228
|
+
*/
|
|
3229
|
+
_nameInCloseFrame(closeError) {
|
|
3230
|
+
const text = closeError && typeof closeError.message === 'string' ? closeError.message : '';
|
|
3231
|
+
const quoted = /'([^']*)'/.exec(text);
|
|
3232
|
+
return quoted ? quoted[1] : null;
|
|
3233
|
+
}
|
|
3234
|
+
|
|
3235
|
+
/**
|
|
3236
|
+
* Count one send as unconfirmed on `channel`, and hand back the ONE way to stop
|
|
3237
|
+
* counting it.
|
|
3238
|
+
*
|
|
3239
|
+
* The count exists for `_closeFrameNamesThisSend()`: it is what says whether a numbered
|
|
3240
|
+
* close frame can only be about this send. It is the client's own rather than amqplib's
|
|
3241
|
+
* `unconfirmed` array, because that array is emptied by the very event this has to
|
|
3242
|
+
* measure — `toClosed()` rejects every pending publish, and only then do the callbacks
|
|
3243
|
+
* that ask this question run.
|
|
3244
|
+
*
|
|
3245
|
+
* A closure rather than a bare decrement at each site, because a send ends in three
|
|
3246
|
+
* places (the callback, a synchronous throw, and nowhere at all when neither happens),
|
|
3247
|
+
* and a count that is decremented twice or not at all makes the reading above answer
|
|
3248
|
+
* for the wrong number of messages.
|
|
3249
|
+
*
|
|
3250
|
+
* @param {Object} channel - The channel the send is going out on.
|
|
3251
|
+
* @returns {function(): void} Idempotent; call it when the send is no longer unconfirmed.
|
|
3252
|
+
* @private
|
|
3253
|
+
*/
|
|
3254
|
+
/**
|
|
3255
|
+
* The channel of one send role, as it stands NOW.
|
|
3256
|
+
*
|
|
3257
|
+
* A getter rather than a captured value, because the member is replaced when the
|
|
3258
|
+
* channel dies (`_attachPublisherChannelHandlers()` clears it, the next publish
|
|
3259
|
+
* recreates it), and a caller holding the previous one would be sending on a corpse.
|
|
3260
|
+
*
|
|
3261
|
+
* @param {'publisher'|'telemetry'} role
|
|
3262
|
+
* @returns {Object|null}
|
|
3263
|
+
* @private
|
|
3264
|
+
*/
|
|
3265
|
+
_sendChannelOf(role) {
|
|
3266
|
+
return role === 'telemetry' ? this._telemetryChannel : this._channel;
|
|
3267
|
+
}
|
|
3268
|
+
|
|
3269
|
+
/**
|
|
3270
|
+
* The one place a send channel is assigned — so "which member does this role live in"
|
|
3271
|
+
* is written once rather than at every site that creates, clears or tears one down.
|
|
3272
|
+
*
|
|
3273
|
+
* @param {'publisher'|'telemetry'} role
|
|
3274
|
+
* @param {Object|null} channel
|
|
3275
|
+
* @private
|
|
3276
|
+
*/
|
|
3277
|
+
_setSendChannelOf(role, channel) {
|
|
3278
|
+
if (role === 'telemetry') {
|
|
3279
|
+
this._telemetryChannel = channel;
|
|
3280
|
+
} else {
|
|
3281
|
+
this._channel = channel;
|
|
3282
|
+
}
|
|
3283
|
+
}
|
|
3284
|
+
|
|
3285
|
+
/**
|
|
3286
|
+
* Which channel a publish goes out on, from the options it was given.
|
|
3287
|
+
*
|
|
3288
|
+
* Explicit, never derived from the queue name (`architecture-principles.md` §8, §9): a
|
|
3289
|
+
* telemetry publish says so because `BaseClient.publishTelemetry()` says so, and a name
|
|
3290
|
+
* that merely looks like a monitoring queue does not move a caller's message onto
|
|
3291
|
+
* another channel.
|
|
3292
|
+
*
|
|
3293
|
+
* @param {Object} options - The publish options as the caller handed them over.
|
|
3294
|
+
* @returns {'publisher'|'telemetry'}
|
|
3295
|
+
* @private
|
|
3296
|
+
*/
|
|
3297
|
+
_sendRoleFor(options) {
|
|
3298
|
+
return options && options.channelRole === 'telemetry' ? 'telemetry' : 'publisher';
|
|
3299
|
+
}
|
|
3300
|
+
|
|
3301
|
+
_countSendInFlight(channel) {
|
|
3302
|
+
channel._unconfirmedSends = (channel._unconfirmedSends || 0) + 1;
|
|
3303
|
+
let counted = true;
|
|
3304
|
+
return () => {
|
|
3305
|
+
if (!counted) return;
|
|
3306
|
+
counted = false;
|
|
3307
|
+
channel._unconfirmedSends -= 1;
|
|
3308
|
+
};
|
|
3309
|
+
}
|
|
3310
|
+
|
|
2732
3311
|
/**
|
|
2733
3312
|
* Declare an exchange.
|
|
2734
3313
|
*
|
|
@@ -2961,11 +3540,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
2961
3540
|
* @private
|
|
2962
3541
|
*/
|
|
2963
3542
|
async _publishOnce(queue, buffer, options = {}) {
|
|
2964
|
-
//
|
|
2965
|
-
|
|
2966
|
-
|
|
3543
|
+
// Whose channel this send goes out on (d.724). Read once, here: the role cannot
|
|
3544
|
+
// change inside one attempt, and the channel it names can — so everything below asks
|
|
3545
|
+
// `_sendChannelOf(sendRole)` rather than holding a member.
|
|
3546
|
+
const sendRole = this._sendRoleFor(options);
|
|
3547
|
+
|
|
3548
|
+
// Ensure the send channel exists and is open (auto-recreates if closed)
|
|
3549
|
+
await this._ensurePublisherChannel(sendRole);
|
|
3550
|
+
|
|
2967
3551
|
// Track operation for debugging
|
|
2968
|
-
this._trackChannelOperation(this.
|
|
3552
|
+
this._trackChannelOperation(this._sendChannelOf(sendRole), `publish to ${queue}`);
|
|
2969
3553
|
|
|
2970
3554
|
// Asked with `=== undefined`, not with `||`: the empty string IS a value here
|
|
2971
3555
|
// — it names the default exchange — and the third leg was a third copy of a
|
|
@@ -3018,13 +3602,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3018
3602
|
// If queue doesn't exist (404), we should NOT auto-create it - infrastructure queues must be created explicitly
|
|
3019
3603
|
// This prevents creating queues with wrong arguments (no TTL) which causes 406 errors later
|
|
3020
3604
|
if (!skipQueueExistencePrecheck) {
|
|
3605
|
+
// Declared OUTSIDE the `try`, because the `catch` below has to ask whether
|
|
3606
|
+
// the channel that CARRIED the RPC is still alive — that reading is what
|
|
3607
|
+
// tells "the broker answered" from "the answer never came" (d.683).
|
|
3608
|
+
let queueChannel = null;
|
|
3021
3609
|
try {
|
|
3022
3610
|
// The queue channel comes from the ONE rail (d.282): ensure it, refuse
|
|
3023
3611
|
// it unless it is ALIVE, record the operation. Until d.284 this site
|
|
3024
3612
|
// assembled that sequence itself — `_ensureQueueChannel()` plus
|
|
3025
3613
|
// `_isChannelAlive()` by hand — differing from the rail in nothing but
|
|
3026
3614
|
// the error type, which is `_publishPathTransient()`'s job below.
|
|
3027
|
-
let queueChannel;
|
|
3028
3615
|
try {
|
|
3029
3616
|
queueChannel = await this._requireQueueChannel('checkQueue', queue);
|
|
3030
3617
|
} catch (railErr) {
|
|
@@ -3045,7 +3632,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3045
3632
|
// → QueueNotFoundError(kind='business'). Publisher je nikdy nevytváří.
|
|
3046
3633
|
// - Ostatní jména (ani infra, ani business vzor) nemá kdo vlastnit, takže
|
|
3047
3634
|
// je nezakládá nikdo → QueueNotFoundError(kind='unowned'), d.419.
|
|
3048
|
-
if (checkErr.code ===
|
|
3635
|
+
if (checkErr.code === AMQP_NOT_FOUND) {
|
|
3049
3636
|
// `queueConfig` is loaded once at the top of this module. It used to be
|
|
3050
3637
|
// required here, inside a try/catch whose `catch` re-implemented
|
|
3051
3638
|
// `isInfrastructureQueue` as an inline prefix list — a fallback
|
|
@@ -3067,7 +3654,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3067
3654
|
// registration; a publisher that creates it is a programming error.
|
|
3068
3655
|
//
|
|
3069
3656
|
// The rule and this exact wording came from
|
|
3070
|
-
// `conn-infra-mq/src/transports/rabbitmqClient.js
|
|
3657
|
+
// `conn-infra-mq/src/transports/rabbitmqClient.js` — a transport
|
|
3071
3658
|
// that nothing in `src/` instantiated, so the guard protected nobody
|
|
3072
3659
|
// while this file, the one every client actually runs, auto-created the
|
|
3073
3660
|
// queue and only printed a warning.
|
|
@@ -3075,7 +3662,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3075
3662
|
throw new QueueNotFoundError(queue, false, checkErr, 'business');
|
|
3076
3663
|
}
|
|
3077
3664
|
// Nobody owns this name. Until d.419 that opened a THIRD case beside
|
|
3078
|
-
// the two above: the publish path asked
|
|
3665
|
+
// the two above: the publish path asked the recovery worker whether this
|
|
3079
3666
|
// client might declare it and, when the answer was yes, declared it —
|
|
3080
3667
|
// `durable=true, arguments={}`, no TTL, no dead-letter route (measured
|
|
3081
3668
|
// live, d.410). The concept had already ruled that out: "queues come
|
|
@@ -3100,6 +3687,34 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3100
3687
|
// "nothing declares this name", and the publish path must not flatten the
|
|
3101
3688
|
// two into one message.
|
|
3102
3689
|
throw new QueueNotFoundError(queue, false, checkErr, 'unowned');
|
|
3690
|
+
} else if (
|
|
3691
|
+
!(checkErr instanceof TransientPublishError)
|
|
3692
|
+
&& checkErr.code === undefined
|
|
3693
|
+
&& !this._isChannelAlive(queueChannel)
|
|
3694
|
+
) {
|
|
3695
|
+
// THE RPC NEVER GOT AN ANSWER. amqplib answers the request that was in
|
|
3696
|
+
// flight when a channel closed with the broker's own close frame — a
|
|
3697
|
+
// NUMBER in `code` — and rejects whatever was queued behind it with a
|
|
3698
|
+
// bare `new Error('Channel ended, no reply will be forthcoming')`
|
|
3699
|
+
// (`_rejectPending`, `node_modules/amqplib/lib/channel.js`), whose
|
|
3700
|
+
// only own properties are `stack` and `message`. Until d.683 that object
|
|
3701
|
+
// travelled untouched to `classifyPublishError()`, which had to keep its
|
|
3702
|
+
// verdict by matching the sentence — the one text branch d.677 could not
|
|
3703
|
+
// remove from where it stood, because the sentence was all there was.
|
|
3704
|
+
//
|
|
3705
|
+
// This site does not have to read it: it knows which channel carried the
|
|
3706
|
+
// RPC and can see that the channel is gone, so it answers in the dialect
|
|
3707
|
+
// every other leg of this path already speaks. The three conditions are
|
|
3708
|
+
// each doing work:
|
|
3709
|
+
// - `code === undefined` — a numbered refusal is an ANSWER about this
|
|
3710
|
+
// request (405 RESOURCE_LOCKED, measured), and repeating the request
|
|
3711
|
+
// repeats it. It stays permanent;
|
|
3712
|
+
// - the channel is dead — a failure on a LIVE channel did get through
|
|
3713
|
+
// to the broker and back, whatever it says;
|
|
3714
|
+
// - not already typed — `_publishPathTransient()` above throws into
|
|
3715
|
+
// this same `catch`, and wrapping a verdict in itself would bury the
|
|
3716
|
+
// rail's own sentence one level deeper for no reader.
|
|
3717
|
+
throw this._publishPathTransient('checkQueue', queue, checkErr);
|
|
3103
3718
|
} else {
|
|
3104
3719
|
// Other error - rethrow
|
|
3105
3720
|
throw checkErr;
|
|
@@ -3123,8 +3738,16 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3123
3738
|
let timeout = null;
|
|
3124
3739
|
let safetyTimer = null;
|
|
3125
3740
|
let settled = false;
|
|
3741
|
+
// Assigned once the send is on its way (below), and released here with the
|
|
3742
|
+
// timers: the attempt ends in five places — the callback, a synchronous throw,
|
|
3743
|
+
// the confirmation timeout, the liveness check before the send, and the
|
|
3744
|
+
// watchdog — and a send still counted after its attempt has ended would make
|
|
3745
|
+
// the next close frame on this channel look as if it could be about two
|
|
3746
|
+
// messages (`_closeFrameNamesThisSend()`, d.723).
|
|
3747
|
+
let sendSettled = () => {};
|
|
3126
3748
|
const disarm = () => {
|
|
3127
3749
|
settled = true;
|
|
3750
|
+
sendSettled();
|
|
3128
3751
|
this._clearTimeout(timeout);
|
|
3129
3752
|
this._clearTimeout(safetyTimer);
|
|
3130
3753
|
timeout = null;
|
|
@@ -3147,7 +3770,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3147
3770
|
// during the awaits above (queue existence pre-check), so this is not
|
|
3148
3771
|
// a repeat of `_ensurePublisherChannel()` — it is the last reading
|
|
3149
3772
|
// before the payload leaves, and it reads the one liveness source.
|
|
3150
|
-
if (!this._isChannelAlive(this.
|
|
3773
|
+
if (!this._isChannelAlive(this._sendChannelOf(sendRole))) {
|
|
3151
3774
|
disarm();
|
|
3152
3775
|
reject(new ConnectionError(
|
|
3153
3776
|
`[RabbitMQClient] Cannot publish: channel is closed for queue "${queue}" - `
|
|
@@ -3158,9 +3781,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3158
3781
|
}
|
|
3159
3782
|
|
|
3160
3783
|
// Track original channel to detect if it was closed/recreated during publish
|
|
3161
|
-
const originalChannel = this.
|
|
3784
|
+
const originalChannel = this._sendChannelOf(sendRole);
|
|
3162
3785
|
let callbackInvoked = false;
|
|
3163
|
-
|
|
3786
|
+
// This send is unconfirmed from the moment it is handed over until the broker
|
|
3787
|
+
// answers it, the channel dies under it, or amqplib refuses it outright — and
|
|
3788
|
+
// that number is what decides whose answer a close frame is (d.723).
|
|
3789
|
+
sendSettled = this._countSendInFlight(originalChannel);
|
|
3790
|
+
|
|
3164
3791
|
// Send message with callback
|
|
3165
3792
|
try {
|
|
3166
3793
|
originalChannel.sendToQueue(
|
|
@@ -3169,14 +3796,45 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3169
3796
|
buildAmqpMessageProperties(options, persistent, headers),
|
|
3170
3797
|
(err, ok) => {
|
|
3171
3798
|
callbackInvoked = true;
|
|
3799
|
+
// Read BEFORE `disarm()`, which sets it: the question is whether the budget
|
|
3800
|
+
// had already answered the caller, and after `disarm()` nobody can tell.
|
|
3801
|
+
const answeredAlready = settled;
|
|
3172
3802
|
disarm();
|
|
3173
|
-
|
|
3803
|
+
|
|
3174
3804
|
// CRITICAL: the delivery tag belongs to the channel that SENT the
|
|
3175
3805
|
// message, so that channel's liveness is the whole question. One
|
|
3176
3806
|
// reading answers it: a channel that was replaced is a channel that
|
|
3177
3807
|
// died first (`_ensurePublisherChannel()` recreates only a dead one),
|
|
3178
3808
|
// so identity and `_createdAt` comparisons said nothing this does not.
|
|
3179
3809
|
if (!this._isChannelAlive(originalChannel)) {
|
|
3810
|
+
// WHY the channel died decides first. A numbered close frame is the
|
|
3811
|
+
// broker's answer about THIS message, and no reconnect changes it
|
|
3812
|
+
// (`_refusedSendPermanent()`, d.685a) — so it is asked before the
|
|
3813
|
+
// reconnect wait below, which would otherwise re-offer a message the
|
|
3814
|
+
// broker will refuse again.
|
|
3815
|
+
const refused = this._refusedSendPermanent(originalChannel, 'queue', queue);
|
|
3816
|
+
if (refused) {
|
|
3817
|
+
if (answeredAlready) {
|
|
3818
|
+
this._reportLateSendVerdict(refused, 'queue', queue, queue, options);
|
|
3819
|
+
} else {
|
|
3820
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] ${refused.message}`);
|
|
3821
|
+
}
|
|
3822
|
+
reject(refused);
|
|
3823
|
+
return;
|
|
3824
|
+
}
|
|
3825
|
+
// The frame was there but it was about a DIFFERENT send on this channel
|
|
3826
|
+
// (d.723): this message was never answered, so it is republished rather
|
|
3827
|
+
// than dropped on somebody else's refusal.
|
|
3828
|
+
const collateral = this._collateralSendTransient(originalChannel, 'queue', queue);
|
|
3829
|
+
if (collateral) {
|
|
3830
|
+
if (answeredAlready) {
|
|
3831
|
+
this._reportLateSendVerdict(collateral, 'queue', queue, queue, options);
|
|
3832
|
+
} else {
|
|
3833
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] ${collateral.message}`);
|
|
3834
|
+
}
|
|
3835
|
+
reject(collateral);
|
|
3836
|
+
return;
|
|
3837
|
+
}
|
|
3180
3838
|
// Channel was closed or recreated - delivery tag is invalid
|
|
3181
3839
|
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed/recreated during publish to queue "${queue}", ignoring callback`);
|
|
3182
3840
|
// If we're reconnecting, wait and retry
|
|
@@ -3200,29 +3858,24 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3200
3858
|
}
|
|
3201
3859
|
|
|
3202
3860
|
if (err) {
|
|
3203
|
-
//
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
}
|
|
3222
|
-
} else {
|
|
3223
|
-
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
|
|
3224
|
-
reject(err);
|
|
3225
|
-
}
|
|
3861
|
+
// A confirm callback is handed ONE of three things by amqplib, and this is
|
|
3862
|
+
// the whole list: `null` (the broker confirmed), `new Error('message
|
|
3863
|
+
// nacked')` and `new Error('channel closed')` — composed in
|
|
3864
|
+
// `lib/channel.js` (the `Channel` constructor's confirm listeners) and
|
|
3865
|
+
// `lib/channel_model.js` (`ConfirmChannel.waitForConfirms()`), with no `code` on
|
|
3866
|
+
// either. The callback cannot see WHY the channel died; the channel can,
|
|
3867
|
+
// and the branch above reads it (`_refusedSendPermanent()`, d.685a).
|
|
3868
|
+
//
|
|
3869
|
+
// So there is no delivery-tag case to catch here, and until d.698 one was
|
|
3870
|
+
// written anyway: a substring test for the words a 406 reply text carries.
|
|
3871
|
+
// It matched none of the three, and what it did was not neutral: it
|
|
3872
|
+
// answered with a `PublishError`, which the
|
|
3873
|
+
// classification reads as TRANSIENT, so a message would have been retried
|
|
3874
|
+
// `publishMaxRetries` times and then buffered on the strength of a wording.
|
|
3875
|
+
// The live carrier of that sentence is a 406 close frame on the channel's
|
|
3876
|
+
// `error` event (measured d.693/d.698), never an argument to this callback.
|
|
3877
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Send callback error for queue "${queue}":`, err.message);
|
|
3878
|
+
reject(err);
|
|
3226
3879
|
} else {
|
|
3227
3880
|
// Structured log: publish confirmed
|
|
3228
3881
|
const wfId = workflowIdOf(headers);
|
|
@@ -3257,8 +3910,27 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3257
3910
|
}
|
|
3258
3911
|
} catch (sendErr) {
|
|
3259
3912
|
disarm();
|
|
3260
|
-
//
|
|
3261
|
-
|
|
3913
|
+
// THE CHANNEL ANSWERS, NOT THE SENTENCE (d.685c). A synchronous throw from
|
|
3914
|
+
// `sendToQueue()` means amqplib refused to send at all, and the one question
|
|
3915
|
+
// that decides what to do with the message is whether the channel it was
|
|
3916
|
+
// handed is still alive: a dead one will be replaced by the reconnect, so the
|
|
3917
|
+
// message is republished over its successor; a live one did accept the call
|
|
3918
|
+
// and failed for its own reason, which no reconnect changes.
|
|
3919
|
+
//
|
|
3920
|
+
// Until this batch the question was asked of the message text
|
|
3921
|
+
// (`includes('Channel ended') || includes('Channel closed')`), which is the
|
|
3922
|
+
// defect d.677 removed from the classification: a wording decided whether a
|
|
3923
|
+
// message was re-offered to the broker. It was also matching a sentence that
|
|
3924
|
+
// cannot arrive here — `'Channel ended, no reply will be forthcoming'` is
|
|
3925
|
+
// built for an RPC awaiting a REPLY (`_rejectPending()`, amqplib
|
|
3926
|
+
// `lib/channel.js`), and a send is not an RPC.
|
|
3927
|
+
//
|
|
3928
|
+
// Measured carrier, live against the dev broker: amqplib invalidates a dead
|
|
3929
|
+
// channel's send methods (`invalidateSend()`, `lib/channel.js`), so
|
|
3930
|
+
// what arrives is its `IllegalOperationError` with `'Channel closed'` — no
|
|
3931
|
+
// code, and `channel._closeError` null. Liveness from the one source
|
|
3932
|
+
// (`_isChannelAlive`, d.260), like the two callbacks above.
|
|
3933
|
+
if (!this._isChannelAlive(originalChannel)) {
|
|
3262
3934
|
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during sendToQueue, will retry after reconnection`);
|
|
3263
3935
|
// Wait for reconnection and retry
|
|
3264
3936
|
if (this._reconnecting) {
|
|
@@ -3302,8 +3974,20 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3302
3974
|
|
|
3303
3975
|
// Use callback-based confirmation - kanály jsou spolehlivé
|
|
3304
3976
|
const confirmPromise = new Promise((resolve, reject) => {
|
|
3977
|
+
// Assigned once the send is on its way (below). It is declared HERE because the
|
|
3978
|
+
// count has to be released on EVERY ending of this attempt, and the earliest of
|
|
3979
|
+
// them — the confirmation timeout — is armed before the send exists. A send
|
|
3980
|
+
// still counted after its attempt has ended would make the next close frame on
|
|
3981
|
+
// this channel look as if it could be about two messages (d.723).
|
|
3982
|
+
let sendSettled = () => {};
|
|
3983
|
+
// Whether the budget has already answered the caller — read by the callback, so
|
|
3984
|
+
// a verdict that arrives afterwards can be REPORTED instead of handed to a
|
|
3985
|
+
// settled promise (d.801b). The queue leg keeps the same fact in `settled`.
|
|
3986
|
+
let answeredAlready = false;
|
|
3305
3987
|
// Set timeout for exchange publish confirmation
|
|
3306
3988
|
const timeout = this._setTimeout(() => {
|
|
3989
|
+
answeredAlready = true;
|
|
3990
|
+
sendSettled();
|
|
3307
3991
|
reject(new PublishError(
|
|
3308
3992
|
`[RabbitMQClient] Exchange publish confirmation timeout for exchange "${exchange}" after ${this._brokerAnswerTimeout}ms - `
|
|
3309
3993
|
+ 'Expected: the broker to confirm the message within brokerAnswerTimeout. '
|
|
@@ -3313,14 +3997,79 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3313
3997
|
));
|
|
3314
3998
|
}, this._brokerAnswerTimeout);
|
|
3315
3999
|
|
|
3316
|
-
|
|
4000
|
+
// The channel that SENDS is the channel whose liveness answers the
|
|
4001
|
+
// callback below — read once here, like the queue leg does, because
|
|
4002
|
+
// the member may already hold the replacement by the time the
|
|
4003
|
+
// callback runs.
|
|
4004
|
+
const sendingChannel = this._sendChannelOf(sendRole);
|
|
4005
|
+
// Unconfirmed until answered — the same count the queue leg keeps, and for the
|
|
4006
|
+
// same reason: it is what decides whose answer a close frame is (d.723).
|
|
4007
|
+
sendSettled = this._countSendInFlight(sendingChannel);
|
|
4008
|
+
|
|
4009
|
+
sendingChannel.publish(
|
|
3317
4010
|
exchange,
|
|
3318
4011
|
routingKey,
|
|
3319
4012
|
buffer,
|
|
3320
4013
|
buildAmqpMessageProperties(options, persistent, headers),
|
|
3321
4014
|
(err, ok) => {
|
|
4015
|
+
sendSettled();
|
|
3322
4016
|
this._clearTimeout(timeout);
|
|
3323
4017
|
if (err) {
|
|
4018
|
+
// THE CONFIRM NEVER CAME. amqplib answers every unconfirmed publish of
|
|
4019
|
+
// a channel that has just closed with a bare `new Error('channel
|
|
4020
|
+
// closed')` (the `Channel` constructor's `'close'` listener, `node_modules/amqplib/lib/channel.js`) — class `Error`,
|
|
4021
|
+
// no code, own properties `stack` and `message`. The broker's own
|
|
4022
|
+
// numbered refusal is NOT what arrives here: it goes to the channel's
|
|
4023
|
+
// `error` event (measured d.677, carrier B1 — a publish through an
|
|
4024
|
+
// exchange nobody declared).
|
|
4025
|
+
//
|
|
4026
|
+
// The queue leg above has read the sending channel's liveness since
|
|
4027
|
+
// long before this batch, which is why a refused send already reached
|
|
4028
|
+
// the classification as this package's own `PublishError` (carrier E1).
|
|
4029
|
+
// This leg had no such reading, so its bare sentinel travelled untouched
|
|
4030
|
+
// and `classifyPublishError()` had to keep its verdict by the sentence.
|
|
4031
|
+
// Now both legs answer the same way, and the sentinel is kept in `cause`
|
|
4032
|
+
// rather than read (d.683).
|
|
4033
|
+
//
|
|
4034
|
+
// A live channel means the broker DID answer — a nack is the measured
|
|
4035
|
+
// case (carrier G, `'message nacked'`, amqplib `lib/channel.js`, the `Channel` constructor's `'nack'` listener) —
|
|
4036
|
+
// and that answer is about this message, so it stays what it was.
|
|
4037
|
+
if (!this._isChannelAlive(sendingChannel)) {
|
|
4038
|
+
// The same question this leg's sibling asks one branch up: did the broker
|
|
4039
|
+
// REFUSE this send, or did the channel merely die under it (d.685a)?
|
|
4040
|
+
const refused = this._refusedSendPermanent(sendingChannel, 'exchange', exchange);
|
|
4041
|
+
if (refused) {
|
|
4042
|
+
if (answeredAlready) {
|
|
4043
|
+
this._reportLateSendVerdict(refused, 'exchange', exchange, queue, options);
|
|
4044
|
+
} else {
|
|
4045
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] ${refused.message}`);
|
|
4046
|
+
}
|
|
4047
|
+
reject(refused);
|
|
4048
|
+
return;
|
|
4049
|
+
}
|
|
4050
|
+
// Somebody else's refusal closed the channel under this send (d.723) —
|
|
4051
|
+
// never answered, therefore never refused, therefore republished.
|
|
4052
|
+
const collateral = this._collateralSendTransient(sendingChannel, 'exchange', exchange);
|
|
4053
|
+
if (collateral) {
|
|
4054
|
+
if (answeredAlready) {
|
|
4055
|
+
this._reportLateSendVerdict(collateral, 'exchange', exchange, queue, options);
|
|
4056
|
+
} else {
|
|
4057
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] ${collateral.message}`);
|
|
4058
|
+
}
|
|
4059
|
+
reject(collateral);
|
|
4060
|
+
return;
|
|
4061
|
+
}
|
|
4062
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] [PUBLISH] Channel closed during publish to exchange "${exchange}", classifying as transient`);
|
|
4063
|
+
reject(new PublishError(
|
|
4064
|
+
`[RabbitMQClient] Channel closed during publish to exchange "${exchange}" - message delivery not guaranteed. `
|
|
4065
|
+
+ 'Expected: the channel to stay open until the broker confirms. '
|
|
4066
|
+
+ 'Fix: republish the message — the confirm never arrived, so it may or may not have been routed; '
|
|
4067
|
+
+ 'read error.cause for what amqplib said.',
|
|
4068
|
+
exchange,
|
|
4069
|
+
err
|
|
4070
|
+
));
|
|
4071
|
+
return;
|
|
4072
|
+
}
|
|
3324
4073
|
this._logger.error(`[RabbitMQClient] [mq-client-core] [PUBLISH] Exchange publish callback error:`, err.message);
|
|
3325
4074
|
reject(err);
|
|
3326
4075
|
} else {
|
|
@@ -3338,9 +4087,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3338
4087
|
// Pokud channel closed, vyhodíme TransientPublishError a necháme PublishLayer řešit retry
|
|
3339
4088
|
|
|
3340
4089
|
// Všechny chyby projdou jednotnou klasifikací do Transient/Permanent/QueueNotFound
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
4090
|
+
//
|
|
4091
|
+
// A VERDICT ABOUT A MESSAGE GOES ONE WAY: to whoever sent it. Until d.811b this line
|
|
4092
|
+
// also put the classified failure on the client's `error` channel, which is the
|
|
4093
|
+
// channel the rest of this client uses for the CONNECTION. Measured (d.811): it
|
|
4094
|
+
// fired for every classification, transient as well as permanent; nobody read it
|
|
4095
|
+
// (no `.on('error')` on this client anywhere in `shared/**` or `infra/**`, and zero
|
|
4096
|
+
// callers of `onError()` in production code); the concept never asked for it
|
|
4097
|
+
// (`docs/architecture/mq-publish-reliability.md` does not mention the channel); and
|
|
4098
|
+
// the only rail that ever met it had to learn to IGNORE it, because one concurrent
|
|
4099
|
+
// publish failing while the link came back rejected every reconnect waiter
|
|
4100
|
+
// (`_waitForReconnection()`, d.435/d.435b). It made work for another rail instead of
|
|
4101
|
+
// serving a reader.
|
|
4102
|
+
throw classifyPublishError(err, queue);
|
|
3344
4103
|
}
|
|
3345
4104
|
}
|
|
3346
4105
|
|
|
@@ -3378,7 +4137,11 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3378
4137
|
this._prefetchTracking.set(queue, {
|
|
3379
4138
|
prefetchCount: prefetch,
|
|
3380
4139
|
inFlight: 0,
|
|
3381
|
-
lastCheck: Date.now()
|
|
4140
|
+
lastCheck: Date.now(),
|
|
4141
|
+
// Which side of the threshold this queue was on when the verdict was last
|
|
4142
|
+
// taken. It is what makes the alarm an EPISODE rather than a line per tick
|
|
4143
|
+
// — see `_checkPrefetchUtilization()`.
|
|
4144
|
+
utilizationHigh: false
|
|
3382
4145
|
});
|
|
3383
4146
|
}
|
|
3384
4147
|
|
|
@@ -3496,7 +4259,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3496
4259
|
//
|
|
3497
4260
|
// The ONE queue this demand cannot be made of is the one at the END of a
|
|
3498
4261
|
// chain: a queue something else dead-letters TO. `workflow.failed` is named
|
|
3499
|
-
// in `x-dead-letter-routing-key` by `workflow.init
|
|
4262
|
+
// in `x-dead-letter-routing-key` by `workflow.init`,
|
|
3500
4263
|
// `<service>.dlq` by every business template — and the confirmed topology
|
|
3501
4264
|
// leaves exactly those without a route of their own, because a dead-letter
|
|
3502
4265
|
// route on a dead-letter queue is a loop or a chain with no end
|
|
@@ -3566,7 +4329,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3566
4329
|
await queueChannel.checkQueue(queue);
|
|
3567
4330
|
this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Infrastructure queue ${queue} exists (consumer will proceed)`);
|
|
3568
4331
|
} catch (checkErr) {
|
|
3569
|
-
if (checkErr.code ===
|
|
4332
|
+
if (checkErr.code === AMQP_NOT_FOUND) {
|
|
3570
4333
|
// Typed and coded since d.297: a bare `Error` lost the queue AND the
|
|
3571
4334
|
// class, so the layer above could not tell "this queue does not exist"
|
|
3572
4335
|
// from "the policy refused this queue" and told the reader the wrong
|
|
@@ -3582,8 +4345,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3582
4345
|
}
|
|
3583
4346
|
} else {
|
|
3584
4347
|
// Business queue (or unknown) — declared on the ONE public rail, the same
|
|
3585
|
-
// `assertQueue()`
|
|
3586
|
-
// `RecoveryWorker.createQueue()` (d.277b) go through. It ensures the queue
|
|
4348
|
+
// `assertQueue()` every collaborator that declares a queue goes through. It ensures the queue
|
|
3587
4349
|
// channel, refuses one that is not alive (d.282) and resolves the arguments
|
|
3588
4350
|
// from `queueConfig.declarationOptions()` — so a declaration made here and a
|
|
3589
4351
|
// declaration made anywhere else are the same declaration by construction,
|
|
@@ -3649,56 +4411,14 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3649
4411
|
// `false` = the budget is one attempt, so the first failure rejects into
|
|
3650
4412
|
// `<svc>.dlq` — exactly what a `permanent` classification already does.
|
|
3651
4413
|
// `true` or absent = the budget configuration decides. There is no third
|
|
3652
|
-
// meaning, and the combination that would need one is refused
|
|
4414
|
+
// meaning, and the combination that would need one is refused rather
|
|
3653
4415
|
// than resolved by silently preferring one of the two.
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
);
|
|
3661
|
-
}
|
|
3662
|
-
|
|
3663
|
-
const noRequeue = options.requeueOnError === false;
|
|
3664
|
-
|
|
3665
|
-
if (noRequeue && options.maxAttempts !== undefined && options.maxAttempts !== 1) {
|
|
3666
|
-
throw new ValidationError(
|
|
3667
|
-
'[RabbitMQClient] Contradictory delivery policy - requeueOnError: false means one attempt, '
|
|
3668
|
-
+ `but maxAttempts says ${JSON.stringify(options.maxAttempts)}. Expected: one statement about `
|
|
3669
|
-
+ 'how many times a handler may run. Fix: drop requeueOnError to keep the budget, or drop '
|
|
3670
|
-
+ 'maxAttempts to dead-letter on the first failure.'
|
|
3671
|
-
);
|
|
3672
|
-
}
|
|
3673
|
-
|
|
3674
|
-
const maxAttempts = noRequeue
|
|
3675
|
-
? 1
|
|
3676
|
-
: options.maxAttempts === undefined
|
|
3677
|
-
? runtimeCfg.get('maxDeliveryAttempts', this._config.maxDeliveryAttempts)
|
|
3678
|
-
: options.maxAttempts;
|
|
3679
|
-
|
|
3680
|
-
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
|
|
3681
|
-
throw new ValidationError(
|
|
3682
|
-
'[RabbitMQClient] Invalid delivery budget - maxAttempts must be an integer >= 1, got '
|
|
3683
|
-
+ `${JSON.stringify(maxAttempts)}. Fix: pass consume(queue, handler, { maxAttempts: <integer >= 1> }), `
|
|
3684
|
-
+ 'or leave it out and let the module default (defaults.js maxDeliveryAttempts, '
|
|
3685
|
-
+ 'env RABBITMQ_MAX_DELIVERY_ATTEMPTS) decide.'
|
|
3686
|
-
);
|
|
3687
|
-
}
|
|
3688
|
-
|
|
3689
|
-
const classify = options.classify === undefined
|
|
3690
|
-
? deliveryPolicy.defaultClassification
|
|
3691
|
-
: options.classify;
|
|
3692
|
-
|
|
3693
|
-
if (typeof classify !== 'function') {
|
|
3694
|
-
throw new ValidationError(
|
|
3695
|
-
'[RabbitMQClient] Invalid error classifier - classify must be a function returning '
|
|
3696
|
-
+ `'transient' or 'permanent', got ${typeof options.classify}. Fix: pass consume(queue, handler, `
|
|
3697
|
-
+ '{ classify: (error) => … }), or leave it out and every error is treated as transient.'
|
|
3698
|
-
);
|
|
3699
|
-
}
|
|
3700
|
-
|
|
3701
|
-
const policy = { maxAttempts, classify };
|
|
4416
|
+
// The rules have ONE owner, exported for every consumer's test double
|
|
4417
|
+
// (d.1005): `deliveryPolicy.validateConsumeOptions`.
|
|
4418
|
+
const policy = deliveryPolicy.validateConsumeOptions(
|
|
4419
|
+
options,
|
|
4420
|
+
() => runtimeCfg.get('maxDeliveryAttempts', this._config.maxDeliveryAttempts)
|
|
4421
|
+
);
|
|
3702
4422
|
const onDeadLetter = options.onDeadLetter;
|
|
3703
4423
|
if (onDeadLetter !== undefined && typeof onDeadLetter !== 'function') {
|
|
3704
4424
|
throw new ValidationError(
|
|
@@ -3786,120 +4506,162 @@ class RabbitMQClient extends EventEmitter {
|
|
|
3786
4506
|
}
|
|
3787
4507
|
});
|
|
3788
4508
|
|
|
3789
|
-
//
|
|
4509
|
+
// ONE delivery raises this counter once and lowers it once. The
|
|
4510
|
+
// `finally` at the end of this callback is that second half, and since
|
|
4511
|
+
// d.680 it is the ONLY place the counter goes down
|
|
4512
|
+
// (`change-discipline.md` § One rail per concern).
|
|
4513
|
+
//
|
|
4514
|
+
// MEASURED on production (d.680, 2026-09-20, the dispatcher):
|
|
4515
|
+
//
|
|
4516
|
+
// Prefetch utilization high: queue 'workflow.completed' has 43/5
|
|
4517
|
+
// messages in-flight (860%)
|
|
4518
|
+
//
|
|
4519
|
+
// — and the number grew with every workflow, never falling. Until d.680
|
|
4520
|
+
// the decrement lived in the branches that settle a message HERE: the
|
|
4521
|
+
// auto-ack, the delivery policy, and the rejection of an unreadable
|
|
4522
|
+
// counter. Every OTHER way a delivery can end left the increment
|
|
4523
|
+
// standing — a handler that settles the message itself (the public
|
|
4524
|
+
// `ack()`/`nack()` set `msg._mqProcessed`, which is exactly what makes
|
|
4525
|
+
// the auto-ack branch stand down), a `noAck` consumer, and the two early
|
|
4526
|
+
// returns taken when the channel died under a running handler.
|
|
4527
|
+
//
|
|
4528
|
+
// Nothing stalled, because the window is enforced by the BROKER and not
|
|
4529
|
+
// by this number. The only casualty was the alarm, which from then on
|
|
4530
|
+
// fired for every delivery whatever the load — the warning that cannot
|
|
4531
|
+
// fail to happen that d.550 removed for prefetch 1, and that
|
|
4532
|
+
// `automation-gates.md` §5 counts as a defect of its own.
|
|
4533
|
+
//
|
|
4534
|
+
// The counter is therefore bound to the DELIVERY, not to the branch that
|
|
4535
|
+
// settles it: it measures what this consumer is holding, and it stops
|
|
4536
|
+
// holding a delivery when this callback returns — whoever settled it,
|
|
4537
|
+
// and however many times.
|
|
3790
4538
|
const tracking = this._prefetchTracking.get(queue);
|
|
3791
4539
|
if (tracking) {
|
|
3792
4540
|
tracking.inFlight++;
|
|
3793
4541
|
tracking.lastCheck = Date.now();
|
|
3794
4542
|
this._checkPrefetchUtilization(queue, tracking);
|
|
3795
4543
|
}
|
|
3796
|
-
|
|
3797
|
-
// Which attempt this delivery is, read from the header this library
|
|
3798
|
-
// writes (`../config/deliveryPolicy.js`). `null` = the header carries a
|
|
3799
|
-
// value this library never wrote, so the message cannot be bounded and
|
|
3800
|
-
// the handler is NOT run for it; it is rejected below.
|
|
3801
|
-
const attemptsMade = deliveryPolicy.readAttemptsMade(msgHeaders);
|
|
3802
|
-
const attempt = attemptsMade === null ? null : attemptsMade + 1;
|
|
3803
|
-
|
|
3804
|
-
if (attempt === null) {
|
|
3805
|
-
this._logger.error(
|
|
3806
|
-
'[RabbitMQClient] Unreadable delivery counter - the '
|
|
3807
|
-
+ `${deliveryPolicy.ATTEMPTS_HEADER} header of a message on "${queue}" holds `
|
|
3808
|
-
+ `${JSON.stringify(msgHeaders[deliveryPolicy.ATTEMPTS_HEADER])}, which is not a `
|
|
3809
|
-
+ 'non-negative integer. Expected: only this library writes that header. '
|
|
3810
|
-
+ 'Fix: the message is rejected into the dead-letter queue unread — reading a '
|
|
3811
|
-
+ 'malformed counter as zero would make the redelivery loop unbounded again.',
|
|
3812
|
-
{ queue }
|
|
3813
|
-
);
|
|
3814
|
-
await this._rejectDelivery({
|
|
3815
|
-
msg,
|
|
3816
|
-
queue,
|
|
3817
|
-
channelForMsg,
|
|
3818
|
-
noAck,
|
|
3819
|
-
error: new Error(
|
|
3820
|
-
`[RabbitMQClient] Message on "${queue}" carries an unreadable `
|
|
3821
|
-
+ `${deliveryPolicy.ATTEMPTS_HEADER} header`
|
|
3822
|
-
),
|
|
3823
|
-
classification: deliveryPolicy.UNCLASSIFIABLE,
|
|
3824
|
-
attempts: null,
|
|
3825
|
-
maxAttempts: policy.maxAttempts,
|
|
3826
|
-
onDeadLetter
|
|
3827
|
-
});
|
|
3828
|
-
if (tracking) {
|
|
3829
|
-
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
3830
|
-
tracking.lastCheck = Date.now();
|
|
3831
|
-
}
|
|
3832
|
-
return;
|
|
3833
|
-
}
|
|
3834
4544
|
|
|
3835
4545
|
try {
|
|
3836
|
-
//
|
|
3837
|
-
//
|
|
3838
|
-
//
|
|
3839
|
-
//
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
4546
|
+
// Which attempt this delivery is, read from the header this library
|
|
4547
|
+
// writes (`../config/deliveryPolicy.js`). `null` = the header carries a
|
|
4548
|
+
// value this library never wrote, so the message cannot be bounded and
|
|
4549
|
+
// the handler is NOT run for it; it is rejected below.
|
|
4550
|
+
const attemptsMade = deliveryPolicy.readAttemptsMade(msgHeaders);
|
|
4551
|
+
const attempt = attemptsMade === null ? null : attemptsMade + 1;
|
|
4552
|
+
|
|
4553
|
+
if (attempt === null) {
|
|
4554
|
+
this._logger.error(
|
|
4555
|
+
'[RabbitMQClient] Unreadable delivery counter - the '
|
|
4556
|
+
+ `${deliveryPolicy.ATTEMPTS_HEADER} header of a message on "${queue}" holds `
|
|
4557
|
+
+ `${JSON.stringify(msgHeaders[deliveryPolicy.ATTEMPTS_HEADER])}, which is not a `
|
|
4558
|
+
+ 'non-negative integer. Expected: only this library writes that header. '
|
|
4559
|
+
+ 'Fix: the message is rejected into the dead-letter queue unread — reading a '
|
|
4560
|
+
+ 'malformed counter as zero would make the redelivery loop unbounded again.',
|
|
4561
|
+
{ queue }
|
|
4562
|
+
);
|
|
4563
|
+
await this._rejectDelivery({
|
|
4564
|
+
msg,
|
|
4565
|
+
queue,
|
|
4566
|
+
channelForMsg,
|
|
4567
|
+
noAck,
|
|
4568
|
+
error: new Error(
|
|
4569
|
+
`[RabbitMQClient] Message on "${queue}" carries an unreadable `
|
|
4570
|
+
+ `${deliveryPolicy.ATTEMPTS_HEADER} header`
|
|
4571
|
+
),
|
|
4572
|
+
classification: deliveryPolicy.UNCLASSIFIABLE,
|
|
4573
|
+
attempts: null,
|
|
4574
|
+
maxAttempts: policy.maxAttempts,
|
|
4575
|
+
onDeadLetter
|
|
4576
|
+
});
|
|
4577
|
+
return;
|
|
4578
|
+
}
|
|
4579
|
+
|
|
4580
|
+
try {
|
|
4581
|
+
// The handler is told WHICH attempt it is on, so a side effect that
|
|
4582
|
+
// must happen once — the RPC error reply above all — happens on the
|
|
4583
|
+
// last attempt only, instead of once per redelivery. A handler that
|
|
4584
|
+
// takes one argument ignores the second.
|
|
4585
|
+
await onMessage(msg, {
|
|
4586
|
+
attempt,
|
|
4587
|
+
maxAttempts: policy.maxAttempts,
|
|
4588
|
+
isFinalAttempt: attempt >= policy.maxAttempts
|
|
4589
|
+
});
|
|
4590
|
+
// Acknowledge message after successful processing
|
|
4591
|
+
if (!noAck && !msg._mqProcessed) {
|
|
4592
|
+
// Liveness comes from the ONE source the channel handlers set
|
|
4593
|
+
// first-hand (`_isChannelAlive()`, d.260). The reading this
|
|
4594
|
+
// replaced, `channelForMsg.closed`, asked amqplib for a property it
|
|
4595
|
+
// does not define, so it answered "alive" for the channel the
|
|
4596
|
+
// broker had just killed and handed the corpse an `ack`.
|
|
4597
|
+
if (!this._isChannelAlive(channelForMsg)) {
|
|
4598
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] [CONSUMER] Cannot ack - consumer channel is closed/recreated (message will be requeued by broker)');
|
|
3864
4599
|
return;
|
|
3865
4600
|
}
|
|
3866
|
-
|
|
4601
|
+
try {
|
|
4602
|
+
channelForMsg.ack(msg);
|
|
4603
|
+
msg._mqProcessed = true;
|
|
4604
|
+
} catch (ackErr) {
|
|
4605
|
+
// A REFUSED CALL IS A CLASS, NOT A SENTENCE (d.698). amqplib replaces a
|
|
4606
|
+
// dead channel's send methods with one that throws its own
|
|
4607
|
+
// `IllegalOperationError` (`invalidateSend()`/`invalidOp()`,
|
|
4608
|
+
// `lib/channel.js`), so that class IS "the channel would not take
|
|
4609
|
+
// this ack" — and it is the only thing `ack()` ever throws.
|
|
4610
|
+
//
|
|
4611
|
+
// The sentence branch this replaced listed `'unknown delivery tag'`,
|
|
4612
|
+
// `'PRECONDITION_FAILED'` and `'Channel closed'`. The first two cannot
|
|
4613
|
+
// arrive: an ack with a tag the broker does not hold does not throw at all
|
|
4614
|
+
// — the broker answers by closing the channel with a 406 frame on its
|
|
4615
|
+
// `error` event (measured live, d.698). The third is only ONE of amqplib's
|
|
4616
|
+
// two wordings, so the other, `'Channel closing'` (`toClosing()`, `:167`),
|
|
4617
|
+
// was rethrown — into the handler's own catch, which republished a RETRY
|
|
4618
|
+
// COPY of a message whose handler had already succeeded.
|
|
4619
|
+
if (ackErr instanceof IllegalOperationError) {
|
|
4620
|
+
msg._mqProcessed = true;
|
|
4621
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] [CONSUMER] Ack refused by a closed/closing channel (message delivery will be handled by broker)', { error: ackErr.message });
|
|
4622
|
+
return;
|
|
4623
|
+
}
|
|
4624
|
+
throw ackErr;
|
|
4625
|
+
}
|
|
4626
|
+
this._logger.debug('[RabbitMQClient] MSG_ACKED', {
|
|
4627
|
+
workflow_id: wfId,
|
|
4628
|
+
action: 'MSG_ACKED',
|
|
4629
|
+
handler: 'consume',
|
|
4630
|
+
function: 'RabbitMQClient.consume',
|
|
4631
|
+
input: { queue },
|
|
4632
|
+
output: { status: 'acknowledged', queue }
|
|
4633
|
+
});
|
|
3867
4634
|
}
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
4635
|
+
} catch (handlerErr) {
|
|
4636
|
+
// The dead-letter policy decides what a failed handler costs: one more
|
|
4637
|
+
// attempt (a copy carrying the advanced counter, then an ack of the
|
|
4638
|
+
// original) or a rejection into `<svc>.dlq`. Until 2026-09-11 this
|
|
4639
|
+
// branch was an unconditional `nack(msg, false, true)` with no counter,
|
|
4640
|
+
// so a permanently failing message was redelivered for ever and the DLQ
|
|
4641
|
+
// the topology declares was never reached from the client side
|
|
4642
|
+
// (`docs/governance/confirmations/mq-consumer-contract.md` 002 point 3).
|
|
4643
|
+
await this._applyDeliveryPolicy({
|
|
4644
|
+
msg,
|
|
4645
|
+
queue,
|
|
4646
|
+
channelForMsg,
|
|
4647
|
+
noAck,
|
|
4648
|
+
error: handlerErr,
|
|
4649
|
+
attempt,
|
|
4650
|
+
policy,
|
|
4651
|
+
onDeadLetter
|
|
3875
4652
|
});
|
|
3876
|
-
|
|
3877
|
-
if (tracking) {
|
|
3878
|
-
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
3879
|
-
tracking.lastCheck = Date.now();
|
|
3880
|
-
}
|
|
3881
4653
|
}
|
|
3882
|
-
}
|
|
3883
|
-
// The dead-letter policy decides what a failed handler costs: one more
|
|
3884
|
-
// attempt (a copy carrying the advanced counter, then an ack of the
|
|
3885
|
-
// original) or a rejection into `<svc>.dlq`. Until 2026-09-11 this
|
|
3886
|
-
// branch was an unconditional `nack(msg, false, true)` with no counter,
|
|
3887
|
-
// so a permanently failing message was redelivered for ever and the DLQ
|
|
3888
|
-
// the topology declares was never reached from the client side
|
|
3889
|
-
// (`docs/governance/confirmations/mq-consumer-contract.md` 002 point 3).
|
|
3890
|
-
await this._applyDeliveryPolicy({
|
|
3891
|
-
msg,
|
|
3892
|
-
queue,
|
|
3893
|
-
channelForMsg,
|
|
3894
|
-
noAck,
|
|
3895
|
-
error: handlerErr,
|
|
3896
|
-
attempt,
|
|
3897
|
-
policy,
|
|
3898
|
-
onDeadLetter
|
|
3899
|
-
});
|
|
4654
|
+
} finally {
|
|
3900
4655
|
if (tracking) {
|
|
3901
4656
|
tracking.inFlight = Math.max(0, tracking.inFlight - 1);
|
|
3902
4657
|
tracking.lastCheck = Date.now();
|
|
4658
|
+
// The verdict is taken on BOTH edges (d.736a). Until this call the
|
|
4659
|
+
// window was measured only while it FILLED, so "the queue came back
|
|
4660
|
+
// under the threshold" was a fact nothing ever stated — measured in
|
|
4661
|
+
// `api_delivery_endpoint`'s log on 2026-09-18: three warnings about
|
|
4662
|
+
// one standing condition in 68 ms, and no line of any kind about its
|
|
4663
|
+
// end. See `_checkPrefetchUtilization()`.
|
|
4664
|
+
this._checkPrefetchUtilization(queue, tracking);
|
|
3903
4665
|
}
|
|
3904
4666
|
}
|
|
3905
4667
|
};
|
|
@@ -4079,7 +4841,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4079
4841
|
// Re-register all consumers - use internal helper to avoid the "channel already open" check
|
|
4080
4842
|
consumerReRegResult = await this._reRegisterConsumers();
|
|
4081
4843
|
channelState.consumer = true;
|
|
4082
|
-
|
|
4844
|
+
// The same summary rule as `_ensureConsumerChannel()` (d.1003d, d.1003e): a
|
|
4845
|
+
// recovery that left a consumer unattached is not a ✓ line.
|
|
4846
|
+
if (consumerReRegResult.failed > 0) {
|
|
4847
|
+
const unattached = this._unattachedConsumerQueues();
|
|
4848
|
+
this._logger.warn(
|
|
4849
|
+
`[RabbitMQClient] Consumer channel recreated, but ${unattached.length} consumer(s) are not attached: `
|
|
4850
|
+
+ `${unattached.join(', ')} (re-registered: ${consumerReRegResult.reRegistered}, failed: ${consumerReRegResult.failed})`
|
|
4851
|
+
);
|
|
4852
|
+
} else {
|
|
4853
|
+
this._logger.info(
|
|
4854
|
+
`[RabbitMQClient] ✓ Consumer channel recreated (re-registered: ${consumerReRegResult.reRegistered}, failed: ${consumerReRegResult.failed})`
|
|
4855
|
+
);
|
|
4856
|
+
}
|
|
4083
4857
|
} catch (err) {
|
|
4084
4858
|
this._logger.error('[RabbitMQClient] Failed to recreate consumer channel:', err.message);
|
|
4085
4859
|
this._consumerChannel = null;
|
|
@@ -4187,7 +4961,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4187
4961
|
* Attach event handlers to publisher channel
|
|
4188
4962
|
* @private
|
|
4189
4963
|
*/
|
|
4190
|
-
_attachPublisherChannelHandlers(channel) {
|
|
4964
|
+
_attachPublisherChannelHandlers(channel, role) {
|
|
4191
4965
|
// One closure, one notification. amqplib announces a broker-side channel
|
|
4192
4966
|
// death TWICE — `error` first, then `close` (measured against amqplib 0.10.9
|
|
4193
4967
|
// on the live broker, 2026-09-12: a `checkQueue()` on a missing queue yields
|
|
@@ -4200,6 +4974,13 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4200
4974
|
// without a preceding `error`. The `error` handler only records WHY the
|
|
4201
4975
|
// channel is dying, here, for the one notification to carry.
|
|
4202
4976
|
channel._closeError = null;
|
|
4977
|
+
// The publishes this channel is still carrying, and the snapshot of that number taken
|
|
4978
|
+
// when a close frame arrives — `_closeFrameNamesThisSend()` is the one reader of both
|
|
4979
|
+
// (d.723). Declared here because this is the one place a PUBLISHER channel is
|
|
4980
|
+
// prepared, and a member nobody declared would make that reading answer for
|
|
4981
|
+
// `undefined` instead of for a number.
|
|
4982
|
+
channel._unconfirmedSends = 0;
|
|
4983
|
+
channel._unconfirmedAtCloseFrame = null;
|
|
4203
4984
|
channel._alive = true;
|
|
4204
4985
|
channel.on('error', async (err) => {
|
|
4205
4986
|
channel._alive = false;
|
|
@@ -4208,19 +4989,46 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4208
4989
|
}
|
|
4209
4990
|
const reason = `Publisher channel error: ${err.message} (code: ${err.code || 'unknown'})`;
|
|
4210
4991
|
channel._closeReason = reason;
|
|
4992
|
+
// How many publishes this frame could possibly be an answer about (d.723). Taken
|
|
4993
|
+
// HERE and nowhere else: amqplib emits `error` before `close`, and it is `close`
|
|
4994
|
+
// that rejects the unconfirmed publishes — by the time their callbacks ask, the
|
|
4995
|
+
// count has already been unwound. `_closeFrameNamesThisSend()` reads it.
|
|
4996
|
+
channel._unconfirmedAtCloseFrame = channel._unconfirmedSends;
|
|
4997
|
+
// WHY the channel is dying is recorded FIRST, for every reason there is.
|
|
4998
|
+
//
|
|
4999
|
+
// Since d.685a the publish path answers a refused send from this member
|
|
5000
|
+
// (`_refusedSendPermanent()`), so a branch that returns before writing it leaves the
|
|
5001
|
+
// publish path blind: the send reads a channel that died for no stated reason and
|
|
5002
|
+
// calls it transient — retried `publishMaxRetries` times and then buffered, for a
|
|
5003
|
+
// message the broker refused. The `'unknown delivery tag'` branch below did exactly
|
|
5004
|
+
// that until d.696, and the frame it catches is a NUMBERED refusal like any other
|
|
5005
|
+
// (measured live, d.693: `Error` with `code: 406`, arriving on this very event).
|
|
5006
|
+
// Recording first and branching afterwards changes what the branch DOES in nothing.
|
|
5007
|
+
channel._closeError = err;
|
|
4211
5008
|
|
|
4212
|
-
//
|
|
4213
|
-
//
|
|
4214
|
-
//
|
|
4215
|
-
|
|
4216
|
-
|
|
5009
|
+
// A 406 ON THE PUBLISHER CHANNEL IS THE PUBLISH PATH'S BUSINESS, and the CODE is
|
|
5010
|
+
// what says so (d.698).
|
|
5011
|
+
//
|
|
5012
|
+
// The publisher channel is a ConfirmChannel; this client sends no `basic.ack` on it,
|
|
5013
|
+
// so the sentence this branch read until d.698 — `'unknown delivery tag'` — never
|
|
5014
|
+
// reached it. What does reach it is the broker refusing the SEND with 406: a
|
|
5015
|
+
// `user_id` property that is not the authenticated user (measured d.685a, `classId
|
|
5016
|
+
// 60 / methodId 40`). That refusal is already answered where it belongs — the send
|
|
5017
|
+
// reads this very `_closeError` and rejects the caller with a `PermanentPublishError`
|
|
5018
|
+
// (`_refusedSendPermanent()`) — so raising it a second time as a client fault would
|
|
5019
|
+
// report one event twice, once to the publisher and once to everybody.
|
|
5020
|
+
//
|
|
5021
|
+
// Only 406. A `404 NOT-FOUND` says nothing declares the exchange, which no publisher
|
|
5022
|
+
// can fix and no caller of THIS message is the right audience for, and a codeless
|
|
5023
|
+
// death is not an answer about any request at all: both stay raised, below.
|
|
5024
|
+
if (err.code === AMQP_PRECONDITION_FAILED) {
|
|
5025
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Publisher channel refused the send with ${AMQP_PRECONDITION_FAILED} (answered to the publisher): ${err.message}`);
|
|
4217
5026
|
// Don't try to recreate - channel is already closing
|
|
4218
|
-
// Don't emit error - publish
|
|
5027
|
+
// Don't emit error - the publish path answers the caller from `_closeError`
|
|
4219
5028
|
return;
|
|
4220
5029
|
}
|
|
4221
|
-
|
|
5030
|
+
|
|
4222
5031
|
this._logger.error(`[RabbitMQClient] [mq-client-core] ${reason}`);
|
|
4223
|
-
channel._closeError = err;
|
|
4224
5032
|
// Recovery is NOT started here — one rail, the `close` handler (d.261).
|
|
4225
5033
|
this.emit('error', err);
|
|
4226
5034
|
});
|
|
@@ -4242,15 +5050,37 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4242
5050
|
if (this._channel === channel) {
|
|
4243
5051
|
this._channel = null;
|
|
4244
5052
|
}
|
|
5053
|
+
// The same obituary for the other send role (d.724). Without this line a dead
|
|
5054
|
+
// telemetry channel would stay in its member and every later telemetry publish
|
|
5055
|
+
// would be handed a corpse — `_ensurePublisherChannel('telemetry')` recreates only
|
|
5056
|
+
// what it finds cleared.
|
|
5057
|
+
if (this._telemetryChannel === channel) {
|
|
5058
|
+
this._telemetryChannel = null;
|
|
5059
|
+
}
|
|
4245
5060
|
|
|
4246
5061
|
// Only try to recreate if connection is available
|
|
4247
5062
|
// If connection is closed, reconnection logic will handle channel recreation
|
|
5063
|
+
//
|
|
5064
|
+
// Obnovuje se ROLE, která zemřela (d.1003b): tenhle handler visí na obou send
|
|
5065
|
+
// kanálech a do d.1003b volal `_ensurePublisherChannel()` bez role — po smrti
|
|
5066
|
+
// telemetry kanálu tedy „obnovil" živý publisher (no-op) a telemetry zůstal
|
|
5067
|
+
// prázdný až do příští události.
|
|
4248
5068
|
if (this._connectionAlive) {
|
|
4249
5069
|
try {
|
|
4250
|
-
await this._ensurePublisherChannel();
|
|
5070
|
+
await this._ensurePublisherChannel(role);
|
|
4251
5071
|
} catch (err) {
|
|
4252
|
-
//
|
|
4253
|
-
|
|
5072
|
+
// Přechodné selhání obnovy (spojení odchází — d.1000/d.1003) je jen `warn`:
|
|
5073
|
+
// kanál zůstane prázdný a příští publish ho po reconnectu obnoví sám
|
|
5074
|
+
// (`_ensurePublisherChannel()`). Cokoli jiného je nehlášené selhání, a to
|
|
5075
|
+
// má v tomhle klientovi kolej `error` (viz `_reportRecoveryFailure()`) —
|
|
5076
|
+
// do d.1003 ho tenhle catch polykal jako `warn`, takže se o trvale
|
|
5077
|
+
// nepoužitelném kanálu dozvěděl až první, kdo zkusil publikovat.
|
|
5078
|
+
if (err instanceof TransientPublishError) {
|
|
5079
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate ${role} channel (connection may be closing): ${err.message}`);
|
|
5080
|
+
} else {
|
|
5081
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Failed to recreate ${role} channel after it closed: ${err.message}`);
|
|
5082
|
+
this.emit('error', err);
|
|
5083
|
+
}
|
|
4254
5084
|
}
|
|
4255
5085
|
}
|
|
4256
5086
|
});
|
|
@@ -4305,12 +5135,27 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4305
5135
|
try {
|
|
4306
5136
|
await this._ensureQueueChannel();
|
|
4307
5137
|
} catch (err) {
|
|
4308
|
-
//
|
|
4309
|
-
|
|
5138
|
+
// The connection is going away — amqplib's terminal-state class, measured
|
|
5139
|
+
// live (d.693 step b): `createChannel()` on a closed connection answers
|
|
5140
|
+
// `IllegalOperationError('Connection closed (by client)')`. The class is the
|
|
5141
|
+
// fact; the sentence was only carrying it (d.696).
|
|
5142
|
+
//
|
|
5143
|
+
// It is read from `cause`, because the rail this failure travels wraps it:
|
|
5144
|
+
// `_ensureQueueChannel()` answers with a `ConnectionError` naming the fix and
|
|
5145
|
+
// keeps amqplib's object underneath. One level, and first-hand — that IS the
|
|
5146
|
+
// wrapper's own cause, not a chain walked in hope. The rule itself has one
|
|
5147
|
+
// body, `_connectionIsGoingAway()` (d.1003c), shared with the send and the
|
|
5148
|
+
// consumer channel.
|
|
5149
|
+
//
|
|
5150
|
+
// Anything else is an unannounced failure and goes to `error` (d.1003c) —
|
|
5151
|
+
// until then it was a `warn`, like the transient case. Recovery is NOT added
|
|
5152
|
+
// here: the channel stays cleared and the next queue operation recreates it
|
|
5153
|
+
// (`_requireQueueChannel()`), exactly as before.
|
|
5154
|
+
if (this._connectionIsGoingAway(err.cause)) {
|
|
4310
5155
|
this._logger.warn(`[RabbitMQClient] [mq-client-core] Queue channel close during connection closure (expected - reconnection will handle)`);
|
|
4311
5156
|
} else {
|
|
4312
|
-
|
|
4313
|
-
this.
|
|
5157
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Failed to recreate queue channel after it closed: ${err.message} (the next queue operation tries again)`);
|
|
5158
|
+
this.emit('error', err);
|
|
4314
5159
|
}
|
|
4315
5160
|
}
|
|
4316
5161
|
} else {
|
|
@@ -4365,8 +5210,19 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4365
5210
|
try {
|
|
4366
5211
|
await this._ensureConsumerChannel();
|
|
4367
5212
|
} catch (err) {
|
|
4368
|
-
//
|
|
4369
|
-
|
|
5213
|
+
// Táž kolej jako u send kanálu (d.1003b): selhání, které jen hlásí odcházející
|
|
5214
|
+
// spojení, je `warn` — reconnect kanál obnoví a konzumenty zaregistruje znovu.
|
|
5215
|
+
// Cokoli jiného je nehlášené selhání a jde na `error`; do d.1003b ho tenhle
|
|
5216
|
+
// catch polykal jako `warn`, takže konzumenti tiše přestali dostávat zprávy.
|
|
5217
|
+
// `_ensureConsumerChannel()` odpovídá `ConnectionError` s chybou amqplibu
|
|
5218
|
+
// v `cause` — jeden článek, vlastní příčina obalu (vzor obnovy frontového
|
|
5219
|
+
// kanálu níž, d.696).
|
|
5220
|
+
if (this._connectionIsGoingAway(err.cause)) {
|
|
5221
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel (connection may be closing): ${err.message}`);
|
|
5222
|
+
} else {
|
|
5223
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Failed to recreate consumer channel after it closed: ${err.message}`);
|
|
5224
|
+
this.emit('error', err);
|
|
5225
|
+
}
|
|
4370
5226
|
}
|
|
4371
5227
|
}
|
|
4372
5228
|
});
|
|
@@ -4389,6 +5245,42 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4389
5245
|
* The threshold itself is unchanged and no new one is introduced: a window
|
|
4390
5246
|
* larger than one message is measured exactly as before.
|
|
4391
5247
|
*
|
|
5248
|
+
* THE VERDICT IS TAKEN ON THE TRANSITION, BOTH WAYS (d.736a). Until this batch
|
|
5249
|
+
* the check ran only while the window FILLED, and it spoke every time it found
|
|
5250
|
+
* the condition standing. MEASURED (`api_delivery_endpoint`, 2026-09-18,
|
|
5251
|
+
* one consumer):
|
|
5252
|
+
*
|
|
5253
|
+
* 08:00:10.928 WARN … queue 'delivery.health.events' has 8/10 … (80%)
|
|
5254
|
+
* 08:00:10.995 WARN … queue 'delivery.health.events' has 8/10 … (80%)
|
|
5255
|
+
* 08:00:10.996 WARN … queue 'delivery.health.events' has 9/10 … (90%)
|
|
5256
|
+
*
|
|
5257
|
+
* — three lines in 68 ms about ONE standing condition, and never a line saying
|
|
5258
|
+
* the queue came back under the threshold. The operator reading that log can
|
|
5259
|
+
* see an alarm start and can never see it end, which makes the line unusable
|
|
5260
|
+
* for the only question it exists to answer.
|
|
5261
|
+
*
|
|
5262
|
+
* The shape is the one the monitoring concept settled the same day: a rail that
|
|
5263
|
+
* says a standing condition ONCE — OPEN states it, everything after it is the
|
|
5264
|
+
* same episode, and the end is a RECORD rather than a second alarm
|
|
5265
|
+
* (`infra/api_monitoring`, `c996215c` telemetry-intake rejections, `1b1c3912`
|
|
5266
|
+
* registry health events; `watchers/alertLifecycle.js`
|
|
5267
|
+
* § closeRecoveredInfrastructureAlerts: "each episode's one message went out
|
|
5268
|
+
* when it opened"). This library keeps no episode store, so its record is the
|
|
5269
|
+
* log: the onset stays a `warn`, the return is one `info`.
|
|
5270
|
+
*
|
|
5271
|
+
* WHAT DOES NOT CHANGE: the callback and the `prefetch:high-utilization` event
|
|
5272
|
+
* still announce the onset, with the same payload — they simply fire once per
|
|
5273
|
+
* episode now. The return gets no callback and no event of its own: both would
|
|
5274
|
+
* be a declaration with no consumer anywhere in this workspace
|
|
5275
|
+
* (`change-discipline.md` § Removing something removes its declaration), while
|
|
5276
|
+
* the log line has a measured reader.
|
|
5277
|
+
*
|
|
5278
|
+
* NO HYSTERESIS is introduced. Monitoring damps a flapping subject with a floor
|
|
5279
|
+
* taken from configuration, and it needs one because a notification stands
|
|
5280
|
+
* behind each of its episodes; here the episode is a log line, and a floor
|
|
5281
|
+
* written into this file would be a number with no owner
|
|
5282
|
+
* (`architecture-principles.md` §2). The edge alone removes the repetition.
|
|
5283
|
+
*
|
|
4392
5284
|
* @private
|
|
4393
5285
|
* @param {string} queue - Queue name
|
|
4394
5286
|
* @param {Object} tracking - Prefetch tracking object
|
|
@@ -4399,38 +5291,65 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4399
5291
|
}
|
|
4400
5292
|
|
|
4401
5293
|
if (tracking.prefetchCount <= 1) {
|
|
4402
|
-
|
|
5294
|
+
// One-message window: 0 % or 100 %, and 100 % is the working state. There is
|
|
5295
|
+
// no condition to stand, so an episode carried over from a larger window
|
|
5296
|
+
// (the prefetch of a re-registered consumer can change) ends here — silently,
|
|
5297
|
+
// because what it was measured against no longer exists.
|
|
5298
|
+
tracking.utilizationHigh = false;
|
|
5299
|
+
return;
|
|
4403
5300
|
}
|
|
4404
5301
|
|
|
4405
5302
|
const utilization = tracking.inFlight / tracking.prefetchCount;
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
5303
|
+
const high = utilization >= this._prefetchUtilizationThreshold;
|
|
5304
|
+
|
|
5305
|
+
if (high === tracking.utilizationHigh) {
|
|
5306
|
+
return; // The same condition as at the last verdict — it has already been said.
|
|
5307
|
+
}
|
|
5308
|
+
|
|
5309
|
+
tracking.utilizationHigh = high;
|
|
5310
|
+
|
|
5311
|
+
const state = `queue '${queue}' has ${tracking.inFlight}/${tracking.prefetchCount} messages in-flight (${Math.round(utilization * 100)}%, threshold: ${Math.round(this._prefetchUtilizationThreshold * 100)}%)`;
|
|
5312
|
+
|
|
5313
|
+
if (!high) {
|
|
5314
|
+
// The end of the episode: a record, not an alarm.
|
|
5315
|
+
this._logger.info(`[RabbitMQClient] [mq-client-core] Prefetch utilization back below threshold: ${state}`);
|
|
5316
|
+
return;
|
|
5317
|
+
}
|
|
5318
|
+
|
|
5319
|
+
const message = `Prefetch utilization high: ${state}`;
|
|
5320
|
+
this._logger.warn(`[RabbitMQClient] [mq-client-core] ⚠️ ${message}`);
|
|
5321
|
+
|
|
5322
|
+
// Call alert callback if provided
|
|
5323
|
+
if (this._prefetchAlertCallback) {
|
|
5324
|
+
try {
|
|
5325
|
+
this._prefetchAlertCallback(queue, utilization, tracking.inFlight, tracking.prefetchCount);
|
|
5326
|
+
} catch (alertErr) {
|
|
5327
|
+
this._logger.error(`[RabbitMQClient] [mq-client-core] Prefetch alert callback error:`, alertErr.message);
|
|
4418
5328
|
}
|
|
4419
|
-
|
|
4420
|
-
// Emit event for external listeners
|
|
4421
|
-
this.emit('prefetch:high-utilization', {
|
|
4422
|
-
queue,
|
|
4423
|
-
utilization,
|
|
4424
|
-
inFlight: tracking.inFlight,
|
|
4425
|
-
prefetchCount: tracking.prefetchCount,
|
|
4426
|
-
threshold: this._prefetchUtilizationThreshold,
|
|
4427
|
-
timestamp: new Date().toISOString()
|
|
4428
|
-
});
|
|
4429
5329
|
}
|
|
5330
|
+
|
|
5331
|
+
// Emit event for external listeners
|
|
5332
|
+
this.emit('prefetch:high-utilization', {
|
|
5333
|
+
queue,
|
|
5334
|
+
utilization,
|
|
5335
|
+
inFlight: tracking.inFlight,
|
|
5336
|
+
prefetchCount: tracking.prefetchCount,
|
|
5337
|
+
threshold: this._prefetchUtilizationThreshold,
|
|
5338
|
+
timestamp: new Date().toISOString()
|
|
5339
|
+
});
|
|
4430
5340
|
}
|
|
4431
5341
|
|
|
4432
5342
|
/**
|
|
4433
5343
|
* Start periodic prefetch monitoring
|
|
5344
|
+
*
|
|
5345
|
+
* WHAT IS LEFT FOR THIS TICK TO SAY since d.736a: the verdict is taken on both
|
|
5346
|
+
* edges of every delivery (`consume()`), and `_checkPrefetchUtilization()` returns
|
|
5347
|
+
* in silence when the condition is the one already announced — so a tick that sees
|
|
5348
|
+
* the same `inFlight` and the same `prefetchCount` says nothing. It speaks only
|
|
5349
|
+
* where the verdict changed without a delivery in between, and the one quantity
|
|
5350
|
+
* that can move that way is `prefetchCount`: a consumer re-registered with a
|
|
5351
|
+
* different window (`_applyConsumerPrefetch()`).
|
|
5352
|
+
*
|
|
4434
5353
|
* @private
|
|
4435
5354
|
*/
|
|
4436
5355
|
_startPrefetchMonitoring() {
|
|
@@ -4438,7 +5357,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4438
5357
|
return; // Already started
|
|
4439
5358
|
}
|
|
4440
5359
|
|
|
4441
|
-
this._prefetchCheckTimer =
|
|
5360
|
+
this._prefetchCheckTimer = this._setInterval(() => {
|
|
4442
5361
|
const now = Date.now();
|
|
4443
5362
|
for (const [queue, tracking] of this._prefetchTracking.entries()) {
|
|
4444
5363
|
// Only check if we have recent activity (within last 30s)
|
|
@@ -4457,7 +5376,7 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4457
5376
|
*/
|
|
4458
5377
|
_stopPrefetchMonitoring() {
|
|
4459
5378
|
if (this._prefetchCheckTimer) {
|
|
4460
|
-
|
|
5379
|
+
this._clearInterval(this._prefetchCheckTimer);
|
|
4461
5380
|
this._prefetchCheckTimer = null;
|
|
4462
5381
|
}
|
|
4463
5382
|
}
|
|
@@ -4736,10 +5655,12 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4736
5655
|
this._consumerChannel.ack(msg);
|
|
4737
5656
|
msg._mqProcessed = true;
|
|
4738
5657
|
} catch (err) {
|
|
4739
|
-
|
|
4740
|
-
|
|
5658
|
+
// The same reading as the auto-ack inside `consume()` — see the note there (d.698).
|
|
5659
|
+
// amqplib's terminal-state class is the only thing `ack()` throws; anything else
|
|
5660
|
+
// reaching this catch came from outside amqplib and is a defect that must be heard.
|
|
5661
|
+
if (err instanceof IllegalOperationError) {
|
|
4741
5662
|
msg._mqProcessed = true;
|
|
4742
|
-
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot ack - consumer channel is closed/
|
|
5663
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot ack - consumer channel is closed/closing (the broker returns the unacked delivery)', { error: err.message });
|
|
4743
5664
|
return;
|
|
4744
5665
|
}
|
|
4745
5666
|
this.emit('error', err);
|
|
@@ -4777,10 +5698,10 @@ class RabbitMQClient extends EventEmitter {
|
|
|
4777
5698
|
this._consumerChannel.nack(msg, false, requeue);
|
|
4778
5699
|
msg._mqProcessed = true;
|
|
4779
5700
|
} catch (err) {
|
|
4780
|
-
|
|
4781
|
-
if (
|
|
5701
|
+
// The same reading as `ack()` above (d.698): the class amqplib refuses a call with.
|
|
5702
|
+
if (err instanceof IllegalOperationError) {
|
|
4782
5703
|
msg._mqProcessed = true;
|
|
4783
|
-
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot nack - consumer channel is closed/
|
|
5704
|
+
this._logger.warn('[RabbitMQClient] [mq-client-core] Cannot nack - consumer channel is closed/closing (the broker returns the unacked delivery)', { error: err.message });
|
|
4784
5705
|
return;
|
|
4785
5706
|
}
|
|
4786
5707
|
this.emit('error', err);
|