@onlineapps/mq-client-core 5.0.0 → 5.1.0

package/src/transports/rabbitmqClient.js

@@ -125,11 +125,10 @@ const REFUSAL_REPLY_CODES = [403, 530, 406];
  *  - `workflow_id` is the only spelling the envelope contract admits, and a
  *    reader that accepts the camelCase alias is forbidden as such, not merely a
  *    producer that writes it (`docs/biz/70-contracts/workflow-message.md` §2.2,
- *    owner decision `workflow-message-aliases` 001). The alias is still written
- *    today by the gateway's webhook submit, whose removal is its owner's
- *    (`docs/standards/JSON_NAMING_CONVENTION.md` § alias read, `infra/TODO.md`);
- *    a reader that keeps honouring it is what makes the pair impossible to
- *    retire, because nothing then shows which producers still write it.
+ *    owner decision `workflow-message-aliases` 001). No producer in `shared/**`
+ *    or `infra/**` writes the alias (re-measured d.1134: the gateway's headers
+ *    carry `workflow_id` alone, `infra/api_gateway/shared/workflowMessageHeaders.js`),
+ *    and a reader that honoured it would hide the next producer that did.
  *  - a missing id is reported as missing. `'unknown'` is a value, and a log line
  *    carrying it claims a workflow by that name — the same reason
  *    `BaseClient._emitDeadLetterEvent()` resolves an absent id to `null` and
@@ -282,6 +281,9 @@ class RabbitMQClient extends EventEmitter {
     // socket whose channels are still being recreated is not yet usable (d.340).
     // Channels do the same through `channel._alive` (see `_isChannelAlive()`).
     this._connectionAlive = false;
+    // The broker's reason while it blocks this connection (`connection.blocked`), `null` otherwise
+    // (d.1145). Written by `_onConnectionBlocked()` / `_onConnectionUnblocked()`.
+    this._connectionBlockedBy = null;
     this._channel = null; // ConfirmChannel for publish operations
     this._queueChannel = null; // Regular channel for queue operations (assertQueue, checkQueue)
     this._consumerChannel = null; // Dedicated channel for consume operations
@@ -366,12 +368,14 @@ class RabbitMQClient extends EventEmitter {
     this._publishRetryBaseDelay = this._config.publishRetryBaseDelay;
     this._publishRetryMaxDelay = this._config.publishRetryMaxDelay;
     this._publishRetryBackoffMultiplier = this._config.publishRetryBackoffMultiplier;
-    // ONE number, two consumers, and therefore a name that belongs to neither of
-    // them alone: how long this client waits for an ANSWER from the broker. The
-    // publish confirm waits for it (`_publishOnce`), and so does the whole
-    // teardown of `disconnect()` (d.283 chose deliberately not to invent a second
-    // number for that; the name it inherited then said "publish confirmation",
-    // which was true of one consumer and false of the other).
+    // ONE number, and therefore a name that belongs to none of its consumers
+    // alone: how long this client waits for an ANSWER from the broker. The
+    // publish confirm waits for it (`_publishOnce`), so does the whole teardown
+    // of `disconnect()` (d.283 chose deliberately not to invent a second number
+    // for that; the name it inherited then said "publish confirmation", which was
+    // true of one consumer and false of the other), and so does every request a
+    // channel sends and waits on (`_askBroker()`, conf `mq-broker-answer-budget`
+    // 001, d.1135).
     this._brokerAnswerTimeout = this._config.brokerAnswerTimeout;
     // How long the connect handshake may take before the attempt is abandoned.
     this._connectTimeout = this._config.connectTimeout;
@@ -785,6 +789,71 @@ class RabbitMQClient extends EventEmitter {
       this.emit('error', err);
     });
     connection.on('close', () => this._onConnectionDied());
+    // A broker in a memory or disk alarm blocks a connection that publishes and reads nothing
+    // more from it until the alarm clears; the connection stays alive, so no reconnect — and
+    // no flush — follows (measured d.1143 on the platform's image). amqplib announces the
+    // capability and re-emits both frames (`lib/connect.js`, `lib/channel_model.js`).
+    this._connectionBlockedBy = null;
+    connection.on('blocked', (reason) => this._onConnectionBlocked(connection, reason));
+    connection.on('unblocked', () => this._onConnectionUnblocked(connection));
+  }
+
+  /**
+   * The broker blocked this connection (`connection.blocked`, d.1145).
+   *
+   * Recorded as state — the publish path names it when it refuses (`_publishPathTransient()`)
+   * — and reported once as a warning. A frame of a stale connection (one this client already
+   * replaced) says nothing about the current one and is ignored.
+   *
+   * @param {Object} connection - the amqplib connection that received the frame
+   * @param {string} reason - the broker's reason (RabbitMQ: "low on memory", "low on disk")
+   * @private
+   */
+  _onConnectionBlocked(connection, reason) {
+    if (connection !== this._connection) {
+      return;
+    }
+    this._connectionBlockedBy = String(reason);
+    this._logger.warn(
+      `[RabbitMQClient] The broker blocked this connection (reason: ${this._connectionBlockedBy}) - `
+        + 'Expected: an alarm-free broker. It reads nothing this client sends until the alarm clears: '
+        + 'publishes and every request wait, and end on brokerAnswerTimeout. '
+        + 'Fix: clear the broker alarm (management API /api/health/checks/alarms names it); the buffer '
+        + 're-sends what never left once the broker unblocks the connection.'
+    );
+  }
+
+  /**
+   * The broker unblocked this connection (`connection.unblocked`, d.1145).
+   *
+   * The flush after it is the one rail `PublishLayer.flushBuffered()` with the trigger
+   * `'unblocked'`: only the entries whose frame never left the client are re-sent; the broker
+   * delivers the rest by itself (conf `mq-publish-buffer` 001). A failed flush is reported the
+   * way the flush after a reconnect reports it.
+   *
+   * @param {Object} connection - the amqplib connection that received the frame
+   * @private
+   */
+  _onConnectionUnblocked(connection) {
+    if (connection !== this._connection) {
+      return;
+    }
+    const wasBlockedBy = this._connectionBlockedBy;
+    this._connectionBlockedBy = null;
+    this._logger.warn(
+      `[RabbitMQClient] The broker unblocked this connection (it had been blocked: ${wasBlockedBy === null ? 'no blocked frame seen' : wasBlockedBy}) - `
+        + 're-sending the buffered messages whose frame never left this client.'
+    );
+    this._publishLayer.flushBuffered('unblocked').then(
+      (flushed) => {
+        this._publishMonitor.trackFlushed(flushed);
+        this._logger.info(`[RabbitMQClient] ✓ Flushed ${flushed} buffered messages after the broker unblocked the connection`);
+      },
+      (flushErr) => {
+        this._logger.warn(`[RabbitMQClient] Failed to flush buffered messages after unblocked: ${flushErr.message}`);
+        this.emit('buffer:flush:failed', { error: flushErr.message, bufferSize: this._publishLayer._buffer.size() });
+      }
+    );
   }
 
   /**
@@ -1390,6 +1459,8 @@ class RabbitMQClient extends EventEmitter {
    */
   _onConnectionDied() {
     this._connectionAlive = false;
+    // A dead connection is no longer blocked; a new one starts unblocked (d.1145).
+    this._connectionBlockedBy = null;
     this._channel = null;
     this._telemetryChannel = null;
     this._queueChannel = null;
@@ -1437,12 +1508,56 @@ class RabbitMQClient extends EventEmitter {
       return { status: 'absent', error: null };
     }
 
-    const UNANSWERED = Symbol('teardown-budget-spent');
+    const outcome = await this._awaitBrokerAnswer(() => handle.close(), budgetMs);
+
+    if (outcome.status === 'unanswered') {
+      this._logger.warn(
+        `[RabbitMQClient] The broker did not answer the ${what} close within the teardown budget `
+          + `(${this._brokerAnswerTimeout}ms) - `
+          + 'Expected: a close-ok frame for the close this client sent. '
+          + 'Fix: nothing to do here — the teardown stops waiting and drops the socket, so '
+          + 'the teardown ends (disconnect(), or a connect() that failed half-way) and the process '
+          + 'can exit; a broker that never answers a close is a broker-side or network fault, and '
+          + 'the declared brokerAnswerTimeout sets how long this client gives it.'
+      );
+      return { status: 'unanswered', error: null };
+    }
+    if (outcome.status === 'refused') {
+      return { status: 'refused', error: outcome.error };
+    }
+    return { status: 'closed', error: null };
+  }
+
+  /**
+   * Wait for the broker's answer to ONE request, never longer than `budgetMs`.
+   *
+   * THE one clock for "how long does this client wait for the broker to answer" (conf
+   * `mq-broker-answer-budget` 001, d.1135). Its two readers are `_askBroker()` — every
+   * request a channel or the connection sends and waits on — and
+   * `_closeWithinTeardownBudget()`, which asks it for the `close-ok` of a teardown with what
+   * is left of that teardown's budget. The publish confirm keeps its own timer inside the
+   * send callback (`_publishOnce()`), because it is not a promise to race: it is a callback
+   * that also has to be disarmed, counted and attributed.
+   *
+   * `request` is called here, synchronously, so a request that throws before it is sent
+   * reads as a refusal rather than escaping the budget as an exception.
+   *
+   * @param {() => Promise<*>} request - sends the request and answers its reply
+   * @param {number} budgetMs - how long the answer may take
+   * @returns {Promise<{status: 'answered', value: *, pending: Promise<*>}
+   *   |{status: 'refused', error: Error, pending: Promise<*>}
+   *   |{status: 'unanswered', pending: Promise<*>}>} `pending` is the request's own
+   *   promise, for a caller that must deal with an answer arriving after the budget
+   * @private
+   */
+  async _awaitBrokerAnswer(request, budgetMs) {
+    const pending = new Promise((resolve) => resolve(request()));
+    const UNANSWERED = Symbol('broker-answer-budget-spent');
     let deadlineTimer = null;
     const outcome = await Promise.race([
-      handle.close().then(
-        () => ({ status: 'closed', error: null }),
-        (err) => ({ status: 'refused', error: err })
+      pending.then(
+        (value) => ({ status: 'answered', value }),
+        (error) => ({ status: 'refused', error })
       ),
       new Promise((resolve) => {
         deadlineTimer = this._setTimeout(() => resolve(UNANSWERED), budgetMs);
@@ -1451,19 +1566,145 @@ class RabbitMQClient extends EventEmitter {
     this._clearTimeout(deadlineTimer);
 
     if (outcome === UNANSWERED) {
-      this._logger.warn(
-        `[RabbitMQClient] The broker did not answer the ${what} close within the teardown budget `
-          + `(${this._brokerAnswerTimeout}ms) - `
-          + 'Expected: a close-ok frame for the close this client sent. '
-          + 'Fix: nothing to do here — the teardown stops waiting and drops the socket, so '
-          + 'disconnect() ends and the process can exit; a broker that never answers a close is '
-          + 'a broker-side or network fault, and the declared brokerAnswerTimeout sets how '
-          + 'long this client gives it.'
+      return { status: 'unanswered', pending };
+    }
+    return { ...outcome, pending };
+  }
+
+  /**
+   * Ask the broker one thing and wait for its answer within `brokerAnswerTimeout`.
+   *
+   * ONE rail for every request this client sends and then waits on (conf
+   * `mq-broker-answer-budget` 001, verdict A; d.1135): queue and exchange declarations,
+   * `checkQueue`, `bindQueue`, `purgeQueue`, `deleteQueue`, `basic.consume`, `basic.cancel`,
+   * `basic.qos` and every channel open. Until d.1135 none of them had a limit. A broker in a
+   * memory alarm keeps the connection open and its heartbeats flowing while it answers
+   * nothing — measured by d.1126 on an isolated broker of the platform's image: every one of
+   * these calls still pending after 300 s, settled only when the alarm was cleared; a paused
+   * broker was ended by amqplib's heartbeat after ~90 s. The heartbeat is not part of this
+   * rail: in an alarm it never fires, so only the budget ends the wait.
+   *
+   * The budget is `brokerAnswerTimeout` and never a second number — its declared meaning is
+   * exactly this question (`config/configSchema.js`).
+   *
+   * WHEN THE BUDGET IS SPENT:
+   *   - the channel that carried the request is DISCARDED (`_discardUnansweredChannel()`):
+   *     amqplib queues the requests of one channel behind each other, so every later request
+   *     on it would wait behind the silent one. The next operation opens a fresh channel;
+   *   - a request that OPENS a channel has no channel yet; the one the broker may still open
+   *     later is closed as soon as it arrives, so an alarm does not leak channels;
+   *   - the caller gets `ConnectionError` naming the operation, its target and the budget.
+   *     The publish path translates it into `TransientPublishError`
+   *     (`_publishPathTransient()`), so retry, the buffer and `bufferOnFailure: false` apply
+   *     to it exactly as to any other transient failure.
+   *
+   * A bare `Error('Channel ended, no reply will be forthcoming')` — amqplib's
+   * `_rejectPending()` answering a request whose channel or connection died first, no class,
+   * no `code` — is the same fact ("no answer came") and becomes the same class. It is
+   * recognised by the carrier's STATE, never by the sentence (d.683): no `code`, a plain
+   * `Error`, and a carrier that is dead by the time the rejection arrives (amqplib emits
+   * `close` synchronously before the rejection's handlers run). A numbered refusal is an
+   * ANSWER and passes through untouched, as does anything already typed.
+   *
+   * @param {Object} request
+   * @param {string} request.operation - the request's name, as the message states it
+   * @param {string} request.target - what it is about (queue, exchange, channel)
+   * @param {Object|null} request.channel - the channel carrying it; `null` for a channel
+   *   open, which the connection carries
+   * @param {() => Promise<*>} send - sends the request and answers its reply
+   * @returns {Promise<*>} the broker's answer
+   * @throws {ConnectionError} when no answer came within the budget, or the carrier ended
+   *   before it could
+   * @private
+   */
+  async _askBroker({ operation, target, channel }, send) {
+    const budgetMs = this._brokerAnswerTimeout;
+    const outcome = await this._awaitBrokerAnswer(send, budgetMs);
+
+    if (outcome.status === 'answered') {
+      return outcome.value;
+    }
+
+    if (outcome.status === 'unanswered') {
+      if (channel) {
+        this._discardUnansweredChannel(channel, `${operation} ${target}`);
+      } else {
+        outcome.pending.then(
+          (late) => this._closeLateChannel(late, target),
+          () => {}
+        );
+      }
+      throw new ConnectionError(
+        `[RabbitMQClient] The broker did not answer ${operation} ${target} within brokerAnswerTimeout `
+          + `(${budgetMs}ms) - Expected: an answer to the request this client sent; a broker that `
+          + 'blocks publishers (memory or disk alarm) or is frozen keeps the connection open and '
+          + 'answers nothing. Fix: check the broker for an alarm (management API '
+          + '/api/health/checks/alarms) or a paused node; the channel that carried the request '
+          + 'is discarded and the next operation opens a fresh one. brokerAnswerTimeout is the '
+          + 'one budget for every answer this client waits for.'
+      );
+    }
+
+    const err = outcome.error;
+    const carrierDead = channel ? !this._isChannelAlive(channel) : !this._connectionAlive;
+    const bare = err instanceof Error && err.constructor === Error && err.code === undefined;
+    if (bare && carrierDead) {
+      throw new ConnectionError(
+        `[RabbitMQClient] ${operation} ${target} got no answer: the ${channel ? 'channel' : 'connection'} `
+          + 'carrying it ended before the broker answered - Expected: an answer to the request '
+          + 'this client sent. Fix: nothing is refused — the request never got a reply; run the '
+          + 'operation again once the channel is back (error.cause carries amqplib\'s rejection).',
+        err
       );
-      return { status: 'unanswered', error: null };
     }
+    throw err;
+  }
 
-    return outcome;
+  /**
+   * Take a channel whose request the broker did not answer out of service.
+   *
+   * It is marked dead first (`_isChannelAlive()`, the one liveness source — d.260), then
+   * dropped from whichever slot still holds it — only if it still holds THIS channel, the
+   * same obituary rule the close handlers keep — and then asked to close. The close is not
+   * awaited: it is an answer from the same silent broker. amqplib sends it at once and
+   * ignores every frame but `close-ok` from then on, so a broker that wakes up closes the
+   * channel cleanly (its consumers cancelled, its unacked deliveries requeued) instead of
+   * keeping it open with nobody holding it; its `close` event then reaches the handlers
+   * attached to it, exactly as for any other channel that ends.
+   *
+   * @param {Object} channel - the amqplib channel that carried the unanswered request
+   * @param {string} reason - which request went unanswered, for the post-mortem
+   * @private
+   */
+  _discardUnansweredChannel(channel, reason) {
+    channel._alive = false;
+    channel._closeReason = `No answer from the broker to ${reason} within brokerAnswerTimeout (${this._brokerAnswerTimeout}ms)`;
+    if (this._channel === channel) this._channel = null;
+    if (this._telemetryChannel === channel) this._telemetryChannel = null;
+    if (this._queueChannel === channel) this._queueChannel = null;
+    if (this._consumerChannel === channel) this._consumerChannel = null;
+    channel.close().catch((err) => {
+      this._logger.debug(`[RabbitMQClient] Discarded channel (${reason}) did not close: ${err.message}`);
+    });
+  }
+
+  /**
+   * Close a channel the broker opened after this client stopped waiting for it. Nobody
+   * holds it, so left open it would be a channel on the broker with no owner.
+   *
+   * @param {Object} channel - the late amqplib channel
+   * @param {string} what - which channel was being opened, for the log
+   * @private
+   */
+  _closeLateChannel(channel, what) {
+    this._logger.warn(
+      `[RabbitMQClient] The broker opened the ${what} after brokerAnswerTimeout (${this._brokerAnswerTimeout}ms) - `
+        + 'Expected: nothing to do; the open had already been reported as unanswered. '
+        + 'Fix: none — the late channel is closed now, so it is not left open with no owner.'
+    );
+    channel.close().catch((err) => {
+      this._logger.debug(`[RabbitMQClient] Late ${what} did not close: ${err.message}`);
+    });
   }
 
   /**
@@ -1698,7 +1939,10 @@ class RabbitMQClient extends EventEmitter {
       this._attachConnectionHandlers(this._connection);
 
       // Use ConfirmChannel to enable publisher confirms for publish operations
-      this._channel = await this._connection.createConfirmChannel();
+      this._channel = await this._askBroker(
+        { operation: 'createConfirmChannel', target: 'publisher channel', channel: null },
+        () => this._connection.createConfirmChannel()
+      );
       
       // Track channel metadata for debugging
       this._channel._createdAt = new Date().toISOString();
@@ -1711,7 +1955,10 @@ class RabbitMQClient extends EventEmitter {
       
       // Create a separate regular channel for queue operations (assertQueue, checkQueue)
       // This avoids RPC reply queue issues with ConfirmChannel.assertQueue()
-      this._queueChannel = await this._connection.createChannel();
+      this._queueChannel = await this._askBroker(
+        { operation: 'createChannel', target: 'queue channel', channel: null },
+        () => this._connection.createChannel()
+      );
       
       // Track channel metadata for debugging
       this._queueChannel._createdAt = new Date().toISOString();
@@ -1724,7 +1971,10 @@ class RabbitMQClient extends EventEmitter {
       
       // Create a dedicated channel for consume operations
       // This prevents channel conflicts between publish (ConfirmChannel) and consume operations
-      this._consumerChannel = await this._connection.createChannel();
+      this._consumerChannel = await this._askBroker(
+        { operation: 'createChannel', target: 'consumer channel', channel: null },
+        () => this._connection.createChannel()
+      );
       
       // Track channel metadata for debugging
       this._consumerChannel._createdAt = new Date().toISOString();
@@ -1743,10 +1993,15 @@ class RabbitMQClient extends EventEmitter {
       // Cleanup partially created resources
       this._connectionAlive = false;
       if (this._connection) {
-        try {
-          await this._connection.close();
-        } catch (_) {
-          /* ignore */
+        // Within the one budget (d.1135): a connect that failed because the broker stopped
+        // answering (a channel open above) would otherwise wait for a `close-ok` from that
+        // same silent broker. What the close answers is not the caller's business here —
+        // the failure that brought us into this catch is — so only a close that got no
+        // answer at all is acted on: its socket is dropped, as the teardown does.
+        const halfOpen = this._connection;
+        const closing = await this._closeWithinTeardownBudget('connection', halfOpen, this._brokerAnswerTimeout);
+        if (closing.status === 'unanswered') {
+          this._dropUnansweredConnection(halfOpen);
         }
         this._connection = null;
       }
@@ -1818,10 +2073,17 @@ class RabbitMQClient extends EventEmitter {
 
       const timeout = this._setTimeout(() => {
         done();
+        // The sentence names the lever that is READ (d.1138): the configuration key when
+        // the caller wrote it, the derivation in the constructor otherwise. It named an
+        // environment variable until d.1138 that nothing reads.
+        const budgetSource = this._config.reconnectWaitTimeout === undefined
+          ? 'not set, so derived as max(60000, maxReconnectAttempts × reconnectMaxDelay)'
+          : 'set in the client configuration';
         reject(new ConnectionError(
           `[RabbitMQClient] Reconnection timeout after ${this._reconnectWaitTimeout}ms - `
-            + 'Expected: the broker to accept a new connection within RABBITMQ_RECONNECT_WAIT_TIMEOUT. '
-            + 'Fix: check the broker is up and reachable; raise that timeout only if a longer outage must be survived in-process.'
+            + `Expected: the broker to accept a new connection within reconnectWaitTimeout (${this._reconnectWaitTimeout}ms, ${budgetSource}). `
+            + 'Fix: check the broker is up and reachable; raise reconnectWaitTimeout in the configuration handed to '
+            + 'the client only if a longer outage must be survived in-process.'
         ));
       }, this._reconnectWaitTimeout);
       
@@ -1979,7 +2241,10 @@ class RabbitMQClient extends EventEmitter {
     
     try {
       // Create new ConfirmChannel
-      const channel = await this._connection.createConfirmChannel();
+      const channel = await this._askBroker(
+        { operation: 'createConfirmChannel', target: `${role} channel`, channel: null },
+        () => this._connection.createConfirmChannel()
+      );
 
       // Track channel metadata for debugging
       channel._createdAt = new Date().toISOString();
@@ -2030,6 +2295,24 @@ class RabbitMQClient extends EventEmitter {
           err
         );
       }
+      // Třetí nosič (d.1135): broker na otevření kanálu neodpověděl v `brokerAnswerTimeout`
+      // (alarm paměti, zmrazený broker) — `_askBroker()` to hlásí `ConnectionError`, jinou
+      // chybu z otevření kanálu tahle třída nenese. Je to týž stav „odpověď nepřišla“, ne
+      // odmítnutí, takže přechodný jako oba nosiče výše. Stojí ZA nimi: spojení, které
+      // odchází, má svou vlastní (přesnější) větu i tehdy, když amqplib odmítnutí složil
+      // `_askBroker()` do `ConnectionError`.
+      if (err instanceof ConnectionError) {
+        this._logger.warn(
+          `[RabbitMQClient] [mq-client-core] Failed to recreate ${role} channel (no answer from the broker): ${msg}`
+        );
+        throw new TransientPublishError(
+          `[RabbitMQClient] ${role} channel cannot be recreated: the broker did not answer the open (transient condition) - `
+            + `Expected: an answer within brokerAnswerTimeout (${this._brokerAnswerTimeout}ms). `
+            + 'Fix: retry the publish; PublishLayer does this on its own and buffers the message meanwhile. '
+            + 'error.cause names the unanswered request.',
+          err
+        );
+      }
 
       // Ostatní chyby považujeme za permanentní problém na úrovni kanálu/spojení.
       throw new PermanentPublishError(
@@ -2062,7 +2345,10 @@ class RabbitMQClient extends EventEmitter {
     
     try {
       // Create new regular channel
-      this._queueChannel = await this._connection.createChannel();
+      this._queueChannel = await this._askBroker(
+        { operation: 'createChannel', target: 'queue channel', channel: null },
+        () => this._connection.createChannel()
+      );
       
       // Track channel metadata for debugging
       this._queueChannel._createdAt = new Date().toISOString();
@@ -2137,7 +2423,10 @@ class RabbitMQClient extends EventEmitter {
     
     try {
       // Create new regular channel
-      this._consumerChannel = await this._connection.createChannel();
+      this._consumerChannel = await this._askBroker(
+        { operation: 'createChannel', target: 'consumer channel', channel: null },
+        () => this._connection.createChannel()
+      );
       
       // Track channel metadata for debugging
       this._consumerChannel._createdAt = new Date().toISOString();
@@ -2212,6 +2501,33 @@ class RabbitMQClient extends EventEmitter {
     return { reRegistered, failed };
   }
 
+  /**
+   * The consumer channel, if it is alive — otherwise a refusal that names why.
+   *
+   * The one question every step that is about to use the consumer channel asks
+   * (`_isChannelAlive()`, the one liveness source, d.260). A refusal here is not a broker
+   * verdict: the consumer stays tracked and the next recovery re-attaches it.
+   *
+   * @param {string} action - what was about to happen, as the refusal states it
+   * @param {string} queue - the queue it is about
+   * @returns {Object} the live consumer channel
+   * @throws {ConnectionError} when the channel is dead or gone
+   * @private
+   */
+  _liveConsumerChannel(action, queue) {
+    const channel = this._consumerChannel;
+    if (!this._isChannelAlive(channel)) {
+      throw new ConnectionError(
+        `[RabbitMQClient] Cannot ${action} ${queue}: consumer channel is not available - `
+          + 'Expected: a live consumer channel for the whole re-registration. '
+          + 'Fix: nothing to do by hand — the channel died during the recovery (closed by the broker, or '
+          + 'discarded after brokerAnswerTimeout); the consumer stays tracked and the next channel recreate '
+          + 're-attaches it. Listen for consumer:re-registration:failed to see which ones wait.'
+      );
+    }
+    return channel;
+  }
+
   /**
    * Re-attach ONE tracked consumer on the current consumer channel.
    *
@@ -2235,19 +2551,29 @@ class RabbitMQClient extends EventEmitter {
    */
   async _reRegisterConsumer(queue, consumerInfo) {
     try {
+      // Asked before EACH re-registration, not once per loop (d.1138): a channel that
+      // died after the previous consumer — closed by the broker, or discarded by the
+      // broker-answer budget (d.1135) — leaves the slot empty, and reaching into it was a
+      // TypeError naming a JavaScript accident instead of the channel's state. Asked
+      // before the queue step, so a consumer that cannot be attached declares nothing.
+      this._liveConsumerChannel('re-register the consumer of', queue);
+
       await this._prepareQueueForConsume(queue, {
         durable: consumerInfo.options.durable,
         queueOptions: consumerInfo.options.queueOptions
       });
 
+      // …and again after it: the queue step awaits the broker on another channel,
+      // and the consumer channel can die meanwhile.
+      const consumerChannel = this._liveConsumerChannel('re-register the consumer of', queue);
+
       // The consumer's prefetch is re-applied here for the same reason the
       // queue step is: the channel is new, and QoS lives on the channel.
-      await this._applyConsumerPrefetch(queue, consumerInfo.options.prefetch);
+      await this._applyConsumerPrefetch(consumerChannel, queue, consumerInfo.options.prefetch);
 
-      const consumeResult = await this._consumerChannel.consume(
-        queue,
-        consumerInfo.handler,
-        { noAck: consumerInfo.options.noAck }
+      const consumeResult = await this._askBroker(
+        { operation: 'consume', target: queue, channel: consumerChannel },
+        () => consumerChannel.consume(queue, consumerInfo.handler, { noAck: consumerInfo.options.noAck })
       );
       consumerInfo.consumerTag = consumeResult.consumerTag;
       this._logger.info(`[RabbitMQClient] [mq-client-core] ✓ Re-registered consumer for queue: ${queue} (consumerTag: ${consumeResult.consumerTag})`);
@@ -2500,7 +2826,11 @@ class RabbitMQClient extends EventEmitter {
         try {
           // Check if queue exists
           this._trackChannelOperation(this._queueChannel, `checkQueue ${queue}`);
-          const queueInfo = await this._queueChannel.checkQueue(queue);
+          const queueChannel = this._queueChannel;
+          const queueInfo = await this._askBroker(
+            { operation: 'checkQueue', target: queue, channel: queueChannel },
+            () => queueChannel.checkQueue(queue)
+          );
           if (queueInfo) {
             // Queue exists - is THIS client's consumer attached to it?
             //
@@ -2854,7 +3184,10 @@ class RabbitMQClient extends EventEmitter {
     const channel = await this._requireQueueChannel('assertQueue', queue);
 
     try {
-      return await channel.assertQueue(queue, queueOptions);
+      return await this._askBroker(
+        { operation: 'assertQueue', target: queue, channel },
+        () => channel.assertQueue(queue, queueOptions)
+      );
     } catch (err) {
       // The SAME verdict the consume rail states, through the SAME function
       // (d.582). This is the rail FÁZE 0.8 walks: a service declaring its own
@@ -2984,10 +3317,34 @@ class RabbitMQClient extends EventEmitter {
    * @private
    */
   _publishPathTransient(verb, queue, cause) {
+    // The sentence is chosen from this client's STATE, never from the cause's text (d.1145,
+    // finding of d.1143): since d.1135 this refusal also ends a pre-check the broker never
+    // answered, and on the platform's image the connection was then alive and BLOCKED by a
+    // memory alarm — "connection may be closed" sent the reader to a reconnect that was not
+    // coming. `sent` is false on every one of these (no frame was written), which is what the
+    // flush after `unblocked` reads.
+    let state;
+    let fix;
+    if (!this._connectionAlive) {
+      state = 'the connection is closed';
+      fix = 'Fix: nothing to do here — a reconnect restores it; PublishLayer retries and buffers the message meanwhile, '
+        + 'and the buffer re-sends it after the reconnect.';
+    } else if (this._connectionBlockedBy !== null) {
+      state = `the broker has blocked this connection (reason: ${this._connectionBlockedBy}) and answers nothing until it clears the alarm`;
+      fix = 'Fix: clear the broker alarm (management API /api/health/checks/alarms names it); PublishLayer retries and '
+        + 'buffers the message meanwhile, and the buffer re-sends it once the broker unblocks the connection (its frame never left).';
+    } else {
+      // Neither closed nor blocked: the cause says what happened (a request unanswered
+      // within brokerAnswerTimeout, or a channel open the broker refused) — this sentence
+      // does not guess which.
+      state = 'the queue channel is not available on a live, unblocked connection (error.cause names the request '
+        + 'that went unanswered or was refused)';
+      fix = 'Fix: read error.cause; an unanswered request points at a paused or overloaded broker. PublishLayer retries '
+        + 'and buffers the message meanwhile.';
+    }
     return new TransientPublishError(
-      `[RabbitMQClient] Cannot ${verb} ${queue}: queue channel is not available (connection may be closed) - `
-        + `Expected: an open queue channel before ${verb} on the publish path. `
-        + 'Fix: retry the publish after the reconnect; PublishLayer does this on its own and buffers the message meanwhile.',
+      `[RabbitMQClient] Cannot ${verb} ${queue}: ${state} - `
+        + `Expected: an answering broker on an open queue channel before ${verb} on the publish path. ${fix}`,
       cause
     );
   }
@@ -3354,7 +3711,10 @@ class RabbitMQClient extends EventEmitter {
       ...options,
       durable: options.durable !== undefined ? options.durable : this._config.durable
     };
-    return await channel.assertExchange(exchange, type, exchangeOptions);
+    return await this._askBroker(
+      { operation: 'assertExchange', target: exchange, channel },
+      () => channel.assertExchange(exchange, type, exchangeOptions)
+    );
   }
 
   /**
@@ -3398,7 +3758,10 @@ class RabbitMQClient extends EventEmitter {
     }
 
     const channel = await this._requireQueueChannel('bindQueue', `${queue} -> ${exchange}`);
-    return await channel.bindQueue(queue, exchange, pattern);
+    return await this._askBroker(
+      { operation: 'bindQueue', target: `${queue} -> ${exchange}`, channel },
+      () => channel.bindQueue(queue, exchange, pattern)
+    );
   }
 
   /**
@@ -3481,7 +3844,11 @@ class RabbitMQClient extends EventEmitter {
     }
 
     this._trackChannelOperation(this._consumerChannel, `cancelConsumer ${queue}`);
-    const result = await this._consumerChannel.cancel(info.consumerTag);
+    const consumerChannel = this._consumerChannel;
+    const result = await this._askBroker(
+      { operation: 'cancel', target: queue, channel: consumerChannel },
+      () => consumerChannel.cancel(info.consumerTag)
+    );
     // Dropped only after the broker accepted the cancel: a consumer that is
     // still delivering must stay on the list recovery re-attaches.
     this._activeConsumers.delete(queue);
@@ -3498,7 +3865,10 @@ class RabbitMQClient extends EventEmitter {
    */
   async checkQueue(queue) {
     const channel = await this._requireQueueChannel('checkQueue', queue);
-    return await channel.checkQueue(queue);
+    return await this._askBroker(
+      { operation: 'checkQueue', target: queue, channel },
+      () => channel.checkQueue(queue)
+    );
   }
 
   /**
@@ -3507,7 +3877,10 @@ class RabbitMQClient extends EventEmitter {
    */
   async purgeQueue(queue) {
     const channel = await this._requireQueueChannel('purgeQueue', queue);
-    return await channel.purgeQueue(queue);
+    return await this._askBroker(
+      { operation: 'purgeQueue', target: queue, channel },
+      () => channel.purgeQueue(queue)
+    );
   }
 
   /**
@@ -3517,7 +3890,10 @@ class RabbitMQClient extends EventEmitter {
    */
   async deleteQueue(queue, options = {}) {
     const channel = await this._requireQueueChannel('deleteQueue', queue);
-    return await channel.deleteQueue(queue, options);
+    return await this._askBroker(
+      { operation: 'deleteQueue', target: queue, channel },
+      () => channel.deleteQueue(queue, options)
+    );
   }
 
   /**
@@ -3536,10 +3912,50 @@ class RabbitMQClient extends EventEmitter {
   }
 
   /**
-   * Internal publish implementation - single attempt without retry
+   * One publish attempt, without retry — and, when it fails, whether its frame LEFT THE
+   * CLIENT (d.1145, conf `mq-publish-buffer` 001).
+   *
+   * Every error this attempt throws carries `sent` (boolean): `true` once `sendToQueue()` /
+   * `publish()` on the channel returned without throwing — the frame was written to the
+   * channel — and `false` for everything that failed before that (the pre-check, the channel
+   * ensure, a throwing send). It says nothing about whether the broker stored the frame; it is
+   * what tells a message the broker may still deliver (held in a blocked socket, measured
+   * d.1143) from one it never received. `PublishLayer` folds the attempts of one call into the
+   * fact it writes on its own error and on the buffer entry.
+   *
+   * A nested republish from the confirm path (see `_publishOneAttempt`) may throw an error
+   * that already says `sent: true`; that fact is kept, never lowered.
+   *
+   * @param {string} queue
+   * @param {Buffer} buffer
+   * @param {Object} [options]
+   * @returns {Promise<void>}
+   * @throws {Error} the attempt's failure, with `sent` written on it
    * @private
    */
   async _publishOnce(queue, buffer, options = {}) {
+    const attempt = { sent: false };
+    try {
+      return await this._publishOneAttempt(queue, buffer, options, attempt);
+    } catch (err) {
+      if (err !== null && typeof err === 'object') {
+        err.sent = attempt.sent || err.sent === true;
+      }
+      throw err;
+    }
+  }
+
+  /**
+   * The body of one publish attempt. `attempt.sent` turns `true` the moment the frame is
+   * written to the channel; `_publishOnce()` reads it.
+   *
+   * @param {string} queue
+   * @param {Buffer} buffer
+   * @param {Object} options
+   * @param {{sent: boolean}} attempt - written here, read by `_publishOnce()`
+   * @private
+   */
+  async _publishOneAttempt(queue, buffer, options, attempt) {
     // Whose channel this send goes out on (d.724). Read once, here: the role cannot
     // change inside one attempt, and the channel it names can — so everything below asks
     // `_sendChannelOf(sendRole)` rather than holding a member.
@@ -3620,7 +4036,11 @@ class RabbitMQClient extends EventEmitter {
             }
             throw railErr;
           }
-          await queueChannel.checkQueue(queue);
+          const checkedOn = queueChannel;
+          await this._askBroker(
+            { operation: 'checkQueue', target: queue, channel: checkedOn },
+            () => checkedOn.checkQueue(queue)
+          );
           // Queue exists - proceed to publish
         } catch (checkErr) {
           // If queue doesn't exist (404), this je ERROR – ale rozlišujeme infra vs. non-infra:
@@ -3714,6 +4134,12 @@ class RabbitMQClient extends EventEmitter {
             //   - not already typed — `_publishPathTransient()` above throws into
             //     this same `catch`, and wrapping a verdict in itself would bury the
             //     rail's own sentence one level deeper for no reader.
+            //
+            // Since d.1135 the check goes through `_askBroker()`, so what arrives here
+            // is its `ConnectionError`: either the budget ran out (the channel is then
+            // discarded — dead) or amqplib's bare rejection, folded where it arrived. Both
+            // meet the three conditions unchanged; the bare error sits one level down, in
+            // `cause.cause`.
             throw this._publishPathTransient('checkQueue', queue, checkErr);
           } else {
             // Other error - rethrow
@@ -3844,12 +4270,20 @@ class RabbitMQClient extends EventEmitter {
                     return this.publish(queue, buffer, options);
                   }).then(resolve).catch(reject);
                 } else {
-                  // Not reconnecting - this is a real error
-                  // We cannot guarantee the message was delivered - must fail
+                  // No reconnect is in flight YET — which says nothing about whether one
+                  // follows. When the connection dies, amqplib closes its channels before
+                  // it announces the connection's own `close` (`lib/connection.js`,
+                  // `toClosed()`), and that announcement is what starts the reconnect, so
+                  // this callback runs before it (d.1134). The sentence therefore states
+                  // what is known here — the channel closed with the confirm outstanding —
+                  // and the classification makes it transient, so the publish path
+                  // retries and buffers it.
                   reject(new PublishError(
-                    `[RabbitMQClient] Channel closed during publish to queue "${queue}" and not reconnecting - message delivery not guaranteed. `
-                      + 'Expected: the channel to stay open until the broker confirms, or a reconnect to be in flight. '
-                      + 'Fix: republish the message — the confirm never arrived, so it may or may not have been stored.',
+                    `[RabbitMQClient] Channel closed during publish to queue "${queue}" before the broker confirmed the message - `
+                      + 'message delivery not guaranteed. '
+                      + 'Expected: the channel to stay open until the broker confirms. '
+                      + 'Fix: republish the message — the confirm never arrived, so it may or may not have been stored; '
+                      + 'error.buffered says whether the buffer holds it for the next reconnect.',
                     queue,
                     null
                   ));
@@ -3890,6 +4324,8 @@ class RabbitMQClient extends EventEmitter {
                 resolve();
               }
             });
+            // The frame is written to the channel: `sendToQueue()` returned without throwing.
+            attempt.sent = true;
             
             // Set a safety timeout - if callback wasn't invoked and channel closed, retry.
             // A broker confirm can land inside `sendToQueue()` itself, so the
@@ -4077,6 +4513,8 @@ class RabbitMQClient extends EventEmitter {
               resolve();
             }
           });
+          // The frame is written to the channel: `publish()` returned without throwing.
+          attempt.sent = true;
         });
         
         await confirmPromise;
@@ -4118,17 +4556,22 @@ class RabbitMQClient extends EventEmitter {
    * A non-numeric prefetch (the caller asked for none) sets none, and records
    * none: there is nothing to measure utilisation against.
    *
+   * @param {Object} consumerChannel - The consumer channel the caller holds; the
+   *   re-registration hands over the one it has just asked is alive (d.1138)
    * @param {string} queue - Queue name
    * @param {number|null|undefined} prefetch - The consumer's prefetch
    * @returns {Promise<void>}
    * @private
    */
-  async _applyConsumerPrefetch(queue, prefetch) {
+  async _applyConsumerPrefetch(consumerChannel, queue, prefetch) {
     if (typeof prefetch !== 'number') {
       return;
     }
 
-    await this._consumerChannel.prefetch(prefetch);
+    await this._askBroker(
+      { operation: 'prefetch', target: queue, channel: consumerChannel },
+      () => consumerChannel.prefetch(prefetch)
+    );
 
     const tracking = this._prefetchTracking.get(queue);
     if (tracking) {
@@ -4326,7 +4769,10 @@ class RabbitMQClient extends EventEmitter {
       // If missing, fail-fast. The owning infra service must recreate on startup.
       const queueChannel = await this._requireQueueChannel('checkQueue', queue);
       try {
-        await queueChannel.checkQueue(queue);
+        await this._askBroker(
+          { operation: 'checkQueue', target: queue, channel: queueChannel },
+          () => queueChannel.checkQueue(queue)
+        );
         this._logger.debug(`[RabbitMQClient] [mq-client-core] [CONSUMER] ✓ Infrastructure queue ${queue} exists (consumer will proceed)`);
       } catch (checkErr) {
         if (checkErr.code === AMQP_NOT_FOUND) {
@@ -4451,7 +4897,7 @@ class RabbitMQClient extends EventEmitter {
       // Set prefetch if provided (on consumer channel). ONE rail, shared with
       // `_reRegisterConsumers()` — prefetch belongs to the consumer, not to the
       // channel that happened to be open when it first attached.
-      await this._applyConsumerPrefetch(queue, prefetch);
+      await this._applyConsumerPrefetch(this._consumerChannel, queue, prefetch);
 
       // CRITICAL: Log before calling amqplib's consume()
       // amqplib's consume() may internally call assertQueue() without parameters if queue doesn't exist
@@ -4666,10 +5112,10 @@ class RabbitMQClient extends EventEmitter {
         }
       };
       
-      const consumeResult = await this._consumerChannel.consume(
-        queue,
-        messageHandler,
-        { noAck }
+      const consumerChannel = this._consumerChannel;
+      const consumeResult = await this._askBroker(
+        { operation: 'consume', target: queue, channel: consumerChannel },
+        () => consumerChannel.consume(queue, messageHandler, { noAck })
       );
       
       // Track consumer for auto re-registration if channel closes
@@ -4778,13 +5224,23 @@ class RabbitMQClient extends EventEmitter {
           this._logger.info(
             '[RabbitMQClient] Client was closed by its caller during recovery - closing the re-established connection'
           );
-          try {
-            await revived.close();
-          } catch (closeErr) {
+          // Within the one budget (d.1138, conf `mq-broker-answer-budget` 001): this close
+          // waits for a `close-ok` like every teardown close, and a broker that stopped
+          // answering right now would otherwise hold this round — and `_reconnecting` —
+          // open forever. Unanswered, the socket is dropped the way the teardown drops it.
+          const closing = await this._closeWithinTeardownBudget(
+            'connection re-established after disconnect',
+            revived,
+            this._brokerAnswerTimeout
+          );
+          if (closing.status === 'refused') {
             this._logger.warn(
-              `[RabbitMQClient] Error closing the connection re-established after disconnect: ${closeErr.message}`
+              `[RabbitMQClient] Error closing the connection re-established after disconnect: ${closing.error.message}`
             );
           }
+          if (closing.status === 'unanswered') {
+            this._dropUnansweredConnection(revived);
+          }
           this._connection = null;
           this._connectionAlive = false;
           this._reconnecting = false;
@@ -4831,7 +5287,10 @@ class RabbitMQClient extends EventEmitter {
         
         // Recreate consumer channel and re-register all consumers
         try {
-          this._consumerChannel = await this._connection.createChannel();
+          this._consumerChannel = await this._askBroker(
+            { operation: 'createChannel', target: 'consumer channel', channel: null },
+            () => this._connection.createChannel()
+          );
           this._consumerChannel._createdAt = new Date().toISOString();
           this._consumerChannel._closeReason = null;
           this._consumerChannel._lastOperation = 'Recreated via _reconnectWithBackoff()';
@@ -4882,7 +5341,7 @@ class RabbitMQClient extends EventEmitter {
         // Flush buffered messages po reconnectu
         let bufferFlushResult = { inMemory: 0, flushed: false };
         try {
-          const flushResult = await this._publishLayer.flushBuffered();
+          const flushResult = await this._publishLayer.flushBuffered('reconnected');
           bufferFlushResult = { inMemory: flushResult, flushed: true };
           this._publishMonitor.trackFlushed(bufferFlushResult.inMemory);
           this._logger.info(`[RabbitMQClient] ✓ Flushed ${bufferFlushResult.inMemory} buffered messages after reconnection`);
@@ -5033,7 +5492,19 @@ class RabbitMQClient extends EventEmitter {
       this.emit('error', err);
     });
     
-    channel.on('close', async () => {
+    // PREPENDED, not appended (d.1134). amqplib registers its own `close` listener in the
+    // channel's constructor — the one that answers every unconfirmed publish with the bare
+    // `new Error('channel closed')` (`lib/channel.js`, the `Channel` constructor) — and a
+    // constructor runs before anybody can attach anything, so an appended handler always
+    // ran SECOND. The publish callbacks then read a channel that was still alive, took the
+    // branch meant for a broker answer on a live channel, and the sentinel reached the
+    // classification untyped: permanent, not buffered, not retried — for a message the
+    // broker said nothing about (měřeno d.1126, `320 CONNECTION_FORCED` with a confirm in
+    // flight). `connection.toClosed()` closes every channel before the connection's own
+    // events (`lib/connection.js`), so this handler is the only one that can mark the
+    // channel dead in time. Prepending makes it run before amqplib's own listener — an
+    // order the emitter guarantees, with no timer and no race.
+    channel.prependListener('close', async () => {
       if (this._closedByCaller) {
         return;
       }