@onlineapps/mq-client-core 5.0.0 → 6.0.0

package/src/layers/PublishLayer.js

@@ -2,10 +2,11 @@
 
 const {
   TransientPublishError,
+  RecoveryEndedPublishError,
   PermanentPublishError,
   QueueNotFoundError,
 } = require('../utils/publishErrors');
-const { ValidationError, PublishError } = require('../utils/errorHandler');
+const { ValidationError, PublishError, carryVerdict } = require('../utils/errorHandler');
 const MessageBuffer = require('../buffer/MessageBuffer');
 const { assertLogger } = require('@onlineapps/logger-contract');
 
@@ -23,6 +24,8 @@ class PublishLayer {
    * @param {Object} options.client - Instance RabbitMQClient
    * @param {Object} options.logger - Logger s info/warn/error/debug (povinný)
    * @param {Object} [options.bufferConfig] - Konfigurace pro MessageBuffer
+   * @param {{setTimeout: Function, clearTimeout: Function}} options.timers - The clock the
+   *   buffer expires its entries on (required, d.1275; the transport passes its own timer rail).
    * @param {boolean} [options.retryEnabled=true] - Zapnout retry
    * @param {number} [options.maxRetries=3] - Max počet pokusů
    * @param {number} [options.retryBaseDelay=100] - Base delay v ms
@@ -114,7 +117,11 @@ class PublishLayer {
     this._buffer = new MessageBuffer({
       inMemory: options.bufferConfig?.inMemory,
       logger: this._logger,
+      timers: options.timers,
     });
+    // Read by the one sentence that tells a caller how long the buffer holds its message
+    // (`_holdOnFailure()`); `InMemoryBuffer` has already refused a missing value above.
+    this._bufferTtlMs = options.bufferConfig.inMemory.ttlMs;
   }
 
   /**
@@ -122,8 +129,35 @@ class PublishLayer {
    * @param {string} queue
    * @param {Buffer} buffer
    * @param {Object} [options]
+   * @param {boolean} [options.bufferOnFailure=true] - Whether a message whose publish failed
+   *   transiently is held in the buffer and replayed after the next reconnect. `false` is for
+   *   a caller that keeps the message durably elsewhere (conf `step-publish-failure` 001):
+   *   nothing is held, so nothing replays behind its back. On each of the three transient
+   *   paths (retries exhausted, retry disabled, the awaited reconnect failed) the thrown
+   *   error carries `buffered` (boolean) — `true` only when the buffer really holds the
+   *   message. A non-boolean value is refused before any attempt.
+   *
+   * Every error this method throws also carries `sent` (boolean, d.1145, conf
+   * `mq-publish-buffer` 001): `true` when ANY attempt of this call wrote the frame to the
+   * channel, so the broker may hold — and after a memory alarm deliver — the message even
+   * though the call failed; `false` when no attempt got that far. The buffer entry records the
+   * same fact, and the flush after `unblocked` reads it.
+   * @throws {ValidationError} When `options.bufferOnFailure` is present and not a boolean.
    */
   async publish(queue, buffer, options = {}) {
+    // Refused HERE, before any attempt: a value that cannot say yes or no must not be read
+    // as either — `'false'` is truthy, and `0` would be read as "no" by accident
+    // (`architecture-principles.md` §3, §4). The one owner of the option is this layer;
+    // `BaseClient.publish()` only carries it.
+    if (options.bufferOnFailure !== undefined && typeof options.bufferOnFailure !== 'boolean') {
+      throw new ValidationError(
+        `[PublishLayer] options.bufferOnFailure must be a boolean - got ${JSON.stringify(options.bufferOnFailure)} `
+          + `for queue "${queue}". Expected: true (hold a transiently failed message and replay it after `
+          + 'the reconnect — the default) or false (hold nothing; the caller keeps the message). '
+          + 'Fix: pass bufferOnFailure: true or bufferOnFailure: false, or omit it.'
+      );
+    }
+
     // `priority` still orders the buffer's replay (critical first). What it no
     // longer does is choose a DIFFERENT buffer: the persistent one was a stub
     // that stored nothing, so routing critical messages to it dropped them
@@ -145,10 +179,40 @@ class PublishLayer {
     try {
       await this._client._publishOnce(queue, buffer, options);
     } catch (err) {
-      return await this._handlePublishError(err, queue, buffer, options, priority);
+      const sent = this._sentOf(err, queue);
+      return await this._handlePublishError(err, queue, buffer, options, priority, sent);
     }
   }
 
+  /**
+   * Whether the transport says the failed attempt's frame left the client.
+   *
+   * The transport writes `sent` on every error of `_publishOnce()` (d.1145). An error without
+   * it is a defect of the neighbour, and it is refused here by name rather than read as
+   * `false`: a message whose frame did leave would then be re-sent after `unblocked`, and
+   * that is the duplicate conf `mq-publish-buffer` 001 exists to prevent.
+   *
+   * @param {*} err - what `client._publishOnce()` threw
+   * @param {string} queue
+   * @returns {boolean}
+   * @throws {Error} when the error carries no boolean `sent`
+   * @private
+   */
+  _sentOf(err, queue) {
+    if (err !== null && typeof err === 'object' && typeof err.sent === 'boolean') {
+      return err.sent;
+    }
+    throw new Error(
+      '[PublishLayer] The transport reported a failed attempt without saying whether its frame was sent - '
+        + `queue "${queue}", error: ${err && err.message ? err.message : String(err)}. `
+        + 'Expected: every error of client._publishOnce() carries sent (boolean) — RabbitMQClient writes it on '
+        + 'every attempt (conf mq-publish-buffer 001). Fix: a stand-in transport must write err.sent as the '
+        + 'real one does; the message is neither buffered nor retried, because nobody can say whether '
+        + 'the broker may already hold it.',
+      { cause: err }
+    );
+  }
+
   /**
    * Publish s retry logikou
    * @private
@@ -171,6 +235,14 @@ class PublishLayer {
      * (the transport's `channel._closeError` + `_closeFrameNamesThisSend()`), the event is
      * the one already emitted, and the subscription lives exactly as long as this publish.
      */
+    // Whether ANY attempt of this call wrote the frame to the channel (d.1145). Written on
+    // every error this loop throws — `said()` — and on the buffer entry.
+    let sentAny = false;
+    const said = (error) => {
+      error.sent = sentAny;
+      return error;
+    };
+
     let lateRefusal = null;
     const onRefusedLate = (event) => {
       if (event.queue === queue && event.verdict === 'PermanentPublishError' && event.error) {
@@ -196,7 +268,7 @@ class PublishLayer {
           retryable: false,
           refusedLate: true
         });
-        throw lateRefusal;
+        throw said(lateRefusal);
       }
 
       attempt++;
@@ -234,6 +306,7 @@ class PublishLayer {
 
       } catch (err) {
         lastError = err;
+        sentAny = this._sentOf(err, queue) || sentAny;
 
         // Non-retryable errors - fail immediately
         if (err instanceof PermanentPublishError || err instanceof QueueNotFoundError) {
@@ -243,7 +316,22 @@ class PublishLayer {
             error: err.message,
             retryable: false
           });
-          throw err;
+          throw said(err);
+        }
+
+        // The attempt could not get its send channel because the connection stayed down
+        // (B1, d.1300): the recovery it waited for ended in standby or fatal, or none may
+        // start. Retrying NOW is pointless — the next use of the client starts the next
+        // cycle — so the message is held at once, on the same rail as the retry below that
+        // waited for a reconnect and saw it fail. Asked BEFORE the generic transient branch,
+        // because the class is a transient one (the message is held and replayed).
+        if (err instanceof RecoveryEndedPublishError) {
+          throw said(await this._failedAfterRecovery({
+            queue, buffer, options, priority, attempt, transient: err, recoveryError: err.cause || err, sentAny,
+            headline: '[PublishLayer] Publish failed: the recovery this publish waited for ended without a connection '
+              + `(attempt ${attempt} for queue "${queue}") - ${err.message} `
+              + 'Expected: a live connection before the send. '
+          }));
         }
 
         // Transient errors - check if we should retry or buffer
@@ -258,22 +346,12 @@ class PublishLayer {
                 this._logger.debug(`[PublishLayer] ✓ Reconnection completed, proceeding with retry ${attempt + 1}/${this._maxRetries}`);
                 continue; // Retry immediately after reconnect
               } catch (reconnectErr) {
-                // Reconnection failed - buffer and fail
-                await this._bufferMessage(queue, buffer, options, priority, err);
-                this._client.emit('publish:failed', {
-                  queue,
-                  attempt,
-                  error: `Reconnection failed: ${reconnectErr.message}`,
-                  retryable: false,
-                  reconnectFailed: true
-                });
-                throw new PublishError(
-                  `[PublishLayer] Publish failed: reconnection failed after ${attempt} attempts for queue "${queue}" - `
+                // Reconnection failed - buffer (unless the caller said not to) and fail
+                throw said(await this._failedAfterRecovery({
+                  queue, buffer, options, priority, attempt, transient: err, recoveryError: reconnectErr, sentAny,
+                  headline: `[PublishLayer] Publish failed: reconnection failed after ${attempt} attempts for queue "${queue}" - `
                     + 'Expected: the transport to re-establish the connection before the retry. '
-                    + 'Fix: read error.cause for the reconnect reason; the message itself is buffered, so it is not lost.',
-                  queue,
-                  reconnectErr
-                );
+                }));
               }
             }
 
@@ -305,8 +383,8 @@ class PublishLayer {
             continue;
           }
 
-          // Max retries exceeded - buffer and fail
-          await this._bufferMessage(queue, buffer, options, priority, err);
+          // Max retries exceeded - buffer (unless the caller said not to) and fail
+          const held = await this._holdOnFailure(queue, buffer, options, priority, err, sentAny);
           this._client.emit('publish:failed', {
             queue,
             attempt,
@@ -314,21 +392,23 @@ class PublishLayer {
             retryable: true,
             maxRetriesExceeded: true
           });
-          throw new PublishError(
+          const failure = new PublishError(
             `[PublishLayer] Publish failed after ${attempt} attempts for queue "${queue}" - `
               + 'Expected: a retryable broker error to clear within maxRetries. '
-              + 'Fix: read error.cause for the last broker reason; the message is buffered, so it is not lost.',
+              + `Fix: read error.cause for the last broker reason; ${held.sentence}`,
             queue,
             err
           );
+          failure.buffered = held.buffered;
+          throw said(failure);
         }
 
         // Unknown error - fail immediately
-        throw err;
+        throw said(err);
       }
     }
 
-    throw (
+    throw said(
       lastError
       || new PublishError(
         `[PublishLayer] Publish failed for queue "${queue}" after ${attempt} attempts - `
@@ -345,48 +425,174 @@ class PublishLayer {
     }
   }
 
+  /**
+   * The one ending for "the recovery this publish depended on did not arrive": hold the
+   * message (unless the caller said not to), report the failure, and build the error the
+   * caller gets — with `buffered` written on it and the verdict of a fatal ending carried on
+   * top (`carryVerdict()`, d.1297/d.1300).
+   *
+   * Two callers, one body (d.1300): the retry that waited for a reconnect in flight and saw
+   * it fail, and the attempt whose send channel could not be had because the connection
+   * stayed down (`RecoveryEndedPublishError`). Until d.1300 only the first held the message;
+   * the second threw the wait's ending raw, so two near-identical situations ended two ways
+   * (INFRA-DOCS ID254 point 1).
+   *
+   * @private
+   * @returns {Promise<PublishError>} the failure to throw; the caller writes `sent` on it
+   */
+  async _failedAfterRecovery({ queue, buffer, options, priority, attempt, transient, recoveryError, sentAny, headline }) {
+    const held = await this._holdOnFailure(queue, buffer, options, priority, transient, sentAny);
+    this._client.emit('publish:failed', {
+      queue,
+      attempt,
+      error: `Reconnection failed: ${recoveryError.message}`,
+      retryable: false,
+      reconnectFailed: true
+    });
+    const failure = new PublishError(
+      `${headline}Fix: read error.cause for the reconnect reason; ${held.sentence}`,
+      queue,
+      recoveryError
+    );
+    failure.buffered = held.buffered;
+    return carryVerdict(failure, recoveryError);
+  }
+
   /**
    * Handle publish error - buffer transient errors
    * @private
    */
-  async _handlePublishError(err, queue, buffer, options, priority) {
+  async _handlePublishError(err, queue, buffer, options, priority, sent) {
     if (err instanceof QueueNotFoundError) {
       this._logger.error(`[PublishLayer] QueueNotFoundError for '${queue}' (infra=${err.isInfrastructure}): ${err.message}`);
       throw err;
     }
 
     if (err instanceof TransientPublishError) {
-      await this._bufferMessage(queue, buffer, options, priority, err);
+      const held = await this._holdOnFailure(queue, buffer, options, priority, err, sent);
+      // The transport's own error, thrown as it came — this path has always done that —
+      // with the one fact only this layer knows written on it.
+      err.buffered = held.buffered;
       throw err;
     }
 
     throw err;
   }
 
+  /**
+   * Hold a transiently failed message in the buffer — or, when the caller passed
+   * `bufferOnFailure: false`, deliberately do not — and say which happened.
+   *
+   * The one decision point of the three transient paths, so the option cannot be honoured
+   * on one path and forgotten on another.
+   *
+   * @private
+   * @returns {Promise<{buffered: boolean, sentence: string}>} `buffered` is `true` only when
+   *   the buffer really holds the message; `sentence` ends the thrown error's `Fix:` with
+   *   what the caller can rely on.
+   */
+  async _holdOnFailure(queue, buffer, options, priority, err, sent) {
+    if (options.bufferOnFailure === false) {
+      this._logger.warn(
+        `[PublishLayer] Message for queue '${queue}' NOT buffered (bufferOnFailure: false) after a transient error: ${err.message}`
+      );
+      return {
+        buffered: false,
+        sentence: 'the message is NOT buffered (bufferOnFailure: false), so nothing replays it — '
+          + 'the caller holds the only copy (error.buffered === false).'
+      };
+    }
+    // Asked AFTER the caller's explicit choice above (`architecture-principles.md` §8: the
+    // option the caller passed is what its sentence names), and before any hold.
+    // A client that will never reconnect on its own cannot replay anything: the two flushes
+    // are `reconnected` and `unblocked`, and neither comes after the connection is declared
+    // lost, after `disconnect()`, or with `reconnectEnabled: false`. Holding there is a loss
+    // with a delay (reported LOST at TTL), not a safety — so nothing is held, and the error
+    // says so (d.1301, the lead's decision narrowing B1 of d.1300). The transport writes the
+    // fact (`replayable`) where it knows the state; this layer only reads it.
+    if (err instanceof RecoveryEndedPublishError && err.replayable === false) {
+      this._logger.warn(
+        `[PublishLayer] Message for queue '${queue}' NOT buffered: this client will not reconnect, so no replay could come - ${err.message}`
+      );
+      return {
+        buffered: false,
+        sentence: 'the message is NOT buffered: this client will not reconnect on its own (the connection is '
+          + 'permanently lost, the caller closed it, or reconnectEnabled is false), so no reconnect could replay '
+          + 'it — the caller holds the only copy (error.buffered === false).'
+      };
+    }
+    if (await this._bufferMessage(queue, buffer, options, priority, err, sent)) {
+      // What the buffer WILL do, and nothing more (d.1275). It used to say "the message is
+      // buffered, so it is not lost" — while the only two flushes are the reconnect and
+      // `unblocked`, so an outage of the publisher channel alone ended with the message
+      // expiring (INFRA 531, W224). Whether a channel recovery should flush too is an open
+      // decision (a sent entry re-sent after it may be a duplicate); until it is made, the
+      // sentence states the gap instead of papering over it.
+      return {
+        buffered: true,
+        sentence: `the message is buffered for up to ${this._bufferTtlMs} ms (publishBufferTtlMs) and `
+          + 'replayed only after a reconnect or after the broker unblocks the connection — a recovered '
+          + 'publisher channel alone does not replay it; if neither comes within that time it expires and '
+          + (sent
+            ? 'is reported as an expired message whose frame had been sent (the broker may hold a copy) '
+            : 'is reported as LOST ')
+          + '(an error line naming the queue or exchange), and nothing replays it then.'
+      };
+    }
+    return {
+      buffered: false,
+      sentence: 'the message is NOT buffered (the buffer refused it — see the error log line before this one), '
+        + 'so nothing replays it (error.buffered === false).'
+    };
+  }
+
   /**
    * Buffer message při transient error
    * @private
+   * @returns {Promise<boolean>} whether the buffer now holds the message
    */
-  async _bufferMessage(queue, buffer, options, priority, err) {
+  async _bufferMessage(queue, buffer, options, priority, err, sent) {
     try {
-      await this._buffer.add(queue, buffer, options, { priority });
+      await this._buffer.add(queue, buffer, options, { priority, sent });
       this._logger.warn(`[PublishLayer] Buffered message for queue '${queue}' due to transient error: ${err.message}`);
       this._client.emit('publish:buffered', { queue });
+      return true;
     } catch (bufferErr) {
       this._logger.error(`[PublishLayer] Failed to buffer message for queue '${queue}': ${bufferErr.message}`);
+      return false;
     }
   }
 
   /**
-   * Flush buffered messages po reconnectu
+   * Re-send what the buffer holds — the one flush, with two triggers (d.1145, conf
+   * `mq-publish-buffer` 001):
+   *
+   *   - `'reconnected'` — a new connection: every entry is re-sent, as it always was;
+   *   - `'unblocked'` — the broker ended a memory alarm on the SAME connection: only the
+   *     entries whose frame never left the client are re-sent; the sent ones are dropped with a
+   *     warning, because the broker delivers those itself (d.1143 measured it).
+   *
+   * Every entry is re-sent with the publish options it was held with — nothing else is
+   * merged in (d.1145 removed the `flushOptions` argument nobody passed).
+   *
+   * @param {'reconnected'|'unblocked'} trigger - what ended the outage
+   * @returns {Promise<number>} how many entries were re-sent successfully
+   * @throws {Error} when the trigger is not one of the two
    */
-  async flushBuffered(flushOptions = {}) {
+  async flushBuffered(trigger) {
+    const SENT_ENTRIES = { reconnected: 'resend', unblocked: 'drop' };
+    if (!Object.prototype.hasOwnProperty.call(SENT_ENTRIES, trigger)) {
+      throw new Error(
+        `[PublishLayer] flushBuffered() needs its trigger - got ${JSON.stringify(trigger)}. `
+          + "Expected: 'reconnected' or 'unblocked' — they differ in what happens to a message whose frame "
+          + 'was already sent (conf mq-publish-buffer 001). Fix: pass the event that ended the outage.'
+      );
+    }
     const result = await this._buffer.flush(async (queue, buf, opts) => {
-      const mergedOptions = Object.assign({}, opts, flushOptions);
-      await this._client._publishOnce(queue, buf, mergedOptions);
-    });
+      await this._client._publishOnce(queue, buf, opts);
+    }, { sentEntries: SENT_ENTRIES[trigger] });
 
-    this._logger.info(`[PublishLayer] Flushed ${result} buffered message(s)`);
+    this._logger.info(`[PublishLayer] Flushed ${result} buffered message(s) after ${trigger}`);
     return result;
   }
 }

package/src/monitoring-initiated.js

@@ -8,15 +8,12 @@
  * run `active` on it (`api/infra/api_monitoring/src/consumer/index.js`). A service that
  * does not file it starts runs that are invisible for their whole life.
  *
- * Until now two services built that event, each in its own words: the gateway
- * (`infra/api_gateway/shared/monitoringInitiatedCopy.js`, with a redacted cookbook) and
- * the scheduler (`infra/api_scheduler/src/publisher/workflowPublisher.js`, inline, the
- * cookbook verbatim and no `tenant_uuid` at all). Two implementations of one concern are
- * a defect by default (`change-discipline.md` § One rail per concern), and the owner
- * settled which rail survives: this one, in the library that already owns the transport
- * half of the same wire (`docs/governance/confirmations/scheduler.md` 009). Both copies
- * are retired by their own services in the change that adopts this module — no period
- * with both (`architecture-principles.md` §11).
+ * One producer rail, by the owner's decision (`docs/governance/confirmations/scheduler.md`
+ * 009 and 024): the event is built and filed by this module, in the library that already
+ * owns the transport half of the same wire (`./monitoring-publish.js`). Every service that
+ * starts a run files it through here — the decision names the gateway and the scheduler.
+ * A service that builds the event in its own words is a second implementation of one
+ * concern, a defect by default (`change-discipline.md` § One rail per concern).
  *
  * WHAT THIS MODULE DOES NOT DO, deliberately:
  *