@onlineapps/mq-client-core 5.0.0 → 6.0.0

package/src/buffer/InMemoryBuffer.js

@@ -10,6 +10,19 @@ const { assertLogger } = require('@onlineapps/logger-contract');
  * Primárně určený pro transient scénáře (health checky apod.).
  */
 
+/**
+ * How a log line names where a buffered message was going: its exchange when it was published
+ * to one, its queue otherwise.
+ *
+ * @param {{queue: string, options: Object}} item
+ * @returns {string}
+ */
+function describeTarget(item) {
+  return item.options && typeof item.options.exchange === 'string' && item.options.exchange !== ''
+    ? `exchange "${item.options.exchange}"`
+    : `queue "${item.queue}"`;
+}
+
 class InMemoryBuffer {
   /**
    * Both numbers are REQUIRED and have no stand-in here.
@@ -30,6 +43,10 @@ class InMemoryBuffer {
    * @param {number} options.maxSize - Max messages held; 0 means hold none
    * @param {number} options.ttlMs - How long one message may wait, in ms
    * @param {Object} options.logger - Logger s info/warn/error/debug (povinný)
+   * @param {{setTimeout: Function, clearTimeout: Function}} options.timers - The clock the
+   *   expiry runs on (required, d.1275): `setTimeout(callback, ms)` returns a handle,
+   *   `clearTimeout(handle)` disarms it. The transport hands its own timer rail
+   *   (`_setTimeout()` / `_clearTimeout()`), so `disconnect()` ends this timer too.
    */
   constructor(options = {}) {
     // The logger first, so a caller who forgot it gets the contract message
@@ -58,10 +75,23 @@ class InMemoryBuffer {
           + 'or pass ttlMs explicitly.'
       );
     }
+    if (!options.timers || typeof options.timers.setTimeout !== 'function'
+      || typeof options.timers.clearTimeout !== 'function') {
+      throw new Error(
+        '[InMemoryBuffer] Missing required option - timers is required and has no default here. '
+          + 'Expected: { setTimeout(callback, ms), clearTimeout(handle) } — the clock a held message '
+          + 'expires on at its TTL, rather than whenever the buffer is next touched. '
+          + 'Fix: build this buffer through PublishLayer, which hands it the transport\'s timer rail, '
+          + 'or pass options.timers explicitly.'
+      );
+    }
     this._maxSize = options.maxSize;
     this._ttlMs = options.ttlMs;
+    this._timers = options.timers;
+    /** The one armed expiry timer, aimed at the oldest entry's deadline; null while empty. */
+    this._expiryTimer = null;
 
-    /** @type {Array<{queue:string, buffer:Buffer, options:Object, createdAt:number, priority:string}>} */
+    /** @type {Array<{queue:string, buffer:Buffer, options:Object, createdAt:number, priority:string, sent:boolean}>} */
     this._items = [];
   }
 
@@ -69,7 +99,18 @@ class InMemoryBuffer {
    * Přidá zprávu do bufferu.
    * Staré/expirující zprávy jsou průběžně čištěny, při překročení kapacity odhazujeme nejstarší.
    */
-  async add(queue, buffer, options = {}, priority = 'normal') {
+  async add(queue, buffer, options = {}, priority = 'normal', sent) {
+    // Whether the frame of this message already LEFT the client (d.1145, conf
+    // `mq-publish-buffer` 001): the flush after `unblocked` re-sends only the entries that
+    // never did, because the broker delivers the frames it was holding by itself. Required —
+    // an entry that cannot say would be re-sent or dropped by a guess.
+    if (typeof sent !== 'boolean') {
+      throw new Error(
+        `[InMemoryBuffer] Cannot hold a message for queue '${queue}' without knowing whether its frame was sent - `
+          + `got sent=${JSON.stringify(sent)}. Expected: a boolean from the failed publish (error.sent). `
+          + 'Fix: pass it from PublishLayer, which reads it off the transport\'s error.'
+      );
+    }
     const now = Date.now();
     this._cleanupExpired(now);
 
@@ -86,7 +127,9 @@ class InMemoryBuffer {
       options,
       createdAt: now,
       priority,
+      sent,
     });
+    this._armExpiry(now);
   }
 
   /**
@@ -94,7 +137,20 @@ class InMemoryBuffer {
    * @param {Function} flushFn - async (queue, buffer, options) => void
    * @returns {Promise<number>} - počet úspěšně flushnutých zpráv
    */
-  async flush(flushFn) {
+  async flush(flushFn, { sentEntries } = {}) {
+    // What happens to an entry whose frame already LEFT the client (d.1145, conf
+    // `mq-publish-buffer` 001): 'resend' after a reconnect — the old socket and whatever the
+    // broker had not read from it are gone; 'drop' after the broker unblocked the connection —
+    // it reads what was waiting in the socket and delivers those frames by itself, so a
+    // re-send would be one more copy. Required: a flush that guessed would do one or the other
+    // wrong.
+    if (sentEntries !== 'resend' && sentEntries !== 'drop') {
+      throw new Error(
+        `[InMemoryBuffer] flush() needs to know what to do with a sent entry - got sentEntries=${JSON.stringify(sentEntries)}. `
+          + "Expected: 'resend' (after a reconnect) or 'drop' (after the broker unblocked the connection). "
+          + 'Fix: flush through PublishLayer.flushBuffered(trigger), which maps its trigger to one of them.'
+      );
+    }
     const now = Date.now();
     this._cleanupExpired(now);
 
@@ -103,8 +159,24 @@ class InMemoryBuffer {
     }
 
     // Zkopírujeme a vyprázdníme interní pole, abychom se vyhnuli reentrancy problémům
-    const items = this._items.slice();
+    let items = this._items.slice();
     this._items = [];
+    // Nothing is held while the flush runs; a replay that fails re-arms through `add()`.
+    this._armExpiry(now);
+
+    if (sentEntries === 'drop') {
+      for (const item of items.filter((entry) => entry.sent)) {
+        this._logger.warn(
+          `[InMemoryBuffer] A buffered message for ${describeTarget(item)} is not re-sent after the broker unblocked `
+            + `the connection: its frame had been sent, and it was buffered ${now - item.createdAt} ms ago - Expected: the broker delivers `
+            + 'the frames it held during the alarm by itself once it reads the socket again, so a re-send would '
+            + 'be one more copy. Fix: nothing to do here; a consumer that must not see a message twice '
+            + 'deduplicates it (conf mq-consumer-contract 009), and a message the broker did NOT keep is '
+            + 'a broker-side loss this client cannot see.'
+        );
+      }
+      items = items.filter((entry) => !entry.sent);
+    }
 
     // Kritické zprávy (priority === 'critical') házíme dopředu
     items.sort((a, b) => {
@@ -125,7 +197,9 @@ class InMemoryBuffer {
         this._logger.warn(
           `[InMemoryBuffer] Failed to flush message for queue '${item.queue}': ${err.message}`
         );
-        await this.add(item.queue, item.buffer, item.options, item.priority);
+        // The replay is one more attempt: if IT wrote the frame, the entry says so from now on.
+        const sent = item.sent || (err !== null && typeof err === 'object' && err.sent === true);
+        await this.add(item.queue, item.buffer, item.options, item.priority, sent);
       }
     }
 
@@ -140,15 +214,89 @@ class InMemoryBuffer {
     return this._items.length;
   }
 
+  /**
+   * Aim the one expiry timer at the oldest entry's deadline — or disarm it when nothing is held.
+   *
+   * WHY A TIMER (d.1275, INFRA 531). Until d.1275 expiry ran only inside `add()`, `flush()` and
+   * `size()`. The buffer is emptied by a reconnect or by `unblocked`; an outage of a CHANNEL
+   * alone ends with neither, so an entry outlived its TTL until some unrelated publish touched
+   * the buffer — measured on the dev scheduler at 584 843 ms and 5 267 969 ms against a TTL of
+   * 300 000, and the ERROR line saying the message was lost came that late. The deadline is
+   * the TTL; the line now comes at the deadline.
+   *
+   * One timer, re-aimed on every change, rather than one per entry: entries are FIFO by
+   * `createdAt`, so the oldest one's deadline is the next one that matters. Re-aiming always
+   * clears the previous handle first, so a handle the transport's teardown already swept
+   * (`_clearAllTimers()`) is never trusted to still be armed.
+   *
+   * @param {number} now - the clock reading the caller took
+   * @private
+   */
+  _armExpiry(now) {
+    if (this._expiryTimer !== null) {
+      this._timers.clearTimeout(this._expiryTimer);
+      this._expiryTimer = null;
+    }
+    if (this._items.length === 0) {
+      return;
+    }
+    const oldest = this._items.reduce((min, item) => Math.min(min, item.createdAt), Infinity);
+    // `_cleanupExpired()` drops an entry once its age is MORE than the TTL, so the timer
+    // fires one millisecond past the deadline.
+    const delay = Math.max(0, oldest + this._ttlMs + 1 - now);
+    this._expiryTimer = this._timers.setTimeout(() => {
+      this._expiryTimer = null;
+      const firedAt = Date.now();
+      this._cleanupExpired(firedAt);
+      this._armExpiry(firedAt);
+    }, delay);
+  }
+
+  /**
+   * Drop the entries older than the TTL — and say, as an ERROR, what was dropped (d.1145, conf
+   * `mq-publish-buffer` 001).
+   *
+   * Until d.1145 this was one `info` line with a count and nothing else, the only trace of a
+   * message nobody would ever deliver (a memory alarm keeps the connection alive, so no
+   * reconnect flush comes — d.1143). One error per destination and `sent` value, with the
+   * count and the age of the oldest: a message whose frame never left is LOST, one whose frame
+   * had been sent may have been delivered by the broker on its own, and the two are never
+   * folded into one sentence.
+   *
+   * @param {number} now - the clock reading the caller took
+   * @private
+   */
   _cleanupExpired(now) {
-    const before = this._items.length;
-    this._items = this._items.filter(
-      (item) => now - item.createdAt <= this._ttlMs
-    );
-    const removed = before - this._items.length;
-    if (removed > 0) {
-      this._logger.info(
-        `[InMemoryBuffer] Removed ${removed} expired buffered messages`
+    const expired = this._items.filter((item) => now - item.createdAt > this._ttlMs);
+    if (expired.length === 0) {
+      return;
+    }
+    this._items = this._items.filter((item) => now - item.createdAt <= this._ttlMs);
+
+    const groups = new Map();
+    for (const item of expired) {
+      const key = `${describeTarget(item)}|${item.sent}`;
+      const group = groups.get(key) || { target: describeTarget(item), sent: item.sent, count: 0, oldestMs: 0 };
+      group.count += 1;
+      group.oldestMs = Math.max(group.oldestMs, now - item.createdAt);
+      groups.set(key, group);
+    }
+    for (const { target, sent, count, oldestMs } of groups.values()) {
+      // `LOST after <ms>` is the phrase an operator greps for, and only a never-sent message
+      // carries it: a sent one may have been delivered by the broker (conf `mq-publish-buffer`
+      // 001 — the two are never folded into one sentence).
+      this._logger.error(
+        `[InMemoryBuffer] ${count} buffered message(s) for ${target} `
+          + (sent ? `expired after ${oldestMs} ms` : `LOST after ${oldestMs} ms`)
+          + ` (buffer TTL ${this._ttlMs} ms; oldest ${oldestMs} ms), sent=${sent} - `
+          + (sent
+            ? 'the frame had been sent, so the broker may have delivered it on its own; nothing re-sends it now. '
+            : 'the frame never left this client, so the message is LOST; nothing re-sends it now. ')
+          + 'Expected: a reconnect, or the broker unblocking the connection, within publishBufferTtlMs '
+          + '(a recovered publisher channel alone replays nothing). '
+          + 'Fix: find why the broker stayed away that long (an alarm: management API /api/health/checks/alarms; '
+          + 'an outage: the connection log); a message that must survive longer belongs to a caller that keeps '
+          + 'it durably and publishes with bufferOnFailure: false.'
       );
     }
   }

package/src/buffer/MessageBuffer.js

@@ -22,6 +22,8 @@ class MessageBuffer {
    * @param {Object} options
    * @param {Object} [options.inMemory] - In-memory buffer options
    * @param {Object} options.logger - Logger s info/warn/error/debug (povinny)
+   * @param {{setTimeout: Function, clearTimeout: Function}} options.timers - The expiry clock,
+   *   passed through unread: `InMemoryBuffer` owns the rule that it is required (d.1275).
    */
   constructor(options = {}) {
     this._logger = assertLogger(
@@ -35,6 +37,7 @@ class MessageBuffer {
       maxSize: options.inMemory?.maxSize,
       ttlMs: options.inMemory?.ttlMs,
       logger: this._logger,
+      timers: options.timers,
     });
 
   }
@@ -50,7 +53,9 @@ class MessageBuffer {
   async add(queue, buffer, options = {}, meta = {}) {
     const priority = meta.priority || 'normal';
 
-    await this._inMemory.add(queue, buffer, options, priority);
+    // `sent` is passed through unread: the in-memory buffer owns the rule that it is required
+    // (d.1145, conf `mq-publish-buffer` 001).
+    await this._inMemory.add(queue, buffer, options, priority, meta.sent);
   }
 
   /**
@@ -58,8 +63,8 @@ class MessageBuffer {
    * @param {Function} flushFn - async (queue, buffer, options) => void
    * @returns {Promise<number>} how many messages were replayed
    */
-  async flush(flushFn) {
-    return await this._inMemory.flush(flushFn);
+  async flush(flushFn, flushPolicy) {
+    return await this._inMemory.flush(flushFn, flushPolicy);
   }
 
   /**

package/src/config/configSchema.js

@@ -136,8 +136,12 @@ module.exports = {
       type: 'integer',
       minimum: 0,
       description:
-        'AMQP heartbeat in SECONDS, handed to amqplib at connect. Explicit value, else ENV '
-        + 'RABBITMQ_HEARTBEAT, else the module default (../defaults.js heartbeatSeconds).',
+        'AMQP heartbeat in SECONDS, proposed to the broker in the connection URL (?heartbeat=) at '
+        + 'connect. Explicit value, else ENV RABBITMQ_HEARTBEAT, else the module default '
+        + '(../defaults.js heartbeatSeconds). 0 does NOT switch the heartbeat off: amqplib\'s '
+        + 'negotiation takes whichever side sets a limit, so the broker\'s own value applies '
+        + '(RabbitMQ 60 s unless its configuration says otherwise; measured d.1126: a paused broker '
+        + 'detected after 179 s). Only a broker that also offers 0 disables it.',
     },
     serviceName: {
       type: 'string',
@@ -287,10 +291,12 @@ module.exports = {
       type: 'integer',
       minimum: 0,
       description:
-        'How long this client waits for an ANSWER from the broker, in milliseconds: the confirm of '
-        + 'one publish attempt, and the close-ok of each handle disconnect() closes. One number, two '
-        + 'consumers — it was named publishConfirmationTimeout until d.299, after the teardown had '
-        + 'been spending it since d.283.',
+        'How long this client waits for an ANSWER from the broker, in milliseconds — every wait: the '
+        + 'confirm of one publish attempt, the close-ok of each handle disconnect() closes, and every '
+        + 'request a channel sends and waits on (queue/exchange declarations, checkQueue, bindQueue, '
+        + 'purge, delete, consume, cancel, prefetch, channel open; conf mq-broker-answer-budget 001, '
+        + 'd.1135). One number for one question — it was named publishConfirmationTimeout until d.299, '
+        + 'after the teardown had been spending it since d.283.',
     },
     connectTimeout: {
       type: 'integer',
@@ -389,19 +395,9 @@ module.exports = {
     healthCriticalCallback: callbackOrNull(
       'Async function (health) => void, or null. Called when a health result is critical.'
     ),
-    criticalHealthShutdown: {
-      type: 'boolean',
-      description:
-        'Whether a sustained critical health result makes the transport emit `health:shutdown`. '
-        + 'Anything other than `false` means enabled. It only arms the event; ending the '
-        + 'process remains the owner\'s decision.',
-    },
-    criticalHealthShutdownDelay: {
-      type: 'integer',
-      minimum: 0,
-      description:
-        'How long health must stay critical before that event is emitted, in milliseconds.',
-    },
+    // NO `criticalHealthShutdown` / `criticalHealthShutdownDelay` since d.1275: the
+    // `health:shutdown` event they armed reached no listener. `BaseClient` refuses both
+    // by name before this schema is applied (`NOT_A_KEY_OF_THIS_CLIENT`).
 
     // ---------------------------------------------------------------------
     // `queueCreationFilter` and `queueCreationCallback` stood here until d.419,

package/src/config/defaultConfig.js

@@ -120,12 +120,13 @@ module.exports = {
   publishRetryMaxDelay: 5000,
   publishRetryBackoffMultiplier: 2,
 
-  // How long this client waits for an ANSWER from the broker, in ms: the confirm
-  // of one publish attempt, and the `close-ok` of each handle `disconnect()`
-  // closes — one number for two consumers, which is why it carries the name of
-  // neither. It was `publishConfirmationTimeout` until d.299, a name that was
-  // true of the publish path and false of the teardown that had used it since
-  // d.283.
+  // How long this client waits for an ANSWER from the broker, in ms — every wait:
+  // the confirm of one publish attempt, the `close-ok` of each handle
+  // `disconnect()` closes, and every request a channel sends and waits on
+  // (conf `mq-broker-answer-budget` 001, d.1135) — one number for all of them,
+  // which is why it carries the name of none. It was `publishConfirmationTimeout`
+  // until d.299, a name that was true of the publish path and false of the
+  // teardown that had used it since d.283.
   brokerAnswerTimeout: 5000,
 
   // How long the connect handshake may take before the attempt is abandoned, in
@@ -190,11 +191,9 @@ module.exports = {
   healthReportCallback: null,
   healthCriticalCallback: null,
 
-  // Does critical health end this client, and how long it stays critical first
-  // (ms)? The library only reports; the shutdown it asks for is the owner's
-  // `onFatal` path.
-  criticalHealthShutdown: true,
-  criticalHealthShutdownDelay: 60000,
+  // NO `criticalHealthShutdown` / `criticalHealthShutdownDelay` (removed in d.1275,
+  // refused by name in `BaseClient`): they armed a `health:shutdown` event no one
+  // listened to. Critical health is reported; the process is the owner's.
 
   // `recoveryScope` stood here until d.514, with the two callbacks it filtered
   // (`queueCreationFilter`, `queueCreationCallback`, d.419) one batch ahead of

package/src/config/queueConfig.js

@@ -93,6 +93,28 @@ const INFRASTRUCTURE_PREFIX_LOOKUPS = Object.freeze({
 
 const INFRASTRUCTURE_PREFIXES = Object.freeze(Object.keys(INFRASTRUCTURE_PREFIX_LOOKUPS));
 
+/**
+ * The OBSERVERS' COPIES of a run (d.1336): the queues `monitoring.workflow.fanout` copies a
+ * run's lifecycle events into for a reader that watches the run and is not part of it — the
+ * monitoring consumer's intake, the delivery endpoint's WS feed, the scheduler's end-of-run
+ * watch. A copy whose reader is away dies on its TTL and dead-letters while the run goes on,
+ * so a `message_dlq` from one of them is a lost copy of the run's telemetry, not a lost run;
+ * the monitoring consumer and the scheduler both decide on that (INFRA 559).
+ *
+ * DECLARED, NOT DERIVED — measured on this file: membership cannot be read off an entry. The
+ * binding to the fanout is each consumer's own act at boot (`bindQueueToFanout()`), written on
+ * the entries only as a comment; TTL plus a dead-letter route is what every family here
+ * carries. So the list is written ONCE, here, as `[section, key]` pairs, and every name is
+ * composed by `queueName()` — which refuses an entry this file does not declare, so a release
+ * that dropped one stops the importer instead of shortening the list. A new observer of a run
+ * joins by one line here, beside its declaration below.
+ */
+const WORKFLOW_OBSERVER_COPIES = Object.freeze([
+  ['monitoring', 'workflow'],
+  ['deliveryEvents', 'workflow.events'],
+  ['scheduler', 'workflow.events']
+]);
+
 /** Every section declaring names under one prefix — derived, never a second list. */
 const sectionsForPrefix = (prefix) =>
   Object.keys(SECTION_PREFIXES).filter((section) => SECTION_PREFIXES[section] === prefix);
@@ -725,7 +747,8 @@ module.exports = {
 
     /**
      * monitoring.workflow.fanout - Fanout exchange for workflow events
-     * Publishers send to this exchange; monitoring.workflow + delivery.workflow.events both receive copies.
+     * Publishers send to this exchange; the queues bound to it — the observers' copies of a
+     * run — are listed once, in `WORKFLOW_OBSERVER_COPIES` at the top of this file.
      */
     'workflow.fanout': {
       type: 'exchange',
@@ -1302,6 +1325,17 @@ module.exports = {
     return INFRASTRUCTURE_PREFIXES;
   },
 
+  /**
+   * The names of the observers' copies of a run — `WORKFLOW_OBSERVER_COPIES` above, each
+   * composed by `queueName()`. Published as `topology.workflowObserverCopyQueues`; this is
+   * the one place the list is read from its declaration (d.1336).
+   * @returns {ReadonlyArray<string>} Frozen queue names, in declaration order.
+   * @throws {ValidationError} when a listed entry is not declared in this file.
+   */
+  workflowObserverCopyQueues() {
+    return Object.freeze(WORKFLOW_OBSERVER_COPIES.map(([section, key]) => this.queueName(section, key)));
+  },
+
   /**
    * Check if queue name is an infrastructure queue.
    *

package/src/index.js

@@ -23,6 +23,7 @@ const {
 } = require('./utils/errorHandler');
 const {
   TransientPublishError,
+  RecoveryEndedPublishError,
   PermanentPublishError,
   QueueNotFoundError,
   classifyPublishError,
@@ -87,6 +88,7 @@ module.exports.errorCodes = Object.freeze({
 });
 module.exports.publishErrors = {
   TransientPublishError,
+  RecoveryEndedPublishError,
   PermanentPublishError,
   QueueNotFoundError,
   classifyPublishError,
@@ -139,6 +141,26 @@ module.exports.queueConfig = queueConfig;
  * gone. Frozen, because an importer must not be able to rewrite platform topology.
  */
 module.exports.topology = Object.freeze({
+  /**
+   * The queue every infrastructure service publishes its health checks to and the
+   * registry drains (`docs/standards/queue-ownership.md` § Infrastructure Queue Creation
+   * Order). Declared in `queueConfig` all along; composed here since d.1307, because the
+   * name was carried as a literal one floor up — `@onlineapps/service-common`
+   * `src/defaults.js` `infrastructureHealthQueueName`, which the registry, the health
+   * publisher and the wrapper read and call "the one owner" — so the platform had two
+   * declaring owners of one wire name (finding d.1306). Which side cites this export is
+   * the owner's decision (d.1306 A/B); the export is the step both ways share.
+   */
+  infrastructureHealthChecksQueue: queueConfig.queueName('infrastructure', 'health.checks'),
+  /**
+   * The observers' copies of a run (d.1336, INFRA 559): the queues bound to
+   * `monitoring.workflow.fanout` for a reader that watches the run and is not part of it.
+   * A dead letter from one of them is a lost copy, not a lost run — the monitoring consumer
+   * (`observer_copy`) and the scheduler both decide on that and selected the queues by hand,
+   * twice. Declared once in `queueConfig` (`WORKFLOW_OBSERVER_COPIES`), names composed by
+   * `queueName()`; a frozen array.
+   */
+  workflowObserverCopyQueues: queueConfig.workflowObserverCopyQueues(),
   /** Fanout exchange: registry publishes infrastructure health events here. */
   infrastructureHealthEventsExchange: queueConfig.queueName('infrastructure', 'health.events'),
   /** The monitoring consumer's queue bound to that exchange. */