@zdavison/matador 4.0.2 → 4.0.3

package/dist/core/fanout.d.ts

@@ -13,6 +13,34 @@ export interface FanoutConfig {
     readonly topology: Topology;
     readonly defaultQueue: string;
     readonly maxRetryBufferSize?: number | undefined;
+    /**
+     * Maximum number of retry attempts for a buffered message before it's
+     * dropped and reported via onEnqueueError, instead of being retried forever.
+     *
+     * @default undefined (no limit — retries until the buffer flushes successfully)
+     */
+    readonly maxRetryAttempts?: number | undefined;
+    /**
+     * How often (in ms) to attempt flushing the retry buffer, independent of
+     * transport reconnect events. This covers the case where the transport
+     * stays connected but an individual publish keeps failing (e.g. broker
+     * nack, publish confirm timeout) — onConnected never fires again for
+     * that, so nothing would otherwise retry it.
+     *
+     * @default 30000. Pass 0 to disable the interval and rely on
+     * onConnected/manual flushes only.
+     */
+    readonly retryIntervalMs?: number | undefined;
+    /**
+     * Number of consecutive flush failures tolerated within a single flush
+     * pass before bailing out early and re-buffering the untried remainder,
+     * instead of retrying every buffered message against a broker that's
+     * rejecting everything.
+     *
+     * @default 10. Pass 0 (or a negative number) to disable the breaker and
+     * always attempt every buffered message on every flush pass.
+     */
+    readonly maxConsecutiveFlushFailures?: number | undefined;
 }
 /**
  * Result of sending an event.
@@ -47,9 +75,13 @@ export declare class FanoutEngine {
     private readonly topology;
     private readonly defaultQueue;
     private enqueuingCount;
+    private flushInFlightCount;
     private readonly retryBuffer;
     private readonly maxRetryBufferSize;
+    private readonly maxRetryAttempts;
+    private readonly maxConsecutiveFlushFailures;
     private readonly disposeOnConnected;
+    private readonly retryTimer;
     constructor(config: FanoutConfig);
     dispose(): void;
     /**
@@ -60,12 +92,31 @@ export declare class FanoutEngine {
      * Sends an event to all registered subscribers.
      */
     send<T>(eventClass: EventClass<T>, event: Event<T>, options?: EventOptions): Promise<SendResult>;
+    /**
+     * Attempts a single buffered item during a flush pass
+     * @returns true if the attempt succeeded; false on failure
+     */
+    private attemptFlushItem;
     /**
      * Flushes the retry buffer
      *
      * This is called when the transport reconnects, and is used to retry any messages that were buffered while the transport was disconnected
      */
     private flushRetryBuffer;
+    /**
+     * Re-buffers messages that were never attempted because a flush pass
+     * stopped out early. Respects maxRetryBufferSize: concurrent sends can have
+     * refilled the buffer while this flush was running, so anything beyond
+     * remaining capacity is dropped and reported instead of silently growing
+     * the buffer past its cap.
+     */
+    private rebufferUntried;
+    /**
+     * Handles a failed flush attempt for a single buffered item: drops it once
+     * maxRetryAttempts is exceeded, otherwise re-buffers it (unless the buffer
+     * is full, in which case it's dropped too).
+     */
+    private handleFlushFailure;
     private isSubscriberEnabled;
 }
 //# sourceMappingURL=fanout.d.ts.map

package/dist/core/fanout.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fanout.d.ts","sourceRoot":"","sources":["../../src/core/fanout.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAe,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,KAAK,EAGV,KAAK,EACL,UAAU,EACV,YAAY,EACb,MAAM,mBAAmB,CAAC;AAa3B;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClD;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAY;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA2B;gBAElD,MAAM,EAAE,YAAY;IAehC,OAAO,IAAI,IAAI;IAKf;;OAEG;IACH,IAAI,wBAAwB,IAAI,MAAM,CAErC;IAED;;OAEG;IACG,IAAI,CAAC,CAAC,EACV,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EACzB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EACf,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,UAAU,CAAC;IAyHtB;;;;OAIG;YACW,gBAAgB;YAsDhB,mBAAmB;CAelC"}
+{"version":3,"file":"fanout.d.ts","sourceRoot":"","sources":["../../src/core/fanout.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAe,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,KAAK,EAGV,KAAK,EACL,UAAU,EACV,YAAY,EACb,MAAM,mBAAmB,CAAC;AAsB3B;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEjD;;;;;OAKG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE/C;;;;;;;;;OASG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE9C;;;;;;;;OAQG;IACH,QAAQ,CAAC,2BAA2B,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3D;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAY;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAS;IACrD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA2B;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6C;gBAE5D,MAAM,EAAE,YAAY;IAkChC,OAAO,IAAI,IAAI;IAMf;;OAEG;IACH,IAAI,wBAAwB,IAAI,MAAM,CAErC;IAED;;OAEG;IACG,IAAI,CAAC,CAAC,EACV,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EACzB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EACf,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,UAAU,CAAC;IAgItB;;;OAGG;YACW,gBAAgB;IAsB9B;;;;OAIG;YACW,gBAAgB;IA2D9B;;;;;;OAMG;YACW,eAAe;IAqC7B;;;;OAIG;YACW,kBAAkB;YAiClB,mBAAmB;CAelC"}

package/dist/core/fanout.js

@@ -1,6 +1,12 @@
 import { TransportSendError } from '../errors/index.js';
 import { resolveTargetQueueName } from '../topology/index.js';
 import { createEnvelope } from '../types/index.js';
+/**
+ * Default number of consecutive flush failures tolerated within a single
+ * flush pass before bailing out early and re-buffering the untried
+ * remainder.
+ */
+const DEFAULT_MAX_CONSECUTIVE_FLUSH_FAILURES = 10;
 /**
  * Engine for fanning out events to subscribers.
  *
@@ -17,9 +23,13 @@ export class FanoutEngine {
     topology;
     defaultQueue;
     enqueuingCount = 0;
+    flushInFlightCount = 0;
     retryBuffer = [];
     maxRetryBufferSize;
+    maxRetryAttempts;
+    maxConsecutiveFlushFailures;
     disposeOnConnected;
+    retryTimer;
     constructor(config) {
         this.transport = config.transport;
         this.schema = config.schema;
@@ -27,21 +37,39 @@ export class FanoutEngine {
         this.topology = config.topology;
         this.defaultQueue = config.defaultQueue;
         this.maxRetryBufferSize = config.maxRetryBufferSize ?? 5000;
+        this.maxRetryAttempts = config.maxRetryAttempts;
+        const maxConsecutiveFlushFailures = config.maxConsecutiveFlushFailures ??
+            DEFAULT_MAX_CONSECUTIVE_FLUSH_FAILURES;
+        this.maxConsecutiveFlushFailures =
+            maxConsecutiveFlushFailures > 0
+                ? maxConsecutiveFlushFailures
+                : Number.POSITIVE_INFINITY;
         // If the transport supports it, add a callback to flush the retry buffer when the transport reconnects
         // The returned value is a function to unsubscribe from the callback
         this.disposeOnConnected = this.transport.onConnected?.(() => {
             void this.flushRetryBuffer();
         });
+        // Also flush periodically: onConnected only fires on connection-level
+        // reconnects, so it never fires when the transport stays connected but
+        // an individual publish keeps failing (e.g. broker nack, confirm timeout).
+        const retryIntervalMs = config.retryIntervalMs ?? 30_000;
+        if (retryIntervalMs > 0) {
+            this.retryTimer = setInterval(() => {
+                void this.flushRetryBuffer();
+            }, retryIntervalMs);
+            this.retryTimer.unref?.();
+        }
     }
     dispose() {
         // Unsubscribe from the callback (that flushes the retry buffer when the transport reconnects)
         this.disposeOnConnected?.();
+        clearInterval(this.retryTimer);
     }
     /**
      * Current count of events being enqueued.
      */
     get eventsBeingEnqueuedCount() {
-        return this.enqueuingCount;
+        return this.enqueuingCount + this.flushInFlightCount;
     }
     /**
      * Sends an event to all registered subscribers.
@@ -101,8 +129,14 @@ export class FanoutEngine {
                         envelope,
                         sendOptions,
                         subscriberName: subscriber.name,
+                        attempts: 0,
+                    });
+                    this.hooks.logger.warn(`[Matador] 🟡 Message for '${subscriber.name}' buffered for retry on reconnect.`, {
+                        queue: qualifiedQueue,
+                        subscriberName: subscriber.name,
+                        bufferSize: this.retryBuffer.length,
+                        maxBufferSize: this.maxRetryBufferSize,
                     });
-                    this.hooks.logger.warn(`[Matador] 🟡 Message for '${subscriber.name}' buffered for retry on reconnect (buffer: ${this.retryBuffer.length}/${this.maxRetryBufferSize}).`);
                     if (options.throwOnBufferedFailure) {
                         const err = new TransportSendError(qualifiedQueue, cause);
                         // By adding the error, the main Send in Matador will throw
@@ -146,6 +180,29 @@ export class FanoutEngine {
             errors,
         };
     }
+    /**
+     * Attempts a single buffered item during a flush pass
+     * @returns true if the attempt succeeded; false on failure
+     */
+    async attemptFlushItem(item) {
+        this.enqueuingCount++;
+        try {
+            const usedTransport = await this.transport.send(item.queue, item.envelope, item.sendOptions);
+            await this.hooks.onEnqueueSuccess({
+                envelope: item.envelope,
+                queue: item.queue,
+                transport: usedTransport,
+            });
+            return true;
+        }
+        catch (error) {
+            await this.handleFlushFailure(item, error);
+            return false;
+        }
+        finally {
+            this.enqueuingCount--;
+        }
+    }
     /**
      * Flushes the retry buffer
      *
@@ -157,43 +214,99 @@ export class FanoutEngine {
         this.hooks.logger.info(`[Matador] ⏳ Flushing ${this.retryBuffer.length} buffered message(s)...`);
         // Drain the buffer atomically so concurrent flush calls don't double-send.
         const toFlush = this.retryBuffer.splice(0);
-        for (const item of toFlush) {
-            this.enqueuingCount++;
-            try {
-                const usedTransport = await this.transport.send(item.queue, item.envelope, item.sendOptions);
-                await this.hooks.onEnqueueSuccess({
-                    envelope: item.envelope,
-                    queue: item.queue,
-                    transport: usedTransport,
-                });
-            }
-            catch (error) {
-                // Re-buffer on failure; it will be retried on the next reconnect.
-                if (this.retryBuffer.length < this.maxRetryBufferSize) {
-                    this.retryBuffer.push(item);
+        // Tracked separately from enqueuingCount so shutdown doesn't see a it falsely idle while items are being flushed
+        this.flushInFlightCount += toFlush.length;
+        let consecutiveFailures = 0;
+        try {
+            for (let i = 0; i < toFlush.length; i++) {
+                const item = toFlush[i];
+                if (!item) {
+                    this.flushInFlightCount--;
+                    continue;
                 }
-                else {
-                    const cause = error instanceof Error ? error : new Error(String(error));
-                    const err = new TransportSendError(item.queue, cause);
-                    await this.hooks.onEnqueueError({
-                        envelope: item.envelope,
-                        error: err,
-                        transport: this.transport.name,
-                    });
+                const succeeded = await this.attemptFlushItem(item);
+                this.flushInFlightCount--;
+                // Small sleep to avoid overwhelming the transport
+                // Random sleep to avoid thundering herd (between 0 and 10ms)
+                const randomMs = Math.random() * 10;
+                await new Promise((resolve) => setTimeout(resolve, randomMs));
+                // A success means the transport is actually working, so it resets the consecutive failures streak
+                consecutiveFailures = succeeded ? 0 : consecutiveFailures + 1;
+                if (consecutiveFailures >= this.maxConsecutiveFlushFailures) {
+                    // A run of failures this long is more likely a connection-level issue (broker down, slow, rejecting everything)
+                    // So we stop early to avoid retrying 5k messages for nothing
+                    const untried = toFlush.slice(i + 1);
+                    this.flushInFlightCount -= untried.length;
+                    if (untried.length > 0) {
+                        await this.rebufferUntried(untried, consecutiveFailures);
+                    }
+                    break;
                 }
             }
-            finally {
-                this.enqueuingCount--;
-            }
+        }
+        finally {
+            this.flushInFlightCount = 0;
         }
         const remaining = this.retryBuffer.length;
         if (remaining > 0) {
-            this.hooks.logger.warn(`[Matador] 🟡 ${remaining} buffered message(s) could not be flushed; will retry on next reconnect.`);
+            this.hooks.logger.warn(`[Matador] 🟡 ${remaining} buffered message(s) could not be flushed; will retry later.`);
         }
         else {
             this.hooks.logger.info('[Matador] 🟢 All buffered messages flushed successfully.');
         }
     }
+    /**
+     * Re-buffers messages that were never attempted because a flush pass
+     * stopped out early. Respects maxRetryBufferSize: concurrent sends can have
+     * refilled the buffer while this flush was running, so anything beyond
+     * remaining capacity is dropped and reported instead of silently growing
+     * the buffer past its cap.
+     */
+    async rebufferUntried(untried, consecutiveFailures) {
+        const capacity = Math.max(0, this.maxRetryBufferSize - this.retryBuffer.length);
+        const toRebuffer = untried.slice(0, capacity);
+        const dropped = untried.slice(capacity);
+        if (toRebuffer.length > 0) {
+            this.retryBuffer.unshift(...toRebuffer);
+        }
+        const droppedSuffix = dropped.length > 0 ? `, ${dropped.length} dropped (buffer full).` : '.';
+        this.hooks.logger.warn(`[Matador] 🟡 Stopping this flush pass after ${consecutiveFailures} consecutive failures; ${toRebuffer.length} untried message(s) re-buffered for the next attempt${droppedSuffix}`);
+        for (const item of dropped) {
+            this.hooks.logger.error(`[Matador] 🔴 Retry buffer full (${this.maxRetryBufferSize}). Message for '${item.subscriberName}' dropped and will not be retried.`);
+            const err = new TransportSendError(item.queue, new Error('Retry buffer full'));
+            await this.hooks.onEnqueueError({
+                envelope: item.envelope,
+                error: err,
+                transport: this.transport.name,
+            });
+        }
+    }
+    /**
+     * Handles a failed flush attempt for a single buffered item: drops it once
+     * maxRetryAttempts is exceeded, otherwise re-buffers it (unless the buffer
+     * is full, in which case it's dropped too).
+     */
+    async handleFlushFailure(item, error) {
+        item.attempts++;
+        const cause = error instanceof Error ? error : new Error(String(error));
+        const exceededAttempts = this.maxRetryAttempts !== undefined &&
+            item.attempts >= this.maxRetryAttempts;
+        if (!exceededAttempts &&
+            this.retryBuffer.length < this.maxRetryBufferSize) {
+            // Re-buffer on failure; it will be retried on the next reconnect or flush interval.
+            this.retryBuffer.push(item);
+            return;
+        }
+        if (exceededAttempts) {
+            this.hooks.logger.error(`[Matador] 🔴 Message for '${item.subscriberName}' exceeded max retry attempts (${this.maxRetryAttempts}) and will not be retried further.`);
+        }
+        const err = new TransportSendError(item.queue, cause);
+        await this.hooks.onEnqueueError({
+            envelope: item.envelope,
+            error: err,
+            transport: this.transport.name,
+        });
+    }
     async isSubscriberEnabled(subscriber) {
         if (!subscriber.enabled) {
             return true;