@zdavison/matador 4.0.2 → 4.0.3

package/dist/index.cjs

@@ -202,6 +202,16 @@ var DelayedMessagesNotSupportedError = class extends MatadorError {
   transportName;
   description = "Delayed messages were requested but the transport does not support them. ACTION: For RabbitMQ, install the rabbitmq_delayed_message_exchange plugin. Run: rabbitmq-plugins enable rabbitmq_delayed_message_exchange Then restart RabbitMQ and reconnect. Alternatively, remove delayMs from your event options if delays are not required.";
 };
+var LocalTransportNoActiveSubscriberError = class extends MatadorError {
+  constructor(queue) {
+    super(
+      `LocalTransport has no active subscriber for queue "${queue}"; message cannot be delivered.`
+    );
+    this.queue = queue;
+  }
+  queue;
+  description = "The LocalTransport has no active subscriber for this queue in the current process, so it cannot guarantee delivery. ACTION: If the subscriber is a SubscriberStub (implemented in another service), the local transport can never deliver it locally \u2014 a distributed transport like RabbitMQ is required. If the subscriber should run in this process, ensure matador.start() has subscribed to this queue before sending.";
+};
 var EventNotRegisteredError = class extends MatadorError {
   constructor(eventKey) {
     super(
@@ -252,16 +262,6 @@ var SubscriberIsStubError = class extends MatadorError {
   subscriberName;
   description = "A SubscriberStub was registered in a consuming schema. ACTION: SubscriberStubs should only be used in producer schemas to declare that a subscriber exists in another service. In the consumer service, provide a full Subscriber with a callback function. Remove the stub from the consumer schema and add the actual implementation.";
 };
-var LocalTransportCannotProcessStubError = class extends MatadorError {
-  constructor(subscriberName) {
-    super(
-      `LocalTransport cannot process stub subscriber "${subscriberName}". Stub subscribers require a distributed transport like RabbitMQ.`
-    );
-    this.subscriberName = subscriberName;
-  }
-  subscriberName;
-  description = "The LocalTransport cannot process events for SubscriberStubs. ACTION: SubscriberStubs represent remote implementations that only RabbitMQ can route. If using LocalTransport for testing, provide mock implementations instead of stubs. For production fallback scenarios, be aware that stub-targeted events will be dropped.";
-};
 var UnknownQueueReferenceError = class extends MatadorError {
   constructor(queueName) {
     super(
@@ -801,6 +801,7 @@ function createSubscriberStub(input) {
 }
 
 // src/core/fanout.ts
+var DEFAULT_MAX_CONSECUTIVE_FLUSH_FAILURES = 10;
 var FanoutEngine = class {
   transport;
   schema;
@@ -808,9 +809,13 @@ var FanoutEngine = class {
   topology;
   defaultQueue;
   enqueuingCount = 0;
+  flushInFlightCount = 0;
   retryBuffer = [];
   maxRetryBufferSize;
+  maxRetryAttempts;
+  maxConsecutiveFlushFailures;
   disposeOnConnected;
+  retryTimer;
   constructor(config) {
     this.transport = config.transport;
     this.schema = config.schema;
@@ -818,18 +823,29 @@ var FanoutEngine = class {
     this.topology = config.topology;
     this.defaultQueue = config.defaultQueue;
     this.maxRetryBufferSize = config.maxRetryBufferSize ?? 5e3;
+    this.maxRetryAttempts = config.maxRetryAttempts;
+    const maxConsecutiveFlushFailures = config.maxConsecutiveFlushFailures ?? DEFAULT_MAX_CONSECUTIVE_FLUSH_FAILURES;
+    this.maxConsecutiveFlushFailures = maxConsecutiveFlushFailures > 0 ? maxConsecutiveFlushFailures : Number.POSITIVE_INFINITY;
     this.disposeOnConnected = this.transport.onConnected?.(() => {
       void this.flushRetryBuffer();
     });
+    const retryIntervalMs = config.retryIntervalMs ?? 3e4;
+    if (retryIntervalMs > 0) {
+      this.retryTimer = setInterval(() => {
+        void this.flushRetryBuffer();
+      }, retryIntervalMs);
+      this.retryTimer.unref?.();
+    }
   }
   dispose() {
     this.disposeOnConnected?.();
+    clearInterval(this.retryTimer);
   }
   /**
    * Current count of events being enqueued.
    */
   get eventsBeingEnqueuedCount() {
-    return this.enqueuingCount;
+    return this.enqueuingCount + this.flushInFlightCount;
   }
   /**
    * Sends an event to all registered subscribers.
@@ -883,10 +899,17 @@ var FanoutEngine = class {
             queue: qualifiedQueue,
             envelope,
             sendOptions,
-            subscriberName: subscriber.name
+            subscriberName: subscriber.name,
+            attempts: 0
           });
           this.hooks.logger.warn(
-            `[Matador] \u{1F7E1} Message for '${subscriber.name}' buffered for retry on reconnect (buffer: ${this.retryBuffer.length}/${this.maxRetryBufferSize}).`
+            `[Matador] \u{1F7E1} Message for '${subscriber.name}' buffered for retry on reconnect.`,
+            {
+              queue: qualifiedQueue,
+              subscriberName: subscriber.name,
+              bufferSize: this.retryBuffer.length,
+              maxBufferSize: this.maxRetryBufferSize
+            }
           );
           if (options.throwOnBufferedFailure) {
             const err = new TransportSendError(qualifiedQueue, cause);
@@ -930,6 +953,31 @@ var FanoutEngine = class {
       errors
     };
   }
+  /**
+   * Attempts a single buffered item during a flush pass
+   * @returns true if the attempt succeeded; false on failure
+   */
+  async attemptFlushItem(item) {
+    this.enqueuingCount++;
+    try {
+      const usedTransport = await this.transport.send(
+        item.queue,
+        item.envelope,
+        item.sendOptions
+      );
+      await this.hooks.onEnqueueSuccess({
+        envelope: item.envelope,
+        queue: item.queue,
+        transport: usedTransport
+      });
+      return true;
+    } catch (error) {
+      await this.handleFlushFailure(item, error);
+      return false;
+    } finally {
+      this.enqueuingCount--;
+    }
+  }
   /**
    * Flushes the retry buffer
    *
@@ -941,39 +989,36 @@ var FanoutEngine = class {
       `[Matador] \u23F3 Flushing ${this.retryBuffer.length} buffered message(s)...`
     );
     const toFlush = this.retryBuffer.splice(0);
-    for (const item of toFlush) {
-      this.enqueuingCount++;
-      try {
-        const usedTransport = await this.transport.send(
-          item.queue,
-          item.envelope,
-          item.sendOptions
-        );
-        await this.hooks.onEnqueueSuccess({
-          envelope: item.envelope,
-          queue: item.queue,
-          transport: usedTransport
-        });
-      } catch (error) {
-        if (this.retryBuffer.length < this.maxRetryBufferSize) {
-          this.retryBuffer.push(item);
-        } else {
-          const cause = error instanceof Error ? error : new Error(String(error));
-          const err = new TransportSendError(item.queue, cause);
-          await this.hooks.onEnqueueError({
-            envelope: item.envelope,
-            error: err,
-            transport: this.transport.name
-          });
+    this.flushInFlightCount += toFlush.length;
+    let consecutiveFailures = 0;
+    try {
+      for (let i = 0; i < toFlush.length; i++) {
+        const item = toFlush[i];
+        if (!item) {
+          this.flushInFlightCount--;
+          continue;
+        }
+        const succeeded = await this.attemptFlushItem(item);
+        this.flushInFlightCount--;
+        const randomMs = Math.random() * 10;
+        await new Promise((resolve) => setTimeout(resolve, randomMs));
+        consecutiveFailures = succeeded ? 0 : consecutiveFailures + 1;
+        if (consecutiveFailures >= this.maxConsecutiveFlushFailures) {
+          const untried = toFlush.slice(i + 1);
+          this.flushInFlightCount -= untried.length;
+          if (untried.length > 0) {
+            await this.rebufferUntried(untried, consecutiveFailures);
+          }
+          break;
         }
-      } finally {
-        this.enqueuingCount--;
       }
+    } finally {
+      this.flushInFlightCount = 0;
     }
     const remaining = this.retryBuffer.length;
     if (remaining > 0) {
       this.hooks.logger.warn(
-        `[Matador] \u{1F7E1} ${remaining} buffered message(s) could not be flushed; will retry on next reconnect.`
+        `[Matador] \u{1F7E1} ${remaining} buffered message(s) could not be flushed; will retry later.`
       );
     } else {
       this.hooks.logger.info(
@@ -981,6 +1026,67 @@ var FanoutEngine = class {
       );
     }
   }
+  /**
+   * Re-buffers messages that were never attempted because a flush pass
+   * stopped out early. Respects maxRetryBufferSize: concurrent sends can have
+   * refilled the buffer while this flush was running, so anything beyond
+   * remaining capacity is dropped and reported instead of silently growing
+   * the buffer past its cap.
+   */
+  async rebufferUntried(untried, consecutiveFailures) {
+    const capacity = Math.max(
+      0,
+      this.maxRetryBufferSize - this.retryBuffer.length
+    );
+    const toRebuffer = untried.slice(0, capacity);
+    const dropped = untried.slice(capacity);
+    if (toRebuffer.length > 0) {
+      this.retryBuffer.unshift(...toRebuffer);
+    }
+    const droppedSuffix = dropped.length > 0 ? `, ${dropped.length} dropped (buffer full).` : ".";
+    this.hooks.logger.warn(
+      `[Matador] \u{1F7E1} Stopping this flush pass after ${consecutiveFailures} consecutive failures; ${toRebuffer.length} untried message(s) re-buffered for the next attempt${droppedSuffix}`
+    );
+    for (const item of dropped) {
+      this.hooks.logger.error(
+        `[Matador] \u{1F534} Retry buffer full (${this.maxRetryBufferSize}). Message for '${item.subscriberName}' dropped and will not be retried.`
+      );
+      const err = new TransportSendError(
+        item.queue,
+        new Error("Retry buffer full")
+      );
+      await this.hooks.onEnqueueError({
+        envelope: item.envelope,
+        error: err,
+        transport: this.transport.name
+      });
+    }
+  }
+  /**
+   * Handles a failed flush attempt for a single buffered item: drops it once
+   * maxRetryAttempts is exceeded, otherwise re-buffers it (unless the buffer
+   * is full, in which case it's dropped too).
+   */
+  async handleFlushFailure(item, error) {
+    item.attempts++;
+    const cause = error instanceof Error ? error : new Error(String(error));
+    const exceededAttempts = this.maxRetryAttempts !== void 0 && item.attempts >= this.maxRetryAttempts;
+    if (!exceededAttempts && this.retryBuffer.length < this.maxRetryBufferSize) {
+      this.retryBuffer.push(item);
+      return;
+    }
+    if (exceededAttempts) {
+      this.hooks.logger.error(
+        `[Matador] \u{1F534} Message for '${item.subscriberName}' exceeded max retry attempts (${this.maxRetryAttempts}) and will not be retried further.`
+      );
+    }
+    const err = new TransportSendError(item.queue, cause);
+    await this.hooks.onEnqueueError({
+      envelope: item.envelope,
+      error: err,
+      transport: this.transport.name
+    });
+  }
   async isSubscriberEnabled(subscriber) {
     if (!subscriber.enabled) {
       return true;
@@ -2273,7 +2379,11 @@ var Matador = class {
       schema: this.schema,
       hooks: this.hooks,
       topology: this.topology,
-      defaultQueue
+      defaultQueue,
+      maxRetryBufferSize: config.maxRetryBufferSize,
+      maxRetryAttempts: config.maxRetryAttempts,
+      retryIntervalMs: config.retryIntervalMs,
+      maxConsecutiveFlushFailures: config.maxConsecutiveFlushFailures
     });
     this.shutdownManager = new ShutdownManager(
       () => this.fanout.eventsBeingEnqueuedCount,
@@ -2459,7 +2569,7 @@ var Matador = class {
   }
   async unsubscribeAll() {
     for (const subscription of this.subscriptions) {
-      await subscription.unsubscribe();
+      await (subscription.pauseForShutdown ?? subscription.unsubscribe)();
     }
     this.subscriptions.length = 0;
   }
@@ -2621,6 +2731,7 @@ var localCapabilities = {
   ordering: "queue",
   priorities: false
 };
+var MAX_COMPLETED_MESSAGES = 1e3;
 var LocalTransport = class {
   name = "local";
   capabilities = localCapabilities;
@@ -2664,7 +2775,7 @@ var LocalTransport = class {
         topology.prefix
       );
       if (!this.queues.has(queueName)) {
-        this.queues.set(queueName, []);
+        this.queues.set(queueName, /* @__PURE__ */ new Map());
       }
     }
   }
@@ -2696,6 +2807,18 @@ var LocalTransport = class {
     this.delayedTimers.add(timer);
   }
   async enqueue(queue, envelope) {
+    const subs = this.subscriptions.get(queue);
+    if (!subs || subs.length === 0) {
+      throw new LocalTransportNoActiveSubscriberError(queue);
+    }
+    await this.storeAndDeliver(queue, envelope);
+  }
+  /**
+   * Stores a message and delivers it to any active subscribers, without
+   * requiring one to be present. Dead-letter queues are meant to hold
+   * messages for later manual inspection, so they don't need a live consumer.
+   */
+  async storeAndDeliver(queue, envelope) {
     const messages = this.getOrCreateQueue(queue);
     const messageId = `${++this.messageIdCounter}`;
     const queuedMessage = {
@@ -2703,17 +2826,11 @@ var LocalTransport = class {
       id: messageId,
       completed: false
     };
-    messages.push(queuedMessage);
+    messages.set(messageId, queuedMessage);
     await this.deliverToSubscribers(queue, queuedMessage);
   }
   async deliverToSubscribers(queue, message) {
-    const subs = this.subscriptions.get(queue);
-    if (!subs) {
-      this.logger.warn(
-        `[Matador][LocalTransport] \u{1F7E1} No subscriptions found for queue '${queue}', message will be lost.`
-      );
-      return;
-    }
+    const subs = this.subscriptions.get(queue) ?? [];
     for (const sub of subs) {
       if (!sub.active || message.completed) continue;
       const receipt = {
@@ -2746,7 +2863,7 @@ var LocalTransport = class {
     const subs = this.subscriptions.get(queue) ?? [];
     subs.push(subscription);
     this.subscriptions.set(queue, subs);
-    const messages = this.queues.get(queue) ?? [];
+    const messages = this.queues.get(queue)?.values() ?? [];
     for (const message of messages) {
       if (message.completed) continue;
       await this.deliverToSubscribers(queue, message);
@@ -2761,6 +2878,11 @@ var LocalTransport = class {
           this.subscriptions.delete(queue);
         }
       },
+      // No-op: LocalTransport is only ever fed from within the process
+      // So, when shutting down, it needs to continue and handle new messages until the end
+      // As, an ongoing message (before shutdown) could create new (local) messages
+      pauseForShutdown: async () => {
+      },
       get isActive() {
         return subscription.active;
       }
@@ -2770,10 +2892,14 @@ var LocalTransport = class {
     const message = receipt.handle;
     message.completed = true;
     this.completedMessages.push(receipt);
+    if (this.completedMessages.length > MAX_COMPLETED_MESSAGES) {
+      this.completedMessages.shift();
+    }
+    this.queues.get(receipt.sourceQueue)?.delete(message.id);
   }
   async sendToDeadLetter(receipt, dlqName, envelope, _reason) {
     const dlqQueueName = `${receipt.sourceQueue}.${dlqName}`;
-    await this.enqueue(dlqQueueName, envelope);
+    await this.storeAndDeliver(dlqQueueName, envelope);
     await this.complete(receipt);
   }
   // Test helpers
@@ -2781,9 +2907,7 @@ var LocalTransport = class {
    * Gets the current size of a queue.
    */
   getQueueSize(queue) {
-    const messages = this.queues.get(queue);
-    if (!messages) return 0;
-    return messages.filter((m) => !m.completed).length;
+    return this.queues.get(queue)?.size ?? 0;
   }
   /**
    * Gets all completed message receipts.
@@ -2797,7 +2921,7 @@ var LocalTransport = class {
   getPendingMessages(queue) {
     const messages = this.queues.get(queue);
     if (!messages) return [];
-    return messages.filter((m) => !m.completed).map((m) => m.envelope);
+    return Array.from(messages.values()).map((m) => m.envelope);
   }
   /**
    * Clears all state (for test isolation).
@@ -2819,7 +2943,7 @@ var LocalTransport = class {
   async receiveOne(queue) {
     const messages = this.queues.get(queue);
     if (!messages) return null;
-    const pending = messages.find((m) => !m.completed);
+    const pending = messages.values().next().value;
     if (!pending) return null;
     const receipt = {
       handle: pending,
@@ -2834,7 +2958,7 @@ var LocalTransport = class {
   getOrCreateQueue(queue) {
     let messages = this.queues.get(queue);
     if (!messages) {
-      messages = [];
+      messages = /* @__PURE__ */ new Map();
       this.queues.set(queue, messages);
     }
     return messages;
@@ -2977,6 +3101,11 @@ var MultiTransport = class {
       unsubscribe: async () => {
         await Promise.all(subscriptions.map((s) => s.unsubscribe()));
       },
+      pauseForShutdown: async () => {
+        await Promise.all(
+          subscriptions.map((s) => (s.pauseForShutdown ?? s.unsubscribe)())
+        );
+      },
       get isActive() {
         return subscriptions.some((s) => s.isActive);
       }
@@ -3153,33 +3282,37 @@ var RabbitMQTransport = class {
     };
     this.subscriptionIntents.push(intent);
     await this.activateIntent(intent);
-    return {
-      unsubscribe: async () => {
-        intent.active = false;
-        const idx = this.subscriptionIntents.indexOf(intent);
-        if (idx !== -1) this.subscriptionIntents.splice(idx, 1);
-        const consumer = intent.currentConsumer;
-        if (consumer) {
-          consumer.active = false;
-          const queueChannel = this.queueChannels.get(queue);
-          if (queueChannel) {
+    const unsubscribe = async () => {
+      intent.active = false;
+      const idx = this.subscriptionIntents.indexOf(intent);
+      if (idx !== -1) this.subscriptionIntents.splice(idx, 1);
+      const consumer = intent.currentConsumer;
+      if (consumer) {
+        consumer.active = false;
+        const queueChannel = this.queueChannels.get(queue);
+        if (queueChannel) {
+          try {
+            await queueChannel.channel.cancel(consumer.consumerTag);
+          } catch {
+          }
+          const cIdx = queueChannel.consumers.indexOf(consumer);
+          if (cIdx !== -1) queueChannel.consumers.splice(cIdx, 1);
+          if (queueChannel.consumers.length === 0) {
             try {
-              await queueChannel.channel.cancel(consumer.consumerTag);
+              await queueChannel.channel.close();
             } catch {
             }
-            const cIdx = queueChannel.consumers.indexOf(consumer);
-            if (cIdx !== -1) queueChannel.consumers.splice(cIdx, 1);
-            if (queueChannel.consumers.length === 0) {
-              try {
-                await queueChannel.channel.close();
-              } catch {
-              }
-              this.queueChannels.delete(queue);
-            }
+            this.queueChannels.delete(queue);
           }
-          intent.currentConsumer = null;
         }
-      },
+        intent.currentConsumer = null;
+      }
+    };
+    return {
+      unsubscribe,
+      // Messages on this transport comes from outside (RabbitMQ), so, when shutting down, we need to unsubscribe directly
+      // Otherwise, we could receive traffic forever, and an idle state would never be reached.
+      pauseForShutdown: unsubscribe,
       get isActive() {
         return intent.active;
       }
@@ -3634,7 +3767,7 @@ exports.InvalidEventError = InvalidEventError;
 exports.InvalidSchemaError = InvalidSchemaError;
 exports.JsonCodec = JsonCodec;
 exports.LocalTransport = LocalTransport;
-exports.LocalTransportCannotProcessStubError = LocalTransportCannotProcessStubError;
+exports.LocalTransportNoActiveSubscriberError = LocalTransportNoActiveSubscriberError;
 exports.Matador = Matador;
 exports.MatadorError = MatadorError;
 exports.MatadorEvent = MatadorEvent;