@linagora/rabbitmq-client 0.1.2 → 0.2.1

package/README.md

@@ -37,9 +37,10 @@ Only `url` is required. Everything else has defaults.
 | `connectionRetryDelay` | `5000` | Milliseconds between reconnection attempts |
 | `initMaxAttempts` | `5` | Connection attempts on startup before giving up |
 | `publishMaxAttempts` | `5` | Publish retries (exponential backoff, capped at 60s) |
-| `prefetch` | `10` | Channel prefetch count |
+| `prefetch` | `10` | Channel prefetch: how many messages the broker delivers into consumer memory before waiting for acks |
+| `concurrency` | = `prefetch` | How many handlers run at once per subscription. Prefetch buffers messages in memory; concurrency bounds how many are processed simultaneously. Only throttles when set below `prefetch` (the broker never delivers more than `prefetch` unacked at a time); `0` means no limit. Override per queue via `subscribe(..., { concurrency })` |
 | `closeTimeout` | `5000` | Milliseconds to wait for in-flight messages when closing |
-| `logger` | -- | Parent [pino](https://github.com/pinojs/pino) instance (the library creates its own if omitted) |
+| `logger` | console (warn/error) | Any `ILogger` (`console`, Winston, pino, Bunyan). Omitted: logs warn/error to the console, stays quiet otherwise. Pass `silentLogger` for no output. |
 | `hooks` | -- | Observability callbacks (see [Hooks](#hooks)) |
 
 ## How it works
@@ -79,6 +80,15 @@ await client.subscribe('events', 'order.placed', 'order-queue', handler, {
 
 Custom arguments are merged with the DLQ wiring defaults, so you can swap the queue type or add a max-length without losing dead-letter routing.
 
+To limit how many messages this queue processes at once (for example, to protect a downstream database from a burst), set `concurrency` on the subscription. It defaults to the client-level `concurrency`, which in turn defaults to `prefetch`:
+
+```typescript
+// Buffer up to 50 messages in memory, but process at most 4 at a time.
+await client.subscribe('events', 'order.placed', 'order-queue', handler, {
+  concurrency: 4,
+})
+```
+
 ### Unsubscribing
 
 Cancel a consumer and remove it from the auto-restoration list:

package/dist/index.cjs

@@ -30,7 +30,9 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
-  RabbitMQClient: () => RabbitMQClient
+  RabbitMQClient: () => RabbitMQClient,
+  defaultLogger: () => defaultLogger,
+  silentLogger: () => silentLogger
 });
 module.exports = __toCommonJS(src_exports);
 
@@ -38,12 +40,69 @@ module.exports = __toCommonJS(src_exports);
 var import_amqplib = __toESM(require("amqplib"), 1);
 
 // src/logger.ts
-var import_pino = __toESM(require("pino"), 1);
-var createLogger = (parent) => {
-  if (parent) {
-    return parent.child({ service: "rabbitmq-client" });
+var defaultLogger = {
+  debug: () => {
+  },
+  info: () => {
+  },
+  warn: (message, ...args) => console.warn("[rabbitmq-client]", message, ...args),
+  error: (message, ...args) => console.error("[rabbitmq-client]", message, ...args)
+};
+var silentLogger = {
+  debug: () => {
+  },
+  info: () => {
+  },
+  warn: () => {
+  },
+  error: () => {
+  }
+};
+
+// src/semaphore.ts
+var Semaphore = class {
+  permits;
+  waiters = [];
+  constructor(permits) {
+    if (!Number.isInteger(permits) || permits < 1) {
+      throw new Error(`Semaphore requires a positive integer of permits, got ${permits}`);
+    }
+    this.permits = permits;
+  }
+  /**
+   * Acquires a permit, resolving immediately when one is free or queueing
+   * (FIFO) until another holder releases. Always pair with `release()` in a
+   * `finally` so a throwing handler never leaks a permit.
+   */
+  acquire() {
+    if (this.permits > 0) {
+      this.permits--;
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+      this.waiters.push(resolve);
+    });
+  }
+  /**
+   * Releases a permit. Hands it directly to the next waiter if any are queued,
+   * otherwise returns it to the pool.
+   */
+  release() {
+    const next = this.waiters.shift();
+    if (next) {
+      next();
+    } else {
+      this.permits++;
+    }
+  }
+  /** Permits currently available. Exposed for tests and introspection. */
+  get available() {
+    return this.permits;
+  }
+  /** Number of callers currently queued waiting for a permit. */
+  get waiting() {
+    return this.waiters.length;
   }
-  return (0, import_pino.default)({ name: "rabbitmq-client" });
 };
 
 // src/client.ts
@@ -70,6 +129,11 @@ var RabbitMQClient = class {
   reconnectionPromise = null;
   assertedExchanges = /* @__PURE__ */ new Set();
   consumerTags = /* @__PURE__ */ new Map();
+  // Per-queue concurrency limiter, keyed by queue name. Persisted across
+  // reconnects (unlike consumerTags) so handlers still running when a
+  // connection drops keep holding their permits, and the new consumer cannot
+  // exceed the ceiling while the old pipeline drains. `null` = no limit.
+  subscriptionSemaphores = /* @__PURE__ */ new Map();
   inflightCount = 0;
   drainResolve = null;
   constructor(options) {
@@ -81,9 +145,14 @@ var RabbitMQClient = class {
       initMaxAttempts: options.initMaxAttempts ?? DEFAULTS.initMaxAttempts,
       publishMaxAttempts: options.publishMaxAttempts ?? DEFAULTS.publishMaxAttempts,
       prefetch: options.prefetch ?? DEFAULTS.prefetch,
+      // Default concurrency tracks prefetch, preserving the previous behaviour
+      // (handlers ran fire-and-forget, so up to `prefetch` ran at once) while
+      // making the ceiling explicit and independently tunable. A value <= 0
+      // (including an unlimited `prefetch: 0`) means "no concurrency limit".
+      concurrency: options.concurrency ?? options.prefetch ?? DEFAULTS.prefetch,
       closeTimeout: options.closeTimeout ?? DEFAULTS.closeTimeout
     };
-    this.logger = createLogger(options.logger);
+    this.logger = options.logger ?? defaultLogger;
     this.hooks = options.hooks ?? {};
   }
   /**
@@ -117,10 +186,10 @@ var RabbitMQClient = class {
         this.channel = await this.connection.createConfirmChannel();
         this.logger.info("Confirm channel created");
         await this.channel.prefetch(this.options.prefetch);
-        this.logger.info({ prefetch: this.options.prefetch }, "Channel prefetch set");
+        this.logger.info("Channel prefetch set", { prefetch: this.options.prefetch });
         this.connection.on("error", (error) => {
           this.connected = false;
-          this.logger.error({ error }, "Connection error");
+          this.logger.error("Connection error", { error });
         });
         this.connection.on("close", () => {
           this.connected = false;
@@ -128,7 +197,7 @@ var RabbitMQClient = class {
           this.reconnectWithRetry();
         });
         this.channel.on("error", (error) => {
-          this.logger.error({ error }, "Channel error");
+          this.logger.error("Channel error", { error });
           this.handleChannelFailure();
         });
         this.channel.on("close", () => {
@@ -141,20 +210,17 @@ var RabbitMQClient = class {
         attempts++;
         this.connected = false;
         if (maxAttempts !== void 0 && attempts >= maxAttempts) {
-          this.logger.error({ error, attempts, maxAttempts }, "Connection failed after maximum attempts");
+          this.logger.error("Connection failed after maximum attempts", { error, attempts, maxAttempts });
           throw new Error(
             `Failed to connect to RabbitMQ after ${attempts} attempts. Check RABBITMQ_URL configuration and RabbitMQ server availability.`
           );
         }
-        this.logger.warn(
-          {
-            error,
-            attempt: attempts,
-            maxAttempts: maxAttempts ?? "unlimited",
-            retryDelayMs: this.options.connectionRetryDelay
-          },
-          "Connection attempt failed, retrying..."
-        );
+        this.logger.warn("Connection attempt failed, retrying...", {
+          error,
+          attempt: attempts,
+          maxAttempts: maxAttempts ?? "unlimited",
+          retryDelayMs: this.options.connectionRetryDelay
+        });
         await this.sleep(this.options.connectionRetryDelay);
       }
     }
@@ -202,21 +268,18 @@ var RabbitMQClient = class {
         });
         await this.channel.waitForConfirms();
         if (attempts > 0) {
-          this.logger.info({ exchange, routingKey, messageSize: content.length, attempts }, "Published message after retries");
+          this.logger.info("Published message after retries", { exchange, routingKey, messageSize: content.length, attempts });
         } else {
-          this.logger.info({ exchange, routingKey, messageSize: content.length }, "Published message");
+          this.logger.info("Published message", { exchange, routingKey, messageSize: content.length });
         }
-        this.logger.debug({ exchange, routingKey, payload: message }, "Published message payload");
+        this.logger.debug("Published message payload", { exchange, routingKey, payload: message });
         this.callHook(this.hooks.onPublish, { exchange, routingKey, attempts: attempts + 1 });
         return;
       } catch (error) {
         attempts++;
         this.connected = false;
         if (attempts >= maxAttempts) {
-          this.logger.error(
-            { error, exchange, routingKey, attempts, maxAttempts },
-            "Publish failed after max attempts"
-          );
+          this.logger.error("Publish failed after max attempts", { error, exchange, routingKey, attempts, maxAttempts });
           throw new Error(
             `Failed to publish to ${exchange}/${routingKey} after ${attempts} attempts: ${error instanceof Error ? error.message : String(error)}`
           );
@@ -225,10 +288,7 @@ var RabbitMQClient = class {
           baseDelay * Math.pow(2, attempts - 1),
           MAX_PUBLISH_RETRY_DELAY_MS
         );
-        this.logger.warn(
-          { error, exchange, routingKey, attempt: attempts, maxAttempts, retryDelayMs: retryDelay },
-          "Publish attempt failed, retrying"
-        );
+        this.logger.warn("Publish attempt failed, retrying", { error, exchange, routingKey, attempt: attempts, maxAttempts, retryDelayMs: retryDelay });
         await this.sleep(retryDelay);
       }
     }
@@ -242,7 +302,7 @@ var RabbitMQClient = class {
   async close(clearSubscriptions = true) {
     try {
       if (this.inflightCount > 0) {
-        this.logger.info({ inflightCount: this.inflightCount }, "Waiting for in-flight messages to drain");
+        this.logger.info("Waiting for in-flight messages to drain", { inflightCount: this.inflightCount });
         await this.waitForDrain(this.options.closeTimeout);
       }
       await this.channel?.close();
@@ -256,10 +316,11 @@ var RabbitMQClient = class {
       this.consumerTags.clear();
       if (clearSubscriptions) {
         this.subscriptions = [];
+        this.subscriptionSemaphores.clear();
       }
       this.logger.info("Connection closed");
     } catch (error) {
-      this.logger.error({ error }, "Error closing connection");
+      this.logger.error("Error closing connection", { error });
       throw error;
     }
   }
@@ -285,7 +346,7 @@ var RabbitMQClient = class {
     if (this.subscriptions.length === 0) {
       return;
     }
-    this.logger.info({ count: this.subscriptions.length }, "Re-establishing subscriptions");
+    this.logger.info("Re-establishing subscriptions", { count: this.subscriptions.length });
     const subs = [...this.subscriptions];
     const results = await Promise.allSettled(
       subs.map(async (sub) => {
@@ -301,14 +362,14 @@ var RabbitMQClient = class {
       } else {
         const queue = subs[index].queue;
         failed.push(queue);
-        this.logger.error({ error: result.reason, queue }, "Failed to re-subscribe to queue");
+        this.logger.error("Failed to re-subscribe to queue", { error: result.reason, queue });
       }
     });
     if (succeeded.length > 0) {
-      this.logger.info({ count: succeeded.length, queues: succeeded }, "Successfully re-subscribed to queues");
+      this.logger.info("Successfully re-subscribed to queues", { count: succeeded.length, queues: succeeded });
     }
     if (failed.length > 0) {
-      this.logger.warn({ count: failed.length, queues: failed }, "Some subscriptions failed to restore");
+      this.logger.warn("Some subscriptions failed to restore", { count: failed.length, queues: failed });
     }
     this.callHook(this.hooks.onReconnect, { subscriptionsRestored: succeeded.length, subscriptionsFailed: failed.length });
   }
@@ -328,6 +389,7 @@ var RabbitMQClient = class {
       this.subscriptions.push(sub);
     } else {
       this.subscriptions[existingIndex] = sub;
+      this.subscriptionSemaphores.delete(queue);
     }
     await this.setupSubscription(sub);
   }
@@ -340,22 +402,24 @@ var RabbitMQClient = class {
       await this.channel.cancel(tag);
     }
     this.consumerTags.delete(queue);
+    this.subscriptionSemaphores.delete(queue);
     this.subscriptions = this.subscriptions.filter((s) => s.queue !== queue);
-    this.logger.info({ queue }, "Unsubscribed from queue");
+    this.logger.info("Unsubscribed from queue", { queue });
   }
   async setupSubscription(sub) {
     if (!this.channel) {
       throw new Error("Channel not available");
     }
+    const channel = this.channel;
     const { exchange, routingKey, queue, handler, options } = sub;
     const dlxExchange = `${exchange}.dlx`;
     const dlqQueue = `${queue}.dlq`;
     const dlqRoutingKey = `${routingKey}.dead`;
-    await this.channel.assertExchange(dlxExchange, "topic", { durable: true });
-    await this.channel.assertQueue(dlqQueue, { durable: true });
-    await this.channel.bindQueue(dlqQueue, dlxExchange, dlqRoutingKey);
+    await channel.assertExchange(dlxExchange, "topic", { durable: true });
+    await channel.assertQueue(dlqQueue, { durable: true });
+    await channel.bindQueue(dlqQueue, dlxExchange, dlqRoutingKey);
     if (!this.assertedExchanges.has(exchange)) {
-      await this.channel.assertExchange(exchange, "topic", { durable: true });
+      await channel.assertExchange(exchange, "topic", { durable: true });
       this.assertedExchanges.add(exchange);
     }
     const queueType = options?.queueArguments?.["x-queue-type"] ?? "quorum";
@@ -366,101 +430,102 @@ var RabbitMQClient = class {
     if (queueType === "quorum") {
       queueArgs["x-dead-letter-strategy"] = "at-least-once";
     }
-    await this.channel.assertQueue(queue, {
+    await channel.assertQueue(queue, {
       durable: true,
       deadLetterExchange: dlxExchange,
       deadLetterRoutingKey: dlqRoutingKey,
       arguments: { ...queueArgs, ...options?.queueArguments }
     });
-    await this.channel.bindQueue(queue, exchange, routingKey);
-    const { consumerTag } = await this.channel.consume(
+    await channel.bindQueue(queue, exchange, routingKey);
+    if (!this.subscriptionSemaphores.has(queue)) {
+      const concurrency = options?.concurrency ?? this.options.concurrency;
+      this.subscriptionSemaphores.set(queue, concurrency > 0 ? new Semaphore(concurrency) : null);
+    }
+    const semaphore = this.subscriptionSemaphores.get(queue) ?? null;
+    const { consumerTag } = await channel.consume(
       queue,
       (message) => {
         if (message) {
-          this.handleWithRetry(message, handler).catch((error) => {
-            this.logger.error({ error }, "Unhandled error in message handler");
-          });
+          this.dispatch(message, handler, semaphore, channel);
         }
       },
       { noAck: false }
     );
     this.consumerTags.set(queue, consumerTag);
-    this.logger.info({ exchange, routingKey, queue }, "Subscribed to queue");
+    this.logger.info("Subscribed to queue", { exchange, routingKey, queue });
   }
-  async handleWithRetry(message, handler) {
-    const channel = this.channel;
-    if (!channel) {
-      this.logger.warn("Cannot process message -- channel unavailable");
+  /**
+   * Gates a delivered message on the subscription's concurrency semaphore, then
+   * processes it. A message counts as in-flight from delivery until its handler
+   * settles (including time spent waiting for a permit), so graceful `close()`
+   * drains queued messages as well as actively-processing ones.
+   */
+  dispatch(message, handler, semaphore, channel) {
+    this.inflightCount++;
+    const acquire = semaphore ? semaphore.acquire() : Promise.resolve();
+    acquire.then(() => this.handleWithRetry(message, handler, channel)).catch((error) => {
+      this.logger.error("Unhandled error in message handler", { error });
+    }).finally(() => {
+      semaphore?.release();
+      this.inflightCount--;
+      if (this.inflightCount === 0 && this.drainResolve) {
+        this.drainResolve();
+      }
+    });
+  }
+  async handleWithRetry(message, handler, channel) {
+    if (channel !== this.channel) {
+      this.logger.warn("Skipping message from a superseded channel; it will be redelivered");
       return;
     }
-    this.inflightCount++;
+    const startTime = Date.now();
+    let attempts = 0;
+    const routingKey = message.fields.routingKey;
+    const exchange = message.fields.exchange;
+    let content;
     try {
-      const startTime = Date.now();
-      let attempts = 0;
-      const routingKey = message.fields.routingKey;
-      const exchange = message.fields.exchange;
-      let content;
+      content = JSON.parse(message.content.toString());
+    } catch (parseError) {
+      const rawContent = message.content.toString();
+      const rawPreview = rawContent.substring(0, 100);
+      this.logger.error("Failed to parse message JSON, sending to DLQ", {
+        error: parseError,
+        exchange,
+        routingKey,
+        rawContentPreview: rawPreview + (rawContent.length > 100 ? "..." : "")
+      });
+      channel.nack(message, false, false);
+      this.callHook(this.hooks.onMessageDlq, { exchange, routingKey, duration: 0, reason: "invalid_json" });
+      return;
+    }
+    this.logger.debug("Message received, processing", { exchange, routingKey, payload: content });
+    while (attempts < this.options.maxRetries) {
       try {
-        content = JSON.parse(message.content.toString());
-      } catch (parseError) {
-        const rawContent = message.content.toString();
-        const rawPreview = rawContent.substring(0, 100);
-        this.logger.error(
-          {
-            error: parseError,
-            exchange,
-            routingKey,
-            rawContentPreview: rawPreview + (rawContent.length > 100 ? "..." : "")
-          },
-          "Failed to parse message JSON, sending to DLQ"
-        );
-        channel.nack(message, false, false);
-        this.callHook(this.hooks.onMessageDlq, { exchange, routingKey, duration: 0, reason: "invalid_json" });
+        await handler(content);
+        const duration2 = Date.now() - startTime;
+        this.logger.info("Message processed successfully", { exchange, routingKey, duration: duration2, attempts: attempts + 1 });
+        channel.ack(message);
+        this.callHook(this.hooks.onMessageProcessed, { exchange, routingKey, duration: duration2, attempts: attempts + 1 });
         return;
-      }
-      this.logger.debug({ exchange, routingKey, payload: content }, "Message received, processing");
-      while (attempts < this.options.maxRetries) {
-        try {
-          await handler(content);
-          const duration2 = Date.now() - startTime;
-          this.logger.info(
-            { exchange, routingKey, duration: duration2, attempts: attempts + 1 },
-            "Message processed successfully"
-          );
-          channel.ack(message);
-          this.callHook(this.hooks.onMessageProcessed, { exchange, routingKey, duration: duration2, attempts: attempts + 1 });
-          return;
-        } catch (error) {
-          attempts++;
-          this.logger.error(
-            {
-              error: error instanceof Error ? error.message : error,
-              stack: error instanceof Error ? error.stack : void 0,
-              exchange,
-              routingKey,
-              attempt: attempts,
-              maxRetries: this.options.maxRetries
-            },
-            "Handler failed"
-          );
-          if (attempts < this.options.maxRetries) {
-            await this.sleep(this.options.retryDelay);
-          }
+      } catch (error) {
+        attempts++;
+        this.logger.error("Handler failed", {
+          error: error instanceof Error ? error.message : error,
+          stack: error instanceof Error ? error.stack : void 0,
+          exchange,
+          routingKey,
+          attempt: attempts,
+          maxRetries: this.options.maxRetries
+        });
+        if (attempts < this.options.maxRetries) {
+          await this.sleep(this.options.retryDelay);
         }
       }
-      const duration = Date.now() - startTime;
-      this.logger.error(
-        { exchange, routingKey, maxRetries: this.options.maxRetries, duration },
-        "Message failed after max retries, sending to DLQ"
-      );
-      channel.nack(message, false, false);
-      this.callHook(this.hooks.onMessageDlq, { exchange, routingKey, duration, reason: "max_retries_exhausted" });
-    } finally {
-      this.inflightCount--;
-      if (this.inflightCount === 0 && this.drainResolve) {
-        this.drainResolve();
-      }
     }
+    const duration = Date.now() - startTime;
+    this.logger.error("Message failed after max retries, sending to DLQ", { exchange, routingKey, maxRetries: this.options.maxRetries, duration });
+    channel.nack(message, false, false);
+    this.callHook(this.hooks.onMessageDlq, { exchange, routingKey, duration, reason: "max_retries_exhausted" });
   }
   /**
    * Lightweight liveness probe: creates and immediately deletes a temporary
@@ -478,7 +543,7 @@ var RabbitMQClient = class {
       await this.channel.deleteQueue(queue);
       return true;
     } catch (error) {
-      this.logger.warn({ error }, "Health check failed");
+      this.logger.warn("Health check failed", { error });
       return false;
     }
   }
@@ -486,10 +551,7 @@ var RabbitMQClient = class {
     if (this.inflightCount === 0) return Promise.resolve();
     return new Promise((resolve) => {
       const timer = setTimeout(() => {
-        this.logger.warn(
-          { inflightCount: this.inflightCount },
-          "Close timeout reached with messages still in flight"
-        );
+        this.logger.warn("Close timeout reached with messages still in flight", { inflightCount: this.inflightCount });
         this.drainResolve = null;
         resolve();
       }, timeout);
@@ -505,7 +567,7 @@ var RabbitMQClient = class {
     try {
       hook?.(info);
     } catch (error) {
-      this.logger.debug({ error }, "Observability hook error");
+      this.logger.debug("Observability hook error", { error });
     }
   }
   sleep(ms) {
@@ -514,6 +576,8 @@ var RabbitMQClient = class {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  RabbitMQClient
+  RabbitMQClient,
+  defaultLogger,
+  silentLogger
 });
 //# sourceMappingURL=index.cjs.map