@mayurbhusare/resilient-queue 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mayur Bhusare
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,109 @@
+# @mayurbhusare/resilient-queue
+
+Minimal resilient Redis-backed job queue for Node.js.
+
+Provides exponential retry, dead-letter queue (DLQ) handling, idempotency guarantees, and graceful shutdown — without heavy frameworks.
+
+---
+
+## ✨ Features
+
+- Redis-backed job queue
+- Exponential backoff retry
+- Dead Letter Queue (DLQ) support
+- Idempotency protection
+- Graceful shutdown support
+- Lightweight and minimal design
+- Fully tested with Jest
+
+---
+
+## 📦 Installation
+
+```bash
+npm install @mayurbhusare/resilient-queue
+```
+
+## 🚀 Quick Example
+```bash
+
+const {
+  ResilientQueue,
+  RetryableError,
+  FatalError
+} = require("@mayurbhusare/resilient-queue");
+
+const queue = new ResilientQueue({
+  redisUrl: "redis://127.0.0.1:6379",
+  maxRetries: 2,
+  baseDelay: 500
+});
+
+queue.process(async (job) => {
+  console.log("Processing:", job.data);
+
+  if (job.data.retry) {
+    throw new RetryableError("Temporary failure");
+  }
+
+  if (job.data.fatal) {
+    throw new FatalError("Permanent failure");
+  }
+
+  console.log("Completed");
+});
+
+queue.enqueue({ message: "Hello World" });
+
+```
+## 🧠 How It Works
+Main Queue
+
+Jobs are pushed into:
+```bash
+rq:main
+```
+Workers consume jobs using blocking Redis BLPOP.
+
+## Retry Strategy
+
+Retryable errors trigger exponential backoff:
+```bash
+delay = baseDelay * 2^attempt
+```
+
+After exceeding maxRetries, the job is moved to the DLQ.
+
+## Dead Letter Queue (DLQ)
+
+Failed jobs are stored in:
+```bash
+rq:dlq
+```
+
+### DLQ entries contain:
+
+- failure reason
+- attempt count
+- timestamps
+
+## Idempotency
+
+If an `idempotencyKey` is provided:
+
+- First execution succeeds
+- Duplicate submissions are ignored
+- Retries are not blocked
+
+## Graceful Shutdown
+```bash
+await queue.close();
+```
+Safely stops worker and closes Redis connections.
+
+## 🎯 Use Cases
+- Email processing
+- Webhook consumers
+- Payment confirmation retries
+- Background jobs
+- Distributed microservices tasks

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@mayurbhusare/resilient-queue",
+  "version": "1.0.0",
+  "description": "A lightweight Redis-backed job queue with exponential retry, dead-letter queue support, and idempotency guarantees.",
+  "main": "src/index.js",
+  "type": "commonjs",
+  "license": "MIT",
+  "keywords": [
+    "redis",
+    "queue",
+    "job-queue",
+    "retry",
+    "dead-letter-queue",
+    "dlq",
+    "idempotency",
+    "distributed-systems",
+    "nodejs",
+    "backend"
+  ],
+  "author": "Mayur Bhusare",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mayurbhusare/resilient-queue.git"
+  },
+  "bugs": {
+    "url": "https://github.com/mayurbhusare/resilient-queue/issues"
+  },
+  "homepage": "https://github.com/mayurbhusare/resilient-queue#readme",
+  "scripts": {
+    "test": "jest --runInBand"
+  },
+  "dependencies": {
+    "ioredis": "^5.9.3",
+    "uuid": "^8.3.2"
+  },
+  "devDependencies": {
+    "jest": "^30.2.0"
+  }
+}

package/src/core/ResilientQueue.js

@@ -0,0 +1,157 @@
+const { v4: uuidv4 } = require("uuid");
+
+const RedisClient = require("../redis/RedisClient");
+const IdempotencyManager = require("../managers/IdempotencyManager");
+const DLQManager = require("../managers/DLQManager");
+const RetryManager = require("../managers/RetryManager");
+
+/**
+ * Main queue class.
+ */
+class ResilientQueue {
+  constructor(options = {}) {
+    const {
+      redisUrl,
+      maxRetries = 3,
+      baseDelay = 500,
+      keyPrefix = "rq"
+    } = options;
+
+    if (!redisUrl) {
+      throw new Error("redisUrl is required to initialize ResilientQueue");
+    }
+
+    this.keyPrefix = keyPrefix;
+    this.mainQueueKey = `${keyPrefix}:main`;
+    this.isRunning = false;
+
+    const redisClientWrapper = new RedisClient(redisUrl);
+
+    this.producer = redisClientWrapper.getProducer();
+    this.consumer = redisClientWrapper.getConsumer();
+
+    this.idempotencyManager = new IdempotencyManager(
+      this.producer,
+      keyPrefix
+    );
+
+    this.dlqManager = new DLQManager(
+      this.producer,
+      keyPrefix
+    );
+
+    this.retryManager = new RetryManager(
+      this.producer,
+      this.dlqManager,
+      { maxRetries, baseDelay, keyPrefix }
+    );
+  }
+
+  /**
+   * Enqueue a new job.
+   */
+  async enqueue(data, options = {}) {
+    const { idempotencyKey, ttlSeconds } = options;
+
+    const job = {
+      id: uuidv4(),
+      data,
+      attempt: 0,
+      idempotencyKey: idempotencyKey || null,
+      ttlSeconds: ttlSeconds || null,
+      createdAt: Date.now()
+    };
+
+    await this.producer.rpush(
+      this.mainQueueKey,
+      JSON.stringify(job)
+    );
+
+    return job.id;
+  }
+
+  /**
+   * Start processing jobs.
+   */
+  async process(handler) {
+    if (typeof handler !== "function") {
+      throw new Error("A handler function must be provided to process()");
+    }
+
+    this.isRunning = true;
+
+    while (this.isRunning) {
+      try {
+        // IMPORTANT: timeout = 1 (not 0)
+        const result = await this.consumer.blpop(
+          this.mainQueueKey,
+          1
+        );
+
+        if (!result || result.length < 2) {
+          continue;
+        }
+
+        const rawJob = result[1];
+        let job;
+
+        try {
+          job = JSON.parse(rawJob);
+        } catch {
+          continue;
+        }
+
+        let shouldProceed = true;
+
+        // Apply idempotency only on first attempt
+        if (job.attempt === 0) {
+          shouldProceed =
+            await this.idempotencyManager.shouldProcess(
+              job.idempotencyKey,
+              job.ttlSeconds
+            );
+        }
+
+        if (!shouldProceed) {
+          continue;
+        }
+
+        try {
+          await handler(job);
+        } catch (error) {
+          // Prevent retry during shutdown
+          if (this.isRunning) {
+            await this.retryManager.handleFailure(job, error);
+          }
+        }
+
+      } catch {
+        // Silent safety net
+      }
+    }
+  }
+
+  /**
+   * Stop the worker loop.
+   */
+  async stop() {
+    this.isRunning = false;
+  }
+
+  /**
+   * Stop worker and close Redis connections safely.
+   */
+  async close() {
+    this.isRunning = false;
+
+    // Wait for BLPOP (timeout 1s) to exit naturally
+    await new Promise(resolve => setTimeout(resolve, 1100));
+
+    await Promise.all([
+      this.producer.quit(),
+      this.consumer.quit()
+    ]);
+  }
+}
+
+module.exports = ResilientQueue;

package/src/errors/Errors.js

@@ -0,0 +1,26 @@
+/**
+ * Error used to indicate a job can be retried.
+ */
+class RetryableError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "RetryableError";
+    Error.captureStackTrace(this, this.constructor);
+  }
+}
+
+/**
+ * Error used to indicate a job should not be retried.
+ */
+class FatalError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "FatalError";
+    Error.captureStackTrace(this, this.constructor);
+  }
+}
+
+module.exports = {
+  RetryableError,
+  FatalError
+};

package/src/index.js

@@ -0,0 +1,8 @@
+const ResilientQueue = require("./core/ResilientQueue");
+const { RetryableError, FatalError } = require("./errors/Errors");
+
+module.exports = {
+  ResilientQueue,
+  RetryableError,
+  FatalError
+};

package/src/managers/DLQManager.js

@@ -0,0 +1,30 @@
+/**
+ * Handles Dead Letter Queue (DLQ) operations.
+ */
+class DLQManager {
+  constructor(redisClient, keyPrefix = "rq") {
+    if (!redisClient) {
+      throw new Error("Redis client is required for DLQManager");
+    }
+
+    this.redis = redisClient;
+    this.keyPrefix = keyPrefix;
+  }
+
+  /**
+   * Moves a failed job to DLQ.
+   */
+  async moveToDLQ(job, reason) {
+    const dlqKey = `${this.keyPrefix}:dlq`;
+
+    const failedJob = {
+      ...job,
+      failedAt: Date.now(),
+      failureReason: reason?.message || "Unknown error"
+    };
+
+    await this.redis.rpush(dlqKey, JSON.stringify(failedJob));
+  }
+}
+
+module.exports = DLQManager;

package/src/managers/IdempotencyManager.js

@@ -0,0 +1,47 @@
+/**
+ * Handles idempotency checks using Redis.
+ */
+class IdempotencyManager {
+  constructor(redisClient, keyPrefix = "rq") {
+    if (!redisClient) {
+      throw new Error("Redis client is required for IdempotencyManager");
+    }
+
+    this.redis = redisClient;
+    this.keyPrefix = keyPrefix;
+  }
+
+  /**
+   * Returns true if job should proceed.
+   * Returns false if duplicate.
+   */
+  async shouldProcess(idempotencyKey, ttlSeconds = null) {
+    if (!idempotencyKey) {
+      return true;
+    }
+
+    const redisKey = `${this.keyPrefix}:idempotency:${idempotencyKey}`;
+
+    let result;
+
+    if (ttlSeconds) {
+      result = await this.redis.set(
+        redisKey,
+        "1",
+        "EX",
+        ttlSeconds,
+        "NX"
+      );
+    } else {
+      result = await this.redis.set(
+        redisKey,
+        "1",
+        "NX"
+      );
+    }
+
+    return result === "OK";
+  }
+}
+
+module.exports = IdempotencyManager;

package/src/managers/RetryManager.js

@@ -0,0 +1,68 @@
+const { RetryableError } = require("../errors/Errors");
+
+/**
+ * Handles retry logic with exponential backoff.
+ */
+class RetryManager {
+  constructor(redisClient, dlqManager, options = {}) {
+    if (!redisClient) {
+      throw new Error("Redis client is required for RetryManager");
+    }
+
+    if (!dlqManager) {
+      throw new Error("DLQManager is required for RetryManager");
+    }
+
+    this.redis = redisClient;
+    this.dlqManager = dlqManager;
+
+    this.maxRetries = options.maxRetries ?? 3;
+    this.baseDelay = options.baseDelay ?? 500;
+    this.keyPrefix = options.keyPrefix ?? "rq";
+  }
+
+  /**
+   * Handle job failure and decide whether to retry or send to DLQ.
+   */
+  async handleFailure(job, error) {
+    const mainQueueKey = `${this.keyPrefix}:main`;
+
+    // If error is not retryable → send to DLQ immediately
+    if (!(error instanceof RetryableError)) {
+      await this.dlqManager.moveToDLQ(job, error);
+      return;
+    }
+
+    const nextAttempt = (job.attempt || 0) + 1;
+
+    // If max retries exceeded → send to DLQ
+    if (nextAttempt > this.maxRetries) {
+      await this.dlqManager.moveToDLQ(
+        { ...job, attempt: nextAttempt },
+        error
+      );
+      return;
+    }
+
+    const delay = this.baseDelay * Math.pow(2, nextAttempt);
+
+    const retryJob = {
+      ...job,
+      attempt: nextAttempt,
+      lastError: error.message,
+      retriedAt: Date.now()
+    };
+
+    // Requeue job after exponential delay
+    setTimeout(() => {
+      this.redis
+        .rpush(mainQueueKey, JSON.stringify(retryJob))
+        .catch(err => {
+          // If requeue fails, move job to DLQ
+          this.dlqManager.moveToDLQ(retryJob, err);
+        });
+    }, delay);
+  }
+}
+
+module.exports = RetryManager;

package/src/redis/RedisClient.js

@@ -0,0 +1,67 @@
+const Redis = require("ioredis");
+
+/**
+ * RedisClient
+ *
+ * Creates separate Redis connections for:
+ *  - Producer (writes: RPUSH, SET, etc.)
+ *  - Consumer (blocking reads: BLPOP)
+ *
+ * This separation is critical because a connection
+ * blocked by BLPOP cannot execute other commands.
+ */
+class RedisClient {
+  constructor(redisUrl) {
+    if (!redisUrl) {
+      throw new Error("redisUrl is required to initialize RedisClient");
+    }
+
+    // Producer connection (writes)
+    this.producer = new Redis(redisUrl, {
+      maxRetriesPerRequest: null
+    });
+
+    // Consumer connection (blocking reads)
+    this.consumer = new Redis(redisUrl, {
+      maxRetriesPerRequest: null
+    });
+
+    this._attachEventHandlers();
+  }
+
+  /**
+   * Attach safe error handlers.
+   * Library should not log by default.
+   * Host application can manage Redis errors.
+   */
+  _attachEventHandlers() {
+    this.producer.on("error", () => {});
+    this.consumer.on("error", () => {});
+  }
+
+  /**
+   * Returns producer connection.
+   */
+  getProducer() {
+    return this.producer;
+  }
+
+  /**
+   * Returns consumer connection.
+   */
+  getConsumer() {
+    return this.consumer;
+  }
+
+  /**
+   * Gracefully close connections.
+   */
+  async disconnect() {
+    await Promise.all([
+      this.producer.quit(),
+      this.consumer.quit()
+    ]);
+  }
+}
+
+module.exports = RedisClient;