@koala42/redis-highway 0.1.2 → 0.1.4

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 KOALA42
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,6 +1,16 @@
 # @koala42/redis-highway
 
-High performance Redis stream-based queue for Node.js.
+High performance Redis stream-based queue for Node.js. Supports Redis single instances and Valkey single instances
+
+## Missing features ATM
+- Better gracefull shutdown handling
+- In worker process functions expose more than just data: T like job id and current status
+- Option to customize/enable/disable metrics
+- Enable custom logger for workers and producers instead of console.logs
+
+
+## Roadmap
+- Support redis cluster, that is probably possible only for job payloads and DLQ, since the stream is only one
 
 ## Installation
 
@@ -21,7 +31,7 @@ const producer = new Producer(redis, 'my-stream');
 
 // Send job
 await producer.push(
-  JSON.stringify({ hello: 'world' }), 
+  JSON.stringify({ hello: 'world' }), // Message serialization is not done automatically
   ['group-A', 'group-B'] // Target specific consumer groups
 );
 ```
@@ -32,8 +42,8 @@ await producer.push(
 import { Redis } from 'ioredis';
 import { Worker } from '@koala42/redis-highway';
 
-class MyWorker extends Worker {
-  async process(data: any) {
+class MyWorker extends Worker<T> {
+  async process(data: T) {
     console.log('Processing:', data);
     // throw new Error('fail'); // Automatic retry/DLQ logic
   }
@@ -56,8 +66,59 @@ const metrics = new Metrics(redis, 'my-stream');
 const payload = await metrics.getPrometheusMetrics(['group-A']);
 ```
 
-## Features
+## Usage with NestJS
+
+```typescript
+
+// Producer
+@Injectable()
+export class EntryService {
+  privater readonly producer: Producer;
+  
+  constructor(){
+    this.producer = new Producer(
+      new Redis(...), // Or reuse existing ioredis connection
+      'my-stream'
+    )
+  }
+  
+  public async sth(): Promise<void>{
+    await producer.push(
+      JSON.stringify({ hello: 'world' }), // Message serialization is not done automatically
+      ['group-A', 'group-B'] // Target specific consumer groups
+    );
+  }
+}
+
 
+// Processor
+@Injectable()
+export class ProcessorService extends Worker<T> implements OnModuleInit, OnModuleDestroy {
+  constructor(){
+    super(
+      new Redis(...), // or reuse existing redis conn
+      'group-A',
+      'my-stream',
+      50 // concurrency
+    )
+  }
+  
+  async onModuleInit(): Promise<void>{
+    await this.start()
+  }
+  
+  onModuleDestroy(){
+    this.stop()
+  }
+  
+  async process(data: T): Promise<void>{
+    console.log("Processing job", JSON.stringify(data))
+  }
+}
+````
+
+## Features
+- **Lightweight**: Uses light Lua scripts and pipelines wherever possible, making it highly concurrents for inserts and for processing as well, because of the reduced I/O load compared to BullMQ
 - **Granular Retries**: If one group fails, only that group retries.
 - **DLQ**: Dead Letter Queue support after max retries.
 - **Metrics**: Throughput, Waiting, DLQ, Prometheus export.

package/dist/lua.d.ts

@@ -1 +1 @@
-export declare const LUA_MARK_DONE = "\n-- KEYS[1] = status key status key for jog\n-- KEYS[2] = data key for job\n-- KEYS[3] = stream key\n-- KEYS[4] = group name\n-- KEYS[5] = metrics key\n-- KEYS[6] = total metrics key(persistent)\n\n-- ARGV[1] = route name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId - redis stream item ID\n\n-- 1. Ack the stream message\nredis.call('XACK', KEYS[3], KEYS[4], ARGV[3])\n\n-- 2. in status key mark the current route as done by saving timestamp\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 3. Increment throughput metric\nif KEYS[5] then\n    redis.call('INCR', KEYS[5])\n    redis.call('EXPIRE', KEYS[5], 86400)\nend\n\n-- 3.1 Increment Total Metric\nif KEYS[6] then\n    redis.call('INCR', KEYS[6])\nend\n\n-- 4. Check for completed routes\nlocal current_fields = redis.call('HLEN', KEYS[1])\n\n-- 5. Get the target completed routes\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 6. If completed routes is status hash length - 1 -> all were done and we can cleanup\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1], KEYS[2])\n    return 1 -- Cleanup, DONE\nend\n\nreturn 0 -- Some routes are not done yet\n";
+export declare const LUA_MARK_DONE = "\n-- KEYS[1] = status key status key for jog\n-- KEYS[2] = data key for job\n-- KEYS[3] = stream key\n-- KEYS[4] = group name\n-- KEYS[5] = metrics key\n-- KEYS[6] = total metrics key(persistent)\n\n-- ARGV[1] = route name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId - redis stream item ID\n\n-- 1. Ack the stream message\nredis.call('XACK', KEYS[3], KEYS[4], ARGV[3])\n\n-- 2. in status key mark the current route as done by saving timestamp\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 3. Increment throughput metric\nif KEYS[5] then\n    redis.call('INCR', KEYS[5])\n    redis.call('EXPIRE', KEYS[5], 86400)\nend\n\n-- 3.1 Increment Total Metric\nif KEYS[6] then\n    redis.call('INCR', KEYS[6])\nend\n\n-- 4. Check for completed routes\nlocal current_fields = redis.call('HLEN', KEYS[1])\n\n-- 5. Get the target completed routes\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 6. If completed routes is status hash length - 1 -> all were done and we can cleanup\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1], KEYS[2])\n    redis.call('XDEL', KEYS[3], ARGV[3])\n    return 1 -- Cleanup, DONE\nend\n\nreturn 0 -- Some routes are not done yet\n";

package/dist/lua.js

@@ -44,6 +44,7 @@ end
 -- 6. If completed routes is status hash length - 1 -> all were done and we can cleanup
 if current_fields >= (target + 1) then
     redis.call('DEL', KEYS[1], KEYS[2])
+    redis.call('XDEL', KEYS[3], ARGV[3])
     return 1 -- Cleanup, DONE
 end
 

package/dist/queue.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/queue.spec.js

@@ -0,0 +1,206 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const ioredis_1 = __importDefault(require("ioredis"));
+const producer_1 = require("./producer");
+const worker_1 = require("./worker");
+const metrics_1 = require("./metrics");
+const uuid_1 = require("uuid");
+const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
+class TestWorker extends worker_1.Worker {
+    constructor(redis, groupName, streamName, concurrency = 1, blockTimeMs = 100) {
+        super(redis, groupName, streamName, concurrency, blockTimeMs);
+        this.processedCount = 0;
+        this.lastProcessedId = null;
+        this.shouldFail = false;
+        this.failCount = 0;
+        this.maxFails = 0;
+    }
+    async process(data) {
+        if (this.shouldFail) {
+            this.failCount++;
+            if (this.maxFails > 0 && this.failCount > this.maxFails) {
+                // Stop failing after maxFails
+            }
+            else {
+                throw new Error("Simulated Failure");
+            }
+        }
+        this.processedCount++;
+        if (data && data.id) {
+            this.lastProcessedId = data.id;
+        }
+    }
+}
+(0, vitest_1.describe)('Redis Queue Integration', () => {
+    let redis;
+    let producer;
+    let streamName;
+    let workers = [];
+    (0, vitest_1.beforeEach)(() => {
+        redis = new ioredis_1.default(REDIS_URL);
+        streamName = `test-queue-${(0, uuid_1.v7)()}`;
+        producer = new producer_1.Producer(redis, streamName);
+        workers = [];
+    });
+    (0, vitest_1.afterEach)(async () => {
+        for (const w of workers) {
+            w.stop();
+        }
+        await new Promise(r => setTimeout(r, 500));
+        // Cleanup Redis keys using the existing connection before closing
+        if (redis.status === 'ready') {
+            const keys = await redis.keys(`${streamName}*`);
+            if (keys.length)
+                await redis.del(...keys);
+        }
+        redis.disconnect();
+    });
+    const waitFor = async (condition, timeout = 5000) => {
+        const start = Date.now();
+        while (Date.now() - start < timeout) {
+            if (await condition())
+                return true;
+            await new Promise(r => setTimeout(r, 100));
+        }
+        return false;
+    };
+    (0, vitest_1.describe)('Core Functionality', () => {
+        (0, vitest_1.it)('Should deliver message to all target groups', async () => {
+            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 100);
+            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 100);
+            workers.push(w1, w2);
+            await w1.start();
+            await w2.start();
+            const id = await producer.push(JSON.stringify({ id: 'msg-1' }), ['group-A', 'group-B']);
+            await waitFor(() => w1.processedCount === 1 && w2.processedCount === 1);
+            (0, vitest_1.expect)(w1.processedCount).toBe(1);
+            (0, vitest_1.expect)(w2.processedCount).toBe(1);
+            // Test cleanup keys (but not explicit XDEL here yet, checking keys gone)
+            const statusKey = `${streamName}:status:${id}`;
+            const dataKey = `${streamName}:data:${id}`;
+            (0, vitest_1.expect)(await redis.exists(statusKey)).toBe(0);
+            (0, vitest_1.expect)(await redis.exists(dataKey)).toBe(0);
+        });
+        (0, vitest_1.it)('Should only deliver to targeted groups', async () => {
+            const wA = new TestWorker(redis, 'group-A', streamName, 1, 100);
+            const wB = new TestWorker(redis, 'group-B', streamName, 1, 100);
+            workers.push(wA, wB);
+            await wA.start();
+            await wB.start();
+            await producer.push(JSON.stringify({ id: 'msg-only-a' }), ['group-A']);
+            await waitFor(() => wA.processedCount === 1);
+            (0, vitest_1.expect)(wA.processedCount).toBe(1);
+            (0, vitest_1.expect)(wB.processedCount).toBe(0);
+        });
+        (0, vitest_1.it)('Should retry only the failed group', async () => {
+            const wOk = new TestWorker(redis, 'group-Ok', streamName, 1, 100);
+            const wFail = new TestWorker(redis, 'group-Fail', streamName, 1, 100);
+            wFail.shouldFail = true;
+            wFail.maxFails = 1; // Fail once, then succeed
+            workers.push(wOk, wFail);
+            await wOk.start();
+            await wFail.start();
+            await producer.push(JSON.stringify({ id: 'retry-test' }), ['group-Ok', 'group-Fail']);
+            // Wait for wOk to finish and wFail to try at least twice (fail + success)
+            await waitFor(() => wOk.processedCount === 1 && wFail.processedCount === 1, 8000);
+            (0, vitest_1.expect)(wOk.processedCount).toBe(1); // Processed once
+            (0, vitest_1.expect)(wFail.failCount).toBeGreaterThanOrEqual(1); // Failed at least once
+            (0, vitest_1.expect)(wFail.processedCount).toBe(1); // Eventually succeeded
+            (0, vitest_1.expect)(wOk.processedCount).toBe(1); // wOk should NOT process the retry
+        });
+        (0, vitest_1.it)('Should move to DLQ after max retries', async () => {
+            const wDead = new TestWorker(redis, 'group-Dead', streamName, 1, 100);
+            wDead.shouldFail = true;
+            wDead.maxFails = 10; // Fail forever (more than max retries which is 3)
+            workers.push(wDead);
+            await wDead.start();
+            const id = await producer.push(JSON.stringify({ id: 'dlq-test' }), ['group-Dead']);
+            await waitFor(async () => {
+                const len = await redis.xlen(`${streamName}:dlq`);
+                return len > 0;
+            }, 10000);
+            const dlqLen = await redis.xlen(`${streamName}:dlq`);
+            (0, vitest_1.expect)(dlqLen).toBe(1);
+            (0, vitest_1.expect)(await redis.exists(`${streamName}:status:${id}`)).toBe(0);
+        });
+    });
+    (0, vitest_1.describe)('Metrics & Monitoring', () => {
+        (0, vitest_1.it)('Should track throughput and queue size', async () => {
+            const w = new TestWorker(redis, 'group-Metrics', streamName, 1, 100);
+            const metricsService = new metrics_1.Metrics(redis, streamName);
+            workers.push(w);
+            await w.start();
+            let metrics = await metricsService.getMetrics(['group-Metrics']);
+            (0, vitest_1.expect)(metrics.dlqLength).toBe(0);
+            const id = await producer.push(JSON.stringify({ id: 'metrics-1' }), ['group-Metrics']);
+            await waitFor(() => w.processedCount === 1);
+            metrics = await metricsService.getMetrics(['group-Metrics']);
+            (0, vitest_1.expect)(metrics.throughput['group-Metrics']).toBeGreaterThanOrEqual(1);
+            w.shouldFail = true;
+            w.maxFails = 10;
+            await producer.push(JSON.stringify({ id: 'metrics-fail' }), ['group-Metrics']);
+            await waitFor(() => redis.xlen(`${streamName}:dlq`).then(len => len > 0));
+            metrics = await metricsService.getMetrics(['group-Metrics']);
+            (0, vitest_1.expect)(metrics.dlqLength).toBe(1);
+        });
+        (0, vitest_1.it)('Should export Prometheus metrics', async () => {
+            const w = new TestWorker(redis, 'group-Prom', streamName, 1, 100);
+            const metricsService = new metrics_1.Metrics(redis, streamName);
+            workers.push(w);
+            await w.start();
+            await producer.push(JSON.stringify({ id: 'prom-1' }), ['group-Prom']);
+            await producer.push(JSON.stringify({ id: 'prom-2' }), ['group-Prom']);
+            await waitFor(() => w.processedCount === 2);
+            const promOutput = await metricsService.getPrometheusMetrics(['group-Prom']);
+            (0, vitest_1.expect)(promOutput).toContain(`# TYPE redis_highway_queue_throughput_1m gauge`);
+            (0, vitest_1.expect)(promOutput).toContain(`redis_highway_queue_throughput_1m{stream="${streamName}", group="group-Prom"} 2`);
+            (0, vitest_1.expect)(promOutput).toContain(`# TYPE redis_highway_queue_jobs_total counter`);
+            (0, vitest_1.expect)(promOutput).toContain(`redis_highway_queue_jobs_total{stream="${streamName}", group="group-Prom"} 2`);
+            (0, vitest_1.expect)(promOutput).toContain(`# TYPE redis_highway_queue_waiting_jobs gauge`);
+        });
+    });
+    (0, vitest_1.describe)('Stream Cleanup', () => {
+        (0, vitest_1.it)('Should delete message from stream after processing', async () => {
+            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 100);
+            workers.push(w1);
+            await w1.start();
+            const id = await producer.push(JSON.stringify({ id: 'msg-cleanup' }), ['group-A']);
+            // Wait for processing
+            await waitFor(() => w1.processedCount === 1);
+            (0, vitest_1.expect)(w1.processedCount).toBe(1);
+            // Wait for stream to be empty
+            const success = await waitFor(async () => {
+                const len = await redis.xlen(streamName);
+                return len === 0;
+            });
+            (0, vitest_1.expect)(success).toBe(true);
+            const messages = await redis.xrange(streamName, '-', '+');
+            (0, vitest_1.expect)(messages.length).toBe(0);
+        });
+        (0, vitest_1.it)('Should delete message from stream only after ALL groups processed it', async () => {
+            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 100);
+            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 100);
+            workers.push(w1, w2);
+            await w1.start(); // Only start w1
+            const id = await producer.push(JSON.stringify({ id: 'msg-multi' }), ['group-A', 'group-B']);
+            // Wait for w1 to process
+            await waitFor(() => w1.processedCount === 1);
+            let len = await redis.xlen(streamName);
+            (0, vitest_1.expect)(len).toBe(1); // Should still exist because group-B pending
+            // Start w2
+            await w2.start();
+            // Wait for w2 to process
+            await waitFor(() => w2.processedCount === 1);
+            // Wait for stream to be empty
+            const success = await waitFor(async () => {
+                len = await redis.xlen(streamName);
+                return len === 0;
+            });
+            (0, vitest_1.expect)(success).toBe(true);
+        });
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {
@@ -16,8 +16,10 @@
   ],
   "scripts": {
     "clean": "rimraf dist",
+    "test": "vitest run src/queue.spec.ts",
     "build": "npm run clean && tsc",
-    "prepublish": "npm run build"
+    "prepublish": "npm run build",
+    "publish": "npm run test && npm run clean && npm run build && npm publish --public"
   },
   "keywords": [
     "redis",