@koala42/redis-highway 0.1.0

package/README.md

@@ -0,0 +1,63 @@
+# @koala42/redis-highway
+
+High performance Redis stream-based queue for Node.js.
+
+## Installation
+
+```bash
+npm install @koala42/redis-highway
+```
+
+## Usage
+
+### Producer
+
+```typescript
+import { Redis } from 'ioredis';
+import { Producer } from '@koala42/redis-highway';
+
+const redis = new Redis();
+const producer = new Producer(redis, 'my-stream');
+
+// Send job
+await producer.push(
+  JSON.stringify({ hello: 'world' }), 
+  ['group-A', 'group-B'] // Target specific consumer groups
+);
+```
+
+### Worker
+
+```typescript
+import { Redis } from 'ioredis';
+import { Worker } from '@koala42/redis-highway';
+
+class MyWorker extends Worker {
+  async process(data: any) {
+    console.log('Processing:', data);
+    // throw new Error('fail'); // Automatic retry/DLQ logic
+  }
+}
+
+const redis = new Redis();
+const worker = new MyWorker(redis, 'group-A', 'my-stream');
+
+await worker.start();
+```
+
+### Metrics
+
+```typescript
+import { Metrics } from '@koala42/redis-highway';
+
+const metrics = new Metrics(redis, 'my-stream');
+
+// Prometheus format
+const payload = await metrics.getPrometheusMetrics(['group-A']);
+```
+
+## Features
+
+- **Granular Retries**: If one group fails, only that group retries.
+- **DLQ**: Dead Letter Queue support after max retries.
+- **Metrics**: Throughput, Waiting, DLQ, Prometheus export.

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export * from './producer';
+export * from './worker';
+export * from './metrics';
+export * from './keys';
+export * from './interfaces';

package/dist/index.js

@@ -0,0 +1,21 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./producer"), exports);
+__exportStar(require("./worker"), exports);
+__exportStar(require("./metrics"), exports);
+__exportStar(require("./keys"), exports);
+__exportStar(require("./interfaces"), exports);

package/dist/interfaces.d.ts

@@ -0,0 +1,2 @@
+export type StreamMessage = [string, string[]];
+export type XReadGroupResponse = [string, StreamMessage[]][];

package/dist/interfaces.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/keys.d.ts

@@ -0,0 +1,10 @@
+export declare class KeyManager {
+    private readonly streamName;
+    constructor(streamName: string);
+    getStreamKey(): string;
+    getJobStatusKey(id: string): string;
+    getJobDataKey(id: string): string;
+    getDlqStreamKey(): string;
+    getThroughputKey(groupName: string, timestamp: number): string;
+    getTotalKey(groupName: string): string;
+}

package/dist/keys.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KeyManager = void 0;
+class KeyManager {
+    constructor(streamName) {
+        this.streamName = streamName;
+    }
+    getStreamKey() {
+        return this.streamName;
+    }
+    getJobStatusKey(id) {
+        return `${this.streamName}:status:${id}`;
+    }
+    getJobDataKey(id) {
+        return `${this.streamName}:data:${id}`;
+    }
+    getDlqStreamKey() {
+        return `${this.streamName}:dlq`;
+    }
+    getThroughputKey(groupName, timestamp) {
+        const minute = Math.floor(timestamp / 60000) * 60000;
+        return `metrics:throughput:${this.streamName}:${groupName}:${minute}`;
+    }
+    getTotalKey(groupName) {
+        return `metrics:total:${this.streamName}:${groupName}`;
+    }
+}
+exports.KeyManager = KeyManager;

package/dist/lua.d.ts

@@ -0,0 +1 @@
+export declare const LUA_MARK_DONE = "\n-- KEYS[1] = status key status key for jog\n-- KEYS[2] = data key for job\n-- KEYS[3] = stream key\n-- KEYS[4] = group name\n-- KEYS[5] = metrics key\n-- KEYS[6] = total metrics key(persistent)\n\n-- ARGV[1] = route name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId - redis stream item ID\n\n-- 1. Ack the stream message\nredis.call('XACK', KEYS[3], KEYS[4], ARGV[3])\n\n-- 2. in status key mark the current route as done by saving timestamp\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 3. Increment throughput metric\nif KEYS[5] then\n    redis.call('INCR', KEYS[5])\n    redis.call('EXPIRE', KEYS[5], 86400)\nend\n\n-- 3.1 Increment Total Metric\nif KEYS[6] then\n    redis.call('INCR', KEYS[6])\nend\n\n-- 4. Check for completed routes\nlocal current_fields = redis.call('HLEN', KEYS[1])\n\n-- 5. Get the target completed routes\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 6. If completed routes is status hash length - 1 -> all were done and we can cleanup\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1], KEYS[2])\n    return 1 -- Cleanup, DONE\nend\n\nreturn 0 -- Some routes are not done yet\n";

package/dist/lua.js

@@ -0,0 +1,51 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LUA_MARK_DONE = void 0;
+exports.LUA_MARK_DONE = `
+-- KEYS[1] = status key status key for jog
+-- KEYS[2] = data key for job
+-- KEYS[3] = stream key
+-- KEYS[4] = group name
+-- KEYS[5] = metrics key
+-- KEYS[6] = total metrics key(persistent)
+
+-- ARGV[1] = route name
+-- ARGV[2] = timestamp
+-- ARGV[3] = msgId - redis stream item ID
+
+-- 1. Ack the stream message
+redis.call('XACK', KEYS[3], KEYS[4], ARGV[3])
+
+-- 2. in status key mark the current route as done by saving timestamp
+redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+
+-- 3. Increment throughput metric
+if KEYS[5] then
+    redis.call('INCR', KEYS[5])
+    redis.call('EXPIRE', KEYS[5], 86400)
+end
+
+-- 3.1 Increment Total Metric
+if KEYS[6] then
+    redis.call('INCR', KEYS[6])
+end
+
+-- 4. Check for completed routes
+local current_fields = redis.call('HLEN', KEYS[1])
+
+-- 5. Get the target completed routes
+local target_str = redis.call('HGET', KEYS[1], '__target')
+local target = tonumber(target_str)
+
+if not target then
+    return 0
+end
+
+-- 6. If completed routes is status hash length - 1 -> all were done and we can cleanup
+if current_fields >= (target + 1) then
+    redis.call('DEL', KEYS[1], KEYS[2])
+    return 1 -- Cleanup, DONE
+end
+
+return 0 -- Some routes are not done yet
+`;

package/dist/metrics.d.ts

@@ -0,0 +1,24 @@
+import { Redis } from 'ioredis';
+export interface QueueMetrics {
+    streamLength: number;
+    dlqLength: number;
+    throughput: Record<string, number>;
+}
+export declare class Metrics {
+    private readonly redis;
+    private readonly streamName;
+    private keys;
+    constructor(redis: Redis, streamName: string);
+    /**
+     * Get current metrics for the queue
+     * @param groupNames - List of consumer groups to fetch throughput for
+     */
+    getMetrics(groupNames: string[]): Promise<QueueMetrics>;
+    /**
+     * Get prometheus compatible metrics
+     * @param groupNames target group names for throughput metrics
+     * @param prefix - export prefix
+     * @returns metrics as string
+     */
+    getPrometheusMetrics(groupNames: string[], prefix?: string): Promise<string>;
+}

package/dist/metrics.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Metrics = void 0;
+const keys_1 = require("./keys");
+class Metrics {
+    constructor(redis, streamName) {
+        this.redis = redis;
+        this.streamName = streamName;
+        this.keys = new keys_1.KeyManager(streamName);
+    }
+    /**
+     * Get current metrics for the queue
+     * @param groupNames - List of consumer groups to fetch throughput for
+     */
+    async getMetrics(groupNames) {
+        const pipeline = this.redis.pipeline();
+        // 1. Stream Length
+        pipeline.xlen(this.streamName);
+        // 2. DLQ Length
+        pipeline.xlen(this.keys.getDlqStreamKey());
+        // 3. Throughput keys for current minute
+        const timestamp = Date.now();
+        groupNames.forEach(group => {
+            pipeline.get(this.keys.getThroughputKey(group, timestamp));
+        });
+        const results = await pipeline.exec();
+        if (!results) {
+            throw new Error("Pipeline execution failed");
+        }
+        // Helper to safely extract result
+        const getResult = (index) => {
+            const [err, res] = results[index];
+            if (err)
+                throw err;
+            return res;
+        };
+        const streamLength = getResult(0);
+        const dlqLength = getResult(1);
+        const throughput = {};
+        groupNames.forEach((group, index) => {
+            // Offset by 2 because first two are xlen calls
+            const val = getResult(index + 2);
+            throughput[group] = parseInt(val || '0', 10);
+        });
+        return {
+            streamLength,
+            dlqLength,
+            throughput
+        };
+    }
+    /**
+     * Get prometheus compatible metrics
+     * @param groupNames target group names for throughput metrics
+     * @param prefix - export prefix
+     * @returns metrics as string
+     */
+    async getPrometheusMetrics(groupNames, prefix = 'redis_highway_queue') {
+        const pipeline = this.redis.pipeline();
+        pipeline.xlen(this.streamName);
+        pipeline.xlen(this.keys.getDlqStreamKey());
+        const timestamp = Date.now();
+        groupNames.forEach(group => {
+            pipeline.get(this.keys.getThroughputKey(group, timestamp));
+            pipeline.get(this.keys.getTotalKey(group));
+        });
+        const results = await pipeline.exec();
+        if (!results)
+            throw new Error("Pipeline execution failed");
+        const getResult = (index) => {
+            const [err, res] = results[index];
+            if (err)
+                throw err;
+            return res;
+        };
+        const streamLength = getResult(0);
+        const dlqLength = getResult(1);
+        const response = [];
+        response.push(`# HELP ${prefix}_waiting_jobs Total jobs waiting in stream`, `# TYPE ${prefix}_waiting_jobs gauge`, `${prefix}_waiting_jobs{stream="${this.streamName}"} ${streamLength}`, `# HELP ${prefix}_dlq_jobs Total jobs in DLQ`, `# TYPE ${prefix}_dlq_jobs gauge`, `${prefix}_dlq_jobs{stream="${this.streamName}"} ${dlqLength}`);
+        groupNames.forEach((group, index) => {
+            const baseIndex = 2 + (index * 2);
+            const throughputVal = parseInt(getResult(baseIndex) || '0', 10);
+            const totalVal = parseInt(getResult(baseIndex + 1) || '0', 10);
+            response.push(`# HELP ${prefix}_throughput_1m Jobs processed in the last minute`, `# TYPE ${prefix}_throughput_1m gauge`, `${prefix}_throughput_1m{stream="${this.streamName}", group="${group}"} ${throughputVal}`, `# HELP ${prefix}_jobs_total Total jobs processed`, `# TYPE ${prefix}_jobs_total counter`, `${prefix}_jobs_total{stream="${this.streamName}", group="${group}"} ${totalVal}`);
+        });
+        return response.join('\n');
+    }
+}
+exports.Metrics = Metrics;

package/dist/producer.d.ts

@@ -0,0 +1,19 @@
+import { Redis } from 'ioredis';
+export interface JobOptions {
+    ttl?: number | null;
+    streamName?: string;
+}
+export declare class Producer {
+    private readonly redis;
+    private readonly streamName;
+    private keys;
+    constructor(redis: Redis, streamName: string);
+    /**
+     * Push message to queue
+     * @param payload - serialized string payload
+     * @param targetGroups - target consumers
+     * @param opts - Job options
+     * @returns Created job ID (uuidv7)
+     */
+    push(payload: string, targetGroups: string[], opts?: JobOptions): Promise<string>;
+}

package/dist/producer.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Producer = void 0;
+const uuid_1 = require("uuid");
+const keys_1 = require("./keys");
+class Producer {
+    constructor(redis, streamName) {
+        this.redis = redis;
+        this.streamName = streamName;
+        this.keys = new keys_1.KeyManager(streamName);
+    }
+    /**
+     * Push message to queue
+     * @param payload - serialized string payload
+     * @param targetGroups - target consumers
+     * @param opts - Job options
+     * @returns Created job ID (uuidv7)
+     */
+    async push(payload, targetGroups, opts) {
+        const id = (0, uuid_1.v7)();
+        const ttl = opts?.ttl || null; // 24 hours in seconds
+        const pipeline = this.redis.pipeline();
+        const dataKey = this.keys.getJobDataKey(id);
+        const statusKey = this.keys.getJobStatusKey(id);
+        // Create job data
+        if (ttl) {
+            pipeline.set(dataKey, payload, 'EX', ttl);
+        }
+        else {
+            pipeline.set(dataKey, payload);
+        }
+        // Initialize job metadata - status
+        pipeline.hset(statusKey, '__target', targetGroups.length);
+        if (ttl) {
+            pipeline.expire(statusKey, ttl);
+        }
+        // Push message to stream
+        pipeline.xadd(this.streamName, '*', 'id', id, 'target', targetGroups.join(','));
+        await pipeline.exec();
+        return id;
+    }
+}
+exports.Producer = Producer;

package/dist/worker.d.ts

@@ -0,0 +1,27 @@
+import Redis from "ioredis";
+export declare abstract class Worker<T = any> {
+    protected redis: Redis;
+    protected groupName: string;
+    protected streamName: string;
+    protected concurrency: number;
+    protected blockTimeMs: number;
+    private isRunning;
+    private activeCount;
+    private readonly events;
+    private keys;
+    private readonly MAX_RETRIES;
+    constructor(redis: Redis, groupName: string, streamName: string, concurrency?: number, blockTimeMs?: number);
+    /**
+     * Start worker
+     * @returns
+     */
+    start(): Promise<void>;
+    stop(): void;
+    private fetchLoop;
+    private spawnWorker;
+    private processInternal;
+    private handleFailure;
+    private finalize;
+    private consumerName;
+    abstract process(data: T): Promise<void>;
+}

package/dist/worker.js

@@ -0,0 +1,145 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Worker = void 0;
+const events_1 = require("events");
+const lua_1 = require("./lua");
+const keys_1 = require("./keys");
+class Worker {
+    constructor(redis, groupName, streamName, concurrency = 1, blockTimeMs = 2000) {
+        this.redis = redis;
+        this.groupName = groupName;
+        this.streamName = streamName;
+        this.concurrency = concurrency;
+        this.blockTimeMs = blockTimeMs;
+        this.isRunning = false;
+        this.activeCount = 0;
+        this.events = new events_1.EventEmitter();
+        this.MAX_RETRIES = 3;
+        this.events.setMaxListeners(100);
+        this.keys = new keys_1.KeyManager(streamName);
+    }
+    /**
+     * Start worker
+     * @returns
+     */
+    async start() {
+        if (this.isRunning) {
+            return;
+        }
+        this.isRunning = true;
+        try {
+            await this.redis.xgroup('CREATE', this.streamName, this.groupName, '0', 'MKSTREAM');
+        }
+        catch (e) {
+            if (!e.message.includes('BUSYGROUP')) {
+                throw e;
+            }
+        }
+        this.fetchLoop();
+    }
+    stop() {
+        this.isRunning = false;
+    }
+    async fetchLoop() {
+        while (this.isRunning) {
+            const freeSlots = this.concurrency - this.activeCount;
+            if (freeSlots <= 0) {
+                await new Promise((resolve) => this.events.once('job_finished', resolve));
+                continue;
+            }
+            try {
+                const results = await this.redis.xreadgroup('GROUP', this.groupName, this.consumerName(), 'COUNT', freeSlots, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
+                if (results) {
+                    const messages = results[0][1];
+                    for (const msg of messages) {
+                        this.spawnWorker(msg);
+                    }
+                }
+            }
+            catch (err) {
+                console.error(`[${this.groupName}] Fetch Error:`, err);
+                await new Promise((resolve) => setTimeout(resolve, 1000));
+            }
+        }
+    }
+    spawnWorker(msg) {
+        this.activeCount++;
+        this.processInternal(msg).finally(() => {
+            this.activeCount--;
+            this.events.emit('job_finished');
+        });
+    }
+    async processInternal(msg) {
+        const msgId = msg[0];
+        const messageFields = msg[1];
+        const fields = {};
+        for (let i = 0; i < messageFields.length; i += 2) {
+            fields[messageFields[i]] = messageFields[i + 1];
+        }
+        const messageUuid = fields['id'];
+        const routes = fields['target'];
+        const retryCount = parseInt(fields['retryCount'] || '0', 10);
+        if (!routes.includes(this.groupName)) {
+            await this.redis.xack(this.streamName, this.groupName, msgId);
+            return;
+        }
+        try {
+            const dataKey = this.keys.getJobDataKey(messageUuid);
+            const payload = await this.redis.get(dataKey);
+            if (!payload) {
+                // Data missing or expired
+                await this.finalize(messageUuid, msgId);
+                return;
+            }
+            await this.process(JSON.parse(payload));
+            await this.finalize(messageUuid, msgId);
+        }
+        catch (err) {
+            console.error(`[${this.groupName}] Job failed ${messageUuid}`, err);
+            await this.handleFailure(messageUuid, msgId, retryCount, err.message);
+        }
+    }
+    async handleFailure(uuid, msgId, currentRetries, errorMsg) {
+        // 1. ACK the failed message (we are done with THIS stream entry)
+        await this.redis.xack(this.streamName, this.groupName, msgId);
+        if (currentRetries < this.MAX_RETRIES) {
+            // 2a. RETRY: Re-queue to main stream targeting ONLY this group
+            console.log(`[${this.groupName}] Retrying job ${uuid} (Attempt ${currentRetries + 1}/${this.MAX_RETRIES})`);
+            const pipeline = this.redis.pipeline();
+            // Refresh TTL to ensure data persists through retries (e.g., +1 hour)
+            pipeline.expire(this.keys.getJobDataKey(uuid), 3600);
+            pipeline.expire(this.keys.getJobStatusKey(uuid), 3600);
+            pipeline.xadd(this.streamName, '*', 'id', uuid, 'target', this.groupName, // Only target THIS group
+            'retryCount', currentRetries + 1);
+            await pipeline.exec();
+        }
+        else {
+            // 2b. DEAD LETTER QUEUE (DLQ)
+            console.error(`[${this.groupName}] Job ${uuid} exhausted retries. Moving to DLQ.`);
+            const dlqStream = `${this.streamName}:dlq`;
+            // We need the payload to store in DLQ permanently?
+            // Or just reference the key? Creating a self-contained DLQ entry is safer.
+            const payload = await this.redis.get(this.keys.getJobDataKey(uuid));
+            await this.redis.xadd(dlqStream, '*', 'id', uuid, 'group', this.groupName, 'error', errorMsg, 'payload', payload || 'MISSING', 'failedAt', Date.now());
+            // We still run finalize to clean up atomic counters/status
+            // (Treat it as "done" for the main flow, even though it failed)
+            await this.finalize(uuid, msgId, true); // true = skip Lua DEL? No, finalize deletes keys if everyone is done.
+            // Actually, if we move to DLQ, we should probably let finalize clean up the hot keys.
+            // The DLQ entry contains the payload copy, so losing the hot key is fine.
+        }
+    }
+    async finalize(messageUuid, msgId, fromError = false) {
+        // If we call finalize from error handler, we already ACKed inside handleFailure.
+        // But the Lua script does ACK too. It is idempotent so it's fine.
+        const timestamp = Date.now();
+        const statusKey = this.keys.getJobStatusKey(messageUuid);
+        const dataKey = this.keys.getJobDataKey(messageUuid);
+        const throughputKey = this.keys.getThroughputKey(this.groupName, timestamp);
+        const totalKey = this.keys.getTotalKey(this.groupName);
+        await this.redis.eval(lua_1.LUA_MARK_DONE, 6, statusKey, dataKey, this.streamName, this.groupName, throughputKey, totalKey, this.groupName, timestamp, msgId);
+    }
+    consumerName() {
+        return `${this.groupName}-${process.pid}`;
+    }
+}
+exports.Worker = Worker;

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@koala42/redis-highway",
+  "version": "0.1.0",
+  "description": "High performance redis queue",
+  "license": "MIT",
+  "author": {
+    "email": "stranavadavid@protonmail.com",
+    "name": "David Stranava",
+    "url": "https://github.com/stranavad"
+  },
+  "type": "commonjs",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "clean": "rimraf dist",
+    "build": "npm run clean && tsc",
+    "prepublish": "npm run build"
+  },
+  "keywords": [
+    "redis",
+    "queue",
+    "valkey",
+    "high-throughput"
+  ],
+  "dependencies": {
+    "ioredis": "^5.9.1",
+    "uuid": "^13.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.0.6",
+    "rimraf": "^6.1.2",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.16"
+  }
+}