@koala42/redis-highway 0.2.12 → 0.2.13

package/README.md

@@ -92,6 +92,57 @@ const batchWorker = new MyBatchWorker(
 await batchWorker.start();
 ```
 
+### DLQ Worker
+Process messages from the Dead Letter Queue. Use this to handle jobs that have exhausted all retries.
+
+**Important:** DLQ Worker has no built-in error handling or retry policy. If `process()` throws an error, the message is lost. This is by design - DLQ processing is meant for manual intervention, logging, or forwarding to external systems.
+
+```typescript
+import { Redis } from 'ioredis';
+import { DlqWorker, DlqMessageEntity } from '@koala42/redis-highway';
+
+class MyDlqWorker extends DlqWorker<{hello: string}> {
+  async process(message: DlqMessageEntity<{hello: string}>) {
+    console.log('Failed job data:', message.data);
+    console.log('Original error:', message.errorMessage);
+    console.log('Failed at:', new Date(message.failedAt));
+    console.log('Original consumer group:', message.group);
+
+    // Example: Log to external system, send alert, or store for manual review
+    await externalLogger.log(message);
+  }
+}
+
+const redis = new Redis();
+const dlqWorker = new MyDlqWorker(redis, {
+  streamName: 'my-stream' // Must match your main worker's stream
+});
+
+await dlqWorker.start();
+
+// To stop gracefully
+// await dlqWorker.stop();
+```
+
+#### DLQ Worker Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `streamName` | string | - | **Required**. The Redis stream key (same as your main workers). |
+| `blockTimeoutMs` | number | 5000 | Redis XREADGROUP block duration in milliseconds. |
+| `waitTimeoutMs` | number | 5000 | Wait time between processing cycles when no messages are available. |
+
+#### DlqMessageEntity Properties
+
+| Property | Type | Description |
+|----------|------|-------------|
+| `data` | T | The original job payload. |
+| `errorMessage` | string | The error message from the last failed attempt. |
+| `failedAt` | number | Unix timestamp when the job was moved to DLQ. |
+| `group` | string | The consumer group that failed to process this job. |
+| `messageUuid` | string | The original job's unique identifier. |
+| `streamMessageId` | string | The Redis stream message ID. |
+
 ### Metrics
 
 ```typescript

package/dist/base-worker.js

@@ -6,6 +6,7 @@ const keys_1 = require("./keys");
 const uuid_1 = require("uuid");
 const stream_message_entity_1 = require("./stream-message-entity");
 const lua_1 = require("./lua");
+const dlq_message_entity_1 = require("./dlq-message-entity");
 class BaseWorker {
     constructor(redis, options, controlOptions, metricsOptions) {
         this.redis = redis;
@@ -44,8 +45,8 @@ class BaseWorker {
                 throw e;
             }
         }
-        this._fetchLoop();
-        this._autoClaimLoop();
+        this._fetchLoop().catch((e) => console.error("Fetch loop crashed", e));
+        this._autoClaimLoop().catch((e) => console.error('Auto claim loop crashed', e));
     }
     /**
      * Gracefully stops the worker
@@ -60,6 +61,7 @@ class BaseWorker {
         while (this._activeCount > 0) {
             await new Promise((resolve) => setTimeout(resolve, 50));
         }
+        await this.redis.xgroup('DELCONSUMER', this._streamName, this._groupName, this._consumerName).catch();
     }
     /**
      * Auto claim loop
@@ -116,7 +118,7 @@ class BaseWorker {
             if (message.retryCount < this._maxRetries) {
                 retryCountIncr++;
                 const newJobId = (0, uuid_1.v7)();
-                pipeline.xadd(this._streamName, '*', 'id', newJobId, 'target', this._groupName, 'retryCount', message.retryCount + 1, 'data', message.serializedData);
+                pipeline.xadd(this._streamName, '*', ...stream_message_entity_1.StreamMessageEntity.getStreamFields(newJobId, this._groupName, message.serializedData, message.retryCount + 1));
                 const newStatusKey = this._keys.getJobStatusKey(newJobId);
                 pipeline.hset(newStatusKey, '__target', 1);
                 const statusKey = this._keys.getJobStatusKey(message.messageUuid);
@@ -126,7 +128,7 @@ class BaseWorker {
                 console.error(`[${this._groupName}] Job ${message.messageUuid} run out of retries. Moving to DLQ`);
                 messagesToDLQ.push(message);
                 // Add message to DLQ stream
-                pipeline.xadd(this._keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this._groupName, 'error', errorMessage, 'payload', message.serializedData, 'failedAt', timestamp);
+                pipeline.xadd(this._keys.getDlqStreamKey(), '*', ...dlq_message_entity_1.DlqMessageEntity.getStreamFields(message.messageUuid, this._groupName, errorMessage, message.serializedData, timestamp));
                 const statusKey = this._keys.getJobStatusKey(message.messageUuid);
                 pipeline.eval(lua_1.LUA_FINALIZE, 2, statusKey, this._streamName, this._groupName, timestamp, message.streamMessageId);
             }

package/dist/dlq-message-entity.d.ts

@@ -0,0 +1,19 @@
+import { StreamMessage } from "./interfaces";
+export declare class DlqMessageEntity<T extends Record<string, unknown>> {
+    private readonly _streamMessageId;
+    private readonly _rawFields;
+    private readonly _fields;
+    private readonly _group;
+    private readonly _errorMessage;
+    private readonly _failedAt;
+    private readonly _messageUuid;
+    private readonly _data;
+    constructor(message: StreamMessage);
+    get data(): T;
+    get streamMessageId(): string;
+    get messageUuid(): string;
+    get group(): string;
+    get errorMessage(): string;
+    get failedAt(): number;
+    static getStreamFields(id: string, group: string, error: string, payload: string, failedAt: number): (string | number)[];
+}

package/dist/dlq-message-entity.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DlqMessageEntity = void 0;
+class DlqMessageEntity {
+    constructor(message) {
+        this._rawFields = [];
+        this._fields = {};
+        this._streamMessageId = message[0];
+        this._rawFields = message[1];
+        for (let i = 0; i < this._rawFields.length; i += 2) {
+            this._fields[this._rawFields[i]] = this._rawFields[i + 1];
+        }
+        this._messageUuid = this._fields['id'];
+        this._group = this._fields['group'];
+        this._errorMessage = this._fields['error'];
+        this._failedAt = Number(this._fields['failedAt']);
+        this._data = JSON.parse(this._fields['payload']);
+    }
+    get data() {
+        return this._data;
+    }
+    get streamMessageId() {
+        return this._streamMessageId;
+    }
+    get messageUuid() {
+        return this._messageUuid;
+    }
+    get group() {
+        return this._group;
+    }
+    get errorMessage() {
+        return this._errorMessage;
+    }
+    get failedAt() {
+        return this._failedAt;
+    }
+    static getStreamFields(id, group, error, payload, failedAt) {
+        return [
+            'id', id,
+            'group', group,
+            'error', error,
+            'payload', payload,
+            'failedAt', failedAt
+        ];
+    }
+}
+exports.DlqMessageEntity = DlqMessageEntity;

package/dist/dlq-worker.d.ts

@@ -0,0 +1,35 @@
+import Redis from "ioredis";
+import { KeyManager } from "./keys";
+import { DlqWorkerOptions, XReadGroupResponse } from "./interfaces";
+import { DlqMessageEntity } from "./dlq-message-entity";
+export declare abstract class DlqWorker<T extends Record<string, unknown>> {
+    protected readonly _redis: Redis;
+    protected _isRunning: boolean;
+    protected readonly _keys: KeyManager;
+    protected readonly _consumerId: string;
+    protected readonly _dlqStreamName: string;
+    protected readonly _groupName = "dlq-worker";
+    protected readonly _consumerName: string;
+    protected readonly _blockTimeoutMs: number;
+    protected readonly _waitTimeoutMs: number;
+    constructor(_redis: Redis, options: DlqWorkerOptions);
+    /**
+     * Start DLQ worker
+     * @returns
+     */
+    protected start(): Promise<void>;
+    /**
+     * Stop DLQ loop
+     */
+    protected stop(): Promise<void>;
+    /**
+     * Background* DLQ loopw
+     */
+    protected dlqLoop(): Promise<void>;
+    /**
+     * Reads job from DLQ
+     * @returns
+     */
+    protected _readGroup(): Promise<XReadGroupResponse | null>;
+    abstract process(data: DlqMessageEntity<T>): Promise<void>;
+}

package/dist/dlq-worker.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DlqWorker = void 0;
+const keys_1 = require("./keys");
+const uuid_1 = require("uuid");
+const dlq_message_entity_1 = require("./dlq-message-entity");
+class DlqWorker {
+    constructor(_redis, options) {
+        this._redis = _redis;
+        this._isRunning = false;
+        this._consumerId = (0, uuid_1.v7)();
+        this._groupName = 'dlq-worker';
+        this._consumerName = `${this._groupName}-${this._consumerId}`;
+        this._keys = new keys_1.KeyManager(options.streamName);
+        this._dlqStreamName = this._keys.getDlqStreamKey();
+        this._blockTimeoutMs = options.blockTimeoutMs ?? 5000;
+        this._waitTimeoutMs = options.waitTimeoutMs ?? 5000;
+    }
+    /**
+     * Start DLQ worker
+     * @returns
+     */
+    async start() {
+        if (this._isRunning) {
+            return;
+        }
+        this._isRunning = true;
+        try {
+            await this._redis.xgroup('CREATE', this._dlqStreamName, this._groupName, '0', 'MKSTREAM');
+        }
+        catch (e) {
+            if (!e.message.includes('BUSYGROUP')) {
+                throw e;
+            }
+        }
+        this.dlqLoop().catch((e) => console.error('DLQ loop crashed', e));
+    }
+    /**
+     * Stop DLQ loop
+     */
+    async stop() {
+        this._isRunning = false;
+        await this._redis.xgroup('DELCONSUMER', this._dlqStreamName, this._groupName, this._consumerName).catch();
+    }
+    /**
+     * Background* DLQ loopw
+     */
+    async dlqLoop() {
+        while (this._isRunning) {
+            try {
+                const results = await this._readGroup();
+                if (!results) {
+                    await new Promise((resolve) => setTimeout(resolve, this._waitTimeoutMs));
+                    continue;
+                }
+                const message = results[0][1][0];
+                if (!message) {
+                    await new Promise((resolve) => setTimeout(resolve, this._waitTimeoutMs));
+                    continue;
+                }
+                const dlqMessage = new dlq_message_entity_1.DlqMessageEntity(message);
+                // Ack and delete the message. XACKDEL is supported from 8.2
+                await this._redis.multi()
+                    .xack(this._dlqStreamName, this._groupName, dlqMessage.streamMessageId)
+                    .xdel(this._dlqStreamName, dlqMessage.streamMessageId)
+                    .exec();
+                await this.process(dlqMessage);
+            }
+            catch (e) {
+                console.error(`[${this._groupName}] Failed processing DLQ job`, e);
+                await new Promise((resolve) => setTimeout(resolve, this._waitTimeoutMs));
+            }
+        }
+    }
+    /**
+     * Reads job from DLQ
+     * @returns
+     */
+    async _readGroup() {
+        return this._redis.xreadgroup('GROUP', this._groupName, this._consumerName, 'COUNT', 1, 'BLOCK', this._blockTimeoutMs, 'STREAMS', this._dlqStreamName, '>');
+    }
+}
+exports.DlqWorker = DlqWorker;

package/dist/index.d.ts

@@ -6,3 +6,5 @@ export * from './interfaces';
 export * from './batch-worker';
 export * from './stream-message-entity';
 export * from './base-worker';
+export * from './dlq-message-entity';
+export * from './dlq-worker';

package/dist/index.js

@@ -22,3 +22,5 @@ __exportStar(require("./interfaces"), exports);
 __exportStar(require("./batch-worker"), exports);
 __exportStar(require("./stream-message-entity"), exports);
 __exportStar(require("./base-worker"), exports);
+__exportStar(require("./dlq-message-entity"), exports);
+__exportStar(require("./dlq-worker"), exports);

package/dist/interfaces.d.ts

@@ -5,6 +5,11 @@ export interface BaseWorkerOptions {
     streamName: string;
     concurrency: number;
 }
+export interface DlqWorkerOptions {
+    streamName: string;
+    blockTimeoutMs?: number;
+    waitTimeoutMs?: number;
+}
 export interface BaseWorkerControlOptions {
     maxRetries: number;
     blockTimeMs: number;

package/dist/producer.d.ts

@@ -1,7 +1,6 @@
 import { Redis } from 'ioredis';
 export interface JobOptions {
     ttl?: number | null;
-    streamName?: string;
 }
 export declare class Producer<T extends Record<string, unknown>> {
     private readonly redis;

package/dist/producer.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Producer = void 0;
 const uuid_1 = require("uuid");
 const keys_1 = require("./keys");
+const stream_message_entity_1 = require("./stream-message-entity");
 class Producer {
     constructor(redis, streamName) {
         this.redis = redis;
@@ -19,17 +20,16 @@ class Producer {
     async push(payload, targetGroups, opts) {
         const serializedPayload = JSON.stringify(payload);
         const id = (0, uuid_1.v7)();
-        const ttl = opts?.ttl || null; // 24 hours in seconds
+        const ttl = opts?.ttl || null; // Defaults to null, no expiry
         const pipeline = this.redis.pipeline();
         const statusKey = this.keys.getJobStatusKey(id);
         // Initialize job metadata - status
-        // TODO: improve target groups use groups join by "," instead of groups length
         pipeline.hset(statusKey, '__target', targetGroups.length);
         if (ttl) {
             pipeline.expire(statusKey, ttl);
         }
         // Push message to stream
-        pipeline.xadd(this.streamName, '*', 'id', id, 'target', targetGroups.join(','), 'data', serializedPayload);
+        pipeline.xadd(this.streamName, '*', ...stream_message_entity_1.StreamMessageEntity.getStreamFields(id, targetGroups, serializedPayload));
         await pipeline.exec();
         return id;
     }

package/dist/stream-message-entity.d.ts

@@ -15,4 +15,5 @@ export declare class StreamMessageEntity<T extends Record<string, unknown>> {
     get messageUuid(): string;
     get routes(): string[];
     get retryCount(): number;
+    static getStreamFields(id: string, target: string | string[], serializedPayload: string, retryCount?: number): (string | number)[];
 }

package/dist/stream-message-entity.js

@@ -35,5 +35,12 @@ class StreamMessageEntity {
     get retryCount() {
         return this._retryCount;
     }
+    static getStreamFields(id, target, serializedPayload, retryCount) {
+        const fields = ['id', id, 'target', Array.isArray(target) ? target.join(',') : target, 'data', serializedPayload];
+        if (retryCount !== undefined) {
+            fields.push('retryCount', retryCount);
+        }
+        return fields;
+    }
 }
 exports.StreamMessageEntity = StreamMessageEntity;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.2.12",
+  "version": "0.2.13",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {