@koala42/redis-highway 0.2.12 → 0.2.14

package/README.md

@@ -9,6 +9,7 @@ Designed for high throughput and massive concurrency with low overhead.
 - **Granular Retries**: Consumer group isolation - if one group fails, only that group retries.
 - **Reliability**: Auto-claiming of stuck messages (crashed consumers) and Dead Letter Queue (DLQ) support.
 - **Metrics**: Built-in tracking for throughput, queue depth, DLQ size, and retries. Prometheus export ready.
+- **ZSTD Compression**: Optional payload compression using Node.js built-in ZSTD. Workers auto-detect compressed messages.
 
 ## Installation
 
@@ -25,7 +26,10 @@ import { Redis } from 'ioredis';
 import { Producer } from '@koala42/redis-highway';
 
 const redis = new Redis();
-const producer = new Producer<{hello: string}>(redis, 'my-stream');
+const producer = new Producer<{hello: string}>(redis, {
+  streamName: 'my-stream',
+  compression: false // Set to true to enable ZSTD compression
+});
 
 // Send job
 await producer.push(
@@ -92,6 +96,57 @@ const batchWorker = new MyBatchWorker(
 await batchWorker.start();
 ```
 
+### DLQ Worker
+Process messages from the Dead Letter Queue. Use this to handle jobs that have exhausted all retries.
+
+**Important:** DLQ Worker has no built-in error handling or retry policy. If `process()` throws an error, the message is lost. This is by design - DLQ processing is meant for manual intervention, logging, or forwarding to external systems.
+
+```typescript
+import { Redis } from 'ioredis';
+import { DlqWorker, DlqMessageEntity } from '@koala42/redis-highway';
+
+class MyDlqWorker extends DlqWorker<{hello: string}> {
+  async process(message: DlqMessageEntity<{hello: string}>) {
+    console.log('Failed job data:', message.data);
+    console.log('Original error:', message.errorMessage);
+    console.log('Failed at:', new Date(message.failedAt));
+    console.log('Original consumer group:', message.group);
+
+    // Example: Log to external system, send alert, or store for manual review
+    await externalLogger.log(message);
+  }
+}
+
+const redis = new Redis();
+const dlqWorker = new MyDlqWorker(redis, {
+  streamName: 'my-stream' // Must match your main worker's stream
+});
+
+await dlqWorker.start();
+
+// To stop gracefully
+// await dlqWorker.stop();
+```
+
+#### DLQ Worker Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `streamName` | string | - | **Required**. The Redis stream key (same as your main workers). |
+| `blockTimeoutMs` | number | 5000 | Redis XREADGROUP block duration in milliseconds. |
+| `waitTimeoutMs` | number | 5000 | Wait time between processing cycles when no messages are available. |
+
+#### DlqMessageEntity Properties
+
+| Property | Type | Description |
+|----------|------|-------------|
+| `data` | T | The original job payload. |
+| `errorMessage` | string | The error message from the last failed attempt. |
+| `failedAt` | number | Unix timestamp when the job was moved to DLQ. |
+| `group` | string | The consumer group that failed to process this job. |
+| `messageUuid` | string | The original job's unique identifier. |
+| `streamMessageId` | string | The Redis stream message ID. |
+
 ### Metrics
 
 ```typescript
@@ -107,6 +162,26 @@ console.log(stats.throughput);
 const promMetrics = await metrics.getPrometheusMetrics(['group-A'], 'my_app_queue');
 ```
 
+### Compression
+
+Enable ZSTD compression to reduce Redis memory usage and network bandwidth for large payloads.
+
+```typescript
+const producer = new Producer<{hello: string}>(redis, {
+  streamName: 'my-stream',
+  compression: true // Enable ZSTD compression
+});
+
+// Messages are automatically compressed before being sent to Redis
+await producer.push({ hello: 'world' }, ['group-A']);
+```
+
+**Key points:**
+- Compression uses Node.js built-in ZSTD (no external dependencies required, Node.js 22+)
+- Workers automatically detect and decompress compressed messages
+- No configuration changes needed on workers - they handle both compressed and uncompressed messages
+- Recommended for payloads larger than 1KB where compression benefits outweigh CPU overhead
+
 ## Configuration
 
 ### Worker Options
@@ -150,7 +225,10 @@ export class EntryService {
   private readonly producer: Producer<MyPayload>;
 
   constructor(@InjectRedis() private readonly redis: Redis) {
-    this.producer = new Producer(this.redis, 'my-stream');
+    this.producer = new Producer(this.redis, {
+      streamName: 'my-stream',
+      compression: false
+    });
   }
 
   async addToQueue(data: MyPayload) {

package/dist/base-worker.js

@@ -6,6 +6,7 @@ const keys_1 = require("./keys");
 const uuid_1 = require("uuid");
 const stream_message_entity_1 = require("./stream-message-entity");
 const lua_1 = require("./lua");
+const dlq_message_entity_1 = require("./dlq-message-entity");
 class BaseWorker {
     constructor(redis, options, controlOptions, metricsOptions) {
         this.redis = redis;
@@ -44,8 +45,8 @@ class BaseWorker {
                 throw e;
             }
         }
-        this._fetchLoop();
-        this._autoClaimLoop();
+        this._fetchLoop().catch((e) => console.error("Fetch loop crashed", e));
+        this._autoClaimLoop().catch((e) => console.error('Auto claim loop crashed', e));
     }
     /**
      * Gracefully stops the worker
@@ -60,6 +61,7 @@ class BaseWorker {
         while (this._activeCount > 0) {
             await new Promise((resolve) => setTimeout(resolve, 50));
         }
+        await this.redis.xgroup('DELCONSUMER', this._streamName, this._groupName, this._consumerName).catch();
     }
     /**
      * Auto claim loop
@@ -116,7 +118,7 @@ class BaseWorker {
             if (message.retryCount < this._maxRetries) {
                 retryCountIncr++;
                 const newJobId = (0, uuid_1.v7)();
-                pipeline.xadd(this._streamName, '*', 'id', newJobId, 'target', this._groupName, 'retryCount', message.retryCount + 1, 'data', message.serializedData);
+                pipeline.xadd(this._streamName, '*', ...stream_message_entity_1.StreamMessageEntity.getStreamFields(newJobId, this._groupName, message.serializedData, message.compressed, message.retryCount + 1));
                 const newStatusKey = this._keys.getJobStatusKey(newJobId);
                 pipeline.hset(newStatusKey, '__target', 1);
                 const statusKey = this._keys.getJobStatusKey(message.messageUuid);
@@ -126,7 +128,7 @@ class BaseWorker {
                 console.error(`[${this._groupName}] Job ${message.messageUuid} run out of retries. Moving to DLQ`);
                 messagesToDLQ.push(message);
                 // Add message to DLQ stream
-                pipeline.xadd(this._keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this._groupName, 'error', errorMessage, 'payload', message.serializedData, 'failedAt', timestamp);
+                pipeline.xadd(this._keys.getDlqStreamKey(), '*', ...dlq_message_entity_1.DlqMessageEntity.getStreamFields(message.messageUuid, this._groupName, errorMessage, message.serializedData, timestamp));
                 const statusKey = this._keys.getJobStatusKey(message.messageUuid);
                 pipeline.eval(lua_1.LUA_FINALIZE, 2, statusKey, this._streamName, this._groupName, timestamp, message.streamMessageId);
             }

package/dist/dlq-message-entity.d.ts

@@ -0,0 +1,19 @@
+import { StreamMessage } from "./interfaces";
+export declare class DlqMessageEntity<T extends Record<string, unknown>> {
+    private readonly _streamMessageId;
+    private readonly _rawFields;
+    private readonly _fields;
+    private readonly _group;
+    private readonly _errorMessage;
+    private readonly _failedAt;
+    private readonly _messageUuid;
+    private readonly _data;
+    constructor(message: StreamMessage);
+    get data(): T;
+    get streamMessageId(): string;
+    get messageUuid(): string;
+    get group(): string;
+    get errorMessage(): string;
+    get failedAt(): number;
+    static getStreamFields(id: string, group: string, error: string, payload: string, failedAt: number): (string | number)[];
+}

package/dist/dlq-message-entity.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DlqMessageEntity = void 0;
+class DlqMessageEntity {
+    constructor(message) {
+        this._rawFields = [];
+        this._fields = {};
+        this._streamMessageId = message[0];
+        this._rawFields = message[1];
+        for (let i = 0; i < this._rawFields.length; i += 2) {
+            this._fields[this._rawFields[i]] = this._rawFields[i + 1];
+        }
+        this._messageUuid = this._fields['id'];
+        this._group = this._fields['group'];
+        this._errorMessage = this._fields['error'];
+        this._failedAt = Number(this._fields['failedAt']);
+        this._data = JSON.parse(this._fields['payload']);
+    }
+    get data() {
+        return this._data;
+    }
+    get streamMessageId() {
+        return this._streamMessageId;
+    }
+    get messageUuid() {
+        return this._messageUuid;
+    }
+    get group() {
+        return this._group;
+    }
+    get errorMessage() {
+        return this._errorMessage;
+    }
+    get failedAt() {
+        return this._failedAt;
+    }
+    static getStreamFields(id, group, error, payload, failedAt) {
+        return [
+            'id', id,
+            'group', group,
+            'error', error,
+            'payload', payload,
+            'failedAt', failedAt
+        ];
+    }
+}
+exports.DlqMessageEntity = DlqMessageEntity;

package/dist/dlq-worker.d.ts

@@ -0,0 +1,35 @@
+import Redis from "ioredis";
+import { KeyManager } from "./keys";
+import { DlqWorkerOptions, XReadGroupResponse } from "./interfaces";
+import { DlqMessageEntity } from "./dlq-message-entity";
+export declare abstract class DlqWorker<T extends Record<string, unknown>> {
+    protected readonly _redis: Redis;
+    protected _isRunning: boolean;
+    protected readonly _keys: KeyManager;
+    protected readonly _consumerId: string;
+    protected readonly _dlqStreamName: string;
+    protected readonly _groupName = "dlq-worker";
+    protected readonly _consumerName: string;
+    protected readonly _blockTimeoutMs: number;
+    protected readonly _waitTimeoutMs: number;
+    constructor(_redis: Redis, options: DlqWorkerOptions);
+    /**
+     * Start DLQ worker
+     * @returns
+     */
+    protected start(): Promise<void>;
+    /**
+     * Stop DLQ loop
+     */
+    protected stop(): Promise<void>;
+    /**
+     * Background* DLQ loopw
+     */
+    protected dlqLoop(): Promise<void>;
+    /**
+     * Reads job from DLQ
+     * @returns
+     */
+    protected _readGroup(): Promise<XReadGroupResponse | null>;
+    abstract process(data: DlqMessageEntity<T>): Promise<void>;
+}

package/dist/dlq-worker.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DlqWorker = void 0;
+const keys_1 = require("./keys");
+const uuid_1 = require("uuid");
+const dlq_message_entity_1 = require("./dlq-message-entity");
+class DlqWorker {
+    constructor(_redis, options) {
+        this._redis = _redis;
+        this._isRunning = false;
+        this._consumerId = (0, uuid_1.v7)();
+        this._groupName = 'dlq-worker';
+        this._consumerName = `${this._groupName}-${this._consumerId}`;
+        this._keys = new keys_1.KeyManager(options.streamName);
+        this._dlqStreamName = this._keys.getDlqStreamKey();
+        this._blockTimeoutMs = options.blockTimeoutMs ?? 5000;
+        this._waitTimeoutMs = options.waitTimeoutMs ?? 5000;
+    }
+    /**
+     * Start DLQ worker
+     * @returns
+     */
+    async start() {
+        if (this._isRunning) {
+            return;
+        }
+        this._isRunning = true;
+        try {
+            await this._redis.xgroup('CREATE', this._dlqStreamName, this._groupName, '0', 'MKSTREAM');
+        }
+        catch (e) {
+            if (!e.message.includes('BUSYGROUP')) {
+                throw e;
+            }
+        }
+        this.dlqLoop().catch((e) => console.error('DLQ loop crashed', e));
+    }
+    /**
+     * Stop DLQ loop
+     */
+    async stop() {
+        this._isRunning = false;
+        await this._redis.xgroup('DELCONSUMER', this._dlqStreamName, this._groupName, this._consumerName).catch();
+    }
+    /**
+     * Background* DLQ loopw
+     */
+    async dlqLoop() {
+        while (this._isRunning) {
+            try {
+                const results = await this._readGroup();
+                if (!results) {
+                    await new Promise((resolve) => setTimeout(resolve, this._waitTimeoutMs));
+                    continue;
+                }
+                const message = results[0][1][0];
+                if (!message) {
+                    await new Promise((resolve) => setTimeout(resolve, this._waitTimeoutMs));
+                    continue;
+                }
+                const dlqMessage = new dlq_message_entity_1.DlqMessageEntity(message);
+                // Ack and delete the message. XACKDEL is supported from 8.2
+                await this._redis.multi()
+                    .xack(this._dlqStreamName, this._groupName, dlqMessage.streamMessageId)
+                    .xdel(this._dlqStreamName, dlqMessage.streamMessageId)
+                    .exec();
+                await this.process(dlqMessage);
+            }
+            catch (e) {
+                console.error(`[${this._groupName}] Failed processing DLQ job`, e);
+                await new Promise((resolve) => setTimeout(resolve, this._waitTimeoutMs));
+            }
+        }
+    }
+    /**
+     * Reads job from DLQ
+     * @returns
+     */
+    async _readGroup() {
+        return this._redis.xreadgroup('GROUP', this._groupName, this._consumerName, 'COUNT', 1, 'BLOCK', this._blockTimeoutMs, 'STREAMS', this._dlqStreamName, '>');
+    }
+}
+exports.DlqWorker = DlqWorker;

package/dist/index.d.ts

@@ -6,3 +6,6 @@ export * from './interfaces';
 export * from './batch-worker';
 export * from './stream-message-entity';
 export * from './base-worker';
+export * from './dlq-message-entity';
+export * from './dlq-worker';
+export * from './serializer';

package/dist/index.js

@@ -22,3 +22,6 @@ __exportStar(require("./interfaces"), exports);
 __exportStar(require("./batch-worker"), exports);
 __exportStar(require("./stream-message-entity"), exports);
 __exportStar(require("./base-worker"), exports);
+__exportStar(require("./dlq-message-entity"), exports);
+__exportStar(require("./dlq-worker"), exports);
+__exportStar(require("./serializer"), exports);

package/dist/interfaces.d.ts

@@ -5,6 +5,11 @@ export interface BaseWorkerOptions {
     streamName: string;
     concurrency: number;
 }
+export interface DlqWorkerOptions {
+    streamName: string;
+    blockTimeoutMs?: number;
+    waitTimeoutMs?: number;
+}
 export interface BaseWorkerControlOptions {
     maxRetries: number;
     blockTimeMs: number;
@@ -21,3 +26,7 @@ export interface BatchWorkerOptions extends BaseWorkerOptions {
     batchSize: number;
     maxFetchCount: number;
 }
+export interface ProducerOptions {
+    streamName: string;
+    compression: boolean;
+}

package/dist/producer.d.ts

@@ -1,13 +1,14 @@
 import { Redis } from 'ioredis';
+import { ProducerOptions } from './interfaces';
 export interface JobOptions {
     ttl?: number | null;
-    streamName?: string;
 }
 export declare class Producer<T extends Record<string, unknown>> {
-    private readonly redis;
-    private readonly streamName;
-    private keys;
-    constructor(redis: Redis, streamName: string);
+    private readonly _keys;
+    private readonly _redis;
+    private readonly _streamName;
+    private readonly _compression;
+    constructor(redis: Redis, options: ProducerOptions);
     /**
      * Push message to queue
      * @param payload

package/dist/producer.js

@@ -3,11 +3,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Producer = void 0;
 const uuid_1 = require("uuid");
 const keys_1 = require("./keys");
+const stream_message_entity_1 = require("./stream-message-entity");
+const serializer_1 = require("./serializer");
 class Producer {
-    constructor(redis, streamName) {
-        this.redis = redis;
-        this.streamName = streamName;
-        this.keys = new keys_1.KeyManager(streamName);
+    constructor(redis, options) {
+        this._redis = redis;
+        this._streamName = options.streamName;
+        this._keys = new keys_1.KeyManager(this._streamName);
+        this._compression = options.compression;
     }
     /**
      * Push message to queue
@@ -17,19 +20,18 @@ class Producer {
      * @returns Created job ID (uuidv7)
      */
     async push(payload, targetGroups, opts) {
-        const serializedPayload = JSON.stringify(payload);
         const id = (0, uuid_1.v7)();
-        const ttl = opts?.ttl || null; // 24 hours in seconds
-        const pipeline = this.redis.pipeline();
-        const statusKey = this.keys.getJobStatusKey(id);
+        const ttl = opts?.ttl || null; // Defaults to null, no expiry
+        const pipeline = this._redis.pipeline();
+        const statusKey = this._keys.getJobStatusKey(id);
         // Initialize job metadata - status
-        // TODO: improve target groups use groups join by "," instead of groups length
         pipeline.hset(statusKey, '__target', targetGroups.length);
         if (ttl) {
             pipeline.expire(statusKey, ttl);
         }
+        const serializedPayload = this._compression ? await serializer_1.Serializer.compressPayload(payload) : JSON.stringify(payload);
         // Push message to stream
-        pipeline.xadd(this.streamName, '*', 'id', id, 'target', targetGroups.join(','), 'data', serializedPayload);
+        pipeline.xadd(this._streamName, '*', ...stream_message_entity_1.StreamMessageEntity.getStreamFields(id, targetGroups, serializedPayload, this._compression));
         await pipeline.exec();
         return id;
     }

package/dist/serializer.d.ts

@@ -0,0 +1,20 @@
+export declare class Serializer {
+    /**
+     * Compress payload using zstd
+     * @param payload - JSON like payload
+     * @returns base64 encoded payload
+     */
+    static compressPayload(payload: Record<string, unknown>): Promise<string>;
+    /**
+     * Decompress payload using zstd
+     * @param compressedPayload - base64 encoded compressed payload
+     * @returns Parsed JSON payload
+     */
+    static decompressPayload(compressedPayload: string): Promise<Record<string, unknown>>;
+    /**
+     * Decompress payload sync
+     * @param compressedPayload - base64 encoded payload
+     * @returns Parsed JSON payload
+     */
+    static decompressPayloadSync<T extends Record<string, unknown>>(compressedPayload: string): Promise<T>;
+}

package/dist/serializer.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Serializer = void 0;
+const zlib_1 = require("zlib");
+class Serializer {
+    /**
+     * Compress payload using zstd
+     * @param payload - JSON like payload
+     * @returns base64 encoded payload
+     */
+    static async compressPayload(payload) {
+        const minifiedPayload = JSON.stringify(payload);
+        const compressedPayload = await new Promise((resolve) => {
+            (0, zlib_1.zstdCompress)(Buffer.from(minifiedPayload), (_, result) => resolve(result));
+        });
+        return Buffer.from(compressedPayload).toString("base64");
+    }
+    /**
+     * Decompress payload using zstd
+     * @param compressedPayload - base64 encoded compressed payload
+     * @returns Parsed JSON payload
+     */
+    static async decompressPayload(compressedPayload) {
+        const decoded = Buffer.from(compressedPayload, "base64");
+        const decompressed = await new Promise((resolve) => {
+            (0, zlib_1.zstdDecompress)(decoded, (_, result) => resolve(result));
+        });
+        return JSON.parse(decompressed.toString());
+    }
+    /**
+     * Decompress payload sync
+     * @param compressedPayload - base64 encoded payload
+     * @returns Parsed JSON payload
+     */
+    static decompressPayloadSync(compressedPayload) {
+        const decoded = Buffer.from(compressedPayload, "base64");
+        const decompressed = (0, zlib_1.zstdDecompressSync)(decoded);
+        return JSON.parse(decompressed.toString());
+    }
+}
+exports.Serializer = Serializer;

package/dist/stream-message-entity.d.ts

@@ -7,12 +7,15 @@ export declare class StreamMessageEntity<T extends Record<string, unknown>> {
     private readonly _messageUuid;
     private readonly _retryCount;
     private readonly _data;
+    private readonly _compressed;
     private readonly _rawData;
     constructor(message: StreamMessage);
     get data(): T;
     get serializedData(): string;
+    get compressed(): boolean;
     get streamMessageId(): string;
     get messageUuid(): string;
     get routes(): string[];
     get retryCount(): number;
+    static getStreamFields(id: string, target: string | string[], serializedPayload: string, compression: boolean, retryCount?: number): (string | number)[];
 }

package/dist/stream-message-entity.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StreamMessageEntity = void 0;
+const serializer_1 = require("./serializer");
 class StreamMessageEntity {
     constructor(message) {
         this._rawFields = [];
@@ -14,7 +15,8 @@ class StreamMessageEntity {
         this._messageUuid = this._fields['id'];
         this._routes = this._fields['target'].split(',');
         this._retryCount = parseInt(this._fields['retryCount'] || '0', 10);
-        this._data = JSON.parse(this._fields['data']);
+        this._compressed = this._fields['zstd'] === 'true';
+        this._data = this._compressed ? serializer_1.Serializer.decompressPayloadSync(this._fields['data']) : JSON.parse(this._fields['data']);
         this._rawData = this._fields['data'];
     }
     get data() {
@@ -23,6 +25,9 @@ class StreamMessageEntity {
     get serializedData() {
         return this._rawData;
     }
+    get compressed() {
+        return this._compressed;
+    }
     get streamMessageId() {
         return this._streamMessageId;
     }
@@ -35,5 +40,15 @@ class StreamMessageEntity {
     get retryCount() {
         return this._retryCount;
     }
+    static getStreamFields(id, target, serializedPayload, compression, retryCount) {
+        const fields = ['id', id, 'target', Array.isArray(target) ? target.join(',') : target, 'data', serializedPayload];
+        if (retryCount !== undefined) {
+            fields.push('retryCount', retryCount);
+        }
+        if (compression) {
+            fields.push('zstd', 'true');
+        }
+        return fields;
+    }
 }
 exports.StreamMessageEntity = StreamMessageEntity;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.2.12",
+  "version": "0.2.14",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {