@koala42/redis-highway 0.1.8 → 0.1.10

package/dist/batch-worker.d.ts

@@ -5,16 +5,27 @@ export declare abstract class BatchWorker<T extends Record<string, unknown>> {
     protected streamName: string;
     protected batchSize: number;
     protected concurrency: number;
+    protected maxFetchSize: number;
     protected maxRetries: number;
     protected blockTimeMs: number;
+    protected maxFetchCount: number;
+    protected claimIntervalMs: number;
+    protected minIdleTimeMs: number;
     private isRunning;
     private activeCount;
-    private readonly events;
     private keys;
+    private blockingRedis;
+    private readonly events;
     private readonly consumerId;
-    constructor(redis: Redis, groupName: string, streamName: string, batchSize?: number, concurrency?: number, maxRetries?: number, blockTimeMs?: number);
+    constructor(redis: Redis, groupName: string, streamName: string, batchSize?: number, // How many jobs are passed to the process function (max)
+    concurrency?: number, // How many concurrent loops should run
+    maxFetchSize?: number, // How many jobs are fetched at once from redis stream
+    maxRetries?: number, blockTimeMs?: number, // How long should the blocking redis wait for logs from stream
+    maxFetchCount?: number, claimIntervalMs?: number, // Check for stuck jobs every minute
+    minIdleTimeMs?: number);
     start(): Promise<void>;
     stop(): Promise<void>;
+    private autoClaimLoop;
     private fetchLoop;
     /**
      * Spawn worker for current processing

package/dist/batch-worker.js

@@ -7,14 +7,23 @@ const stream_message_entity_1 = require("./stream-message-entity");
 const lua_1 = require("./lua");
 const uuid_1 = require("uuid");
 class BatchWorker {
-    constructor(redis, groupName, streamName, batchSize = 10, concurrency = 1, maxRetries = 3, blockTimeMs = 2000) {
+    constructor(redis, groupName, streamName, batchSize = 10, // How many jobs are passed to the process function (max)
+    concurrency = 1, // How many concurrent loops should run
+    maxFetchSize = 20, // How many jobs are fetched at once from redis stream
+    maxRetries = 3, blockTimeMs = 2000, // How long should the blocking redis wait for logs from stream
+    maxFetchCount = 5000, claimIntervalMs = 60000, // Check for stuck jobs every minute
+    minIdleTimeMs = 120000) {
         this.redis = redis;
         this.groupName = groupName;
         this.streamName = streamName;
         this.batchSize = batchSize;
         this.concurrency = concurrency;
+        this.maxFetchSize = maxFetchSize;
         this.maxRetries = maxRetries;
         this.blockTimeMs = blockTimeMs;
+        this.maxFetchCount = maxFetchCount;
+        this.claimIntervalMs = claimIntervalMs;
+        this.minIdleTimeMs = minIdleTimeMs;
         this.isRunning = false;
         this.activeCount = 0;
         this.events = new events_1.EventEmitter();
@@ -24,6 +33,7 @@ class BatchWorker {
         }
         this.events.setMaxListeners(100);
         this.keys = new keys_1.KeyManager(streamName);
+        this.blockingRedis = this.redis.duplicate();
     }
     async start() {
         if (this.isRunning) {
@@ -39,14 +49,62 @@ class BatchWorker {
             }
         }
         this.fetchLoop();
+        this.autoClaimLoop();
     }
     async stop() {
         this.isRunning = false;
         this.events.emit('job_finished');
+        if (this.blockingRedis) {
+            try {
+                await this.blockingRedis.quit();
+            }
+            catch (e) {
+                // whatever
+            }
+        }
         while (this.activeCount > 0) {
             await new Promise((resolve) => setTimeout(resolve, 50));
         }
     }
+    async autoClaimLoop() {
+        while (this.isRunning) {
+            try {
+                await new Promise(resolve => setTimeout(resolve, this.claimIntervalMs));
+                if (!this.isRunning) {
+                    break;
+                }
+                let cursor = '0-0';
+                let continueClaiming = true;
+                while (continueClaiming && this.isRunning) {
+                    const result = await this.redis.xautoclaim(this.streamName, this.groupName, this.getConsumerName(), this.minIdleTimeMs, cursor, 'COUNT', this.batchSize);
+                    if (!result || !result.length) {
+                        continueClaiming = false;
+                        break;
+                    }
+                    const [nextCursor, messages] = result;
+                    cursor = nextCursor;
+                    if (messages && messages.length > 0) {
+                        console.log(`[${this.groupName}] Recovered ${messages.length} stuck messages`);
+                        if (this.activeCount < this.concurrency) {
+                            continueClaiming = false;
+                        }
+                        this.spawnWorker(messages);
+                    }
+                    else {
+                        continueClaiming = false;
+                    }
+                    if (nextCursor === '0-0') {
+                        continueClaiming = false;
+                    }
+                }
+            }
+            catch (e) {
+                if (this.isRunning) {
+                    console.error(`[${this.groupName}] Auto claim err:`, e.message);
+                }
+            }
+        }
+    }
     async fetchLoop() {
         while (this.isRunning) {
             const freeSlots = this.concurrency - this.activeCount;
@@ -54,9 +112,10 @@ class BatchWorker {
                 await new Promise((resolve) => this.events.once('job_finished', resolve));
                 continue;
             }
-            const itemsCount = freeSlots * this.batchSize;
+            const calculatedCount = freeSlots * this.batchSize;
+            const itemsCount = Math.min(calculatedCount, this.maxFetchCount);
             try {
-                const results = await this.redis.xreadgroup('GROUP', this.groupName, this.getConsumerName(), 'COUNT', itemsCount, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
+                const results = await this.blockingRedis.xreadgroup('GROUP', this.groupName, this.getConsumerName(), 'COUNT', itemsCount, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
                 if (!results) {
                     continue;
                 }
@@ -67,8 +126,10 @@ class BatchWorker {
                 }
             }
             catch (err) {
-                console.error(`[${this.groupName}] Fetch Error: `, err);
-                await new Promise((resolve) => setTimeout(resolve, 1000));
+                if (this.isRunning) { // Quicker grace shutdown
+                    console.error(`[${this.groupName}] Fetch Error: `, err);
+                    await new Promise((resolve) => setTimeout(resolve, 1000));
+                }
             }
         }
     }
@@ -103,79 +164,43 @@ class BatchWorker {
             }
             await pipeline.exec();
         }
-        // If no messsages need to be process, return. Fires job finished event for another loop to pickup next logs
         if (!messages.length) {
             return;
         }
-        // Get jobs data
-        const pipeline = this.redis.pipeline();
-        for (const message of messages) {
-            pipeline.get(this.keys.getJobDataKey(message.messageUuid));
-        }
-        const response = await pipeline.exec();
-        // TODO: Add error handling
-        if (!response) {
-            return;
-        }
-        // Parse job data into message entities (lol, titties)
-        messages.forEach((message, index) => {
-            const foundData = response[index] || null;
-            if (!foundData) {
-                return;
-            }
-            const [error, data] = foundData;
-            if (error) {
-                console.error(`[${this.groupName}] Failed getting job data err: `, error);
-                return;
-            }
-            if (!data) {
-                console.error(`[${this.groupName}] Data not found for job`);
-                return;
-            }
-            message.data = JSON.parse(data);
-        });
-        const messagesData = [];
-        const messagesToFinalize = [];
-        messages.forEach((message) => {
-            messagesToFinalize.push(message);
-            if (message.data) {
-                messagesData.push(message.data);
-            }
-        });
-        // TODO improve error handling
-        if (!messagesData.length) {
-            return;
-        }
+        const messagesData = messages.map((msg) => msg.data);
         try {
             await this.process(messagesData);
-            await this.finalize(messagesToFinalize);
+            await this.finalize(messages);
         }
         catch (err) {
-            console.error(`[${this.groupName}] Jobs failed`, err);
+            console.error(`[${this.groupName}] Processing failed`, err);
             await this.handleFailure(messages, err.message);
         }
     }
     async handleFailure(messages, errorMessage) {
         const pipeline = this.redis.pipeline();
-        // 1. ACK the failed message - removes from stream later (or rather, confirms we processed this specific delivery)
+        // ack
         for (const message of messages) {
             pipeline.xack(this.streamName, this.groupName, message.streamMessageId);
         }
         const messagesToDlq = [];
         for (const message of messages) {
-            if (message.retryCount < this.maxRetries) {
-                // Retry
-                console.log(`[${this.groupName}] Retrying job ${message.messageUuid} (Attempt ${message.retryCount + 1}/${this.maxRetries})`);
-                // Refresh TTL
-                pipeline.expire(this.keys.getJobDataKey(message.messageUuid), 3600);
-                pipeline.expire(this.keys.getJobStatusKey(message.messageUuid), 3600);
-                pipeline.xadd(this.streamName, '*', 'id', message.messageUuid, 'target', this.groupName, 'retryCount', message.retryCount + 1);
+            if (message.routes.includes(this.groupName)) {
+                if (message.retryCount < this.maxRetries && message.data) {
+                    console.log(`[${this.groupName}] Retrying job ${message.messageUuid} attempt ${message.retryCount + 1}/${this.maxRetries}`);
+                    const payloadString = JSON.stringify(message.data);
+                    pipeline.xadd(this.streamName, '*', 'id', message.messageUuid, 'target', this.groupName, 'retryCount', message.retryCount + 1, 'data', payloadString);
+                }
+                else {
+                    console.error(`[${this.groupName}] Job ${message.messageUuid} run out of retries. Moving to DLQ`);
+                    messagesToDlq.push(message);
+                    pipeline.xadd(this.keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this.groupName, 'error', errorMessage, 'payload', message.data ? JSON.stringify(message.data) : 'MISSING', 'failedAt', Date.now());
+                }
             }
             else {
-                // DLQ
-                console.error(`[${this.groupName}] Job ${message.messageUuid} exhausted retries. Moving to DLQ.`);
+                console.error(`[${this.groupName}] Job ${message.messageUuid} failed but not routed to this group. Moving to DLQ.`);
                 messagesToDlq.push(message);
-                pipeline.xadd(this.keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this.groupName, 'error', errorMessage, 'payload', message.data ? JSON.stringify(message.data) : 'MISSING', 'failedAt', Date.now());
+                pipeline.xadd(this.keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this.groupName, 'error', `Failed but not routed to ${this.groupName}: ${errorMessage}`, 'payload', JSON.stringify(message.data), 'failedAt', Date.now());
             }
         }
         await pipeline.exec();
@@ -184,26 +209,21 @@ class BatchWorker {
         }
     }
     async finalize(messages) {
-        if (messages.length === 0)
+        if (messages.length === 0) {
             return;
+        }
         const pipeline = this.redis.pipeline();
         const timestamp = Date.now();
         const throughputKey = this.keys.getThroughputKey(this.groupName, timestamp);
         const totalKey = this.keys.getTotalKey(this.groupName);
-        // 1. Batch xacks
         const ids = messages.map(m => m.streamMessageId);
         pipeline.xack(this.streamName, this.groupName, ...ids);
-        // 2. Batch metrics
         pipeline.incrby(throughputKey, ids.length);
         pipeline.expire(throughputKey, 86400);
         pipeline.incrby(totalKey, ids.length);
-        // Lua scripts to only check if data should be deleted
         for (const msg of messages) {
             const statusKey = this.keys.getJobStatusKey(msg.messageUuid);
-            const dataKey = this.keys.getJobDataKey(msg.messageUuid);
-            pipeline.eval(lua_1.LUA_FINALIZE_COMPLEX, 3, statusKey, dataKey, this.streamName, // Keys
-            this.groupName, timestamp, msg.streamMessageId // args
-            );
+            pipeline.eval(lua_1.LUA_FINALIZE_COMPLEX, 2, statusKey, this.streamName, this.groupName, timestamp, msg.streamMessageId);
         }
         await pipeline.exec();
     }

package/dist/batch-worker.spec.js

@@ -11,7 +11,8 @@ const uuid_1 = require("uuid");
 const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
 class TestBatchWorker extends batch_worker_1.BatchWorker {
     constructor(redis, groupName, streamName, batchSize = 10, concurrency = 1, maxRetries = 3, blockTimeMs = 100) {
-        super(redis, groupName, streamName, batchSize, concurrency, maxRetries, blockTimeMs);
+        // Fix argument order: batchSize, concurrency, maxFetchSize, maxRetries, blockTimeMs
+        super(redis, groupName, streamName, batchSize, concurrency, 20, maxRetries, blockTimeMs);
         this.processedBatches = [];
         this.shouldFail = false;
         this.failCount = 0;

package/dist/lua.d.ts

@@ -1,2 +1,2 @@
-export declare const LUA_MARK_DONE = "\n-- KEYS[1] = status key status key for jog\n-- KEYS[2] = data key for job\n-- KEYS[3] = stream key\n-- KEYS[4] = group name\n-- KEYS[5] = metrics key\n-- KEYS[6] = total metrics key(persistent)\n\n-- ARGV[1] = route name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId - redis stream item ID\n\n-- 1. Ack the stream message\nredis.call('XACK', KEYS[3], KEYS[4], ARGV[3])\n\n-- 2. in status key mark the current route as done by saving timestamp\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 3. Increment throughput metric\nif KEYS[5] then\n    redis.call('INCR', KEYS[5])\n    redis.call('EXPIRE', KEYS[5], 86400)\nend\n\n-- 3.1 Increment Total Metric\nif KEYS[6] then\n    redis.call('INCR', KEYS[6])\nend\n\n-- 4. Check for completed routes\nlocal current_fields = redis.call('HLEN', KEYS[1])\n\n-- 5. Get the target completed routes\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 6. If completed routes is status hash length - 1 -> all were done and we can cleanup\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1], KEYS[2])\n    redis.call('XDEL', KEYS[3], ARGV[3])\n    return 1 -- Cleanup, DONE\nend\n\nreturn 0 -- Some routes are not done yet\n";
-export declare const LUA_FINALIZE_COMPLEX = "\n-- KEYS[1] = status key\n-- KEYS[2] = data key\n-- KEYS[3] = stream key\n-- ARGV[1] = group name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId\n\n-- 1. Update status\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 2. Check completions\nlocal current_fields = redis.call('HLEN', KEYS[1])\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 3. Cleanup if done\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1], KEYS[2])\n    redis.call('XDEL', KEYS[3], ARGV[3])\n    return 1\nend\n\nreturn 0\n";
+export declare const LUA_MARK_DONE = "\n-- KEYS[1] = status key status key for jog\n-- KEYS[2] = stream key\n-- KEYS[3] = group name\n-- KEYS[4] = metrics key\n-- KEYS[5] = total metrics key(persistent)\n\n-- ARGV[1] = route name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId - redis stream item ID\n\n-- 1 Ack the stream message\nredis.call('XACK', KEYS[2], KEYS[3], ARGV[3])\n\n-- 2 in status key mark the current route as done by saving timestamp\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 3 Increment throughput metric\nif KEYS[5] then\n    redis.call('INCR', KEYS[4])\n    redis.call('EXPIRE', KEYS[4], 86400)\nend\n\n-- 4 Increment Total Metric\nredis.call('INCR', KEYS[5])\n\n-- 5 Check for completed routes\nlocal current_fields = redis.call('HLEN', KEYS[1])\n\n-- 6 Get the target completed routes\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 7 If completed routes is status hash length - 1 -> all were done and we can cleanup\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1]) -- Only delete status key\n    redis.call('XDEL', KEYS[2], ARGV[3])\n    return 1 -- Cleanup, DONE\nend\n\nreturn 0 -- Some routes are not done yet\n";
+export declare const LUA_FINALIZE_COMPLEX = "\n-- KEYS[1] = status key\n-- KEYS[2] = stream key\n-- ARGV[1] = group name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId\n\n-- 1. Update status\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 2. Check completions\nlocal current_fields = redis.call('HLEN', KEYS[1])\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 3. Cleanup if done\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1])\n    redis.call('XDEL', KEYS[2], ARGV[3])\n    return 1\nend\n\nreturn 0\n";

package/dist/lua.js

@@ -3,37 +3,34 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.LUA_FINALIZE_COMPLEX = exports.LUA_MARK_DONE = void 0;
 exports.LUA_MARK_DONE = `
 -- KEYS[1] = status key status key for jog
--- KEYS[2] = data key for job
--- KEYS[3] = stream key
--- KEYS[4] = group name
--- KEYS[5] = metrics key
--- KEYS[6] = total metrics key(persistent)
+-- KEYS[2] = stream key
+-- KEYS[3] = group name
+-- KEYS[4] = metrics key
+-- KEYS[5] = total metrics key(persistent)
 
 -- ARGV[1] = route name
 -- ARGV[2] = timestamp
 -- ARGV[3] = msgId - redis stream item ID
 
--- 1. Ack the stream message
-redis.call('XACK', KEYS[3], KEYS[4], ARGV[3])
+-- 1 Ack the stream message
+redis.call('XACK', KEYS[2], KEYS[3], ARGV[3])
 
--- 2. in status key mark the current route as done by saving timestamp
+-- 2 in status key mark the current route as done by saving timestamp
 redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
 
--- 3. Increment throughput metric
+-- 3 Increment throughput metric
 if KEYS[5] then
-    redis.call('INCR', KEYS[5])
-    redis.call('EXPIRE', KEYS[5], 86400)
+    redis.call('INCR', KEYS[4])
+    redis.call('EXPIRE', KEYS[4], 86400)
 end
 
--- 3.1 Increment Total Metric
-if KEYS[6] then
-    redis.call('INCR', KEYS[6])
-end
+-- 4 Increment Total Metric
+redis.call('INCR', KEYS[5])
 
--- 4. Check for completed routes
+-- 5 Check for completed routes
 local current_fields = redis.call('HLEN', KEYS[1])
 
--- 5. Get the target completed routes
+-- 6 Get the target completed routes
 local target_str = redis.call('HGET', KEYS[1], '__target')
 local target = tonumber(target_str)
 
@@ -41,10 +38,10 @@ if not target then
     return 0
 end
 
--- 6. If completed routes is status hash length - 1 -> all were done and we can cleanup
+-- 7 If completed routes is status hash length - 1 -> all were done and we can cleanup
 if current_fields >= (target + 1) then
-    redis.call('DEL', KEYS[1], KEYS[2])
-    redis.call('XDEL', KEYS[3], ARGV[3])
+    redis.call('DEL', KEYS[1]) -- Only delete status key
+    redis.call('XDEL', KEYS[2], ARGV[3])
     return 1 -- Cleanup, DONE
 end
 
@@ -52,8 +49,7 @@ return 0 -- Some routes are not done yet
 `;
 exports.LUA_FINALIZE_COMPLEX = `
 -- KEYS[1] = status key
--- KEYS[2] = data key
--- KEYS[3] = stream key
+-- KEYS[2] = stream key
 -- ARGV[1] = group name
 -- ARGV[2] = timestamp
 -- ARGV[3] = msgId
@@ -72,8 +68,8 @@ end
 
 -- 3. Cleanup if done
 if current_fields >= (target + 1) then
-    redis.call('DEL', KEYS[1], KEYS[2])
-    redis.call('XDEL', KEYS[3], ARGV[3])
+    redis.call('DEL', KEYS[1])
+    redis.call('XDEL', KEYS[2], ARGV[3])
     return 1
 end
 

package/dist/producer.js

@@ -21,15 +21,7 @@ class Producer {
         const id = (0, uuid_1.v7)();
         const ttl = opts?.ttl || null; // 24 hours in seconds
         const pipeline = this.redis.pipeline();
-        const dataKey = this.keys.getJobDataKey(id);
         const statusKey = this.keys.getJobStatusKey(id);
-        // Create job data
-        if (ttl) {
-            pipeline.set(dataKey, serializedPayload, 'EX', ttl);
-        }
-        else {
-            pipeline.set(dataKey, serializedPayload);
-        }
         // Initialize job metadata - status
         // TODO: improve target groups use groups join by "," instead of groups length
         pipeline.hset(statusKey, '__target', targetGroups.length);
@@ -37,7 +29,7 @@ class Producer {
             pipeline.expire(statusKey, ttl);
         }
         // Push message to stream
-        pipeline.xadd(this.streamName, '*', 'id', id, 'target', targetGroups.join(','));
+        pipeline.xadd(this.streamName, '*', 'id', id, 'target', targetGroups.join(','), 'data', serializedPayload);
         await pipeline.exec();
         return id;
     }

package/dist/queue.spec.js

@@ -11,8 +11,8 @@ const metrics_1 = require("./metrics");
 const uuid_1 = require("uuid");
 const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
 class TestWorker extends worker_1.Worker {
-    constructor(redis, groupName, streamName, concurrency = 1, blockTimeMs = 100) {
-        super(redis, groupName, streamName, concurrency, 3, blockTimeMs);
+    constructor(redis, groupName, streamName, concurrency = 1, maxRetries = 3, blockTimeMs = 100, claimIntervalMs = 60000, minIdleTimeMs = 300000) {
+        super(redis, groupName, streamName, concurrency, maxRetries, blockTimeMs, claimIntervalMs, minIdleTimeMs);
         this.processedCount = 0;
         this.lastProcessedId = null;
         this.shouldFail = false;
@@ -70,8 +70,8 @@ class TestWorker extends worker_1.Worker {
     };
     (0, vitest_1.describe)('Core Functionality', () => {
         (0, vitest_1.it)('Should deliver message to all target groups', async () => {
-            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 100);
-            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 100);
+            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
+            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 3, 100);
             workers.push(w1, w2);
             await w1.start();
             await w2.start();
@@ -86,8 +86,8 @@ class TestWorker extends worker_1.Worker {
             (0, vitest_1.expect)(await redis.exists(dataKey)).toBe(0);
         });
         (0, vitest_1.it)('Should only deliver to targeted groups', async () => {
-            const wA = new TestWorker(redis, 'group-A', streamName, 1, 100);
-            const wB = new TestWorker(redis, 'group-B', streamName, 1, 100);
+            const wA = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
+            const wB = new TestWorker(redis, 'group-B', streamName, 1, 3, 100);
             workers.push(wA, wB);
             await wA.start();
             await wB.start();
@@ -97,8 +97,8 @@ class TestWorker extends worker_1.Worker {
             (0, vitest_1.expect)(wB.processedCount).toBe(0);
         });
         (0, vitest_1.it)('Should retry only the failed group', async () => {
-            const wOk = new TestWorker(redis, 'group-Ok', streamName, 1, 100);
-            const wFail = new TestWorker(redis, 'group-Fail', streamName, 1, 100);
+            const wOk = new TestWorker(redis, 'group-Ok', streamName, 1, 3, 100);
+            const wFail = new TestWorker(redis, 'group-Fail', streamName, 1, 3, 100);
             wFail.shouldFail = true;
             wFail.maxFails = 1; // Fail once, then succeed
             workers.push(wOk, wFail);
@@ -113,7 +113,7 @@ class TestWorker extends worker_1.Worker {
             (0, vitest_1.expect)(wOk.processedCount).toBe(1); // wOk should NOT process the retry
         });
         (0, vitest_1.it)('Should move to DLQ after max retries', async () => {
-            const wDead = new TestWorker(redis, 'group-Dead', streamName, 1, 100);
+            const wDead = new TestWorker(redis, 'group-Dead', streamName, 1, 3, 100);
             wDead.shouldFail = true;
             wDead.maxFails = 10; // Fail forever (more than max retries which is 3)
             workers.push(wDead);
@@ -130,7 +130,7 @@ class TestWorker extends worker_1.Worker {
     });
     (0, vitest_1.describe)('Metrics & Monitoring', () => {
         (0, vitest_1.it)('Should track throughput and queue size', async () => {
-            const w = new TestWorker(redis, 'group-Metrics', streamName, 1, 100);
+            const w = new TestWorker(redis, 'group-Metrics', streamName, 1, 3, 100);
             const metricsService = new metrics_1.Metrics(redis, streamName);
             workers.push(w);
             await w.start();
@@ -148,7 +148,7 @@ class TestWorker extends worker_1.Worker {
             (0, vitest_1.expect)(metrics.dlqLength).toBe(1);
         });
         (0, vitest_1.it)('Should export Prometheus metrics', async () => {
-            const w = new TestWorker(redis, 'group-Prom', streamName, 1, 100);
+            const w = new TestWorker(redis, 'group-Prom', streamName, 1, 3, 100);
             const metricsService = new metrics_1.Metrics(redis, streamName);
             workers.push(w);
             await w.start();
@@ -165,7 +165,7 @@ class TestWorker extends worker_1.Worker {
     });
     (0, vitest_1.describe)('Stream Cleanup', () => {
         (0, vitest_1.it)('Should delete message from stream after processing', async () => {
-            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 100);
+            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
             workers.push(w1);
             await w1.start();
             const id = await producer.push({ id: 'msg-cleanup' }, ['group-A']);
@@ -182,8 +182,8 @@ class TestWorker extends worker_1.Worker {
             (0, vitest_1.expect)(messages.length).toBe(0);
         });
         (0, vitest_1.it)('Should delete message from stream only after ALL groups processed it', async () => {
-            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 100);
-            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 100);
+            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
+            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 3, 100);
             workers.push(w1, w2);
             await w1.start(); // Only start w1
             const id = await producer.push({ id: 'msg-multi' }, ['group-A', 'group-B']);
@@ -203,4 +203,33 @@ class TestWorker extends worker_1.Worker {
             (0, vitest_1.expect)(success).toBe(true);
         });
     });
+    (0, vitest_1.it)('Should recover stuck messages via Auto-Claim', async () => {
+        const groupName = 'group-Recover';
+        // Start worker with short minIdleTime (e.g., 1000ms) to trigger claim quickly
+        // minIdleTimeMs = 1000. claimIntervalMs = 500 (check frequently)
+        const w = new TestWorker(redis, groupName, streamName, 1, 3, 100, 500, 1000);
+        workers.push(w);
+        // 1. Setup group manually
+        await redis.xgroup('CREATE', streamName, groupName, '0', 'MKSTREAM');
+        // 2. Push message
+        const id = await producer.push({ id: 'stuck-msg' }, [groupName]);
+        // 3. Simulate a consumer reading but crashing (no ACK)
+        // consumer name 'bad-consumer'
+        await redis.xreadgroup('GROUP', groupName, 'bad-consumer', 'COUNT', 1, 'STREAMS', streamName, '>');
+        // 4. Wait for minIdleTime (1000ms) + buffer
+        await new Promise(r => setTimeout(r, 1200));
+        // 5. Start our worker
+        await w.start();
+        // 6. Verify worker picks it up
+        await waitFor(() => w.processedCount === 1, 5000);
+        (0, vitest_1.expect)(w.processedCount).toBe(1);
+        (0, vitest_1.expect)(w.lastProcessedId).toBe('stuck-msg');
+        // Verify it was claimed (delivered to new consumer)
+        // We can check PEL or just trust processedCount
+        const pending = await redis.xpending(streamName, groupName);
+        // After processing, it should be ACKed, so pending count => 0 (if deleted)
+        // or if finalize runs, it deletes the message entirely.
+        const len = await redis.xlen(streamName);
+        (0, vitest_1.expect)(len).toBe(0);
+    });
 });

package/dist/stream-message-entity.d.ts

@@ -6,10 +6,9 @@ export declare class StreamMessageEntity<T extends Record<string, unknown>> {
     private readonly _routes;
     private readonly _messageUuid;
     private readonly _retryCount;
-    private _data;
+    private readonly _data;
     constructor(message: StreamMessage);
-    set data(data: T);
-    get data(): T | null;
+    get data(): T;
     get streamMessageId(): string;
     get messageUuid(): string;
     get routes(): string[];

package/dist/stream-message-entity.js

@@ -6,7 +6,6 @@ class StreamMessageEntity {
         this._rawFields = [];
         this._fields = {};
         this._routes = [];
-        this._data = null;
         this._streamMessageId = message[0];
         this._rawFields = message[1];
         for (let i = 0; i < this._rawFields.length; i += 2) {
@@ -15,9 +14,17 @@ class StreamMessageEntity {
         this._messageUuid = this._fields['id'];
         this._routes = this._fields['target'].split(',');
         this._retryCount = parseInt(this._fields['retryCount'] || '0', 10);
-    }
-    set data(data) {
-        this._data = data;
+        try {
+            this._data = JSON.parse(this._fields['data']);
+        }
+        catch (e) {
+            // Handle corrupt or missing data gracefully
+            // We can set it to null (need to update type to T | null) or a dummy.
+            // Since strict T is expected, we might have to cast or throw controlled error.
+            // For now, let's assume T can be null-ish or cast. But getter says T.
+            // Let's coerce to {} as any to avoid crash, let validation downstream handle it.
+            this._data = {};
+        }
     }
     get data() {
         return this._data;

package/dist/worker.d.ts

@@ -6,18 +6,22 @@ export declare abstract class Worker<T extends Record<string, unknown>> {
     protected concurrency: number;
     protected MAX_RETRIES: number;
     protected blockTimeMs: number;
+    protected claimIntervalMs: number;
+    protected minIdleTimeMs: number;
     private isRunning;
     private activeCount;
     private readonly events;
     private keys;
     private consumerId;
-    constructor(redis: Redis, groupName: string, streamName: string, concurrency?: number, MAX_RETRIES?: number, blockTimeMs?: number);
+    private blockingRedis;
+    constructor(redis: Redis, groupName: string, streamName: string, concurrency?: number, MAX_RETRIES?: number, blockTimeMs?: number, claimIntervalMs?: number, minIdleTimeMs?: number);
     /**
      * Start worker
      * @returns
      */
     start(): Promise<void>;
     stop(): Promise<void>;
+    private autoClaimLoop;
     private fetchLoop;
     private spawnWorker;
     private processInternal;

package/dist/worker.js

@@ -7,19 +7,22 @@ const keys_1 = require("./keys");
 const stream_message_entity_1 = require("./stream-message-entity");
 const uuid_1 = require("uuid");
 class Worker {
-    constructor(redis, groupName, streamName, concurrency = 1, MAX_RETRIES = 3, blockTimeMs = 2000) {
+    constructor(redis, groupName, streamName, concurrency = 1, MAX_RETRIES = 3, blockTimeMs = 2000, claimIntervalMs = 60000, minIdleTimeMs = 300000) {
         this.redis = redis;
         this.groupName = groupName;
         this.streamName = streamName;
         this.concurrency = concurrency;
         this.MAX_RETRIES = MAX_RETRIES;
         this.blockTimeMs = blockTimeMs;
+        this.claimIntervalMs = claimIntervalMs;
+        this.minIdleTimeMs = minIdleTimeMs;
         this.isRunning = false;
         this.activeCount = 0;
         this.events = new events_1.EventEmitter();
         this.consumerId = (0, uuid_1.v7)();
         this.events.setMaxListeners(100);
         this.keys = new keys_1.KeyManager(streamName);
+        this.blockingRedis = this.redis.duplicate();
     }
     /**
      * Start worker
@@ -39,15 +42,59 @@ class Worker {
             }
         }
         this.fetchLoop();
+        this.autoClaimLoop();
     }
     async stop() {
         this.isRunning = false;
-        this.events.emit('job_finished'); // Wake up fetch loop if it's waiting
-        // Wait for active jobs to finish
+        this.events.emit('job_finished');
+        if (this.blockingRedis) {
+            try {
+                await this.blockingRedis.quit();
+            }
+            catch (e) { }
+        }
         while (this.activeCount > 0) {
             await new Promise(resolve => setTimeout(resolve, 50));
         }
     }
+    async autoClaimLoop() {
+        while (this.isRunning) {
+            try {
+                await new Promise(resolve => setTimeout(resolve, this.claimIntervalMs));
+                if (!this.isRunning) {
+                    break;
+                }
+                let cursor = '0-0';
+                let continueClaiming = true;
+                while (continueClaiming && this.isRunning) {
+                    const result = await this.redis.xautoclaim(this.streamName, this.groupName, this.consumerName(), this.minIdleTimeMs, cursor, 'COUNT', this.concurrency);
+                    if (!result) {
+                        continueClaiming = false;
+                        break;
+                    }
+                    const [nextCursor, messages] = result;
+                    cursor = nextCursor;
+                    if (messages && messages.length > 0) {
+                        console.log(`[${this.groupName}] Recovered ${messages.length} stuck messages`);
+                        for (const msg of messages) {
+                            this.spawnWorker(msg);
+                        }
+                    }
+                    else {
+                        continueClaiming = false;
+                    }
+                    if (nextCursor === '0-0') {
+                        continueClaiming = false;
+                    }
+                }
+            }
+            catch (e) {
+                if (this.isRunning) {
+                    console.error(`[${this.groupName}] auto claim err:`, e.message);
+                }
+            }
+        }
+    }
     async fetchLoop() {
         while (this.isRunning) {
             const freeSlots = this.concurrency - this.activeCount;
@@ -56,7 +103,7 @@ class Worker {
                 continue;
             }
             try {
-                const results = await this.redis.xreadgroup('GROUP', this.groupName, this.consumerName(), 'COUNT', freeSlots, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
+                const results = await this.blockingRedis.xreadgroup('GROUP', this.groupName, this.consumerName(), 'COUNT', freeSlots, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
                 if (results) {
                     const messages = results[0][1];
                     for (const msg of messages) {
@@ -84,52 +131,37 @@ class Worker {
             return;
         }
         try {
-            const dataKey = this.keys.getJobDataKey(streamMessage.messageUuid);
-            const payload = await this.redis.get(dataKey);
-            if (!payload) {
-                // Data missing or expired
-                await this.finalize(streamMessage.messageUuid, streamMessage.streamMessageId);
-                return;
-            }
-            await this.process(JSON.parse(payload));
+            await this.process(streamMessage.data);
             await this.finalize(streamMessage.messageUuid, streamMessage.streamMessageId);
         }
         catch (err) {
             console.error(`[${this.groupName}] Job failed ${streamMessage.messageUuid}`, err);
-            await this.handleFailure(streamMessage.messageUuid, streamMessage.streamMessageId, streamMessage.retryCount, err.message);
+            await this.handleFailure(streamMessage.messageUuid, streamMessage.streamMessageId, streamMessage.retryCount, err.message, streamMessage.data);
         }
     }
-    async handleFailure(uuid, msgId, currentRetries, errorMsg) {
-        // 1. ACK the failed message - removes from stream later
+    async handleFailure(uuid, msgId, currentRetries, errorMsg, payloadData) {
+        // Ack
         await this.redis.xack(this.streamName, this.groupName, msgId);
-        // If current retries is lower than max retries, enque it back for another run
-        if (currentRetries < this.MAX_RETRIES) {
+        const payloadString = payloadData ? JSON.stringify(payloadData) : '';
+        if (currentRetries < this.MAX_RETRIES && payloadData) {
             console.log(`[${this.groupName}] Retrying job ${uuid} (Attempt ${currentRetries + 1}/${this.MAX_RETRIES})`);
             const pipeline = this.redis.pipeline();
-            // Refresh TTL to ensure data persists through retries (e.g., +1 hour)
-            pipeline.expire(this.keys.getJobDataKey(uuid), 3600);
-            pipeline.expire(this.keys.getJobStatusKey(uuid), 3600);
-            pipeline.xadd(this.streamName, '*', 'id', uuid, 'target', this.groupName, // Instead of all groups, target the failed one
-            'retryCount', currentRetries + 1);
+            pipeline.xadd(this.streamName, '*', 'id', uuid, 'target', this.groupName, 'retryCount', currentRetries + 1, 'data', payloadString);
             await pipeline.exec();
         }
         else {
-            // If retries is larger than allowed, insert the job with all data to dead letter queue
-            // 2b. DEAD LETTER QUEUE (DLQ)
-            console.error(`[${this.groupName}] Job ${uuid} exhausted retries. Moving to DLQ.`);
-            const payload = await this.redis.get(this.keys.getJobDataKey(uuid));
-            await this.redis.xadd(this.keys.getDlqStreamKey(), '*', 'id', uuid, 'group', this.groupName, 'error', errorMsg, 'payload', payload || 'MISSING', 'failedAt', Date.now());
-            // Delete job from stream and mark it as "done"
-            await this.finalize(uuid, msgId, true);
+            console.error(`[${this.groupName}] Job ${uuid} run outof retries. Moving to DLQ`);
+            await this.redis.xadd(this.keys.getDlqStreamKey(), '*', 'id', uuid, 'group', this.groupName, 'error', errorMsg, 'payload', payloadString, 'failedAt', Date.now());
+            await this.finalize(uuid, msgId);
         }
     }
-    async finalize(messageUuid, msgId, fromError = false) {
+    async finalize(messageUuid, msgId) {
         const timestamp = Date.now();
         const statusKey = this.keys.getJobStatusKey(messageUuid);
         const dataKey = this.keys.getJobDataKey(messageUuid);
         const throughputKey = this.keys.getThroughputKey(this.groupName, timestamp);
         const totalKey = this.keys.getTotalKey(this.groupName);
-        await this.redis.eval(lua_1.LUA_MARK_DONE, 6, statusKey, dataKey, this.streamName, this.groupName, throughputKey, totalKey, this.groupName, timestamp, msgId);
+        await this.redis.eval(lua_1.LUA_MARK_DONE, 5, statusKey, this.streamName, this.groupName, throughputKey, totalKey, this.groupName, timestamp, msgId);
     }
     consumerName() {
         return `${this.groupName}-${process.pid}-${this.consumerId}`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.1.8",
+  "version": "0.1.10",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {