@koala42/redis-highway 0.1.10 → 0.1.11

package/dist/batch-worker.js

@@ -84,11 +84,24 @@ class BatchWorker {
                     const [nextCursor, messages] = result;
                     cursor = nextCursor;
                     if (messages && messages.length > 0) {
-                        console.log(`[${this.groupName}] Recovered ${messages.length} stuck messages`);
-                        if (this.activeCount < this.concurrency) {
-                            continueClaiming = false;
+                        const pipeline = this.redis.pipeline();
+                        for (const msg of messages) {
+                            const entity = new stream_message_entity_1.StreamMessageEntity(msg);
+                            pipeline.xack(this.streamName, this.groupName, entity.streamMessageId);
+                            const statusKey = this.keys.getJobStatusKey(entity.messageUuid);
+                            const timestamp = Date.now();
+                            pipeline.eval(lua_1.LUA_FINALIZE_COMPLEX, 2, statusKey, this.streamName, this.groupName, timestamp, entity.streamMessageId);
+                            if (entity.retryCount < this.maxRetries) {
+                                pipeline.xadd(this.streamName, '*', 'id', entity.messageUuid, 'target', entity.routes.join(','), 'retryCount', entity.retryCount + 1, 'data', entity.data ? JSON.stringify(entity.data) : '');
+                                pipeline.hset(statusKey, '__target', entity.routes.length);
+                            }
+                            else {
+                                console.error(`[${this.groupName}] Job ${entity.messageUuid} run out of retries (stuck). Moving to DLQ`);
+                                pipeline.xadd(this.keys.getDlqStreamKey(), '*', 'id', entity.messageUuid, 'group', this.groupName, 'error', 'Stuck message recovered max retries', 'payload', entity.data ? JSON.stringify(entity.data) : 'MISSING', 'failedAt', Date.now());
+                                pipeline.del(statusKey);
+                            }
                         }
-                        this.spawnWorker(messages);
+                        await pipeline.exec();
                     }
                     else {
                         continueClaiming = false;
@@ -187,7 +200,6 @@ class BatchWorker {
         for (const message of messages) {
             if (message.routes.includes(this.groupName)) {
                 if (message.retryCount < this.maxRetries && message.data) {
-                    console.log(`[${this.groupName}] Retrying job ${message.messageUuid} attempt ${message.retryCount + 1}/${this.maxRetries}`);
                     const payloadString = JSON.stringify(message.data);
                     pipeline.xadd(this.streamName, '*', 'id', message.messageUuid, 'target', this.groupName, 'retryCount', message.retryCount + 1, 'data', payloadString);
                 }

package/dist/keys.d.ts

@@ -8,10 +8,6 @@ export declare class KeyManager {
      * And targets add their completed timestamps there
      */
     getJobStatusKey(id: string): string;
-    /**
-     * Job data contains the job payload
-     */
-    getJobDataKey(id: string): string;
     /**
      * Dead letter queue stream name
      */

package/dist/keys.js

@@ -16,12 +16,6 @@ class KeyManager {
     getJobStatusKey(id) {
         return `${this.streamName}:status:${id}`;
     }
-    /**
-     * Job data contains the job payload
-     */
-    getJobDataKey(id) {
-        return `${this.streamName}:data:${id}`;
-    }
     /**
      * Dead letter queue stream name
      */

package/dist/queue.spec.js

@@ -229,6 +229,11 @@ class TestWorker extends worker_1.Worker {
         const pending = await redis.xpending(streamName, groupName);
         // After processing, it should be ACKed, so pending count => 0 (if deleted)
         // or if finalize runs, it deletes the message entirely.
+        // Wait for cleanup (finalize runs after process)
+        await waitFor(async () => {
+            const len = await redis.xlen(streamName);
+            return len === 0;
+        }, 2000);
         const len = await redis.xlen(streamName);
         (0, vitest_1.expect)(len).toBe(0);
     });

package/dist/stream-message-entity.js

@@ -14,17 +14,7 @@ class StreamMessageEntity {
         this._messageUuid = this._fields['id'];
         this._routes = this._fields['target'].split(',');
         this._retryCount = parseInt(this._fields['retryCount'] || '0', 10);
-        try {
-            this._data = JSON.parse(this._fields['data']);
-        }
-        catch (e) {
-            // Handle corrupt or missing data gracefully
-            // We can set it to null (need to update type to T | null) or a dummy.
-            // Since strict T is expected, we might have to cast or throw controlled error.
-            // For now, let's assume T can be null-ish or cast. But getter says T.
-            // Let's coerce to {} as any to avoid crash, let validation downstream handle it.
-            this._data = {};
-        }
+        this._data = JSON.parse(this._fields['data']);
     }
     get data() {
         return this._data;

package/dist/worker.js

@@ -75,10 +75,25 @@ class Worker {
                     const [nextCursor, messages] = result;
                     cursor = nextCursor;
                     if (messages && messages.length > 0) {
-                        console.log(`[${this.groupName}] Recovered ${messages.length} stuck messages`);
+                        const pipeline = this.redis.pipeline();
                         for (const msg of messages) {
-                            this.spawnWorker(msg);
+                            const entity = new stream_message_entity_1.StreamMessageEntity(msg);
+                            pipeline.xack(this.streamName, this.groupName, entity.streamMessageId);
+                            const statusKey = this.keys.getJobStatusKey(entity.messageUuid);
+                            const timestamp = Date.now();
+                            pipeline.eval(lua_1.LUA_FINALIZE_COMPLEX, 2, statusKey, this.streamName, this.groupName, timestamp, entity.streamMessageId);
+                            if (entity.retryCount < this.MAX_RETRIES) {
+                                pipeline.xadd(this.streamName, '*', 'id', entity.messageUuid, 'target', entity.routes.join(','), 'retryCount', entity.retryCount + 1, 'data', entity.data ? JSON.stringify(entity.data) : '');
+                                pipeline.hset(statusKey, '__target', entity.routes.length);
+                            }
+                            else {
+                                console.error(`[${this.groupName}] Job ${entity.messageUuid} run outof retries (stuck). Moving to DLQ`);
+                                // DLQ
+                                pipeline.xadd(this.keys.getDlqStreamKey(), '*', 'id', entity.messageUuid, 'group', this.groupName, 'error', 'Stuck message recovered max retries', 'payload', entity.data ? JSON.stringify(entity.data) : '', 'failedAt', Date.now());
+                                pipeline.del(statusKey);
+                            }
                         }
+                        await pipeline.exec();
                     }
                     else {
                         continueClaiming = false;
@@ -144,7 +159,6 @@ class Worker {
         await this.redis.xack(this.streamName, this.groupName, msgId);
         const payloadString = payloadData ? JSON.stringify(payloadData) : '';
         if (currentRetries < this.MAX_RETRIES && payloadData) {
-            console.log(`[${this.groupName}] Retrying job ${uuid} (Attempt ${currentRetries + 1}/${this.MAX_RETRIES})`);
             const pipeline = this.redis.pipeline();
             pipeline.xadd(this.streamName, '*', 'id', uuid, 'target', this.groupName, 'retryCount', currentRetries + 1, 'data', payloadString);
             await pipeline.exec();
@@ -158,7 +172,6 @@ class Worker {
     async finalize(messageUuid, msgId) {
         const timestamp = Date.now();
         const statusKey = this.keys.getJobStatusKey(messageUuid);
-        const dataKey = this.keys.getJobDataKey(messageUuid);
         const throughputKey = this.keys.getThroughputKey(this.groupName, timestamp);
         const totalKey = this.keys.getTotalKey(this.groupName);
         await this.redis.eval(lua_1.LUA_MARK_DONE, 5, statusKey, this.streamName, this.groupName, throughputKey, totalKey, this.groupName, timestamp, msgId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {