@koala42/redis-highway 0.1.7 → 0.1.9

package/dist/batch-worker.d.ts

@@ -5,13 +5,19 @@ export declare abstract class BatchWorker<T extends Record<string, unknown>> {
     protected streamName: string;
     protected batchSize: number;
     protected concurrency: number;
+    protected maxFetchSize: number;
     protected maxRetries: number;
     protected blockTimeMs: number;
     private isRunning;
     private activeCount;
-    private readonly events;
     private keys;
-    constructor(redis: Redis, groupName: string, streamName: string, batchSize?: number, concurrency?: number, maxRetries?: number, blockTimeMs?: number);
+    private blockingRedis;
+    private readonly events;
+    private readonly consumerId;
+    constructor(redis: Redis, groupName: string, streamName: string, batchSize?: number, // How many jobs are passed to the process function (max)
+    concurrency?: number, // How many concurrent loops should run
+    maxFetchSize?: number, // How many jobs are fetched at once from redis stream
+    maxRetries?: number, blockTimeMs?: number);
     start(): Promise<void>;
     stop(): Promise<void>;
     private fetchLoop;

package/dist/batch-worker.js

@@ -5,18 +5,25 @@ const events_1 = require("events");
 const keys_1 = require("./keys");
 const stream_message_entity_1 = require("./stream-message-entity");
 const lua_1 = require("./lua");
+const uuid_1 = require("uuid");
 class BatchWorker {
-    constructor(redis, groupName, streamName, batchSize = 10, concurrency = 1, maxRetries = 3, blockTimeMs = 2000) {
+    constructor(redis, groupName, streamName, batchSize = 10, // How many jobs are passed to the process function (max)
+    concurrency = 1, // How many concurrent loops should run
+    maxFetchSize = 20, // How many jobs are fetched at once from redis stream
+    maxRetries = 3, blockTimeMs = 2000) {
         this.redis = redis;
         this.groupName = groupName;
         this.streamName = streamName;
         this.batchSize = batchSize;
         this.concurrency = concurrency;
+        this.maxFetchSize = maxFetchSize;
         this.maxRetries = maxRetries;
         this.blockTimeMs = blockTimeMs;
         this.isRunning = false;
         this.activeCount = 0;
+        this.blockingRedis = null;
         this.events = new events_1.EventEmitter();
+        this.consumerId = (0, uuid_1.v7)();
         if (batchSize < 1) {
             throw new Error('Batch size cannot be less then 0');
         }
@@ -36,14 +43,15 @@ class BatchWorker {
                 throw e;
             }
         }
+        this.blockingRedis = this.redis.duplicate();
         this.fetchLoop();
     }
-    // TODO: implement waiting for runnnig jobs
     async stop() {
-        return new Promise((resolve) => {
-            this.isRunning = false;
-            resolve();
-        });
+        this.isRunning = false;
+        this.events.emit('job_finished');
+        while (this.activeCount > 0) {
+            await new Promise((resolve) => setTimeout(resolve, 50));
+        }
     }
     async fetchLoop() {
         while (this.isRunning) {
@@ -52,9 +60,13 @@ class BatchWorker {
                 await new Promise((resolve) => this.events.once('job_finished', resolve));
                 continue;
             }
-            const itemsCount = freeSlots * this.batchSize;
+            const missingItemsCount = freeSlots * this.batchSize;
+            const itemsCount = missingItemsCount > this.maxFetchSize ? this.maxFetchSize : missingItemsCount;
             try {
-                const results = await this.redis.xreadgroup('GROUP', this.groupName, this.getConsumerName(), 'COUNT', itemsCount, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
+                if (!this.blockingRedis) {
+                    throw new Error('Blocking Redis connection missing');
+                }
+                const results = await this.blockingRedis.xreadgroup('GROUP', this.groupName, this.getConsumerName(), 'COUNT', itemsCount, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
                 if (!results) {
                     continue;
                 }
@@ -65,8 +77,10 @@ class BatchWorker {
                 }
             }
             catch (err) {
-                console.error(`[${this.groupName}] Fetch Error: `, err);
-                await new Promise((resolve) => setTimeout(resolve, 1000));
+                if (this.isRunning) { // Quicker grace shutdown
+                    console.error(`[${this.groupName}] Fetch Error: `, err);
+                    await new Promise((resolve) => setTimeout(resolve, 1000));
+                }
             }
         }
     }
@@ -132,19 +146,21 @@ class BatchWorker {
             }
             message.data = JSON.parse(data);
         });
-        const messagesData = messages.reduce((acc, current) => {
-            if (current.data) {
-                acc.push(current.data);
+        const messagesData = [];
+        const messagesToFinalize = [];
+        messages.forEach((message) => {
+            messagesToFinalize.push(message);
+            if (message.data) {
+                messagesData.push(message.data);
             }
-            return acc;
-        }, []);
+        });
         // TODO improve error handling
         if (!messagesData.length) {
             return;
         }
         try {
             await this.process(messagesData);
-            await this.finalize(messages);
+            await this.finalize(messagesToFinalize);
         }
         catch (err) {
             console.error(`[${this.groupName}] Jobs failed`, err);
@@ -180,19 +196,29 @@ class BatchWorker {
         }
     }
     async finalize(messages) {
+        if (messages.length === 0)
+            return;
         const pipeline = this.redis.pipeline();
-        for (const message of messages) {
-            const timestamp = Date.now();
-            const statusKey = this.keys.getJobStatusKey(message.messageUuid);
-            const dataKey = this.keys.getJobDataKey(message.messageUuid);
-            const throughputKey = this.keys.getThroughputKey(this.groupName, timestamp);
-            const totalKey = this.keys.getTotalKey(this.groupName);
-            pipeline.eval(lua_1.LUA_MARK_DONE, 6, statusKey, dataKey, this.streamName, this.groupName, throughputKey, totalKey, this.groupName, timestamp, message.streamMessageId);
+        const timestamp = Date.now();
+        const throughputKey = this.keys.getThroughputKey(this.groupName, timestamp);
+        const totalKey = this.keys.getTotalKey(this.groupName);
+        // 1. Batch xacks
+        const ids = messages.map(m => m.streamMessageId);
+        pipeline.xack(this.streamName, this.groupName, ...ids);
+        // 2. Batch metrics
+        pipeline.incrby(throughputKey, ids.length);
+        pipeline.expire(throughputKey, 86400);
+        pipeline.incrby(totalKey, ids.length);
+        // Lua scripts to only check if data should be deleted
+        for (const msg of messages) {
+            const statusKey = this.keys.getJobStatusKey(msg.messageUuid);
+            const dataKey = this.keys.getJobDataKey(msg.messageUuid);
+            pipeline.eval(lua_1.LUA_FINALIZE_COMPLEX, 3, statusKey, dataKey, this.streamName, this.groupName, timestamp, msg.streamMessageId);
         }
         await pipeline.exec();
     }
     getConsumerName() {
-        return `${this.groupName}-${process.pid}`;
+        return `${this.groupName}-${process.pid}-${this.consumerId}`;
     }
 }
 exports.BatchWorker = BatchWorker;

package/dist/lua.d.ts

@@ -1 +1,2 @@
 export declare const LUA_MARK_DONE = "\n-- KEYS[1] = status key status key for jog\n-- KEYS[2] = data key for job\n-- KEYS[3] = stream key\n-- KEYS[4] = group name\n-- KEYS[5] = metrics key\n-- KEYS[6] = total metrics key(persistent)\n\n-- ARGV[1] = route name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId - redis stream item ID\n\n-- 1. Ack the stream message\nredis.call('XACK', KEYS[3], KEYS[4], ARGV[3])\n\n-- 2. in status key mark the current route as done by saving timestamp\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 3. Increment throughput metric\nif KEYS[5] then\n    redis.call('INCR', KEYS[5])\n    redis.call('EXPIRE', KEYS[5], 86400)\nend\n\n-- 3.1 Increment Total Metric\nif KEYS[6] then\n    redis.call('INCR', KEYS[6])\nend\n\n-- 4. Check for completed routes\nlocal current_fields = redis.call('HLEN', KEYS[1])\n\n-- 5. Get the target completed routes\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 6. If completed routes is status hash length - 1 -> all were done and we can cleanup\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1], KEYS[2])\n    redis.call('XDEL', KEYS[3], ARGV[3])\n    return 1 -- Cleanup, DONE\nend\n\nreturn 0 -- Some routes are not done yet\n";
+export declare const LUA_FINALIZE_COMPLEX = "\n-- KEYS[1] = status key\n-- KEYS[2] = data key\n-- KEYS[3] = stream key\n-- ARGV[1] = group name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId\n\n-- 1. Update status\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 2. Check completions\nlocal current_fields = redis.call('HLEN', KEYS[1])\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 3. Cleanup if done\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1], KEYS[2])\n    redis.call('XDEL', KEYS[3], ARGV[3])\n    return 1\nend\n\nreturn 0\n";

package/dist/lua.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LUA_MARK_DONE = void 0;
+exports.LUA_FINALIZE_COMPLEX = exports.LUA_MARK_DONE = void 0;
 exports.LUA_MARK_DONE = `
 -- KEYS[1] = status key status key for jog
 -- KEYS[2] = data key for job
@@ -50,3 +50,32 @@ end
 
 return 0 -- Some routes are not done yet
 `;
+exports.LUA_FINALIZE_COMPLEX = `
+-- KEYS[1] = status key
+-- KEYS[2] = data key
+-- KEYS[3] = stream key
+-- ARGV[1] = group name
+-- ARGV[2] = timestamp
+-- ARGV[3] = msgId
+
+-- 1. Update status
+redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+
+-- 2. Check completions
+local current_fields = redis.call('HLEN', KEYS[1])
+local target_str = redis.call('HGET', KEYS[1], '__target')
+local target = tonumber(target_str)
+
+if not target then
+    return 0
+end
+
+-- 3. Cleanup if done
+if current_fields >= (target + 1) then
+    redis.call('DEL', KEYS[1], KEYS[2])
+    redis.call('XDEL', KEYS[3], ARGV[3])
+    return 1
+end
+
+return 0
+`;

package/dist/queue.spec.js

@@ -12,7 +12,7 @@ const uuid_1 = require("uuid");
 const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
 class TestWorker extends worker_1.Worker {
     constructor(redis, groupName, streamName, concurrency = 1, blockTimeMs = 100) {
-        super(redis, groupName, streamName, concurrency, blockTimeMs);
+        super(redis, groupName, streamName, concurrency, 3, blockTimeMs);
         this.processedCount = 0;
         this.lastProcessedId = null;
         this.shouldFail = false;
@@ -48,7 +48,7 @@ class TestWorker extends worker_1.Worker {
     });
     (0, vitest_1.afterEach)(async () => {
         for (const w of workers) {
-            w.stop();
+            await w.stop();
         }
         await new Promise(r => setTimeout(r, 500));
         // Cleanup Redis keys using the existing connection before closing

package/dist/worker.d.ts

@@ -10,13 +10,14 @@ export declare abstract class Worker<T extends Record<string, unknown>> {
     private activeCount;
     private readonly events;
     private keys;
+    private consumerId;
     constructor(redis: Redis, groupName: string, streamName: string, concurrency?: number, MAX_RETRIES?: number, blockTimeMs?: number);
     /**
      * Start worker
      * @returns
      */
     start(): Promise<void>;
-    stop(): void;
+    stop(): Promise<void>;
     private fetchLoop;
     private spawnWorker;
     private processInternal;

package/dist/worker.js

@@ -5,6 +5,7 @@ const events_1 = require("events");
 const lua_1 = require("./lua");
 const keys_1 = require("./keys");
 const stream_message_entity_1 = require("./stream-message-entity");
+const uuid_1 = require("uuid");
 class Worker {
     constructor(redis, groupName, streamName, concurrency = 1, MAX_RETRIES = 3, blockTimeMs = 2000) {
         this.redis = redis;
@@ -16,6 +17,7 @@ class Worker {
         this.isRunning = false;
         this.activeCount = 0;
         this.events = new events_1.EventEmitter();
+        this.consumerId = (0, uuid_1.v7)();
         this.events.setMaxListeners(100);
         this.keys = new keys_1.KeyManager(streamName);
     }
@@ -38,8 +40,13 @@ class Worker {
         }
         this.fetchLoop();
     }
-    stop() {
+    async stop() {
         this.isRunning = false;
+        this.events.emit('job_finished'); // Wake up fetch loop if it's waiting
+        // Wait for active jobs to finish
+        while (this.activeCount > 0) {
+            await new Promise(resolve => setTimeout(resolve, 50));
+        }
     }
     async fetchLoop() {
         while (this.isRunning) {
@@ -125,7 +132,7 @@ class Worker {
         await this.redis.eval(lua_1.LUA_MARK_DONE, 6, statusKey, dataKey, this.streamName, this.groupName, throughputKey, totalKey, this.groupName, timestamp, msgId);
     }
     consumerName() {
-        return `${this.groupName}-${process.pid}`;
+        return `${this.groupName}-${process.pid}-${this.consumerId}`;
     }
 }
 exports.Worker = Worker;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.1.7",
+  "version": "0.1.9",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {