@koala42/redis-highway 0.1.8 → 0.1.9

package/dist/batch-worker.d.ts

@@ -5,14 +5,19 @@ export declare abstract class BatchWorker<T extends Record<string, unknown>> {
     protected streamName: string;
     protected batchSize: number;
     protected concurrency: number;
+    protected maxFetchSize: number;
     protected maxRetries: number;
     protected blockTimeMs: number;
     private isRunning;
     private activeCount;
-    private readonly events;
     private keys;
+    private blockingRedis;
+    private readonly events;
     private readonly consumerId;
-    constructor(redis: Redis, groupName: string, streamName: string, batchSize?: number, concurrency?: number, maxRetries?: number, blockTimeMs?: number);
+    constructor(redis: Redis, groupName: string, streamName: string, batchSize?: number, // How many jobs are passed to the process function (max)
+    concurrency?: number, // How many concurrent loops should run
+    maxFetchSize?: number, // How many jobs are fetched at once from redis stream
+    maxRetries?: number, blockTimeMs?: number);
     start(): Promise<void>;
     stop(): Promise<void>;
     private fetchLoop;

package/dist/batch-worker.js

@@ -7,16 +7,21 @@ const stream_message_entity_1 = require("./stream-message-entity");
 const lua_1 = require("./lua");
 const uuid_1 = require("uuid");
 class BatchWorker {
-    constructor(redis, groupName, streamName, batchSize = 10, concurrency = 1, maxRetries = 3, blockTimeMs = 2000) {
+    constructor(redis, groupName, streamName, batchSize = 10, // How many jobs are passed to the process function (max)
+    concurrency = 1, // How many concurrent loops should run
+    maxFetchSize = 20, // How many jobs are fetched at once from redis stream
+    maxRetries = 3, blockTimeMs = 2000) {
         this.redis = redis;
         this.groupName = groupName;
         this.streamName = streamName;
         this.batchSize = batchSize;
         this.concurrency = concurrency;
+        this.maxFetchSize = maxFetchSize;
         this.maxRetries = maxRetries;
         this.blockTimeMs = blockTimeMs;
         this.isRunning = false;
         this.activeCount = 0;
+        this.blockingRedis = null;
         this.events = new events_1.EventEmitter();
         this.consumerId = (0, uuid_1.v7)();
         if (batchSize < 1) {
@@ -38,6 +43,7 @@ class BatchWorker {
                 throw e;
             }
         }
+        this.blockingRedis = this.redis.duplicate();
         this.fetchLoop();
     }
     async stop() {
@@ -54,9 +60,13 @@ class BatchWorker {
                 await new Promise((resolve) => this.events.once('job_finished', resolve));
                 continue;
             }
-            const itemsCount = freeSlots * this.batchSize;
+            const missingItemsCount = freeSlots * this.batchSize;
+            const itemsCount = missingItemsCount > this.maxFetchSize ? this.maxFetchSize : missingItemsCount;
             try {
-                const results = await this.redis.xreadgroup('GROUP', this.groupName, this.getConsumerName(), 'COUNT', itemsCount, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
+                if (!this.blockingRedis) {
+                    throw new Error('Blocking Redis connection missing');
+                }
+                const results = await this.blockingRedis.xreadgroup('GROUP', this.groupName, this.getConsumerName(), 'COUNT', itemsCount, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
                 if (!results) {
                     continue;
                 }
@@ -67,8 +77,10 @@ class BatchWorker {
                 }
             }
             catch (err) {
-                console.error(`[${this.groupName}] Fetch Error: `, err);
-                await new Promise((resolve) => setTimeout(resolve, 1000));
+                if (this.isRunning) { // Quicker grace shutdown
+                    console.error(`[${this.groupName}] Fetch Error: `, err);
+                    await new Promise((resolve) => setTimeout(resolve, 1000));
+                }
             }
         }
     }
@@ -201,9 +213,7 @@ class BatchWorker {
         for (const msg of messages) {
             const statusKey = this.keys.getJobStatusKey(msg.messageUuid);
             const dataKey = this.keys.getJobDataKey(msg.messageUuid);
-            pipeline.eval(lua_1.LUA_FINALIZE_COMPLEX, 3, statusKey, dataKey, this.streamName, // Keys
-            this.groupName, timestamp, msg.streamMessageId // args
-            );
+            pipeline.eval(lua_1.LUA_FINALIZE_COMPLEX, 3, statusKey, dataKey, this.streamName, this.groupName, timestamp, msg.streamMessageId);
         }
         await pipeline.exec();
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.1.8",
+  "version": "0.1.9",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {