@koala42/redis-highway 0.1.10 → 0.2.0

package/dist/src/base-worker.d.ts

@@ -0,0 +1,64 @@
+import { EventEmitter } from "events";
+import { KeyManager } from "./keys";
+import Redis from "ioredis";
+import { BaseWorkerControlOptions, BaseWorkerOptions, StreamMessage, XReadGroupResponse } from "./interfaces";
+import { StreamMessageEntity } from "./stream-message-entity";
+export declare abstract class BaseWorker<T extends Record<string, unknown>> {
+    protected redis: Redis;
+    protected _isRunning: boolean;
+    protected _activeCount: number;
+    protected readonly _consumerName: string;
+    protected readonly _events: EventEmitter<any>;
+    protected readonly _keys: KeyManager;
+    protected readonly _consumerId: string;
+    protected _blockingRedis: Redis;
+    protected readonly _groupName: string;
+    protected readonly _streamName: string;
+    protected readonly _concurrency: number;
+    protected readonly _maxRetries: number;
+    protected readonly _blockTimeMs: number;
+    protected readonly _claimIntervalMs: number;
+    protected readonly _minIdleTimeMs: number;
+    protected readonly _collectMetrics: boolean;
+    constructor(redis: Redis, options: BaseWorkerOptions, controlOptions: BaseWorkerControlOptions);
+    /**
+     * Start the worker process
+     * Starts fetch loop and auto claim loop
+     */
+    protected start(): Promise<void>;
+    /**
+     * Gracefully stops the worker
+     * Waits for any running jobs
+     */
+    protected stop(): Promise<void>;
+    /**
+     * Auto claim loop
+     * Checks which messages are read but not acked for longer than minIdleTimeMs (PEL)
+     * Acks them and based on retry policy either enqueues them again or moves to DLQ
+     */
+    protected _autoClaimLoop(): Promise<void>;
+    protected _handleFailure(messages: StreamMessageEntity<T>[], errorMessage: string): Promise<void>;
+    /**
+     * Helper methods
+     */
+    /**
+     * Read messages from stream
+     * @param count
+     * @returns XReadGroupResponse
+     */
+    protected _readGroup(count: number): Promise<XReadGroupResponse | null>;
+    /**
+     * Auto claim messages
+     * @param count
+     * @param cursor
+     * @returns
+     */
+    protected _autoClaimMessages(count: number, cursor: string): Promise<[string, StreamMessage[]] | null>;
+    /**
+     * Finalize messages
+     * @param messages
+     * @returns
+     */
+    protected _finalize(messages: StreamMessageEntity<T>[]): Promise<void>;
+    protected abstract _fetchLoop(): Promise<void>;
+}

package/dist/src/base-worker.js

@@ -0,0 +1,180 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseWorker = void 0;
+const events_1 = require("events");
+const keys_1 = require("./keys");
+const uuid_1 = require("uuid");
+const stream_message_entity_1 = require("./stream-message-entity");
+const lua_1 = require("./lua");
+class BaseWorker {
+    constructor(redis, options, controlOptions) {
+        this.redis = redis;
+        this._isRunning = false;
+        this._activeCount = 0;
+        this._events = new events_1.EventEmitter();
+        this._consumerId = (0, uuid_1.v7)();
+        this._events.setMaxListeners(100);
+        this._groupName = options.groupName;
+        this._streamName = options.streamName;
+        this._concurrency = options.concurrency;
+        this._maxRetries = controlOptions.maxRetries;
+        this._blockTimeMs = controlOptions.blockTimeMs;
+        this._claimIntervalMs = controlOptions.claimIntervalMs;
+        this._minIdleTimeMs = controlOptions.minIdleTimeMs;
+        this._collectMetrics = controlOptions.collectMetrics;
+        this._consumerName = `${this._groupName}-${this._consumerId}`;
+        this._keys = new keys_1.KeyManager(options.streamName);
+        this._blockingRedis = redis.duplicate();
+    }
+    /**
+     * Start the worker process
+     * Starts fetch loop and auto claim loop
+     */
+    async start() {
+        if (this._isRunning) {
+            return;
+        }
+        this._isRunning = true;
+        try {
+            await this.redis.xgroup('CREATE', this._streamName, this._groupName, '0', 'MKSTREAM');
+        }
+        catch (e) {
+            if (!e.message.includes('BUSYGROUP')) {
+                throw e;
+            }
+        }
+        this._fetchLoop();
+        this._autoClaimLoop();
+    }
+    /**
+     * Gracefully stops the worker
+     * Waits for any running jobs
+     */
+    async stop() {
+        this._isRunning = false;
+        this._events.emit('job_finished');
+        if (this._blockingRedis) {
+            await this._blockingRedis.quit().catch();
+        }
+        while (this._activeCount > 0) {
+            await new Promise((resolve) => setTimeout(resolve, 50));
+        }
+    }
+    /**
+     * Auto claim loop
+     * Checks which messages are read but not acked for longer than minIdleTimeMs (PEL)
+     * Acks them and based on retry policy either enqueues them again or moves to DLQ
+     */
+    async _autoClaimLoop() {
+        while (this._isRunning) {
+            try {
+                await new Promise(resolve => setTimeout(resolve, this._claimIntervalMs));
+                if (!this._isRunning) {
+                    break;
+                }
+                let cursor = '0-0';
+                let continueClaiming = true;
+                while (continueClaiming && this._isRunning) {
+                    const result = await this._autoClaimMessages(this._concurrency, cursor);
+                    if (!result) {
+                        continueClaiming = false;
+                        break;
+                    }
+                    const [nextCursor, msgs] = result;
+                    cursor = nextCursor;
+                    if (msgs && msgs.length > 0) {
+                        const messages = msgs.map((msg) => new stream_message_entity_1.StreamMessageEntity(msg));
+                        await this._handleFailure(messages, 'Stuck messages');
+                    }
+                    else {
+                        continueClaiming = false;
+                    }
+                    if (nextCursor === '0-0') {
+                        continueClaiming = false;
+                    }
+                }
+            }
+            catch (e) {
+                if (this._isRunning) {
+                    console.error(`[${this._groupName}] auto claim err:`, e.message);
+                }
+            }
+        }
+    }
+    async _handleFailure(messages, errorMessage) {
+        if (!messages.length) {
+            return;
+        }
+        const timestamp = Date.now();
+        const pipeline = this.redis.pipeline();
+        const messagesStreamIds = messages.map((message) => message.streamMessageId);
+        pipeline.xack(this._streamName, this._groupName, ...messagesStreamIds);
+        const messagesToDLQ = [];
+        for (const message of messages) {
+            if (message.retryCount < this._maxRetries) {
+                const newJobId = (0, uuid_1.v7)();
+                pipeline.xadd(this._streamName, '*', 'id', newJobId, 'target', this._groupName, 'retryCount', message.retryCount + 1, 'data', message.serializedData);
+                const newStatusKey = this._keys.getJobStatusKey(newJobId);
+                pipeline.hset(newStatusKey, '__target', 1);
+                const statusKey = this._keys.getJobStatusKey(message.messageUuid);
+                pipeline.eval(lua_1.LUA_FINALIZE, 2, statusKey, this._streamName, this._groupName, timestamp, message.streamMessageId);
+            }
+            else {
+                console.error(`[${this._groupName}] Job ${message.messageUuid} run out of retries. Moving to DLQ`);
+                messagesToDLQ.push(message);
+                // Add message to DLQ stream
+                pipeline.xadd(this._keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this._groupName, 'error', errorMessage, 'payload', message.serializedData, 'failedAt', Date.now());
+                const statusKey = this._keys.getJobStatusKey(message.messageUuid);
+                pipeline.eval(lua_1.LUA_FINALIZE, 2, statusKey, this._streamName, this._groupName, timestamp, message.streamMessageId);
+            }
+        }
+        await pipeline.exec();
+    }
+    /**
+     * Helper methods
+     */
+    /**
+     * Read messages from stream
+     * @param count
+     * @returns XReadGroupResponse
+     */
+    async _readGroup(count) {
+        return this._blockingRedis.xreadgroup('GROUP', this._groupName, this._consumerName, 'COUNT', count, 'BLOCK', this._blockTimeMs, 'STREAMS', this._streamName, '>');
+    }
+    /**
+     * Auto claim messages
+     * @param count
+     * @param cursor
+     * @returns
+     */
+    async _autoClaimMessages(count, cursor) {
+        return this.redis.xautoclaim(this._streamName, this._groupName, this._consumerName, this._minIdleTimeMs, cursor, 'COUNT', this._concurrency);
+    }
+    /**
+     * Finalize messages
+     * @param messages
+     * @returns
+     */
+    async _finalize(messages) {
+        if (messages.length === 0) {
+            return;
+        }
+        const pipeline = this.redis.pipeline();
+        const timestamp = Date.now();
+        const throughputKey = this._keys.getThroughputKey(this._groupName, timestamp);
+        const totalKey = this._keys.getTotalKey(this._groupName);
+        const ids = messages.map(m => m.streamMessageId);
+        pipeline.xack(this._streamName, this._groupName, ...ids);
+        if (this._collectMetrics) {
+            pipeline.incrby(throughputKey, ids.length);
+            pipeline.expire(throughputKey, 86400);
+            pipeline.incrby(totalKey, ids.length);
+        }
+        for (const msg of messages) {
+            const statusKey = this._keys.getJobStatusKey(msg.messageUuid);
+            pipeline.eval(lua_1.LUA_FINALIZE, 2, statusKey, this._streamName, this._groupName, timestamp, msg.streamMessageId);
+        }
+        await pipeline.exec();
+    }
+}
+exports.BaseWorker = BaseWorker;

package/dist/src/batch-worker.d.ts

@@ -0,0 +1,16 @@
+import Redis from "ioredis";
+import { BaseWorkerControlOptions, BatchWorkerOptions } from "./interfaces";
+import { BaseWorker } from "./base-worker";
+export declare abstract class BatchWorker<T extends Record<string, unknown>> extends BaseWorker<T> {
+    private readonly _batchSize;
+    private readonly _maxFetchCount;
+    constructor(redis: Redis, options: BatchWorkerOptions, controlOptions?: BaseWorkerControlOptions);
+    protected _fetchLoop(): Promise<void>;
+    /**
+     * Spawn worker for current processing
+     * @param messages
+     */
+    private spawnWorker;
+    private processInternal;
+    abstract process(data: T[]): Promise<void>;
+}

package/dist/src/batch-worker.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BatchWorker = void 0;
+const interfaces_1 = require("./interfaces");
+const stream_message_entity_1 = require("./stream-message-entity");
+const base_worker_1 = require("./base-worker");
+class BatchWorker extends base_worker_1.BaseWorker {
+    constructor(redis, options, controlOptions = interfaces_1.defaultBaseWorkerControlOptions) {
+        super(redis, options, controlOptions);
+        this._batchSize = options.batchSize;
+        this._maxFetchCount = options.maxFetchCount;
+        if (this._batchSize === 1) {
+            console.warn('Why would you create batch worker with batch size 1');
+        }
+        if (this._batchSize < 1) {
+            throw new Error('Batch size cannot be less then 0');
+        }
+    }
+    async _fetchLoop() {
+        while (this._isRunning) {
+            const freeSlots = this._concurrency - this._activeCount;
+            if (freeSlots <= 0) {
+                await new Promise((resolve) => this._events.once('job_finished', resolve));
+                continue;
+            }
+            const calculatedCount = freeSlots * this._batchSize;
+            const itemsCount = Math.min(calculatedCount, this._maxFetchCount);
+            try {
+                const results = await this._readGroup(itemsCount);
+                if (!results) {
+                    continue;
+                }
+                const messages = results[0][1];
+                for (let i = 0; i < messages.length; i += this._batchSize) {
+                    const chunk = messages.slice(i, i + this._batchSize);
+                    this.spawnWorker(chunk);
+                }
+            }
+            catch (err) {
+                if (this._isRunning) { // Quicker grace shutdown
+                    console.error(`[${this._groupName}] Fetch Error: `, err);
+                    await new Promise((resolve) => setTimeout(resolve, 1000));
+                }
+            }
+        }
+    }
+    /**
+     * Spawn worker for current processing
+     * @param messages
+     */
+    spawnWorker(messages) {
+        this._activeCount++;
+        this.processInternal(messages).finally(() => {
+            this._activeCount--;
+            this._events.emit('job_finished');
+        });
+    }
+    async processInternal(rawMessages) {
+        const allMessages = rawMessages.map((msg) => new stream_message_entity_1.StreamMessageEntity(msg));
+        const messages = []; // Messages to process
+        const ignoredMessages = []; // Messages to ignore
+        for (const message of allMessages) {
+            if (message.routes.includes(this._groupName)) {
+                messages.push(message);
+            }
+            else {
+                ignoredMessages.push(message);
+            }
+        }
+        // ACK ignored messages
+        if (ignoredMessages.length) {
+            const ignoredMessagesStreamIds = ignoredMessages.map((msg) => msg.streamMessageId);
+            await this.redis.xack(this._streamName, this._groupName, ...ignoredMessagesStreamIds);
+        }
+        if (!messages.length) {
+            return;
+        }
+        const messagesData = messages.map((msg) => msg.data);
+        try {
+            await this.process(messagesData);
+            await this._finalize(messages);
+        }
+        catch (err) {
+            console.error(`[${this._groupName}] Processing failed`, err);
+            await this._handleFailure(messages, err.message);
+        }
+    }
+}
+exports.BatchWorker = BatchWorker;

package/dist/src/interfaces.d.ts

@@ -0,0 +1,19 @@
+export type StreamMessage = [string, string[]];
+export type XReadGroupResponse = [string, StreamMessage[]][];
+export interface BaseWorkerOptions {
+    groupName: string;
+    streamName: string;
+    concurrency: number;
+}
+export interface BaseWorkerControlOptions {
+    maxRetries: number;
+    blockTimeMs: number;
+    claimIntervalMs: number;
+    minIdleTimeMs: number;
+    collectMetrics: boolean;
+}
+export declare const defaultBaseWorkerControlOptions: BaseWorkerControlOptions;
+export interface BatchWorkerOptions extends BaseWorkerOptions {
+    batchSize: number;
+    maxFetchCount: number;
+}

package/dist/src/interfaces.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultBaseWorkerControlOptions = void 0;
+exports.defaultBaseWorkerControlOptions = {
+    maxRetries: 3,
+    minIdleTimeMs: 120000,
+    blockTimeMs: 2000,
+    claimIntervalMs: 120000,
+    collectMetrics: true
+};

package/dist/{keys.d.ts → src/keys.d.ts}

@@ -8,10 +8,6 @@ export declare class KeyManager {
      * And targets add their completed timestamps there
      */
     getJobStatusKey(id: string): string;
-    /**
-     * Job data contains the job payload
-     */
-    getJobDataKey(id: string): string;
     /**
      * Dead letter queue stream name
      */

package/dist/{keys.js → src/keys.js}

@@ -16,12 +16,6 @@ class KeyManager {
     getJobStatusKey(id) {
         return `${this.streamName}:status:${id}`;
     }
-    /**
-     * Job data contains the job payload
-     */
-    getJobDataKey(id) {
-        return `${this.streamName}:data:${id}`;
-    }
     /**
      * Dead letter queue stream name
      */

package/dist/src/lua.d.ts

@@ -0,0 +1 @@
+export declare const LUA_FINALIZE = "\n-- KEYS[1] = status key\n-- KEYS[2] = stream key\n-- ARGV[1] = group name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId\n\n-- 1. Update status\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 2. Check completions\nlocal current_fields = redis.call('HLEN', KEYS[1])\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 3. Cleanup if done\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1])\n    redis.call('XDEL', KEYS[2], ARGV[3])\n    return 1\nend\n\nreturn 0\n";

package/dist/src/lua.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LUA_FINALIZE = void 0;
+exports.LUA_FINALIZE = `
+-- KEYS[1] = status key
+-- KEYS[2] = stream key
+-- ARGV[1] = group name
+-- ARGV[2] = timestamp
+-- ARGV[3] = msgId
+
+-- 1. Update status
+redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+
+-- 2. Check completions
+local current_fields = redis.call('HLEN', KEYS[1])
+local target_str = redis.call('HGET', KEYS[1], '__target')
+local target = tonumber(target_str)
+
+if not target then
+    return 0
+end
+
+-- 3. Cleanup if done
+if current_fields >= (target + 1) then
+    redis.call('DEL', KEYS[1])
+    redis.call('XDEL', KEYS[2], ARGV[3])
+    return 1
+end
+
+return 0
+`;

package/dist/{stream-message-entity.d.ts → src/stream-message-entity.d.ts}

@@ -7,8 +7,10 @@ export declare class StreamMessageEntity<T extends Record<string, unknown>> {
     private readonly _messageUuid;
     private readonly _retryCount;
     private readonly _data;
+    private readonly _rawData;
     constructor(message: StreamMessage);
     get data(): T;
+    get serializedData(): string;
     get streamMessageId(): string;
     get messageUuid(): string;
     get routes(): string[];

package/dist/{stream-message-entity.js → src/stream-message-entity.js}

@@ -14,21 +14,15 @@ class StreamMessageEntity {
         this._messageUuid = this._fields['id'];
         this._routes = this._fields['target'].split(',');
         this._retryCount = parseInt(this._fields['retryCount'] || '0', 10);
-        try {
-            this._data = JSON.parse(this._fields['data']);
-        }
-        catch (e) {
-            // Handle corrupt or missing data gracefully
-            // We can set it to null (need to update type to T | null) or a dummy.
-            // Since strict T is expected, we might have to cast or throw controlled error.
-            // For now, let's assume T can be null-ish or cast. But getter says T.
-            // Let's coerce to {} as any to avoid crash, let validation downstream handle it.
-            this._data = {};
-        }
+        this._data = JSON.parse(this._fields['data']);
+        this._rawData = this._fields['data'];
     }
     get data() {
         return this._data;
     }
+    get serializedData() {
+        return this._rawData;
+    }
     get streamMessageId() {
         return this._streamMessageId;
     }

package/dist/src/worker.d.ts

@@ -0,0 +1,24 @@
+import Redis from "ioredis";
+import { BaseWorkerOptions, BaseWorkerControlOptions } from "./interfaces";
+import { BaseWorker } from "./base-worker";
+export declare abstract class Worker<T extends Record<string, unknown>> extends BaseWorker<T> {
+    constructor(redis: Redis, options: BaseWorkerOptions, controlOptions?: BaseWorkerControlOptions);
+    /**
+     * Fetch loop (the main loop)
+     * Based on free slots (concurrency - active count) gets new messages
+     * Spawns worker process for them
+     */
+    protected _fetchLoop(): Promise<void>;
+    /**
+     * Spawns async background worker
+     * @param msg
+     */
+    private spawnWorker;
+    /**
+     * Process message
+     * @param msg
+     * @returns
+     */
+    private processInternal;
+    abstract process(data: T): Promise<void>;
+}

package/dist/src/worker.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Worker = void 0;
+const interfaces_1 = require("./interfaces");
+const stream_message_entity_1 = require("./stream-message-entity");
+const base_worker_1 = require("./base-worker");
+class Worker extends base_worker_1.BaseWorker {
+    constructor(redis, options, controlOptions = interfaces_1.defaultBaseWorkerControlOptions) {
+        super(redis, options, controlOptions);
+    }
+    /**
+     * Fetch loop (the main loop)
+     * Based on free slots (concurrency - active count) gets new messages
+     * Spawns worker process for them
+     */
+    async _fetchLoop() {
+        while (this._isRunning) {
+            const freeSlots = this._concurrency - this._activeCount;
+            if (freeSlots <= 0) {
+                await new Promise((resolve) => this._events.once('job_finished', resolve));
+                continue;
+            }
+            try {
+                const results = await this._readGroup(freeSlots);
+                if (results) {
+                    const messages = results[0][1];
+                    for (const msg of messages) {
+                        this.spawnWorker(msg);
+                    }
+                }
+            }
+            catch (err) {
+                console.error(`[${this._groupName}] Fetch Error:`, err);
+                await new Promise((resolve) => setTimeout(resolve, 1000));
+            }
+        }
+    }
+    /**
+     * Spawns async background worker
+     * @param msg
+     */
+    spawnWorker(msg) {
+        this._activeCount++;
+        this.processInternal(msg).finally(() => {
+            this._activeCount--;
+            this._events.emit('job_finished');
+        });
+    }
+    /**
+     * Process message
+     * @param msg
+     * @returns
+     */
+    async processInternal(msg) {
+        const streamMessage = new stream_message_entity_1.StreamMessageEntity(msg);
+        // Message was not targeted to this group
+        // We ACK it for this consumer group but we don't need to update any statuses, since its not for this group
+        if (!streamMessage.routes.includes(this._groupName)) {
+            await this.redis.xack(this._streamName, this._groupName, streamMessage.streamMessageId);
+            return;
+        }
+        try {
+            await this.process(streamMessage.data);
+            await this._finalize([streamMessage]);
+        }
+        catch (err) {
+            console.error(`[${this._groupName}] Job failed ${streamMessage.messageUuid}`, err);
+            await this._handleFailure([streamMessage], err.message);
+        }
+    }
+}
+exports.Worker = Worker;

package/dist/{queue.spec.js → test/queue.spec.js}

@@ -5,20 +5,36 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const vitest_1 = require("vitest");
 const ioredis_1 = __importDefault(require("ioredis"));
-const producer_1 = require("./producer");
-const worker_1 = require("./worker");
-const metrics_1 = require("./metrics");
+const producer_1 = require("../src/producer");
+const worker_1 = require("../src/worker");
+const metrics_1 = require("../src/metrics");
 const uuid_1 = require("uuid");
 const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
 class TestWorker extends worker_1.Worker {
     constructor(redis, groupName, streamName, concurrency = 1, maxRetries = 3, blockTimeMs = 100, claimIntervalMs = 60000, minIdleTimeMs = 300000) {
-        super(redis, groupName, streamName, concurrency, maxRetries, blockTimeMs, claimIntervalMs, minIdleTimeMs);
+        super(redis, {
+            groupName,
+            streamName,
+            concurrency
+        }, {
+            maxRetries,
+            blockTimeMs,
+            claimIntervalMs,
+            minIdleTimeMs,
+            collectMetrics: true
+        });
         this.processedCount = 0;
         this.lastProcessedId = null;
         this.shouldFail = false;
         this.failCount = 0;
         this.maxFails = 0;
     }
+    async start() {
+        return super.start();
+    }
+    async stop() {
+        return super.stop();
+    }
     async process(data) {
         if (this.shouldFail) {
             this.failCount++;
@@ -229,6 +245,11 @@ class TestWorker extends worker_1.Worker {
         const pending = await redis.xpending(streamName, groupName);
         // After processing, it should be ACKed, so pending count => 0 (if deleted)
         // or if finalize runs, it deletes the message entirely.
+        // Wait for cleanup (finalize runs after process)
+        await waitFor(async () => {
+            const len = await redis.xlen(streamName);
+            return len === 0;
+        }, 2000);
         const len = await redis.xlen(streamName);
         (0, vitest_1.expect)(len).toBe(0);
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.1.10",
+  "version": "0.2.0",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {