@koala42/redis-highway 0.1.10 → 0.2.0

package/dist/batch-worker.d.ts

@@ -1,40 +0,0 @@
-import Redis from "ioredis";
-export declare abstract class BatchWorker<T extends Record<string, unknown>> {
-    protected redis: Redis;
-    protected groupName: string;
-    protected streamName: string;
-    protected batchSize: number;
-    protected concurrency: number;
-    protected maxFetchSize: number;
-    protected maxRetries: number;
-    protected blockTimeMs: number;
-    protected maxFetchCount: number;
-    protected claimIntervalMs: number;
-    protected minIdleTimeMs: number;
-    private isRunning;
-    private activeCount;
-    private keys;
-    private blockingRedis;
-    private readonly events;
-    private readonly consumerId;
-    constructor(redis: Redis, groupName: string, streamName: string, batchSize?: number, // How many jobs are passed to the process function (max)
-    concurrency?: number, // How many concurrent loops should run
-    maxFetchSize?: number, // How many jobs are fetched at once from redis stream
-    maxRetries?: number, blockTimeMs?: number, // How long should the blocking redis wait for logs from stream
-    maxFetchCount?: number, claimIntervalMs?: number, // Check for stuck jobs every minute
-    minIdleTimeMs?: number);
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    private autoClaimLoop;
-    private fetchLoop;
-    /**
-     * Spawn worker for current processing
-     * @param messages
-     */
-    private spawnWorker;
-    private processInternal;
-    private handleFailure;
-    private finalize;
-    private getConsumerName;
-    abstract process(data: T[]): Promise<void>;
-}

package/dist/batch-worker.js

@@ -1,234 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BatchWorker = void 0;
-const events_1 = require("events");
-const keys_1 = require("./keys");
-const stream_message_entity_1 = require("./stream-message-entity");
-const lua_1 = require("./lua");
-const uuid_1 = require("uuid");
-class BatchWorker {
-    constructor(redis, groupName, streamName, batchSize = 10, // How many jobs are passed to the process function (max)
-    concurrency = 1, // How many concurrent loops should run
-    maxFetchSize = 20, // How many jobs are fetched at once from redis stream
-    maxRetries = 3, blockTimeMs = 2000, // How long should the blocking redis wait for logs from stream
-    maxFetchCount = 5000, claimIntervalMs = 60000, // Check for stuck jobs every minute
-    minIdleTimeMs = 120000) {
-        this.redis = redis;
-        this.groupName = groupName;
-        this.streamName = streamName;
-        this.batchSize = batchSize;
-        this.concurrency = concurrency;
-        this.maxFetchSize = maxFetchSize;
-        this.maxRetries = maxRetries;
-        this.blockTimeMs = blockTimeMs;
-        this.maxFetchCount = maxFetchCount;
-        this.claimIntervalMs = claimIntervalMs;
-        this.minIdleTimeMs = minIdleTimeMs;
-        this.isRunning = false;
-        this.activeCount = 0;
-        this.events = new events_1.EventEmitter();
-        this.consumerId = (0, uuid_1.v7)();
-        if (batchSize < 1) {
-            throw new Error('Batch size cannot be less then 0');
-        }
-        this.events.setMaxListeners(100);
-        this.keys = new keys_1.KeyManager(streamName);
-        this.blockingRedis = this.redis.duplicate();
-    }
-    async start() {
-        if (this.isRunning) {
-            return;
-        }
-        this.isRunning = true;
-        try {
-            await this.redis.xgroup('CREATE', this.streamName, this.groupName, '0', 'MKSTREAM');
-        }
-        catch (e) {
-            if (!e.message.includes('BUSYGROUP')) {
-                throw e;
-            }
-        }
-        this.fetchLoop();
-        this.autoClaimLoop();
-    }
-    async stop() {
-        this.isRunning = false;
-        this.events.emit('job_finished');
-        if (this.blockingRedis) {
-            try {
-                await this.blockingRedis.quit();
-            }
-            catch (e) {
-                // whatever
-            }
-        }
-        while (this.activeCount > 0) {
-            await new Promise((resolve) => setTimeout(resolve, 50));
-        }
-    }
-    async autoClaimLoop() {
-        while (this.isRunning) {
-            try {
-                await new Promise(resolve => setTimeout(resolve, this.claimIntervalMs));
-                if (!this.isRunning) {
-                    break;
-                }
-                let cursor = '0-0';
-                let continueClaiming = true;
-                while (continueClaiming && this.isRunning) {
-                    const result = await this.redis.xautoclaim(this.streamName, this.groupName, this.getConsumerName(), this.minIdleTimeMs, cursor, 'COUNT', this.batchSize);
-                    if (!result || !result.length) {
-                        continueClaiming = false;
-                        break;
-                    }
-                    const [nextCursor, messages] = result;
-                    cursor = nextCursor;
-                    if (messages && messages.length > 0) {
-                        console.log(`[${this.groupName}] Recovered ${messages.length} stuck messages`);
-                        if (this.activeCount < this.concurrency) {
-                            continueClaiming = false;
-                        }
-                        this.spawnWorker(messages);
-                    }
-                    else {
-                        continueClaiming = false;
-                    }
-                    if (nextCursor === '0-0') {
-                        continueClaiming = false;
-                    }
-                }
-            }
-            catch (e) {
-                if (this.isRunning) {
-                    console.error(`[${this.groupName}] Auto claim err:`, e.message);
-                }
-            }
-        }
-    }
-    async fetchLoop() {
-        while (this.isRunning) {
-            const freeSlots = this.concurrency - this.activeCount;
-            if (freeSlots <= 0) {
-                await new Promise((resolve) => this.events.once('job_finished', resolve));
-                continue;
-            }
-            const calculatedCount = freeSlots * this.batchSize;
-            const itemsCount = Math.min(calculatedCount, this.maxFetchCount);
-            try {
-                const results = await this.blockingRedis.xreadgroup('GROUP', this.groupName, this.getConsumerName(), 'COUNT', itemsCount, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
-                if (!results) {
-                    continue;
-                }
-                const messages = results[0][1];
-                for (let i = 0; i < messages.length; i += this.batchSize) {
-                    const chunk = messages.slice(i, i + this.batchSize);
-                    this.spawnWorker(chunk);
-                }
-            }
-            catch (err) {
-                if (this.isRunning) { // Quicker grace shutdown
-                    console.error(`[${this.groupName}] Fetch Error: `, err);
-                    await new Promise((resolve) => setTimeout(resolve, 1000));
-                }
-            }
-        }
-    }
-    /**
-     * Spawn worker for current processing
-     * @param messages
-     */
-    spawnWorker(messages) {
-        this.activeCount++;
-        this.processInternal(messages).finally(() => {
-            this.activeCount--;
-            this.events.emit('job_finished');
-        });
-    }
-    async processInternal(rawMessages) {
-        const allMessages = rawMessages.map((msg) => new stream_message_entity_1.StreamMessageEntity(msg));
-        const messages = []; // Messages to process
-        const ignoredMessages = []; // Messages to ignore
-        for (const message of allMessages) {
-            if (message.routes.includes(this.groupName)) {
-                messages.push(message);
-            }
-            else {
-                ignoredMessages.push(message);
-            }
-        }
-        // ACK ignored messages
-        if (ignoredMessages.length) {
-            const pipeline = this.redis.pipeline();
-            for (const ignoredMessage of ignoredMessages) {
-                pipeline.xack(this.streamName, this.groupName, ignoredMessage.streamMessageId);
-            }
-            await pipeline.exec();
-        }
-        if (!messages.length) {
-            return;
-        }
-        const messagesData = messages.map((msg) => msg.data);
-        try {
-            await this.process(messagesData);
-            await this.finalize(messages);
-        }
-        catch (err) {
-            console.error(`[${this.groupName}] Processing failed`, err);
-            await this.handleFailure(messages, err.message);
-        }
-    }
-    async handleFailure(messages, errorMessage) {
-        const pipeline = this.redis.pipeline();
-        // ack
-        for (const message of messages) {
-            pipeline.xack(this.streamName, this.groupName, message.streamMessageId);
-        }
-        const messagesToDlq = [];
-        for (const message of messages) {
-            if (message.routes.includes(this.groupName)) {
-                if (message.retryCount < this.maxRetries && message.data) {
-                    console.log(`[${this.groupName}] Retrying job ${message.messageUuid} attempt ${message.retryCount + 1}/${this.maxRetries}`);
-                    const payloadString = JSON.stringify(message.data);
-                    pipeline.xadd(this.streamName, '*', 'id', message.messageUuid, 'target', this.groupName, 'retryCount', message.retryCount + 1, 'data', payloadString);
-                }
-                else {
-                    console.error(`[${this.groupName}] Job ${message.messageUuid} run out of retries. Moving to DLQ`);
-                    messagesToDlq.push(message);
-                    pipeline.xadd(this.keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this.groupName, 'error', errorMessage, 'payload', message.data ? JSON.stringify(message.data) : 'MISSING', 'failedAt', Date.now());
-                }
-            }
-            else {
-                console.error(`[${this.groupName}] Job ${message.messageUuid} failed but not routed to this group. Moving to DLQ.`);
-                messagesToDlq.push(message);
-                pipeline.xadd(this.keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this.groupName, 'error', `Failed but not routed to ${this.groupName}: ${errorMessage}`, 'payload', JSON.stringify(message.data), 'failedAt', Date.now());
-            }
-        }
-        await pipeline.exec();
-        if (messagesToDlq.length > 0) {
-            await this.finalize(messagesToDlq);
-        }
-    }
-    async finalize(messages) {
-        if (messages.length === 0) {
-            return;
-        }
-        const pipeline = this.redis.pipeline();
-        const timestamp = Date.now();
-        const throughputKey = this.keys.getThroughputKey(this.groupName, timestamp);
-        const totalKey = this.keys.getTotalKey(this.groupName);
-        const ids = messages.map(m => m.streamMessageId);
-        pipeline.xack(this.streamName, this.groupName, ...ids);
-        pipeline.incrby(throughputKey, ids.length);
-        pipeline.expire(throughputKey, 86400);
-        pipeline.incrby(totalKey, ids.length);
-        for (const msg of messages) {
-            const statusKey = this.keys.getJobStatusKey(msg.messageUuid);
-            pipeline.eval(lua_1.LUA_FINALIZE_COMPLEX, 2, statusKey, this.streamName, this.groupName, timestamp, msg.streamMessageId);
-        }
-        await pipeline.exec();
-    }
-    getConsumerName() {
-        return `${this.groupName}-${process.pid}-${this.consumerId}`;
-    }
-}
-exports.BatchWorker = BatchWorker;

package/dist/batch-worker.spec.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/batch-worker.spec.js

@@ -1,124 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const vitest_1 = require("vitest");
-const ioredis_1 = __importDefault(require("ioredis"));
-const producer_1 = require("./producer");
-const batch_worker_1 = require("./batch-worker");
-const uuid_1 = require("uuid");
-const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
-class TestBatchWorker extends batch_worker_1.BatchWorker {
-    constructor(redis, groupName, streamName, batchSize = 10, concurrency = 1, maxRetries = 3, blockTimeMs = 100) {
-        // Fix argument order: batchSize, concurrency, maxFetchSize, maxRetries, blockTimeMs
-        super(redis, groupName, streamName, batchSize, concurrency, 20, maxRetries, blockTimeMs);
-        this.processedBatches = [];
-        this.shouldFail = false;
-        this.failCount = 0;
-        this.maxFails = 0;
-    }
-    async process(data) {
-        if (this.shouldFail) {
-            this.failCount++;
-            if (this.maxFails > 0 && this.failCount > this.maxFails) {
-                // Stop failing
-            }
-            else {
-                throw new Error("Simulated Batch Failure");
-            }
-        }
-        this.processedBatches.push(data);
-    }
-}
-(0, vitest_1.describe)('Batch Worker Integration', () => {
-    let redisProducer;
-    let redisWorker;
-    let producer;
-    let streamName;
-    let worker;
-    (0, vitest_1.beforeEach)(() => {
-        // Use separate connections to avoid XREAD BLOCK blocking the producer
-        redisProducer = new ioredis_1.default(REDIS_URL);
-        redisWorker = new ioredis_1.default(REDIS_URL);
-        streamName = `test-batch-queue-${(0, uuid_1.v7)()}`;
-        producer = new producer_1.Producer(redisProducer, streamName);
-    });
-    (0, vitest_1.afterEach)(async () => {
-        if (worker)
-            await worker.stop();
-        // Wait a bit
-        await new Promise(r => setTimeout(r, 100));
-        // Cleanup
-        if (redisProducer.status === 'ready') {
-            const keys = await redisProducer.keys(`${streamName}*`);
-            if (keys.length)
-                await redisProducer.del(...keys);
-        }
-        redisProducer.disconnect();
-        redisWorker.disconnect();
-    });
-    const waitFor = async (condition, timeout = 8000) => {
-        const start = Date.now();
-        while (Date.now() - start < timeout) {
-            if (await condition())
-                return true;
-            await new Promise(r => setTimeout(r, 100));
-        }
-        return false;
-    };
-    (0, vitest_1.it)('Should process a batch of messages', async () => {
-        worker = new TestBatchWorker(redisWorker, 'group-Batch', streamName, 5, 1);
-        // Push messages BEFORE starting worker to ensure a full batch is available
-        // BatchWorker uses '0' pointer for group creation, so it will pick up existing messages
-        for (let i = 0; i < 5; i++) {
-            await producer.push({ id: `msg-${i}` }, ['group-Batch']);
-        }
-        await worker.start();
-        await waitFor(() => worker.processedBatches.length > 0);
-        (0, vitest_1.expect)(worker.processedBatches.length).toBeGreaterThanOrEqual(1);
-        // It might still split it if redis pagination behaves weirdly, but usually it grabs Count
-        const allProcessed = worker.processedBatches.flat();
-        (0, vitest_1.expect)(allProcessed.length).toBe(5);
-        (0, vitest_1.expect)(worker.processedBatches[0].length).toBe(5);
-        (0, vitest_1.expect)(worker.processedBatches[0].map(j => j.id)).toEqual(['msg-0', 'msg-1', 'msg-2', 'msg-3', 'msg-4']);
-    });
-    (0, vitest_1.it)('Should process multiple batches', async () => {
-        worker = new TestBatchWorker(redisWorker, 'group-Batch-Multi', streamName, 2, 1);
-        for (let i = 0; i < 4; i++) {
-            await producer.push({ id: `msg-${i}` }, ['group-Batch-Multi']);
-        }
-        await worker.start();
-        await waitFor(() => worker.processedBatches.flat().length === 4);
-        (0, vitest_1.expect)(worker.processedBatches.length).toBe(2);
-    });
-    (0, vitest_1.it)('Should retry failed batches', async () => {
-        worker = new TestBatchWorker(redisWorker, 'group-Batch-Retry', streamName, 5, 1, 3);
-        worker.shouldFail = true;
-        worker.maxFails = 1;
-        await worker.start();
-        await producer.push({ id: 'retry-1' }, ['group-Batch-Retry']);
-        await waitFor(() => worker.processedBatches.length === 1);
-        (0, vitest_1.expect)(worker.failCount).toBeGreaterThanOrEqual(1);
-        (0, vitest_1.expect)(worker.processedBatches.length).toBe(1);
-        (0, vitest_1.expect)(worker.processedBatches[0][0].id).toBe('retry-1');
-    });
-    (0, vitest_1.it)('Should move to DLQ after max retries', async () => {
-        worker = new TestBatchWorker(redisWorker, 'group-Batch-DLQ', streamName, 1, 1, 2);
-        worker.shouldFail = true;
-        worker.maxFails = 10;
-        await worker.start();
-        const id = await producer.push({ id: 'dlq-1' }, ['group-Batch-DLQ']);
-        await waitFor(async () => {
-            const len = await redisProducer.xlen(`${streamName}:dlq`);
-            return len > 0;
-        }, 10000);
-        const dlqLen = await redisProducer.xlen(`${streamName}:dlq`);
-        (0, vitest_1.expect)(dlqLen).toBe(1);
-        const dlqMsgs = await redisProducer.xrange(`${streamName}:dlq`, '-', '+');
-        const body = dlqMsgs[0][1];
-        const payloadIdx = body.indexOf('payload');
-        const payload = body[payloadIdx + 1];
-        (0, vitest_1.expect)(JSON.parse(payload)).toEqual({ id: 'dlq-1' });
-    });
-});

package/dist/interfaces.d.ts

@@ -1,2 +0,0 @@
-export type StreamMessage = [string, string[]];
-export type XReadGroupResponse = [string, StreamMessage[]][];

package/dist/interfaces.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/lua.d.ts

@@ -1,2 +0,0 @@
-export declare const LUA_MARK_DONE = "\n-- KEYS[1] = status key status key for jog\n-- KEYS[2] = stream key\n-- KEYS[3] = group name\n-- KEYS[4] = metrics key\n-- KEYS[5] = total metrics key(persistent)\n\n-- ARGV[1] = route name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId - redis stream item ID\n\n-- 1 Ack the stream message\nredis.call('XACK', KEYS[2], KEYS[3], ARGV[3])\n\n-- 2 in status key mark the current route as done by saving timestamp\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 3 Increment throughput metric\nif KEYS[5] then\n    redis.call('INCR', KEYS[4])\n    redis.call('EXPIRE', KEYS[4], 86400)\nend\n\n-- 4 Increment Total Metric\nredis.call('INCR', KEYS[5])\n\n-- 5 Check for completed routes\nlocal current_fields = redis.call('HLEN', KEYS[1])\n\n-- 6 Get the target completed routes\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 7 If completed routes is status hash length - 1 -> all were done and we can cleanup\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1]) -- Only delete status key\n    redis.call('XDEL', KEYS[2], ARGV[3])\n    return 1 -- Cleanup, DONE\nend\n\nreturn 0 -- Some routes are not done yet\n";
-export declare const LUA_FINALIZE_COMPLEX = "\n-- KEYS[1] = status key\n-- KEYS[2] = stream key\n-- ARGV[1] = group name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId\n\n-- 1. Update status\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 2. Check completions\nlocal current_fields = redis.call('HLEN', KEYS[1])\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 3. Cleanup if done\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1])\n    redis.call('XDEL', KEYS[2], ARGV[3])\n    return 1\nend\n\nreturn 0\n";

package/dist/lua.js

@@ -1,77 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LUA_FINALIZE_COMPLEX = exports.LUA_MARK_DONE = void 0;
-exports.LUA_MARK_DONE = `
--- KEYS[1] = status key status key for jog
--- KEYS[2] = stream key
--- KEYS[3] = group name
--- KEYS[4] = metrics key
--- KEYS[5] = total metrics key(persistent)
-
--- ARGV[1] = route name
--- ARGV[2] = timestamp
--- ARGV[3] = msgId - redis stream item ID
-
--- 1 Ack the stream message
-redis.call('XACK', KEYS[2], KEYS[3], ARGV[3])
-
--- 2 in status key mark the current route as done by saving timestamp
-redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
-
--- 3 Increment throughput metric
-if KEYS[5] then
-    redis.call('INCR', KEYS[4])
-    redis.call('EXPIRE', KEYS[4], 86400)
-end
-
--- 4 Increment Total Metric
-redis.call('INCR', KEYS[5])
-
--- 5 Check for completed routes
-local current_fields = redis.call('HLEN', KEYS[1])
-
--- 6 Get the target completed routes
-local target_str = redis.call('HGET', KEYS[1], '__target')
-local target = tonumber(target_str)
-
-if not target then
-    return 0
-end
-
--- 7 If completed routes is status hash length - 1 -> all were done and we can cleanup
-if current_fields >= (target + 1) then
-    redis.call('DEL', KEYS[1]) -- Only delete status key
-    redis.call('XDEL', KEYS[2], ARGV[3])
-    return 1 -- Cleanup, DONE
-end
-
-return 0 -- Some routes are not done yet
-`;
-exports.LUA_FINALIZE_COMPLEX = `
--- KEYS[1] = status key
--- KEYS[2] = stream key
--- ARGV[1] = group name
--- ARGV[2] = timestamp
--- ARGV[3] = msgId
-
--- 1. Update status
-redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
-
--- 2. Check completions
-local current_fields = redis.call('HLEN', KEYS[1])
-local target_str = redis.call('HGET', KEYS[1], '__target')
-local target = tonumber(target_str)
-
-if not target then
-    return 0
-end
-
--- 3. Cleanup if done
-if current_fields >= (target + 1) then
-    redis.call('DEL', KEYS[1])
-    redis.call('XDEL', KEYS[2], ARGV[3])
-    return 1
-end
-
-return 0
-`;

package/dist/worker.d.ts

@@ -1,32 +0,0 @@
-import Redis from "ioredis";
-export declare abstract class Worker<T extends Record<string, unknown>> {
-    protected redis: Redis;
-    protected groupName: string;
-    protected streamName: string;
-    protected concurrency: number;
-    protected MAX_RETRIES: number;
-    protected blockTimeMs: number;
-    protected claimIntervalMs: number;
-    protected minIdleTimeMs: number;
-    private isRunning;
-    private activeCount;
-    private readonly events;
-    private keys;
-    private consumerId;
-    private blockingRedis;
-    constructor(redis: Redis, groupName: string, streamName: string, concurrency?: number, MAX_RETRIES?: number, blockTimeMs?: number, claimIntervalMs?: number, minIdleTimeMs?: number);
-    /**
-     * Start worker
-     * @returns
-     */
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    private autoClaimLoop;
-    private fetchLoop;
-    private spawnWorker;
-    private processInternal;
-    private handleFailure;
-    private finalize;
-    private consumerName;
-    abstract process(data: T): Promise<void>;
-}