@koala42/redis-highway 0.1.10 → 0.2.0

package/dist/worker.js

@@ -1,170 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Worker = void 0;
-const events_1 = require("events");
-const lua_1 = require("./lua");
-const keys_1 = require("./keys");
-const stream_message_entity_1 = require("./stream-message-entity");
-const uuid_1 = require("uuid");
-class Worker {
-    constructor(redis, groupName, streamName, concurrency = 1, MAX_RETRIES = 3, blockTimeMs = 2000, claimIntervalMs = 60000, minIdleTimeMs = 300000) {
-        this.redis = redis;
-        this.groupName = groupName;
-        this.streamName = streamName;
-        this.concurrency = concurrency;
-        this.MAX_RETRIES = MAX_RETRIES;
-        this.blockTimeMs = blockTimeMs;
-        this.claimIntervalMs = claimIntervalMs;
-        this.minIdleTimeMs = minIdleTimeMs;
-        this.isRunning = false;
-        this.activeCount = 0;
-        this.events = new events_1.EventEmitter();
-        this.consumerId = (0, uuid_1.v7)();
-        this.events.setMaxListeners(100);
-        this.keys = new keys_1.KeyManager(streamName);
-        this.blockingRedis = this.redis.duplicate();
-    }
-    /**
-     * Start worker
-     * @returns
-     */
-    async start() {
-        if (this.isRunning) {
-            return;
-        }
-        this.isRunning = true;
-        try {
-            await this.redis.xgroup('CREATE', this.streamName, this.groupName, '0', 'MKSTREAM');
-        }
-        catch (e) {
-            if (!e.message.includes('BUSYGROUP')) {
-                throw e;
-            }
-        }
-        this.fetchLoop();
-        this.autoClaimLoop();
-    }
-    async stop() {
-        this.isRunning = false;
-        this.events.emit('job_finished');
-        if (this.blockingRedis) {
-            try {
-                await this.blockingRedis.quit();
-            }
-            catch (e) { }
-        }
-        while (this.activeCount > 0) {
-            await new Promise(resolve => setTimeout(resolve, 50));
-        }
-    }
-    async autoClaimLoop() {
-        while (this.isRunning) {
-            try {
-                await new Promise(resolve => setTimeout(resolve, this.claimIntervalMs));
-                if (!this.isRunning) {
-                    break;
-                }
-                let cursor = '0-0';
-                let continueClaiming = true;
-                while (continueClaiming && this.isRunning) {
-                    const result = await this.redis.xautoclaim(this.streamName, this.groupName, this.consumerName(), this.minIdleTimeMs, cursor, 'COUNT', this.concurrency);
-                    if (!result) {
-                        continueClaiming = false;
-                        break;
-                    }
-                    const [nextCursor, messages] = result;
-                    cursor = nextCursor;
-                    if (messages && messages.length > 0) {
-                        console.log(`[${this.groupName}] Recovered ${messages.length} stuck messages`);
-                        for (const msg of messages) {
-                            this.spawnWorker(msg);
-                        }
-                    }
-                    else {
-                        continueClaiming = false;
-                    }
-                    if (nextCursor === '0-0') {
-                        continueClaiming = false;
-                    }
-                }
-            }
-            catch (e) {
-                if (this.isRunning) {
-                    console.error(`[${this.groupName}] auto claim err:`, e.message);
-                }
-            }
-        }
-    }
-    async fetchLoop() {
-        while (this.isRunning) {
-            const freeSlots = this.concurrency - this.activeCount;
-            if (freeSlots <= 0) {
-                await new Promise((resolve) => this.events.once('job_finished', resolve));
-                continue;
-            }
-            try {
-                const results = await this.blockingRedis.xreadgroup('GROUP', this.groupName, this.consumerName(), 'COUNT', freeSlots, 'BLOCK', this.blockTimeMs, 'STREAMS', this.streamName, '>');
-                if (results) {
-                    const messages = results[0][1];
-                    for (const msg of messages) {
-                        this.spawnWorker(msg);
-                    }
-                }
-            }
-            catch (err) {
-                console.error(`[${this.groupName}] Fetch Error:`, err);
-                await new Promise((resolve) => setTimeout(resolve, 1000));
-            }
-        }
-    }
-    spawnWorker(msg) {
-        this.activeCount++;
-        this.processInternal(msg).finally(() => {
-            this.activeCount--;
-            this.events.emit('job_finished');
-        });
-    }
-    async processInternal(msg) {
-        const streamMessage = new stream_message_entity_1.StreamMessageEntity(msg);
-        if (!streamMessage.routes.includes(this.groupName)) {
-            await this.redis.xack(this.streamName, this.groupName, streamMessage.streamMessageId);
-            return;
-        }
-        try {
-            await this.process(streamMessage.data);
-            await this.finalize(streamMessage.messageUuid, streamMessage.streamMessageId);
-        }
-        catch (err) {
-            console.error(`[${this.groupName}] Job failed ${streamMessage.messageUuid}`, err);
-            await this.handleFailure(streamMessage.messageUuid, streamMessage.streamMessageId, streamMessage.retryCount, err.message, streamMessage.data);
-        }
-    }
-    async handleFailure(uuid, msgId, currentRetries, errorMsg, payloadData) {
-        // Ack
-        await this.redis.xack(this.streamName, this.groupName, msgId);
-        const payloadString = payloadData ? JSON.stringify(payloadData) : '';
-        if (currentRetries < this.MAX_RETRIES && payloadData) {
-            console.log(`[${this.groupName}] Retrying job ${uuid} (Attempt ${currentRetries + 1}/${this.MAX_RETRIES})`);
-            const pipeline = this.redis.pipeline();
-            pipeline.xadd(this.streamName, '*', 'id', uuid, 'target', this.groupName, 'retryCount', currentRetries + 1, 'data', payloadString);
-            await pipeline.exec();
-        }
-        else {
-            console.error(`[${this.groupName}] Job ${uuid} run outof retries. Moving to DLQ`);
-            await this.redis.xadd(this.keys.getDlqStreamKey(), '*', 'id', uuid, 'group', this.groupName, 'error', errorMsg, 'payload', payloadString, 'failedAt', Date.now());
-            await this.finalize(uuid, msgId);
-        }
-    }
-    async finalize(messageUuid, msgId) {
-        const timestamp = Date.now();
-        const statusKey = this.keys.getJobStatusKey(messageUuid);
-        const dataKey = this.keys.getJobDataKey(messageUuid);
-        const throughputKey = this.keys.getThroughputKey(this.groupName, timestamp);
-        const totalKey = this.keys.getTotalKey(this.groupName);
-        await this.redis.eval(lua_1.LUA_MARK_DONE, 5, statusKey, this.streamName, this.groupName, throughputKey, totalKey, this.groupName, timestamp, msgId);
-    }
-    consumerName() {
-        return `${this.groupName}-${process.pid}-${this.consumerId}`;
-    }
-}
-exports.Worker = Worker;