@koala42/redis-highway 0.2.9 → 0.2.11

package/dist/{src/keys.d.ts → keys.d.ts}

@@ -14,8 +14,9 @@ export declare class KeyManager {
     getDlqStreamKey(): string;
     /**
      * Metrics for storing throughput
+     * @param current - if true, takes throughput in the running minute, if false, takes -1 minute for closed throughput bucket
      */
-    getThroughputKey(groupName: string, timestamp: number): string;
+    getThroughputKey(groupName: string, timestamp: number, current?: boolean): string;
     /**
      * Metrics - retries key
      */

package/dist/{src/keys.js → keys.js}

@@ -24,10 +24,11 @@ class KeyManager {
     }
     /**
      * Metrics for storing throughput
+     * @param current - if true, takes throughput in the running minute, if false, takes -1 minute for closed throughput bucket
      */
-    getThroughputKey(groupName, timestamp) {
+    getThroughputKey(groupName, timestamp, current = true) {
         const minute = Math.floor(timestamp / 60000) * 60000;
-        return `metrics:throughput:${this.streamName}:${groupName}:${minute}`;
+        return `metrics:throughput:${this.streamName}:${groupName}:${current ? minute : minute - 60000}`;
     }
     /**
      * Metrics - retries key

package/dist/{src/metrics.d.ts → metrics.d.ts}

@@ -13,12 +13,12 @@ export declare class Metrics {
      * Get current metrics for the queue
      * @param groupNames - List of consumer groups to fetch throughput for
      */
-    getMetrics(groupNames: string[]): Promise<QueueMetrics>;
+    getMetrics(groupNames: string[], current?: boolean): Promise<QueueMetrics>;
     /**
      * Get prometheus compatible metrics
      * @param groupNames target group names for throughput metrics
      * @param prefix - export prefix
      * @returns metrics as string
      */
-    getPrometheusMetrics(groupNames: string[], prefix?: string): Promise<string>;
+    getPrometheusMetrics(groupNames: string[], prefix?: string, current?: boolean): Promise<string>;
 }

package/dist/{src/metrics.js → metrics.js}

@@ -12,13 +12,13 @@ class Metrics {
      * Get current metrics for the queue
      * @param groupNames - List of consumer groups to fetch throughput for
      */
-    async getMetrics(groupNames) {
+    async getMetrics(groupNames, current = false) {
         const pipeline = this.redis.pipeline();
         pipeline.xlen(this.streamName);
         pipeline.xlen(this.keys.getDlqStreamKey());
         const timestamp = Date.now();
         groupNames.forEach(group => {
-            pipeline.get(this.keys.getThroughputKey(group, timestamp));
+            pipeline.get(this.keys.getThroughputKey(group, timestamp, current));
         });
         const results = await pipeline.exec();
         if (!results) {
@@ -49,13 +49,13 @@ class Metrics {
      * @param prefix - export prefix
      * @returns metrics as string
      */
-    async getPrometheusMetrics(groupNames, prefix = 'redis_highway_queue') {
+    async getPrometheusMetrics(groupNames, prefix = 'redis_highway_queue', current = false) {
         const pipeline = this.redis.pipeline();
         pipeline.xlen(this.streamName);
         pipeline.xlen(this.keys.getDlqStreamKey());
         const timestamp = Date.now();
         groupNames.forEach(group => {
-            pipeline.get(this.keys.getThroughputKey(group, timestamp));
+            pipeline.get(this.keys.getThroughputKey(group, timestamp, current));
             pipeline.get(this.keys.getTotalKey(group));
             pipeline.get(this.keys.getRetriesKey(group, timestamp));
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.2.9",
+  "version": "0.2.11",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {
@@ -10,19 +10,17 @@
   },
   "repository": {
     "url": "https://github.com/Koala42/redis-highway",
-    "type": "github",
-    "directory": "packages/redis-highway"
+    "type": "git"
   },
   "type": "commonjs",
-  "main": "dist/src/index.js",
-  "types": "dist/src/index.d.ts",
+  "main": "dist/index.js",
+  "types": "dist/ndex.d.ts",
   "files": [
     "dist"
   ],
   "scripts": {
     "clean": "rimraf dist",
     "test": "vitest run test",
-    "test:all": "vitest run test",
     "build": "npm run clean && tsc",
     "prepublish": "npm run build"
   },
@@ -42,4 +40,4 @@
     "typescript": "^5.9.3",
     "vitest": "^4.0.16"
   }
-}
+}

package/dist/test/queue.spec.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/test/queue.spec.js

@@ -1,256 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const vitest_1 = require("vitest");
-const ioredis_1 = __importDefault(require("ioredis"));
-const producer_1 = require("../src/producer");
-const worker_1 = require("../src/worker");
-const metrics_1 = require("../src/metrics");
-const uuid_1 = require("uuid");
-const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
-class TestWorker extends worker_1.Worker {
-    constructor(redis, groupName, streamName, concurrency = 1, maxRetries = 3, blockTimeMs = 100, claimIntervalMs = 60000, minIdleTimeMs = 300000) {
-        super(redis, {
-            groupName,
-            streamName,
-            concurrency
-        }, {
-            maxRetries,
-            blockTimeMs,
-            claimIntervalMs,
-            minIdleTimeMs,
-            collectMetrics: true
-        });
-        this.processedCount = 0;
-        this.lastProcessedId = null;
-        this.shouldFail = false;
-        this.failCount = 0;
-        this.maxFails = 0;
-    }
-    async start() {
-        return super.start();
-    }
-    async stop() {
-        return super.stop();
-    }
-    async process(data) {
-        if (this.shouldFail) {
-            this.failCount++;
-            if (this.maxFails > 0 && this.failCount > this.maxFails) {
-                // Stop failing after maxFails
-            }
-            else {
-                throw new Error("Simulated Failure");
-            }
-        }
-        this.processedCount++;
-        if (data && data.id) {
-            this.lastProcessedId = data.id;
-        }
-    }
-}
-(0, vitest_1.describe)('Redis Queue Integration', () => {
-    let redis;
-    let producer;
-    let streamName;
-    let workers = [];
-    (0, vitest_1.beforeEach)(() => {
-        redis = new ioredis_1.default(REDIS_URL);
-        streamName = `test-queue-${(0, uuid_1.v7)()}`;
-        producer = new producer_1.Producer(redis, streamName);
-        workers = [];
-    });
-    (0, vitest_1.afterEach)(async () => {
-        for (const w of workers) {
-            await w.stop();
-        }
-        await new Promise(r => setTimeout(r, 500));
-        // Cleanup Redis keys using the existing connection before closing
-        if (redis.status === 'ready') {
-            const keys = await redis.keys(`${streamName}*`);
-            if (keys.length)
-                await redis.del(...keys);
-        }
-        redis.disconnect();
-    });
-    const waitFor = async (condition, timeout = 5000) => {
-        const start = Date.now();
-        while (Date.now() - start < timeout) {
-            if (await condition())
-                return true;
-            await new Promise(r => setTimeout(r, 100));
-        }
-        return false;
-    };
-    (0, vitest_1.describe)('Core Functionality', () => {
-        (0, vitest_1.it)('Should deliver message to all target groups', async () => {
-            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
-            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 3, 100);
-            workers.push(w1, w2);
-            await w1.start();
-            await w2.start();
-            const id = await producer.push({ id: 'msg-1' }, ['group-A', 'group-B']);
-            await waitFor(() => w1.processedCount === 1 && w2.processedCount === 1);
-            (0, vitest_1.expect)(w1.processedCount).toBe(1);
-            (0, vitest_1.expect)(w2.processedCount).toBe(1);
-            // Test cleanup keys (but not explicit XDEL here yet, checking keys gone)
-            const statusKey = `${streamName}:status:${id}`;
-            const dataKey = `${streamName}:data:${id}`;
-            (0, vitest_1.expect)(await redis.exists(statusKey)).toBe(0);
-            (0, vitest_1.expect)(await redis.exists(dataKey)).toBe(0);
-        });
-        (0, vitest_1.it)('Should only deliver to targeted groups', async () => {
-            const wA = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
-            const wB = new TestWorker(redis, 'group-B', streamName, 1, 3, 100);
-            workers.push(wA, wB);
-            await wA.start();
-            await wB.start();
-            await producer.push({ id: 'msg-only-a' }, ['group-A']);
-            await waitFor(() => wA.processedCount === 1);
-            (0, vitest_1.expect)(wA.processedCount).toBe(1);
-            (0, vitest_1.expect)(wB.processedCount).toBe(0);
-        });
-        (0, vitest_1.it)('Should retry only the failed group', async () => {
-            const wOk = new TestWorker(redis, 'group-Ok', streamName, 1, 3, 100);
-            const wFail = new TestWorker(redis, 'group-Fail', streamName, 1, 3, 100);
-            wFail.shouldFail = true;
-            wFail.maxFails = 1; // Fail once, then succeed
-            workers.push(wOk, wFail);
-            await wOk.start();
-            await wFail.start();
-            await producer.push({ id: 'retry-test' }, ['group-Ok', 'group-Fail']);
-            // Wait for wOk to finish and wFail to try at least twice (fail + success)
-            await waitFor(() => wOk.processedCount === 1 && wFail.processedCount === 1, 8000);
-            (0, vitest_1.expect)(wOk.processedCount).toBe(1); // Processed once
-            (0, vitest_1.expect)(wFail.failCount).toBeGreaterThanOrEqual(1); // Failed at least once
-            (0, vitest_1.expect)(wFail.processedCount).toBe(1); // Eventually succeeded
-            (0, vitest_1.expect)(wOk.processedCount).toBe(1); // wOk should NOT process the retry
-        });
-        (0, vitest_1.it)('Should move to DLQ after max retries', async () => {
-            const wDead = new TestWorker(redis, 'group-Dead', streamName, 1, 3, 100);
-            wDead.shouldFail = true;
-            wDead.maxFails = 10; // Fail forever (more than max retries which is 3)
-            workers.push(wDead);
-            await wDead.start();
-            const id = await producer.push({ id: 'dlq-test' }, ['group-Dead']);
-            await waitFor(async () => {
-                const len = await redis.xlen(`${streamName}:dlq`);
-                return len > 0;
-            }, 10000);
-            const dlqLen = await redis.xlen(`${streamName}:dlq`);
-            (0, vitest_1.expect)(dlqLen).toBe(1);
-            (0, vitest_1.expect)(await redis.exists(`${streamName}:status:${id}`)).toBe(0);
-        });
-    });
-    (0, vitest_1.describe)('Metrics & Monitoring', () => {
-        (0, vitest_1.it)('Should track throughput and queue size', async () => {
-            const w = new TestWorker(redis, 'group-Metrics', streamName, 1, 3, 100);
-            const metricsService = new metrics_1.Metrics(redis, streamName);
-            workers.push(w);
-            await w.start();
-            let metrics = await metricsService.getMetrics(['group-Metrics']);
-            (0, vitest_1.expect)(metrics.dlqLength).toBe(0);
-            const id = await producer.push({ id: 'metrics-1' }, ['group-Metrics']);
-            await waitFor(() => w.processedCount === 1);
-            metrics = await metricsService.getMetrics(['group-Metrics']);
-            (0, vitest_1.expect)(metrics.throughput['group-Metrics']).toBeGreaterThanOrEqual(1);
-            w.shouldFail = true;
-            w.maxFails = 10;
-            await producer.push({ id: 'metrics-fail' }, ['group-Metrics']);
-            await waitFor(() => redis.xlen(`${streamName}:dlq`).then(len => len > 0));
-            metrics = await metricsService.getMetrics(['group-Metrics']);
-            (0, vitest_1.expect)(metrics.dlqLength).toBe(1);
-        });
-        (0, vitest_1.it)('Should export Prometheus metrics', async () => {
-            const w = new TestWorker(redis, 'group-Prom', streamName, 1, 3, 100);
-            const metricsService = new metrics_1.Metrics(redis, streamName);
-            workers.push(w);
-            await w.start();
-            await producer.push({ id: 'prom-1' }, ['group-Prom']);
-            await producer.push({ id: 'prom-2' }, ['group-Prom']);
-            await waitFor(() => w.processedCount === 2);
-            const promOutput = await metricsService.getPrometheusMetrics(['group-Prom']);
-            (0, vitest_1.expect)(promOutput).toContain(`# TYPE redis_highway_queue_throughput_1m gauge`);
-            (0, vitest_1.expect)(promOutput).toContain(`redis_highway_queue_throughput_1m{stream="${streamName}", group="group-Prom"} 2`);
-            (0, vitest_1.expect)(promOutput).toContain(`# TYPE redis_highway_queue_jobs_total counter`);
-            (0, vitest_1.expect)(promOutput).toContain(`redis_highway_queue_jobs_total{stream="${streamName}", group="group-Prom"} 2`);
-            (0, vitest_1.expect)(promOutput).toContain(`# TYPE redis_highway_queue_waiting_jobs gauge`);
-        });
-    });
-    (0, vitest_1.describe)('Stream Cleanup', () => {
-        (0, vitest_1.it)('Should delete message from stream after processing', async () => {
-            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
-            workers.push(w1);
-            await w1.start();
-            const id = await producer.push({ id: 'msg-cleanup' }, ['group-A']);
-            // Wait for processing
-            await waitFor(() => w1.processedCount === 1);
-            (0, vitest_1.expect)(w1.processedCount).toBe(1);
-            // Wait for stream to be empty
-            const success = await waitFor(async () => {
-                const len = await redis.xlen(streamName);
-                return len === 0;
-            });
-            (0, vitest_1.expect)(success).toBe(true);
-            const messages = await redis.xrange(streamName, '-', '+');
-            (0, vitest_1.expect)(messages.length).toBe(0);
-        });
-        (0, vitest_1.it)('Should delete message from stream only after ALL groups processed it', async () => {
-            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
-            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 3, 100);
-            workers.push(w1, w2);
-            await w1.start(); // Only start w1
-            const id = await producer.push({ id: 'msg-multi' }, ['group-A', 'group-B']);
-            // Wait for w1 to process
-            await waitFor(() => w1.processedCount === 1);
-            let len = await redis.xlen(streamName);
-            (0, vitest_1.expect)(len).toBe(1); // Should still exist because group-B pending
-            // Start w2
-            await w2.start();
-            // Wait for w2 to process
-            await waitFor(() => w2.processedCount === 1);
-            // Wait for stream to be empty
-            const success = await waitFor(async () => {
-                len = await redis.xlen(streamName);
-                return len === 0;
-            });
-            (0, vitest_1.expect)(success).toBe(true);
-        });
-    });
-    (0, vitest_1.it)('Should recover stuck messages via Auto-Claim', async () => {
-        const groupName = 'group-Recover';
-        // Start worker with short minIdleTime (e.g., 1000ms) to trigger claim quickly
-        // minIdleTimeMs = 1000. claimIntervalMs = 500 (check frequently)
-        const w = new TestWorker(redis, groupName, streamName, 1, 3, 100, 500, 1000);
-        workers.push(w);
-        // 1. Setup group manually
-        await redis.xgroup('CREATE', streamName, groupName, '0', 'MKSTREAM');
-        // 2. Push message
-        const id = await producer.push({ id: 'stuck-msg' }, [groupName]);
-        // 3. Simulate a consumer reading but crashing (no ACK)
-        // consumer name 'bad-consumer'
-        await redis.xreadgroup('GROUP', groupName, 'bad-consumer', 'COUNT', 1, 'STREAMS', streamName, '>');
-        // 4. Wait for minIdleTime (1000ms) + buffer
-        await new Promise(r => setTimeout(r, 1200));
-        // 5. Start our worker
-        await w.start();
-        // 6. Verify worker picks it up
-        await waitFor(() => w.processedCount === 1, 5000);
-        (0, vitest_1.expect)(w.processedCount).toBe(1);
-        (0, vitest_1.expect)(w.lastProcessedId).toBe('stuck-msg');
-        // Verify it was claimed (delivered to new consumer)
-        // We can check PEL or just trust processedCount
-        const pending = await redis.xpending(streamName, groupName);
-        // After processing, it should be ACKed, so pending count => 0 (if deleted)
-        // or if finalize runs, it deletes the message entirely.
-        // Wait for cleanup (finalize runs after process)
-        await waitFor(async () => {
-            const len = await redis.xlen(streamName);
-            return len === 0;
-        }, 2000);
-        const len = await redis.xlen(streamName);
-        (0, vitest_1.expect)(len).toBe(0);
-    });
-});