@koala42/redis-highway 0.2.8 → 0.2.10

package/README.md

@@ -0,0 +1,192 @@
+# @koala42/redis-highway
+
+High performance Redis stream-based queue for Node.js. Supports Redis single instances and Valkey single instances.
+Designed for high throughput and massive concurrency with low overhead.
+
+## Features
+- **Lightweight**: Uses optimized Lua scripts and pipelines for maximum performance and reduced I/O.
+- **Micro-Batching**: Supports batch processing for high-volume message consumption.
+- **Granular Retries**: Consumer group isolation - if one group fails, only that group retries.
+- **Reliability**: Auto-claiming of stuck messages (crashed consumers) and Dead Letter Queue (DLQ) support.
+- **Metrics**: Built-in tracking for throughput, queue depth, DLQ size, and retries. Prometheus export ready.
+
+## Installation
+
+```bash
+npm install @koala42/redis-highway
+```
+
+## Usage
+
+### Producer
+
+```typescript
+import { Redis } from 'ioredis';
+import { Producer } from '@koala42/redis-highway';
+
+const redis = new Redis();
+const producer = new Producer<{hello: string}>(redis, 'my-stream');
+
+// Send job
+await producer.push(
+  { hello: 'world' }, // Type-safe payload
+  ['group-A', 'group-B'], // Target specific consumer groups
+  { ttl: 3600 } // Optional: expiration time in seconds
+);
+```
+
+### Worker
+
+```typescript
+import { Redis } from 'ioredis';
+import { Worker } from '@koala42/redis-highway';
+
+class MyWorker extends Worker<{hello: string}> {
+  async process(data: {hello: string}) {
+    console.log('Processing:', data.hello);
+    // throw new Error('fail'); // Triggers automatic retry logic
+  }
+}
+
+const redis = new Redis();
+const worker = new MyWorker(
+  redis,
+  {
+    groupName: 'group-A',
+    streamName: 'my-stream',
+    concurrency: 10 // Number of concurrent jobs to process
+  }
+);
+
+await worker.start();
+
+// To stop gracefully
+// await worker.stop();
+```
+
+### Batch Worker
+Process messages in batches for higher throughput.
+
+```typescript
+import { Redis } from 'ioredis';
+import { BatchWorker } from '@koala42/redis-highway';
+
+class MyBatchWorker extends BatchWorker<{hello: string}> {
+  async process(batchedData: {hello: string}[]) {
+    console.log(`Processing batch of ${batchedData.length} items`);
+    // Example: Bulk insert into database
+  }
+}
+
+const batchWorker = new MyBatchWorker(
+  redis,
+  {
+    groupName: 'group-B',
+    streamName: 'my-stream',
+    concurrency: 50, // Total items processing limit
+    batchSize: 10,   // Items per batch
+    maxFetchCount: 50
+  }
+);
+
+await batchWorker.start();
+```
+
+### Metrics
+
+```typescript
+import { Metrics } from '@koala42/redis-highway';
+
+const metrics = new Metrics(redis, 'my-stream');
+
+// Get raw metrics object
+const stats = await metrics.getMetrics(['group-A', 'group-B']);
+console.log(stats.throughput);
+
+// Get Prometheus formatted string
+const promMetrics = await metrics.getPrometheusMetrics(['group-A'], 'my_app_queue');
+```
+
+## Configuration
+
+### Worker Options
+The second argument to `Worker` and `BatchWorker` constructors is the primary configuration object.
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `groupName` | string | **Required**. The consumer group name (e.g., 'email-service'). |
+| `streamName` | string | **Required**. The Redis stream key. |
+| `concurrency` | number | **Required**. Maximum number of messages processed in parallel by this worker instance. |
+| `batchSize` | number | **Required (BatchWorker only)**. Number of messages to process in a single call. |
+| `maxFetchCount` | number | **Required (BatchWorker only)**. limit for XREADGROUP count. |
+
+### Control Options
+The third argument is for fine-tuning retry and recovery behavior.
+
+```typescript
+const worker = new MyWorker(redis, { ... }, {
+  maxRetries: 3,         // Default: 3
+  blockTimeMs: 2000,     // Default: 2000. XREADGROUP block time.
+  minIdleTimeMs: 120000, // Default: 2 minutes. Time before a message is considered stuck.
+  claimIntervalMs: 120000,// Default: 2 minutes. How often to check for stuck messages.
+  collectMetrics: true    // Default: true. Enable throughput tracking.
+});
+```
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `maxRetries` | 3 | Number of times to retry a failed message before moving it to DLQ. |
+| `blockTimeMs` | 2000 | Redis blocking timeout for fetching new messages (in ms). |
+| `minIdleTimeMs` | 120000 | Messages pending longer than this are candidates for auto-claim (recovery). |
+| `claimIntervalMs` | 120000 | Interval for checking and claiming stuck messages. |
+| `collectMetrics` | true | If true, increments throughput counters in Redis. |
+
+## Usage with NestJS
+
+```typescript
+// Producer Service
+@Injectable()
+export class EntryService {
+  private readonly producer: Producer<MyPayload>;
+
+  constructor(@InjectRedis() private readonly redis: Redis) {
+    this.producer = new Producer(this.redis, 'my-stream');
+  }
+
+  async addToQueue(data: MyPayload) {
+    await this.producer.push(data, ['group-A']);
+  }
+}
+
+// Worker Service
+@Injectable()
+export class ProcessorService extends Worker<MyPayload> implements OnModuleInit, OnModuleDestroy {
+  constructor(@InjectRedis() redis: Redis) {
+    super(redis, {
+      groupName: 'group-A',
+      streamName: 'my-stream',
+      concurrency: 50
+    });
+  }
+
+  async onModuleInit() {
+    await this.start();
+  }
+
+  async onModuleDestroy() {
+    await this.stop();
+  }
+
+  async process(data: MyPayload) {
+    // Process your job here
+  }
+}
+```
+
+## Roadmap & Missing Features
+tracked in [Github Issues](https://github.com/Koala42/redis-highway/issues)
+
+## AI Usage Disclosure
+- AI will not be used for the development, ever
+- AI may be used to do code reviews
+- AI may be used to write unit tests

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.2.8",
+  "version": "0.2.10",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {
@@ -10,19 +10,17 @@
   },
   "repository": {
     "url": "https://github.com/Koala42/redis-highway",
-    "type": "github",
-    "directory": "packages/redis-highway"
+    "type": "git"
   },
   "type": "commonjs",
-  "main": "dist/src/index.js",
-  "types": "dist/src/index.d.ts",
+  "main": "dist/index.js",
+  "types": "dist/ndex.d.ts",
   "files": [
     "dist"
   ],
   "scripts": {
     "clean": "rimraf dist",
     "test": "vitest run test",
-    "test:all": "vitest run test",
     "build": "npm run clean && tsc",
     "prepublish": "npm run build"
   },

package/dist/test/queue.spec.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/test/queue.spec.js

@@ -1,256 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const vitest_1 = require("vitest");
-const ioredis_1 = __importDefault(require("ioredis"));
-const producer_1 = require("../src/producer");
-const worker_1 = require("../src/worker");
-const metrics_1 = require("../src/metrics");
-const uuid_1 = require("uuid");
-const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
-class TestWorker extends worker_1.Worker {
-    constructor(redis, groupName, streamName, concurrency = 1, maxRetries = 3, blockTimeMs = 100, claimIntervalMs = 60000, minIdleTimeMs = 300000) {
-        super(redis, {
-            groupName,
-            streamName,
-            concurrency
-        }, {
-            maxRetries,
-            blockTimeMs,
-            claimIntervalMs,
-            minIdleTimeMs,
-            collectMetrics: true
-        });
-        this.processedCount = 0;
-        this.lastProcessedId = null;
-        this.shouldFail = false;
-        this.failCount = 0;
-        this.maxFails = 0;
-    }
-    async start() {
-        return super.start();
-    }
-    async stop() {
-        return super.stop();
-    }
-    async process(data) {
-        if (this.shouldFail) {
-            this.failCount++;
-            if (this.maxFails > 0 && this.failCount > this.maxFails) {
-                // Stop failing after maxFails
-            }
-            else {
-                throw new Error("Simulated Failure");
-            }
-        }
-        this.processedCount++;
-        if (data && data.id) {
-            this.lastProcessedId = data.id;
-        }
-    }
-}
-(0, vitest_1.describe)('Redis Queue Integration', () => {
-    let redis;
-    let producer;
-    let streamName;
-    let workers = [];
-    (0, vitest_1.beforeEach)(() => {
-        redis = new ioredis_1.default(REDIS_URL);
-        streamName = `test-queue-${(0, uuid_1.v7)()}`;
-        producer = new producer_1.Producer(redis, streamName);
-        workers = [];
-    });
-    (0, vitest_1.afterEach)(async () => {
-        for (const w of workers) {
-            await w.stop();
-        }
-        await new Promise(r => setTimeout(r, 500));
-        // Cleanup Redis keys using the existing connection before closing
-        if (redis.status === 'ready') {
-            const keys = await redis.keys(`${streamName}*`);
-            if (keys.length)
-                await redis.del(...keys);
-        }
-        redis.disconnect();
-    });
-    const waitFor = async (condition, timeout = 5000) => {
-        const start = Date.now();
-        while (Date.now() - start < timeout) {
-            if (await condition())
-                return true;
-            await new Promise(r => setTimeout(r, 100));
-        }
-        return false;
-    };
-    (0, vitest_1.describe)('Core Functionality', () => {
-        (0, vitest_1.it)('Should deliver message to all target groups', async () => {
-            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
-            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 3, 100);
-            workers.push(w1, w2);
-            await w1.start();
-            await w2.start();
-            const id = await producer.push({ id: 'msg-1' }, ['group-A', 'group-B']);
-            await waitFor(() => w1.processedCount === 1 && w2.processedCount === 1);
-            (0, vitest_1.expect)(w1.processedCount).toBe(1);
-            (0, vitest_1.expect)(w2.processedCount).toBe(1);
-            // Test cleanup keys (but not explicit XDEL here yet, checking keys gone)
-            const statusKey = `${streamName}:status:${id}`;
-            const dataKey = `${streamName}:data:${id}`;
-            (0, vitest_1.expect)(await redis.exists(statusKey)).toBe(0);
-            (0, vitest_1.expect)(await redis.exists(dataKey)).toBe(0);
-        });
-        (0, vitest_1.it)('Should only deliver to targeted groups', async () => {
-            const wA = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
-            const wB = new TestWorker(redis, 'group-B', streamName, 1, 3, 100);
-            workers.push(wA, wB);
-            await wA.start();
-            await wB.start();
-            await producer.push({ id: 'msg-only-a' }, ['group-A']);
-            await waitFor(() => wA.processedCount === 1);
-            (0, vitest_1.expect)(wA.processedCount).toBe(1);
-            (0, vitest_1.expect)(wB.processedCount).toBe(0);
-        });
-        (0, vitest_1.it)('Should retry only the failed group', async () => {
-            const wOk = new TestWorker(redis, 'group-Ok', streamName, 1, 3, 100);
-            const wFail = new TestWorker(redis, 'group-Fail', streamName, 1, 3, 100);
-            wFail.shouldFail = true;
-            wFail.maxFails = 1; // Fail once, then succeed
-            workers.push(wOk, wFail);
-            await wOk.start();
-            await wFail.start();
-            await producer.push({ id: 'retry-test' }, ['group-Ok', 'group-Fail']);
-            // Wait for wOk to finish and wFail to try at least twice (fail + success)
-            await waitFor(() => wOk.processedCount === 1 && wFail.processedCount === 1, 8000);
-            (0, vitest_1.expect)(wOk.processedCount).toBe(1); // Processed once
-            (0, vitest_1.expect)(wFail.failCount).toBeGreaterThanOrEqual(1); // Failed at least once
-            (0, vitest_1.expect)(wFail.processedCount).toBe(1); // Eventually succeeded
-            (0, vitest_1.expect)(wOk.processedCount).toBe(1); // wOk should NOT process the retry
-        });
-        (0, vitest_1.it)('Should move to DLQ after max retries', async () => {
-            const wDead = new TestWorker(redis, 'group-Dead', streamName, 1, 3, 100);
-            wDead.shouldFail = true;
-            wDead.maxFails = 10; // Fail forever (more than max retries which is 3)
-            workers.push(wDead);
-            await wDead.start();
-            const id = await producer.push({ id: 'dlq-test' }, ['group-Dead']);
-            await waitFor(async () => {
-                const len = await redis.xlen(`${streamName}:dlq`);
-                return len > 0;
-            }, 10000);
-            const dlqLen = await redis.xlen(`${streamName}:dlq`);
-            (0, vitest_1.expect)(dlqLen).toBe(1);
-            (0, vitest_1.expect)(await redis.exists(`${streamName}:status:${id}`)).toBe(0);
-        });
-    });
-    (0, vitest_1.describe)('Metrics & Monitoring', () => {
-        (0, vitest_1.it)('Should track throughput and queue size', async () => {
-            const w = new TestWorker(redis, 'group-Metrics', streamName, 1, 3, 100);
-            const metricsService = new metrics_1.Metrics(redis, streamName);
-            workers.push(w);
-            await w.start();
-            let metrics = await metricsService.getMetrics(['group-Metrics']);
-            (0, vitest_1.expect)(metrics.dlqLength).toBe(0);
-            const id = await producer.push({ id: 'metrics-1' }, ['group-Metrics']);
-            await waitFor(() => w.processedCount === 1);
-            metrics = await metricsService.getMetrics(['group-Metrics']);
-            (0, vitest_1.expect)(metrics.throughput['group-Metrics']).toBeGreaterThanOrEqual(1);
-            w.shouldFail = true;
-            w.maxFails = 10;
-            await producer.push({ id: 'metrics-fail' }, ['group-Metrics']);
-            await waitFor(() => redis.xlen(`${streamName}:dlq`).then(len => len > 0));
-            metrics = await metricsService.getMetrics(['group-Metrics']);
-            (0, vitest_1.expect)(metrics.dlqLength).toBe(1);
-        });
-        (0, vitest_1.it)('Should export Prometheus metrics', async () => {
-            const w = new TestWorker(redis, 'group-Prom', streamName, 1, 3, 100);
-            const metricsService = new metrics_1.Metrics(redis, streamName);
-            workers.push(w);
-            await w.start();
-            await producer.push({ id: 'prom-1' }, ['group-Prom']);
-            await producer.push({ id: 'prom-2' }, ['group-Prom']);
-            await waitFor(() => w.processedCount === 2);
-            const promOutput = await metricsService.getPrometheusMetrics(['group-Prom']);
-            (0, vitest_1.expect)(promOutput).toContain(`# TYPE redis_highway_queue_throughput_1m gauge`);
-            (0, vitest_1.expect)(promOutput).toContain(`redis_highway_queue_throughput_1m{stream="${streamName}", group="group-Prom"} 2`);
-            (0, vitest_1.expect)(promOutput).toContain(`# TYPE redis_highway_queue_jobs_total counter`);
-            (0, vitest_1.expect)(promOutput).toContain(`redis_highway_queue_jobs_total{stream="${streamName}", group="group-Prom"} 2`);
-            (0, vitest_1.expect)(promOutput).toContain(`# TYPE redis_highway_queue_waiting_jobs gauge`);
-        });
-    });
-    (0, vitest_1.describe)('Stream Cleanup', () => {
-        (0, vitest_1.it)('Should delete message from stream after processing', async () => {
-            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
-            workers.push(w1);
-            await w1.start();
-            const id = await producer.push({ id: 'msg-cleanup' }, ['group-A']);
-            // Wait for processing
-            await waitFor(() => w1.processedCount === 1);
-            (0, vitest_1.expect)(w1.processedCount).toBe(1);
-            // Wait for stream to be empty
-            const success = await waitFor(async () => {
-                const len = await redis.xlen(streamName);
-                return len === 0;
-            });
-            (0, vitest_1.expect)(success).toBe(true);
-            const messages = await redis.xrange(streamName, '-', '+');
-            (0, vitest_1.expect)(messages.length).toBe(0);
-        });
-        (0, vitest_1.it)('Should delete message from stream only after ALL groups processed it', async () => {
-            const w1 = new TestWorker(redis, 'group-A', streamName, 1, 3, 100);
-            const w2 = new TestWorker(redis, 'group-B', streamName, 1, 3, 100);
-            workers.push(w1, w2);
-            await w1.start(); // Only start w1
-            const id = await producer.push({ id: 'msg-multi' }, ['group-A', 'group-B']);
-            // Wait for w1 to process
-            await waitFor(() => w1.processedCount === 1);
-            let len = await redis.xlen(streamName);
-            (0, vitest_1.expect)(len).toBe(1); // Should still exist because group-B pending
-            // Start w2
-            await w2.start();
-            // Wait for w2 to process
-            await waitFor(() => w2.processedCount === 1);
-            // Wait for stream to be empty
-            const success = await waitFor(async () => {
-                len = await redis.xlen(streamName);
-                return len === 0;
-            });
-            (0, vitest_1.expect)(success).toBe(true);
-        });
-    });
-    (0, vitest_1.it)('Should recover stuck messages via Auto-Claim', async () => {
-        const groupName = 'group-Recover';
-        // Start worker with short minIdleTime (e.g., 1000ms) to trigger claim quickly
-        // minIdleTimeMs = 1000. claimIntervalMs = 500 (check frequently)
-        const w = new TestWorker(redis, groupName, streamName, 1, 3, 100, 500, 1000);
-        workers.push(w);
-        // 1. Setup group manually
-        await redis.xgroup('CREATE', streamName, groupName, '0', 'MKSTREAM');
-        // 2. Push message
-        const id = await producer.push({ id: 'stuck-msg' }, [groupName]);
-        // 3. Simulate a consumer reading but crashing (no ACK)
-        // consumer name 'bad-consumer'
-        await redis.xreadgroup('GROUP', groupName, 'bad-consumer', 'COUNT', 1, 'STREAMS', streamName, '>');
-        // 4. Wait for minIdleTime (1000ms) + buffer
-        await new Promise(r => setTimeout(r, 1200));
-        // 5. Start our worker
-        await w.start();
-        // 6. Verify worker picks it up
-        await waitFor(() => w.processedCount === 1, 5000);
-        (0, vitest_1.expect)(w.processedCount).toBe(1);
-        (0, vitest_1.expect)(w.lastProcessedId).toBe('stuck-msg');
-        // Verify it was claimed (delivered to new consumer)
-        // We can check PEL or just trust processedCount
-        const pending = await redis.xpending(streamName, groupName);
-        // After processing, it should be ACKed, so pending count => 0 (if deleted)
-        // or if finalize runs, it deletes the message entirely.
-        // Wait for cleanup (finalize runs after process)
-        await waitFor(async () => {
-            const len = await redis.xlen(streamName);
-            return len === 0;
-        }, 2000);
-        const len = await redis.xlen(streamName);
-        (0, vitest_1.expect)(len).toBe(0);
-    });
-});