@koala42/redis-highway 0.2.3 → 0.2.8

package/dist/src/base-worker.d.ts

@@ -1,7 +1,7 @@
 import { EventEmitter } from "events";
 import { KeyManager } from "./keys";
 import Redis from "ioredis";
-import { BaseWorkerControlOptions, BaseWorkerOptions, StreamMessage, XReadGroupResponse } from "./interfaces";
+import { BaseWorkerControlOptions, BaseWorkerCustomMetricsOptions, BaseWorkerOptions, StreamMessage, XReadGroupResponse } from "./interfaces";
 import { StreamMessageEntity } from "./stream-message-entity";
 export declare abstract class BaseWorker<T extends Record<string, unknown>> {
     protected redis: Redis;
@@ -20,7 +20,8 @@ export declare abstract class BaseWorker<T extends Record<string, unknown>> {
     protected readonly _claimIntervalMs: number;
     protected readonly _minIdleTimeMs: number;
     protected readonly _collectMetrics: boolean;
-    constructor(redis: Redis, options: BaseWorkerOptions, controlOptions: BaseWorkerControlOptions);
+    protected readonly _finalIncrementMetricKey: ((item: T) => string | null) | null;
+    constructor(redis: Redis, options: BaseWorkerOptions, controlOptions: BaseWorkerControlOptions, metricsOptions: BaseWorkerCustomMetricsOptions<T>);
     /**
      * Start the worker process
      * Starts fetch loop and auto claim loop

package/dist/src/base-worker.js

@@ -7,7 +7,7 @@ const uuid_1 = require("uuid");
 const stream_message_entity_1 = require("./stream-message-entity");
 const lua_1 = require("./lua");
 class BaseWorker {
-    constructor(redis, options, controlOptions) {
+    constructor(redis, options, controlOptions, metricsOptions) {
         this.redis = redis;
         this._isRunning = false;
         this._activeCount = 0;
@@ -22,6 +22,7 @@ class BaseWorker {
         this._claimIntervalMs = controlOptions.claimIntervalMs;
         this._minIdleTimeMs = controlOptions.minIdleTimeMs;
         this._collectMetrics = controlOptions.collectMetrics;
+        this._finalIncrementMetricKey = metricsOptions.finalIncrementMetricKey ?? null;
         this._consumerName = `${this._groupName}-${this._consumerId}`;
         this._keys = new keys_1.KeyManager(options.streamName);
         this._blockingRedis = redis.duplicate();
@@ -178,7 +179,13 @@ class BaseWorker {
         }
         for (const msg of messages) {
             const statusKey = this._keys.getJobStatusKey(msg.messageUuid);
-            pipeline.eval(lua_1.LUA_FINALIZE, 2, statusKey, this._streamName, this._groupName, timestamp, msg.streamMessageId);
+            const customMetricKey = this._finalIncrementMetricKey ? this._finalIncrementMetricKey(msg.data) : null;
+            if (customMetricKey) {
+                pipeline.eval(lua_1.LUA_FINALIZE_CUSTOM_METRIC, 3, statusKey, this._streamName, customMetricKey, this._groupName, timestamp, msg.streamMessageId);
+            }
+            else {
+                pipeline.eval(lua_1.LUA_FINALIZE, 2, statusKey, this._streamName, this._groupName, timestamp, msg.streamMessageId);
+            }
         }
         await pipeline.exec();
     }

package/dist/src/batch-worker.d.ts

@@ -1,10 +1,10 @@
 import Redis from "ioredis";
-import { BaseWorkerControlOptions, BatchWorkerOptions } from "./interfaces";
+import { BaseWorkerControlOptions, BaseWorkerCustomMetricsOptions, BatchWorkerOptions } from "./interfaces";
 import { BaseWorker } from "./base-worker";
 export declare abstract class BatchWorker<T extends Record<string, unknown>> extends BaseWorker<T> {
     private readonly _batchSize;
     private readonly _maxFetchCount;
-    constructor(redis: Redis, options: BatchWorkerOptions, controlOptions?: BaseWorkerControlOptions);
+    constructor(redis: Redis, options: BatchWorkerOptions, controlOptions?: BaseWorkerControlOptions, metricsOptions?: BaseWorkerCustomMetricsOptions<T>);
     protected _fetchLoop(): Promise<void>;
     /**
      * Spawn worker for current processing

package/dist/src/batch-worker.js

@@ -5,8 +5,8 @@ const interfaces_1 = require("./interfaces");
 const stream_message_entity_1 = require("./stream-message-entity");
 const base_worker_1 = require("./base-worker");
 class BatchWorker extends base_worker_1.BaseWorker {
-    constructor(redis, options, controlOptions = interfaces_1.defaultBaseWorkerControlOptions) {
-        super(redis, options, controlOptions);
+    constructor(redis, options, controlOptions = interfaces_1.defaultBaseWorkerControlOptions, metricsOptions = interfaces_1.defaultBaseWorkerCustomMetrics) {
+        super(redis, options, controlOptions, metricsOptions);
         this._batchSize = options.batchSize;
         this._maxFetchCount = options.maxFetchCount;
         if (this._batchSize === 1) {

package/dist/src/interfaces.d.ts

@@ -12,6 +12,10 @@ export interface BaseWorkerControlOptions {
     minIdleTimeMs: number;
     collectMetrics: boolean;
 }
+export interface BaseWorkerCustomMetricsOptions<T extends Record<string, unknown>> {
+    finalIncrementMetricKey?: ((item: T) => string | null) | null;
+}
+export declare const defaultBaseWorkerCustomMetrics: BaseWorkerCustomMetricsOptions<{}>;
 export declare const defaultBaseWorkerControlOptions: BaseWorkerControlOptions;
 export interface BatchWorkerOptions extends BaseWorkerOptions {
     batchSize: number;

package/dist/src/interfaces.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.defaultBaseWorkerControlOptions = void 0;
+exports.defaultBaseWorkerControlOptions = exports.defaultBaseWorkerCustomMetrics = void 0;
+exports.defaultBaseWorkerCustomMetrics = {
+    finalIncrementMetricKey: null
+};
 exports.defaultBaseWorkerControlOptions = {
     maxRetries: 3,
     minIdleTimeMs: 120000,

package/dist/src/lua.d.ts

@@ -1 +1,2 @@
 export declare const LUA_FINALIZE = "\n-- KEYS[1] = status key\n-- KEYS[2] = stream key\n-- ARGV[1] = group name\n-- ARGV[2] = timestamp\n-- ARGV[3] = msgId\n\n-- 1. Update status\nredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n-- 2. Check completions\nlocal current_fields = redis.call('HLEN', KEYS[1])\nlocal target_str = redis.call('HGET', KEYS[1], '__target')\nlocal target = tonumber(target_str)\n\nif not target then\n    return 0\nend\n\n-- 3. Cleanup if done\nif current_fields >= (target + 1) then\n    redis.call('DEL', KEYS[1])\n    redis.call('XDEL', KEYS[2], ARGV[3])\n    return 1\nend\n\nreturn 0\n";
+export declare const LUA_FINALIZE_CUSTOM_METRIC = "\n  -- KEYS[1] = status key\n  -- KEYS[2] = stream key\n  -- KEYS[3] = custom metric increment key\n  -- ARGV[1] = group name\n  -- ARGV[2] = timestamp\n  -- ARGV[3] = msgId\n\n  -- 1. Update status\n  redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])\n\n  -- 2. Check completions\n  local current_fields = redis.call('HLEN', KEYS[1])\n  local target_str = redis.call('HGET', KEYS[1], '__target')\n  local target = tonumber(target_str)\n\n  if not target then\n      return 0\n  end\n\n  -- 3. Cleanup if done\n  if current_fields >= (target + 1) then\n      redis.call('DEL', KEYS[1])\n      redis.call('XDEL', KEYS[2], ARGV[3])\n      redis.pcall('INCRBY', KEYS[3], 1)\n      return 1\n  end\n\n  return 0\n";

package/dist/src/lua.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LUA_FINALIZE = void 0;
+exports.LUA_FINALIZE_CUSTOM_METRIC = exports.LUA_FINALIZE = void 0;
 exports.LUA_FINALIZE = `
 -- KEYS[1] = status key
 -- KEYS[2] = stream key
@@ -29,3 +29,33 @@ end
 
 return 0
 `;
+exports.LUA_FINALIZE_CUSTOM_METRIC = `
+  -- KEYS[1] = status key
+  -- KEYS[2] = stream key
+  -- KEYS[3] = custom metric increment key
+  -- ARGV[1] = group name
+  -- ARGV[2] = timestamp
+  -- ARGV[3] = msgId
+
+  -- 1. Update status
+  redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+
+  -- 2. Check completions
+  local current_fields = redis.call('HLEN', KEYS[1])
+  local target_str = redis.call('HGET', KEYS[1], '__target')
+  local target = tonumber(target_str)
+
+  if not target then
+      return 0
+  end
+
+  -- 3. Cleanup if done
+  if current_fields >= (target + 1) then
+      redis.call('DEL', KEYS[1])
+      redis.call('XDEL', KEYS[2], ARGV[3])
+      redis.pcall('INCRBY', KEYS[3], 1)
+      return 1
+  end
+
+  return 0
+`;

package/dist/src/worker.d.ts

@@ -1,8 +1,8 @@
 import Redis from "ioredis";
-import { BaseWorkerOptions, BaseWorkerControlOptions } from "./interfaces";
+import { BaseWorkerOptions, BaseWorkerControlOptions, BaseWorkerCustomMetricsOptions } from "./interfaces";
 import { BaseWorker } from "./base-worker";
 export declare abstract class Worker<T extends Record<string, unknown>> extends BaseWorker<T> {
-    constructor(redis: Redis, options: BaseWorkerOptions, controlOptions?: BaseWorkerControlOptions);
+    constructor(redis: Redis, options: BaseWorkerOptions, controlOptions?: BaseWorkerControlOptions, metricsOptions?: BaseWorkerCustomMetricsOptions<T>);
     /**
      * Fetch loop (the main loop)
      * Based on free slots (concurrency - active count) gets new messages

package/dist/src/worker.js

@@ -5,8 +5,8 @@ const interfaces_1 = require("./interfaces");
 const stream_message_entity_1 = require("./stream-message-entity");
 const base_worker_1 = require("./base-worker");
 class Worker extends base_worker_1.BaseWorker {
-    constructor(redis, options, controlOptions = interfaces_1.defaultBaseWorkerControlOptions) {
-        super(redis, options, controlOptions);
+    constructor(redis, options, controlOptions = interfaces_1.defaultBaseWorkerControlOptions, metricsOptions = interfaces_1.defaultBaseWorkerCustomMetrics) {
+        super(redis, options, controlOptions, metricsOptions);
     }
     /**
      * Fetch loop (the main loop)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.2.3",
+  "version": "0.2.8",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {
@@ -8,6 +8,11 @@
     "name": "David Stranava",
     "url": "https://github.com/stranavad"
   },
+  "repository": {
+    "url": "https://github.com/Koala42/redis-highway",
+    "type": "github",
+    "directory": "packages/redis-highway"
+  },
   "type": "commonjs",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -16,7 +21,8 @@
   ],
   "scripts": {
     "clean": "rimraf dist",
-    "test:all": "vitest run test/queue.spec.ts test/batch-worker.spec.ts",
+    "test": "vitest run test",
+    "test:all": "vitest run test",
     "build": "npm run clean && tsc",
     "prepublish": "npm run build"
   },
@@ -36,4 +42,4 @@
     "typescript": "^5.9.3",
     "vitest": "^4.0.16"
   }
-}
+}

package/README.md

@@ -1,124 +0,0 @@
-# @koala42/redis-highway
-
-High performance Redis stream-based queue for Node.js. Supports Redis single instances and Valkey single instances
-
-## Missing features ATM
-- Better gracefull shutdown handling
-- In worker process functions expose more than just data: T like job id and current status
-- Option to customize/enable/disable metrics
-- Enable custom logger for workers and producers instead of console.logs
-
-
-## Roadmap
-- Support redis cluster, that is probably possible only for job payloads and DLQ, since the stream is only one
-
-## Installation
-
-```bash
-npm install @koala42/redis-highway
-```
-
-## Usage
-
-### Producer
-
-```typescript
-import { Redis } from 'ioredis';
-import { Producer } from '@koala42/redis-highway';
-
-const redis = new Redis();
-const producer = new Producer(redis, 'my-stream');
-
-// Send job
-await producer.push(
-  JSON.stringify({ hello: 'world' }), // Message serialization is not done automatically
-  ['group-A', 'group-B'] // Target specific consumer groups
-);
-```
-
-### Worker
-
-```typescript
-import { Redis } from 'ioredis';
-import { Worker } from '@koala42/redis-highway';
-
-class MyWorker extends Worker<T> {
-  async process(data: T) {
-    console.log('Processing:', data);
-    // throw new Error('fail'); // Automatic retry/DLQ logic
-  }
-}
-
-const redis = new Redis();
-const worker = new MyWorker(redis, 'group-A', 'my-stream');
-
-await worker.start();
-```
-
-### Metrics
-
-```typescript
-import { Metrics } from '@koala42/redis-highway';
-
-const metrics = new Metrics(redis, 'my-stream');
-
-// Prometheus format
-const payload = await metrics.getPrometheusMetrics(['group-A']);
-```
-
-## Usage with NestJS
-
-```typescript
-
-// Producer
-@Injectable()
-export class EntryService {
-  privater readonly producer: Producer;
-  
-  constructor(){
-    this.producer = new Producer(
-      new Redis(...), // Or reuse existing ioredis connection
-      'my-stream'
-    )
-  }
-  
-  public async sth(): Promise<void>{
-    await producer.push(
-      JSON.stringify({ hello: 'world' }), // Message serialization is not done automatically
-      ['group-A', 'group-B'] // Target specific consumer groups
-    );
-  }
-}
-
-
-// Processor
-@Injectable()
-export class ProcessorService extends Worker<T> implements OnModuleInit, OnModuleDestroy {
-  constructor(){
-    super(
-      new Redis(...), // or reuse existing redis conn
-      'group-A',
-      'my-stream',
-      50 // concurrency
-    )
-  }
-  
-  async onModuleInit(): Promise<void>{
-    await this.start()
-  }
-  
-  onModuleDestroy(){
-    this.stop()
-  }
-  
-  async process(data: T): Promise<void>{
-    console.log("Processing job", JSON.stringify(data))
-  }
-}
-````
-
-## Features
-- **Lightweight**: Uses light Lua scripts and pipelines wherever possible, making it highly concurrents for inserts and for processing as well, because of the reduced I/O load compared to BullMQ
-- **Granular Retries**: If one group fails, only that group retries.
-- **DLQ**: Dead Letter Queue support after max retries.
-- **Metrics**: Throughput, Waiting, DLQ, Prometheus export.