@koala42/redis-highway 0.2.1 → 0.2.3

package/dist/src/base-worker.js

@@ -110,8 +110,10 @@ class BaseWorker {
         const messagesStreamIds = messages.map((message) => message.streamMessageId);
         pipeline.xack(this._streamName, this._groupName, ...messagesStreamIds);
         const messagesToDLQ = [];
+        let retryCountIncr = 0;
         for (const message of messages) {
             if (message.retryCount < this._maxRetries) {
+                retryCountIncr++;
                 const newJobId = (0, uuid_1.v7)();
                 pipeline.xadd(this._streamName, '*', 'id', newJobId, 'target', this._groupName, 'retryCount', message.retryCount + 1, 'data', message.serializedData);
                 const newStatusKey = this._keys.getJobStatusKey(newJobId);
@@ -123,11 +125,15 @@ class BaseWorker {
                 console.error(`[${this._groupName}] Job ${message.messageUuid} run out of retries. Moving to DLQ`);
                 messagesToDLQ.push(message);
                 // Add message to DLQ stream
-                pipeline.xadd(this._keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this._groupName, 'error', errorMessage, 'payload', message.serializedData, 'failedAt', Date.now());
+                pipeline.xadd(this._keys.getDlqStreamKey(), '*', 'id', message.messageUuid, 'group', this._groupName, 'error', errorMessage, 'payload', message.serializedData, 'failedAt', timestamp);
                 const statusKey = this._keys.getJobStatusKey(message.messageUuid);
                 pipeline.eval(lua_1.LUA_FINALIZE, 2, statusKey, this._streamName, this._groupName, timestamp, message.streamMessageId);
             }
         }
+        if (retryCountIncr) {
+            const retryCountKey = this._keys.getRetriesKey(this._groupName, timestamp);
+            pipeline.incrby(retryCountKey, retryCountIncr);
+        }
         await pipeline.exec();
     }
     /**

package/dist/src/keys.d.ts

@@ -13,9 +13,13 @@ export declare class KeyManager {
      */
     getDlqStreamKey(): string;
     /**
-     * Metrics Hash for storing throughput
+     * Metrics for storing throughput
      */
     getThroughputKey(groupName: string, timestamp: number): string;
+    /**
+     * Metrics - retries key
+     */
+    getRetriesKey(groupName: string, timestamp: number): string;
     /**
      * Total jobs processed metrics
      */

package/dist/src/keys.js

@@ -23,12 +23,19 @@ class KeyManager {
         return `${this.streamName}:dlq`;
     }
     /**
-     * Metrics Hash for storing throughput
+     * Metrics for storing throughput
      */
     getThroughputKey(groupName, timestamp) {
         const minute = Math.floor(timestamp / 60000) * 60000;
         return `metrics:throughput:${this.streamName}:${groupName}:${minute}`;
     }
+    /**
+     * Metrics - retries key
+     */
+    getRetriesKey(groupName, timestamp) {
+        const minute = Math.floor(timestamp / 60000) * 60000;
+        return `metrics:retry-count:${this.streamName}:${groupName}:${minute}`;
+    }
     /**
      * Total jobs processed metrics
      */

package/dist/src/metrics.js

@@ -57,6 +57,7 @@ class Metrics {
         groupNames.forEach(group => {
             pipeline.get(this.keys.getThroughputKey(group, timestamp));
             pipeline.get(this.keys.getTotalKey(group));
+            pipeline.get(this.keys.getRetriesKey(group, timestamp));
         });
         const results = await pipeline.exec();
         if (!results)
@@ -72,10 +73,11 @@ class Metrics {
         const response = [];
         response.push(`# HELP ${prefix}_waiting_jobs Total jobs waiting in stream`, `# TYPE ${prefix}_waiting_jobs gauge`, `${prefix}_waiting_jobs{stream="${this.streamName}"} ${streamLength}`, `# HELP ${prefix}_dlq_jobs Total jobs in DLQ`, `# TYPE ${prefix}_dlq_jobs gauge`, `${prefix}_dlq_jobs{stream="${this.streamName}"} ${dlqLength}`);
         groupNames.forEach((group, index) => {
-            const baseIndex = 2 + (index * 2);
+            const baseIndex = 2 + (index * 3);
             const throughputVal = parseInt(getResult(baseIndex) || '0', 10);
             const totalVal = parseInt(getResult(baseIndex + 1) || '0', 10);
-            response.push(`# HELP ${prefix}_throughput_1m Jobs processed in the last minute`, `# TYPE ${prefix}_throughput_1m gauge`, `${prefix}_throughput_1m{stream="${this.streamName}", group="${group}"} ${throughputVal}`, `# HELP ${prefix}_jobs_total Total jobs processed`, `# TYPE ${prefix}_jobs_total counter`, `${prefix}_jobs_total{stream="${this.streamName}", group="${group}"} ${totalVal}`);
+            const retryCountVal = parseInt(getResult(baseIndex + 2) || '0', 10);
+            response.push(`# HELP ${prefix}_throughput_1m Jobs processed in the last minute`, `# TYPE ${prefix}_throughput_1m gauge`, `${prefix}_throughput_1m{stream="${this.streamName}", group="${group}"} ${throughputVal}`, `# HELP ${prefix}_retries_1m Retries in the last minute`, `# TYPE ${prefix}_retries_1m gauge`, `${prefix}_retries_1m{stream="${this.streamName}", group="${group}"} ${retryCountVal}`, `# HELP ${prefix}_jobs_total Total jobs processed`, `# TYPE ${prefix}_jobs_total counter`, `${prefix}_jobs_total{stream="${this.streamName}", group="${group}"} ${totalVal}`);
         });
         return response.join('\n');
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koala42/redis-highway",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "description": "High performance redis queue",
   "license": "MIT",
   "author": {
@@ -16,8 +16,7 @@
   ],
   "scripts": {
     "clean": "rimraf dist",
-    "test": "vitest run src/queue.spec.ts",
-    "test:all": "vitest run src",
+    "test:all": "vitest run test/queue.spec.ts test/batch-worker.spec.ts",
     "build": "npm run clean && tsc",
     "prepublish": "npm run build"
   },
@@ -28,7 +27,7 @@
     "high-throughput"
   ],
   "dependencies": {
-    "ioredis": "^5.8.2",
+    "ioredis": "5.8.2",
     "uuid": "^13.0.0"
   },
   "devDependencies": {