@hile/redis-stream-queue 3.0.2 → 3.0.4

package/dist/queue.d.ts

@@ -9,6 +9,7 @@ export declare class RedisStreamQueue {
     readDeadLetters<TData>(definition: QueueDefinition<TData>, options?: ReadDeadLettersOptions): Promise<Array<QueueDeadLetter<TData>>>;
     runWorkerOnce<TData>(definition: QueueDefinition<TData>, handler: QueueWorkerHandler<TData>, options: RequiredWorkerOptions): Promise<number>;
     private processEntry;
+    private acknowledge;
     private handleFailure;
     private createJob;
     private promoteDelayed;
@@ -41,7 +42,10 @@ type RequiredWorkerOptions = {
     concurrency: number;
     block: number;
     pollInterval: number;
+    errorRetryInterval: number;
     claimIdle: number;
     claimCount: number;
+    removeOnAck: boolean;
+    onError?: (err: unknown) => void | Promise<void>;
 };
 export {};

package/dist/queue.js

@@ -6,7 +6,16 @@ const DEFAULT_MAX_ATTEMPTS = 1;
 const DEFAULT_CONCURRENCY = 1;
 const DEFAULT_CLAIM_IDLE = 60_000;
 const DEFAULT_POLL_INTERVAL = 1_000;
+const DEFAULT_ERROR_RETRY_INTERVAL = 1_000;
 const DEFAULT_READ_BLOCK = 0;
+const PROMOTE_DELAYED_SCRIPT = `
+if redis.call('ZSCORE', KEYS[1], ARGV[1]) == false then
+  return 0
+end
+local streamId = redis.call('XADD', KEYS[2], '*', 'job', ARGV[1])
+redis.call('ZREM', KEYS[1], ARGV[1])
+return streamId
+`;
 export class RedisStreamQueue {
     redis;
     prefix;
@@ -128,13 +137,20 @@ export class RedisStreamQueue {
             else {
                 await run();
             }
-            await this.redis.xack(this.streamKey(definition), options.group, streamId);
         }
         catch (err) {
-            await this.handleFailure(definition, stored, streamId, attempt, options.group, err);
+            await this.handleFailure(definition, stored, streamId, attempt, options, err);
+            return;
         }
+        await this.acknowledge(definition, options, streamId);
     }
-    async handleFailure(definition, stored, streamId, attempt, group, err) {
+    async acknowledge(definition, options, streamId) {
+        await this.redis.xack(this.streamKey(definition), options.group, streamId);
+        if (options.removeOnAck) {
+            await this.redis.xdel(this.streamKey(definition), streamId);
+        }
+    }
+    async handleFailure(definition, stored, streamId, attempt, options, err) {
         const reason = errorReason(err);
         const failed = {
             ...stored,
@@ -150,7 +166,7 @@ export class RedisStreamQueue {
         else {
             await this.redis.xadd(this.deadLetterKey(definition), '*', 'job', this.encodeJob(definition, failed));
         }
-        await this.redis.xack(this.streamKey(definition), group, streamId);
+        await this.acknowledge(definition, options, streamId);
     }
     createJob(definition, stored, streamId, attempt) {
         return {
@@ -170,11 +186,9 @@ export class RedisStreamQueue {
         const members = await this.redis.zrangebyscore(this.delayedKey(definition), '-inf', this.now(), 'LIMIT', 0, limit);
         let promoted = 0;
         for (const member of members) {
-            const removed = await this.redis.zrem(this.delayedKey(definition), member);
-            if (removed === 0)
-                continue;
-            await this.redis.xadd(this.streamKey(definition), '*', 'job', member);
-            promoted++;
+            const streamId = await this.redis.eval(PROMOTE_DELAYED_SCRIPT, 2, this.delayedKey(definition), this.streamKey(definition), member);
+            if (streamId !== 0)
+                promoted++;
         }
         return promoted;
     }
@@ -276,9 +290,22 @@ export class RedisStreamQueueWorker {
     }
     async runLoop() {
         while (this.running) {
-            const processed = await this.runOnce();
-            if (processed === 0) {
-                await sleep(this.options.pollInterval);
+            try {
+                const processed = await this.runOnce();
+                if (processed === 0) {
+                    await sleep(this.options.pollInterval);
+                }
+            }
+            catch (err) {
+                try {
+                    await this.options.onError?.(err);
+                }
+                catch {
+                    // Error observers must not stop a worker that is already recovering.
+                }
+                if (this.running) {
+                    await sleep(this.options.errorRetryInterval);
+                }
             }
         }
     }
@@ -291,8 +318,11 @@ function normalizeWorkerOptions(definition, options) {
         concurrency,
         block: assertNonNegativeInteger(options.block ?? DEFAULT_READ_BLOCK, 'block'),
         pollInterval: assertNonNegativeInteger(options.pollInterval ?? DEFAULT_POLL_INTERVAL, 'pollInterval'),
+        errorRetryInterval: assertNonNegativeInteger(options.errorRetryInterval ?? DEFAULT_ERROR_RETRY_INTERVAL, 'errorRetryInterval'),
         claimIdle: assertNonNegativeInteger(options.claimIdle ?? DEFAULT_CLAIM_IDLE, 'claimIdle'),
         claimCount: assertPositiveInteger(options.claimCount ?? concurrency, 'claimCount'),
+        removeOnAck: options.removeOnAck ?? false,
+        onError: options.onError,
     };
 }
 function normalizeBackoff(backoff) {

package/dist/types.d.ts

@@ -66,8 +66,11 @@ export type QueueWorkerOptions = {
     concurrency?: number;
     block?: number;
     pollInterval?: number;
+    errorRetryInterval?: number;
     claimIdle?: number;
     claimCount?: number;
+    removeOnAck?: boolean;
+    onError?: (err: unknown) => void | Promise<void>;
 };
 export type RedisStreamEntry = [id: string, fields: string[]];
 export type RedisStreamReadResult = Array<[stream: string, entries: RedisStreamEntry[]]> | null;
@@ -79,10 +82,12 @@ export interface RedisStreamQueueLike {
     xpending(...args: any[]): Promise<any>;
     xclaim(...args: any[]): Promise<any>;
     xack(...args: any[]): Promise<any>;
+    xdel(...args: any[]): Promise<any>;
     xrange(...args: any[]): Promise<any>;
     zadd(...args: any[]): Promise<any>;
     zrangebyscore(...args: any[]): Promise<any>;
     zrem(...args: any[]): Promise<any>;
+    eval(...args: any[]): Promise<any>;
     set(...args: any[]): Promise<any>;
     get(...args: any[]): Promise<any>;
     del(...args: any[]): Promise<any>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hile/redis-stream-queue",
-  "version": "3.0.2",
+  "version": "3.0.4",
   "description": "Redis Streams backed durable job queue primitives for Hile applications",
   "type": "module",
   "main": "./dist/index.js",
@@ -26,5 +26,5 @@
   "dependencies": {
     "@hile/context": "^3.0.2"
   },
-  "gitHead": "0985b6f8abc1f4de0a36324063585fdc3ac1375b"
+  "gitHead": "f5d5970c964440047dda973d3f4eda92116ca809"
 }