@fgiova/sqs-consumer 3.2.1 → 3.4.0

package/README.md

@@ -78,13 +78,50 @@ const consumer = new SQSConsumer({
 | destroySigner    | boolean                | true    | Whether to destroy the signer when the consumer is destroyed.                                             |
 
 ### ConsumerOptions
-| Option                | Type     | Default | Description                                                |
-|-----------------------|----------|---------|------------------------------------------------------------|
-| visibilityTimeout     | number   | 30      | The visibility timeout for the messages in seconds         |
-| waitTimeSeconds       | number   | 20      | The wait time for the receiveMessage call in seconds       |
-| itemsPerRequest       | number   | 10      | The maximum number of messages to be received at once      |
-| messageAttributeNames | string[] | []      | The message attribute names to be included in the response |
-| attributeNames        | string[] | []      | The attribute names to be included in the response         |
+| Option                | Type                        | Default | Description                                                                                                                                  |
+|-----------------------|-----------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------|
+| visibilityTimeout     | number                      | 30      | The visibility timeout for the messages in seconds                                                                                           |
+| waitTimeSeconds       | number                      | 20      | The wait time for the receiveMessage call in seconds                                                                                         |
+| itemsPerRequest       | number                      | 10      | The maximum number of messages to be received at once                                                                                        |
+| messageAttributeNames | string[]                    | []      | The message attribute names to be included in the response                                                                                   |
+| attributeNames        | string[]                    | []      | The attribute names to be included in the response                                                                                           |
+| errorDelaySeconds     | number                      | 0       | Seconds a failed message stays invisible before it can be received again                                                                     |
+| errorDelayBackoff     | (attempt: number) => number |         | Returns the delay in seconds for a failed message, given its `ApproximateReceiveCount`. Takes precedence over `errorDelaySeconds` |
+| maxReceiveCount       | number / (attempt: number) => number |  | Receive count after which a failed message triggers `onMaxReceiveCountExceeded`. A function receives the `ApproximateReceiveCount` and returns the threshold |
+
+Delays are truncated to an integer and clamped to the SQS range 0–43200 seconds. When `errorDelayBackoff` is set, `ApproximateReceiveCount` is added to `attributeNames` automatically. If `errorDelayBackoff` throws, the error is reported to `onSQSError` and `errorDelaySeconds` is used instead.
+
+> **Note:** SQS counts the 12-hour visibility limit from when the message was received, not from when the delay is set. A delay that exceeds the remaining time (roughly `43200 - executionTimeout - visibility extensions`) is silently rejected by SQS and the message is retried after its current visibility timeout. Keep delays well below 12 hours, and keep their sum below the queue's `MessageRetentionPeriod`, or the message expires without reaching the DLQ.
+>
+> Every retry increments `ApproximateReceiveCount`, so with a redrive policy the message moves to the DLQ after `maxReceiveCount` attempts: size the backoff accordingly. On FIFO queues a delayed message blocks its whole message group until the delay expires.
+
+```typescript
+const consumer = new SQSConsumer({
+	queueARN: "arn:aws:sqs:eu-central-1:000000000000:test",
+	handler: async (message) => { /* ... */ },
+	consumerOptions: {
+		// 1st failure: 2s, 2nd: 4s, 3rd: 8s ... capped at 15 minutes
+		errorDelayBackoff: (attempt) => Math.min(2 ** attempt, 900),
+	},
+});
+```
+
+When a message fails and its `ApproximateReceiveCount` reaches `maxReceiveCount`, the `onMaxReceiveCountExceeded` hook is called with the message and its receive count. **The message is deleted unless a hook returns `false`**, including when no hook is registered: return `false` to release it with the usual error delay instead. When `maxReceiveCount` is set, `ApproximateReceiveCount` is added to `attributeNames` automatically. If the `maxReceiveCount` function throws, the error is reported to `onSQSError` and the message is released.
+
+> **Note:** if the queue has a redrive policy, set `maxReceiveCount` below the queue's own `maxReceiveCount`, or SQS moves the message to the DLQ before the hook is called.
+
+```typescript
+const consumer = new SQSConsumer({
+	queueARN: "arn:aws:sqs:eu-central-1:000000000000:test",
+	handler: async (message) => { /* ... */ },
+	consumerOptions: {
+		maxReceiveCount: 5,
+	},
+});
+consumer.addHook("onMaxReceiveCountExceeded", async (message, count) => {
+	await archive(message); // keep a copy, then let the consumer delete it
+});
+```
 
 ### Hooks
 | Option           | Type                                                                  | Description                                                         |
@@ -94,6 +131,7 @@ const consumer = new SQSConsumer({
 | onHandlerSuccess | (message: Message) => Promise<Message>                                | Called when the consumer handles a message successfully             |
 | onHandlerTimeout | (message: Message) => Promise<boolean>                                | Called when the consumer handler execution exceed executionTimeout  |
 | onHandlerError   | (message: Message, error: Error) => Promise<boolean>                  | Called when the consumer handler execution throws an error          |
+| onMaxReceiveCountExceeded | (message: Message, count: number) => Promise<boolean> | Called when a failed message reaches `maxReceiveCount`. Return `false` to keep the message, otherwise it is deleted |
 | onSuccess        | (message: Message) => Promise<boolean>                                | Called when the consumer handler execution finishes successfully    |
 | onError          | (hook: HookName, message: Message, error: Error) => Promise<boolean>  | Called when the consumer handler execution throws an uncaught error |
 | onSQSError       | (error: Error, message?: Message) => Promise<void>                    | Called when the consumer receives an error from the SQS service     |

package/dist/cjs/hooks.d.ts

@@ -1,7 +1,7 @@
 /** biome-ignore-all lint/complexity/noBannedTypes: Function required as generic */
 import type { Message } from "@fgiova/mini-sqs-client";
 import type { Logger } from "./logger";
-export type HookName = "onStart" | "onStop" | "onPoll" | "onMessage" | "onHandlerSuccess" | "onHandlerTimeout" | "onHandlerError" | "onSuccess" | "onError" | "onSQSError";
+export type HookName = "onStart" | "onStop" | "onPoll" | "onMessage" | "onHandlerSuccess" | "onHandlerTimeout" | "onHandlerError" | "onMaxReceiveCountExceeded" | "onSuccess" | "onError" | "onSQSError";
 export type HookSignatureMap = {
     onStart: {
         args: [sqsConsumer: unknown];
@@ -31,6 +31,10 @@ export type HookSignatureMap = {
         args: [message: Message, error: Error];
         return: boolean;
     };
+    onMaxReceiveCountExceeded: {
+        args: [message: Message, count: number];
+        return: boolean;
+    };
     onSuccess: {
         args: [message: Message];
         return: boolean;

package/dist/cjs/hooks.js

@@ -39,6 +39,11 @@ class Hooks {
                 throwable: false,
                 type: "boolean-return",
             },
+            onMaxReceiveCountExceeded: {
+                S: Symbol("maxReceiveCountExceeded"),
+                throwable: false,
+                type: "boolean-return",
+            },
             onSuccess: {
                 S: Symbol("success"),
                 throwable: false,
@@ -64,6 +69,7 @@ class Hooks {
             [this.hookSymbols.onHandlerSuccess.S]: [],
             [this.hookSymbols.onHandlerTimeout.S]: [],
             [this.hookSymbols.onHandlerError.S]: [],
+            [this.hookSymbols.onMaxReceiveCountExceeded.S]: [],
             [this.hookSymbols.onSuccess.S]: [],
             [this.hookSymbols.onError.S]: [],
             [this.hookSymbols.onSQSError.S]: [],

package/dist/cjs/index.d.ts

@@ -23,6 +23,9 @@ type ConsumerOptions = {
     itemsPerRequest?: number;
     messageAttributeNames?: string[];
     attributeNames?: string[];
+    errorDelaySeconds?: number;
+    errorDelayBackoff?: (attempt: number) => number;
+    maxReceiveCount?: number | ((attempt: number) => number);
 };
 type ConsumerHookName = Exclude<HookName, "onStart" | "onStop">;
 export type HooksOptions = {

package/dist/cjs/index.js

@@ -42,7 +42,18 @@ class SQSConsumer {
             waitTimeSeconds: options.consumerOptions?.waitTimeSeconds ?? 20,
             itemsPerRequest: options.consumerOptions?.itemsPerRequest ?? 10,
             messageAttributeNames: options.consumerOptions?.messageAttributeNames ?? [],
-            attributeNames: options.consumerOptions?.attributeNames ?? [],
+            attributeNames: [
+                ...new Set([
+                    ...(options.consumerOptions?.attributeNames ?? []),
+                    ...(options.consumerOptions?.errorDelayBackoff ||
+                        options.consumerOptions?.maxReceiveCount !== undefined
+                        ? ["ApproximateReceiveCount"]
+                        : []),
+                ]),
+            ],
+            errorDelaySeconds: options.consumerOptions?.errorDelaySeconds ?? 0,
+            errorDelayBackoff: options.consumerOptions?.errorDelayBackoff,
+            maxReceiveCount: options.consumerOptions?.maxReceiveCount,
         };
         this.sqsClient =
             options.clientOptions?.sqsClient ??
@@ -120,9 +131,11 @@ class SQSConsumer {
         }
     }
     haExtendVisibilityTimeout(pendingReceipts, visibilityTimeout) {
-        if (this.handlerOptions.extendVisibilityTimeout !== false &&
-            visibilityTimeout > 0) {
+        if (this.handlerOptions.extendVisibilityTimeout && visibilityTimeout > 0) {
             return setInterval(async () => {
+                /* c8 ignore next 1 */
+                if (!pendingReceipts.size)
+                    return;
                 try {
                     await this.sqsClient.changeMessageVisibilityBatch(this.queueARN, [...pendingReceipts], visibilityTimeout);
                 } /* c8 ignore next 3 */
@@ -138,10 +151,46 @@ class SQSConsumer {
         const candidateToRelease = [];
         for (const result of messagesResults) {
             if (result.error) {
-                // biome-ignore lint/style/noNonNullAssertion: ReceiptHandle must be present here
-                candidateToRelease.push(result.message.ReceiptHandle);
+                let delaySeconds = this.consumerOptions.errorDelaySeconds;
+                const maxReceiveCount = this.consumerOptions.maxReceiveCount;
+                if (maxReceiveCount !== undefined) {
+                    try {
+                        const receiveCount = Number(
+                        /* c8 ignore next 1 */
+                        result.message.Attributes?.ApproximateReceiveCount ?? 0);
+                        if (receiveCount >=
+                            (typeof maxReceiveCount === "function"
+                                ? maxReceiveCount(receiveCount)
+                                : maxReceiveCount)) {
+                            const toDelete = await this.hooks.runHook("onMaxReceiveCountExceeded", result.message, receiveCount);
+                            if (toDelete) {
+                                // biome-ignore lint/style/noNonNullAssertion: ReceiptHandle must be present here
+                                candidatesToDelete.push(result.message.ReceiptHandle);
+                                continue;
+                            }
+                        }
+                    }
+                    catch (e) {
+                        await this.hooks.runHook("onSQSError", e);
+                    }
+                }
+                if (this.consumerOptions.errorDelayBackoff) {
+                    try {
+                        delaySeconds = this.consumerOptions.errorDelayBackoff(
+                        /* c8 ignore next 1 */
+                        Number(result.message.Attributes?.ApproximateReceiveCount ?? 0));
+                    }
+                    catch (e) {
+                        await this.hooks.runHook("onSQSError", e);
+                    }
+                }
+                candidateToRelease.push({
+                    // biome-ignore lint/style/noNonNullAssertion: ReceiptHandle must be present here
+                    receiptHandle: result.message.ReceiptHandle,
+                    visibilityTimeout: Math.min(43200, Math.max(0, Math.trunc(delaySeconds) || 0)),
+                });
             }
-            else if (this.handlerOptions.deleteMessage !== false) {
+            else if (this.handlerOptions.deleteMessage) {
                 // biome-ignore lint/style/noNonNullAssertion: ReceiptHandle must be present here
                 candidatesToDelete.push(result.message.ReceiptHandle);
             }
@@ -150,15 +199,14 @@ class SQSConsumer {
             await this.sqsClient.deleteMessageBatch(this.queueARN, candidatesToDelete);
         }
         if (candidateToRelease.length) {
-            await this.sqsClient.changeMessageVisibilityBatch(this.queueARN, candidateToRelease, 0);
+            await this.sqsClient.changeMessageVisibilityBatch(this.queueARN, candidateToRelease);
         }
     }
     async pollMessages() {
         let consecutiveErrors = 0;
         while (this.running) {
             try {
-                /* c8 ignore next 1 */
-                const visibilityTimeout = this.consumerOptions.visibilityTimeout ?? 30;
+                const visibilityTimeout = this.consumerOptions.visibilityTimeout;
                 const messagesResult = await this.sqsClient.receiveMessage(this.queueARN, {
                     WaitTimeSeconds: this.consumerOptions.waitTimeSeconds,
                     MaxNumberOfMessages: this.consumerOptions.itemsPerRequest,
@@ -175,13 +223,14 @@ class SQSConsumer {
                     messages.map((message) => message.ReceiptHandle));
                     const haTimeout = this.haExtendVisibilityTimeout(pendingReceipts, visibilityTimeout);
                     try {
-                        if (this.handlerOptions.parallelExecution !== false) {
+                        if (this.handlerOptions.parallelExecution) {
                             const results = await (0, p_map_1.default)(messages, async (message) => {
                                 return await this.runOnMessageHook(message);
                             }, {
                                 concurrency: 10,
                                 stopOnError: false,
                             });
+                            pendingReceipts.clear();
                             await this.deleteMessages(results);
                         }
                         else {

package/dist/esm/hooks.d.mts

@@ -1,7 +1,7 @@
 /** biome-ignore-all lint/complexity/noBannedTypes: Function required as generic */
 import type { Message } from "@fgiova/mini-sqs-client";
 import type { Logger } from "./logger.mjs";
-export type HookName = "onStart" | "onStop" | "onPoll" | "onMessage" | "onHandlerSuccess" | "onHandlerTimeout" | "onHandlerError" | "onSuccess" | "onError" | "onSQSError";
+export type HookName = "onStart" | "onStop" | "onPoll" | "onMessage" | "onHandlerSuccess" | "onHandlerTimeout" | "onHandlerError" | "onMaxReceiveCountExceeded" | "onSuccess" | "onError" | "onSQSError";
 export type HookSignatureMap = {
     onStart: {
         args: [sqsConsumer: unknown];
@@ -31,6 +31,10 @@ export type HookSignatureMap = {
         args: [message: Message, error: Error];
         return: boolean;
     };
+    onMaxReceiveCountExceeded: {
+        args: [message: Message, count: number];
+        return: boolean;
+    };
     onSuccess: {
         args: [message: Message];
         return: boolean;

package/dist/esm/hooks.mjs

@@ -36,6 +36,11 @@ export class Hooks {
                 throwable: false,
                 type: "boolean-return",
             },
+            onMaxReceiveCountExceeded: {
+                S: Symbol("maxReceiveCountExceeded"),
+                throwable: false,
+                type: "boolean-return",
+            },
             onSuccess: {
                 S: Symbol("success"),
                 throwable: false,
@@ -61,6 +66,7 @@ export class Hooks {
             [this.hookSymbols.onHandlerSuccess.S]: [],
             [this.hookSymbols.onHandlerTimeout.S]: [],
             [this.hookSymbols.onHandlerError.S]: [],
+            [this.hookSymbols.onMaxReceiveCountExceeded.S]: [],
             [this.hookSymbols.onSuccess.S]: [],
             [this.hookSymbols.onError.S]: [],
             [this.hookSymbols.onSQSError.S]: [],

package/dist/esm/index.d.mts

@@ -23,6 +23,9 @@ type ConsumerOptions = {
     itemsPerRequest?: number;
     messageAttributeNames?: string[];
     attributeNames?: string[];
+    errorDelaySeconds?: number;
+    errorDelayBackoff?: (attempt: number) => number;
+    maxReceiveCount?: number | ((attempt: number) => number);
 };
 type ConsumerHookName = Exclude<HookName, "onStart" | "onStop">;
 export type HooksOptions = {

package/dist/esm/index.mjs

@@ -35,7 +35,18 @@ export class SQSConsumer {
             waitTimeSeconds: options.consumerOptions?.waitTimeSeconds ?? 20,
             itemsPerRequest: options.consumerOptions?.itemsPerRequest ?? 10,
             messageAttributeNames: options.consumerOptions?.messageAttributeNames ?? [],
-            attributeNames: options.consumerOptions?.attributeNames ?? [],
+            attributeNames: [
+                ...new Set([
+                    ...(options.consumerOptions?.attributeNames ?? []),
+                    ...(options.consumerOptions?.errorDelayBackoff ||
+                        options.consumerOptions?.maxReceiveCount !== undefined
+                        ? ["ApproximateReceiveCount"]
+                        : []),
+                ]),
+            ],
+            errorDelaySeconds: options.consumerOptions?.errorDelaySeconds ?? 0,
+            errorDelayBackoff: options.consumerOptions?.errorDelayBackoff,
+            maxReceiveCount: options.consumerOptions?.maxReceiveCount,
         };
         this.sqsClient =
             options.clientOptions?.sqsClient ??
@@ -113,9 +124,11 @@ export class SQSConsumer {
         }
     }
     haExtendVisibilityTimeout(pendingReceipts, visibilityTimeout) {
-        if (this.handlerOptions.extendVisibilityTimeout !== false &&
-            visibilityTimeout > 0) {
+        if (this.handlerOptions.extendVisibilityTimeout && visibilityTimeout > 0) {
             return setInterval(async () => {
+                /* c8 ignore next 1 */
+                if (!pendingReceipts.size)
+                    return;
                 try {
                     await this.sqsClient.changeMessageVisibilityBatch(this.queueARN, [...pendingReceipts], visibilityTimeout);
                 } /* c8 ignore next 3 */
@@ -131,10 +144,46 @@ export class SQSConsumer {
         const candidateToRelease = [];
         for (const result of messagesResults) {
             if (result.error) {
-                // biome-ignore lint/style/noNonNullAssertion: ReceiptHandle must be present here
-                candidateToRelease.push(result.message.ReceiptHandle);
+                let delaySeconds = this.consumerOptions.errorDelaySeconds;
+                const maxReceiveCount = this.consumerOptions.maxReceiveCount;
+                if (maxReceiveCount !== undefined) {
+                    try {
+                        const receiveCount = Number(
+                        /* c8 ignore next 1 */
+                        result.message.Attributes?.ApproximateReceiveCount ?? 0);
+                        if (receiveCount >=
+                            (typeof maxReceiveCount === "function"
+                                ? maxReceiveCount(receiveCount)
+                                : maxReceiveCount)) {
+                            const toDelete = await this.hooks.runHook("onMaxReceiveCountExceeded", result.message, receiveCount);
+                            if (toDelete) {
+                                // biome-ignore lint/style/noNonNullAssertion: ReceiptHandle must be present here
+                                candidatesToDelete.push(result.message.ReceiptHandle);
+                                continue;
+                            }
+                        }
+                    }
+                    catch (e) {
+                        await this.hooks.runHook("onSQSError", e);
+                    }
+                }
+                if (this.consumerOptions.errorDelayBackoff) {
+                    try {
+                        delaySeconds = this.consumerOptions.errorDelayBackoff(
+                        /* c8 ignore next 1 */
+                        Number(result.message.Attributes?.ApproximateReceiveCount ?? 0));
+                    }
+                    catch (e) {
+                        await this.hooks.runHook("onSQSError", e);
+                    }
+                }
+                candidateToRelease.push({
+                    // biome-ignore lint/style/noNonNullAssertion: ReceiptHandle must be present here
+                    receiptHandle: result.message.ReceiptHandle,
+                    visibilityTimeout: Math.min(43200, Math.max(0, Math.trunc(delaySeconds) || 0)),
+                });
             }
-            else if (this.handlerOptions.deleteMessage !== false) {
+            else if (this.handlerOptions.deleteMessage) {
                 // biome-ignore lint/style/noNonNullAssertion: ReceiptHandle must be present here
                 candidatesToDelete.push(result.message.ReceiptHandle);
             }
@@ -143,15 +192,14 @@ export class SQSConsumer {
             await this.sqsClient.deleteMessageBatch(this.queueARN, candidatesToDelete);
         }
         if (candidateToRelease.length) {
-            await this.sqsClient.changeMessageVisibilityBatch(this.queueARN, candidateToRelease, 0);
+            await this.sqsClient.changeMessageVisibilityBatch(this.queueARN, candidateToRelease);
         }
     }
     async pollMessages() {
         let consecutiveErrors = 0;
         while (this.running) {
             try {
-                /* c8 ignore next 1 */
-                const visibilityTimeout = this.consumerOptions.visibilityTimeout ?? 30;
+                const visibilityTimeout = this.consumerOptions.visibilityTimeout;
                 const messagesResult = await this.sqsClient.receiveMessage(this.queueARN, {
                     WaitTimeSeconds: this.consumerOptions.waitTimeSeconds,
                     MaxNumberOfMessages: this.consumerOptions.itemsPerRequest,
@@ -168,13 +216,14 @@ export class SQSConsumer {
                     messages.map((message) => message.ReceiptHandle));
                     const haTimeout = this.haExtendVisibilityTimeout(pendingReceipts, visibilityTimeout);
                     try {
-                        if (this.handlerOptions.parallelExecution !== false) {
+                        if (this.handlerOptions.parallelExecution) {
                             const results = await pMap(messages, async (message) => {
                                 return await this.runOnMessageHook(message);
                             }, {
                                 concurrency: 10,
                                 stopOnError: false,
                             });
+                            pendingReceipts.clear();
                             await this.deleteMessages(results);
                         }
                         else {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@fgiova/sqs-consumer",
-	"version": "3.2.1",
+	"version": "3.4.0",
 	"main": "./dist/cjs/index.js",
 	"types": "./dist/cjs/index.d.ts",
 	"exports": {
@@ -56,7 +56,7 @@
 		]
 	},
 	"dependencies": {
-		"@fgiova/mini-sqs-client": "^5.1.0",
+		"@fgiova/mini-sqs-client": "^5.2.0",
 		"@watchable/unpromise": "^1.0.2",
 		"p-map": "^7.0.3"
 	},