@fgiova/sqs-consumer 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +91 -0
- package/dist/errors.d.ts +3 -0
- package/dist/errors.js +11 -0
- package/dist/hooks.d.ts +14 -0
- package/dist/hooks.js +113 -0
- package/dist/index.d.ts +76 -0
- package/dist/index.js +217 -0
- package/dist/logger.d.ts +11 -0
- package/dist/logger.js +2 -0
- package/package.json +66 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Francesco Giovannini
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# SQS consumer using undici
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/@fgiova/sqs-consumer)
|
|
4
|
+

|
|
5
|
+
[](http://www.typescriptlang.org/)
|
|
6
|
+
[](https://codeclimate.com/github/fgiova/sqs-consumer/maintainability)
|
|
7
|
+
[](https://codeclimate.com/github/fgiova/mini-sqs-client/test_coverage)
|
|
8
|
+
|
|
9
|
+
## Description
|
|
10
|
+
This module allows consuming SQS messages using @fgiova/mini-sqs-client thorough the aws-json protocol with "undici" as http agent .<br />
|
|
11
|
+
The @fgiova/aws-signature module is used for signing requests to optimize performance. <br />
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
```bash
|
|
15
|
+
npm install @fgiova/sqs-consumer
|
|
16
|
+
```
|
|
17
|
+
## Usage
|
|
18
|
+
|
|
19
|
+
```typescript
|
|
20
|
+
import {SQSConsumer} from '@fgiova/sqs-consumer'
|
|
21
|
+
|
|
22
|
+
const consumer = new SQSConsumer({
|
|
23
|
+
queueARN: "arn:aws:sqs:eu-central-1:000000000000:test-queue-hooks",
|
|
24
|
+
handler: async (message) => {
|
|
25
|
+
console.log(message.Body);
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Options
|
|
32
|
+
| Option | Type | Default | Description |
|
|
33
|
+
|-----------------|-------------------------------------|---------|----------------------------------------------------|
|
|
34
|
+
| queueARN | string | | The ARN of the queue to consume |
|
|
35
|
+
| handler | (message: Message) => Promise<void> | | The handler function to be called for each message |
|
|
36
|
+
| logger | Logger | console | The logger to be used |
|
|
37
|
+
| autoStart | boolean | true | Whether to start the consumer automatically |
|
|
38
|
+
| handlerOptions | HandlerOptions | | The options for the handler |
|
|
39
|
+
| clientOptions | ClientOptions | | The options for the client |
|
|
40
|
+
| consumerOptions | ConsumerOptions | | The options for the consumer |
|
|
41
|
+
| hooks | Hooks | | The hooks to be called on specific events |
|
|
42
|
+
|
|
43
|
+
### HandlerOptions
|
|
44
|
+
| Option | Type | Default | Description |
|
|
45
|
+
|-------------------|---------|---------|----------------------------------------------------------------------------------------------------------|
|
|
46
|
+
| deleteMessage | boolean | true | Whether to delete the message after handling (if handler execute without any error) |
|
|
47
|
+
| extendVisibility | boolean | true | Whether to extend the visibility timeout during message handling |
|
|
48
|
+
| excuteTimeout | number | 30000 | The timeout for the handler execution in ms |
|
|
49
|
+
| parallelExecution | boolean | true | If true execute handler in parallel for each batch of messages received, otherwise execute consecutively |
|
|
50
|
+
|
|
51
|
+
### ClientOptions
|
|
52
|
+
| Option | Type | Default | Description |
|
|
53
|
+
|------------------|------------------------|---------|-----------------------------------------------------------------------------------------------------------|
|
|
54
|
+
| sqsClient | MiniSQSClient | | The MiniSQSClient client to be used. If not provided, a new client will be created using the queueARN. |
|
|
55
|
+
| endpoint | string | | The endpoint to be used for the client. If not provided, the endpoint will be inferred from the queueARN. |
|
|
56
|
+
| undiciOptions | Pool.Options | | The options for the undici client. |
|
|
57
|
+
| signer | Signer / SignerOptions | | The signer to be used for signing requests. If not provided, a new singleton signer will be created. |
|
|
58
|
+
| destroySigner | boolean | false | Whether to destroy the signer when the consumer is destroyed. |
|
|
59
|
+
|
|
60
|
+
### ConsumerOptions
|
|
61
|
+
| Option | Type | Default | Description |
|
|
62
|
+
|-------------------------|----------|---------|------------------------------------------------------------|
|
|
63
|
+
| visibilityTimeout | number | 30 | The visibility timeout for the messages in seconds |
|
|
64
|
+
| waitTimeSeconds | number | 20 | The wait time for the receiveMessage call in seconds |
|
|
65
|
+
| itemsPerRequest | number | 10 | The maximum number of messages to be received at once |
|
|
66
|
+
| messageAttributeNames | string[] | [] | The message attribute names to be included in the response |
|
|
67
|
+
|
|
68
|
+
### Hooks
|
|
69
|
+
| Option | Type | Description |
|
|
70
|
+
|------------------|-----------------------------------------------------------------------|----------------------------------------------------------------------|
|
|
71
|
+
| onPoll | (messages: Message[]) => Promise<Message[]> | Called when the consumer polls for messages |
|
|
72
|
+
| onMessage | (message: Message) => Promise<Message> | Called when the consumer receives a message |
|
|
73
|
+
| onHandle | (message: Message) => Promise<Message> | Called when the consumer handles a message |
|
|
74
|
+
| onHandlerSuccess | (message: Message) => Promise<Message> | Called when the consumer handles a message successfully |
|
|
75
|
+
| onHandlerTimeout | (message: Message) => Promise<Message> | Called when the consumer handler execution exceed executionTimeout |
|
|
76
|
+
| onHandlerError | (message: Message, error: Error) => Promise<Boolean> | Called when the consumer handler execution throws an error |
|
|
77
|
+
| onSuccess | (message: Message) => Promise<Message> | Called when the consumer handler execution finishes successfully |
|
|
78
|
+
| onError | ( hook: HookName, message: Message, error: Error) => Promise<Boolean> | Called when the consumer handler execution throws an uncaught error |
|
|
79
|
+
| onSQSError | (error: Error, message?: Message) => Promise<void> | Called when the consumer receives an error from the SQS service |
|
|
80
|
+
|
|
81
|
+
## API
|
|
82
|
+
|
|
83
|
+
```typescript
|
|
84
|
+
SQSConsumer(options: SQSConsumerOptions)
|
|
85
|
+
SQSConsumer.start(): Promise<void>
|
|
86
|
+
SQSConsumer.stop(destroyConsumer = false): Promise<void>
|
|
87
|
+
SQSConsumer.addHook(hookname: string, func: Function): void
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## License
|
|
91
|
+
Licensed under [MIT](./LICENSE).
|
package/dist/errors.d.ts
ADDED
package/dist/errors.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TimeoutError = void 0;
|
|
4
|
+
class TimeoutError extends Error {
|
|
5
|
+
constructor(message) {
|
|
6
|
+
/* c8 ignore next 1 */
|
|
7
|
+
super(message || "Timeout Error");
|
|
8
|
+
this.name = "TimeoutError";
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
exports.TimeoutError = TimeoutError;
|
package/dist/hooks.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Logger } from "./logger";
|
|
2
|
+
import { Message } from "@fgiova/mini-sqs-client";
|
|
3
|
+
export type HookName = "onPoll" | "onMessage" | "onHandlerSuccess" | "onHandlerTimeout" | "onHandlerError" | "onSuccess" | "onError" | "onSQSError";
|
|
4
|
+
export declare class Hooks {
|
|
5
|
+
private readonly hookSymbols;
|
|
6
|
+
private readonly hooks;
|
|
7
|
+
private readonly logger;
|
|
8
|
+
constructor(logger?: Logger);
|
|
9
|
+
addHook(hook: HookName, fn: Function): void;
|
|
10
|
+
runHook(hook: HookName, ...args: any[]): Promise<boolean | void | Message | Message[]>;
|
|
11
|
+
private runHookWithVoidReturn;
|
|
12
|
+
private runHookWithMessageReturn;
|
|
13
|
+
private runHookWithBooleanReturn;
|
|
14
|
+
}
|
package/dist/hooks.js
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Hooks = void 0;
|
|
4
|
+
class Hooks {
|
|
5
|
+
hookSymbols = {
|
|
6
|
+
onPoll: {
|
|
7
|
+
S: Symbol("poll"),
|
|
8
|
+
throwable: true,
|
|
9
|
+
type: "message-return"
|
|
10
|
+
},
|
|
11
|
+
onMessage: {
|
|
12
|
+
S: Symbol("message"),
|
|
13
|
+
throwable: true,
|
|
14
|
+
type: "message-return"
|
|
15
|
+
},
|
|
16
|
+
onHandlerSuccess: {
|
|
17
|
+
S: Symbol("handlerSuccess"),
|
|
18
|
+
throwable: false,
|
|
19
|
+
type: "message-return"
|
|
20
|
+
},
|
|
21
|
+
onHandlerTimeout: {
|
|
22
|
+
S: Symbol("handlerTimeout"),
|
|
23
|
+
throwable: false,
|
|
24
|
+
type: "boolean-return"
|
|
25
|
+
},
|
|
26
|
+
onHandlerError: {
|
|
27
|
+
S: Symbol("handlerError"),
|
|
28
|
+
throwable: false,
|
|
29
|
+
type: "boolean-return"
|
|
30
|
+
},
|
|
31
|
+
onSuccess: {
|
|
32
|
+
S: Symbol("success"),
|
|
33
|
+
throwable: false,
|
|
34
|
+
type: "boolean-return"
|
|
35
|
+
},
|
|
36
|
+
onError: {
|
|
37
|
+
S: Symbol("error"),
|
|
38
|
+
throwable: false,
|
|
39
|
+
type: "boolean-return"
|
|
40
|
+
},
|
|
41
|
+
onSQSError: {
|
|
42
|
+
S: Symbol("sqsError"),
|
|
43
|
+
throwable: false,
|
|
44
|
+
type: "void"
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
hooks;
|
|
48
|
+
logger;
|
|
49
|
+
constructor(logger = console) {
|
|
50
|
+
this.logger = logger;
|
|
51
|
+
this.hooks = {
|
|
52
|
+
[this.hookSymbols.onPoll.S]: [],
|
|
53
|
+
[this.hookSymbols.onMessage.S]: [],
|
|
54
|
+
[this.hookSymbols.onHandlerSuccess.S]: [],
|
|
55
|
+
[this.hookSymbols.onHandlerTimeout.S]: [],
|
|
56
|
+
[this.hookSymbols.onHandlerError.S]: [],
|
|
57
|
+
[this.hookSymbols.onSuccess.S]: [],
|
|
58
|
+
[this.hookSymbols.onError.S]: [],
|
|
59
|
+
[this.hookSymbols.onSQSError.S]: [],
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
addHook(hook, fn) {
|
|
63
|
+
const hookSymbol = this.hookSymbols[hook];
|
|
64
|
+
if (!hookSymbol)
|
|
65
|
+
throw new Error(`Invalid hook ${hook}`);
|
|
66
|
+
this.hooks[hookSymbol.S]?.push(fn);
|
|
67
|
+
}
|
|
68
|
+
async runHook(hook, ...args) {
|
|
69
|
+
const hookSymbol = this.hookSymbols[hook];
|
|
70
|
+
try {
|
|
71
|
+
if (!hookSymbol)
|
|
72
|
+
throw new Error(`Invalid hook ${hook}`);
|
|
73
|
+
switch (hookSymbol.type) {
|
|
74
|
+
case "boolean-return":
|
|
75
|
+
return await this.runHookWithBooleanReturn(hookSymbol, ...args);
|
|
76
|
+
case "message-return":
|
|
77
|
+
const [message, ...someArgs] = args;
|
|
78
|
+
return await this.runHookWithMessageReturn(hookSymbol, message, ...someArgs);
|
|
79
|
+
case "void":
|
|
80
|
+
return await this.runHookWithVoidReturn(hookSymbol, ...args);
|
|
81
|
+
}
|
|
82
|
+
/* c8 ignore next 1 */
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
error.message = `Error running hook ${hook}: ${error.message}`;
|
|
86
|
+
if (hookSymbol.throwable !== false)
|
|
87
|
+
throw error;
|
|
88
|
+
this.logger.error(error);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
async runHookWithVoidReturn(hookSymbol, ...args) {
|
|
92
|
+
for (const fn of this.hooks[hookSymbol.S]) {
|
|
93
|
+
await fn(...args);
|
|
94
|
+
}
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
async runHookWithMessageReturn(hookSymbol, message, ...args) {
|
|
98
|
+
for (const fn of this.hooks[hookSymbol.S]) {
|
|
99
|
+
message = await fn(message, ...args);
|
|
100
|
+
}
|
|
101
|
+
return message;
|
|
102
|
+
}
|
|
103
|
+
async runHookWithBooleanReturn(hookSymbol, ...args) {
|
|
104
|
+
for (const fn of this.hooks[hookSymbol.S]) {
|
|
105
|
+
const continueLoop = await fn(...args);
|
|
106
|
+
if (!continueLoop) {
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return true;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
exports.Hooks = Hooks;
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { Signer, SignerOptions } from "@fgiova/aws-signature";
|
|
2
|
+
import type { Pool } from "undici";
|
|
3
|
+
import { MiniSQSClient, Message } from "@fgiova/mini-sqs-client";
|
|
4
|
+
import { HookName } from "./hooks";
|
|
5
|
+
import { Logger } from "./logger";
|
|
6
|
+
type HandlerOptions = {
|
|
7
|
+
deleteMessage?: boolean;
|
|
8
|
+
extendVisibilityTimeout?: boolean;
|
|
9
|
+
executionTimeout?: number;
|
|
10
|
+
parallelExecution?: boolean;
|
|
11
|
+
};
|
|
12
|
+
type ClientOptions = {
|
|
13
|
+
sqsClient?: MiniSQSClient;
|
|
14
|
+
endpoint?: string;
|
|
15
|
+
undiciOptions?: Pool.Options;
|
|
16
|
+
signer?: Signer | SignerOptions;
|
|
17
|
+
destroySigner?: boolean;
|
|
18
|
+
};
|
|
19
|
+
type ConsumerOptions = {
|
|
20
|
+
visibilityTimeout?: number;
|
|
21
|
+
waitTimeSeconds?: number;
|
|
22
|
+
itemsPerRequest?: number;
|
|
23
|
+
messageAttributeNames?: string[];
|
|
24
|
+
};
|
|
25
|
+
type HooksOptions = {
|
|
26
|
+
onPoll?: (messages: Message[]) => Promise<Message[]>;
|
|
27
|
+
onMessage?: (message: Message) => Promise<Message>;
|
|
28
|
+
onHandlerSuccess?: (message: Message) => Promise<Message>;
|
|
29
|
+
onHandlerTimeout?: (message: Message) => Promise<Boolean>;
|
|
30
|
+
onHandlerError?: (message: Message, error: Error) => Promise<Boolean>;
|
|
31
|
+
onSuccess?: (message: Message) => Promise<Boolean>;
|
|
32
|
+
onError?: (hook: HookName, message: Message, error: Error) => Promise<Boolean>;
|
|
33
|
+
onSQSError?: (error: Error, message?: Message) => Promise<void>;
|
|
34
|
+
};
|
|
35
|
+
export type SQSConsumerOptions = {
|
|
36
|
+
queueARN: string;
|
|
37
|
+
handler: (message: Message) => Promise<any>;
|
|
38
|
+
logger?: Logger;
|
|
39
|
+
autostart?: boolean;
|
|
40
|
+
handlerOptions?: HandlerOptions;
|
|
41
|
+
clientOptions?: ClientOptions;
|
|
42
|
+
consumerOptions?: ConsumerOptions;
|
|
43
|
+
hooks?: HooksOptions;
|
|
44
|
+
};
|
|
45
|
+
export declare class SQSConsumer {
|
|
46
|
+
private readonly queueARN;
|
|
47
|
+
private readonly handlerOptions;
|
|
48
|
+
private readonly clientOptions;
|
|
49
|
+
private readonly consumerOptions;
|
|
50
|
+
private readonly sqsClient;
|
|
51
|
+
private readonly hooks;
|
|
52
|
+
private readonly messageHandler;
|
|
53
|
+
private messagesOnFly;
|
|
54
|
+
private running;
|
|
55
|
+
private polling;
|
|
56
|
+
private destroyed;
|
|
57
|
+
private readonly logger;
|
|
58
|
+
constructor(options: SQSConsumerOptions);
|
|
59
|
+
private runOnMessageHook;
|
|
60
|
+
private runHandler;
|
|
61
|
+
private haExtendVisibilityTimeout;
|
|
62
|
+
private deleteMessages;
|
|
63
|
+
private pollMessages;
|
|
64
|
+
start(): Promise<void>;
|
|
65
|
+
stop(destroy?: boolean): Promise<void>;
|
|
66
|
+
addHook(hookName: "onPoll", value: (messages: Message[]) => Promise<Message[]>): void;
|
|
67
|
+
addHook(hookName: "onMessage", value: (message: Message) => Promise<Message>): void;
|
|
68
|
+
addHook(hookName: "onHandlerSuccess", value: (message: Message) => Promise<Message>): void;
|
|
69
|
+
addHook(hookName: "onHandlerTimeout", value: (message: Message) => Promise<Boolean>): void;
|
|
70
|
+
addHook(hookName: "onHandlerError", value: (message: Message, error: Error) => Promise<Boolean>): void;
|
|
71
|
+
addHook(hookName: "onSuccess", value: (message: Message) => Promise<Boolean>): void;
|
|
72
|
+
addHook(hookName: "onError", value: (hook: HookName, message: Message, error: Error) => Promise<Boolean>): void;
|
|
73
|
+
addHook(hookName: "onSQSError", value: (error: Error, message?: Message) => Promise<void>): void;
|
|
74
|
+
addHook(hookName: HookName, value: Function): void;
|
|
75
|
+
}
|
|
76
|
+
export {};
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.SQSConsumer = void 0;
|
|
7
|
+
const mini_sqs_client_1 = require("@fgiova/mini-sqs-client");
|
|
8
|
+
const hooks_1 = require("./hooks");
|
|
9
|
+
const errors_1 = require("./errors");
|
|
10
|
+
const promises_1 = require("timers/promises");
|
|
11
|
+
const p_map_1 = __importDefault(require("p-map"));
|
|
12
|
+
class SQSConsumer {
|
|
13
|
+
queueARN;
|
|
14
|
+
handlerOptions;
|
|
15
|
+
clientOptions;
|
|
16
|
+
consumerOptions;
|
|
17
|
+
sqsClient;
|
|
18
|
+
hooks;
|
|
19
|
+
messageHandler;
|
|
20
|
+
messagesOnFly = 0;
|
|
21
|
+
running = false;
|
|
22
|
+
polling = false;
|
|
23
|
+
destroyed = false;
|
|
24
|
+
logger;
|
|
25
|
+
constructor(options) {
|
|
26
|
+
if (!options.queueARN)
|
|
27
|
+
throw new Error("queueARN is required");
|
|
28
|
+
if (!options.handler || typeof options.handler !== "function")
|
|
29
|
+
throw new Error("handler is required and must be a function");
|
|
30
|
+
this.logger = options.logger ?? console;
|
|
31
|
+
this.hooks = new hooks_1.Hooks(this.logger);
|
|
32
|
+
this.queueARN = options.queueARN;
|
|
33
|
+
const region = this.queueARN.split(":").reverse()[2];
|
|
34
|
+
this.clientOptions = { destroySigner: options.clientOptions?.destroySigner ?? true };
|
|
35
|
+
this.handlerOptions = {
|
|
36
|
+
deleteMessage: options.handlerOptions?.deleteMessage ?? true,
|
|
37
|
+
extendVisibilityTimeout: options.handlerOptions?.extendVisibilityTimeout ?? true,
|
|
38
|
+
executionTimeout: options.handlerOptions?.executionTimeout ?? 30000,
|
|
39
|
+
parallelExecution: options.handlerOptions?.parallelExecution ?? true
|
|
40
|
+
};
|
|
41
|
+
this.consumerOptions = {
|
|
42
|
+
visibilityTimeout: options.consumerOptions?.visibilityTimeout ?? 30,
|
|
43
|
+
waitTimeSeconds: options.consumerOptions?.waitTimeSeconds ?? 20,
|
|
44
|
+
itemsPerRequest: options.consumerOptions?.itemsPerRequest ?? 10,
|
|
45
|
+
messageAttributeNames: options.consumerOptions?.messageAttributeNames ?? []
|
|
46
|
+
};
|
|
47
|
+
this.sqsClient = options.clientOptions?.sqsClient ?? new mini_sqs_client_1.MiniSQSClient(region, options.clientOptions?.endpoint, options.clientOptions?.undiciOptions, options.clientOptions?.signer);
|
|
48
|
+
if (options.hooks) {
|
|
49
|
+
for (const [key, value] of Object.entries(options.hooks)) {
|
|
50
|
+
if (typeof value !== "function")
|
|
51
|
+
throw new Error(`${key} must be a function`);
|
|
52
|
+
this.addHook(key, value);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
this.messageHandler = options.handler;
|
|
56
|
+
if (options.autostart !== false) {
|
|
57
|
+
void this.start();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async runOnMessageHook(message) {
|
|
61
|
+
try {
|
|
62
|
+
await this.hooks.runHook("onMessage", message);
|
|
63
|
+
const result = await this.runHandler(this.messageHandler, message);
|
|
64
|
+
await this.hooks.runHook("onSuccess", message);
|
|
65
|
+
return {
|
|
66
|
+
message,
|
|
67
|
+
result
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
catch (e) {
|
|
71
|
+
await this.hooks.runHook("onError", "onMessage", message, e);
|
|
72
|
+
return {
|
|
73
|
+
message,
|
|
74
|
+
error: e
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
async runHandler(handler, message) {
|
|
79
|
+
try {
|
|
80
|
+
let handlerResult;
|
|
81
|
+
if (this.handlerOptions.executionTimeout) {
|
|
82
|
+
const response = await Promise.race([
|
|
83
|
+
handler(message),
|
|
84
|
+
new Promise((resolve, reject) => {
|
|
85
|
+
setTimeout(() => {
|
|
86
|
+
reject(new errors_1.TimeoutError("Handler execution timed out"));
|
|
87
|
+
}, this.handlerOptions.executionTimeout);
|
|
88
|
+
})
|
|
89
|
+
]);
|
|
90
|
+
handlerResult = response;
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
handlerResult = await handler(message);
|
|
94
|
+
}
|
|
95
|
+
await Promise.all([
|
|
96
|
+
this.hooks.runHook("onHandlerSuccess", message),
|
|
97
|
+
]);
|
|
98
|
+
return handlerResult;
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
if (error instanceof errors_1.TimeoutError) {
|
|
102
|
+
await this.hooks.runHook("onHandlerTimeout", message);
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
await this.hooks.runHook("onHandlerError", message, error);
|
|
106
|
+
}
|
|
107
|
+
throw error;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
haExtendVisibilityTimeout(messages, visibilityTimeout) {
|
|
111
|
+
if (this.handlerOptions.extendVisibilityTimeout !== false) {
|
|
112
|
+
const processingMessages = messages.map(message => message.ReceiptHandle);
|
|
113
|
+
return setInterval(async () => {
|
|
114
|
+
try {
|
|
115
|
+
await this.sqsClient.changeMessageVisibilityBatch(this.queueARN, processingMessages, visibilityTimeout);
|
|
116
|
+
}
|
|
117
|
+
/* c8 ignore next 3 */
|
|
118
|
+
catch (e) {
|
|
119
|
+
await this.hooks.runHook("onSQSError", e);
|
|
120
|
+
}
|
|
121
|
+
}, (visibilityTimeout * 1000 - 5));
|
|
122
|
+
}
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
async deleteMessages(messagesResults) {
|
|
126
|
+
const candidatesToDelete = [];
|
|
127
|
+
const candidateToRelease = [];
|
|
128
|
+
for (const result of messagesResults) {
|
|
129
|
+
if (result.result && this.handlerOptions.deleteMessage !== false) {
|
|
130
|
+
candidatesToDelete.push(result.message.ReceiptHandle);
|
|
131
|
+
}
|
|
132
|
+
else if (result.error) {
|
|
133
|
+
candidateToRelease.push(result.message.ReceiptHandle);
|
|
134
|
+
}
|
|
135
|
+
if (candidatesToDelete.length)
|
|
136
|
+
await this.sqsClient.deleteMessageBatch(this.queueARN, candidatesToDelete);
|
|
137
|
+
if (candidateToRelease.length)
|
|
138
|
+
await this.sqsClient.changeMessageVisibilityBatch(this.queueARN, candidateToRelease, 0);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
async pollMessages() {
|
|
142
|
+
if (!this.running)
|
|
143
|
+
return;
|
|
144
|
+
try {
|
|
145
|
+
this.polling = true;
|
|
146
|
+
const visibilityTimeout = this.consumerOptions.visibilityTimeout;
|
|
147
|
+
const messagesResult = await this.sqsClient.receiveMessage(this.queueARN, {
|
|
148
|
+
WaitTimeSeconds: this.consumerOptions.waitTimeSeconds,
|
|
149
|
+
MaxNumberOfMessages: this.consumerOptions.itemsPerRequest,
|
|
150
|
+
VisibilityTimeout: visibilityTimeout,
|
|
151
|
+
MessageAttributeNames: this.consumerOptions.messageAttributeNames,
|
|
152
|
+
});
|
|
153
|
+
/* c8 ignore next 1 */
|
|
154
|
+
const messages = messagesResult.Messages ?? [];
|
|
155
|
+
const totalMessages = messages.length;
|
|
156
|
+
this.messagesOnFly += totalMessages;
|
|
157
|
+
await this.hooks.runHook("onPoll", messages);
|
|
158
|
+
if (messages.length) {
|
|
159
|
+
const haTimeout = this.haExtendVisibilityTimeout(messages, visibilityTimeout);
|
|
160
|
+
this.messagesOnFly += (messages.length - totalMessages);
|
|
161
|
+
if (this.handlerOptions.parallelExecution !== false) {
|
|
162
|
+
const results = await (0, p_map_1.default)(messages, async (message) => {
|
|
163
|
+
return await this.runOnMessageHook(message);
|
|
164
|
+
}, {
|
|
165
|
+
concurrency: 10,
|
|
166
|
+
stopOnError: false,
|
|
167
|
+
});
|
|
168
|
+
await this.deleteMessages(results);
|
|
169
|
+
this.messagesOnFly -= messages.length;
|
|
170
|
+
}
|
|
171
|
+
else {
|
|
172
|
+
const handlingMessages = [...messages];
|
|
173
|
+
for (let i = 0; i < handlingMessages.length; i++) {
|
|
174
|
+
const result = await this.runOnMessageHook(handlingMessages[i]);
|
|
175
|
+
await this.deleteMessages([result]);
|
|
176
|
+
messages.splice(i, 1);
|
|
177
|
+
this.messagesOnFly -= 1;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
if (haTimeout)
|
|
181
|
+
clearInterval(haTimeout);
|
|
182
|
+
}
|
|
183
|
+
else {
|
|
184
|
+
this.messagesOnFly -= totalMessages;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
catch (e) {
|
|
188
|
+
await this.hooks.runHook("onSQSError", e);
|
|
189
|
+
}
|
|
190
|
+
this.polling = false;
|
|
191
|
+
await this.pollMessages();
|
|
192
|
+
}
|
|
193
|
+
async start() {
|
|
194
|
+
if (this.running)
|
|
195
|
+
throw new Error("Consumer is already running");
|
|
196
|
+
if (this.destroyed)
|
|
197
|
+
throw new Error("Consumer is destroyed");
|
|
198
|
+
this.running = true;
|
|
199
|
+
void this.pollMessages();
|
|
200
|
+
}
|
|
201
|
+
async stop(destroy = false) {
|
|
202
|
+
if (!this.running)
|
|
203
|
+
throw new Error("Consumer is not running");
|
|
204
|
+
this.running = false;
|
|
205
|
+
while (this.messagesOnFly > 0 || this.polling) {
|
|
206
|
+
await (0, promises_1.setTimeout)(500);
|
|
207
|
+
}
|
|
208
|
+
if (destroy) {
|
|
209
|
+
await this.sqsClient.destroy(this.clientOptions.destroySigner);
|
|
210
|
+
this.destroyed = true;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
addHook(hookName, value) {
|
|
214
|
+
return this.hooks.addHook(hookName, value);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
exports.SQSConsumer = SQSConsumer;
|
package/dist/logger.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Represents a generic logger that could be a simple console, pino etc.
|
|
3
|
+
*/
|
|
4
|
+
export interface Logger {
|
|
5
|
+
trace(message?: any, ...optionalParams: any[]): void;
|
|
6
|
+
debug(message?: any, ...optionalParams: any[]): void;
|
|
7
|
+
info(message?: any, ...optionalParams: any[]): void;
|
|
8
|
+
warn(message?: any, ...optionalParams: any[]): void;
|
|
9
|
+
error(message?: any, ...optionalParams: any[]): void;
|
|
10
|
+
[x: string]: any;
|
|
11
|
+
}
|
package/dist/logger.js
ADDED
package/package.json
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@fgiova/sqs-consumer",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"main": "dist/index.js",
|
|
5
|
+
"files": [
|
|
6
|
+
"dist"
|
|
7
|
+
],
|
|
8
|
+
"scripts": {
|
|
9
|
+
"prebuild": "rimraf dist",
|
|
10
|
+
"build": "tsc",
|
|
11
|
+
"lint": "eslint 'src/**/*.ts'",
|
|
12
|
+
"test": "tap",
|
|
13
|
+
"test:debug": "tap --only --timeout=0",
|
|
14
|
+
"test:coverage": "tap --coverage-report=lcovonly --coverage-report=text",
|
|
15
|
+
"test:local": "TEST_LOCAL=true tap"
|
|
16
|
+
},
|
|
17
|
+
"repository": {
|
|
18
|
+
"type": "git",
|
|
19
|
+
"url": "git+https://github.com/fgiova/sqs-consumer.git"
|
|
20
|
+
},
|
|
21
|
+
"author": "Francesco Giovannini <fgiova@fgiova.com>",
|
|
22
|
+
"license": "MIT",
|
|
23
|
+
"keywords": [
|
|
24
|
+
"aws",
|
|
25
|
+
"sqs",
|
|
26
|
+
"sqs-consumer"
|
|
27
|
+
],
|
|
28
|
+
"publishConfig": {
|
|
29
|
+
"access": "public"
|
|
30
|
+
},
|
|
31
|
+
"engines": {
|
|
32
|
+
"node": ">=18.0.0"
|
|
33
|
+
},
|
|
34
|
+
"tap": {
|
|
35
|
+
"show-full-coverage": true,
|
|
36
|
+
"before": "./test/scripts/executors/before.js",
|
|
37
|
+
"after": "./test/scripts/executors/teardown.js",
|
|
38
|
+
"exclude": [
|
|
39
|
+
"test/helpers/**/*",
|
|
40
|
+
"test/scripts/**/*"
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
"dependencies": {
|
|
44
|
+
"@fgiova/mini-sqs-client": "^2.2.1",
|
|
45
|
+
"p-map": "^4.0.0"
|
|
46
|
+
},
|
|
47
|
+
"peerDependencies": {
|
|
48
|
+
"@fgiova/aws-signature": "^2.0.0"
|
|
49
|
+
},
|
|
50
|
+
"devDependencies": {
|
|
51
|
+
"@aws-sdk/client-sqs": "^3.451.0",
|
|
52
|
+
"@semantic-release/changelog": "^6.0.3",
|
|
53
|
+
"@semantic-release/git": "^10.0.1",
|
|
54
|
+
"@types/node": "^18.18.6",
|
|
55
|
+
"@typescript-eslint/eslint-plugin": "^6.4.0",
|
|
56
|
+
"@typescript-eslint/parser": "^6.4.0",
|
|
57
|
+
"conventional-changelog-conventionalcommits": "^6.1.0",
|
|
58
|
+
"eslint": "^8.47.0",
|
|
59
|
+
"eslint-plugin-import": "^2.28.1",
|
|
60
|
+
"semantic-release": "^21.0.9",
|
|
61
|
+
"tap": "^18.6.1",
|
|
62
|
+
"testcontainers": "^10.2.1",
|
|
63
|
+
"ts-node": "^10.9.1",
|
|
64
|
+
"typescript": "^5.2.2"
|
|
65
|
+
}
|
|
66
|
+
}
|