@fgiova/fastify-sqs-consumer 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +69 -0
  3. package/dist/index.d.ts +31 -0
  4. package/dist/index.js +133 -0
  5. package/package.json +70 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Francesco Giovannini
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,69 @@
1
+ # fastify sqs-consumer
2
+ By: Francesco Giovannini <fgiova@fgiova.com>
3
+
4
+ version: 1.0.0
5
+ ## Description
6
+ This plugin for fastify 4.x allows you to consume messages from AWS SQS queues.
7
+ On fastify shutdown a simple wait function is called to wait for the end of the processing of the messages in progress.
8
+
9
+ **Warning**<br>
10
+ To use this plugin, you must have correctly configured your [AWS credentials](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html).
11
+
12
+ ## Install
13
+ ```bash
14
+ npm i @fgiova/fastify-sqs-consumer
15
+ ```
16
+ ### Usage
17
+ ```js
18
+ const fastify = require("fastify")()
19
+
20
+ fastify.register(require("fastify-sqs-consumer"), [
21
+ {
22
+ url: "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
23
+ waitTimeSeconds: 20,
24
+ timeout: 10_000,
25
+ batchSize: 10,
26
+ handlerFunction: async (message, fastify) => {
27
+ return true;
28
+ }
29
+ }
30
+ ]);
31
+ ```
32
+ ### Options
33
+ Options are an array of objects with the following properties (one more for each queue):
34
+
35
+ | Option | Type | Description |
36
+ |------------------|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
37
+ | url* | string | The URL of the Amazon SQS queue from which messages are received. |
38
+ | waitTimeSeconds | number | The duration (in seconds, default 20s) for which the call waits for a message to arrive in the queue before returning. If a message is available, the call returns sooner than WaitTimeSeconds. If no messages are available and the wait time expires, the call returns successfully with an empty list of messages. |
39
+ | timeout | number | The duration before the message is considered as failed: default 90000ms. |
40
+ | batchSize | number | The maximum number of messages to return. Amazon SQS never returns more messages than this value (however, fewer messages might be returned). Valid values: 1 to 10. Default: 1. |
41
+ | handlerFunction* | function | The function that will be called for each message. |
42
+ | attributeNames | string[] | Array of caught attributes for each message |
43
+ | events | object | Events functions for the consumer (detail in next table) |
44
+ | sqs | SQSClient | Initialized SQS Client (useful for testing sessions) |
45
+
46
+ ### handlerFunction
47
+
48
+ Handler function is an async function called for each message received from the queue.
49
+ Each Error thrown by the function is caught and the message is not deleted from the queue.
50
+ Otherwise, the message is deleted from the queue.
51
+
52
+ #### Events
53
+
54
+ | Event name | Arguments | Description |
55
+ |-------------------|-----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|
56
+ | error | error Object | Fired when a general error occurs |
57
+ | processingError | error Object, message SQS.Message | Fired when an error occurs pending message handling |
58
+ | timeoutError | error Object, message SQS.Message | Fired when processing of message exceed timeout duration<br/> If function return false message are discarded, otherwise message is bounced to queue. |
59
+ | messageReceived | message SQS.Message | Fired when message is received |
60
+ | messageProcessed | message SQS.Message | Fired when message is successfully processed |
61
+ | responseProcessed | void | Fired after one batch of items (up to batchSize) has been successfully processed. |
62
+ | stopped | void | Fired when the consumer finally stops its work. |
63
+ | empty | void | Fired when the queue is empty (All messages have been consumed). |
64
+
65
+ ## License
66
+ Licensed under [MIT](./LICENSE).
67
+
68
+ ### Acknowledgements
69
+ This project is kindly sponsored by: isendu Srl [www.isendu.com](https://www.isendu.com)
package/dist/index.d.ts ADDED
@@ -0,0 +1,31 @@
1
+ import { Consumer } from "sqs-consumer";
2
+ import type { FastifyInstance } from "fastify";
3
+ import type { Message } from "@aws-sdk/client-sqs";
4
+ import type { SQSClient } from "@aws-sdk/client-sqs";
5
+ declare module "fastify" {
6
+ interface FastifyInstance {
7
+ sqsConsumers: Consumer[];
8
+ }
9
+ }
10
+ type eventsHandlers = {
11
+ error?: (error: Error, message: void | Message | Message[]) => void;
12
+ processingError?: (error: Error, message: Message) => void;
13
+ timeoutError?: (error: Error, message: Message) => void | boolean;
14
+ messageReceived?: (message: Message) => void;
15
+ messageProcessed?: (message: Message) => void;
16
+ responseProcessed?: () => void;
17
+ stopped?: () => void;
18
+ empty?: () => void;
19
+ };
20
+ declare function sqsConsumerPlugin(fastify: FastifyInstance, options: {
21
+ queueUrl: string;
22
+ handlerFunction: (message: Message, fastify: FastifyInstance) => Promise<any>;
23
+ timeout?: number;
24
+ waitTimeSeconds?: number;
25
+ batchSize?: number;
26
+ attributeNames?: string[];
27
+ events?: eventsHandlers;
28
+ sqs?: SQSClient;
29
+ }[], done: any): void;
30
+ declare const _default: typeof sqsConsumerPlugin;
31
+ export = _default;
package/dist/index.js ADDED
@@ -0,0 +1,133 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ const fastify_plugin_1 = __importDefault(require("fastify-plugin"));
6
+ const sqs_consumer_1 = require("sqs-consumer");
7
+ const promises_1 = require("timers/promises");
8
+ ;
9
+ function createConsumer(fastify, queueUrl, handlerFunction, meta, timeout = 90000, waitTimeSeconds = 20, batchSize = 1, attributeNames = [], events, sqs) {
10
+ return sqs_consumer_1.Consumer.create({
11
+ queueUrl,
12
+ handleMessageTimeout: 0,
13
+ waitTimeSeconds,
14
+ visibilityTimeout: timeout / 1000,
15
+ batchSize,
16
+ sqs,
17
+ attributeNames,
18
+ handleMessage: async function handleMessageFunction(message) {
19
+ meta.pendingMessages++;
20
+ try {
21
+ const result = await Promise.race([
22
+ (0, promises_1.setTimeout)(timeout, "timeout"),
23
+ handlerFunction(message, fastify),
24
+ ]);
25
+ if (result === "timeout") {
26
+ const TimeoutError = new Error(`Message ${message.MessageId} Timeout after ${timeout}ms`);
27
+ if (events?.timeoutError && typeof events.timeoutError === "function") {
28
+ const timeout = events.timeoutError(TimeoutError, message);
29
+ /* istanbul ignore else */
30
+ if (timeout === false) {
31
+ meta.pendingMessages--;
32
+ return;
33
+ }
34
+ }
35
+ throw TimeoutError;
36
+ }
37
+ meta.pendingMessages--;
38
+ }
39
+ catch (e) {
40
+ meta.pendingMessages--;
41
+ fastify.log.error(e);
42
+ throw e;
43
+ }
44
+ return;
45
+ }
46
+ })
47
+ .on("error", (err, message) => {
48
+ /* istanbul ignore else */
49
+ if (events?.error) {
50
+ events.error(err, message);
51
+ }
52
+ else {
53
+ fastify.log.error(`[fastify-sqs-consumer] ${err.message}`);
54
+ }
55
+ })
56
+ .on("processing_error", (err, message) => {
57
+ /* istanbul ignore else */
58
+ if (events?.processingError) {
59
+ events.processingError(err, message);
60
+ }
61
+ else {
62
+ fastify.log.error(`[fastify-sqs-consumer] ${err.message}`);
63
+ }
64
+ })
65
+ .on("message_received", (message) => {
66
+ if (events?.messageReceived)
67
+ events.messageReceived(message);
68
+ })
69
+ .on("message_processed", (message) => {
70
+ if (events?.messageProcessed)
71
+ events.messageProcessed(message);
72
+ })
73
+ .on("response_processed", () => {
74
+ if (events?.responseProcessed)
75
+ events.responseProcessed();
76
+ }).on("stopped", () => {
77
+ if (events?.stopped)
78
+ events.stopped();
79
+ })
80
+ .on("empty", () => {
81
+ /* istanbul ignore next */
82
+ if (events?.empty)
83
+ events.empty();
84
+ });
85
+ }
86
+ function sqsConsumerPlugin(fastify, options, done) {
87
+ const metaData = {
88
+ pendingMessages: 0
89
+ };
90
+ const consumers = [];
91
+ fastify.decorate("sqsConsumers", consumers);
92
+ for (const handler of options) {
93
+ const { queueUrl, handlerFunction, timeout, waitTimeSeconds, batchSize, attributeNames, events, sqs } = handler;
94
+ consumers.push(createConsumer(fastify, queueUrl, handlerFunction, metaData, timeout, waitTimeSeconds, batchSize, attributeNames, events, sqs));
95
+ }
96
+ fastify.addHook("onReady", (done) => {
97
+ for (const consumer of consumers) {
98
+ consumer.start();
99
+ }
100
+ done();
101
+ });
102
+ let allStopped = false;
103
+ fastify.addHook("onClose", (fastify, done) => {
104
+ for (const consumer of consumers) {
105
+ consumer.stop();
106
+ consumer.removeAllListeners();
107
+ }
108
+ const interval = setInterval(() => {
109
+ const isRunning = consumers.some((consumer) => consumer.isRunning);
110
+ /* istanbul ignore next */
111
+ if (isRunning) {
112
+ fastify.log.debug("[fastify-sqs-consumer] Some consumers still running... please wait");
113
+ return null;
114
+ }
115
+ else if (!allStopped) {
116
+ allStopped = true;
117
+ fastify.log.debug("[fastify-sqs-consumer] All consumers are stopped");
118
+ }
119
+ if (metaData.pendingMessages <= 0) {
120
+ clearInterval(interval);
121
+ return done();
122
+ }
123
+ else {
124
+ fastify.log.debug(`[fastify-sqs-consumer] ${metaData.pendingMessages} pending messages`);
125
+ }
126
+ }, 50);
127
+ });
128
+ done();
129
+ }
130
+ module.exports = (0, fastify_plugin_1.default)(sqsConsumerPlugin, {
131
+ name: "fastify-sqs-consumer",
132
+ fastify: ">=4.x"
133
+ });
package/package.json ADDED
@@ -0,0 +1,70 @@
1
+ {
2
+ "name": "@fgiova/fastify-sqs-consumer",
3
+ "description": "sqs-consumer fastify plugin",
4
+ "version": "1.0.0",
5
+ "main": "dist/index.js",
6
+ "files": [
7
+ "dist"
8
+ ],
9
+ "scripts": {
10
+ "lint": "eslint 'src/**/*.ts'",
11
+ "clean": "rimraf dist",
12
+ "build": "tsc",
13
+ "test": "tap -t90 --before=test/scripts/executors/before.js --after=test/scripts/executors/after.js",
14
+ "test:local": "TEST_LOCAL=true tap -t90",
15
+ "test:debug": "tap -T -O --before=test/scripts/executors/before.js --after=test/scripts/executors/after.js"
16
+ },
17
+ "repository": {
18
+ "type": "git",
19
+ "url": "git+https://github.com/fgiova/fastify-sqs-consumer.git"
20
+ },
21
+ "author": "Francesco Giovannini <fgiova@fgiova.com>",
22
+ "license": "MIT",
23
+ "keywords": [
24
+ "fastify",
25
+ "sqs",
26
+ "sqs-consumer",
27
+ "consumer",
28
+ "aws",
29
+ "amazon"
30
+ ],
31
+ "publishConfig": {
32
+ "access": "public"
33
+ },
34
+ "engines": {
35
+ "node": ">=18.0.0"
36
+ },
37
+ "tap": {
38
+ "node-arg": "--require=ts-node/register",
39
+ "ts": true,
40
+ "lines": 100,
41
+ "branches": 100,
42
+ "statements": 100,
43
+ "functions": 100
44
+ },
45
+ "devDependencies": {
46
+ "@aws-sdk/client-sqs": "^3.398.0",
47
+ "@semantic-release/changelog": "^6.0.3",
48
+ "@semantic-release/git": "^10.0.1",
49
+ "@types/node": "^18.17.1",
50
+ "@types/tap": "^15.0.5",
51
+ "@typescript-eslint/eslint-plugin": "^6.4.1",
52
+ "@typescript-eslint/parser": "^6.4.1",
53
+ "conventional-changelog-conventionalcommits": "^6.1.0",
54
+ "cross-env": "^7.0.3",
55
+ "dotenv": "^10.0.0",
56
+ "eslint": "^8.1.0",
57
+ "eslint-plugin-import": "^2.25.2",
58
+ "fastify": "^4.21.0",
59
+ "nodemon": "^2.0.14",
60
+ "semantic-release": "^21.1.0",
61
+ "tap": "^16.3.8",
62
+ "testcontainers": "^10.2.1",
63
+ "ts-node": "^10.9.1",
64
+ "typescript": "^5.1.6"
65
+ },
66
+ "dependencies": {
67
+ "fastify-plugin": "^4.5.1",
68
+ "sqs-consumer": "^7.2.2"
69
+ }
70
+ }