ac-sqs 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/README.md +4 -1
- package/index.js +359 -84
- package/package.json +7 -7
package/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,53 @@
|
|
|
1
1
|
|
|
2
|
+
# [3.3.0](https://github.com/admiralcloud/ac-sqs/compare/v3.2.2..v3.3.0) (2025-09-26 17:27:53)
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
### Feature
|
|
6
|
+
|
|
7
|
+
* **App:** Add function to create missing queues | MP | [fa112ba346687587530c03fd470402b1ca3d0f98](https://github.com/admiralcloud/ac-sqs/commit/fa112ba346687587530c03fd470402b1ca3d0f98)
|
|
8
|
+
Add function to create missing queues
|
|
9
|
+
Related issues:
|
|
10
|
+
### Chores
|
|
11
|
+
|
|
12
|
+
* **Misc:** Updated packages | MP | [ccde2c1292e657aaceff82d9978459efc3bf24f7](https://github.com/admiralcloud/ac-sqs/commit/ccde2c1292e657aaceff82d9978459efc3bf24f7)
|
|
13
|
+
Updated packages
|
|
14
|
+
Related issues:
|
|
15
|
+
|
|
16
|
+
## [3.2.2](https://github.com/admiralcloud/ac-sqs/compare/v3.2.1..v3.2.2) (2025-09-19 05:21:22)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
### Bug Fix
|
|
20
|
+
|
|
21
|
+
* **Misc:** Package updates | MP | [feed53274af621d33f5c03bf946d9d3274ad1986](https://github.com/admiralcloud/ac-sqs/commit/feed53274af621d33f5c03bf946d9d3274ad1986)
|
|
22
|
+
Package updates
|
|
23
|
+
Related issues:
|
|
24
|
+
|
|
25
|
+
## [3.2.1](https://github.com/admiralcloud/ac-sqs/compare/v3.2.0..v3.2.1) (2025-07-18 12:57:51)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
### Bug Fix
|
|
29
|
+
|
|
30
|
+
* **App:** Reduced code complexity | MP | [0313cc01d90d9e4d1ad3b7ef3136af7dfc72d717](https://github.com/admiralcloud/ac-sqs/commit/0313cc01d90d9e4d1ad3b7ef3136af7dfc72d717)
|
|
31
|
+
Create smaller functions
|
|
32
|
+
Related issues: [admiralcloud/ac-sqs#1](https://github.com/admiralcloud/ac-sqs/issues/1) [admiralcloud/ac-api-server#340](https://github.com/admiralcloud/ac-api-server/issues/340)
|
|
33
|
+
* **App:** Improved code quality | MP | [0d7e40518804f8c227cbc20c602510edada2e02e](https://github.com/admiralcloud/ac-sqs/commit/0d7e40518804f8c227cbc20c602510edada2e02e)
|
|
34
|
+
Separated huge function into smaller parts
|
|
35
|
+
Related issues: [admiralcloud/ac-sqs#1](https://github.com/admiralcloud/ac-sqs/issues/1) [admiralcloud/ac-api-server#340](https://github.com/admiralcloud/ac-api-server/issues/340)
|
|
36
|
+
* **App:** Improved code quality | MP | [cd9f5d3969d4aac9c3c87e7e84dd5788e9929cce](https://github.com/admiralcloud/ac-sqs/commit/cd9f5d3969d4aac9c3c87e7e84dd5788e9929cce)
|
|
37
|
+
Improved code quality
|
|
38
|
+
Related issues:
|
|
39
|
+
* **App:** Requested code changes | MP | [39aa6c33b9087fcb2c3cfa075716a5dd83e07550](https://github.com/admiralcloud/ac-sqs/commit/39aa6c33b9087fcb2c3cfa075716a5dd83e07550)
|
|
40
|
+
Improved code quality
|
|
41
|
+
Related issues:
|
|
42
|
+
* **App:** Improved visibility management | MP | [d45111846668fe3db268fd40987c726e82e1074c](https://github.com/admiralcloud/ac-sqs/commit/d45111846668fe3db268fd40987c726e82e1074c)
|
|
43
|
+
Add batch processing and throttling for visibility extension, better cleanup, graceful shutdown and stats
|
|
44
|
+
Related issues:
|
|
45
|
+
### Chores
|
|
46
|
+
|
|
47
|
+
* **App:** Updated packages | MP | [c6a2d43081047626aa60d5cb5f97427760143880](https://github.com/admiralcloud/ac-sqs/commit/c6a2d43081047626aa60d5cb5f97427760143880)
|
|
48
|
+
Updated packages
|
|
49
|
+
Related issues:
|
|
50
|
+
|
|
2
51
|
# [3.2.0](https://github.com/admiralcloud/ac-sqs/compare/v3.1.2..v3.2.0) (2025-05-11 11:56:57)
|
|
3
52
|
|
|
4
53
|
|
package/README.md
CHANGED
|
@@ -112,6 +112,9 @@ An array of metadata to get. By default only "ApproximateNumberOfMessages" is re
|
|
|
112
112
|
|
|
113
113
|
https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_GetQueueAttributes.html
|
|
114
114
|
|
|
115
|
+
## createQueue
|
|
116
|
+
Call with all lists and let the function check if the queues exist. If not, the missing queue is created.
|
|
117
|
+
|
|
115
118
|
# Test
|
|
116
119
|
You can run tests using **yarn run test**.
|
|
117
120
|
|
|
@@ -120,7 +123,7 @@ Preparations you have to make before running the tests:
|
|
|
120
123
|
+ export the AWS profile to use for tests (if it is not your default profile) using **export AWS_PROFILE=development**
|
|
121
124
|
+ export the AWS account id using **export awsaccount=12345**
|
|
122
125
|
+ create a SQS list named "test_acsqs"
|
|
123
|
-
+ create a bucket and export the name using **export bucket=
|
|
126
|
+
+ create a bucket and export the name using **export bucket=sqstest.admiralcloud.com**
|
|
124
127
|
+ export the node test environment using **export NODE_ENV=test**
|
|
125
128
|
|
|
126
129
|
**ATTENTION**: Tests may fail when checking the SQS length. This is a by-design failure:
|
package/index.js
CHANGED
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
const _ = require('lodash')
|
|
2
2
|
const { v4: uuidV4 } = require('uuid')
|
|
3
|
+
const { setTimeout: sleep } = require('timers/promises')
|
|
3
4
|
|
|
4
|
-
const { SQSClient, SendMessageCommand, SendMessageBatchCommand, ReceiveMessageCommand, DeleteMessageBatchCommand, GetQueueAttributesCommand,
|
|
5
|
+
const { SQSClient, SendMessageCommand, SendMessageBatchCommand, ReceiveMessageCommand, DeleteMessageBatchCommand, GetQueueAttributesCommand, ChangeMessageVisibilityBatchCommand } = require('@aws-sdk/client-sqs')
|
|
5
6
|
const { S3Client, GetObjectCommand, PutObjectCommand, DeleteObjectsCommand } = require("@aws-sdk/client-s3")
|
|
6
7
|
|
|
7
|
-
|
|
8
8
|
class ACSQS {
|
|
9
|
-
constructor({ region = 'eu-central-1', account, availableLists, useS3 = { enabled: true, bucket: undefined }, messageThreshold = 250e3, debug, logger=console, throwError = false }) {
|
|
9
|
+
constructor({ region = 'eu-central-1', account, availableLists, useS3 = { enabled: true, bucket: undefined }, messageThreshold = 250e3, debug, logger = console, throwError = false, maxConcurrentMessages = 3000 }) {
|
|
10
10
|
this.region = region
|
|
11
11
|
this.account = account
|
|
12
12
|
this.availableLists = availableLists
|
|
13
13
|
this.logger = logger
|
|
14
14
|
this.throwError = throwError
|
|
15
|
-
this.
|
|
15
|
+
this.maxConcurrentMessages = maxConcurrentMessages
|
|
16
|
+
|
|
17
|
+
// Improved visibility management
|
|
18
|
+
this.visibilityManagement = new Map()
|
|
19
|
+
this.batchExtendRunning = false
|
|
20
|
+
this.stopBatchExtend = false
|
|
21
|
+
this.batchExtendInterval = 5000 // Check every 5 seconds
|
|
16
22
|
|
|
17
23
|
const awsConfig = {
|
|
18
24
|
region,
|
|
@@ -26,6 +32,281 @@ class ACSQS {
|
|
|
26
32
|
this.bucket = _.get(useS3, 'bucket')
|
|
27
33
|
this.s3 = new S3Client(awsConfig)
|
|
28
34
|
}
|
|
35
|
+
|
|
36
|
+
// Start batch extend loop
|
|
37
|
+
this.startBatchExtendTimer()
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async startBatchExtendTimer() {
|
|
41
|
+
if (this.batchExtendRunning) return
|
|
42
|
+
|
|
43
|
+
this.batchExtendRunning = true
|
|
44
|
+
this.stopBatchExtend = false
|
|
45
|
+
|
|
46
|
+
while (true) {
|
|
47
|
+
try {
|
|
48
|
+
if (this.stopBatchExtend) break
|
|
49
|
+
|
|
50
|
+
await sleep(this.batchExtendInterval)
|
|
51
|
+
|
|
52
|
+
if (this.stopBatchExtend) break
|
|
53
|
+
|
|
54
|
+
await this.processBatchExtensions()
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
this.logger.error('ACSQS | startBatchExtendTimer | Error in batch extend loop | %s', error?.message)
|
|
58
|
+
// Don't break the loop on errors, just log and continue
|
|
59
|
+
await sleep(1000) // Wait 1s on error before retrying
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
this.batchExtendRunning = false
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
stopBatchExtendTimer() {
|
|
67
|
+
this.stopBatchExtend = true
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async processBatchExtensions() {
|
|
71
|
+
if (this.visibilityManagement.size === 0) return
|
|
72
|
+
|
|
73
|
+
// Group messages by queue for batch processing
|
|
74
|
+
const queueGroups = new Map()
|
|
75
|
+
const now = Date.now()
|
|
76
|
+
|
|
77
|
+
for (const [messageId, messageData] of this.visibilityManagement) {
|
|
78
|
+
// Check if message still exists in tracking (might have been deleted)
|
|
79
|
+
if (!this.visibilityManagement.has(messageId)) continue
|
|
80
|
+
|
|
81
|
+
// Check if message needs extension
|
|
82
|
+
if (now >= messageData.nextExtendTime) {
|
|
83
|
+
// Check max extensions
|
|
84
|
+
if (messageData.extensionCount >= messageData.maxExtensions) {
|
|
85
|
+
this.logger.warn('ACSQS | processBatchExtensions | Max extensions reached | %s | %s', messageData.queueName, messageId)
|
|
86
|
+
this.removeVisibilityTracking(messageId)
|
|
87
|
+
continue
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (!queueGroups.has(messageData.queueName)) {
|
|
91
|
+
queueGroups.set(messageData.queueName, [])
|
|
92
|
+
}
|
|
93
|
+
queueGroups.get(messageData.queueName).push(messageData)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Process each queue's extensions in batch
|
|
98
|
+
for (const [queueName, messages] of queueGroups) {
|
|
99
|
+
await this.extendVisibilityBatch(queueName, messages)
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async extendVisibilityBatch(queueName, messages) {
|
|
104
|
+
if (messages.length === 0) return
|
|
105
|
+
|
|
106
|
+
const config = _.find(this.availableLists, { name: queueName })
|
|
107
|
+
if (!config) return
|
|
108
|
+
|
|
109
|
+
const chunks = this.chunkMessages(messages)
|
|
110
|
+
await this.processAllChunks(queueName, chunks, config)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
chunkMessages(messages) {
|
|
114
|
+
return _.chunk(messages, 10)
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async processAllChunks(queueName, chunks, config) {
|
|
118
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
119
|
+
const chunk = chunks[i]
|
|
120
|
+
|
|
121
|
+
// Add delay between chunks to avoid throttling (except for first chunk)
|
|
122
|
+
if (i > 0) {
|
|
123
|
+
await sleep(100)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
await this.processChunk(queueName, chunk, config, i + 1, chunks.length)
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async processChunk(queueName, chunk, config, chunkNumber, totalChunks) {
|
|
131
|
+
const validChunk = this.getValidChunk(chunk)
|
|
132
|
+
if (validChunk.length === 0) return
|
|
133
|
+
|
|
134
|
+
const sqsParams = await this.buildSQSParams(validChunk, config)
|
|
135
|
+
await this.executeChunkWithRetry(queueName, validChunk, sqsParams, config, chunkNumber, totalChunks)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
getValidChunk(chunk) {
|
|
139
|
+
return chunk.filter(messageData => this.visibilityManagement.has(messageData.messageId))
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async buildSQSParams(validChunk, config) {
|
|
143
|
+
const visibilityTimeout = _.get(config, 'visibilityTimeout', 30)
|
|
144
|
+
const entries = validChunk.map(messageData => ({
|
|
145
|
+
Id: messageData.messageId,
|
|
146
|
+
ReceiptHandle: messageData.receiptHandle,
|
|
147
|
+
VisibilityTimeout: visibilityTimeout
|
|
148
|
+
}))
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
QueueUrl: await this.getQueueUrl(config),
|
|
152
|
+
Entries: entries
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async executeChunkWithRetry(queueName, validChunk, sqsParams, config, chunkNumber, totalChunks) {
|
|
157
|
+
const maxRetries = 2
|
|
158
|
+
let lastError = null
|
|
159
|
+
|
|
160
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
161
|
+
if (attempt > 1) {
|
|
162
|
+
await this.handleRetryDelay(lastError, attempt, maxRetries, queueName, chunkNumber, totalChunks)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
try {
|
|
166
|
+
const command = new ChangeMessageVisibilityBatchCommand(sqsParams)
|
|
167
|
+
const response = await this.sqs.send(command)
|
|
168
|
+
|
|
169
|
+
const visibilityTimeout = _.get(config, 'visibilityTimeout', 30)
|
|
170
|
+
this.handleChunkResponse(queueName, validChunk, response, config, visibilityTimeout)
|
|
171
|
+
return
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
lastError = error
|
|
175
|
+
|
|
176
|
+
if (this.isPermanentError(error)) {
|
|
177
|
+
this.logger.warn('ACSQS | processChunk | Permanent error, not retrying | %s | %s', queueName, error.message)
|
|
178
|
+
break
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (attempt < maxRetries) {
|
|
182
|
+
this.logger.warn('ACSQS | processChunk | Attempt %s failed, will retry | %s | %s',
|
|
183
|
+
attempt, queueName, error.message)
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
this.handleAllRetriesFailed(queueName, validChunk, lastError)
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async handleRetryDelay(lastError, attempt, maxRetries, queueName, chunkNumber, totalChunks) {
|
|
192
|
+
const isThrottled = lastError?.message?.includes('throttled')
|
|
193
|
+
const delayMs = isThrottled ? 500 : 200
|
|
194
|
+
|
|
195
|
+
this.logger.warn('ACSQS | processChunk | Retry attempt %s/%s | %s | Chunk %s/%s | Waiting %sms',
|
|
196
|
+
attempt, maxRetries, queueName, chunkNumber, totalChunks, delayMs)
|
|
197
|
+
|
|
198
|
+
await sleep(delayMs)
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
isPermanentError(error) {
|
|
202
|
+
return error.message?.includes('ReceiptHandleIsInvalid') ||
|
|
203
|
+
error.message?.includes('MessageNotInflight')
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
handleAllRetriesFailed(queueName, validChunk, lastError) {
|
|
207
|
+
this.logger.error('ACSQS | processChunk | All retries failed | %s | Chunk size: %s | %s',
|
|
208
|
+
queueName, validChunk.length, lastError?.message)
|
|
209
|
+
|
|
210
|
+
for (const messageData of validChunk) {
|
|
211
|
+
this.removeVisibilityTracking(messageData.messageId)
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
handleChunkResponse(queueName, validChunk, response, config, visibilityTimeout) {
|
|
216
|
+
const successful = new Set((response.Successful || []).map(s => s.Id))
|
|
217
|
+
const failed = new Set((response.Failed || []).map(f => f.Id))
|
|
218
|
+
|
|
219
|
+
for (const messageData of validChunk) {
|
|
220
|
+
// Check again if message still exists (might have been deleted during AWS call)
|
|
221
|
+
if (!this.visibilityManagement.has(messageData.messageId)) continue
|
|
222
|
+
|
|
223
|
+
if (successful.has(messageData.messageId)) {
|
|
224
|
+
this.updateSuccessfulMessage(messageData, visibilityTimeout, queueName, config)
|
|
225
|
+
}
|
|
226
|
+
else if (failed.has(messageData.messageId)) {
|
|
227
|
+
this.handleFailedMessage(messageData, response.Failed, queueName, config)
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
updateSuccessfulMessage(messageData, visibilityTimeout, queueName, config) {
|
|
233
|
+
messageData.extensionCount++
|
|
234
|
+
messageData.nextExtendTime = Date.now() + (visibilityTimeout * 0.8 * 1000)
|
|
235
|
+
|
|
236
|
+
if (config.debug) {
|
|
237
|
+
this.logger.debug('ACSQS | extendVisibilityBatch | Success | %s | M %s | %ss | Count: %s',
|
|
238
|
+
queueName, messageData.messageId, visibilityTimeout, messageData.extensionCount)
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
handleFailedMessage(messageData, failedItems, queueName, config) {
|
|
243
|
+
const failedItem = failedItems.find(f => f.Id === messageData.messageId)
|
|
244
|
+
|
|
245
|
+
// Log only if it's not an expired receipt handle (which is normal)
|
|
246
|
+
if (failedItem?.Code === 'ReceiptHandleIsInvalid') {
|
|
247
|
+
if (config.debug) {
|
|
248
|
+
this.logger.debug('ACSQS | extendVisibilityBatch | Receipt handle expired (normal) | %s | M %s',
|
|
249
|
+
queueName, messageData.messageId)
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
else {
|
|
253
|
+
this.logger.warn('ACSQS | extendVisibilityBatch | Failed | %s | M %s | %s',
|
|
254
|
+
queueName, messageData.messageId, failedItem?.Message || 'Unknown error')
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Remove from tracking if receipt handle expired or other permanent error
|
|
258
|
+
if (failedItem?.Code === 'ReceiptHandleIsInvalid' || failedItem?.Code === 'MessageNotInflight') {
|
|
259
|
+
this.removeVisibilityTracking(messageData.messageId)
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
addVisibilityTracking(messageId, queueName, receiptHandle, config) {
|
|
264
|
+
// Check if we're at max capacity
|
|
265
|
+
if (this.visibilityManagement.size >= this.maxConcurrentMessages) {
|
|
266
|
+
this.logger.warn('ACSQS | addVisibilityTracking | Max concurrent messages reached | %s', this.maxConcurrentMessages)
|
|
267
|
+
return false
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const visibilityTimeout = _.get(config, 'visibilityTimeout', 30)
|
|
271
|
+
const maxExtensions = _.get(config, 'maxVisibilityExtensions', 12)
|
|
272
|
+
|
|
273
|
+
this.visibilityManagement.set(messageId, {
|
|
274
|
+
messageId,
|
|
275
|
+
queueName,
|
|
276
|
+
receiptHandle,
|
|
277
|
+
extensionCount: 0,
|
|
278
|
+
maxExtensions,
|
|
279
|
+
nextExtendTime: Date.now() + (visibilityTimeout * 0.8 * 1000),
|
|
280
|
+
createdAt: Date.now()
|
|
281
|
+
})
|
|
282
|
+
|
|
283
|
+
return true
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
removeVisibilityTracking(messageId) {
|
|
287
|
+
this.visibilityManagement.delete(messageId)
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// Legacy method for backwards compatibility - now uses batch processing
|
|
291
|
+
async extendVisibility({ name, message, throwError }) {
|
|
292
|
+
const config = _.find(this.availableLists, { name })
|
|
293
|
+
if (!config) {
|
|
294
|
+
this.logger.error('AWS | extendVisibility | configurationMissing | %s', name)
|
|
295
|
+
throw new Error('configurationForListMissing')
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const { MessageId: messageId } = message
|
|
299
|
+
|
|
300
|
+
// Check if message is already being tracked
|
|
301
|
+
if (this.visibilityManagement.has(messageId)) {
|
|
302
|
+
// Force immediate extension by setting nextExtendTime to now
|
|
303
|
+
const messageData = this.visibilityManagement.get(messageId)
|
|
304
|
+
messageData.nextExtendTime = Date.now()
|
|
305
|
+
return
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// Add to tracking if not already present
|
|
309
|
+
this.addVisibilityTracking(messageId, name, message.ReceiptHandle, config)
|
|
29
310
|
}
|
|
30
311
|
|
|
31
312
|
async getAllLists({ throwError = false } = {}) {
|
|
@@ -68,6 +349,25 @@ class ACSQS {
|
|
|
68
349
|
}
|
|
69
350
|
}
|
|
70
351
|
|
|
352
|
+
async createQueue({ lists }) {
|
|
353
|
+
for (const list of lists) {
|
|
354
|
+
const config = _.find(this.availableLists, { name: list.name })
|
|
355
|
+
if (!config) {
|
|
356
|
+
this.logger.error('AWS | createQueue | configurationMissing | %s', list.name)
|
|
357
|
+
throw new Error('configurationForListMissing')
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
const queueUrl = await this.getQueueUrl(config)
|
|
361
|
+
const command = new CreateQueueCommand({ QueueName: queueUrl })
|
|
362
|
+
try {
|
|
363
|
+
await this.sqs.send(command)
|
|
364
|
+
}
|
|
365
|
+
catch(e) {
|
|
366
|
+
this.logger.error('AWS | createQueue | %s | %s', list.name, e?.message)
|
|
367
|
+
if (this.throwError) throw e
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
71
371
|
|
|
72
372
|
async sendSQSMessage({ name, message, messageGroupId, deDuplicationId, delay, throwError, debug }) {
|
|
73
373
|
const config = _.find(this.availableLists, { name })
|
|
@@ -165,62 +465,6 @@ class ACSQS {
|
|
|
165
465
|
}
|
|
166
466
|
}
|
|
167
467
|
|
|
168
|
-
async extendVisibility({ name, message, throwError }) {
|
|
169
|
-
const config = _.find(this.availableLists, { name })
|
|
170
|
-
if (!config) {
|
|
171
|
-
this.logger.error('AWS | extendVisibility | configurationMissing | %s', name)
|
|
172
|
-
throw new Error('configurationForListMissing')
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
const visibilityTimeout = _.get(config, 'visibilityTimeout', 15)
|
|
176
|
-
const maxVisibilityExtensions = _.get(config, 'maxVisibilityExtensions', 12) // max number of times the extension can me made (12 x 15s = 3min)
|
|
177
|
-
|
|
178
|
-
const { MessageId: messageId, ReceiptHandle: receiptHandle } = message
|
|
179
|
-
|
|
180
|
-
// Check if we've reached maximum extensions
|
|
181
|
-
if (this.visibilityTimer[messageId] && this.visibilityTimer[messageId].visibilityExtensionCount >= maxVisibilityExtensions) {
|
|
182
|
-
this.logger.warn('ACSQS | extendVisibility | %s | M %s | Max extensions reached | %s | %j', name, messageId, maxVisibilityExtensions, message)
|
|
183
|
-
this.deleteVisibilityTimer({ messageId })
|
|
184
|
-
return
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// Track extension count
|
|
188
|
-
if (this.visibilityTimer[messageId]) {
|
|
189
|
-
this.visibilityTimer[messageId].visibilityExtensionCount++
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
const sqsParams = {
|
|
193
|
-
QueueUrl: await this.getQueueUrl(config),
|
|
194
|
-
ReceiptHandle: receiptHandle,
|
|
195
|
-
VisibilityTimeout: visibilityTimeout
|
|
196
|
-
}
|
|
197
|
-
const command = new ChangeMessageVisibilityCommand(sqsParams)
|
|
198
|
-
try {
|
|
199
|
-
const response = await this.sqs.send(command)
|
|
200
|
-
if (config.debug) {
|
|
201
|
-
const visibilityExtensionCount = this.visibilityTimer[messageId] ? this.visibilityTimer[messageId].visibilityExtensionCount : 0
|
|
202
|
-
this.logger.debug('ACSQS | extendVisibility | %s | M %s | %ss | %s | %j', name, messageId, visibilityTimeout, visibilityExtensionCount, message)
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
return response
|
|
206
|
-
}
|
|
207
|
-
catch(e) {
|
|
208
|
-
this.logger.error('ACSQS | extendVisibility | %s | %s', name, e?.message)
|
|
209
|
-
this.deleteVisibilityTimer({ messageId })
|
|
210
|
-
if (this.throwError || throwError) throw e
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
deleteVisibilityTimer({ messageId }) {
|
|
215
|
-
if (this.visibilityTimer[messageId]) {
|
|
216
|
-
clearTimeout(this.visibilityTimer[messageId].timer)
|
|
217
|
-
const self = this
|
|
218
|
-
setTimeout(() => {
|
|
219
|
-
delete self.visibilityTimer[messageId]
|
|
220
|
-
}, 1000)
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
468
|
async receiveSQSMessages({ name, throwError, debug }) {
|
|
225
469
|
const config = _.find(this.availableLists, { name })
|
|
226
470
|
if (!config) {
|
|
@@ -241,7 +485,6 @@ class ACSQS {
|
|
|
241
485
|
const result = await this.sqs.send(command)
|
|
242
486
|
if (!_.size(result.Messages)) return
|
|
243
487
|
|
|
244
|
-
// Benutze Arrow-Funktion, um `this` beizubehalten
|
|
245
488
|
const messages = await Promise.all(result.Messages.map(async (message) => {
|
|
246
489
|
if (message.Body.startsWith('s3:')) {
|
|
247
490
|
const key = message.Body.replace('s3:', '')
|
|
@@ -251,27 +494,14 @@ class ACSQS {
|
|
|
251
494
|
message.s3key = key
|
|
252
495
|
}
|
|
253
496
|
catch(e) {
|
|
254
|
-
this.logger.error('ACSQS | receiveSQSMessages | s3KeyInvalid | %s', name, key)
|
|
497
|
+
this.logger.error('ACSQS | receiveSQSMessages | s3KeyInvalid | %s | %s', name, key)
|
|
255
498
|
}
|
|
256
499
|
}
|
|
257
500
|
|
|
258
501
|
if (visibilityTimeout > 0) {
|
|
259
|
-
//
|
|
260
|
-
const { MessageId: messageId } = message
|
|
261
|
-
|
|
262
|
-
const self = this // `this` als lokale Variable speichern
|
|
263
|
-
|
|
264
|
-
this.visibilityTimer[messageId] = {
|
|
265
|
-
// Arrow-Funktion für setInterval damit `this` erhalten bleibt,
|
|
266
|
-
// oder verwende die lokale Variable `self`
|
|
267
|
-
timer: setInterval(() => {
|
|
268
|
-
this.extendVisibility({ name, message })
|
|
269
|
-
.catch(e => {
|
|
270
|
-
this.logger.error('ACSQS | AutoExtendVisibility | Failed %s', e.message)
|
|
271
|
-
})
|
|
272
|
-
}, timeoutMs),
|
|
273
|
-
visibilityExtensionCount: 0
|
|
274
|
-
}
|
|
502
|
+
// Add to visibility tracking instead of individual timers
|
|
503
|
+
const { MessageId: messageId, ReceiptHandle: receiptHandle } = message
|
|
504
|
+
this.addVisibilityTracking(messageId, name, receiptHandle, config)
|
|
275
505
|
}
|
|
276
506
|
|
|
277
507
|
return message
|
|
@@ -305,12 +535,10 @@ class ACSQS {
|
|
|
305
535
|
if (item.s3key) {
|
|
306
536
|
s3keys.push({ Key: item.s3key })
|
|
307
537
|
}
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
}
|
|
538
|
+
// Remove from visibility tracking
|
|
539
|
+
this.removeVisibilityTracking(messageId)
|
|
311
540
|
}
|
|
312
541
|
|
|
313
|
-
|
|
314
542
|
let sqsParams = {
|
|
315
543
|
QueueUrl: await this.getQueueUrl(config),
|
|
316
544
|
Entries: entries
|
|
@@ -327,7 +555,7 @@ class ACSQS {
|
|
|
327
555
|
Objects: s3keys,
|
|
328
556
|
}
|
|
329
557
|
}
|
|
330
|
-
const command = new DeleteObjectsCommand(input)
|
|
558
|
+
const command = new DeleteObjectsCommand(input)
|
|
331
559
|
this.s3.send(command)
|
|
332
560
|
}
|
|
333
561
|
return response
|
|
@@ -338,6 +566,54 @@ class ACSQS {
|
|
|
338
566
|
}
|
|
339
567
|
}
|
|
340
568
|
|
|
569
|
+
// Cleanup method for graceful shutdown
|
|
570
|
+
async shutdown() {
|
|
571
|
+
this.stopBatchExtendTimer()
|
|
572
|
+
|
|
573
|
+
// Wait for batch extend loop to finish
|
|
574
|
+
while (this.batchExtendRunning) {
|
|
575
|
+
await sleep(100)
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
this.visibilityManagement.clear()
|
|
579
|
+
this.logger.info('ACSQS | shutdown | Visibility management stopped and cleared')
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
// Get visibility tracking stats for monitoring
|
|
583
|
+
getVisibilityStats() {
|
|
584
|
+
const stats = {
|
|
585
|
+
totalTracked: this.visibilityManagement.size,
|
|
586
|
+
queueBreakdown: {},
|
|
587
|
+
oldestMessage: null,
|
|
588
|
+
avgExtensions: 0
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
let totalExtensions = 0
|
|
592
|
+
let oldestTime = Date.now()
|
|
593
|
+
|
|
594
|
+
for (const [messageId, data] of this.visibilityManagement) {
|
|
595
|
+
if (!stats.queueBreakdown[data.queueName]) {
|
|
596
|
+
stats.queueBreakdown[data.queueName] = 0
|
|
597
|
+
}
|
|
598
|
+
stats.queueBreakdown[data.queueName]++
|
|
599
|
+
totalExtensions += data.extensionCount
|
|
600
|
+
|
|
601
|
+
if (data.createdAt < oldestTime) {
|
|
602
|
+
oldestTime = data.createdAt
|
|
603
|
+
stats.oldestMessage = {
|
|
604
|
+
messageId,
|
|
605
|
+
age: Date.now() - data.createdAt,
|
|
606
|
+
extensions: data.extensionCount
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
if (this.visibilityManagement.size > 0) {
|
|
612
|
+
stats.avgExtensions = totalExtensions / this.visibilityManagement.size
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
return stats
|
|
616
|
+
}
|
|
341
617
|
|
|
342
618
|
// helpers
|
|
343
619
|
async fetchS3Object({ key }) {
|
|
@@ -356,7 +632,6 @@ class ACSQS {
|
|
|
356
632
|
throw e
|
|
357
633
|
}
|
|
358
634
|
}
|
|
359
|
-
|
|
360
635
|
}
|
|
361
636
|
|
|
362
637
|
module.exports = ACSQS
|
package/package.json
CHANGED
|
@@ -3,24 +3,24 @@
|
|
|
3
3
|
"author": "Mark Poepping (https://www.admiralcloud.com)",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": "admiralcloud/ac-sqs",
|
|
6
|
-
"version": "3.
|
|
6
|
+
"version": "3.3.0",
|
|
7
7
|
"dependencies": {
|
|
8
|
-
"@aws-sdk/client-s3": "^3.
|
|
9
|
-
"@aws-sdk/client-sqs": "^3.
|
|
8
|
+
"@aws-sdk/client-s3": "^3.896.0",
|
|
9
|
+
"@aws-sdk/client-sqs": "^3.896.0",
|
|
10
10
|
"lodash": "^4.17.21",
|
|
11
11
|
"uuid": "^11.1.0"
|
|
12
12
|
},
|
|
13
13
|
"devDependencies": {
|
|
14
|
-
"ac-semantic-release": "^0.4.
|
|
14
|
+
"ac-semantic-release": "^0.4.8",
|
|
15
15
|
"chai": "^4.5.0",
|
|
16
|
-
"eslint": "^9.
|
|
17
|
-
"mocha": "^11.
|
|
16
|
+
"eslint": "^9.36.0",
|
|
17
|
+
"mocha": "^11.7.2"
|
|
18
18
|
},
|
|
19
19
|
"scripts": {
|
|
20
20
|
"test": "mocha --reporter spec"
|
|
21
21
|
},
|
|
22
22
|
"engines": {
|
|
23
|
-
"node": ">=
|
|
23
|
+
"node": ">=20.0.0"
|
|
24
24
|
},
|
|
25
25
|
"resolutions": {
|
|
26
26
|
"mocha/chokidar/braces": "^3.0.3"
|