ac-sqs 3.2.0 → 3.2.1

package/CHANGELOG.md

@@ -1,3 +1,29 @@
+
+## [3.2.1](https://github.com/admiralcloud/ac-sqs/compare/v3.2.0..v3.2.1) (2025-07-18 12:57:51)
+
+
+### Bug Fix
+
+* **App:** Reduced code complexity | MP | [0313cc01d90d9e4d1ad3b7ef3136af7dfc72d717](https://github.com/admiralcloud/ac-sqs/commit/0313cc01d90d9e4d1ad3b7ef3136af7dfc72d717)    
+Create smaller functions  
+Related issues: [admiralcloud/ac-sqs#1](https://github.com/admiralcloud/ac-sqs/issues/1) [admiralcloud/ac-api-server#340](https://github.com/admiralcloud/ac-api-server/issues/340)
+* **App:** Improved code quality | MP | [0d7e40518804f8c227cbc20c602510edada2e02e](https://github.com/admiralcloud/ac-sqs/commit/0d7e40518804f8c227cbc20c602510edada2e02e)    
+Separated huge function into smaller parts  
+Related issues: [admiralcloud/ac-sqs#1](https://github.com/admiralcloud/ac-sqs/issues/1) [admiralcloud/ac-api-server#340](https://github.com/admiralcloud/ac-api-server/issues/340)
+* **App:** Improved code quality | MP | [cd9f5d3969d4aac9c3c87e7e84dd5788e9929cce](https://github.com/admiralcloud/ac-sqs/commit/cd9f5d3969d4aac9c3c87e7e84dd5788e9929cce)    
+Improved code quality  
+Related issues:
+* **App:** Requested code changes | MP | [39aa6c33b9087fcb2c3cfa075716a5dd83e07550](https://github.com/admiralcloud/ac-sqs/commit/39aa6c33b9087fcb2c3cfa075716a5dd83e07550)    
+Improved code quality  
+Related issues:
+* **App:** Improved visibility management | MP | [d45111846668fe3db268fd40987c726e82e1074c](https://github.com/admiralcloud/ac-sqs/commit/d45111846668fe3db268fd40987c726e82e1074c)    
+Add batch processing and throttling for visibility extension, better cleanup, graceful shutdown and stats  
+Related issues:
+### Chores
+
+* **App:** Updated packages | MP | [c6a2d43081047626aa60d5cb5f97427760143880](https://github.com/admiralcloud/ac-sqs/commit/c6a2d43081047626aa60d5cb5f97427760143880)    
+Updated packages  
+Related issues:
  
 # [3.2.0](https://github.com/admiralcloud/ac-sqs/compare/v3.1.2..v3.2.0) (2025-05-11 11:56:57)
 

package/README.md

@@ -120,7 +120,7 @@ Preparations you have to make before running the tests:
 + export the AWS profile to use for tests (if it is not your default profile) using  **export AWS_PROFILE=development**
 + export the AWS account id using **export awsaccount=12345**
 + create a SQS list named "test_acsqs"
-+ create a bucket and export the name using **export bucket=acsqs-test-bucket**
++ create a bucket and export the name using **export bucket=sqstest.admiralcloud.com**
 + export the node test environment using **export NODE_ENV=test**
 
 **ATTENTION**: Tests may fail when checking the SQS length. This is a by-design failure:

package/index.js

@@ -1,18 +1,24 @@
 const _ = require('lodash')
 const { v4: uuidV4 } = require('uuid')
+const { setTimeout: sleep } = require('timers/promises')
 
-const { SQSClient, SendMessageCommand, SendMessageBatchCommand, ReceiveMessageCommand, DeleteMessageBatchCommand, GetQueueAttributesCommand, ChangeMessageVisibilityCommand } = require('@aws-sdk/client-sqs')
+const { SQSClient, SendMessageCommand, SendMessageBatchCommand, ReceiveMessageCommand, DeleteMessageBatchCommand, GetQueueAttributesCommand, ChangeMessageVisibilityBatchCommand } = require('@aws-sdk/client-sqs')
 const { S3Client, GetObjectCommand, PutObjectCommand, DeleteObjectsCommand } = require("@aws-sdk/client-s3")
 
-
 class ACSQS {
-  constructor({ region = 'eu-central-1', account, availableLists, useS3 = { enabled: true, bucket: undefined }, messageThreshold = 250e3, debug, logger=console, throwError = false }) {
+  constructor({ region = 'eu-central-1', account, availableLists, useS3 = { enabled: true, bucket: undefined }, messageThreshold = 250e3, debug, logger = console, throwError = false, maxConcurrentMessages = 3000 }) {
     this.region = region
     this.account = account
     this.availableLists = availableLists
     this.logger = logger
     this.throwError = throwError
-    this.visibilityTimer = {}
+    this.maxConcurrentMessages = maxConcurrentMessages
+    
+    // Improved visibility management
+    this.visibilityManagement = new Map()
+    this.batchExtendRunning = false
+    this.stopBatchExtend = false
+    this.batchExtendInterval = 5000 // Check every 5 seconds
 
     const awsConfig = {
       region,
@@ -26,6 +32,281 @@ class ACSQS {
       this.bucket = _.get(useS3, 'bucket')
       this.s3 = new S3Client(awsConfig)
     }
+
+    // Start batch extend loop
+    this.startBatchExtendTimer()
+  }
+
+  async startBatchExtendTimer() {
+    if (this.batchExtendRunning) return
+    
+    this.batchExtendRunning = true
+    this.stopBatchExtend = false
+
+    while (true) {
+      try {
+        if (this.stopBatchExtend) break
+        
+        await sleep(this.batchExtendInterval)
+        
+        if (this.stopBatchExtend) break
+        
+        await this.processBatchExtensions()
+      }
+      catch (error) {
+        this.logger.error('ACSQS | startBatchExtendTimer | Error in batch extend loop | %s', error?.message)
+        // Don't break the loop on errors, just log and continue
+        await sleep(1000) // Wait 1s on error before retrying
+      }
+    }
+    
+    this.batchExtendRunning = false
+  }
+
+  stopBatchExtendTimer() {
+    this.stopBatchExtend = true
+  }
+
+  async processBatchExtensions() {
+    if (this.visibilityManagement.size === 0) return
+
+    // Group messages by queue for batch processing
+    const queueGroups = new Map()
+    const now = Date.now()
+
+    for (const [messageId, messageData] of this.visibilityManagement) {
+      // Check if message still exists in tracking (might have been deleted)
+      if (!this.visibilityManagement.has(messageId)) continue
+      
+      // Check if message needs extension
+      if (now >= messageData.nextExtendTime) {
+        // Check max extensions
+        if (messageData.extensionCount >= messageData.maxExtensions) {
+          this.logger.warn('ACSQS | processBatchExtensions | Max extensions reached | %s | %s', messageData.queueName, messageId)
+          this.removeVisibilityTracking(messageId)
+          continue
+        }
+
+        if (!queueGroups.has(messageData.queueName)) {
+          queueGroups.set(messageData.queueName, [])
+        }
+        queueGroups.get(messageData.queueName).push(messageData)
+      }
+    }
+
+    // Process each queue's extensions in batch
+    for (const [queueName, messages] of queueGroups) {
+      await this.extendVisibilityBatch(queueName, messages)
+    }
+  }
+
+  async extendVisibilityBatch(queueName, messages) {
+    if (messages.length === 0) return
+
+    const config = _.find(this.availableLists, { name: queueName })
+    if (!config) return
+
+    const chunks = this.chunkMessages(messages)
+    await this.processAllChunks(queueName, chunks, config)
+  }
+
+  chunkMessages(messages) {
+    return _.chunk(messages, 10)
+  }
+
+  async processAllChunks(queueName, chunks, config) {
+    for (let i = 0; i < chunks.length; i++) {
+      const chunk = chunks[i]
+      
+      // Add delay between chunks to avoid throttling (except for first chunk)
+      if (i > 0) {
+        await sleep(100)
+      }
+
+      await this.processChunk(queueName, chunk, config, i + 1, chunks.length)
+    }
+  }
+
+  async processChunk(queueName, chunk, config, chunkNumber, totalChunks) {
+    const validChunk = this.getValidChunk(chunk)
+    if (validChunk.length === 0) return
+
+    const sqsParams = await this.buildSQSParams(validChunk, config)
+    await this.executeChunkWithRetry(queueName, validChunk, sqsParams, config, chunkNumber, totalChunks)
+  }
+
+  getValidChunk(chunk) {
+    return chunk.filter(messageData => this.visibilityManagement.has(messageData.messageId))
+  }
+
+  async buildSQSParams(validChunk, config) {
+    const visibilityTimeout = _.get(config, 'visibilityTimeout', 30)
+    const entries = validChunk.map(messageData => ({
+      Id: messageData.messageId,
+      ReceiptHandle: messageData.receiptHandle,
+      VisibilityTimeout: visibilityTimeout
+    }))
+
+    return {
+      QueueUrl: await this.getQueueUrl(config),
+      Entries: entries
+    }
+  }
+
+  async executeChunkWithRetry(queueName, validChunk, sqsParams, config, chunkNumber, totalChunks) {
+    const maxRetries = 2
+    let lastError = null
+
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      if (attempt > 1) {
+        await this.handleRetryDelay(lastError, attempt, maxRetries, queueName, chunkNumber, totalChunks)
+      }
+
+      try {
+        const command = new ChangeMessageVisibilityBatchCommand(sqsParams)
+        const response = await this.sqs.send(command)
+        
+        const visibilityTimeout = _.get(config, 'visibilityTimeout', 30)
+        this.handleChunkResponse(queueName, validChunk, response, config, visibilityTimeout)
+        return
+      }
+      catch (error) {
+        lastError = error
+        
+        if (this.isPermanentError(error)) {
+          this.logger.warn('ACSQS | processChunk | Permanent error, not retrying | %s | %s', queueName, error.message)
+          break
+        }
+        
+        if (attempt < maxRetries) {
+          this.logger.warn('ACSQS | processChunk | Attempt %s failed, will retry | %s | %s', 
+            attempt, queueName, error.message)
+        }
+      }
+    }
+
+    this.handleAllRetriesFailed(queueName, validChunk, lastError)
+  }
+
+  async handleRetryDelay(lastError, attempt, maxRetries, queueName, chunkNumber, totalChunks) {
+    const isThrottled = lastError?.message?.includes('throttled')
+    const delayMs = isThrottled ? 500 : 200
+    
+    this.logger.warn('ACSQS | processChunk | Retry attempt %s/%s | %s | Chunk %s/%s | Waiting %sms', 
+      attempt, maxRetries, queueName, chunkNumber, totalChunks, delayMs)
+      
+    await sleep(delayMs)
+  }
+
+  isPermanentError(error) {
+    return error.message?.includes('ReceiptHandleIsInvalid') || 
+           error.message?.includes('MessageNotInflight')
+  }
+
+  handleAllRetriesFailed(queueName, validChunk, lastError) {
+    this.logger.error('ACSQS | processChunk | All retries failed | %s | Chunk size: %s | %s', 
+      queueName, validChunk.length, lastError?.message)
+    
+    for (const messageData of validChunk) {
+      this.removeVisibilityTracking(messageData.messageId)
+    }
+  }
+
+  handleChunkResponse(queueName, validChunk, response, config, visibilityTimeout) {
+    const successful = new Set((response.Successful || []).map(s => s.Id))
+    const failed = new Set((response.Failed || []).map(f => f.Id))
+
+    for (const messageData of validChunk) {
+      // Check again if message still exists (might have been deleted during AWS call)
+      if (!this.visibilityManagement.has(messageData.messageId)) continue
+      
+      if (successful.has(messageData.messageId)) {
+        this.updateSuccessfulMessage(messageData, visibilityTimeout, queueName, config)
+      }
+      else if (failed.has(messageData.messageId)) {
+        this.handleFailedMessage(messageData, response.Failed, queueName, config)
+      }
+    }
+  }
+
+  updateSuccessfulMessage(messageData, visibilityTimeout, queueName, config) {
+    messageData.extensionCount++
+    messageData.nextExtendTime = Date.now() + (visibilityTimeout * 0.8 * 1000)
+    
+    if (config.debug) {
+      this.logger.debug('ACSQS | extendVisibilityBatch | Success | %s | M %s | %ss | Count: %s', 
+        queueName, messageData.messageId, visibilityTimeout, messageData.extensionCount)
+    }
+  }
+
+  handleFailedMessage(messageData, failedItems, queueName, config) {
+    const failedItem = failedItems.find(f => f.Id === messageData.messageId)
+    
+    // Log only if it's not an expired receipt handle (which is normal)
+    if (failedItem?.Code === 'ReceiptHandleIsInvalid') {
+      if (config.debug) {
+        this.logger.debug('ACSQS | extendVisibilityBatch | Receipt handle expired (normal) | %s | M %s', 
+          queueName, messageData.messageId)
+      }
+    }
+    else {
+      this.logger.warn('ACSQS | extendVisibilityBatch | Failed | %s | M %s | %s', 
+        queueName, messageData.messageId, failedItem?.Message || 'Unknown error')
+    }
+    
+    // Remove from tracking if receipt handle expired or other permanent error
+    if (failedItem?.Code === 'ReceiptHandleIsInvalid' || failedItem?.Code === 'MessageNotInflight') {
+      this.removeVisibilityTracking(messageData.messageId)
+    }
+  }
+
+  addVisibilityTracking(messageId, queueName, receiptHandle, config) {
+    // Check if we're at max capacity
+    if (this.visibilityManagement.size >= this.maxConcurrentMessages) {
+      this.logger.warn('ACSQS | addVisibilityTracking | Max concurrent messages reached | %s', this.maxConcurrentMessages)
+      return false
+    }
+
+    const visibilityTimeout = _.get(config, 'visibilityTimeout', 30)
+    const maxExtensions = _.get(config, 'maxVisibilityExtensions', 12)
+
+    this.visibilityManagement.set(messageId, {
+      messageId,
+      queueName,
+      receiptHandle,
+      extensionCount: 0,
+      maxExtensions,
+      nextExtendTime: Date.now() + (visibilityTimeout * 0.8 * 1000),
+      createdAt: Date.now()
+    })
+
+    return true
+  }
+
+  removeVisibilityTracking(messageId) {
+    this.visibilityManagement.delete(messageId)
+  }
+
+  // Legacy method for backwards compatibility - now uses batch processing
+  async extendVisibility({ name, message, throwError }) {
+    const config = _.find(this.availableLists, { name })
+    if (!config) {
+      this.logger.error('AWS | extendVisibility | configurationMissing | %s', name)
+      throw new Error('configurationForListMissing')
+    }
+
+    const { MessageId: messageId } = message
+    
+    // Check if message is already being tracked
+    if (this.visibilityManagement.has(messageId)) {
+      // Force immediate extension by setting nextExtendTime to now
+      const messageData = this.visibilityManagement.get(messageId)
+      messageData.nextExtendTime = Date.now()
+      return
+    }
+
+    // Add to tracking if not already present
+    this.addVisibilityTracking(messageId, name, message.ReceiptHandle, config)
   }
 
   async getAllLists({ throwError = false } = {}) {
@@ -68,7 +349,6 @@ class ACSQS {
     }
   }
 
-
   async sendSQSMessage({ name, message, messageGroupId, deDuplicationId, delay, throwError, debug }) {
     const config = _.find(this.availableLists, { name })
     if (!config) {
@@ -165,62 +445,6 @@ class ACSQS {
     }
   }
 
-  async extendVisibility({ name, message, throwError }) {
-    const config = _.find(this.availableLists, { name })
-    if (!config) {
-      this.logger.error('AWS | extendVisibility | configurationMissing | %s', name)
-      throw new Error('configurationForListMissing')
-    }
-
-    const visibilityTimeout = _.get(config, 'visibilityTimeout', 15)
-    const maxVisibilityExtensions = _.get(config, 'maxVisibilityExtensions', 12) // max number of times the extension can me made (12 x 15s = 3min)
-    
-    const { MessageId: messageId, ReceiptHandle: receiptHandle } = message
-
-     // Check if we've reached maximum extensions
-    if (this.visibilityTimer[messageId] && this.visibilityTimer[messageId].visibilityExtensionCount >= maxVisibilityExtensions) {
-      this.logger.warn('ACSQS | extendVisibility | %s | M %s | Max extensions reached | %s | %j', name, messageId, maxVisibilityExtensions, message)
-      this.deleteVisibilityTimer({ messageId })
-      return
-    }
-    
-    // Track extension count
-    if (this.visibilityTimer[messageId]) {
-      this.visibilityTimer[messageId].visibilityExtensionCount++
-    }
-    
-    const sqsParams = {
-      QueueUrl: await this.getQueueUrl(config),
-      ReceiptHandle: receiptHandle,
-      VisibilityTimeout: visibilityTimeout
-    }
-    const command = new ChangeMessageVisibilityCommand(sqsParams)
-    try {
-      const response = await this.sqs.send(command)
-      if (config.debug) {
-        const visibilityExtensionCount = this.visibilityTimer[messageId] ? this.visibilityTimer[messageId].visibilityExtensionCount : 0
-        this.logger.debug('ACSQS | extendVisibility | %s |  M %s | %ss | %s | %j', name, messageId, visibilityTimeout, visibilityExtensionCount, message)
-      }
-
-      return response
-    }
-    catch(e) {
-      this.logger.error('ACSQS | extendVisibility | %s | %s', name, e?.message)
-      this.deleteVisibilityTimer({ messageId })
-      if (this.throwError || throwError) throw e
-    }
-  }
-
-  deleteVisibilityTimer({ messageId }) {
-    if (this.visibilityTimer[messageId]) {
-      clearTimeout(this.visibilityTimer[messageId].timer)
-      const self = this
-      setTimeout(() => {
-        delete self.visibilityTimer[messageId]
-      }, 1000)
-    }
-  }
-
   async receiveSQSMessages({ name, throwError, debug }) {
     const config = _.find(this.availableLists, { name })
     if (!config) {
@@ -241,7 +465,6 @@ class ACSQS {
       const result = await this.sqs.send(command)
       if (!_.size(result.Messages)) return
       
-      // Benutze Arrow-Funktion, um `this` beizubehalten
       const messages = await Promise.all(result.Messages.map(async (message) => {
         if (message.Body.startsWith('s3:')) {
           const key = message.Body.replace('s3:', '')
@@ -251,27 +474,14 @@ class ACSQS {
             message.s3key = key
           }
           catch(e) {
-            this.logger.error('ACSQS | receiveSQSMessages | s3KeyInvalid | %s', name, key)         
+            this.logger.error('ACSQS | receiveSQSMessages | s3KeyInvalid | %s | %s', name, key)         
           }
         }
   
         if (visibilityTimeout > 0) {
-          // start visibility timer that automatically extends visibility of the message if required
-          const { MessageId: messageId } = message
-          const timeoutMs = Math.floor(visibilityTimeout * 0.8 * 1000)
-          const self = this // `this` als lokale Variable speichern
-          
-          this.visibilityTimer[messageId] = {
-            // Arrow-Funktion für setInterval damit `this` erhalten bleibt,
-            // oder verwende die lokale Variable `self`
-            timer: setInterval(() => {
-              this.extendVisibility({ name, message })
-                .catch(e => {
-                  this.logger.error('ACSQS | AutoExtendVisibility | Failed %s', e.message)
-                })
-            }, timeoutMs),
-            visibilityExtensionCount: 0
-          }
+          // Add to visibility tracking instead of individual timers
+          const { MessageId: messageId, ReceiptHandle: receiptHandle } = message
+          this.addVisibilityTracking(messageId, name, receiptHandle, config)
         }
   
         return message
@@ -305,12 +515,10 @@ class ACSQS {
       if (item.s3key) {
         s3keys.push({ Key: item.s3key })
       }
-      if (this.visibilityTimer[messageId]) {
-        this.deleteVisibilityTimer({ messageId })
-      }
+      // Remove from visibility tracking
+      this.removeVisibilityTracking(messageId)
     }
 
-
     let sqsParams = {
       QueueUrl: await this.getQueueUrl(config),
       Entries: entries
@@ -327,7 +535,7 @@ class ACSQS {
             Objects: s3keys,
           }
         }
-        const command = new DeleteObjectsCommand(input);
+        const command = new DeleteObjectsCommand(input)
         this.s3.send(command)
       }
       return response
@@ -338,6 +546,54 @@ class ACSQS {
     }
   }
 
+  // Cleanup method for graceful shutdown
+  async shutdown() {
+    this.stopBatchExtendTimer()
+    
+    // Wait for batch extend loop to finish
+    while (this.batchExtendRunning) {
+      await sleep(100)
+    }
+    
+    this.visibilityManagement.clear()
+    this.logger.info('ACSQS | shutdown | Visibility management stopped and cleared')
+  }
+
+  // Get visibility tracking stats for monitoring
+  getVisibilityStats() {
+    const stats = {
+      totalTracked: this.visibilityManagement.size,
+      queueBreakdown: {},
+      oldestMessage: null,
+      avgExtensions: 0
+    }
+
+    let totalExtensions = 0
+    let oldestTime = Date.now()
+
+    for (const [messageId, data] of this.visibilityManagement) {
+      if (!stats.queueBreakdown[data.queueName]) {
+        stats.queueBreakdown[data.queueName] = 0
+      }
+      stats.queueBreakdown[data.queueName]++
+      totalExtensions += data.extensionCount
+      
+      if (data.createdAt < oldestTime) {
+        oldestTime = data.createdAt
+        stats.oldestMessage = {
+          messageId,
+          age: Date.now() - data.createdAt,
+          extensions: data.extensionCount
+        }
+      }
+    }
+
+    if (this.visibilityManagement.size > 0) {
+      stats.avgExtensions = totalExtensions / this.visibilityManagement.size
+    }
+
+    return stats
+  }
 
   // helpers
   async fetchS3Object({ key }) {
@@ -356,7 +612,6 @@ class ACSQS {
       throw e
     }
   }
-
 }
 
 module.exports = ACSQS

package/package.json

@@ -3,24 +3,24 @@
   "author": "Mark Poepping (https://www.admiralcloud.com)",
   "license": "MIT",
   "repository": "admiralcloud/ac-sqs",
-  "version": "3.2.0",
+  "version": "3.2.1",
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.806.0",
-    "@aws-sdk/client-sqs": "^3.806.0",
+    "@aws-sdk/client-s3": "^3.846.0",
+    "@aws-sdk/client-sqs": "^3.846.0",
     "lodash": "^4.17.21",
     "uuid": "^11.1.0"
   },
   "devDependencies": {
     "ac-semantic-release": "^0.4.6",
     "chai": "^4.5.0",
-    "eslint": "^9.26.0",
-    "mocha": "^11.2.2"
+    "eslint": "^9.31.0",
+    "mocha": "^11.7.1"
   },
   "scripts": {
     "test": "mocha --reporter spec"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=20.0.0"
   },
   "resolutions": {
     "mocha/chokidar/braces": "^3.0.3"