ac-sqs 0.1.0

package/.acsemver.js

@@ -0,0 +1,9 @@
+module.exports = {
+  repository: {
+    url: 'https://github.com/admiralcloud/ac-sqs'
+  },
+  changelogFile: __dirname + '/CHANGELOG.md',
+  sections: [
+    { name: 'App' }
+  ],
+}

package/.eslintrc.js

@@ -0,0 +1,27 @@
+const config = {
+  root: true,
+  'env': {
+    'commonjs': true,
+    'es6': true,
+    'node': true
+  },
+  'extends': 'eslint:recommended',
+  "rules": {
+    "space-before-function-paren": 0,
+    "no-extra-semi": 0,
+    "object-curly-spacing": ["error", "always"],
+    "brace-style": ["error", "stroustrup", { "allowSingleLine": true }],
+    "no-useless-escape": 0,
+    "standard/no-callback-literal": 0,
+    "new-cap": 0
+  },
+  globals: {
+    describe: true,
+    it: true
+  },
+  'parserOptions': {
+    'ecmaVersion': 2022
+},
+}
+
+module.exports = config

package/CHANGELOG.md

@@ -0,0 +1,10 @@
+<a name="0.1.0"></a>
+ 
+# [0.1.0](https://github.com/admiralcloud/ac-sqs/compare/..v0.1.0) (2023-04-29 06:14:09)
+
+
+### Feature
+
+* **App:** Initial version | MP | [b4e87cdce80dd218d1be1c9dafca786d3fea15d0](https://github.com/admiralcloud/ac-sqs/commit/b4e87cdce80dd218d1be1c9dafca786d3fea15d0)    
+Initial version  
+Related issues: [admiralcloud/ac-sqs#1](https://github.com/admiralcloud/ac-sqs/issues/1) [admiralcloud/ac-api-server#340](https://github.com/admiralcloud/ac-api-server/issues/340)

package/Makefile

@@ -0,0 +1,17 @@
+MOCHA_OPTS= --slow 0 -A
+REPORTER = spec
+
+lint-fix:
+	./node_modules/.bin/eslint --fix index.js test/test.js
+
+lint-check:
+	./node_modules/.bin/eslint index.js test/test.js
+
+commit:
+	@node ./node_modules/ac-semantic-release/lib/commit.js
+
+release:
+	@node ./node_modules/ac-semantic-release/lib/release.js
+
+
+.PHONY: check

package/README.md

@@ -0,0 +1,132 @@
+# AC SQS
+This tool is a wrapper for AWS SDK's SQS function. It includes handling of big SQS messages using S3.
+
+# Usage
+
+```
+yarn add ac-sqs
+
+const acsqs = new ACSQS({
+  profile: 'development', // Optional AWS profile, see below
+  account: '123456789', // AWS account id
+  availableLists: [{
+    name: 'listName'
+    batchSize: 10
+  }],
+  useS3: {
+    enabled: true,
+    bucket: 'ac-sqs-messages' // s3 bucket
+  }
+})
+
+await acsqs.sendSQSMessage({ name: listName, message: 'This is my message' })
+```
+
+# Available methods/calls
+## Initiate class
+**Account**
+The ***account*** id of your AWS account. This is required
+
+**Available lists**
+Array of AWS SQS lists that will be used by this function. Every item in the list must be an object with the following properties:
++ name -> the name of the list in AWS SQS. See below for more info.
++ batchSize -> number of messages to fetch per call. Max 10, defaults to 1
++ visibilityTimeout -> see AWS SQS for details, defaults to 30
++ waitTime -> see AWS SQS for details, defaults to 20
++ fifo -> set to true, if this is a fifo list
++ localPrefix -> set your local prefix. See below for more info
+
+Name should be the plain name of the list. Parameters like fifo or test (in test environment) or localPrefixes (for local development) should not be part of the list name. LocalPrefix can be used if multiple developers work on your project and you want to make sure they all work on their own SQS list without changing the name of all SQS lists in your main project.
+
+Example for a FIFO list:
+Let the name be "mylist". This will automatically create "test_mylist" in test environment (NODE_ENV=test) and "local_LOCALPREFIX_mylist" if you set localPrefix.
+
+**Profile [optional]**
+By default the first AWS credentials in ~/.aws/credentials will be used.  You can also export an environment variable ***profile*** or send a named AWS profile.
+
+**useS3 [optional]**
+AWS SQS only allows a certain message size (262kb at the time of the documentation). To process messages with a bigger payload, the actual message content will be stored in a file on AWS S3. The feature is enabled by default and you must make sure to set a bucket.
+
+```
+useS3: {
+  enabled: true,
+  bucket: 'ac-sqs-message'
+}
+```
+
+Make sure your function can read, write and delete messages in the bucket.
+
+## sendSQSMessage
+Create a messsage in a SQS list.
+
+**name**
+The name of the list to send a SQS message to.
+
+**message**
+The actual message (as plain test). If you want to send JSON, please stringify!
+
+**messageGroupId [optional]**
+See AWS SQS for details
+
+**deDuplicationId [optional]**
+See AWS SQS for details
+
+**delay [optional]**
+See AWS SQS for details
+
+https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html
+
+## receiveSQSMessages
+Receive messages from SQS. By default up to 10 messages are fetched per call. You set a lower number using the batchSize parameter in the corresponding availableLists entry. 
+
+**name**
+The name of the list to receive a SQS messages from.
+
+https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html
+
+## deleteSQSMessages
+Deletes one or multiple SQS messages and also cleans up/deletes related S3 files.
+
+**name**
+The name of the list to delete SQS messages from.
+
+**items**
+An array of objects to delete. Every objects must a least have properties ***Id*** and ***ReceiptHandle***. If the messages was originally stored on S3 the entry must also contain ***Key***.
+
+The ***Id*** parameter is the message id, ***ReceiptHandle*** is an identifier you get with the receive request.
+
+https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_DeleteMessageBatch.html
+
+## getQueueAttributes
+Get information about a list
+
+**name**
+The name of the list to get information about.
+
+**attributes**
+An array of metadata to get. By default only "ApproximateNumberOfMessages" is requested. Please see AWS SQS for other metadata options.
+
+https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_GetQueueAttributes.html
+
+# Test
+You can run tests using **yarn run test**.
+
+Preparations you have to make before running the tests:
+
++ export the AWS profile to use for tests (if it is not your default profile) using  **export profile=development**
++ export the AWS account id using **export awsaccount=12345**
++ create a SQS list named "test_acsqs"
++ create a bucket and export the name using **export bucket=acsqs-test-bucket**
+
+**ATTENTION**: Tests may fail when checking the SQS length. This is a by-design failure:
+"ApproximateNumberOfMessages metrics may not achieve consistency until at least 1 minute after the producers stop sending messages."
+
+See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_GetQueueAttributes.html
+
+# Misc
+## Links
+- [Website](https://www.admiralcloud.com/)
+- [Facebook](https://www.facebook.com/MediaAssetManagement/)
+
+## License
+[MIT License](https://opensource.org/licenses/MIT) Copyright © 2009-present, AdmiralCloud AG, Mark Poepping

package/index.js

@@ -0,0 +1,209 @@
+const _ = require('lodash')
+const { v4: uuidV4 } = require('uuid')
+const https = require('https')
+
+const { fromNodeProviderChain } = require('@aws-sdk/credential-providers')
+const { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageBatchCommand, GetQueueAttributesCommand } = require('@aws-sdk/client-sqs')
+const { S3Client, GetObjectCommand, PutObjectCommand, DeleteObjectsCommand } = require("@aws-sdk/client-s3")
+
+
+class ACSQS {
+  constructor({ region = 'eu-central-1', account, availableLists, profile = process.env['profile'], useS3 = { enabled: true, bucket: undefined }, messageThreshold = 250e3, debug, logger=console }) {
+    const httpOptions = {
+      keepAlive: true
+    }
+
+    this.region = region
+    this.account = account
+    this.availableLists = availableLists
+    this.logger = logger
+
+    const awsConfig = {
+      region,
+      credentials: fromNodeProviderChain({ profile, ignoreCache: true }),
+      httpOptions: {
+        agent: new https.Agent(httpOptions)
+      }
+    }
+    if (debug && profile) {
+      this.logger.log('ACSQS | Using AWS profile | %s', profile)
+    }
+    this.sqs = new SQSClient(awsConfig)
+
+    // store huge messages in S3 and let SQS be the link to that message
+    if (_.get(useS3, 'enabled')) {
+      this.useS3 = true
+      this.messageThreshold = messageThreshold
+      this.bucket = _.get(useS3, 'bucket')
+      this.s3 = new S3Client(awsConfig)
+    }
+  }
+
+  async getQueueUrl({ name, fifo, localPrefix }) {
+    let queueUrl = `https://sqs.${this.region}.amazonaws.com/${this.account}/` 
+    if (localPrefix) queueUrl += `local_${localPrefix}_`
+    if (process.env['NODE_ENV'] === 'test') queueUrl += 'test_'
+    queueUrl += name
+    if (fifo) queueUrl += '.fifo'
+    return queueUrl
+  }
+
+  async getQueueAttributes({ name, attributes = ['ApproximateNumberOfMessages'] }) {
+    const config = _.find(this.availableLists, { name })
+    if (!config) {
+      this.logger.error('ACSQS | getQueueAttributes | configurationMissing | %s', name)
+      throw new Error('configurationForListMissing')
+    }
+    let sqsParams = {
+      QueueUrl: await this.getQueueUrl(config),
+      AttributeNames: attributes
+    }
+    const command = new GetQueueAttributesCommand(sqsParams)
+    try {
+      return await this.sqs.send(command)
+    }
+    catch(e) {
+      this.logger.error('ACSQS | getQueueAttributes | %s | %s', name, e?.message)
+    }
+  }
+
+
+  async sendSQSMessage({ name, message, messageGroupId, deDuplicationId, delay }) {
+    const config = _.find(this.availableLists, { name })
+    if (!config) {
+      this.logger.error('AWS | sendSQSMessage | configurationMissing | %s', name)
+      throw new Error('configurationForListMissing')
+    }
+
+    if (this.useS3 && message.length > this.messageThreshold) {
+      // store message in S3
+      const key = uuidV4()
+      const input = {
+        Bucket: this.bucket,
+        Key: key,
+        ContentType: 'text/plain',
+        Body: Buffer.from(message, 'utf-8')     
+      }
+      const command = new PutObjectCommand(input)
+      await this.s3.send(command)
+      message = `s3:${key}`
+    }
+
+    const sqsParams = {
+      QueueUrl: await this.getQueueUrl(config),
+      MessageBody: message
+    }
+    if (messageGroupId) _.set(sqsParams, 'MessageGroupId', messageGroupId)
+    if (deDuplicationId) _.set(sqsParams, 'MessageDeduplicationId', deDuplicationId)
+    if (delay) _.set(sqsParams, 'DelaySeconds', delay)
+
+    const command = new SendMessageCommand(sqsParams)
+    try {
+      const response = await this.sqs.send(command)
+      return response
+    }
+    catch(e) {
+      this.logger.error('ACSQS | sendSQSMessage | %s | %s', name, e?.message)
+    }
+  }
+
+  async receiveSQSMessages({ name }) {
+    const config = _.find(this.availableLists, { name })
+    if (!config) {
+      this.logger.error('AWS | receiveSQSMessage | configurationMissing | %s', name)
+      throw new Error('configurationForListMissing')
+    }
+    let sqsParams = {
+      QueueUrl: await this.getQueueUrl(config),
+      MaxNumberOfMessages: _.get(config, 'batchSize', 10),
+      VisibilityTimeout: _.get(config, 'visibilityTimeout', 30),
+      WaitTimeSeconds: _.get(config, 'waitTime', 20)
+    }
+    const command = new ReceiveMessageCommand(sqsParams)
+    try {
+      const result = await this.sqs.send(command)
+      const messages = await Promise.all(result.Messages.map(async message => {
+        if (message.Body.startsWith('s3:')) {
+          const key = message.Body.replace('s3:', '')
+          try {
+            const objectData = await this.fetchS3Object({ key })
+            message.Body = objectData
+            message.s3key = key
+          }
+          catch(e) {
+            this.logger.error('AWS | receiveSQSMessages | s3KeyInvalid | %s', name, key)         
+          }
+        }
+        return message
+      }))
+      return messages
+    }
+    catch(e) {
+      this.logger.error('ACSQS | receiveSQSMessage | %s | %s', name, e?.message)
+    }
+  }
+
+  // items -> [{ Id, ReceiptHandle }]
+  async deleteSQSMessages({ name, items }) {
+    const config = _.find(this.availableLists, { name })
+    if (!config) {
+      this.logger.error('AWS | deleteSQSMessage | configurationMissing | %s', name)
+      throw new Error('configurationForListMissing')
+    }
+
+    const entries = []
+    const s3keys = []
+    for (const item of items) {
+      entries.push({ Id: item.Id, ReceiptHandle: item.ReceiptHandle })
+      if (item.s3key) {
+        s3keys.push({ Key: item.s3key })
+      }
+    }
+
+    let sqsParams = {
+      QueueUrl: await this.getQueueUrl(config),
+      Entries: entries
+    }
+    const command = new DeleteMessageBatchCommand(sqsParams)
+    try {
+      const response = await this.sqs.send(command)
+      // check cleaning up s3
+      if (this.useS3 && _.size(s3keys)) {
+        const input = {
+          Bucket: this.bucket,
+          Delete: { 
+            Objects: s3keys,
+          }
+        }
+        const command = new DeleteObjectsCommand(input);
+        this.s3.send(command)
+      }
+      return response
+    }
+    catch(e) {
+      this.logger.error('ACSQS | deleteSQSMessage | %s | %s', name, e?.message)
+    }
+  }
+
+
+  // helpers
+  async fetchS3Object({ key }) {
+    const input = {
+      Bucket: this.bucket,
+      Key: key
+    }
+    const command = new GetObjectCommand(input)
+  
+    try {
+      const response = await this.s3.send(command)
+      return await response.Body.transformToString()
+    } 
+    catch (e) {
+      this.logger.error('ACSQS | fetchS3Object | %j | %s', input, e?.message)
+      throw e
+    }
+  }
+
+}
+
+module.exports = ACSQS

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "ac-sqs",
+  "author": "Mark Poepping (https://www.admiralcloud.com)",
+  "license": "MIT",
+  "repository": "admiralcloud/ac-sqs",
+  "version": "0.1.0",
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.321.1",
+    "@aws-sdk/client-sqs": "^3.321.1",
+    "@aws-sdk/credential-providers": "^3.321.1",
+    "lodash": "^4.17.21",
+    "uuid": "^9.0.0"
+  },
+  "devDependencies": {
+    "ac-semantic-release": "^0.4.1",
+    "chai": "^4.3.7",
+    "eslint": "^8.39.0",
+    "mocha": "^10.x"
+  },
+  "scripts": {
+    "test": "mocha --reporter spec"
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  }
+}