mimi-messaging-sqs_sns 0.8.0 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf7944b9a3e10f154f738d66544cfa63611b17a742b156460969f70821ad86bb
-  data.tar.gz: f26e3e3623208170de192174c9c0fdb2ef5e26fa8cca831e73065eb86e21b0b5
+  metadata.gz: 9ebbd0b876cc696936356711703245e45106ff85ad152bde59fd2826b6cbc21f
+  data.tar.gz: ac2167ece90f7de977fee85c66137e5cb9f232e139fcb8ab797c27467cffa540
 SHA512:
-  metadata.gz: 37bbb87ba16df4886a5e525ca13732b3712021c5269418d8d8c90e1af125cbaa7e32aa5757a96293569a1717cc4c00542aaf20d12f3fc7514477cd3d708cd615
-  data.tar.gz: 9d53cc0479c47af711f0e89203f9ef1ced767115743fb2fa685ea559ebb0b891226a248219ff2103ec95207b678814c1c0b983f82a22eda8e741feac01a2d4dc
+  metadata.gz: 0cdc8a69e02de3533b0871fb9727fbe63e47081fe6a0ac66bf093cacf2f99d16f697034987adc507078bcf14d4864dd7ff78a2025e49305b4ae48c32313b1992
+  data.tar.gz: 56665e8adb58d3d17c1fcb7e3b5ef5eea8231d24a130de1bfd589acb90169473bd6088a5a227d3eb6d3141ce19af7939d2a874132607b45579b4a02d4b4c3103

data/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## UNRELEASED
 
+## v0.8.2
+
+* [#7](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/7)
+  * Fixed an issue in the error processing, which could potentially crash the consumer threads
+
+## v0.8.1
+
+* [#5](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/5)
+  * Refactored worker pool based message processing and error handling
+
 ## v0.8.0
 
 * [#3](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/3)

data/TODO.md

@@ -3,7 +3,7 @@
 List of missing features planned for future releases.
 
 * [X] Log error and recover if the reply cannot be sent
-* [ ] Threadsafe TimeoutQueue
-* [ ] Multithreaded consumers
+* [X] Threadsafe TimeoutQueue
+* [X] Multithreaded consumers
 * [ ] Subscribe to topic without a queue (Temporary queues)
 

data/examples/event.rb

@@ -4,8 +4,8 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 

data/examples/mass-query.rb

@@ -6,8 +6,8 @@ COUNT = 10
 THREADS = 10
 QUERY_TIMEOUT = 60
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"

data/examples/query.rb

@@ -4,8 +4,8 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"

data/examples/responder.rb

@@ -3,8 +3,8 @@
 require "mimi/messaging/sqs_sns"
 
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"

data/examples/subscriber.rb

@@ -3,8 +3,8 @@
 require "mimi/messaging/sqs_sns"
 
 AWS_REGION = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 

data/lib/mimi/messaging/sqs_sns/adapter.rb

@@ -185,15 +185,8 @@ module Mimi
           @consumers ||= []
           opts = opts.dup
           queue_url = find_or_create_queue(queue_name)
-          @consumers << Consumer.new(self, queue_url) do |m|
-            worker_pool.post do
-              process_request_message(processor, m)
-            end
-          rescue Concurrent::RejectedExecutionError
-            # the backlog is overflown, put the message back
-            Mimi::Messaging.log "Worker pool backlog is full, nack-ing the message " \
-              "(workers:#{worker_pool.length}, backlog:#{worker_pool.queue_length})"
-            raise Mimi::Messaging::NACK # exception raised in Consumer thread
+          @consumers << Consumer.new(self, queue_url, worker_pool) do |m|
+            process_request_message(processor, m)
           end
         end
 
@@ -209,15 +202,8 @@ module Mimi
           topic_arn = find_or_create_topic(topic_name) # TODO: or find_topic!(...) ?
           queue_url = find_or_create_queue(queue_name)
           subscribe_topic_queue(topic_arn, queue_url)
-          @consumers << Consumer.new(self, queue_url) do |m|
-            worker_pool.post do
-              process_event_message(processor, m)
-            end
-          rescue Concurrent::RejectedExecutionError
-            # the backlog is overflown, put the message back
-            Mimi::Messaging.log "Worker pool backlog is full, nack-ing the message " \
-              "(workers:#{worker_pool.length}, backlog:#{worker_pool.queue_length})"
-            raise Mimi::Messaging::NACK # exception raised in Consumer thread
+          @consumers << Consumer.new(self, queue_url, worker_pool) do |m|
+            process_event_message(processor, m)
           end
         end
 

data/lib/mimi/messaging/sqs_sns/consumer.rb

@@ -10,12 +10,20 @@ module Mimi
         # (seconds) determines how soon the NACK-ed message becomes visible to other consumers
         NACK_VISIBILITY_TIMEOUT = 1
 
-        def initialize(adapter, queue_url, &block)
+        # Creates a new Consumer to read and process messages from the given queue.
+        #
+        # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
+        # @param queue_url [String]
+        # @param worker_pool [Concurrent::ThreadPoolExecutor, nil] an optional worker pool to be used
+        #
+        def initialize(adapter, queue_url, worker_pool = nil, &block)
           @stop_requested = false
-          Mimi::Messaging.log "Starting consumer for: #{queue_url}"
+          @worker_pool = worker_pool
+          Mimi::Messaging.log "Starting consumer for: #{queue_url}, " \
+            " worker_pool: #{worker_pool ? "yes" : "no"}"
           @consumer_thread = Thread.new do
             while not @stop_requested
-              read_and_process_message(adapter, queue_url, block)
+              read_and_process_message(adapter, queue_url, worker_pool, block)
             end
             Mimi::Messaging.log "Stopping consumer for: #{queue_url}"
           end
@@ -41,25 +49,33 @@ module Mimi
         #
         # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
         # @param queue_url [String]
+        # @param worker_pool [Concurrent::ThreadPoolExecutor,nil]
         # @param block [Proc] a block to be invoked when a message is received
         #
-        def read_and_process_message(adapter, queue_url, block)
+        def read_and_process_message(adapter, queue_url, worker_pool, block)
           message = read_message(adapter, queue_url)
           return unless message
 
           Mimi::Messaging.log "Read message from: #{queue_url}"
-          block.call(message)
-          ack_message(adapter, queue_url, message)
-        rescue Mimi::Messaging::NACK
-          Mimi::Messaging.log "NACK-ing message from: #{queue_url}"
-          nack_message(adapter, queue_url, message)
+          if worker_pool
+            process_message_worker_pool(adapter, queue_url, worker_pool, message, block)
+          else
+            process_message(adapter, queue_url, message, block)
+          end
         rescue StandardError => e
           Mimi::Messaging.logger&.error(
-            "#{self.class}: failed to read and process message from: #{queue_url}," \
+            "#{self.class}: failed to read or process message from: #{queue_url}," \
+            " worker_pool: #{worker_pool ? "yes" : "no"}" \
             " error: (#{e.class}) #{e}"
           )
         end
 
+        # Returns a message read from SQS queue
+        #
+        # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
+        # @param queue_url [String]
+        # @return [Hash] with stringified keys
+        #
         def read_message(adapter, queue_url)
           result = adapter.sqs_client.receive_message(
             queue_url: queue_url,
@@ -73,6 +89,40 @@ module Mimi
           raise Mimi::Messaging::ConnectionError, "Unexpected number of messages read"
         end
 
+        # Processes a received message using the worker pool
+        #
+        # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
+        # @param queue_url [String]
+        # @param worker_pool [Concurrent::ThreadPoolExecutor,nil]
+        # @param block [Proc] a block to be invoked when a message is received
+        #
+        def process_message_worker_pool(adapter, queue_url, worker_pool, message, block)
+          begin
+            worker_pool.post do
+              process_message(adapter, queue_url, message, block)
+            end
+          rescue Concurrent::RejectedExecutionError
+            # the backlog is overflown, put the message back
+            Mimi::Messaging.log "Worker pool backlog is full, nack-ing the message " \
+              "(workers:#{worker_pool.length}, backlog:#{worker_pool.queue_length})"
+            nack_message(adapter, queue_url, message)
+          end
+        end
+
+        def process_message(adapter, queue_url, message, block)
+          block.call(message)
+          ack_message(adapter, queue_url, message)
+        rescue Mimi::Messaging::NACK
+          Mimi::Messaging.log "NACK-ing message from: #{queue_url}"
+          nack_message(adapter, queue_url, message)
+        rescue StandardError => e
+          Mimi::Messaging.logger&.error(
+            "#{self.class}: failed to process message from: #{queue_url}," \
+            " error: (#{e.class}) #{e}"
+          )
+          # NOTE: error is recovered and the message is neither ACKed or NACKed
+        end
+
         # ACK-ing the message indicates successfull processing of it
         # and removes the message from the queue
         #
@@ -91,6 +141,12 @@ module Mimi
             receipt_handle: msg.receipt_handle,
             visibility_timeout: NACK_VISIBILITY_TIMEOUT
           )
+        rescue StandardError => e
+          Mimi::Messaging.logger&.error(
+            "#{self.class}: failed to NACK message from: #{queue_url}," \
+            " error: (#{e.class}) #{e}"
+          )
+          # NOTE: error is recovered and the message is neither ACKed or NACKed
         end
       end # class Consumer
     end # module SQS_SNS

data/lib/mimi/messaging/sqs_sns/version.rb

@@ -3,7 +3,7 @@
 module Mimi
   module Messaging
     module SQS_SNS
-      VERSION = "0.8.0"
+      VERSION = "0.8.3"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mimi-messaging-sqs_sns
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.3
 platform: ruby
 authors:
 - Alex Kukushkin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-16 00:00:00.000000000 Z
+date: 2022-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mimi-messaging