mimi-messaging-sqs_sns 0.5.0 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 297b98675dafdc02dc9cbc9256339b3ecb696875
-  data.tar.gz: db1e2efc5ac9f8b98b1de5fb5dff0c76ee21930c
+SHA256:
+  metadata.gz: e106fbe54e012ffd59a6126292242b23c28b3086ff5dfd4866acb64bad559ccf
+  data.tar.gz: 2fce6feb750d6cf27925e79646b8e76359188343f663ee8d20f30f09eedc2ba0
 SHA512:
-  metadata.gz: c9d87b2c28ed31a1836070e09adc8732d3becc46364b3d04836e031bb9ae424e02aad22519a1632a1c205900a10a8ba7e2650c1e80b9947b40fa8dea5cd95331
-  data.tar.gz: 0d64bb39d6858fddb0645561180695fc9c6bb4ae44cc2f79027e36f9bab4bc4ff5efa69814da79a3046a0ead1412897ec00c0a9902f9d0b8a85c08f124029ee8
+  metadata.gz: a830a70afd7b0a5f268d600ca9673a8f3fc36566201510263008f517d32354a020213de0b51497c446e99a768089f2fd9a1392dccdf5bbbc5fc352d3557a2a4e
+  data.tar.gz: 41dc62d85b21f8736011d974b2749e5f6ce093d3f3cc3746f037f8a986ef01621bc2a61ae60f8f96a5405dcc6672bd1468fec0e3ea695e0b1f1e4a123d71c962

data/CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+
+## UNRELEASED
+
+## v0.8.2
+
+* [#7](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/7)
+  * Fixed an issue in the error processing, which could potentially crash the consumer threads
+
+## v0.8.1
+
+* [#5](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/5)
+  * Refactored worker pool based message processing and error handling
+
+## v0.8.0
+
+* [#3](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/3)
+  * Added a worker pool:
+    * now processing of messages from a single queue can be done in multiple parallel threads (workers)
+    * the worker threads are shared across all message consumers which read messages from different queues
+    * the size of the worker pool is limited, to have at most `mq_worker_pool_max_threads` processing messages in parallel
+    * `mq_worker_pool_min_threads` determines the target minimal number of threads in the pool, waiting for new messages
+    * `mq_worker_pool_max_backlog` controls how many messages which are read from SQS queues can be put in the worker pool backlog; if a new message is read from SQS queue and the backlog is full, this message is NACK-ed (put back into SQS queue for the other consumers to process)
+  * Improved thread-safety of the adapter: reply consumer, TimeoutQueue#pop
+
+## v0.7.0
+
+* [#1](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/1)
+  * Added KMS support for creating queues/topics with sever-side encryption enabled
+  * Optimized stopping of the adapter: stopping all consumers in parallel
+
+
+## v0.6.x
+
+* Basic functionality implemented, see missing features in [TODO](TODO.md)

data/README.md

@@ -2,6 +2,9 @@
 
 AWS SQS/SNS adapter for [mimi-messaging](https://github.com/kukushkin/mimi-messaging).
 
+* [Changes](CHANGELOG.md)
+
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -43,7 +46,6 @@ Mimi::Messaging.configure(
 Mimi::Messaging.start
 ```
 
-
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/kukushkin/mimi-messaging-sqs_sns. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.

data/TODO.md

@@ -0,0 +1,9 @@
+# TODO
+
+List of missing features planned for future releases.
+
+* [X] Log error and recover if the reply cannot be sent
+* [X] Threadsafe TimeoutQueue
+* [X] Multithreaded consumers
+* [ ] Subscribe to topic without a queue (Temporary queues)
+

data/examples/event.rb

@@ -4,8 +4,8 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 
@@ -30,5 +30,5 @@ COUNT.times do |i|
   t = Time.now
   puts "Publishing event: #{i}"
   adapter.event("hello#tested", i: i) # rand(100))
-  sleep 1
+  sleep 0.1
 end

data/examples/mass-query.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "mimi/messaging/sqs_sns"
+
+COUNT = 10
+THREADS = 10
+QUERY_TIMEOUT = 60
+AWS_REGION            = "eu-west-1"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_ACCESS_KEY_ID     = "foo"
+AWS_SECRET_ACCESS_KEY = "bar"
+AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+Mimi::Messaging.use(logger: logger, serializer: Mimi::Messaging::JsonSerializer)
+Mimi::Messaging.configure(
+  mq_adapter: "sqs_sns",
+  mq_aws_access_key_id:     AWS_ACCESS_KEY_ID,
+  mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
+  mq_aws_region:            AWS_REGION,
+  mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID,
+  mq_default_query_timeout: QUERY_TIMEOUT
+)
+adapter = Mimi::Messaging.adapter
+
+adapter.start
+
+t_start = Time.now
+t_queries = []
+threads = []
+THREADS.times do |ti|
+  threads << Thread.new do
+    COUNT.times do |i|
+      t = Time.now
+      result = adapter.query("test/hello", i: i) # rand(100))
+      t = Time.now - t
+      t_queries << t
+      puts "result: #{result.to_h}, t: %.3fs" % t
+      sleep 0.1
+    end
+  end
+end
+
+threads.each(&:join)
+
+t_elapsed = Time.now - t_start
+puts "t_elapsed: %.3fs" % t_elapsed
+adapter.stop
+puts "t.avg: %.3fs" % (t_queries.sum / t_queries.count)

data/examples/query.rb

@@ -4,10 +4,11 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
+AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"
 
 logger = Logger.new(STDOUT)
 logger.level = Logger::INFO
@@ -18,7 +19,8 @@ Mimi::Messaging.configure(
   mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
-  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID
 )
 adapter = Mimi::Messaging.adapter
 

data/examples/responder.rb

@@ -2,11 +2,12 @@
 
 require "mimi/messaging/sqs_sns"
 
-AWS_REGION = "eu-west-1"
-AWS_SQS_ENDPOINT_URL = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL = "http://localstack:4575"
-AWS_ACCESS_KEY_ID = "foo"
+AWS_REGION            = "eu-west-1"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
+AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"
 
 class Processor
   def self.call_command(method_name, message, opts)
@@ -16,13 +17,14 @@ class Processor
   def self.call_query(method_name, message, opts)
     # puts "QUERY: #{method_name}, #{message}, headers: #{opts[:headers]}"
     puts "QUERY: #{method_name}, headers: #{opts[:headers]}"
+    sleep 1 # imitate work
     {}
   end
 end # class Processor
 
 
 logger = Logger.new(STDOUT)
-logger.level = Logger::INFO
+logger.level = Logger::DEBUG
 Mimi::Messaging.use(logger: logger, serializer: Mimi::Messaging::JsonSerializer)
 Mimi::Messaging.configure(
   mq_adapter: "sqs_sns",
@@ -30,7 +32,12 @@ Mimi::Messaging.configure(
   mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
-  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID,
+  mq_worker_pool_min_threads: 1,
+  mq_worker_pool_max_threads: 2,
+  mq_worker_pool_max_backlog: 4,
+  mq_log_at_level: :info
 )
 adapter = Mimi::Messaging.adapter
 queue_name = "test"
@@ -47,4 +54,3 @@ ensure
   puts "Stopping adapter"
   adapter.stop
 end
-

data/examples/subscriber.rb

@@ -3,8 +3,8 @@
 require "mimi/messaging/sqs_sns"
 
 AWS_REGION = "eu-west-1"
-AWS_SQS_ENDPOINT_URL = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 
@@ -20,6 +20,7 @@ class Processor
 
   def self.call_event(event_type, message, opts)
     puts "EVENT: #{event_type}, #{message}, headers: #{message.headers}"
+    sleep 1 # imitate work
   end
 end # class Processor
 
@@ -34,6 +35,9 @@ Mimi::Messaging.configure(
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
   mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_worker_pool_min_threads: 1,
+  mq_worker_pool_max_threads: 2,
+  mq_worker_pool_max_backlog: 4,
   mq_log_at_level: :info
 )
 adapter = Mimi::Messaging.adapter
@@ -52,4 +56,3 @@ ensure
   puts "Stopping adapter"
   adapter.stop
 end
-

data/lib/mimi/messaging/sqs_sns/adapter.rb

@@ -3,8 +3,9 @@
 require "mimi/messaging"
 require "aws-sdk-sqs"
 require "aws-sdk-sns"
-require "timeout"
 require "securerandom"
+require "concurrent"
+require_relative "timeout_queue"
 
 module Mimi
   module Messaging
@@ -40,7 +41,7 @@ module Mimi
           "." => "-"
         }.freeze
 
-        attr_reader :options, :sqs_client, :sns_client
+        attr_reader :options, :sqs_client, :sns_client, :worker_pool
 
         register_adapter_name "sqs_sns"
 
@@ -49,6 +50,11 @@ module Mimi
           mq_default_query_timeout: 15, # seconds,
           mq_reply_queue_prefix: "reply-",
 
+          # worker pool parameters
+          mq_worker_pool_min_threads: 1,
+          mq_worker_pool_max_threads: 16,
+          mq_worker_pool_max_backlog: 16,
+
           # if nil, AWS SDK will guess values from environment
           mq_aws_region: nil,
           mq_aws_access_key_id: nil,
@@ -56,7 +62,8 @@ module Mimi
           mq_aws_sqs_endpoint: nil,
           mq_aws_sns_endpoint: nil,
 
-          mq_aws_sqs_read_timeout: 10, # seconds
+          mq_aws_sqs_sns_kms_master_key_id: nil,
+          mq_aws_sqs_read_timeout: 20, # seconds
         }.freeze
 
         # Initializes SQS/SNS adapter
@@ -73,16 +80,19 @@ module Mimi
         #
         def initialize(options)
           @options = DEFAULT_OPTIONS.merge(options).dup
+          @reply_consumer_mutex = Mutex.new
         end
 
         def start
           @sqs_client = Aws::SQS::Client.new(sqs_client_config)
           @sns_client = Aws::SNS::Client.new(sns_client_config)
+          start_worker_pool!
           check_availability!
         end
 
         def stop
           stop_all_processors
+          stop_worker_pool!
           @sqs_client = nil
           @sns_client = nil
         end
@@ -93,6 +103,7 @@ module Mimi
         # for processors.
         #
         def stop_all_processors
+          @consumers&.each(&:signal_stop)
           @consumers&.each(&:stop)
           @consumers = nil
           @reply_consumer&.stop
@@ -124,7 +135,7 @@ module Mimi
         # @param opts [Hash] additional options, e.g. :timeout
         #
         # @return [Hash]
-        # @raise [SomeError,TimeoutError]
+        # @raise [SomeError,Timeout::Error]
         #
         def query(target, message, opts = {})
           queue_name, method_name = target.split("/")
@@ -140,10 +151,7 @@ module Mimi
           )
           deliver_message_queue(queue_url, message)
           timeout = opts[:timeout] || options[:mq_default_query_timeout]
-          response = nil
-          Timeout::timeout(timeout) do
-            response = reply_queue.pop
-          end
+          response = reply_queue.pop(true, timeout)
           deserialize(response.body)
         end
 
@@ -178,22 +186,7 @@ module Mimi
           opts = opts.dup
           queue_url = find_or_create_queue(queue_name)
           @consumers << Consumer.new(self, queue_url) do |m|
-            message = Mimi::Messaging::Message.new(
-              deserialize(m.body),
-              deserialize_headers(m)
-            )
-            method_name = message.headers[:__method]
-            reply_to = message.headers[:__reply_queue_url]
-            if reply_to
-              response = processor.call_query(method_name, message, {})
-              response_message = Mimi::Messaging::Message.new(
-                response,
-                __request_id: message.headers[:__request_id]
-              )
-              deliver_message_queue(reply_to, response_message)
-            else
-              processor.call_command(method_name, message, {})
-            end
+            process_request_message(processor, m)
           end
         end
 
@@ -210,12 +203,7 @@ module Mimi
           queue_url = find_or_create_queue(queue_name)
           subscribe_topic_queue(topic_arn, queue_url)
           @consumers << Consumer.new(self, queue_url) do |m|
-            message = Mimi::Messaging::Message.new(
-              deserialize(m.body),
-              deserialize_headers(m)
-            )
-            event_type = message.headers[:__event_type]
-            processor.call_event(event_type, message, {})
+            process_event_message(processor, m)
           end
         end
 
@@ -227,13 +215,16 @@ module Mimi
         def create_queue(queue_name)
           fqn = sqs_sns_converted_full_name(queue_name)
           Mimi::Messaging.log "Creating a queue: #{fqn}"
-          result = sqs_client.create_queue(queue_name: fqn)
+          attrs = {}
+          if options[:mq_aws_sqs_sns_kms_master_key_id]
+            attrs["KmsMasterKeyId"] = options[:mq_aws_sqs_sns_kms_master_key_id]
+          end
+          result = sqs_client.create_queue(queue_name: fqn, attributes: attrs)
           result.queue_url
         rescue StandardError => e
           raise Mimi::Messaging::ConnectionError, "Failed to create queue '#{queue_name}': #{e}"
         end
 
-
         # Finds a queue URL for a queue with given name.
         #
         # If an existing queue with this name is not found,
@@ -315,6 +306,7 @@ module Mimi
           unless message.is_a?(Mimi::Messaging::Message)
             raise ArgumentError, "Message is expected as argument"
           end
+
           Mimi::Messaging.log "Delivering message to: #{queue_url}, headers: #{message.headers}"
           sqs_client.send_message(
             queue_url: queue_url,
@@ -327,6 +319,47 @@ module Mimi
           raise Mimi::Messaging::ConnectionError, "Failed to deliver message to '#{queue_url}': #{e}"
         end
 
+        # Processes an incoming COMMAND or QUERY message
+        #
+        # @param processor [#call_query(),#call_command()] request processor object
+        # @param sqs_message
+        #
+        def process_request_message(processor, sqs_message)
+          message = Mimi::Messaging::Message.new(
+            deserialize(sqs_message.body),
+            deserialize_headers(sqs_message)
+          )
+          method_name = message.headers[:__method]
+          reply_to = message.headers[:__reply_queue_url]
+          if reply_to
+            response = processor.call_query(method_name, message, {})
+            response_message = Mimi::Messaging::Message.new(
+              response,
+              __request_id: message.headers[:__request_id]
+            )
+            deliver_query_response(reply_to, response_message)
+          else
+            processor.call_command(method_name, message, {})
+          end
+        end
+
+        # Delivers a message as a response to a QUERY
+        #
+        # Responses are allowed to fail. There can be a number of reasons
+        # why responses fail: reply queue does not exist (anymore?),
+        # response message is too big. In any case the error is reported,
+        # but the QUERY message is acknowledged as a successfully processed.
+        #
+        # @param queue_url [String]
+        # @param message [Mimi::Messaging::Message]
+        #
+        def deliver_query_response(queue_url, message)
+          deliver_message_queue(queue_url, message)
+        rescue Mimi::Messaging::ConnectionError => e
+          Mimi::Messaging.logger&.warn("Failed to deliver QRY response: #{e}")
+          # NOTE: error is recovered
+        end
+
         # Returns URL of a queue with a given name.
         #
         # If the queue with given name does not exist, returns nil
@@ -379,9 +412,11 @@ module Mimi
         # @return [ReplyConsumer]
         #
         def reply_consumer
-          @reply_consumer ||= begin
-            reply_queue_name = options[:mq_reply_queue_prefix] + SecureRandom.hex(8)
-            Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_name)
+          @reply_consumer_mutex.synchronize do
+            @reply_consumer ||= begin
+              reply_queue_name = options[:mq_reply_queue_prefix] + SecureRandom.hex(8)
+              Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_name)
+            end
           end
         end
 
@@ -462,7 +497,11 @@ module Mimi
         def create_topic(topic_name)
           fqn = sqs_sns_converted_full_name(topic_name)
           Mimi::Messaging.log "Creating a topic: #{fqn}"
-          result = sns_client.create_topic(name: fqn)
+          attrs = {}
+          if options[:mq_aws_sqs_sns_kms_master_key_id]
+            attrs["KmsMasterKeyId"] = options[:mq_aws_sqs_sns_kms_master_key_id]
+          end
+          result = sns_client.create_topic(name: fqn, attributes: attrs)
           result.topic_arn
         rescue StandardError => e
           raise Mimi::Messaging::ConnectionError, "Failed to create topic '#{topic_name}': #{e}"
@@ -479,7 +518,7 @@ module Mimi
           )
           queue_arn = result.attributes["QueueArn"]
           Mimi::Messaging.log "Subscribing queue to a topic: '#{topic_arn}'->'#{queue_url}'"
-          result = sns_client.subscribe(
+          _result = sns_client.subscribe(
             topic_arn: topic_arn,
             protocol: "sqs",
             endpoint: queue_arn,
@@ -501,6 +540,7 @@ module Mimi
           unless message.is_a?(Mimi::Messaging::Message)
             raise ArgumentError, "Message is expected as argument"
           end
+
           Mimi::Messaging.log "Delivering message to: #{topic_arn}"
           sns_client.publish(
             topic_arn: topic_arn,
@@ -512,6 +552,46 @@ module Mimi
         rescue StandardError => e
           raise Mimi::Messaging::ConnectionError, "Failed to deliver message to '#{topic_arn}': #{e}"
         end
+
+        # Processes an incoming EVENT message
+        #
+        # @param processor [#call_event()] event processor object
+        # @param sqs_message []
+        #
+        def process_event_message(processor, sqs_message)
+          message = Mimi::Messaging::Message.new(
+            deserialize(sqs_message.body),
+            deserialize_headers(sqs_message)
+          )
+          event_type = message.headers[:__event_type]
+          processor.call_event(event_type, message, {})
+        end
+
+        # Starts the worker pool using current configuration
+        #
+        # @return [Concurrent::ThreadPoolExecutor]
+        #
+        def start_worker_pool!
+          Mimi::Messaging.log "Starting worker pool, " \
+            "min_threads:#{options[:mq_worker_pool_min_threads]}, " \
+            "max_threads:#{options[:mq_worker_pool_max_threads]}, " \
+            "max_backlog:#{options[:mq_worker_pool_max_backlog]}"
+
+          @worker_pool = Concurrent::ThreadPoolExecutor.new(
+            min_threads: options[:mq_worker_pool_min_threads],
+            max_threads: options[:mq_worker_pool_max_threads],
+            max_queue: options[:mq_worker_pool_max_backlog],
+            fallback_policy: :abort
+          )
+        end
+
+        # Gracefully stops the worker pool, allowing all threads to finish their jobs
+        #
+        def stop_worker_pool!
+          Mimi::Messaging.log "Stopping worker pool"
+          @worker_pool.shutdown
+          @worker_pool.wait_for_termination
+        end
       end # class Adapter
     end # module SQS_SNS
   end # module Messaging

data/lib/mimi/messaging/sqs_sns/consumer.rb

@@ -7,17 +7,28 @@ module Mimi
       # Message consumer for SQS queues
       #
       class Consumer
+        # (seconds) determines how soon the NACK-ed message becomes visible to other consumers
+        NACK_VISIBILITY_TIMEOUT = 1
+
         def initialize(adapter, queue_url, &block)
           @stop_requested = false
           Mimi::Messaging.log "Starting consumer for: #{queue_url}"
           @consumer_thread = Thread.new do
-            while not @stop_requested do
+            while not @stop_requested
               read_and_process_message(adapter, queue_url, block)
             end
             Mimi::Messaging.log "Stopping consumer for: #{queue_url}"
           end
         end
 
+        # Requests the Consumer to stop, without actually waiting for it
+        #
+        def signal_stop
+          @stop_requested = true
+        end
+
+        # Requests the Consumer to stop AND waits until it does
+        #
         def stop
           @stop_requested = true
           @consumer_thread.join
@@ -35,12 +46,21 @@ module Mimi
         def read_and_process_message(adapter, queue_url, block)
           message = read_message(adapter, queue_url)
           return unless message
+
           Mimi::Messaging.log "Read message from: #{queue_url}"
-          block.call(message)
-          ack_message(adapter, queue_url, message)
+          begin
+            adapter.worker_pool.post do
+              process_message(adapter, queue_url, message, block)
+            end
+          rescue Concurrent::RejectedExecutionError
+            # the backlog is overflown, put the message back
+            Mimi::Messaging.log "Worker pool backlog is full, nack-ing the message " \
+              "(workers:#{adapter.worker_pool.length}, backlog:#{adapter.worker_pool.queue_length})"
+            nack_message(adapter, queue_url, message)
+          end
         rescue StandardError => e
           Mimi::Messaging.logger&.error(
-            "#{self.class}: failed to read and process message from: #{queue_url}," \
+            "#{self.class}: failed to read or process message from: #{queue_url}," \
             " error: (#{e.class}) #{e}"
           )
         end
@@ -54,14 +74,49 @@ module Mimi
           )
           return nil if result.messages.count == 0
           return result.messages.first if result.messages.count == 1
+
           raise Mimi::Messaging::ConnectionError, "Unexpected number of messages read"
         end
 
+        def process_message(adapter, queue_url, message, block)
+          block.call(message)
+          ack_message(adapter, queue_url, message)
+        rescue Mimi::Messaging::NACK
+          Mimi::Messaging.log "NACK-ing message from: #{queue_url}"
+          nack_message(adapter, queue_url, message)
+        rescue StandardError => e
+          Mimi::Messaging.logger&.error(
+            "#{self.class}: failed to process message from: #{queue_url}," \
+            " error: (#{e.class}) #{e}"
+          )
+          # NOTE: error is recovered and the message is neither ACKed or NACKed
+        end
+
+        # ACK-ing the message indicates successfull processing of it
+        # and removes the message from the queue
+        #
         def ack_message(adapter, queue_url, msg)
           adapter.sqs_client.delete_message(
             queue_url: queue_url, receipt_handle: msg.receipt_handle
           )
         end
+
+        # NACK-ing the message indicates a failure to process the message.
+        # The message becomes immediately available to other consumers.
+        #
+        def nack_message(adapter, queue_url, msg)
+          adapter.sqs_client.change_message_visibility(
+            queue_url: queue_url,
+            receipt_handle: msg.receipt_handle,
+            visibility_timeout: NACK_VISIBILITY_TIMEOUT
+          )
+        rescue StandardError => e
+          Mimi::Messaging.logger&.error(
+            "#{self.class}: failed to NACK message from: #{queue_url}," \
+            " error: (#{e.class}) #{e}"
+          )
+          # NOTE: error is recovered and the message is neither ACKed or NACKed
+        end
       end # class Consumer
     end # module SQS_SNS
   end # module Messaging

data/lib/mimi/messaging/sqs_sns/reply_consumer.rb

@@ -36,7 +36,7 @@ module Mimi
         # @return [Queue] a new Queue object registered for this request_id
         #
         def register_request_id(request_id)
-          queue = Queue.new
+          queue = TimeoutQueue.new
           @mutex.synchronize do
             queue = @queues[request_id] ||= queue
           end

data/lib/mimi/messaging/sqs_sns/timeout_queue.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module Mimi
+  module Messaging
+    module SQS_SNS
+      # TimeoutQueue solves the problem the native Ruby Queue class has with waiting for elements.
+      #
+      # See the excellent blog post discussing the issue:
+      # https://medium.com/workday-engineering/ruby-concurrency-building-a-timeout-queue-5d7c588ca80d
+      #
+      # TLDR -- using Ruby standard Timeout.timeout() around Queue#pop() is unsafe
+      #
+      #
+      class TimeoutQueue
+        def initialize
+          @elems = []
+          @mutex = Mutex.new
+          @cond_var = ConditionVariable.new
+        end
+
+        # Pushes an element into the queue
+        #
+        # @param elem [Object]
+        #
+        def <<(elem)
+          @mutex.synchronize do
+            @elems << elem
+            @cond_var.signal
+          end
+        end
+        alias push <<
+
+        # Pops an element from the queue in either non-blocking
+        # or a blocking (with an optional timeout) way.
+        #
+        # @param blocking [true,false] wait for a new element (true) or return immediately
+        # @param timeout [nil,Integer] if in blocking mode,
+        #                              wait at most given number of seconds or forever (nil)
+        # @raise [Timeout::Error] if a timeout in blocking mode was reached
+        #
+        def pop(blocking = true, timeout = nil)
+          @mutex.synchronize do
+            if blocking
+              if timeout.nil?
+                while @elems.empty?
+                  @cond_var.wait(@mutex)
+                end
+              else
+                timeout_time = Time.now.to_f + timeout
+                while @elems.empty? && (remaining_time = timeout_time - Time.now.to_f) > 0
+                  @cond_var.wait(@mutex, remaining_time)
+                end
+              end
+            end
+            raise Timeout::Error, "queue timeout expired" if @elems.empty?
+
+            @elems.shift
+          end
+        end
+      end # class TimeoutQueue
+    end # module SQS_SNS
+  end # module Messaging
+end # module Mimi

data/lib/mimi/messaging/sqs_sns/version.rb

@@ -3,7 +3,7 @@
 module Mimi
   module Messaging
     module SQS_SNS
-      VERSION = "0.5.0"
+      VERSION = "0.8.2"
     end
   end
 end

data/mimi-messaging-sqs_sns.gemspec

@@ -36,9 +36,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency "mimi-messaging", "~> 1.2"
   spec.add_dependency "aws-sdk-sqs", "~> 1.22"
   spec.add_dependency "aws-sdk-sns", "~> 1.19"
+  spec.add_dependency "concurrent-ruby"
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "pry", "~> 0.12"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mimi-messaging-sqs_sns
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.8.2
 platform: ruby
 authors:
 - Alex Kukushkin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-17 00:00:00.000000000 Z
+date: 2021-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mimi-messaging
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +100,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -118,14 +138,17 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- TODO.md
 - bin/console
 - bin/setup
 - examples/event.rb
+- examples/mass-query.rb
 - examples/query.rb
 - examples/responder.rb
 - examples/subscriber.rb
@@ -134,6 +157,7 @@ files:
 - lib/mimi/messaging/sqs_sns/consumer.rb
 - lib/mimi/messaging/sqs_sns/reply_consumer.rb
 - lib/mimi/messaging/sqs_sns/temporary_queue_consumer.rb
+- lib/mimi/messaging/sqs_sns/timeout_queue.rb
 - lib/mimi/messaging/sqs_sns/version.rb
 - mimi-messaging-sqs_sns.gemspec
 homepage: https://github.com/kukushkin/mimi-messaging-sqs_sns
@@ -158,8 +182,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: AWS SQS/SNS adapter for mimi-messaging