mimi-messaging-sqs_sns 0.4.3 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c519409be703d17780104c7128f1ac45356e6a28
-  data.tar.gz: d79e3369b555015308c3e794bec2e6e5a7990c56
+SHA256:
+  metadata.gz: d8ca2b61a4f7cc802b628397b74d45a19f81dd50cf7d285bfc813922e17e7327
+  data.tar.gz: fea7216f18b7707cedefaac1090349791c01f35b76f0a3019531039e28e8a39e
 SHA512:
-  metadata.gz: 835a5d1103716fa79de1db5b4869650645ebfd472b8e80343741f5af0f666f12ab07bd16b35eb88b6c9a8d601364b2bcebba86f050e0b111fcec0a627ac631e9
-  data.tar.gz: 6980d6feea3ed5c9e5af6a2a5735c05511b19dcc091e7ac0f62c37cfd7357689b4efdfaa7fbc443d266cfbed9746351d102db1c433d9e44cd30df5afa9861e56
+  metadata.gz: c4734aef7603919fe1485b1403fe63cfe3751b25ca0c753336b1567b524817ad68d9a0255fb14a993f365d58963b3a5a894bb02b338eac19d2af01303455510b
+  data.tar.gz: 017f5eb8d84cb0b10196c6a76fee01b5c1f3909c3b3eca2883e729992b02370f9be9827639858cbcb7e68beb02c35e4ac6cc9b5f40726b9e8faaccf5c8ee7eb6

data/CHANGELOG.md

@@ -0,0 +1,30 @@
+# Changelog
+
+## UNRELEASED
+
+## v0.8.1
+
+* [#5](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/5)
+  * Refactored worker pool based message processing and error handling
+
+## v0.8.0
+
+* [#3](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/3)
+  * Added a worker pool:
+    * now processing of messages from a single queue can be done in multiple parallel threads (workers)
+    * the worker threads are shared across all message consumers which read messages from different queues
+    * the size of the worker pool is limited, to have at most `mq_worker_pool_max_threads` processing messages in parallel
+    * `mq_worker_pool_min_threads` determines the target minimal number of threads in the pool, waiting for new messages
+    * `mq_worker_pool_max_backlog` controls how many messages which are read from SQS queues can be put in the worker pool backlog; if a new message is read from SQS queue and the backlog is full, this message is NACK-ed (put back into SQS queue for the other consumers to process)
+  * Improved thread-safety of the adapter: reply consumer, TimeoutQueue#pop
+
+## v0.7.0
+
+* [#1](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/1)
+  * Added KMS support for creating queues/topics with sever-side encryption enabled
+  * Optimized stopping of the adapter: stopping all consumers in parallel
+
+
+## v0.6.x
+
+* Basic functionality implemented, see missing features in [TODO](TODO.md)

data/README.md

@@ -2,6 +2,9 @@
 
 AWS SQS/SNS adapter for [mimi-messaging](https://github.com/kukushkin/mimi-messaging).
 
+* [Changes](CHANGELOG.md)
+
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -43,7 +46,6 @@ Mimi::Messaging.configure(
 Mimi::Messaging.start
 ```
 
-
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/kukushkin/mimi-messaging-sqs_sns. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.

data/TODO.md

@@ -0,0 +1,9 @@
+# TODO
+
+List of missing features planned for future releases.
+
+* [X] Log error and recover if the reply cannot be sent
+* [X] Threadsafe TimeoutQueue
+* [X] Multithreaded consumers
+* [ ] Subscribe to topic without a queue (Temporary queues)
+

data/examples/event.rb

@@ -4,8 +4,8 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 
@@ -30,5 +30,5 @@ COUNT.times do |i|
   t = Time.now
   puts "Publishing event: #{i}"
   adapter.event("hello#tested", i: i) # rand(100))
-  sleep 1
+  sleep 0.1
 end

data/examples/mass-query.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "mimi/messaging/sqs_sns"
+
+COUNT = 10
+THREADS = 10
+QUERY_TIMEOUT = 60
+AWS_REGION            = "eu-west-1"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_ACCESS_KEY_ID     = "foo"
+AWS_SECRET_ACCESS_KEY = "bar"
+AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+Mimi::Messaging.use(logger: logger, serializer: Mimi::Messaging::JsonSerializer)
+Mimi::Messaging.configure(
+  mq_adapter: "sqs_sns",
+  mq_aws_access_key_id:     AWS_ACCESS_KEY_ID,
+  mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
+  mq_aws_region:            AWS_REGION,
+  mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID,
+  mq_default_query_timeout: QUERY_TIMEOUT
+)
+adapter = Mimi::Messaging.adapter
+
+adapter.start
+
+t_start = Time.now
+t_queries = []
+threads = []
+THREADS.times do |ti|
+  threads << Thread.new do
+    COUNT.times do |i|
+      t = Time.now
+      result = adapter.query("test/hello", i: i) # rand(100))
+      t = Time.now - t
+      t_queries << t
+      puts "result: #{result.to_h}, t: %.3fs" % t
+      sleep 0.1
+    end
+  end
+end
+
+threads.each(&:join)
+
+t_elapsed = Time.now - t_start
+puts "t_elapsed: %.3fs" % t_elapsed
+adapter.stop
+puts "t.avg: %.3fs" % (t_queries.sum / t_queries.count)

data/examples/query.rb

@@ -4,10 +4,11 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
+AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"
 
 logger = Logger.new(STDOUT)
 logger.level = Logger::INFO
@@ -18,7 +19,8 @@ Mimi::Messaging.configure(
   mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
-  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID
 )
 adapter = Mimi::Messaging.adapter
 

data/examples/responder.rb

@@ -2,11 +2,12 @@
 
 require "mimi/messaging/sqs_sns"
 
-AWS_REGION = "eu-west-1"
-AWS_SQS_ENDPOINT_URL = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL = "http://localstack:4575"
-AWS_ACCESS_KEY_ID = "foo"
+AWS_REGION            = "eu-west-1"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
+AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"
 
 class Processor
   def self.call_command(method_name, message, opts)
@@ -16,13 +17,14 @@ class Processor
   def self.call_query(method_name, message, opts)
     # puts "QUERY: #{method_name}, #{message}, headers: #{opts[:headers]}"
     puts "QUERY: #{method_name}, headers: #{opts[:headers]}"
+    sleep 1 # imitate work
     {}
   end
 end # class Processor
 
 
 logger = Logger.new(STDOUT)
-logger.level = Logger::INFO
+logger.level = Logger::DEBUG
 Mimi::Messaging.use(logger: logger, serializer: Mimi::Messaging::JsonSerializer)
 Mimi::Messaging.configure(
   mq_adapter: "sqs_sns",
@@ -30,7 +32,12 @@ Mimi::Messaging.configure(
   mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
-  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID,
+  mq_worker_pool_min_threads: 1,
+  mq_worker_pool_max_threads: 2,
+  mq_worker_pool_max_backlog: 4,
+  mq_log_at_level: :info
 )
 adapter = Mimi::Messaging.adapter
 queue_name = "test"
@@ -47,4 +54,3 @@ ensure
   puts "Stopping adapter"
   adapter.stop
 end
-

data/examples/subscriber.rb

@@ -3,8 +3,8 @@
 require "mimi/messaging/sqs_sns"
 
 AWS_REGION = "eu-west-1"
-AWS_SQS_ENDPOINT_URL = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 
@@ -20,6 +20,7 @@ class Processor
 
   def self.call_event(event_type, message, opts)
     puts "EVENT: #{event_type}, #{message}, headers: #{message.headers}"
+    sleep 1 # imitate work
   end
 end # class Processor
 
@@ -34,6 +35,9 @@ Mimi::Messaging.configure(
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
   mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_worker_pool_min_threads: 1,
+  mq_worker_pool_max_threads: 2,
+  mq_worker_pool_max_backlog: 4,
   mq_log_at_level: :info
 )
 adapter = Mimi::Messaging.adapter
@@ -52,4 +56,3 @@ ensure
   puts "Stopping adapter"
   adapter.stop
 end
-

data/lib/mimi/messaging/sqs_sns.rb

@@ -12,4 +12,5 @@ end # module Mimi
 
 require_relative "sqs_sns/adapter"
 require_relative "sqs_sns/consumer"
+require_relative "sqs_sns/temporary_queue_consumer"
 require_relative "sqs_sns/reply_consumer"

data/lib/mimi/messaging/sqs_sns/adapter.rb

@@ -3,8 +3,9 @@
 require "mimi/messaging"
 require "aws-sdk-sqs"
 require "aws-sdk-sns"
-require "timeout"
 require "securerandom"
+require "concurrent"
+require_relative "timeout_queue"
 
 module Mimi
   module Messaging
@@ -40,7 +41,7 @@ module Mimi
           "." => "-"
         }.freeze
 
-        attr_reader :options, :sqs_client, :sns_client
+        attr_reader :options, :sqs_client, :sns_client, :worker_pool
 
         register_adapter_name "sqs_sns"
 
@@ -49,6 +50,11 @@ module Mimi
           mq_default_query_timeout: 15, # seconds,
           mq_reply_queue_prefix: "reply-",
 
+          # worker pool parameters
+          mq_worker_pool_min_threads: 1,
+          mq_worker_pool_max_threads: 16,
+          mq_worker_pool_max_backlog: 16,
+
           # if nil, AWS SDK will guess values from environment
           mq_aws_region: nil,
           mq_aws_access_key_id: nil,
@@ -56,7 +62,8 @@ module Mimi
           mq_aws_sqs_endpoint: nil,
           mq_aws_sns_endpoint: nil,
 
-          mq_aws_sqs_read_timeout: 10, # seconds
+          mq_aws_sqs_sns_kms_master_key_id: nil,
+          mq_aws_sqs_read_timeout: 20, # seconds
         }.freeze
 
         # Initializes SQS/SNS adapter
@@ -73,16 +80,19 @@ module Mimi
         #
         def initialize(options)
           @options = DEFAULT_OPTIONS.merge(options).dup
+          @reply_consumer_mutex = Mutex.new
         end
 
         def start
           @sqs_client = Aws::SQS::Client.new(sqs_client_config)
           @sns_client = Aws::SNS::Client.new(sns_client_config)
+          start_worker_pool!
           check_availability!
         end
 
         def stop
           stop_all_processors
+          stop_worker_pool!
           @sqs_client = nil
           @sns_client = nil
         end
@@ -93,6 +103,7 @@ module Mimi
         # for processors.
         #
         def stop_all_processors
+          @consumers&.each(&:signal_stop)
           @consumers&.each(&:stop)
           @consumers = nil
           @reply_consumer&.stop
@@ -124,7 +135,7 @@ module Mimi
         # @param opts [Hash] additional options, e.g. :timeout
         #
         # @return [Hash]
-        # @raise [SomeError,TimeoutError]
+        # @raise [SomeError,Timeout::Error]
         #
         def query(target, message, opts = {})
           queue_name, method_name = target.split("/")
@@ -140,10 +151,7 @@ module Mimi
           )
           deliver_message_queue(queue_url, message)
           timeout = opts[:timeout] || options[:mq_default_query_timeout]
-          response = nil
-          Timeout::timeout(timeout) do
-            response = reply_queue.pop
-          end
+          response = reply_queue.pop(true, timeout)
           deserialize(response.body)
         end
 
@@ -178,22 +186,7 @@ module Mimi
           opts = opts.dup
           queue_url = find_or_create_queue(queue_name)
           @consumers << Consumer.new(self, queue_url) do |m|
-            message = Mimi::Messaging::Message.new(
-              deserialize(m.body),
-              deserialize_headers(m)
-            )
-            method_name = message.headers[:__method]
-            reply_to = message.headers[:__reply_queue_url]
-            if reply_to
-              response = processor.call_query(method_name, message, {})
-              response_message = Mimi::Messaging::Message.new(
-                response,
-                __request_id: message.headers[:__request_id]
-              )
-              deliver_message_queue(reply_to, response_message)
-            else
-              processor.call_command(method_name, message, {})
-            end
+            process_request_message(processor, m)
           end
         end
 
@@ -210,15 +203,52 @@ module Mimi
           queue_url = find_or_create_queue(queue_name)
           subscribe_topic_queue(topic_arn, queue_url)
           @consumers << Consumer.new(self, queue_url) do |m|
-            message = Mimi::Messaging::Message.new(
-              deserialize(m.body),
-              deserialize_headers(m)
-            )
-            event_type = message.headers[:__event_type]
-            processor.call_event(event_type, message, {})
+            process_event_message(processor, m)
           end
         end
 
+        # Creates a new queue
+        #
+        # @param queue_name [String] name of the topic to be created
+        # @return [String] a new queue URL
+        #
+        def create_queue(queue_name)
+          fqn = sqs_sns_converted_full_name(queue_name)
+          Mimi::Messaging.log "Creating a queue: #{fqn}"
+          attrs = {}
+          if options[:mq_aws_sqs_sns_kms_master_key_id]
+            attrs["KmsMasterKeyId"] = options[:mq_aws_sqs_sns_kms_master_key_id]
+          end
+          result = sqs_client.create_queue(queue_name: fqn, attributes: attrs)
+          result.queue_url
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError, "Failed to create queue '#{queue_name}': #{e}"
+        end
+
+        # Finds a queue URL for a queue with given name.
+        #
+        # If an existing queue with this name is not found,
+        # the method will try to create a new one.
+        #
+        # @param queue_name [String]
+        # @return [String] a queue URL
+        #
+        def find_or_create_queue(queue_name)
+          queue_registry(queue_name) || create_queue(queue_name)
+        end
+
+        # Deletes a queue identified by the queue URL
+        #
+        # @param queue_url [String]
+        #
+        def delete_queue(queue_url)
+          Mimi::Messaging.log "Deleting a queue: #{queue_url}"
+          sqs_client.delete_queue(queue_url: queue_url)
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError,
+            "Failed to delete queue with url '#{queue_url}': #{e}"
+        end
+
         private
 
         # Returns configuration parameters for AWS SQS client
@@ -266,20 +296,6 @@ module Mimi
           end
         end
 
-        # Creates a new queue
-        #
-        # @param queue_name [String] name of the topic to be created
-        # @return [String] a new queue URL
-        #
-        def create_queue(queue_name)
-          fqn = sqs_sns_converted_full_name(queue_name)
-          Mimi::Messaging.log "Creating a queue: #{fqn}"
-          result = sqs_client.create_queue(queue_name: fqn)
-          result.queue_url
-        rescue StandardError => e
-          raise Mimi::Messaging::ConnectionError, "Failed to create queue '#{queue_name}': #{e}"
-        end
-
         # Delivers a message to a queue with given URL.
         #
         # @param queue_url [String]
@@ -290,6 +306,7 @@ module Mimi
           unless message.is_a?(Mimi::Messaging::Message)
             raise ArgumentError, "Message is expected as argument"
           end
+
           Mimi::Messaging.log "Delivering message to: #{queue_url}, headers: #{message.headers}"
           sqs_client.send_message(
             queue_url: queue_url,
@@ -302,6 +319,47 @@ module Mimi
           raise Mimi::Messaging::ConnectionError, "Failed to deliver message to '#{queue_url}': #{e}"
         end
 
+        # Processes an incoming COMMAND or QUERY message
+        #
+        # @param processor [#call_query(),#call_command()] request processor object
+        # @param sqs_message
+        #
+        def process_request_message(processor, sqs_message)
+          message = Mimi::Messaging::Message.new(
+            deserialize(sqs_message.body),
+            deserialize_headers(sqs_message)
+          )
+          method_name = message.headers[:__method]
+          reply_to = message.headers[:__reply_queue_url]
+          if reply_to
+            response = processor.call_query(method_name, message, {})
+            response_message = Mimi::Messaging::Message.new(
+              response,
+              __request_id: message.headers[:__request_id]
+            )
+            deliver_query_response(reply_to, response_message)
+          else
+            processor.call_command(method_name, message, {})
+          end
+        end
+
+        # Delivers a message as a response to a QUERY
+        #
+        # Responses are allowed to fail. There can be a number of reasons
+        # why responses fail: reply queue does not exist (anymore?),
+        # response message is too big. In any case the error is reported,
+        # but the QUERY message is acknowledged as a successfully processed.
+        #
+        # @param queue_url [String]
+        # @param message [Mimi::Messaging::Message]
+        #
+        def deliver_query_response(queue_url, message)
+          deliver_message_queue(queue_url, message)
+        rescue Mimi::Messaging::ConnectionError => e
+          Mimi::Messaging.logger&.warn("Failed to deliver QRY response: #{e}")
+          # NOTE: error is recovered
+        end
+
         # Returns URL of a queue with a given name.
         #
         # If the queue with given name does not exist, returns nil
@@ -349,27 +407,16 @@ module Mimi
           )
         end
 
-        # Finds a queue URL for a queue with given name.
-        #
-        # If an existing queue with this name is not found,
-        # the method will try to create a new one.
-        #
-        # @param queue_name [String]
-        # @return [String] a queue URL
-        #
-        def find_or_create_queue(queue_name)
-          queue_registry(queue_name) || create_queue(queue_name)
-        end
-
         # Returns the configured reply listener for this process
         #
         # @return [ReplyConsumer]
         #
         def reply_consumer
-          @reply_consumer ||= begin
-            reply_queue_name = options[:mq_reply_queue_prefix] + SecureRandom.hex(8)
-            reply_queue_url = create_queue(reply_queue_name)
-            Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_url)
+          @reply_consumer_mutex.synchronize do
+            @reply_consumer ||= begin
+              reply_queue_name = options[:mq_reply_queue_prefix] + SecureRandom.hex(8)
+              Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_name)
+            end
           end
         end
 
@@ -450,7 +497,11 @@ module Mimi
         def create_topic(topic_name)
           fqn = sqs_sns_converted_full_name(topic_name)
           Mimi::Messaging.log "Creating a topic: #{fqn}"
-          result = sns_client.create_topic(name: fqn)
+          attrs = {}
+          if options[:mq_aws_sqs_sns_kms_master_key_id]
+            attrs["KmsMasterKeyId"] = options[:mq_aws_sqs_sns_kms_master_key_id]
+          end
+          result = sns_client.create_topic(name: fqn, attributes: attrs)
           result.topic_arn
         rescue StandardError => e
           raise Mimi::Messaging::ConnectionError, "Failed to create topic '#{topic_name}': #{e}"
@@ -467,7 +518,7 @@ module Mimi
           )
           queue_arn = result.attributes["QueueArn"]
           Mimi::Messaging.log "Subscribing queue to a topic: '#{topic_arn}'->'#{queue_url}'"
-          result = sns_client.subscribe(
+          _result = sns_client.subscribe(
             topic_arn: topic_arn,
             protocol: "sqs",
             endpoint: queue_arn,
@@ -489,6 +540,7 @@ module Mimi
           unless message.is_a?(Mimi::Messaging::Message)
             raise ArgumentError, "Message is expected as argument"
           end
+
           Mimi::Messaging.log "Delivering message to: #{topic_arn}"
           sns_client.publish(
             topic_arn: topic_arn,
@@ -500,6 +552,46 @@ module Mimi
         rescue StandardError => e
           raise Mimi::Messaging::ConnectionError, "Failed to deliver message to '#{topic_arn}': #{e}"
         end
+
+        # Processes an incoming EVENT message
+        #
+        # @param processor [#call_event()] event processor object
+        # @param sqs_message []
+        #
+        def process_event_message(processor, sqs_message)
+          message = Mimi::Messaging::Message.new(
+            deserialize(sqs_message.body),
+            deserialize_headers(sqs_message)
+          )
+          event_type = message.headers[:__event_type]
+          processor.call_event(event_type, message, {})
+        end
+
+        # Starts the worker pool using current configuration
+        #
+        # @return [Concurrent::ThreadPoolExecutor]
+        #
+        def start_worker_pool!
+          Mimi::Messaging.log "Starting worker pool, " \
+            "min_threads:#{options[:mq_worker_pool_min_threads]}, " \
+            "max_threads:#{options[:mq_worker_pool_max_threads]}, " \
+            "max_backlog:#{options[:mq_worker_pool_max_backlog]}"
+
+          @worker_pool = Concurrent::ThreadPoolExecutor.new(
+            min_threads: options[:mq_worker_pool_min_threads],
+            max_threads: options[:mq_worker_pool_max_threads],
+            max_queue: options[:mq_worker_pool_max_backlog],
+            fallback_policy: :abort
+          )
+        end
+
+        # Gracefully stops the worker pool, allowing all threads to finish their jobs
+        #
+        def stop_worker_pool!
+          Mimi::Messaging.log "Stopping worker pool"
+          @worker_pool.shutdown
+          @worker_pool.wait_for_termination
+        end
       end # class Adapter
     end # module SQS_SNS
   end # module Messaging

data/lib/mimi/messaging/sqs_sns/consumer.rb

@@ -4,25 +4,31 @@ module Mimi
   module Messaging
     module SQS_SNS
       #
-      # Message processor for SQS queues
+      # Message consumer for SQS queues
       #
       class Consumer
+        # (seconds) determines how soon the NACK-ed message becomes visible to other consumers
+        NACK_VISIBILITY_TIMEOUT = 1
+
         def initialize(adapter, queue_url, &block)
-          @block = block
           @stop_requested = false
           Mimi::Messaging.log "Starting consumer for: #{queue_url}"
           @consumer_thread = Thread.new do
-            while not @stop_requested do
-              message = read_message(adapter, queue_url)
-              next unless message
-              Mimi::Messaging.log "Read message from: #{queue_url}"
-              block.call(message)
-              ack_message(adapter, queue_url, message)
+            while not @stop_requested
+              read_and_process_message(adapter, queue_url, block)
             end
             Mimi::Messaging.log "Stopping consumer for: #{queue_url}"
           end
         end
 
+        # Requests the Consumer to stop, without actually waiting for it
+        #
+        def signal_stop
+          @stop_requested = true
+        end
+
+        # Requests the Consumer to stop AND waits until it does
+        #
         def stop
           @stop_requested = true
           @consumer_thread.join
@@ -35,16 +41,24 @@ module Mimi
         #
         # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
         # @param queue_url [String]
+        # @param block [Proc] a block to be invoked when a message is received
         #
-        def read_and_process_message(adapter, queue_url)
+        def read_and_process_message(adapter, queue_url, block)
           message = read_message(adapter, queue_url)
           return unless message
+
           Mimi::Messaging.log "Read message from: #{queue_url}"
-          block.call(message)
-          ack_message(adapter, queue_url, message)
+          adapter.worker_pool.post do
+            process_message(adapter, queue_url, message, block)
+          end
+        rescue Concurrent::RejectedExecutionError
+          # the backlog is overflown, put the message back
+          Mimi::Messaging.log "Worker pool backlog is full, nack-ing the message " \
+            "(workers:#{adapter.worker_pool.length}, backlog:#{adapter.worker_pool.queue_length})"
+          nack_message(adapter, queue_url, message)
         rescue StandardError => e
           Mimi::Messaging.logger&.error(
-            "#{self.class}: failed to read and process message from: #{queue_url}," \
+            "#{self.class}: failed to read or process message from: #{queue_url}," \
             " error: (#{e.class}) #{e}"
           )
         end
@@ -58,14 +72,43 @@ module Mimi
           )
           return nil if result.messages.count == 0
           return result.messages.first if result.messages.count == 1
+
           raise Mimi::Messaging::ConnectionError, "Unexpected number of messages read"
         end
 
+        def process_message(adapter, queue_url, message, block)
+          block.call(message)
+          ack_message(adapter, queue_url, message)
+        rescue Mimi::Messaging::NACK
+          Mimi::Messaging.log "NACK-ing message from: #{queue_url}"
+          nack_message(adapter, queue_url, message)
+        rescue StandardError => e
+          Mimi::Messaging.logger&.error(
+            "#{self.class}: failed to process message from: #{queue_url}," \
+            " error: (#{e.class}) #{e}"
+          )
+          # NOTE: message is neither ACKed or NACKed
+        end
+
+        # ACK-ing the message indicates successfull processing of it
+        # and removes the message from the queue
+        #
         def ack_message(adapter, queue_url, msg)
           adapter.sqs_client.delete_message(
             queue_url: queue_url, receipt_handle: msg.receipt_handle
           )
         end
+
+        # NACK-ing the message indicates a failure to process the message.
+        # The message becomes immediately available to other consumers.
+        #
+        def nack_message(adapter, queue_url, msg)
+          adapter.sqs_client.change_message_visibility(
+            queue_url: queue_url,
+            receipt_handle: msg.receipt_handle,
+            visibility_timeout: NACK_VISIBILITY_TIMEOUT
+          )
+        end
       end # class Consumer
     end # module SQS_SNS
   end # module Messaging

data/lib/mimi/messaging/sqs_sns/reply_consumer.rb

@@ -8,25 +8,23 @@ module Mimi
       # and passes them to registered Queues (see Ruby ::Queue class).
       #
       class ReplyConsumer
-        attr_reader :reply_queue_url
+        attr_reader :reply_queue_name, :reply_queue_url
 
-        def initialize(adapter, reply_queue_url)
+        def initialize(adapter, reply_queue_name)
           @mutex = Mutex.new
           @queues = {}
           @adapter = adapter
-          @reply_queue_url = reply_queue_url
-          @consumer = Consumer.new(adapter, reply_queue_url) do |message|
+          @reply_queue_name = reply_queue_name
+          @consumer = TemporaryQueueConsumer.new(adapter, reply_queue_name) do |message|
             dispatch_message(message)
           end
+          @reply_queue_url = @consumer.queue_url
         end
 
         def stop
-          begin
-            @consumer.stop
-          rescue StandardError => e
-            raise Mimi::Messaging::Error, "Failed to stop consumer: #{e}"
-          end
-          # TODO: adapter.sqs_client.delete_queue(reply_queue_url)
+          @consumer.stop
+        rescue StandardError => e
+          raise Mimi::Messaging::Error, "Failed to stop reply consumer: #{e}"
         end
 
         # Register a new request_id to listen for.
@@ -38,7 +36,7 @@ module Mimi
         # @return [Queue] a new Queue object registered for this request_id
         #
         def register_request_id(request_id)
-          queue = Queue.new
+          queue = TimeoutQueue.new
           @mutex.synchronize do
             queue = @queues[request_id] ||= queue
           end

data/lib/mimi/messaging/sqs_sns/temporary_queue_consumer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Mimi
+  module Messaging
+    module SQS_SNS
+      #
+      # Temporary queue consumer creates a temporary queue
+      # and attaches to it. The queue will be deleted
+      # on consumer shutdown.
+      #
+      class TemporaryQueueConsumer
+        attr_reader :queue_url
+
+        def initialize(adapter, queue_name, &block)
+          @adapter = adapter
+          @queue_url = adapter.find_or_create_queue(queue_name)
+          @consumer = Consumer.new(adapter, @queue_url, &block)
+        end
+
+        def stop
+          @consumer.stop
+          @adapter.delete_queue(queue_url)
+        rescue StandardError => e
+          raise Mimi::Messaging::Error, "Failed to stop temporary queue consumer: #{e}"
+        end
+      end # class TemporaryQueueConsumer
+    end # module SQS_SNS
+  end # module Messaging
+end # module Mimi

data/lib/mimi/messaging/sqs_sns/timeout_queue.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module Mimi
+  module Messaging
+    module SQS_SNS
+      # TimeoutQueue solves the problem the native Ruby Queue class has with waiting for elements.
+      #
+      # See the excellent blog post discussing the issue:
+      # https://medium.com/workday-engineering/ruby-concurrency-building-a-timeout-queue-5d7c588ca80d
+      #
+      # TLDR -- using Ruby standard Timeout.timeout() around Queue#pop() is unsafe
+      #
+      #
+      class TimeoutQueue
+        def initialize
+          @elems = []
+          @mutex = Mutex.new
+          @cond_var = ConditionVariable.new
+        end
+
+        # Pushes an element into the queue
+        #
+        # @param elem [Object]
+        #
+        def <<(elem)
+          @mutex.synchronize do
+            @elems << elem
+            @cond_var.signal
+          end
+        end
+        alias push <<
+
+        # Pops an element from the queue in either non-blocking
+        # or a blocking (with an optional timeout) way.
+        #
+        # @param blocking [true,false] wait for a new element (true) or return immediately
+        # @param timeout [nil,Integer] if in blocking mode,
+        #                              wait at most given number of seconds or forever (nil)
+        # @raise [Timeout::Error] if a timeout in blocking mode was reached
+        #
+        def pop(blocking = true, timeout = nil)
+          @mutex.synchronize do
+            if blocking
+              if timeout.nil?
+                while @elems.empty?
+                  @cond_var.wait(@mutex)
+                end
+              else
+                timeout_time = Time.now.to_f + timeout
+                while @elems.empty? && (remaining_time = timeout_time - Time.now.to_f) > 0
+                  @cond_var.wait(@mutex, remaining_time)
+                end
+              end
+            end
+            raise Timeout::Error, "queue timeout expired" if @elems.empty?
+
+            @elems.shift
+          end
+        end
+      end # class TimeoutQueue
+    end # module SQS_SNS
+  end # module Messaging
+end # module Mimi

data/lib/mimi/messaging/sqs_sns/version.rb

@@ -3,7 +3,7 @@
 module Mimi
   module Messaging
     module SQS_SNS
-      VERSION = "0.4.3"
+      VERSION = "0.8.1"
     end
   end
 end

data/mimi-messaging-sqs_sns.gemspec

@@ -36,9 +36,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency "mimi-messaging", "~> 1.2"
   spec.add_dependency "aws-sdk-sqs", "~> 1.22"
   spec.add_dependency "aws-sdk-sns", "~> 1.19"
+  spec.add_dependency "concurrent-ruby"
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "pry", "~> 0.12"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mimi-messaging-sqs_sns
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.8.1
 platform: ruby
 authors:
 - Alex Kukushkin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-21 00:00:00.000000000 Z
+date: 2021-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mimi-messaging
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +100,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -118,14 +138,17 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- TODO.md
 - bin/console
 - bin/setup
 - examples/event.rb
+- examples/mass-query.rb
 - examples/query.rb
 - examples/responder.rb
 - examples/subscriber.rb
@@ -133,6 +156,8 @@ files:
 - lib/mimi/messaging/sqs_sns/adapter.rb
 - lib/mimi/messaging/sqs_sns/consumer.rb
 - lib/mimi/messaging/sqs_sns/reply_consumer.rb
+- lib/mimi/messaging/sqs_sns/temporary_queue_consumer.rb
+- lib/mimi/messaging/sqs_sns/timeout_queue.rb
 - lib/mimi/messaging/sqs_sns/version.rb
 - mimi-messaging-sqs_sns.gemspec
 homepage: https://github.com/kukushkin/mimi-messaging-sqs_sns
@@ -157,8 +182,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: AWS SQS/SNS adapter for mimi-messaging