mimi-messaging-sqs_sns 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b02992abcb101248ebc54069aba5b3887417930b7613a84ed4b6723ad7ce3060
-  data.tar.gz: 3b830249dfb56e42a7d380571402c2abef55d74f3e90bf0b9f537e105078cd27
+  metadata.gz: cf7944b9a3e10f154f738d66544cfa63611b17a742b156460969f70821ad86bb
+  data.tar.gz: f26e3e3623208170de192174c9c0fdb2ef5e26fa8cca831e73065eb86e21b0b5
 SHA512:
-  metadata.gz: 4d6ecfe3d2202124b10ff03129166249f518fca0f15c2e513ba5ab6b2835a713671618eab623fb39e3cbebe3c8351e0688884743e4f69baa3832b1db94a6a858
-  data.tar.gz: 5bb7ae92080d112f16a4127b010f7f102ebd71cb53b4701b756e9bbab8c43fae42618969c148fd71f67678dbf34ed4d6c734307a1c84f7201229d5bd95c3907a
+  metadata.gz: 37bbb87ba16df4886a5e525ca13732b3712021c5269418d8d8c90e1af125cbaa7e32aa5757a96293569a1717cc4c00542aaf20d12f3fc7514477cd3d708cd615
+  data.tar.gz: 9d53cc0479c47af711f0e89203f9ef1ced767115743fb2fa685ea559ebb0b891226a248219ff2103ec95207b678814c1c0b983f82a22eda8e741feac01a2d4dc

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## UNRELEASED
+
+## v0.8.0
+
+* [#3](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/3)
+  * Added a worker pool:
+    * now processing of messages from a single queue can be done in multiple parallel threads (workers)
+    * the worker threads are shared across all message consumers which read messages from different queues
+    * the size of the worker pool is limited, to have at most `mq_worker_pool_max_threads` processing messages in parallel
+    * `mq_worker_pool_min_threads` determines the target minimal number of threads in the pool, waiting for new messages
+    * `mq_worker_pool_max_backlog` controls how many messages which are read from SQS queues can be put in the worker pool backlog; if a new message is read from SQS queue and the backlog is full, this message is NACK-ed (put back into SQS queue for the other consumers to process)
+  * Improved thread-safety of the adapter: reply consumer, TimeoutQueue#pop
+
 ## v0.7.0
 
 * [#1](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/1)

data/TODO.md

@@ -2,8 +2,8 @@
 
 List of missing features planned for future releases.
 
-[X] Log error and recover if the reply cannot be sent
-[ ] Threadsafe TimeoutQueue
-[ ] Multithreaded consumers
-[ ] Subscribe to topic without a queue (Temporary queues)
+* [X] Log error and recover if the reply cannot be sent
+* [ ] Threadsafe TimeoutQueue
+* [ ] Multithreaded consumers
+* [ ] Subscribe to topic without a queue (Temporary queues)
 

data/examples/event.rb

@@ -4,8 +4,8 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 
@@ -30,5 +30,5 @@ COUNT.times do |i|
   t = Time.now
   puts "Publishing event: #{i}"
   adapter.event("hello#tested", i: i) # rand(100))
-  sleep 1
+  sleep 0.1
 end

data/examples/mass-query.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "mimi/messaging/sqs_sns"
+
+COUNT = 10
+THREADS = 10
+QUERY_TIMEOUT = 60
+AWS_REGION            = "eu-west-1"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_ACCESS_KEY_ID     = "foo"
+AWS_SECRET_ACCESS_KEY = "bar"
+AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+Mimi::Messaging.use(logger: logger, serializer: Mimi::Messaging::JsonSerializer)
+Mimi::Messaging.configure(
+  mq_adapter: "sqs_sns",
+  mq_aws_access_key_id:     AWS_ACCESS_KEY_ID,
+  mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
+  mq_aws_region:            AWS_REGION,
+  mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID,
+  mq_default_query_timeout: QUERY_TIMEOUT
+)
+adapter = Mimi::Messaging.adapter
+
+adapter.start
+
+t_start = Time.now
+t_queries = []
+threads = []
+THREADS.times do |ti|
+  threads << Thread.new do
+    COUNT.times do |i|
+      t = Time.now
+      result = adapter.query("test/hello", i: i) # rand(100))
+      t = Time.now - t
+      t_queries << t
+      puts "result: #{result.to_h}, t: %.3fs" % t
+      sleep 0.1
+    end
+  end
+end
+
+threads.each(&:join)
+
+t_elapsed = Time.now - t_start
+puts "t_elapsed: %.3fs" % t_elapsed
+adapter.stop
+puts "t.avg: %.3fs" % (t_queries.sum / t_queries.count)

data/examples/responder.rb

@@ -17,6 +17,7 @@ class Processor
   def self.call_query(method_name, message, opts)
     # puts "QUERY: #{method_name}, #{message}, headers: #{opts[:headers]}"
     puts "QUERY: #{method_name}, headers: #{opts[:headers]}"
+    sleep 1 # imitate work
     {}
   end
 end # class Processor
@@ -32,7 +33,11 @@ Mimi::Messaging.configure(
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
   mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
-  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID,
+  mq_worker_pool_min_threads: 1,
+  mq_worker_pool_max_threads: 2,
+  mq_worker_pool_max_backlog: 4,
+  mq_log_at_level: :info
 )
 adapter = Mimi::Messaging.adapter
 queue_name = "test"

data/examples/subscriber.rb

@@ -3,8 +3,8 @@
 require "mimi/messaging/sqs_sns"
 
 AWS_REGION = "eu-west-1"
-AWS_SQS_ENDPOINT_URL = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 
@@ -20,6 +20,7 @@ class Processor
 
   def self.call_event(event_type, message, opts)
     puts "EVENT: #{event_type}, #{message}, headers: #{message.headers}"
+    sleep 1 # imitate work
   end
 end # class Processor
 
@@ -34,6 +35,9 @@ Mimi::Messaging.configure(
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
   mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_worker_pool_min_threads: 1,
+  mq_worker_pool_max_threads: 2,
+  mq_worker_pool_max_backlog: 4,
   mq_log_at_level: :info
 )
 adapter = Mimi::Messaging.adapter
@@ -52,4 +56,3 @@ ensure
   puts "Stopping adapter"
   adapter.stop
 end
-

data/lib/mimi/messaging/sqs_sns/adapter.rb

@@ -3,8 +3,9 @@
 require "mimi/messaging"
 require "aws-sdk-sqs"
 require "aws-sdk-sns"
-require "timeout"
 require "securerandom"
+require "concurrent"
+require_relative "timeout_queue"
 
 module Mimi
   module Messaging
@@ -40,7 +41,7 @@ module Mimi
           "." => "-"
         }.freeze
 
-        attr_reader :options, :sqs_client, :sns_client
+        attr_reader :options, :sqs_client, :sns_client, :worker_pool
 
         register_adapter_name "sqs_sns"
 
@@ -49,6 +50,11 @@ module Mimi
           mq_default_query_timeout: 15, # seconds,
           mq_reply_queue_prefix: "reply-",
 
+          # worker pool parameters
+          mq_worker_pool_min_threads: 1,
+          mq_worker_pool_max_threads: 16,
+          mq_worker_pool_max_backlog: 16,
+
           # if nil, AWS SDK will guess values from environment
           mq_aws_region: nil,
           mq_aws_access_key_id: nil,
@@ -74,16 +80,19 @@ module Mimi
         #
         def initialize(options)
           @options = DEFAULT_OPTIONS.merge(options).dup
+          @reply_consumer_mutex = Mutex.new
         end
 
         def start
           @sqs_client = Aws::SQS::Client.new(sqs_client_config)
           @sns_client = Aws::SNS::Client.new(sns_client_config)
+          start_worker_pool!
           check_availability!
         end
 
         def stop
           stop_all_processors
+          stop_worker_pool!
           @sqs_client = nil
           @sns_client = nil
         end
@@ -126,7 +135,7 @@ module Mimi
         # @param opts [Hash] additional options, e.g. :timeout
         #
         # @return [Hash]
-        # @raise [SomeError,TimeoutError]
+        # @raise [SomeError,Timeout::Error]
         #
         def query(target, message, opts = {})
           queue_name, method_name = target.split("/")
@@ -142,10 +151,7 @@ module Mimi
           )
           deliver_message_queue(queue_url, message)
           timeout = opts[:timeout] || options[:mq_default_query_timeout]
-          response = nil
-          Timeout::timeout(timeout) do
-            response = reply_queue.pop
-          end
+          response = reply_queue.pop(true, timeout)
           deserialize(response.body)
         end
 
@@ -180,22 +186,14 @@ module Mimi
           opts = opts.dup
           queue_url = find_or_create_queue(queue_name)
           @consumers << Consumer.new(self, queue_url) do |m|
-            message = Mimi::Messaging::Message.new(
-              deserialize(m.body),
-              deserialize_headers(m)
-            )
-            method_name = message.headers[:__method]
-            reply_to = message.headers[:__reply_queue_url]
-            if reply_to
-              response = processor.call_query(method_name, message, {})
-              response_message = Mimi::Messaging::Message.new(
-                response,
-                __request_id: message.headers[:__request_id]
-              )
-              deliver_query_respone(reply_to, response_message)
-            else
-              processor.call_command(method_name, message, {})
+            worker_pool.post do
+              process_request_message(processor, m)
             end
+          rescue Concurrent::RejectedExecutionError
+            # the backlog is overflown, put the message back
+            Mimi::Messaging.log "Worker pool backlog is full, nack-ing the message " \
+              "(workers:#{worker_pool.length}, backlog:#{worker_pool.queue_length})"
+            raise Mimi::Messaging::NACK # exception raised in Consumer thread
           end
         end
 
@@ -212,12 +210,14 @@ module Mimi
           queue_url = find_or_create_queue(queue_name)
           subscribe_topic_queue(topic_arn, queue_url)
           @consumers << Consumer.new(self, queue_url) do |m|
-            message = Mimi::Messaging::Message.new(
-              deserialize(m.body),
-              deserialize_headers(m)
-            )
-            event_type = message.headers[:__event_type]
-            processor.call_event(event_type, message, {})
+            worker_pool.post do
+              process_event_message(processor, m)
+            end
+          rescue Concurrent::RejectedExecutionError
+            # the backlog is overflown, put the message back
+            Mimi::Messaging.log "Worker pool backlog is full, nack-ing the message " \
+              "(workers:#{worker_pool.length}, backlog:#{worker_pool.queue_length})"
+            raise Mimi::Messaging::NACK # exception raised in Consumer thread
           end
         end
 
@@ -333,6 +333,30 @@ module Mimi
           raise Mimi::Messaging::ConnectionError, "Failed to deliver message to '#{queue_url}': #{e}"
         end
 
+        # Processes an incoming COMMAND or QUERY message
+        #
+        # @param processor [#call_query(),#call_command()] request processor object
+        # @param sqs_message
+        #
+        def process_request_message(processor, sqs_message)
+          message = Mimi::Messaging::Message.new(
+            deserialize(sqs_message.body),
+            deserialize_headers(sqs_message)
+          )
+          method_name = message.headers[:__method]
+          reply_to = message.headers[:__reply_queue_url]
+          if reply_to
+            response = processor.call_query(method_name, message, {})
+            response_message = Mimi::Messaging::Message.new(
+              response,
+              __request_id: message.headers[:__request_id]
+            )
+            deliver_query_response(reply_to, response_message)
+          else
+            processor.call_command(method_name, message, {})
+          end
+        end
+
         # Delivers a message as a response to a QUERY
         #
         # Responses are allowed to fail. There can be a number of reasons
@@ -343,7 +367,7 @@ module Mimi
         # @param queue_url [String]
         # @param message [Mimi::Messaging::Message]
         #
-        def deliver_query_respone(queue_url, message)
+        def deliver_query_response(queue_url, message)
           deliver_message_queue(queue_url, message)
         rescue Mimi::Messaging::ConnectionError => e
           Mimi::Messaging.logger&.warn("Failed to deliver QRY response: #{e}")
@@ -402,9 +426,11 @@ module Mimi
         # @return [ReplyConsumer]
         #
         def reply_consumer
-          @reply_consumer ||= begin
-            reply_queue_name = options[:mq_reply_queue_prefix] + SecureRandom.hex(8)
-            Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_name)
+          @reply_consumer_mutex.synchronize do
+            @reply_consumer ||= begin
+              reply_queue_name = options[:mq_reply_queue_prefix] + SecureRandom.hex(8)
+              Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_name)
+            end
           end
         end
 
@@ -540,6 +566,46 @@ module Mimi
         rescue StandardError => e
           raise Mimi::Messaging::ConnectionError, "Failed to deliver message to '#{topic_arn}': #{e}"
         end
+
+        # Processes an incoming EVENT message
+        #
+        # @param processor [#call_event()] event processor object
+        # @param sqs_message []
+        #
+        def process_event_message(processor, sqs_message)
+          message = Mimi::Messaging::Message.new(
+            deserialize(sqs_message.body),
+            deserialize_headers(sqs_message)
+          )
+          event_type = message.headers[:__event_type]
+          processor.call_event(event_type, message, {})
+        end
+
+        # Starts the worker pool using current configuration
+        #
+        # @return [Concurrent::ThreadPoolExecutor]
+        #
+        def start_worker_pool!
+          Mimi::Messaging.log "Starting worker pool, " \
+            "min_threads:#{options[:mq_worker_pool_min_threads]}, " \
+            "max_threads:#{options[:mq_worker_pool_max_threads]}, " \
+            "max_backlog:#{options[:mq_worker_pool_max_backlog]}"
+
+          @worker_pool = Concurrent::ThreadPoolExecutor.new(
+            min_threads: options[:mq_worker_pool_min_threads],
+            max_threads: options[:mq_worker_pool_max_threads],
+            max_queue: options[:mq_worker_pool_max_backlog],
+            fallback_policy: :abort
+          )
+        end
+
+        # Gracefully stops the worker pool, allowing all threads to finish their jobs
+        #
+        def stop_worker_pool!
+          Mimi::Messaging.log "Stopping worker pool"
+          @worker_pool.shutdown
+          @worker_pool.wait_for_termination
+        end
       end # class Adapter
     end # module SQS_SNS
   end # module Messaging

data/lib/mimi/messaging/sqs_sns/consumer.rb

@@ -7,6 +7,9 @@ module Mimi
       # Message consumer for SQS queues
       #
       class Consumer
+        # (seconds) determines how soon the NACK-ed message becomes visible to other consumers
+        NACK_VISIBILITY_TIMEOUT = 1
+
         def initialize(adapter, queue_url, &block)
           @stop_requested = false
           Mimi::Messaging.log "Starting consumer for: #{queue_url}"
@@ -47,6 +50,9 @@ module Mimi
           Mimi::Messaging.log "Read message from: #{queue_url}"
           block.call(message)
           ack_message(adapter, queue_url, message)
+        rescue Mimi::Messaging::NACK
+          Mimi::Messaging.log "NACK-ing message from: #{queue_url}"
+          nack_message(adapter, queue_url, message)
         rescue StandardError => e
           Mimi::Messaging.logger&.error(
             "#{self.class}: failed to read and process message from: #{queue_url}," \
@@ -67,11 +73,25 @@ module Mimi
           raise Mimi::Messaging::ConnectionError, "Unexpected number of messages read"
         end
 
+        # ACK-ing the message indicates successfull processing of it
+        # and removes the message from the queue
+        #
         def ack_message(adapter, queue_url, msg)
           adapter.sqs_client.delete_message(
             queue_url: queue_url, receipt_handle: msg.receipt_handle
           )
         end
+
+        # NACK-ing the message indicates a failure to process the message.
+        # The message becomes immediately available to other consumers.
+        #
+        def nack_message(adapter, queue_url, msg)
+          adapter.sqs_client.change_message_visibility(
+            queue_url: queue_url,
+            receipt_handle: msg.receipt_handle,
+            visibility_timeout: NACK_VISIBILITY_TIMEOUT
+          )
+        end
       end # class Consumer
     end # module SQS_SNS
   end # module Messaging

data/lib/mimi/messaging/sqs_sns/reply_consumer.rb

@@ -36,7 +36,7 @@ module Mimi
         # @return [Queue] a new Queue object registered for this request_id
         #
         def register_request_id(request_id)
-          queue = Queue.new
+          queue = TimeoutQueue.new
           @mutex.synchronize do
             queue = @queues[request_id] ||= queue
           end

data/lib/mimi/messaging/sqs_sns/timeout_queue.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module Mimi
+  module Messaging
+    module SQS_SNS
+      # TimeoutQueue solves the problem the native Ruby Queue class has with waiting for elements.
+      #
+      # See the excellent blog post discussing the issue:
+      # https://medium.com/workday-engineering/ruby-concurrency-building-a-timeout-queue-5d7c588ca80d
+      #
+      # TLDR -- using Ruby standard Timeout.timeout() around Queue#pop() is unsafe
+      #
+      #
+      class TimeoutQueue
+        def initialize
+          @elems = []
+          @mutex = Mutex.new
+          @cond_var = ConditionVariable.new
+        end
+
+        # Pushes an element into the queue
+        #
+        # @param elem [Object]
+        #
+        def <<(elem)
+          @mutex.synchronize do
+            @elems << elem
+            @cond_var.signal
+          end
+        end
+        alias push <<
+
+        # Pops an element from the queue in either non-blocking
+        # or a blocking (with an optional timeout) way.
+        #
+        # @param blocking [true,false] wait for a new element (true) or return immediately
+        # @param timeout [nil,Integer] if in blocking mode,
+        #                              wait at most given number of seconds or forever (nil)
+        # @raise [Timeout::Error] if a timeout in blocking mode was reached
+        #
+        def pop(blocking = true, timeout = nil)
+          @mutex.synchronize do
+            if blocking
+              if timeout.nil?
+                while @elems.empty?
+                  @cond_var.wait(@mutex)
+                end
+              else
+                timeout_time = Time.now.to_f + timeout
+                while @elems.empty? && (remaining_time = timeout_time - Time.now.to_f) > 0
+                  @cond_var.wait(@mutex, remaining_time)
+                end
+              end
+            end
+            raise Timeout::Error, "queue timeout expired" if @elems.empty?
+
+            @elems.shift
+          end
+        end
+      end # class TimeoutQueue
+    end # module SQS_SNS
+  end # module Messaging
+end # module Mimi

data/lib/mimi/messaging/sqs_sns/version.rb

@@ -3,7 +3,7 @@
 module Mimi
   module Messaging
     module SQS_SNS
-      VERSION = "0.7.0"
+      VERSION = "0.8.0"
     end
   end
 end

data/mimi-messaging-sqs_sns.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "mimi-messaging", "~> 1.2"
   spec.add_dependency "aws-sdk-sqs", "~> 1.22"
   spec.add_dependency "aws-sdk-sns", "~> 1.19"
+  spec.add_dependency "concurrent-ruby"
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "pry", "~> 0.12"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mimi-messaging-sqs_sns
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Alex Kukushkin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-14 00:00:00.000000000 Z
+date: 2021-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mimi-messaging
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -134,6 +148,7 @@ files:
 - bin/console
 - bin/setup
 - examples/event.rb
+- examples/mass-query.rb
 - examples/query.rb
 - examples/responder.rb
 - examples/subscriber.rb
@@ -142,6 +157,7 @@ files:
 - lib/mimi/messaging/sqs_sns/consumer.rb
 - lib/mimi/messaging/sqs_sns/reply_consumer.rb
 - lib/mimi/messaging/sqs_sns/temporary_queue_consumer.rb
+- lib/mimi/messaging/sqs_sns/timeout_queue.rb
 - lib/mimi/messaging/sqs_sns/version.rb
 - mimi-messaging-sqs_sns.gemspec
 homepage: https://github.com/kukushkin/mimi-messaging-sqs_sns