mimi-messaging-sqs_sns 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e106fbe54e012ffd59a6126292242b23c28b3086ff5dfd4866acb64bad559ccf
-  data.tar.gz: 2fce6feb750d6cf27925e79646b8e76359188343f663ee8d20f30f09eedc2ba0
+  metadata.gz: 9ebbd0b876cc696936356711703245e45106ff85ad152bde59fd2826b6cbc21f
+  data.tar.gz: ac2167ece90f7de977fee85c66137e5cb9f232e139fcb8ab797c27467cffa540
 SHA512:
-  metadata.gz: a830a70afd7b0a5f268d600ca9673a8f3fc36566201510263008f517d32354a020213de0b51497c446e99a768089f2fd9a1392dccdf5bbbc5fc352d3557a2a4e
-  data.tar.gz: 41dc62d85b21f8736011d974b2749e5f6ce093d3f3cc3746f037f8a986ef01621bc2a61ae60f8f96a5405dcc6672bd1468fec0e3ea695e0b1f1e4a123d71c962
+  metadata.gz: 0cdc8a69e02de3533b0871fb9727fbe63e47081fe6a0ac66bf093cacf2f99d16f697034987adc507078bcf14d4864dd7ff78a2025e49305b4ae48c32313b1992
+  data.tar.gz: 56665e8adb58d3d17c1fcb7e3b5ef5eea8231d24a130de1bfd589acb90169473bd6088a5a227d3eb6d3141ce19af7939d2a874132607b45579b4a02d4b4c3103

data/examples/event.rb

@@ -4,8 +4,8 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 

data/examples/mass-query.rb

@@ -6,8 +6,8 @@ COUNT = 10
 THREADS = 10
 QUERY_TIMEOUT = 60
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"

data/examples/query.rb

@@ -4,8 +4,8 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"

data/examples/responder.rb

@@ -3,8 +3,8 @@
 require "mimi/messaging/sqs_sns"
 
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"

data/examples/subscriber.rb

@@ -3,8 +3,8 @@
 require "mimi/messaging/sqs_sns"
 
 AWS_REGION = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SQS_ENDPOINT_URL  = "http://sqs-sns:4566"
+AWS_SNS_ENDPOINT_URL  = "http://sqs-sns:4566"
 AWS_ACCESS_KEY_ID = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 

data/lib/mimi/messaging/sqs_sns/adapter.rb

@@ -185,7 +185,7 @@ module Mimi
           @consumers ||= []
           opts = opts.dup
           queue_url = find_or_create_queue(queue_name)
-          @consumers << Consumer.new(self, queue_url) do |m|
+          @consumers << Consumer.new(self, queue_url, worker_pool) do |m|
             process_request_message(processor, m)
           end
         end
@@ -202,7 +202,7 @@ module Mimi
           topic_arn = find_or_create_topic(topic_name) # TODO: or find_topic!(...) ?
           queue_url = find_or_create_queue(queue_name)
           subscribe_topic_queue(topic_arn, queue_url)
-          @consumers << Consumer.new(self, queue_url) do |m|
+          @consumers << Consumer.new(self, queue_url, worker_pool) do |m|
             process_event_message(processor, m)
           end
         end

data/lib/mimi/messaging/sqs_sns/consumer.rb

@@ -10,12 +10,20 @@ module Mimi
         # (seconds) determines how soon the NACK-ed message becomes visible to other consumers
         NACK_VISIBILITY_TIMEOUT = 1
 
-        def initialize(adapter, queue_url, &block)
+        # Creates a new Consumer to read and process messages from the given queue.
+        #
+        # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
+        # @param queue_url [String]
+        # @param worker_pool [Concurrent::ThreadPoolExecutor, nil] an optional worker pool to be used
+        #
+        def initialize(adapter, queue_url, worker_pool = nil, &block)
           @stop_requested = false
-          Mimi::Messaging.log "Starting consumer for: #{queue_url}"
+          @worker_pool = worker_pool
+          Mimi::Messaging.log "Starting consumer for: #{queue_url}, " \
+            " worker_pool: #{worker_pool ? "yes" : "no"}"
           @consumer_thread = Thread.new do
             while not @stop_requested
-              read_and_process_message(adapter, queue_url, block)
+              read_and_process_message(adapter, queue_url, worker_pool, block)
             end
             Mimi::Messaging.log "Stopping consumer for: #{queue_url}"
           end
@@ -41,30 +49,33 @@ module Mimi
         #
         # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
         # @param queue_url [String]
+        # @param worker_pool [Concurrent::ThreadPoolExecutor,nil]
         # @param block [Proc] a block to be invoked when a message is received
         #
-        def read_and_process_message(adapter, queue_url, block)
+        def read_and_process_message(adapter, queue_url, worker_pool, block)
           message = read_message(adapter, queue_url)
           return unless message
 
           Mimi::Messaging.log "Read message from: #{queue_url}"
-          begin
-            adapter.worker_pool.post do
-              process_message(adapter, queue_url, message, block)
-            end
-          rescue Concurrent::RejectedExecutionError
-            # the backlog is overflown, put the message back
-            Mimi::Messaging.log "Worker pool backlog is full, nack-ing the message " \
-              "(workers:#{adapter.worker_pool.length}, backlog:#{adapter.worker_pool.queue_length})"
-            nack_message(adapter, queue_url, message)
+          if worker_pool
+            process_message_worker_pool(adapter, queue_url, worker_pool, message, block)
+          else
+            process_message(adapter, queue_url, message, block)
           end
         rescue StandardError => e
           Mimi::Messaging.logger&.error(
             "#{self.class}: failed to read or process message from: #{queue_url}," \
+            " worker_pool: #{worker_pool ? "yes" : "no"}" \
             " error: (#{e.class}) #{e}"
           )
         end
 
+        # Returns a message read from SQS queue
+        #
+        # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
+        # @param queue_url [String]
+        # @return [Hash] with stringified keys
+        #
         def read_message(adapter, queue_url)
           result = adapter.sqs_client.receive_message(
             queue_url: queue_url,
@@ -78,6 +89,26 @@ module Mimi
           raise Mimi::Messaging::ConnectionError, "Unexpected number of messages read"
         end
 
+        # Processes a received message using the worker pool
+        #
+        # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
+        # @param queue_url [String]
+        # @param worker_pool [Concurrent::ThreadPoolExecutor,nil]
+        # @param block [Proc] a block to be invoked when a message is received
+        #
+        def process_message_worker_pool(adapter, queue_url, worker_pool, message, block)
+          begin
+            worker_pool.post do
+              process_message(adapter, queue_url, message, block)
+            end
+          rescue Concurrent::RejectedExecutionError
+            # the backlog is overflown, put the message back
+            Mimi::Messaging.log "Worker pool backlog is full, nack-ing the message " \
+              "(workers:#{worker_pool.length}, backlog:#{worker_pool.queue_length})"
+            nack_message(adapter, queue_url, message)
+          end
+        end
+
         def process_message(adapter, queue_url, message, block)
           block.call(message)
           ack_message(adapter, queue_url, message)

data/lib/mimi/messaging/sqs_sns/version.rb

@@ -3,7 +3,7 @@
 module Mimi
   module Messaging
     module SQS_SNS
-      VERSION = "0.8.2"
+      VERSION = "0.8.3"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mimi-messaging-sqs_sns
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Alex Kukushkin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-03-15 00:00:00.000000000 Z
+date: 2022-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mimi-messaging