mimi-messaging-sqs_sns 0.4.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b740db8e4dab81e38d1b84ecd40c508a67d4c4c8
-  data.tar.gz: b55b8188a57c5e0b8095ba133b27dad09f17f2a2
+SHA256:
+  metadata.gz: 1f7bfa555dbbe8a3d917d0133811c6f2af643016e5864628a8ae7ae79ec2a9fc
+  data.tar.gz: adc4769159cb678ca05e69c0d5b174b2d9a87064da8fcddf16916596ab14889e
 SHA512:
-  metadata.gz: c215ab899f4217ecd11e628f9739671d569da25cd2abd8ea3ce35572c5076b09941898f18b22f9d97926a9e29a93fb1729064919643ae0119a0a4f62c860b3d1
-  data.tar.gz: 28b7ff63ec0db04041fc173a329bf39a3179d16f50e3d72133f776806fb95c2de5572ec51f8b97681a507645a05021fd01df9b2e1d3f36feae98601f8c8f39ee
+  metadata.gz: 377f9dc7b50c0c3c4edd77afec14e2473fdebbfd4751a18d358c27b9c4281e3af2566f5f70c68fb228a75316d985a2f8f4cab4c74b5d233335e2986875fa88b0
+  data.tar.gz: becd20ef9d41f474ee072a8af75b8b885e40f47b8c34ddca61269cd413fb88368d4715808e4ca8a507b2c56e41b6db0aaf1381c0c9e1715dc4397aa4dcf1888b

data/TODO.md

@@ -0,0 +1,9 @@
+# TODO
+
+List of missing features planned for future releases.
+
+[X] Log error and recover if the reply cannot be sent
+[ ] Threadsafe TimeoutQueue
+[ ] Multithreaded consumers
+[ ] Subscribe to topic without a queue (Temporary queues)
+

data/examples/query.rb

@@ -4,8 +4,8 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 

data/examples/responder.rb

@@ -3,8 +3,8 @@
 require "mimi/messaging/sqs_sns"
 
 AWS_REGION = "eu-west-1"
-AWS_SQS_ENDPOINT_URL = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL = "http://localstack:4566"
 AWS_ACCESS_KEY_ID = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 
@@ -22,7 +22,7 @@ end # class Processor
 
 
 logger = Logger.new(STDOUT)
-logger.level = Logger::INFO
+logger.level = Logger::DEBUG
 Mimi::Messaging.use(logger: logger, serializer: Mimi::Messaging::JsonSerializer)
 Mimi::Messaging.configure(
   mq_adapter: "sqs_sns",

data/lib/mimi/messaging/sqs_sns.rb

@@ -12,4 +12,5 @@ end # module Mimi
 
 require_relative "sqs_sns/adapter"
 require_relative "sqs_sns/consumer"
+require_relative "sqs_sns/temporary_queue_consumer"
 require_relative "sqs_sns/reply_consumer"

data/lib/mimi/messaging/sqs_sns/adapter.rb

@@ -78,6 +78,7 @@ module Mimi
         def start
           @sqs_client = Aws::SQS::Client.new(sqs_client_config)
           @sns_client = Aws::SNS::Client.new(sns_client_config)
+          check_availability!
         end
 
         def stop
@@ -189,7 +190,7 @@ module Mimi
                 response,
                 __request_id: message.headers[:__request_id]
               )
-              deliver_message_queue(reply_to, response_message)
+              deliver_query_respone(reply_to, response_message)
             else
               processor.call_command(method_name, message, {})
             end
@@ -218,6 +219,45 @@ module Mimi
           end
         end
 
+        # Creates a new queue
+        #
+        # @param queue_name [String] name of the topic to be created
+        # @return [String] a new queue URL
+        #
+        def create_queue(queue_name)
+          fqn = sqs_sns_converted_full_name(queue_name)
+          Mimi::Messaging.log "Creating a queue: #{fqn}"
+          result = sqs_client.create_queue(queue_name: fqn)
+          result.queue_url
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError, "Failed to create queue '#{queue_name}': #{e}"
+        end
+
+
+        # Finds a queue URL for a queue with given name.
+        #
+        # If an existing queue with this name is not found,
+        # the method will try to create a new one.
+        #
+        # @param queue_name [String]
+        # @return [String] a queue URL
+        #
+        def find_or_create_queue(queue_name)
+          queue_registry(queue_name) || create_queue(queue_name)
+        end
+
+        # Deletes a queue identified by the queue URL
+        #
+        # @param queue_url [String]
+        #
+        def delete_queue(queue_url)
+          Mimi::Messaging.log "Deleting a queue: #{queue_url}"
+          sqs_client.delete_queue(queue_url: queue_url)
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError,
+            "Failed to delete queue with url '#{queue_url}': #{e}"
+        end
+
         private
 
         # Returns configuration parameters for AWS SQS client
@@ -248,18 +288,21 @@ module Mimi
           params.compact
         end
 
-        # Creates a new queue
+        # Checks SQS and SNS clients availability
         #
-        # @param queue_name [String] name of the topic to be created
-        # @return [String] a new queue URL
+        # @raise [Mimi::Messaging::ConnectionError]
         #
-        def create_queue(queue_name)
-          fqn = sqs_sns_converted_full_name(queue_name)
-          Mimi::Messaging.log "Creating a queue: #{fqn}"
-          result = sqs_client.create_queue(queue_name: fqn)
-          result.queue_url
-        rescue StandardError => e
-          raise Mimi::Messaging::ConnectionError, "Failed to create queue '#{queue_name}': #{e}"
+        def check_availability!
+          begin
+            queue_registry("test")
+          rescue StandardError => e
+            raise Mimi::Messaging::ConnectionError, "SQS connection is not available: #{e}"
+          end
+          begin
+            topic_registry("test")
+          rescue StandardError => e
+            raise Mimi::Messaging::ConnectionError, "SNS connection is not available: #{e}"
+          end
         end
 
         # Delivers a message to a queue with given URL.
@@ -272,7 +315,7 @@ module Mimi
           unless message.is_a?(Mimi::Messaging::Message)
             raise ArgumentError, "Message is expected as argument"
           end
-          Mimi::Messaging.log "Delivering message to: #{queue_url}"
+          Mimi::Messaging.log "Delivering message to: #{queue_url}, headers: #{message.headers}"
           sqs_client.send_message(
             queue_url: queue_url,
             message_body: serialize(message),
@@ -284,6 +327,23 @@ module Mimi
           raise Mimi::Messaging::ConnectionError, "Failed to deliver message to '#{queue_url}': #{e}"
         end
 
+        # Delivers a message as a response to a QUERY
+        #
+        # Responses are allowed to fail. There can be a number of reasons
+        # why responses fail: reply queue does not exist (anymore?),
+        # response message is too big. In any case the error is reported,
+        # but the QUERY message is acknowledged as a successfully processed.
+        #
+        # @param queue_url [String]
+        # @param message [Mimi::Messaging::Message]
+        #
+        def deliver_query_respone(queue_url, message)
+          deliver_message_queue(queue_url, message)
+        rescue Mimi::Messaging::ConnectionError => e
+          Mimi::Messaging.logger&.warn("Failed to deliver QRY response: #{e}")
+          # NOTE: error is recovered
+        end
+
         # Returns URL of a queue with a given name.
         #
         # If the queue with given name does not exist, returns nil
@@ -300,6 +360,8 @@ module Mimi
           end
         rescue Aws::SQS::Errors::NonExistentQueue
           nil
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError, "Failed to get queue url '#{queue_name}': #{e}"
         end
 
         # Converts a topic or queue name to a fully qualified (with namespace)
@@ -329,18 +391,6 @@ module Mimi
           )
         end
 
-        # Finds a queue URL for a queue with given name.
-        #
-        # If an existing queue with this name is not found,
-        # the method will try to create a new one.
-        #
-        # @param queue_name [String]
-        # @return [String] a queue URL
-        #
-        def find_or_create_queue(queue_name)
-          queue_registry(queue_name) || create_queue(queue_name)
-        end
-
         # Returns the configured reply listener for this process
         #
         # @return [ReplyConsumer]
@@ -348,8 +398,7 @@ module Mimi
         def reply_consumer
           @reply_consumer ||= begin
             reply_queue_name = options[:mq_reply_queue_prefix] + SecureRandom.hex(8)
-            reply_queue_url = create_queue(reply_queue_name)
-            Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_url)
+            Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_name)
           end
         end
 

data/lib/mimi/messaging/sqs_sns/consumer.rb

@@ -4,20 +4,15 @@ module Mimi
   module Messaging
     module SQS_SNS
       #
-      # Message processor for SQS queues
+      # Message consumer for SQS queues
       #
       class Consumer
         def initialize(adapter, queue_url, &block)
-          @block = block
           @stop_requested = false
           Mimi::Messaging.log "Starting consumer for: #{queue_url}"
           @consumer_thread = Thread.new do
             while not @stop_requested do
-              message = read_message(adapter, queue_url)
-              next unless message
-              Mimi::Messaging.log "Read message from: #{queue_url}"
-              block.call(message)
-              ack_message(adapter, queue_url, message)
+              read_and_process_message(adapter, queue_url, block)
             end
             Mimi::Messaging.log "Stopping consumer for: #{queue_url}"
           end
@@ -30,6 +25,26 @@ module Mimi
 
         private
 
+        # A method invoked in a loop to read/wait for a message
+        # from the associated queue and process it
+        #
+        # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
+        # @param queue_url [String]
+        # @param block [Proc] a block to be invoked when a message is received
+        #
+        def read_and_process_message(adapter, queue_url, block)
+          message = read_message(adapter, queue_url)
+          return unless message
+          Mimi::Messaging.log "Read message from: #{queue_url}"
+          block.call(message)
+          ack_message(adapter, queue_url, message)
+        rescue StandardError => e
+          Mimi::Messaging.logger&.error(
+            "#{self.class}: failed to read and process message from: #{queue_url}," \
+            " error: (#{e.class}) #{e}"
+          )
+        end
+
         def read_message(adapter, queue_url)
           result = adapter.sqs_client.receive_message(
             queue_url: queue_url,

data/lib/mimi/messaging/sqs_sns/reply_consumer.rb

@@ -8,25 +8,23 @@ module Mimi
       # and passes them to registered Queues (see Ruby ::Queue class).
       #
       class ReplyConsumer
-        attr_reader :reply_queue_url
+        attr_reader :reply_queue_name, :reply_queue_url
 
-        def initialize(adapter, reply_queue_url)
+        def initialize(adapter, reply_queue_name)
           @mutex = Mutex.new
           @queues = {}
           @adapter = adapter
-          @reply_queue_url = reply_queue_url
-          @consumer = Consumer.new(adapter, reply_queue_url) do |message|
+          @reply_queue_name = reply_queue_name
+          @consumer = TemporaryQueueConsumer.new(adapter, reply_queue_name) do |message|
             dispatch_message(message)
           end
+          @reply_queue_url = @consumer.queue_url
         end
 
         def stop
-          begin
-            @consumer.stop
-          rescue StandardError => e
-            raise Mimi::Messaging::Error, "Failed to stop consumer: #{e}"
-          end
-          # TODO: adapter.sqs_client.delete_queue(reply_queue_url)
+          @consumer.stop
+        rescue StandardError => e
+          raise Mimi::Messaging::Error, "Failed to stop reply consumer: #{e}"
         end
 
         # Register a new request_id to listen for.
@@ -65,6 +63,7 @@ module Mimi
           @mutex.synchronize do
             headers = deserialize_headers(message)
             request_id = headers[:__request_id]
+            Mimi::Messaging.log "dispatching response, headers:#{headers}"
             queue = @queues.delete(request_id)
           end
           queue&.push(message)

data/lib/mimi/messaging/sqs_sns/temporary_queue_consumer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Mimi
+  module Messaging
+    module SQS_SNS
+      #
+      # Temporary queue consumer creates a temporary queue
+      # and attaches to it. The queue will be deleted
+      # on consumer shutdown.
+      #
+      class TemporaryQueueConsumer
+        attr_reader :queue_url
+
+        def initialize(adapter, queue_name, &block)
+          @adapter = adapter
+          @queue_url = adapter.find_or_create_queue(queue_name)
+          @consumer = Consumer.new(adapter, @queue_url, &block)
+        end
+
+        def stop
+          @consumer.stop
+          @adapter.delete_queue(queue_url)
+        rescue StandardError => e
+          raise Mimi::Messaging::Error, "Failed to stop temporary queue consumer: #{e}"
+        end
+      end # class TemporaryQueueConsumer
+    end # module SQS_SNS
+  end # module Messaging
+end # module Mimi

data/lib/mimi/messaging/sqs_sns/version.rb

@@ -3,7 +3,7 @@
 module Mimi
   module Messaging
     module SQS_SNS
-      VERSION = "0.4.0"
+      VERSION = "0.6.1"
     end
   end
 end

data/mimi-messaging-sqs_sns.gemspec

@@ -39,6 +39,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "pry", "~> 0.12"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mimi-messaging-sqs_sns
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Alex Kukushkin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-13 00:00:00.000000000 Z
+date: 2020-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mimi-messaging
@@ -86,14 +86,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -123,6 +129,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- TODO.md
 - bin/console
 - bin/setup
 - examples/event.rb
@@ -133,6 +140,7 @@ files:
 - lib/mimi/messaging/sqs_sns/adapter.rb
 - lib/mimi/messaging/sqs_sns/consumer.rb
 - lib/mimi/messaging/sqs_sns/reply_consumer.rb
+- lib/mimi/messaging/sqs_sns/temporary_queue_consumer.rb
 - lib/mimi/messaging/sqs_sns/version.rb
 - mimi-messaging-sqs_sns.gemspec
 homepage: https://github.com/kukushkin/mimi-messaging-sqs_sns
@@ -157,8 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: AWS SQS/SNS adapter for mimi-messaging