mimi-messaging-sqs_sns 0.4.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f0919e38a4926e465f4b2017b98ce6b621a41f5b
-  data.tar.gz: f356e4552744757c4e6c77eb3eca359839d7d834
+SHA256:
+  metadata.gz: b02992abcb101248ebc54069aba5b3887417930b7613a84ed4b6723ad7ce3060
+  data.tar.gz: 3b830249dfb56e42a7d380571402c2abef55d74f3e90bf0b9f537e105078cd27
 SHA512:
-  metadata.gz: b45642dc502c2bf8e5debf230460594abef63fde87f4f029d5583fdb942965465581be65e0eb050781d2780453f1843644a9c0539849ef1e562062b3370cd47b
-  data.tar.gz: b5251902e1ac67cad8731f05fdd5a0fe1a7e32db70f4aa62a9f7a29bdb0a2de0c6ff56e82c0fa5b01fabb9d7438fb0411aa62390daf1afe0ceed1ce72c223562
+  metadata.gz: 4d6ecfe3d2202124b10ff03129166249f518fca0f15c2e513ba5ab6b2835a713671618eab623fb39e3cbebe3c8351e0688884743e4f69baa3832b1db94a6a858
+  data.tar.gz: 5bb7ae92080d112f16a4127b010f7f102ebd71cb53b4701b756e9bbab8c43fae42618969c148fd71f67678dbf34ed4d6c734307a1c84f7201229d5bd95c3907a

data/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+## v0.7.0
+
+* [#1](https://github.com/kukushkin/mimi-messaging-sqs_sns/pull/1)
+  * Added KMS support for creating queues/topics with sever-side encryption enabled
+  * Optimized stopping of the adapter: stopping all consumers in parallel
+
+
+## v0.6.x
+
+* Basic functionality implemented, see missing features in [TODO](TODO.md)

data/README.md

@@ -2,6 +2,9 @@
 
 AWS SQS/SNS adapter for [mimi-messaging](https://github.com/kukushkin/mimi-messaging).
 
+* [Changes](CHANGELOG.md)
+
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -43,7 +46,6 @@ Mimi::Messaging.configure(
 Mimi::Messaging.start
 ```
 
-
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/kukushkin/mimi-messaging-sqs_sns. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.

data/TODO.md

@@ -0,0 +1,9 @@
+# TODO
+
+List of missing features planned for future releases.
+
+[X] Log error and recover if the reply cannot be sent
+[ ] Threadsafe TimeoutQueue
+[ ] Multithreaded consumers
+[ ] Subscribe to topic without a queue (Temporary queues)
+

data/examples/query.rb

@@ -4,10 +4,11 @@ require "mimi/messaging/sqs_sns"
 
 COUNT = 10
 AWS_REGION            = "eu-west-1"
-AWS_SQS_ENDPOINT_URL  = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL  = "http://localstack:4575"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
+AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"
 
 logger = Logger.new(STDOUT)
 logger.level = Logger::INFO
@@ -18,7 +19,8 @@ Mimi::Messaging.configure(
   mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
-  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID
 )
 adapter = Mimi::Messaging.adapter
 

data/examples/responder.rb

@@ -2,11 +2,12 @@
 
 require "mimi/messaging/sqs_sns"
 
-AWS_REGION = "eu-west-1"
-AWS_SQS_ENDPOINT_URL = "http://localstack:4576"
-AWS_SNS_ENDPOINT_URL = "http://localstack:4575"
-AWS_ACCESS_KEY_ID = "foo"
+AWS_REGION            = "eu-west-1"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4566"
+AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
+AWS_SQS_SNS_KMS_MASTER_KEY_ID = "blah"
 
 class Processor
   def self.call_command(method_name, message, opts)
@@ -22,7 +23,7 @@ end # class Processor
 
 
 logger = Logger.new(STDOUT)
-logger.level = Logger::INFO
+logger.level = Logger::DEBUG
 Mimi::Messaging.use(logger: logger, serializer: Mimi::Messaging::JsonSerializer)
 Mimi::Messaging.configure(
   mq_adapter: "sqs_sns",
@@ -30,7 +31,8 @@ Mimi::Messaging.configure(
   mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
   mq_aws_region:            AWS_REGION,
   mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
-  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_aws_sqs_sns_kms_master_key_id: AWS_SQS_SNS_KMS_MASTER_KEY_ID
 )
 adapter = Mimi::Messaging.adapter
 queue_name = "test"
@@ -47,4 +49,3 @@ ensure
   puts "Stopping adapter"
   adapter.stop
 end
-

data/lib/mimi/messaging/sqs_sns.rb

@@ -12,4 +12,5 @@ end # module Mimi
 
 require_relative "sqs_sns/adapter"
 require_relative "sqs_sns/consumer"
+require_relative "sqs_sns/temporary_queue_consumer"
 require_relative "sqs_sns/reply_consumer"

data/lib/mimi/messaging/sqs_sns/adapter.rb

@@ -56,7 +56,8 @@ module Mimi
           mq_aws_sqs_endpoint: nil,
           mq_aws_sns_endpoint: nil,
 
-          mq_aws_sqs_read_timeout: 10, # seconds
+          mq_aws_sqs_sns_kms_master_key_id: nil,
+          mq_aws_sqs_read_timeout: 20, # seconds
         }.freeze
 
         # Initializes SQS/SNS adapter
@@ -93,6 +94,7 @@ module Mimi
         # for processors.
         #
         def stop_all_processors
+          @consumers&.each(&:signal_stop)
           @consumers&.each(&:stop)
           @consumers = nil
           @reply_consumer&.stop
@@ -190,7 +192,7 @@ module Mimi
                 response,
                 __request_id: message.headers[:__request_id]
               )
-              deliver_message_queue(reply_to, response_message)
+              deliver_query_respone(reply_to, response_message)
             else
               processor.call_command(method_name, message, {})
             end
@@ -219,6 +221,48 @@ module Mimi
           end
         end
 
+        # Creates a new queue
+        #
+        # @param queue_name [String] name of the topic to be created
+        # @return [String] a new queue URL
+        #
+        def create_queue(queue_name)
+          fqn = sqs_sns_converted_full_name(queue_name)
+          Mimi::Messaging.log "Creating a queue: #{fqn}"
+          attrs = {}
+          if options[:mq_aws_sqs_sns_kms_master_key_id]
+            attrs["KmsMasterKeyId"] = options[:mq_aws_sqs_sns_kms_master_key_id]
+          end
+          result = sqs_client.create_queue(queue_name: fqn, attributes: attrs)
+          result.queue_url
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError, "Failed to create queue '#{queue_name}': #{e}"
+        end
+
+        # Finds a queue URL for a queue with given name.
+        #
+        # If an existing queue with this name is not found,
+        # the method will try to create a new one.
+        #
+        # @param queue_name [String]
+        # @return [String] a queue URL
+        #
+        def find_or_create_queue(queue_name)
+          queue_registry(queue_name) || create_queue(queue_name)
+        end
+
+        # Deletes a queue identified by the queue URL
+        #
+        # @param queue_url [String]
+        #
+        def delete_queue(queue_url)
+          Mimi::Messaging.log "Deleting a queue: #{queue_url}"
+          sqs_client.delete_queue(queue_url: queue_url)
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError,
+            "Failed to delete queue with url '#{queue_url}': #{e}"
+        end
+
         private
 
         # Returns configuration parameters for AWS SQS client
@@ -266,20 +310,6 @@ module Mimi
           end
         end
 
-        # Creates a new queue
-        #
-        # @param queue_name [String] name of the topic to be created
-        # @return [String] a new queue URL
-        #
-        def create_queue(queue_name)
-          fqn = sqs_sns_converted_full_name(queue_name)
-          Mimi::Messaging.log "Creating a queue: #{fqn}"
-          result = sqs_client.create_queue(queue_name: fqn)
-          result.queue_url
-        rescue StandardError => e
-          raise Mimi::Messaging::ConnectionError, "Failed to create queue '#{queue_name}': #{e}"
-        end
-
         # Delivers a message to a queue with given URL.
         #
         # @param queue_url [String]
@@ -290,7 +320,8 @@ module Mimi
           unless message.is_a?(Mimi::Messaging::Message)
             raise ArgumentError, "Message is expected as argument"
           end
-          Mimi::Messaging.log "Delivering message to: #{queue_url}"
+
+          Mimi::Messaging.log "Delivering message to: #{queue_url}, headers: #{message.headers}"
           sqs_client.send_message(
             queue_url: queue_url,
             message_body: serialize(message),
@@ -302,6 +333,23 @@ module Mimi
           raise Mimi::Messaging::ConnectionError, "Failed to deliver message to '#{queue_url}': #{e}"
         end
 
+        # Delivers a message as a response to a QUERY
+        #
+        # Responses are allowed to fail. There can be a number of reasons
+        # why responses fail: reply queue does not exist (anymore?),
+        # response message is too big. In any case the error is reported,
+        # but the QUERY message is acknowledged as a successfully processed.
+        #
+        # @param queue_url [String]
+        # @param message [Mimi::Messaging::Message]
+        #
+        def deliver_query_respone(queue_url, message)
+          deliver_message_queue(queue_url, message)
+        rescue Mimi::Messaging::ConnectionError => e
+          Mimi::Messaging.logger&.warn("Failed to deliver QRY response: #{e}")
+          # NOTE: error is recovered
+        end
+
         # Returns URL of a queue with a given name.
         #
         # If the queue with given name does not exist, returns nil
@@ -349,18 +397,6 @@ module Mimi
           )
         end
 
-        # Finds a queue URL for a queue with given name.
-        #
-        # If an existing queue with this name is not found,
-        # the method will try to create a new one.
-        #
-        # @param queue_name [String]
-        # @return [String] a queue URL
-        #
-        def find_or_create_queue(queue_name)
-          queue_registry(queue_name) || create_queue(queue_name)
-        end
-
         # Returns the configured reply listener for this process
         #
         # @return [ReplyConsumer]
@@ -368,8 +404,7 @@ module Mimi
         def reply_consumer
           @reply_consumer ||= begin
             reply_queue_name = options[:mq_reply_queue_prefix] + SecureRandom.hex(8)
-            reply_queue_url = create_queue(reply_queue_name)
-            Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_url)
+            Mimi::Messaging::SQS_SNS::ReplyConsumer.new(self, reply_queue_name)
           end
         end
 
@@ -450,7 +485,11 @@ module Mimi
         def create_topic(topic_name)
           fqn = sqs_sns_converted_full_name(topic_name)
           Mimi::Messaging.log "Creating a topic: #{fqn}"
-          result = sns_client.create_topic(name: fqn)
+          attrs = {}
+          if options[:mq_aws_sqs_sns_kms_master_key_id]
+            attrs["KmsMasterKeyId"] = options[:mq_aws_sqs_sns_kms_master_key_id]
+          end
+          result = sns_client.create_topic(name: fqn, attributes: attrs)
           result.topic_arn
         rescue StandardError => e
           raise Mimi::Messaging::ConnectionError, "Failed to create topic '#{topic_name}': #{e}"
@@ -467,7 +506,7 @@ module Mimi
           )
           queue_arn = result.attributes["QueueArn"]
           Mimi::Messaging.log "Subscribing queue to a topic: '#{topic_arn}'->'#{queue_url}'"
-          result = sns_client.subscribe(
+          _result = sns_client.subscribe(
             topic_arn: topic_arn,
             protocol: "sqs",
             endpoint: queue_arn,
@@ -489,6 +528,7 @@ module Mimi
           unless message.is_a?(Mimi::Messaging::Message)
             raise ArgumentError, "Message is expected as argument"
           end
+
           Mimi::Messaging.log "Delivering message to: #{topic_arn}"
           sns_client.publish(
             topic_arn: topic_arn,

data/lib/mimi/messaging/sqs_sns/consumer.rb

@@ -4,25 +4,28 @@ module Mimi
   module Messaging
     module SQS_SNS
       #
-      # Message processor for SQS queues
+      # Message consumer for SQS queues
       #
       class Consumer
         def initialize(adapter, queue_url, &block)
-          @block = block
           @stop_requested = false
           Mimi::Messaging.log "Starting consumer for: #{queue_url}"
           @consumer_thread = Thread.new do
-            while not @stop_requested do
-              message = read_message(adapter, queue_url)
-              next unless message
-              Mimi::Messaging.log "Read message from: #{queue_url}"
-              block.call(message)
-              ack_message(adapter, queue_url, message)
+            while not @stop_requested
+              read_and_process_message(adapter, queue_url, block)
             end
             Mimi::Messaging.log "Stopping consumer for: #{queue_url}"
           end
         end
 
+        # Requests the Consumer to stop, without actually waiting for it
+        #
+        def signal_stop
+          @stop_requested = true
+        end
+
+        # Requests the Consumer to stop AND waits until it does
+        #
         def stop
           @stop_requested = true
           @consumer_thread.join
@@ -30,6 +33,27 @@ module Mimi
 
         private
 
+        # A method invoked in a loop to read/wait for a message
+        # from the associated queue and process it
+        #
+        # @param adapter [Mimi::Messaging::SQS_SNS::Adapter]
+        # @param queue_url [String]
+        # @param block [Proc] a block to be invoked when a message is received
+        #
+        def read_and_process_message(adapter, queue_url, block)
+          message = read_message(adapter, queue_url)
+          return unless message
+
+          Mimi::Messaging.log "Read message from: #{queue_url}"
+          block.call(message)
+          ack_message(adapter, queue_url, message)
+        rescue StandardError => e
+          Mimi::Messaging.logger&.error(
+            "#{self.class}: failed to read and process message from: #{queue_url}," \
+            " error: (#{e.class}) #{e}"
+          )
+        end
+
         def read_message(adapter, queue_url)
           result = adapter.sqs_client.receive_message(
             queue_url: queue_url,
@@ -39,6 +63,7 @@ module Mimi
           )
           return nil if result.messages.count == 0
           return result.messages.first if result.messages.count == 1
+
           raise Mimi::Messaging::ConnectionError, "Unexpected number of messages read"
         end
 

data/lib/mimi/messaging/sqs_sns/reply_consumer.rb

@@ -8,25 +8,23 @@ module Mimi
       # and passes them to registered Queues (see Ruby ::Queue class).
       #
       class ReplyConsumer
-        attr_reader :reply_queue_url
+        attr_reader :reply_queue_name, :reply_queue_url
 
-        def initialize(adapter, reply_queue_url)
+        def initialize(adapter, reply_queue_name)
           @mutex = Mutex.new
           @queues = {}
           @adapter = adapter
-          @reply_queue_url = reply_queue_url
-          @consumer = Consumer.new(adapter, reply_queue_url) do |message|
+          @reply_queue_name = reply_queue_name
+          @consumer = TemporaryQueueConsumer.new(adapter, reply_queue_name) do |message|
             dispatch_message(message)
           end
+          @reply_queue_url = @consumer.queue_url
         end
 
         def stop
-          begin
-            @consumer.stop
-          rescue StandardError => e
-            raise Mimi::Messaging::Error, "Failed to stop consumer: #{e}"
-          end
-          # TODO: adapter.sqs_client.delete_queue(reply_queue_url)
+          @consumer.stop
+        rescue StandardError => e
+          raise Mimi::Messaging::Error, "Failed to stop reply consumer: #{e}"
         end
 
         # Register a new request_id to listen for.
@@ -65,6 +63,7 @@ module Mimi
           @mutex.synchronize do
             headers = deserialize_headers(message)
             request_id = headers[:__request_id]
+            Mimi::Messaging.log "dispatching response, headers:#{headers}"
             queue = @queues.delete(request_id)
           end
           queue&.push(message)

data/lib/mimi/messaging/sqs_sns/temporary_queue_consumer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Mimi
+  module Messaging
+    module SQS_SNS
+      #
+      # Temporary queue consumer creates a temporary queue
+      # and attaches to it. The queue will be deleted
+      # on consumer shutdown.
+      #
+      class TemporaryQueueConsumer
+        attr_reader :queue_url
+
+        def initialize(adapter, queue_name, &block)
+          @adapter = adapter
+          @queue_url = adapter.find_or_create_queue(queue_name)
+          @consumer = Consumer.new(adapter, @queue_url, &block)
+        end
+
+        def stop
+          @consumer.stop
+          @adapter.delete_queue(queue_url)
+        rescue StandardError => e
+          raise Mimi::Messaging::Error, "Failed to stop temporary queue consumer: #{e}"
+        end
+      end # class TemporaryQueueConsumer
+    end # module SQS_SNS
+  end # module Messaging
+end # module Mimi

data/lib/mimi/messaging/sqs_sns/version.rb

@@ -3,7 +3,7 @@
 module Mimi
   module Messaging
     module SQS_SNS
-      VERSION = "0.4.1"
+      VERSION = "0.7.0"
     end
   end
 end

data/mimi-messaging-sqs_sns.gemspec

@@ -39,6 +39,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "pry", "~> 0.12"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mimi-messaging-sqs_sns
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Alex Kukushkin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-14 00:00:00.000000000 Z
+date: 2021-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mimi-messaging
@@ -86,14 +86,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -118,11 +124,13 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- TODO.md
 - bin/console
 - bin/setup
 - examples/event.rb
@@ -133,6 +141,7 @@ files:
 - lib/mimi/messaging/sqs_sns/adapter.rb
 - lib/mimi/messaging/sqs_sns/consumer.rb
 - lib/mimi/messaging/sqs_sns/reply_consumer.rb
+- lib/mimi/messaging/sqs_sns/temporary_queue_consumer.rb
 - lib/mimi/messaging/sqs_sns/version.rb
 - mimi-messaging-sqs_sns.gemspec
 homepage: https://github.com/kukushkin/mimi-messaging-sqs_sns
@@ -157,8 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: AWS SQS/SNS adapter for mimi-messaging