mimi-messaging-sqs_sns 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a8b8bacfe2cc6db227168999349d0aa7b3e917e
-  data.tar.gz: e83e778b72017c79d108c919f0495945ae8be41f
+  metadata.gz: b740db8e4dab81e38d1b84ecd40c508a67d4c4c8
+  data.tar.gz: b55b8188a57c5e0b8095ba133b27dad09f17f2a2
 SHA512:
-  metadata.gz: 546272f3b86be0a2218f89051aa090eb0cf5d36addca02abb0f521f5fa4a90ec6dec5d72e5e56a3bb741ac6d54afb6d586d4836050013dad0c1d7781bfcb6633
-  data.tar.gz: cb3d2f9547dd421f21ca9c2f8def098d1c81ba225e68b156ff4f2eeb44b1875f85d2347ff480d399fe0bc2e639db644d2ec72160edc8c9c4db8adfd9f31d8a0d
+  metadata.gz: c215ab899f4217ecd11e628f9739671d569da25cd2abd8ea3ce35572c5076b09941898f18b22f9d97926a9e29a93fb1729064919643ae0119a0a4f62c860b3d1
+  data.tar.gz: 28b7ff63ec0db04041fc173a329bf39a3179d16f50e3d72133f776806fb95c2de5572ec51f8b97681a507645a05021fd01df9b2e1d3f36feae98601f8c8f39ee

data/README.md

@@ -32,6 +32,7 @@ Mimi::Messaging.configure(
   mq_aws_access_key_id: nil,
   mq_aws_secret_access_key: nil,
   mq_aws_sqs_endpoint: nil,
+  mq_aws_sns_endpoint: nil,
 
   mq_aws_sqs_read_timeout: 10, # seconds
   mq_namespace: nil,

data/examples/event.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "mimi/messaging/sqs_sns"
+
+COUNT = 10
+AWS_REGION            = "eu-west-1"
+AWS_SQS_ENDPOINT_URL  = "http://localstack:4576"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4575"
+AWS_ACCESS_KEY_ID     = "foo"
+AWS_SECRET_ACCESS_KEY = "bar"
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+Mimi::Messaging.use(logger: logger, serializer: Mimi::Messaging::JsonSerializer)
+Mimi::Messaging.configure(
+  mq_adapter: "sqs_sns",
+  mq_aws_access_key_id:     AWS_ACCESS_KEY_ID,
+  mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
+  mq_aws_region:            AWS_REGION,
+  mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_log_at_level: :info
+)
+adapter = Mimi::Messaging.adapter
+
+adapter.start
+
+t_start = Time.now
+COUNT.times do |i|
+  t = Time.now
+  puts "Publishing event: #{i}"
+  adapter.event("hello#tested", i: i) # rand(100))
+  sleep 1
+end

data/examples/query.rb

@@ -5,6 +5,7 @@ require "mimi/messaging/sqs_sns"
 COUNT = 10
 AWS_REGION            = "eu-west-1"
 AWS_SQS_ENDPOINT_URL  = "http://localstack:4576"
+AWS_SNS_ENDPOINT_URL  = "http://localstack:4575"
 AWS_ACCESS_KEY_ID     = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 
@@ -16,7 +17,8 @@ Mimi::Messaging.configure(
   mq_aws_access_key_id:     AWS_ACCESS_KEY_ID,
   mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
   mq_aws_region:            AWS_REGION,
-  mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL
+  mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL
 )
 adapter = Mimi::Messaging.adapter
 

data/examples/responder.rb

@@ -4,6 +4,7 @@ require "mimi/messaging/sqs_sns"
 
 AWS_REGION = "eu-west-1"
 AWS_SQS_ENDPOINT_URL = "http://localstack:4576"
+AWS_SNS_ENDPOINT_URL = "http://localstack:4575"
 AWS_ACCESS_KEY_ID = "foo"
 AWS_SECRET_ACCESS_KEY = "bar"
 
@@ -28,7 +29,8 @@ Mimi::Messaging.configure(
   mq_aws_access_key_id:     AWS_ACCESS_KEY_ID,
   mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
   mq_aws_region:            AWS_REGION,
-  mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL
+  mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL
 )
 adapter = Mimi::Messaging.adapter
 queue_name = "test"

data/examples/subscriber.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "mimi/messaging/sqs_sns"
+
+AWS_REGION = "eu-west-1"
+AWS_SQS_ENDPOINT_URL = "http://localstack:4576"
+AWS_SNS_ENDPOINT_URL = "http://localstack:4575"
+AWS_ACCESS_KEY_ID = "foo"
+AWS_SECRET_ACCESS_KEY = "bar"
+
+class Processor
+  def self.call_command(method_name, message, opts)
+    puts "COMMAND: #{method_name}, #{message}, headers: #{message.headers}"
+  end
+
+  def self.call_query(method_name, message, opts)
+    puts "QUERY: #{method_name}, #{message}, headers: #{message.headers}"
+    {}
+  end
+
+  def self.call_event(event_type, message, opts)
+    puts "EVENT: #{event_type}, #{message}, headers: #{message.headers}"
+  end
+end # class Processor
+
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+Mimi::Messaging.use(logger: logger, serializer: Mimi::Messaging::JsonSerializer)
+Mimi::Messaging.configure(
+  mq_adapter: "sqs_sns",
+  mq_aws_access_key_id:     AWS_ACCESS_KEY_ID,
+  mq_aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
+  mq_aws_region:            AWS_REGION,
+  mq_aws_sqs_endpoint:      AWS_SQS_ENDPOINT_URL,
+  mq_aws_sns_endpoint:      AWS_SNS_ENDPOINT_URL,
+  mq_log_at_level: :info
+)
+adapter = Mimi::Messaging.adapter
+
+topic_name = "hello"
+queue_name = "listener.hello"
+adapter.start
+puts "Registering event processor on '#{topic_name}'->'#{queue_name}'"
+adapter.start_event_processor_with_queue(topic_name, queue_name, Processor)
+
+begin
+  loop do
+    sleep 1
+  end
+ensure
+  puts "Stopping adapter"
+  adapter.stop
+end
+

data/lib/mimi/messaging/sqs_sns/adapter.rb

@@ -2,6 +2,7 @@
 
 require "mimi/messaging"
 require "aws-sdk-sqs"
+require "aws-sdk-sns"
 require "timeout"
 require "securerandom"
 
@@ -23,20 +24,37 @@ module Mimi
       # * #stop_all_processors
       #
       class Adapter < Mimi::Messaging::Adapters::Base
-        attr_reader :options, :sqs_client
+        #
+        # NOTE: AWS SQS/SNS alphabet for queue and topic names
+        # is different from what mimi-messaging allows:
+        # '.' is not an allowed character.
+        #
+        # SQS_SNS_ALPHABET_MAP structure is used to convert
+        # names from mimi-messaging alphabet to SQS/SNS alphabet.
+        #
+        # Mimi::Messaging still accepts queue and topic names
+        # containing the '.', but the adapter will convert those
+        # to valid SQS/SNS names using this mapping.
+        #
+        SQS_SNS_ALPHABET_MAP = {
+          "." => "-"
+        }.freeze
+
+        attr_reader :options, :sqs_client, :sns_client
 
         register_adapter_name "sqs_sns"
 
         DEFAULT_OPTIONS = {
           mq_namespace: nil,
           mq_default_query_timeout: 15, # seconds,
-          mq_reply_queue_prefix: "reply.",
+          mq_reply_queue_prefix: "reply-",
 
           # if nil, AWS SDK will guess values from environment
           mq_aws_region: nil,
           mq_aws_access_key_id: nil,
           mq_aws_secret_access_key: nil,
           mq_aws_sqs_endpoint: nil,
+          mq_aws_sns_endpoint: nil,
 
           mq_aws_sqs_read_timeout: 10, # seconds
         }.freeze
@@ -59,11 +77,13 @@ module Mimi
 
         def start
           @sqs_client = Aws::SQS::Client.new(sqs_client_config)
+          @sns_client = Aws::SNS::Client.new(sns_client_config)
         end
 
         def stop
           stop_all_processors
           @sqs_client = nil
+          @sns_client = nil
         end
 
         # Stops all message (command, query and event) processors.
@@ -92,8 +112,8 @@ module Mimi
         def command(target, message, _opts = {})
           queue_name, method_name = target.split("/")
           message = Mimi::Messaging::Message.new(message, __method: method_name)
-          queue_url = find_queue(queue_name)
-          deliver_message(queue_url, message)
+          queue_url = find_queue!(queue_name)
+          deliver_message_queue(queue_url, message)
         end
 
         # Executes the query to the given target and returns response
@@ -107,7 +127,7 @@ module Mimi
         #
         def query(target, message, opts = {})
           queue_name, method_name = target.split("/")
-          queue_url = find_queue(queue_name)
+          queue_url = find_queue!(queue_name)
           request_id = SecureRandom.hex(8)
           reply_queue = reply_consumer.register_request_id(request_id)
 
@@ -117,7 +137,7 @@ module Mimi
             __reply_queue_url: reply_consumer.reply_queue_url,
             __request_id: request_id
           )
-          deliver_message(queue_url, message)
+          deliver_message_queue(queue_url, message)
           timeout = opts[:timeout] || options[:mq_default_query_timeout]
           response = nil
           Timeout::timeout(timeout) do
@@ -134,8 +154,9 @@ module Mimi
         #
         def event(target, message, _opts = {})
           topic_name, event_type = target.split("#")
-
-          raise "Not implemented"
+          message = Mimi::Messaging::Message.new(message, __event_type: event_type)
+          topic_arn = find_or_create_topic(topic_name) # TODO: or find_topic!(...) ?
+          deliver_message_topic(topic_arn, message)
         end
 
         # Starts a request (command/query) processor.
@@ -168,7 +189,7 @@ module Mimi
                 response,
                 __request_id: message.headers[:__request_id]
               )
-              deliver_message(reply_to, response_message)
+              deliver_message_queue(reply_to, response_message)
             else
               processor.call_command(method_name, message, {})
             end
@@ -176,25 +197,25 @@ module Mimi
         end
 
         def start_event_processor(topic_name, processor, opts = {})
+          # NOTE: due to SQS/SNS limitations, implementing this will
+          # require creating a temporary queue and subscribing it to the topic
           raise "Not implemented"
         end
 
         def start_event_processor_with_queue(topic_name, queue_name, processor, opts = {})
-          raise "Not implemented"
-        end
-
-        # Creates a new queue
-        #
-        # @param queue_name [String] name of the topic to be created
-        # @return [String] a new queue URL
-        #
-        def create_queue(queue_name)
-          fqn = full_queue_name(queue_name)
-          Mimi::Messaging.log "Creating a queue: #{fqn}"
-          result = sqs_client.create_queue(queue_name: fqn)
-          result.queue_url
-        rescue StandardError => e
-          raise Mimi::Messaging::ConnectionError, "Failed to create queue '#{queue_name}': #{e}"
+          @consumers ||= []
+          opts = opts.dup
+          topic_arn = find_or_create_topic(topic_name) # TODO: or find_topic!(...) ?
+          queue_url = find_or_create_queue(queue_name)
+          subscribe_topic_queue(topic_arn, queue_url)
+          @consumers << Consumer.new(self, queue_url) do |m|
+            message = Mimi::Messaging::Message.new(
+              deserialize(m.body),
+              deserialize_headers(m)
+            )
+            event_type = message.headers[:__event_type]
+            processor.call_event(event_type, message, {})
+          end
         end
 
         private
@@ -213,12 +234,40 @@ module Mimi
           params.compact
         end
 
+        # Returns configuration parameters for AWS SNS client
+        #
+        # @return [Hash]
+        #
+        def sns_client_config
+          params = {
+            region: options[:mq_aws_region],
+            endpoint: options[:mq_aws_sns_endpoint],
+            access_key_id: options[:mq_aws_access_key_id],
+            secret_access_key: options[:mq_aws_secret_access_key]
+          }
+          params.compact
+        end
+
+        # Creates a new queue
+        #
+        # @param queue_name [String] name of the topic to be created
+        # @return [String] a new queue URL
+        #
+        def create_queue(queue_name)
+          fqn = sqs_sns_converted_full_name(queue_name)
+          Mimi::Messaging.log "Creating a queue: #{fqn}"
+          result = sqs_client.create_queue(queue_name: fqn)
+          result.queue_url
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError, "Failed to create queue '#{queue_name}': #{e}"
+        end
+
         # Delivers a message to a queue with given URL.
         #
         # @param queue_url [String]
         # @param message [Mimi::Messaging::Message]
         #
-        def deliver_message(queue_url, message)
+        def deliver_message_queue(queue_url, message)
           raise ArgumentError, "Non-empty queue URL is expected" unless queue_url
           unless message.is_a?(Mimi::Messaging::Message)
             raise ArgumentError, "Message is expected as argument"
@@ -243,7 +292,7 @@ module Mimi
         # @return [String,nil] queue URL
         #
         def queue_registry(queue_name)
-          fqn = full_queue_name(queue_name)
+          fqn = sqs_sns_converted_full_name(queue_name)
           @queue_registry ||= {}
           @queue_registry[fqn] ||= begin
             result = sqs_client.get_queue_url(queue_name: fqn)
@@ -253,13 +302,18 @@ module Mimi
           nil
         end
 
-        # Converts a queue name to a fully qualified queue name
+        # Converts a topic or queue name to a fully qualified (with namespace)
+        # and in a valid SQS/SNS alphabet.
         #
-        # @param queue_name [String]
-        # @return [String]
+        # @param name [String] a mimi-messaging valid name
+        # @return [String] an SQS/SNS valid name
         #
-        def full_queue_name(queue_name)
-          "#{options[:mq_namespace]}#{queue_name}"
+        def sqs_sns_converted_full_name(name)
+          name = "#{options[:mq_namespace]}#{name}"
+          SQS_SNS_ALPHABET_MAP.each do |from, to|
+            name = name.gsub(from, to)
+          end
+          name
         end
 
         # Finds a queue URL for a queue with a given name,
@@ -268,7 +322,7 @@ module Mimi
         # @param queue_name [String]
         # @return [String] a queue URL
         #
-        def find_queue(queue_name)
+        def find_queue!(queue_name)
           queue_registry(queue_name) || (
             raise Mimi::Messaging::ConnectionError,
               "Failed to find a queue with given name: '#{queue_name}'"
@@ -307,6 +361,125 @@ module Mimi
         def deserialize_headers(message)
           message.message_attributes.to_h.map { |k, v| [k.to_sym, v.string_value] }.to_h
         end
+
+        # Lists all SNS topics by their ARNs.
+        #
+        # NOTE: iterates over all topics at SNS every time
+        #
+        # @return [Array<String>] array of topic ARNs
+        #
+        def sns_list_topics
+          result = []
+          next_token = nil
+          loop do
+            response = sns_client.list_topics(next_token: next_token)
+            result += response.topics.map(&:topic_arn)
+            next_token = response.next_token
+            break unless next_token
+          end
+          result
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError, "Failed to list topics: #{e}"
+        end
+
+        # Returns ARN of a topic with a given name.
+        #
+        # If the topic with given name does not exist, returns nil
+        #
+        # @param topic_name [String]
+        # @return [String,nil] topic ARN or nil, if not found
+        #
+        def topic_registry(topic_name)
+          fqn = sqs_sns_converted_full_name(topic_name)
+          @topic_registry ||= {}
+          @topic_registry[fqn] ||= begin
+            sns_list_topics.find { |topic_arn| topic_arn.split(":").last == fqn }
+          end
+        end
+
+        # Finds a topic ARN for a topic with a given name,
+        # or raises an error if the topic is not found.
+        #
+        # @param topic_name [String]
+        # @return [String] a topic ARN
+        #
+        def find_topic!(topic_name)
+          topic_registry(topic_name) || (
+            raise Mimi::Messaging::ConnectionError,
+              "Failed to find a topic with given name: '#{topic_name}'"
+          )
+        end
+
+        # Finds a topic ARN for a topic with given name.
+        #
+        # If an existing topic with this name is not found,
+        # the method will try to create a new one.
+        #
+        # @param topic_name [String]
+        # @return [String] a topic ARN
+        #
+        def find_or_create_topic(topic_name)
+          topic_registry(topic_name) || create_topic(topic_name)
+        end
+
+        # Creates a new topic
+        #
+        # @param topic_name [String] name of the topic to be created
+        # @return [String] a new topic ARN
+        #
+        def create_topic(topic_name)
+          fqn = sqs_sns_converted_full_name(topic_name)
+          Mimi::Messaging.log "Creating a topic: #{fqn}"
+          result = sns_client.create_topic(name: fqn)
+          result.topic_arn
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError, "Failed to create topic '#{topic_name}': #{e}"
+        end
+
+        # Subscribes an existing queue to an existing topic
+        #
+        # @param topic_arn [String]
+        # @param queue_url [String]
+        #
+        def subscribe_topic_queue(topic_arn, queue_url)
+          result = sqs_client.get_queue_attributes(
+            queue_url: queue_url, attribute_names: ["QueueArn"]
+          )
+          queue_arn = result.attributes["QueueArn"]
+          Mimi::Messaging.log "Subscribing queue to a topic: '#{topic_arn}'->'#{queue_url}'"
+          result = sns_client.subscribe(
+            topic_arn: topic_arn,
+            protocol: "sqs",
+            endpoint: queue_arn,
+            attributes: { "RawMessageDelivery" => "true" }
+          )
+          true
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError,
+            "Failed to subscribe queue to topic '#{topic_arn}'->'#{queue_url}': #{e}"
+        end
+
+        # Delivers a message to a topic with given ARN.
+        #
+        # @param topic_arn [String]
+        # @param message [Mimi::Messaging::Message]
+        #
+        def deliver_message_topic(topic_arn, message)
+          raise ArgumentError, "Non-empty topic ARN is expected" unless topic_arn
+          unless message.is_a?(Mimi::Messaging::Message)
+            raise ArgumentError, "Message is expected as argument"
+          end
+          Mimi::Messaging.log "Delivering message to: #{topic_arn}"
+          sns_client.publish(
+            topic_arn: topic_arn,
+            message: serialize(message),
+            message_attributes: message.headers.map do |k, v|
+              [k.to_s, { data_type: "String", string_value: v.to_s }]
+            end.to_h
+          )
+        rescue StandardError => e
+          raise Mimi::Messaging::ConnectionError, "Failed to deliver message to '#{topic_arn}': #{e}"
+        end
       end # class Adapter
     end # module SQS_SNS
   end # module Messaging

data/lib/mimi/messaging/sqs_sns/version.rb

@@ -3,7 +3,7 @@
 module Mimi
   module Messaging
     module SQS_SNS
-      VERSION = "0.3.0"
+      VERSION = "0.4.0"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mimi-messaging-sqs_sns
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Alex Kukushkin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-09 00:00:00.000000000 Z
+date: 2019-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mimi-messaging
@@ -125,8 +125,10 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- examples/event.rb
 - examples/query.rb
 - examples/responder.rb
+- examples/subscriber.rb
 - lib/mimi/messaging/sqs_sns.rb
 - lib/mimi/messaging/sqs_sns/adapter.rb
 - lib/mimi/messaging/sqs_sns/consumer.rb