realm-sns 0.7.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: aed94ae9cc516c1b6288aa772cc161c26cb1e15653844484a8bdb3a746a23192
+  data.tar.gz: 2e199ef3883732d74df1f2695262ac8d3194dd695c9d039bf5cc3a032f91f04a
+SHA512:
+  metadata.gz: 1af4c9e553f23fbd7ac3284e059203767a457314d9e2b2c7cdea0c2f223c78b1ae6071b9c8769f960a6ba21d2c65f472a35ef74043f02f99dfd26770d4d392ee
+  data.tar.gz: 1e35141ccba9f78085e1f509b0522f45fec26b2a240ae6d519882b16553f69ae62dc23c4d78713afa0cd112bca364fd6039650d8f052e7d59d001508340fa71b

data/lib/realm-sns.rb

@@ -0,0 +1,8 @@
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+Dir[File.join(File.dirname(__FILE__), 'realm', '**', '*.rb')].sort.each do |f|
+  require f
+end
+
+# rubocop:enable Naming/FileName

data/lib/realm/sns/gateway.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'active_support/core_ext/string'
+
+require 'realm/event_handler'
+require 'realm/event_router/gateway'
+require_relative './gateway/queue_manager'
+require_relative './gateway/topic_adapter'
+require_relative './gateway/worker'
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      def initialize(topic_arn:, queue_prefix: nil, event_processing_attempts: 3, **)
+        super
+        @topic = TopicAdapter.new(topic_arn)
+        @queue_prefix = queue_prefix
+        @event_processing_attempts = event_processing_attempts
+        @queue_map = {}
+      end
+
+      def add_listener(event_type, listener, queue_arn: nil)
+        queue = queue_arn ? queue_manager.get(arn: queue_arn) : provide_queue(event_type, listener)
+        @queue_map[queue] = listener
+      end
+
+      def trigger(event_type, attributes = {})
+        create_event(event_type, attributes).tap { |event| @topic.publish(event_type, event.to_json) }
+      end
+
+      def worker(**options)
+        @worker ||= Worker.new(
+          @queue_map,
+          event_factory: @event_factory,
+          logger: @runtime && @runtime.context[:logger],
+          event_processing_attempts: @event_processing_attempts,
+          **options,
+        )
+      end
+
+      def queues
+        @queue_map.keys
+      end
+
+      private
+
+      def provide_queue(event_type, listener)
+        queue_name = [event_type, queue_suffix(listener)].join('-')
+        queue = queue_manager.provide(queue_name)
+        @topic.subscribe(event_type, queue)
+        queue
+      end
+
+      def queue_manager
+        @queue_manager ||= QueueManager.new(prefix: @queue_prefix)
+      end
+
+      def queue_suffix(listener)
+        listener.try(:identifier) || SecureRandom.alphanumeric(16)
+      end
+    end
+  end
+end

data/lib/realm/sns/gateway/queue_adapter.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'realm/event_router/gateway'
+require 'realm/mixins/decorator'
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      # Provides cleaner SDK over Aws::SQS::Queue
+      class QueueAdapter
+        include Mixins::Decorator[:@queue]
+
+        def arn
+          @queue.attributes['QueueArn']
+        end
+
+        def allow_send_messages(source_arn)
+          @queue.set_attributes(attributes: {
+                                  'Policy' => {
+                                    'Version' => '2012-10-17',
+                                    'Statement' => policy_statement(source_arn),
+                                  }.to_json,
+                                })
+        end
+
+        def publish(event_type, message)
+          @queue.send_message(
+            message_body: message,
+            message_attributes: { 'event_type' => { data_type: 'String', string_value: event_type.to_s } },
+          )
+        end
+
+        def empty?
+          attributes.slice(
+            'ApproximateNumberOfMessages',
+            'ApproximateNumberOfMessagesDelayed',
+            'ApproximateNumberOfMessagesNotVisible',
+          ).all? { |_, val| val.to_i.zero? }
+        end
+
+        private
+
+        def policy_statement(source_arn)
+          {
+            'Effect' => 'Allow',
+            'Principal' => { 'AWS' => '*' },
+            'Action' => 'sqs:SendMessage',
+            'Resource' => arn,
+            'Condition' => {
+              'ArnEquals' => { 'aws:SourceArn' => source_arn },
+            },
+          }
+        end
+      end
+    end
+  end
+end

data/lib/realm/sns/gateway/queue_manager.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require 'realm/error'
+require_relative './queue_adapter'
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      class QueueManager
+        class QueueNameTooLong < Realm::Error
+          def initialize(name, msg: "Queue name '#{name}' cannot be longer than 80 characters, " \
+                                    "please provide custom EventHandler identifier if it's auto generated")
+            super(msg)
+          end
+        end
+
+        CleanupWithoutPrefix = Realm::Error[
+          'Cleaning up queues without prefix is not allowed, it can lead to deleting queues from other apps']
+
+        def initialize(prefix: nil, sqs: Aws::SQS::Resource.new)
+          @prefix = prefix
+          @sqs = sqs
+        end
+
+        def get(name: nil, arn: nil)
+          throw ArgumentError, 'You have to provide name or arn of the queue' unless name || arn
+
+          QueueAdapter.new(@sqs.get_queue_by_name(queue_name: name ? prefix_name(name) : arn.split(':')[-1]))
+        end
+
+        def create(queue_name)
+          name = prefix_name(queue_name)
+          raise QueueNameTooLong, name if name.size > 80
+
+          QueueAdapter.new(@sqs.create_queue(queue_name: name))
+        end
+
+        def provide(queue_name)
+          get(name: queue_name)
+        rescue Aws::SQS::Errors::NonExistentQueue
+          create(queue_name)
+        end
+
+        def cleanup(except: [])
+          raise CleanupWithoutPrefix unless @prefix
+
+          except_urls = Array(except).map(&:url)
+          @sqs.queues(queue_name_prefix: @prefix).each do |queue|
+            next if except_urls.include?(queue.url)
+
+            queue.delete if QueueAdapter.new(queue).empty?
+          end
+        end
+
+        private
+
+        def prefix_name(name)
+          [@prefix, name].compact.join('-')
+        end
+      end
+    end
+  end
+end

data/lib/realm/sns/gateway/topic_adapter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sns'
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      # Provides cleaner SDK over Aws::SNS::Topic
+      class TopicAdapter
+        def initialize(topic_or_arn)
+          @topic = topic_or_arn.is_a?(Aws::SNS::Topic) ? topic_or_arn : Aws::SNS::Resource.new.topic(topic_or_arn)
+        end
+
+        def publish(event_type, message)
+          @topic.publish(
+            message: message,
+            message_attributes: { 'event_type' => { data_type: 'String', string_value: event_type.to_s } },
+          )
+        end
+
+        def subscribe(event_type, queue)
+          queue.allow_send_messages(@topic.arn)
+          @topic.subscribe(
+            protocol: 'sqs',
+            endpoint: queue.arn,
+            attributes: subscribe_attributes(event_type),
+          )
+        end
+
+        private
+
+        def subscribe_attributes(event_type)
+          attrs = { 'RawMessageDelivery' => true }
+          attrs['FilterPolicy'] = { 'event_type' => [event_type] } unless event_type == :any
+          attrs.transform_values(&:to_json)
+        end
+      end
+    end
+  end
+end

data/lib/realm/sns/gateway/worker.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash'
+require 'aws-sdk-sqs'
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      class Worker
+        def initialize(queue_map, event_factory:, event_processing_attempts: 3, logger: nil)
+          @queue_map = queue_map
+          @event_factory = event_factory
+          @event_processing_attempts = event_processing_attempts
+          @logger = logger || Logger.new($stdout)
+          @threads = []
+        end
+
+        def start(poller_options: {})
+          @signaler = { exiting: false }
+          @queue_map.each_pair do |queue, listener|
+            @threads << Thread.new { run_poller(queue, listener, @signaler, poller_options) }
+          end
+          self
+        end
+
+        def stop(timeout: 30)
+          Thread.new { @logger.info("Stopping worker (timeout: #{timeout}s)") }.join # Cannot log from trap context
+          @signaler[:exiting] = true
+          join(timeout)
+          @threads.clear
+          self
+        end
+
+        def join(timeout = nil)
+          @threads.each { |thread| thread.join(timeout) }
+        end
+
+        private
+
+        def run_poller(queue, listener, signaler, options)
+          @logger.info("Start polling #{queue.arn}")
+          init_poller(queue, signaler, options).poll do |messages, stats|
+            log_poller_stats(queue, stats)
+            messages.each { |msg| handle_message(listener, msg) }
+          end
+          @logger.info("Polling stopped #{queue.arn}")
+        end
+
+        def init_poller(queue, signaler, options = {})
+          Aws::SQS::QueuePoller.new(
+            queue.url,
+            max_number_of_messages: 10,
+            visibility_timeout: 60,
+            attribute_names: ['ApproximateReceiveCount'],
+            message_attribute_names: ['event_type'],
+            before_request: before_request_proc(queue, signaler),
+            **options,
+          )
+        end
+
+        def before_request_proc(queue, signaler)
+          proc {
+            if signaler[:exiting]
+              @logger.info("Stopping polling #{queue.arn}")
+              throw :stop_polling
+            end
+          }
+        end
+
+        def log_poller_stats(queue, stats)
+          @logger.info(
+            message: "Poller #{queue.arn} stats",
+            request_count: stats.request_count,
+            message_count: stats.received_message_count,
+            last_message_received_at: stats.last_message_received_at,
+          )
+        end
+
+        def handle_message(listener, msg)
+          event = message_to_event(msg)
+          listener.(event)
+        rescue StandardError => e
+          log_error(e, event, msg)
+          # Picks up the message again after visibility_timeout runs out:
+          throw :skip_delete if event && message_receive_count(msg) < @event_processing_attempts
+        end
+
+        def message_to_event(msg)
+          event_type = msg.message_attributes['event_type'].string_value
+          raise 'Message is missing event type' unless event_type
+
+          payload = JSON.parse(msg.body).deep_symbolize_keys
+          @event_factory.create_event(event_type, payload)
+        end
+
+        def message_receive_count(msg)
+          msg.attributes['ApproximateReceiveCount'].to_i
+        end
+
+        def log_error(error, event, msg)
+          return @logger.fatal("Unexpected message in queue: #{msg}; error: #{error.full_message}") unless event
+
+          attempt = message_receive_count(msg)
+          will_retry = attempt < @event_processing_attempts
+          log_line = [
+            "Processing of event failed type=#{event.type} id=#{event.head.id} attempt=#{attempt},",
+            "#{will_retry ? 'will retry' : 'final'}):\n#{error.full_message}",
+          ].join(' ')
+          @logger.send(will_retry ? :warn : :error, log_line)
+        end
+      end
+    end
+  end
+end

data/lib/realm/sns/plugin.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'realm/plugin'
+require_relative 'gateway'
+
+module Realm
+  module SNS
+    class Plugin < Realm::Plugin
+      def self.setup(_config, container)
+        container.register('event_router.gateway_classes.sns', SNS::Gateway)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification
+name: realm-sns
+version: !ruby/object:Gem::Version
+  version: 0.7.0
+platform: ruby
+authors:
+- developers@reevoo.com
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-06-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sns
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.36'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.36'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.34'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.34'
+- !ruby/object:Gem::Dependency
+  name: realm-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/realm-sns.rb
+- lib/realm/sns/gateway.rb
+- lib/realm/sns/gateway/queue_adapter.rb
+- lib/realm/sns/gateway/queue_manager.rb
+- lib/realm/sns/gateway/topic_adapter.rb
+- lib/realm/sns/gateway/worker.rb
+- lib/realm/sns/plugin.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key: 
+specification_version: 4
+summary: SNS/SQS plugin for Realm
+test_files: []