realm-sns 0.7.3 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc34c9cede23ac937d62a7c1f496444a629427765d5bb3d7d4fc4f6eb8921507
-  data.tar.gz: 56571d1ffcff0d8c418de54f9bb3d3b99ef867e4c514d84af73694fc8c05f8d5
+  metadata.gz: 3c3e08ba717bbc19dc0c3541de8e63eabe3107b1275912ca6d137407c1eff68d
+  data.tar.gz: 3acb987d8ddb0ef72dcfb3d4af7b7dd46e94059e29ca5e0d35d4ee399f168f06
 SHA512:
-  metadata.gz: 972425f1c9c07b92209f2ad6eee95381ad8a346547cfe23981d8a936f5ca8d2dd726aa716ebc61db90d77cd0ce6fd6b7ee3f3350b2eb560edf516f975e94703a
-  data.tar.gz: b22a548fa14c648aba403c219b9b32ffcd0336761f363eb6d3874511ca269a92d13794ead1f9f5f676a56803eadc46e9e86f3693b066972df8d24494f7fb2ef6
+  metadata.gz: 68ca252514fa6bc74b3a66d69971895d73824e1d545d627fc3939db7cc923c204664846ccb6637b6c6fde4e0bd172ea763ee0232d06c5e38df85ede8ec2c51a1
+  data.tar.gz: 3a0b06381af32201261843f969dee3ed6b288b8dc6ad413dd621bbfb69a85fce7e2c85effb2f8edb3460ff4b546275c3dade5a894d3db83c217268903cc4f2fd

data/lib/realm-sns.rb

@@ -0,0 +1,16 @@
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+require 'zeitwerk'
+
+loader = Zeitwerk::Loader.for_gem
+loader.ignore(__FILE__)
+loader.inflector.inflect('sns' => 'SNS')
+loader.setup
+
+require 'realm-core'
+require 'aws-sdk-sns'
+require 'aws-sdk-sqs'
+require 'realm/sns/plugin'
+
+# rubocop:enable Naming/FileName

data/lib/realm/sns/gateway.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      def initialize(topic_arn:, queue_prefix: nil, event_processing_attempts: 3, **)
+        super
+        @topic = TopicAdapter.new(topic_arn)
+        @queue_prefix = queue_prefix
+        @event_processing_attempts = event_processing_attempts
+        @queue_map = {}
+      end
+
+      def add_listener(event_type, listener, queue_arn: nil)
+        queue = queue_arn ? queue_manager.get(arn: queue_arn) : provide_queue(event_type, listener)
+        @queue_map[queue] = listener
+      end
+
+      def trigger(event_type, attributes = {})
+        create_event(event_type, attributes).tap { |event| @topic.publish(event_type, event.to_json) }
+      end
+
+      def worker(**options)
+        @worker ||= Worker.new(
+          @queue_map,
+          event_factory: @event_factory,
+          logger: @runtime && @runtime.context[:logger],
+          event_processing_attempts: @event_processing_attempts,
+          **options,
+        )
+      end
+
+      def queues
+        @queue_map.keys
+      end
+
+      private
+
+      def provide_queue(event_type, listener)
+        queue_name = [event_type.to_s.gsub('.', '_'), queue_suffix(listener)].join('-')
+        queue = queue_manager.provide(queue_name)
+        @topic.subscribe(event_type, queue)
+        queue
+      end
+
+      def queue_manager
+        @queue_manager ||= QueueManager.new(prefix: @queue_prefix)
+      end
+
+      def queue_suffix(listener)
+        listener.try(:identifier) || SecureRandom.alphanumeric(16)
+      end
+    end
+  end
+end

data/lib/realm/sns/gateway/queue_adapter.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      # Provides cleaner SDK over Aws::SQS::Queue
+      class QueueAdapter
+        include Mixins::Decorator[:@queue]
+
+        def arn
+          @queue.attributes['QueueArn']
+        end
+
+        def allow_send_messages(source_arn)
+          @queue.set_attributes(attributes: {
+                                  'Policy' => {
+                                    'Version' => '2012-10-17',
+                                    'Statement' => policy_statement(source_arn),
+                                  }.to_json,
+                                })
+        end
+
+        def publish(event_type, message)
+          @queue.send_message(
+            message_body: message,
+            message_attributes: { 'event_type' => { data_type: 'String', string_value: event_type.to_s } },
+          )
+        end
+
+        def empty?
+          attributes.slice(
+            'ApproximateNumberOfMessages',
+            'ApproximateNumberOfMessagesDelayed',
+            'ApproximateNumberOfMessagesNotVisible',
+          ).all? { |_, val| val.to_i.zero? }
+        end
+
+        private
+
+        def policy_statement(source_arn)
+          {
+            'Effect' => 'Allow',
+            'Principal' => { 'AWS' => '*' },
+            'Action' => 'sqs:SendMessage',
+            'Resource' => arn,
+            'Condition' => {
+              'ArnEquals' => { 'aws:SourceArn' => source_arn },
+            },
+          }
+        end
+      end
+    end
+  end
+end

data/lib/realm/sns/gateway/queue_manager.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      class QueueManager
+        class QueueNameTooLong < Realm::Error
+          def initialize(name, msg: "Queue name '#{name}' cannot be longer than 80 characters, " \
+                                    "please provide custom EventHandler identifier if it's auto generated")
+            super(msg)
+          end
+        end
+
+        CleanupWithoutPrefix = Realm::Error[
+          'Cleaning up queues without prefix is not allowed, it can lead to deleting queues from other apps']
+
+        def initialize(prefix: nil, sqs: Aws::SQS::Resource.new)
+          @prefix = prefix
+          @sqs = sqs
+        end
+
+        def get(name: nil, arn: nil)
+          throw ArgumentError, 'You have to provide name or arn of the queue' unless name || arn
+
+          QueueAdapter.new(@sqs.get_queue_by_name(queue_name: name ? prefix_name(name) : arn.split(':')[-1]))
+        end
+
+        def create(queue_name)
+          name = prefix_name(queue_name)
+          raise QueueNameTooLong, name if name.size > 80
+
+          QueueAdapter.new(@sqs.create_queue(queue_name: name))
+        end
+
+        def provide(queue_name)
+          get(name: queue_name)
+        rescue Aws::SQS::Errors::NonExistentQueue
+          create(queue_name)
+        end
+
+        def cleanup(except: [])
+          raise CleanupWithoutPrefix unless @prefix
+
+          except_urls = Array(except).map(&:url)
+          @sqs.queues(queue_name_prefix: @prefix).each do |queue|
+            next if except_urls.include?(queue.url)
+
+            queue.delete if QueueAdapter.new(queue).empty?
+          end
+        end
+
+        private
+
+        def prefix_name(name)
+          [@prefix, name].compact.join('-')
+        end
+      end
+    end
+  end
+end

data/lib/realm/sns/gateway/topic_adapter.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      # Provides cleaner SDK over Aws::SNS::Topic
+      class TopicAdapter
+        class SubscriptionError < Realm::Error
+          def initialize(queue_arn, subscription_attributes)
+            super("Cannot subscribe SQS queue #{queue_arn} with attributes #{subscription_attributes}")
+          end
+        end
+
+        def initialize(topic_or_arn)
+          @topic = topic_or_arn.is_a?(Aws::SNS::Topic) ? topic_or_arn : Aws::SNS::Resource.new.topic(topic_or_arn)
+        end
+
+        def publish(event_type, message)
+          @topic.publish(
+            message: message,
+            message_attributes: { 'event_type' => { data_type: 'String', string_value: event_type.to_s } },
+          )
+        end
+
+        def subscribe(event_type, queue)
+          queue.allow_send_messages(@topic.arn)
+          attributes = subscribe_attributes(event_type)
+          @topic.subscribe(protocol: 'sqs', endpoint: queue.arn, attributes: attributes)
+        rescue Aws::SNS::Errors::InvalidParameter
+          raise SubscriptionError.new(queue.arn, attributes)
+        end
+
+        private
+
+        def subscribe_attributes(event_type)
+          attrs = { 'RawMessageDelivery' => true }
+          attrs['FilterPolicy'] = { 'event_type' => [event_type] } unless event_type == :any
+          attrs.transform_values(&:to_json)
+        end
+      end
+    end
+  end
+end

data/lib/realm/sns/gateway/worker.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Realm
+  module SNS
+    class Gateway < Realm::EventRouter::Gateway
+      class Worker
+        def initialize(queue_map, event_factory:, event_processing_attempts: 3, logger: nil)
+          @queue_map = queue_map
+          @event_factory = event_factory
+          @event_processing_attempts = event_processing_attempts
+          @logger = logger || Logger.new($stdout)
+          @threads = []
+        end
+
+        def start(poller_options: {})
+          @signaler = { exiting: false }
+          @queue_map.each_pair do |queue, listener|
+            @threads << Thread.new { run_poller(queue, listener, @signaler, poller_options) }
+          end
+          self
+        end
+
+        def stop(timeout: 30)
+          Thread.new { @logger.info("Stopping worker (timeout: #{timeout}s)") }.join # Cannot log from trap context
+          @signaler[:exiting] = true
+          join(timeout)
+          @threads.clear
+          self
+        end
+
+        def join(timeout = nil)
+          @threads.each { |thread| thread.join(timeout) }
+        end
+
+        private
+
+        def run_poller(queue, listener, signaler, options)
+          @logger.info("Start polling #{queue.arn}")
+          init_poller(queue, signaler, options).poll do |messages, stats|
+            log_poller_stats(queue, stats)
+            messages.each { |msg| handle_message(listener, msg) }
+          end
+          @logger.info("Polling stopped #{queue.arn}")
+        end
+
+        def init_poller(queue, signaler, options = {})
+          Aws::SQS::QueuePoller.new(
+            queue.url,
+            max_number_of_messages: 10,
+            visibility_timeout: 60,
+            attribute_names: ['ApproximateReceiveCount'],
+            message_attribute_names: ['event_type'],
+            before_request: before_request_proc(queue, signaler),
+            **options,
+          )
+        end
+
+        def before_request_proc(queue, signaler)
+          proc {
+            if signaler[:exiting]
+              @logger.info("Stopping polling #{queue.arn}")
+              throw :stop_polling
+            end
+          }
+        end
+
+        def log_poller_stats(queue, stats)
+          @logger.info(
+            message: "Poller #{queue.arn} stats",
+            request_count: stats.request_count,
+            message_count: stats.received_message_count,
+            last_message_received_at: stats.last_message_received_at,
+          )
+        end
+
+        def handle_message(listener, msg)
+          event = message_to_event(msg)
+          listener.(event)
+        rescue StandardError => e
+          log_error(e, event, msg)
+          # Picks up the message again after visibility_timeout runs out:
+          throw :skip_delete if event && message_receive_count(msg) < @event_processing_attempts
+        end
+
+        def message_to_event(msg)
+          event_type = msg.message_attributes['event_type'].string_value
+          raise 'Message is missing event type' unless event_type
+
+          payload = JSON.parse(msg.body).deep_symbolize_keys
+          @event_factory.create_event(event_type, payload)
+        end
+
+        def message_receive_count(msg)
+          msg.attributes['ApproximateReceiveCount'].to_i
+        end
+
+        def log_error(error, event, msg)
+          return @logger.fatal("Unexpected message in queue: #{msg}; error: #{error.full_message}") unless event
+
+          attempt = message_receive_count(msg)
+          will_retry = attempt < @event_processing_attempts
+          log_line = [
+            "Processing of event failed type=#{event.type} id=#{event.head.id} attempt=#{attempt},",
+            "#{will_retry ? 'will retry' : 'final'}):\n#{error.full_message}",
+          ].join(' ')
+          @logger.send(will_retry ? :warn : :error, log_line)
+        end
+      end
+    end
+  end
+end

data/lib/realm/sns/plugin.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Realm
+  module SNS
+    class Plugin < Realm::Plugin
+      def self.setup(_config, container)
+        container.register('event_router.gateway_classes.sns', SNS::Gateway)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: realm-sns
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.4
 platform: ruby
 authors:
 - developers@reevoo.com
@@ -114,8 +114,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- README.md
-- Rakefile
+- lib/realm-sns.rb
+- lib/realm/sns/gateway.rb
+- lib/realm/sns/gateway/queue_adapter.rb
+- lib/realm/sns/gateway/queue_manager.rb
+- lib/realm/sns/gateway/topic_adapter.rb
+- lib/realm/sns/gateway/worker.rb
+- lib/realm/sns/plugin.rb
 homepage:
 licenses:
 - MIT

data/README.md

@@ -1,40 +0,0 @@
-# Realm
-
-Domain layer framework following Domain-driven/CQRS design principles.
-
-[![Build status](https://badge.buildkite.com/346cce75f6c31e0a41bb98b198e85eb6b722243624459fad9c.svg)](https://buildkite.com/reevoo/realm)
-
-## Service layers
-
-We follow the standard MVC design pattern of Rails but giving the model layer more structure and guidance regarding where
-to put your code. The model is split into domain layer (using our [Realm](https://github.com/reevoo/smart-mono/tree/master/gems/realm) library)
-and persistence layer (using [ROM](https://rom-rb.org/) library). The individual components are explained in the following section.
-
-![Service layers](https://confluence-connect.gliffy.net/embed/image/d02d04b1-5e40-415f-b7ba-3a631efa9bf3.png?utm_medium=live&utm_source=custom)
-
-Advanced components are shown in lighter color, those will be needed only later on as the service domain logic grows.
-
-## Model layer components
-
-![Service external components](https://confluence-connect.gliffy.net/embed/image/c593fcc2-304e-47c3-8e3c-b0cc09e0ed54.png?utm_medium=live&utm_source=custom)
-
-Each service has one **domain** module which consists of multiple [**aggregate**](https://martinfowler.com/bliki/DDD_Aggregate.html) modules.
-Aggregate is a cluster of domain objects that can be treated as a single unit. The only way for outer world to communicate
-with aggregate is by **queries** and **commands**. Query exposes aggregate's internal state and command changes it.
-The state of an aggregate is represented by tree of **entities** with one being the aggregate root and zero or more dependent
-entities with *belongs_to* relation to the root entity. The state of an aggregate (entity tree) is persisted
-and retrieved by **repository**. There is generally one repository per aggregate unless we split the read/write
-(query/command) persistence model for that particular domain. The repository uses **relations** to access the database
-tables. Each relation class represents one table.
-
-
-## Where to put my code as it grows?
-
-TODO
-
-
-## Roadmap
-
-- [ ] Support Ruby 3
-- [ ] Make it work outside of Rails engines
-- [ ] Support multiple persistence gateways in one runtime

data/Rakefile

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-begin
-  require 'bundler/setup'
-rescue LoadError
-  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
-end
-
-require 'rdoc/task'
-
-RDoc::Task.new(:rdoc) do |rdoc|
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title    = 'Realm'
-  rdoc.options << '--line-numbers'
-  rdoc.rdoc_files.include('README.md')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
-
-require 'bundler/gem_tasks'