circuitry 2.1.1 → 3.0.0

data/lib/circuitry/config/file_loader.rb

@@ -0,0 +1,24 @@
+require 'yaml'
+require 'erb'
+require 'fileutils'
+
+module Circuitry
+  module Config
+    module FileLoader
+      def self.load(cfile, environment = 'development')
+        return nil unless File.exist?(cfile)
+
+        opts = {}
+        opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
+        opts = opts.merge(opts.delete(environment) || {})
+
+        publisher_opts = opts.merge(opts.delete('publisher') || {})
+        subscriber_opts = opts.merge(opts.delete('subscriber') || {})
+
+        Circuitry.subscriber_config = subscriber_opts
+        Circuitry.publisher_config = publisher_opts
+        true
+      end
+    end
+  end
+end

data/lib/circuitry/config/publisher_settings.rb

@@ -0,0 +1,16 @@
+require 'virtus'
+require 'circuitry/config/shared_settings'
+
+module Circuitry
+  module Config
+    class PublisherSettings
+      include Virtus::Model
+      include SharedSettings
+
+      def async_strategy=(value)
+        validate_setting(value, Publisher.async_strategies)
+        super
+      end
+    end
+  end
+end

data/lib/circuitry/config/shared_settings.rb

@@ -0,0 +1,39 @@
+require 'logger'
+
+module Circuitry
+  class ConfigError < StandardError; end
+
+  module Config
+    module SharedSettings
+      def self.included(base)
+        base.attribute :access_key, String
+        base.attribute :secret_key, String
+        base.attribute :region, String, default: 'us-east-1'
+        base.attribute :logger, Logger, default: Logger.new(STDERR)
+        base.attribute :error_handler
+        base.attribute :topic_names, Array[String], default: []
+        base.attribute :on_async_exit
+        base.attribute :async_strategy, Symbol, default: ->(_page, _att) { :fork }
+      end
+
+      def middleware
+        @_middleware ||= Circuitry::Middleware::Chain.new
+        yield @_middleware if block_given?
+        @_middleware
+      end
+
+      def aws_options
+        {
+          access_key_id:     access_key,
+          secret_access_key: secret_key,
+          region:            region
+        }
+      end
+
+      def validate_setting(value, permitted_values)
+        return if permitted_values.include?(value)
+        raise ConfigError, "invalid value `#{value}`, must be one of #{permitted_values.inspect}"
+      end
+    end
+  end
+end

data/lib/circuitry/config/subscriber_settings.rb

@@ -0,0 +1,32 @@
+require 'virtus'
+require 'circuitry/config/shared_settings'
+
+module Circuitry
+  module Config
+    class SubscriberSettings
+      include Virtus::Model
+      include SharedSettings
+
+      attribute :queue_name, String
+      attribute :dead_letter_queue_name, String
+      attribute :visibility_timeout, Integer, default: 30 * 60
+      attribute :max_receive_count, Integer, default: 8
+      attribute :lock_strategy, Object, default: ->(_page, _att) { Circuitry::Locks::Memory.new }
+
+      def dead_letter_queue_name
+        super || "#{queue_name}-failures"
+      end
+
+      def async_strategy=(value)
+        validate_setting(value, Subscriber.async_strategies)
+        super
+      end
+
+      def lock_strategy=(value)
+        unless value.is_a?(Circuitry::Locks::Base)
+          raise ConfigError, "invalid lock strategy \"#{value.inspect}\""
+        end
+      end
+    end
+  end
+end

data/lib/circuitry/locks/base.rb

@@ -25,15 +25,15 @@ module Circuitry
 
       protected
 
-      def lock(key, ttl)
+      def lock(_key, _ttl)
         raise NotImplementedError
       end
 
-      def lock!(key, ttl)
+      def lock!(_key, _ttl)
         raise NotImplementedError
       end
 
-      def unlock!(key)
+      def unlock!(_key)
         raise NotImplementedError
       end
 

data/lib/circuitry/locks/memory.rb

@@ -19,7 +19,7 @@ module Circuitry
         reap
 
         store do |store|
-          if store.has_key?(key)
+          if store.key?(key)
             false
           else
             store[key] = Time.now + ttl
@@ -44,16 +44,16 @@ module Circuitry
 
       private
 
-      def store(&block)
+      def store
         semaphore.synchronize do
-          block.call(self.class.store)
+          yield self.class.store
         end
       end
 
       def reap
         store do |store|
           now = Time.now
-          store.delete_if { |key, expires_at| expires_at <= now }
+          store.delete_if { |_, expires_at| expires_at <= now }
         end
       end
 

data/lib/circuitry/locks/noop.rb

@@ -5,7 +5,7 @@ module Circuitry
 
       protected
 
-      def lock(key, ttl)
+      def lock(_key, _ttl)
         true
       end
 

data/lib/circuitry/locks/redis.rb

@@ -40,7 +40,7 @@ module Circuitry
         if pool?
           client.with(&block)
         else
-          block.call(client)
+          yield client
         end
       end
 

data/lib/circuitry/middleware/chain.rb

@@ -54,7 +54,7 @@ module Circuitry
       def invoke(*args)
         chain = build.dup
 
-        traverse_chain = -> do
+        traverse_chain = lambda do
           if chain.empty?
             yield
           else

data/lib/circuitry/processor.rb

@@ -1,6 +1,6 @@
 module Circuitry
   module Processor
-    def process(&block)
+    def process(&_block)
       raise NotImplementedError, "#{self} must implement class method `process`"
     end
 
@@ -8,15 +8,17 @@ module Circuitry
       raise NotImplementedError, "#{self} must implement class method `flush`"
     end
 
+    def on_exit
+      Circuitry.subscriber_config.on_async_exit
+    end
+
     protected
 
-    def safely_process(&block)
-      begin
-        block.call
-      rescue => e
-        logger.error("Error handling message: #{e}")
-        error_handler.call(e) if error_handler
-      end
+    def safely_process
+      yield
+    rescue => e
+      logger.error("Error handling message: #{e}")
+      error_handler.call(e) if error_handler
     end
 
     def pool
@@ -26,11 +28,11 @@ module Circuitry
     private
 
     def logger
-      Circuitry.config.logger
+      Circuitry.subscriber_config.logger
     end
 
     def error_handler
-      Circuitry.config.error_handler
+      Circuitry.subscriber_config.error_handler
     end
   end
 end

data/lib/circuitry/processors/forker.rb

@@ -9,7 +9,7 @@ module Circuitry
         def process(&block)
           pid = fork do
             safely_process(&block)
-            Circuitry.config.on_fork_exit.call if Circuitry.config.on_fork_exit
+            on_exit.call if on_exit
           end
 
           Process.detach(pid)

data/lib/circuitry/processors/threader.rb

@@ -11,7 +11,7 @@ module Circuitry
 
           pool << Thread.new do
             safely_process(&block)
-            Circuitry.config.on_thread_exit.call if Circuitry.config.on_thread_exit
+            on_exit.call if on_exit
           end
         end
 

data/lib/circuitry/provisioning.rb

@@ -0,0 +1,9 @@
+require 'circuitry/provisioning/provisioner'
+
+module Circuitry
+  module Provisioning
+    def self.provision(logger: Logger.new(STDOUT))
+      Provisioning::Provisioner.new(logger).run
+    end
+  end
+end

data/lib/circuitry/provisioning/provisioner.rb

@@ -0,0 +1,71 @@
+require 'circuitry/provisioning/queue_creator'
+require 'circuitry/provisioning/topic_creator'
+require 'circuitry/provisioning/subscription_creator'
+
+module Circuitry
+  module Provisioning
+    class Provisioner
+      def initialize(logger)
+        self.logger = logger
+      end
+
+      def run
+        queue = create_queue
+        return unless queue
+
+        create_topics(:publisher, publisher_config.topic_names)
+        subscribe_topics(queue, create_topics(:subscriber, subscriber_config.topic_names))
+      end
+
+      private
+
+      attr_accessor :logger
+
+      def publisher_config
+        Circuitry.publisher_config
+      end
+
+      def subscriber_config
+        Circuitry.subscriber_config
+      end
+
+      def create_queue
+        safe_aws('Create Queue') do
+          queue = QueueCreator.find_or_create(
+            subscriber_config.queue_name,
+            dead_letter_queue_name: subscriber_config.dead_letter_queue_name,
+            max_receive_count: subscriber_config.max_receive_count,
+            visibility_timeout: subscriber_config.visibility_timeout
+          )
+          logger.info "Created queue #{queue.url}"
+          queue
+        end
+      end
+
+      def create_topics(type, topics)
+        safe_aws("Create #{type.to_s.capitalize} Topics") do
+          topics.map do |topic_name|
+            topic = TopicCreator.find_or_create(topic_name)
+            logger.info "Created topic #{topic.name}"
+            topic
+          end
+        end
+      end
+
+      def subscribe_topics(queue, topics)
+        safe_aws('Subscribe Topics') do
+          SubscriptionCreator.subscribe_all(queue, topics)
+          logger.info "Subscribed all topics to #{queue.name}"
+          true
+        end
+      end
+
+      def safe_aws(desc)
+        yield
+      rescue Aws::SQS::Errors::AccessDenied
+        logger.fatal("#{desc}: Access denied. Check your configured credentials.")
+        nil
+      end
+    end
+  end
+end

data/lib/circuitry/provisioning/queue_creator.rb

@@ -0,0 +1,64 @@
+require 'circuitry/services/sqs'
+require 'circuitry/queue'
+
+module Circuitry
+  module Provisioning
+    class QueueCreator
+      include Services::SQS
+
+      attr_reader :queue_name
+      attr_reader :visibility_timeout
+
+      def self.find_or_create(queue_name,
+                              dead_letter_queue_name: nil,
+                              max_receive_count: 8,
+                              visibility_timeout: 30 * 60)
+        creator = new(queue_name, visibility_timeout)
+        result = creator.create_queue
+        creator.create_dead_letter_queue(dead_letter_queue_name, max_receive_count) if dead_letter_queue_name
+        result
+      end
+
+      def initialize(queue_name, visibility_timeout)
+        self.queue_name = queue_name
+        self.visibility_timeout = visibility_timeout
+      end
+
+      def create_queue
+        @_queue ||= Queue.new(create_primary_queue_internal)
+      end
+
+      def create_dead_letter_queue(name, max_receive_count)
+        @_dl_queue ||= Queue.new(create_dl_queue_internal(name, max_receive_count))
+      end
+
+      private
+
+      attr_writer :queue_name
+      attr_writer :visibility_timeout
+
+      def create_dl_queue_internal(name, max_receive_count)
+        dl_url = sqs.create_queue(queue_name: name).queue_url
+        dl_arn = sqs.get_queue_attributes(
+          queue_url: dl_url,
+          attribute_names: ['QueueArn']
+        ).attributes['QueueArn']
+
+        policy = build_redrive_policy(dl_arn, max_receive_count)
+        sqs.set_queue_attributes(queue_url: create_queue.url, attributes: policy)
+        dl_url
+      end
+
+      def build_redrive_policy(deadletter_arn, max_receive_count)
+        {
+          'RedrivePolicy' => %({"maxReceiveCount":"#{max_receive_count}", "deadLetterTargetArn":"#{deadletter_arn}"})
+        }
+      end
+
+      def create_primary_queue_internal
+        attributes = { 'VisibilityTimeout' => visibility_timeout.to_s }
+        sqs.create_queue(queue_name: queue_name, attributes: attributes).queue_url
+      end
+    end
+  end
+end

data/lib/circuitry/provisioning/subscription_creator.rb

@@ -0,0 +1,65 @@
+require 'circuitry/services/sns'
+require 'circuitry/topic'
+
+module Circuitry
+  module Provisioning
+    class SubscriptionCreator
+      include Services::SNS
+      include Services::SQS
+
+      attr_reader :queue
+      attr_reader :topics
+
+      def self.subscribe_all(queue, topics)
+        new(queue, topics).subscribe_all
+      end
+
+      def initialize(queue, topics)
+        raise ArgumentError, 'queue must be a Circuitry::Queue' unless queue.is_a?(Circuitry::Queue)
+        raise ArgumentError, 'topics must be an array' unless topics.is_a?(Array)
+
+        self.queue = queue
+        self.topics = topics
+      end
+
+      def subscribe_all
+        topics.each do |topic|
+          sns.subscribe(topic_arn: topic.arn, endpoint: queue.arn, protocol: 'sqs')
+        end
+        sqs.set_queue_attributes(
+          queue_url: queue.url,
+          attributes: build_policy
+        )
+      end
+
+      private
+
+      attr_writer :queue
+      attr_writer :topics
+
+      def build_policy
+        # The aws ruby SDK doesn't have a policy builder :{
+        {
+          'Policy' => {
+            'Version'   => '2012-10-17',
+            'Id'        => "#{queue.arn}/SNSPolicy",
+            'Statement' => topics.map { |t| build_policy_statement(t) }
+          }.to_json
+        }
+      end
+
+      def build_policy_statement(topic)
+        {
+          'Sid'       => "Sid#{topic.name}",
+          'Effect'    => 'Allow',
+          'Principal' => { 'AWS' => '*' },
+          'Action'    => 'SQS:SendMessage',
+          'Resource'  => queue.arn,
+          'Condition' => {
+            'ArnEquals' => { 'aws:SourceArn' => topic.arn }
+          }
+        }
+      end
+    end
+  end
+end