circuitry 1.4.1 → 2.0.0

data/lib/circuitry/cli.rb

@@ -0,0 +1,70 @@
+require 'circuitry/provisioner'
+require 'thor'
+
+module Circuitry
+  class CLI < Thor
+    class_option :verbose, aliases: :v, type: :boolean
+
+    desc 'provision <queue> -t <topic> [<topic> ...]', 'Provision a queue subscribed to one or more topics'
+
+    long_desc <<-END
+      Creates an SQS queue with appropriate SNS access policy along with one or more SNS topics
+      named <topic> that has an SQS subscription for each.
+
+      When the queue already exists, its policy will be added or updated.
+
+      When a topic already exists, it will be ignored.
+
+      When a topic subscription already exists, it will be ignored.
+
+      If no dead letter queue is specified, one will be created by default with the
+      name <queue>-failures
+    END
+
+    option :topics, aliases: :t, type: :array, required: :true
+    option :access_key, aliases: :a
+    option :secret_key, aliases: :s
+    option :dead_letter_queue, aliases: :d
+    option :region, aliases: :r
+
+    def provision(queue_name)
+      with_custom_config(queue_name) do |config|
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::INFO if options['verbose']
+        Circuitry::Provisioner.new(config, logger: logger).run
+      end
+    end
+
+    private
+
+    def say(*args)
+      puts(*args) if options['verbose']
+    end
+
+    def with_custom_config(queue_name, &block)
+      original_values = {}
+      %i[access_key secret_key region subscriber_queue_name subscriber_dead_letter_queue_name publisher_topic_names].each do |sym|
+        original_values[sym] = Circuitry.config.send(sym)
+      end
+
+      assign_options_config(queue_name, original_values)
+
+      block.call(Circuitry.config)
+    ensure
+      restore_config(original_values)
+    end
+
+    def assign_options_config(queue_name, original_values)
+      Circuitry.config.access_key = options.fetch('access_key', original_values[:access_key])
+      Circuitry.config.secret_key = options.fetch('secret_key', original_values[:secret_key])
+      Circuitry.config.region = options.fetch('region', original_values[:region])
+      Circuitry.config.subscriber_queue_name = queue_name
+      Circuitry.config.subscriber_dead_letter_queue_name = options.fetch('dead_letter_queue', "#{queue_name}-failures")
+      Circuitry.config.publisher_topic_names = options['topics']
+    end
+
+    def restore_config(original_values)
+      original_values.keys.each { |key| Circuitry.config.send(:"#{key}=", original_values[key]) }
+    end
+  end
+end

data/lib/circuitry/concerns/async.rb

@@ -29,11 +29,11 @@ module Circuitry
 
       def async=(value)
         value = case value
-          when false, nil then false
-          when true then self.class.default_async_strategy
-          when *self.class.async_strategies then value
-          else raise ArgumentError, "Invalid value `#{value.inspect}`, must be one of #{[true, false].concat(self.class.async_strategies).inspect}"
-        end
+                when false, nil then false
+                when true then self.class.default_async_strategy
+                when *self.class.async_strategies then value
+                else raise ArgumentError, async_value_error(value)
+                end
 
         if value == :fork && !platform_supports_forking?
           raise NotSupportedError, 'Your platform does not support forking'
@@ -43,11 +43,16 @@ module Circuitry
       end
 
       def async?
-        !!async
+        ![nil, false].include?(async)
       end
 
       private
 
+      def async_value_error(value)
+        options = [true, false].concat(self.class.async_strategies).inspect
+        "Invalid value `#{value.inspect}`, must be one of #{options}"
+      end
+
       def platform_supports_forking?
         Process.respond_to?(:fork)
       end

data/lib/circuitry/configuration.rb

@@ -5,14 +5,18 @@ module Circuitry
   class Configuration
     include Virtus::Model
 
+    attribute :subscriber_queue_name, String
+    attribute :subscriber_dead_letter_queue_name, String
+    attribute :publisher_topic_names, Array[String]
+
     attribute :access_key, String
     attribute :secret_key, String
     attribute :region, String, default: 'us-east-1'
     attribute :logger, Logger, default: Logger.new(STDERR)
     attribute :error_handler
-    attribute :lock_strategy, Object, default: ->(page, attribute) { Circuitry::Locks::Memory.new }
-    attribute :publish_async_strategy, Symbol, default: ->(page, attribute) { :fork }
-    attribute :subscribe_async_strategy, Symbol, default: ->(page, attribute) { :fork }
+    attribute :lock_strategy, Object, default: ->(_page, _attribute) { Circuitry::Locks::Memory.new }
+    attribute :publish_async_strategy, Symbol, default: ->(_page, _attribute) { :fork }
+    attribute :subscribe_async_strategy, Symbol, default: ->(_page, _attribute) { :fork }
     attribute :on_thread_exit
     attribute :on_fork_exit
 
@@ -26,20 +30,23 @@ module Circuitry
       super
     end
 
+    def subscriber_dead_letter_queue_name
+      super || "#{subscriber_queue_name}-failures"
+    end
+
     def aws_options
       {
-          access_key_id:     access_key,
-          secret_access_key: secret_key,
-          region:            region,
+        access_key_id:     access_key,
+        secret_access_key: secret_key,
+        region:            region
       }
     end
 
     private
 
     def validate(value, permitted_values)
-      unless permitted_values.include?(value)
-        raise ArgumentError, "invalid value `#{value}`, must be one of #{permitted_values.inspect}"
-      end
+      return if permitted_values.include?(value)
+      raise ArgumentError, "invalid value `#{value}`, must be one of #{permitted_values.inspect}"
     end
   end
 end

data/lib/circuitry/provisioner.rb

@@ -0,0 +1,59 @@
+require 'circuitry/queue_creator'
+require 'circuitry/topic_creator'
+require 'circuitry/subscription_creator'
+
+module Circuitry
+  class Provisioner
+    attr_reader :log
+    attr_reader :config
+
+    def initialize(config, logger: Logger.new(STDOUT))
+      @config = config
+      @log = logger
+    end
+
+    def run
+      queue = create_queue
+      topics = create_topics if queue
+      subscribe_topics(queue, topics) if queue && topics
+    end
+
+    private
+
+    def create_queue
+      safe_aws('Create Queue') do
+        queue = Circuitry::QueueCreator.find_or_create(
+          config.subscriber_queue_name,
+          dead_letter_queue_name: config.subscriber_dead_letter_queue_name
+        )
+        log.info "Created queue #{queue.url}"
+        queue
+      end
+    end
+
+    def create_topics
+      safe_aws('Create Topics') do
+        config.publisher_topic_names.map do |topic_name|
+          topic = Circuitry::TopicCreator.find_or_create(topic_name)
+          log.info "Created topic #{topic.name}"
+          topic
+        end
+      end
+    end
+
+    def subscribe_topics(queue, topics)
+      safe_aws('Subscribe Topics') do
+        Circuitry::SubscriptionCreator.subscribe_all(queue, topics)
+        log.info "Subscribed all topics to #{queue.name}"
+        true
+      end
+    end
+
+    def safe_aws(desc)
+      yield
+    rescue Aws::SQS::Errors::AccessDenied
+      log.fatal("#{desc}: Access denied. Check your configured credentials.")
+      nil
+    end
+  end
+end

data/lib/circuitry/publisher.rb

@@ -12,8 +12,8 @@ module Circuitry
     include Services::SNS
 
     DEFAULT_OPTIONS = {
-        async: false,
-        timeout: 15,
+      async: false,
+      timeout: 15
     }.freeze
 
     attr_reader :timeout
@@ -26,23 +26,14 @@ module Circuitry
     end
 
     def publish(topic_name, object)
-      raise ArgumentError.new('topic_name cannot be nil') if topic_name.nil?
-      raise ArgumentError.new('object cannot be nil') if object.nil?
-      raise PublishError.new('AWS configuration is not set') unless can_publish?
-
-      process = -> do
-        Timeout.timeout(timeout) do
-          logger.info("Publishing message to #{topic_name}")
-
-          topic = TopicCreator.find_or_create(topic_name)
-          sns.publish(topic_arn: topic.arn, message: object.to_json)
-        end
-      end
+      raise ArgumentError, 'topic_name cannot be nil' if topic_name.nil?
+      raise ArgumentError, 'object cannot be nil' if object.nil?
+      raise PublishError, 'AWS configuration is not set' unless can_publish?
 
       if async?
-        process_asynchronously(&process)
+        process_asynchronously(& -> { publish_internal(topic_name, object) })
       else
-        process.call
+        publish_internal(topic_name, object)
       end
     end
 
@@ -52,6 +43,17 @@ module Circuitry
 
     protected
 
+    def publish_internal(topic_name, object)
+      # TODO: Don't use ruby timeout.
+      # http://www.mikeperham.com/2015/05/08/timeout-rubys-most-dangerous-api/
+      Timeout.timeout(timeout) do
+        logger.info("Publishing message to #{topic_name}")
+
+        topic = TopicCreator.find_or_create(topic_name)
+        sns.publish(topic_arn: topic.arn, message: object.to_json)
+      end
+    end
+
     attr_writer :timeout
 
     private

data/lib/circuitry/queue.rb

@@ -0,0 +1,27 @@
+require 'circuitry/services/sqs'
+
+module Circuitry
+  class Queue
+    include Services::SQS
+
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+    end
+
+    def name
+      url.split('/').last
+    end
+
+    def arn
+      @arn ||= attribute('QueueArn')
+    end
+
+    private
+
+    def attribute(name)
+      sqs.get_queue_attributes(queue_url: url, attribute_names: [name]).attributes[name]
+    end
+  end
+end

data/lib/circuitry/queue_creator.rb

@@ -0,0 +1,50 @@
+require 'circuitry/services/sqs'
+require 'circuitry/queue'
+
+module Circuitry
+  class QueueCreator
+    include Services::SQS
+
+    attr_reader :queue_name
+
+    def self.find_or_create(queue_name, dead_letter_queue_name: nil, max_receive_count: 8 )
+      creator = new(queue_name)
+      result = creator.create_queue
+      creator.create_dead_letter_queue(dead_letter_queue_name, max_receive_count) if dead_letter_queue_name
+      result
+    end
+
+    def initialize(queue_name)
+      @queue_name = queue_name
+    end
+
+    def create_queue
+      @_queue ||= Queue.new(create_primary_queue_internal)
+    end
+
+    def create_dead_letter_queue(name, max_receive_count)
+      @_dl_queue ||= Queue.new(create_dl_queue_internal(name, max_receive_count))
+    end
+
+    private
+
+    def create_dl_queue_internal(name, max_receive_count)
+      dl_url = sqs.create_queue(queue_name: name).queue_url
+      dl_arn = sqs.get_queue_attributes(queue_url: dl_url, attribute_names: ['QueueArn']).attributes['QueueArn']
+
+      sqs.set_queue_attributes(queue_url: create_queue.url, attributes: build_redrive_policy(dl_arn, max_receive_count))
+      dl_url
+    end
+
+    def build_redrive_policy(deadletter_arn, max_receive_count)
+      {
+        'RedrivePolicy' => %({"maxReceiveCount":"#{max_receive_count}", "deadLetterTargetArn":"#{deadletter_arn}"})
+      }
+    end
+
+    def create_primary_queue_internal
+      attributes = { 'VisibilityTimeout' => (30 * 60).to_s }
+      sqs.create_queue(queue_name: queue_name, attributes: attributes).queue_url
+    end
+  end
+end

data/lib/circuitry/railtie.rb

@@ -0,0 +1,9 @@
+require 'rails'
+
+module Circuitry
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load 'circuitry/tasks.rb'
+    end
+  end
+end

data/lib/circuitry/subscriber.rb

@@ -14,36 +14,33 @@ module Circuitry
     attr_reader :queue, :timeout, :wait_time, :batch_size, :lock
 
     DEFAULT_OPTIONS = {
-        lock: true,
-        async: false,
-        timeout: 15,
-        wait_time: 10,
-        batch_size: 10,
+      lock: true,
+      async: false,
+      timeout: 15,
+      wait_time: 10,
+      batch_size: 10
     }.freeze
 
     CONNECTION_ERRORS = [
-        Aws::SQS::Errors::ServiceError,
+      Aws::SQS::Errors::ServiceError
     ].freeze
 
-    def initialize(queue, options = {})
-      raise ArgumentError.new('queue cannot be nil') if queue.nil?
-
+    def initialize(options = {})
       options = DEFAULT_OPTIONS.merge(options)
 
       self.subscribed = false
-      self.queue = queue
-      self.lock = options[:lock]
-      self.async = options[:async]
-      self.timeout = options[:timeout]
-      self.wait_time = options[:wait_time]
-      self.batch_size = options[:batch_size]
+
+      self.queue = QueueCreator.find_or_create(Circuitry.config.subscriber_queue_name).url
+      %i[lock async timeout wait_time batch_size].each do |sym|
+        send(:"#{sym}=", options[sym])
+      end
 
       trap_signals
     end
 
     def subscribe(&block)
-      raise ArgumentError.new('block required') if block.nil?
-      raise SubscribeError.new('AWS configuration is not set') unless can_subscribe?
+      raise ArgumentError, 'block required' if block.nil?
+      raise SubscribeError, 'AWS configuration is not set' unless can_subscribe?
 
       logger.info("Subscribing to queue: #{queue}")
 
@@ -54,7 +51,7 @@ module Circuitry
       logger.info("Unsubscribed from queue: #{queue}")
     rescue *CONNECTION_ERRORS => e
       logger.error("Connection error to queue: #{queue}: #{e}")
-      raise SubscribeError.new(e)
+      raise SubscribeError, e.message
     end
 
     def subscribed?
@@ -76,17 +73,22 @@ module Circuitry
 
     def lock=(value)
       value = case value
-        when true then Circuitry.config.lock_strategy
-        when false then Circuitry::Locks::NOOP.new
-        when Circuitry::Locks::Base then value
-        else raise ArgumentError, "Invalid value `#{value}`, must be one of `true`, `false`, or instance of `#{Circuitry::Locks::Base}`"
-      end
+              when true then Circuitry.config.lock_strategy
+              when false then Circuitry::Locks::NOOP.new
+              when Circuitry::Locks::Base then value
+              else raise ArgumentError, lock_value_error(value)
+              end
 
       @lock = value
     end
 
     private
 
+    def lock_value_error(value)
+      opts = Circuitry::Locks::Base
+      "Invalid value `#{value}`, must be one of `true`, `false`, or instance of `#{opts}`"
+    end
+
     def trap_signals
       trap('SIGINT') do
         if subscribed?
@@ -109,19 +111,21 @@ module Circuitry
     end
 
     def process_messages(messages, &block)
-      messages.each do |message|
-        process = -> do
-          process_message(message, &block)
-        end
-
-        if async?
-          process_asynchronously(&process)
-        else
-          process.call
-        end
+      if async?
+        process_messages_asynchronously(messages, &block)
+      else
+        process_messages_synchronously(messages, &block)
       end
     end
 
+    def process_messages_asynchronously(messages, &block)
+      messages.each { |message| process_asynchronously(& -> { process_message(message, &block) }) }
+    end
+
+    def process_messages_synchronously(messages, &block)
+      messages.each { |message| process_message(message, &block) }
+    end
+
     def process_message(message, &block)
       message = Message.new(message)
 
@@ -134,20 +138,35 @@ module Circuitry
     end
 
     def handle_message(message, &block)
-      if lock.soft_lock(message.id)
+      handled = try_with_lock(message.id) do
         begin
+          # TODO: Don't use ruby timeout.
+          # http://www.mikeperham.com/2015/05/08/timeout-rubys-most-dangerous-api/
           Timeout.timeout(timeout) do
             block.call(message.body, message.topic.name)
           end
         rescue => e
-          lock.unlock(message.id)
           logger.error("Error handling message #{message.id}: #{e}")
           raise e
         end
+      end
+
+      logger.info("Ignoring duplicate message #{message.id}") unless handled
+    end
+
+    def try_with_lock(handle)
+      if lock.soft_lock(handle)
+        begin
+          yield
+        rescue => e
+          lock.unlock(handle)
+          raise e
+        end
 
-        lock.hard_lock(message.id)
+        lock.hard_lock(handle)
+        true
       else
-        logger.info("Ignoring duplicate message #{message.id}")
+        false
       end
     end