propono 1.7.0 → 2.0.0.rc1

data/lib/propono/components/aws_config.rb

@@ -1,11 +1,6 @@
 module Propono
-
-  def self.aws_options
-    AwsConfig.new(Propono.config).aws_options
-  end
-  
   class AwsConfig
-    
+
     def initialize(config)
       @config = config
     end
@@ -18,11 +13,11 @@ module Propono
         }
       else
         {
-          :aws_access_key_id => @config.access_key,
-          :aws_secret_access_key => @config.secret_key,
+          :access_key_id => @config.access_key,
+          :secret_access_key => @config.secret_key,
           :region => @config.queue_region
         }
       end
     end
   end
-end
+end

data/lib/propono/components/client.rb

@@ -0,0 +1,93 @@
+module Propono
+  class Client
+
+    # Propono configuration settings.
+    #
+    # Settings should be set in an initializer or using some
+    # other method that insures they are set before any
+    # Propono code is used. They can be set as followed:
+    #
+    #   Propono.config.access_key = "my-access-key"
+    #
+    # The following settings are allowed:
+    #
+    # * <tt>:access_key</tt> - The AWS access key
+    # * <tt>:secret_key</tt> - The AWS secret key
+    # * <tt>:queue_region</tt> - The AWS region
+    # * <tt>:application_name</tt> - The name of the application Propono
+    #   is included in.
+    # * <tt>:queue_suffix</tt> - Optional string to append to topic and queue names.
+    # * <tt>:logger</tt> - A logger object that responds to puts.
+
+    attr_reader :config, :aws_client
+    def initialize(settings = {}, &block)
+      @config = Configuration.new
+      if block_given?
+        configure(&block)
+      else
+        settings.each do |key, value|
+          config.send("#{key}=", value)
+        end
+      end
+
+      @aws_client = AwsClient.new(AwsConfig.new(config))
+    end
+
+    def configure(&block)
+      yield config
+    end
+
+    # Publishes a new message into the Propono pub/sub network.
+    #
+    # This requires a topic and a message. By default this pushes
+    # out AWS SNS.
+    #
+    # @param [String] topic The name of the topic to publish to.
+    # @param [String] message The message to post.
+    def publish(topic, message, options = {})
+      suffixed_topic = "#{topic}#{config.queue_suffix}"
+      Publisher.publish(aws_client, config, suffixed_topic, message, options)
+    end
+
+    # Creates a new SNS-SQS subscription on the specified topic.
+    #
+    # This is implicitly called by {#listen_to_queue}.
+    #
+    # @param [String] topic The name of the topic to subscribe to.
+    def subscribe(topic)
+      QueueSubscription.create(aws_client, config, topic)
+    end
+
+    # Listens on a queue and yields for each message
+    #
+    # Calling this will enter a queue-listening loop that
+    # yields the message_processor for each messages.
+    #
+    # This method will automatically create a subscription if
+    # one does not exist, so there is no need to call
+    # <tt>subscribe</tt> in addition.
+    #
+    # @param [String] topic The topic to subscribe to.
+    # @param &message_processor The block to yield for each message.
+    def listen(topic_name, &message_processor)
+      QueueListener.listen(aws_client, config, topic_name, &message_processor)
+    end
+
+    # Listens on a queue and yields for each message
+    #
+    # Calling this will enter a queue-listening loop that
+    # yields the message_processor for each messages.  The
+    # loop will end when all messages have been processed.
+    #
+    # This method will automatically create a subscription if
+    # one does not exist, so there is no need to call
+    # <tt>subscribe</tt> in addition.
+    #
+    # @param [String] topic The topic to subscribe to.
+    # @param &message_processor The block to yield for each message.
+    def drain_queue(topic, &message_processor)
+      QueueListener.drain(aws_client, config, topic, &message_processor)
+    end
+  end
+end
+

data/lib/propono/components/queue.rb

@@ -1,16 +1,14 @@
 module Propono
   class Queue
 
-    include Sqs
-
-    attr_reader :url
-    def initialize(url)
+    attr_reader :url, :attributes
+    def initialize(url, attributes)
       @url = url
+      @attributes = attributes
     end
 
     def arn
-      attributes = sqs.get_queue_attributes(@url, 'QueueArn').body["Attributes"]
-      attributes["QueueArn"]
+      @arn ||= attributes["QueueArn"]
     end
   end
 end

data/lib/propono/components/queue_subscription.rb

@@ -1,37 +1,47 @@
 module Propono
   class QueueSubscription
 
-    include Sns
-    include Sqs
+    attr_reader :aws_client, :propono_config, :topic_arn, :queue_name, :queue, :failed_queue, :corrupt_queue, :slow_queue
 
-    attr_reader :topic_arn, :queue_name, :queue, :failed_queue, :corrupt_queue, :slow_queue
-
-    def self.create(topic_id, options = {})
-      new(topic_id, options).tap do |subscription|
+    def self.create(*args)
+      new(*args).tap do |subscription|
         subscription.create
       end
     end
 
-    def initialize(topic_id, options = {})
-      @topic_id = topic_id
-      @suffixed_topic_id = "#{topic_id}#{Propono.config.queue_suffix}"
-      @suffixed_slow_topic_id = "#{topic_id}#{Propono.config.queue_suffix}-slow"
-      @queue_name = "#{Propono.config.application_name.gsub(" ", "_")}-#{@suffixed_topic_id}"
+    def initialize(aws_client, propono_config, topic_name)
+      @aws_client = aws_client
+      @propono_config = propono_config
+      @topic_name = topic_name
+      @suffixed_topic_name = "#{topic_name}#{propono_config.queue_suffix}"
+      @suffixed_slow_topic_name = "#{topic_name}#{propono_config.queue_suffix}-slow"
+      @queue_name = "#{propono_config.application_name.gsub(" ", "_")}-#{@suffixed_topic_name}"
     end
 
     def create
-      raise ProponoError.new("topic_id is nil") unless @topic_id
-      @topic = TopicCreator.find_or_create(@suffixed_topic_id)
-      @queue = QueueCreator.find_or_create(queue_name)
-      @failed_queue = QueueCreator.find_or_create("#{queue_name}-failed")
-      @corrupt_queue = QueueCreator.find_or_create("#{queue_name}-corrupt")
-      sns.subscribe(@topic.arn, @queue.arn, 'sqs')
-      sqs.set_queue_attributes(@queue.url, "Policy", generate_policy(@queue, @topic))
+      raise ProponoError.new("topic_name is nil") unless @topic_name
+      create_and_subscribe_main_queue
+      create_and_subscribe_slow_queue
+      create_misc_queues
+    end
+
+    def create_and_subscribe_main_queue
+      @queue = aws_client.create_queue(queue_name)
+      topic = aws_client.create_topic(@suffixed_topic_name)
+      aws_client.subscribe_sqs_to_sns(@queue, topic)
+      aws_client.set_sqs_policy(@queue, generate_policy(@queue, topic))
+    end
+
+    def create_misc_queues
+      @failed_queue = aws_client.create_queue("#{queue_name}-failed")
+      @corrupt_queue = aws_client.create_queue("#{queue_name}-corrupt")
+    end
 
-      @slow_queue = QueueCreator.find_or_create("#{queue_name}-slow")
-      @slow_topic = TopicCreator.find_or_create(@suffixed_slow_topic_id)
-      sns.subscribe(@slow_topic.arn, @slow_queue.arn, 'sqs')
-      sqs.set_queue_attributes(@slow_queue.url, "Policy", generate_policy(@slow_queue, @slow_topic))
+    def create_and_subscribe_slow_queue
+      @slow_queue = aws_client.create_queue("#{queue_name}-slow")
+      slow_topic = aws_client.create_topic(@suffixed_slow_topic_name)
+      aws_client.subscribe_sqs_to_sns(@slow_queue, slow_topic)
+      aws_client.set_sqs_policy(@slow_queue, generate_policy(@slow_queue, slow_topic))
     end
 
     private

data/lib/propono/components/sqs_message.rb

@@ -1,18 +1,15 @@
 module Propono
   class SqsMessage
-    include Sqs
-
-    attr_reader :context, :message, :raw_message, :receipt_handle, :failure_count
+    attr_reader :context, :message, :receipt_handle, :failure_count
     def initialize(raw_message)
-      raw_body = raw_message["Body"]
+      raw_body = raw_message.body
       @raw_body_json = JSON.parse(raw_body)
       body = JSON.parse(@raw_body_json["Message"])
 
-      @raw_message    = raw_message
       @context        = Propono::Utils.symbolize_keys body
       @failure_count  = context[:num_failures] || 0
       @message        = context.delete(:message)
-      @receipt_handle = raw_message["receipt_handle"]
+      @receipt_handle = raw_message.receipt_handle
     end
 
     def to_json_with_exception(exception)

data/lib/propono/components/topic.rb

@@ -1,11 +1,12 @@
 module Propono
   class Topic
+    attr_reader :aws_topic
+    def initialize(aws_topic)
+      @aws_topic = aws_topic
+    end
 
-    include Sqs
-
-    attr_reader :arn
-    def initialize(arn)
-      @arn = arn
+    def arn
+      @arn ||= aws_topic.topic_arn
     end
   end
 end

data/lib/propono/configuration.rb

@@ -8,8 +8,6 @@ module Propono
     SETTINGS = [
       :use_iam_profile, :access_key, :secret_key, :queue_region, :queue_suffix,
       :application_name,
-      :udp_host, :udp_port,
-      :tcp_host, :tcp_port,
       :logger,
       :max_retries, :num_messages_per_poll
     ]

data/lib/propono/services/publisher.rb

@@ -5,76 +5,53 @@ module Propono
   end
 
   class Publisher
-    include Sns
-
-    def self.publish(topic_id, message, options = {})
-      new(topic_id, message, options).publish
+    def self.publish(*args)
+      new(*args).publish
     end
 
-    attr_reader :topic_id, :message, :protocol, :id, :async
+    attr_reader :aws_client, :propono_config, :topic_name, :message, :id, :async
 
-    def initialize(topic_id, message, options = {})
-      raise PublisherError.new("Topic is nil") if topic_id.nil?
+    def initialize(aws_client, propono_config, topic_name, message, async: false, id: nil)
+      raise PublisherError.new("Topic is nil") if topic_name.nil?
       raise PublisherError.new("Message is nil") if message.nil?
 
-      options = Propono::Utils.symbolize_keys options
-
-      @topic_id = topic_id
+      @aws_client = aws_client
+      @propono_config = propono_config
+      @topic_name = topic_name
       @message = message
-      @protocol = options.fetch(:protocol, :sns).to_sym
-      @id = SecureRandom.hex(3)
-      @id = "#{options[:id]}-#{@id}" if options[:id]
-      @async = options.fetch(:async, true)
+      @async = async
+
+      random_id = SecureRandom.hex(3)
+      @id = id ? "#{id}-#{random_id}" : random_id
     end
 
     def publish
-      Propono.config.logger.info "Propono [#{id}]: Publishing #{message} to #{topic_id} via #{protocol}"
-      send("publish_via_#{protocol}")
+      propono_config.logger.info "Propono [#{id}]: Publishing #{message} to #{topic_name}"
+      async ? publish_asyncronously : publish_syncronously
     end
 
     private
 
-    def publish_via_sns
-      async ? publish_via_sns_asyncronously : publish_via_sns_syncronously
-    end
-
-    def publish_via_sns_asyncronously
-      Thread.new { publish_via_sns_syncronously }
+    def publish_asyncronously
+      Thread.new { publish_syncronously }
     end
 
-    def publish_via_sns_syncronously
+    def publish_syncronously
       begin
-        topic = TopicCreator.find_or_create(topic_id)
+        topic = aws_client.create_topic(topic_name)
       rescue => e
-        Propono.config.logger.error "Propono [#{id}]: Failed to create topic #{topic_id}: #{e}"
+        propono_config.logger.error "Propono [#{id}]: Failed to get or create topic #{topic_name}: #{e}"
         raise
       end
 
       begin
-        sns.publish(topic.arn, body.to_json)
+        aws_client.publish_to_sns(topic, body)
       rescue => e
-        Propono.config.logger.error "Propono [#{id}]: Failed to send via sns: #{e}"
+        propono_config.logger.error "Propono [#{id}]: Failed to send via sns: #{e}"
         raise
       end
     end
 
-    def publish_via_udp
-      payload = body.merge(topic: topic_id).to_json
-      UDPSocket.new.send(payload, 0, Propono.config.udp_host, Propono.config.udp_port)
-    rescue => e
-      Propono.config.logger.error "Propono [#{id}]: Failed to send : #{e}"
-    end
-
-    def publish_via_tcp
-      payload = body.merge(topic: topic_id).to_json
-
-      socket = TCPSocket.new(Propono.config.tcp_host, Propono.config.tcp_port)
-      socket.write payload
-      socket.close
-    rescue => e
-      Propono.config.logger.error "Propono [#{id}]: Failed to send : #{e}"
-    end
-
     def body
       {
         id: id,

data/lib/propono/services/queue_listener.rb

@@ -1,127 +1,124 @@
 module Propono
 
   class QueueListener
-    include Sqs
-
-    def self.listen(topic_id, &message_processor)
-      new(topic_id, &message_processor).listen
+    def self.listen(*args, &message_processor)
+      new(*args, &message_processor).listen
     end
 
-    def self.drain(topic_id, &message_processor)
-      new(topic_id, &message_processor).drain
+    def self.drain(*args, &message_processor)
+      new(*args, &message_processor).drain
     end
 
-    def initialize(topic_id, &message_processor)
-      @topic_id = topic_id
+    attr_reader :aws_client, :propono_config, :topic_name, :message_processor
+    def initialize(aws_client, propono_config, topic_name, &message_processor)
+      @aws_client = aws_client
+      @propono_config = propono_config
+      @topic_name = topic_name
       @message_processor = message_processor
     end
 
     def listen
-      raise ProponoError.new("topic_id is nil") unless @topic_id
+      raise ProponoError.new("topic_name is nil") unless topic_name
       loop do
-        unless read_messages
-          sleep 10
-        end
+        read_messages
       end
     end
 
     def drain
-      raise ProponoError.new("topic_id is nil") unless @topic_id
-      while read_messages
-        # continue as long as there are messages in the queue
-      end
+      raise ProponoError.new("topic_name is nil") unless topic_name
+      true while read_messages_from_queue(main_queue, 10, long_poll: false)
+      true while read_messages_from_queue(slow_queue, 10, long_poll: false)
     end
 
     private
 
     def read_messages
-      read_messages_from_queue(main_queue_url, Propono.config.num_messages_per_poll) || read_messages_from_queue(slow_queue_url, 1)
+      read_messages_from_queue(main_queue, propono_config.num_messages_per_poll) ||
+      read_messages_from_queue(slow_queue, 1)
     end
 
-    def read_messages_from_queue(queue_url, num_messages)
-      response = sqs.receive_message(queue_url, {'MaxNumberOfMessages' => num_messages} )
-      messages = response.body['Message']
+    def read_messages_from_queue(queue, num_messages, long_poll: true)
+      messages = aws_client.read_from_sqs(queue, num_messages, long_poll: long_poll)
       if messages.empty?
         false
       else
-        messages.each { |msg| process_raw_message(msg, queue_url) }
+        messages.each { |msg| process_raw_message(msg, queue) }
+        true
       end
-    rescue Excon::Errors::Forbidden
-      Propono.config.logger.error "Forbidden error caught and re-raised. #{queue_url}"
-      Propono.config.logger.error $!
-      raise $!
-    rescue
-      Propono.config.logger.error "Unexpected error reading from queue #{queue_url}"
-      Propono.config.logger.error $!
-      Propono.config.logger.error $!.backtrace
+    rescue => e #Aws::Errors => e
+      propono_config.logger.error "Unexpected error reading from queue #{queue.url}"
+      propono_config.logger.error e.class.name
+      propono_config.logger.error e.message
+      propono_config.logger.error e.backtrace
+      false
     end
 
     # The calls to delete_message are deliberately duplicated so
     # as to ensure the message is only deleted if the preceeding line
     # has completed succesfully. We do *not* want to ensure that the
     # message is deleted regardless of what happens in this method.
-    def process_raw_message(raw_sqs_message, queue_url)
-      sqs_message = parse(raw_sqs_message, queue_url)
-      unless sqs_message.nil?
-        Propono.config.logger.info "Propono [#{sqs_message.context[:id]}]: Received from sqs."
-        handle(sqs_message)
-        delete_message(raw_sqs_message, queue_url)
-      end
+    def process_raw_message(raw_sqs_message, queue)
+      sqs_message = parse(raw_sqs_message, queue)
+      return unless sqs_message
+
+      propono_config.logger.info "Propono [#{sqs_message.context[:id]}]: Received from sqs."
+      handle(sqs_message)
+      aws_client.delete_from_sqs(queue, sqs_message.receipt_handle)
     end
 
-    def parse(raw_sqs_message, queue_url)
+    def parse(raw_sqs_message, queue)
       SqsMessage.new(raw_sqs_message)
     rescue
-      Propono.config.logger.error "Error parsing message, moving to corrupt queue", $!, $!.backtrace
+      propono_config.logger.error "Error parsing message, moving to corrupt queue", $!, $!.backtrace
       move_to_corrupt_queue(raw_sqs_message)
-      delete_message(raw_sqs_message, queue_url)
+      aws_client.delete_from_sqs(queue, raw_sqs_message.receipt_handle)
       nil
     end
 
     def handle(sqs_message)
       process_message(sqs_message)
     rescue => e
-      Propono.config.logger.error("Failed to handle message #{e.message} #{e.backtrace}")
+      propono_config.logger.error("Failed to handle message #{e.message} #{e.backtrace}")
       requeue_message_on_failure(sqs_message, e)
     end
 
     def process_message(sqs_message)
-      @message_processor.call(sqs_message.message, sqs_message.context)
+      return false unless message_processor
+      message_processor.call(sqs_message.message, sqs_message.context)
     end
 
     def move_to_corrupt_queue(raw_sqs_message)
-      sqs.send_message(corrupt_queue_url, raw_sqs_message["Body"])
+      aws_client.send_to_sqs(corrupt_queue, raw_sqs_message.body)
     end
 
     def requeue_message_on_failure(sqs_message, exception)
-      next_queue = (sqs_message.failure_count < Propono.config.max_retries) ? main_queue_url : failed_queue_url
-      Propono.config.logger.error "Error proessing message, moving to queue: #{next_queue}"
-      sqs.send_message(next_queue, sqs_message.to_json_with_exception(exception))
+      next_queue = (sqs_message.failure_count < propono_config.max_retries) ? main_queue : failed_queue
+      propono_config.logger.error "Error proessing message, moving to queue: #{next_queue}"
+      aws_client.send_to_sqs(next_queue, sqs_message.to_json_with_exception(exception))
     end
 
-    def delete_message(raw_sqs_message, queue_url)
-      sqs.delete_message(queue_url, raw_sqs_message['ReceiptHandle'])
+    def delete_message(raw_sqs_message, queue)
+      aws_client.delete_from_sqs(queue, raw_sqs_message.receipt_handle)
     end
 
-    def main_queue_url
-      @main_queue_url ||= subscription.queue.url
+    def main_queue
+      @main_queue ||= subscription.queue
     end
 
-    def failed_queue_url
-      @failed_queue_url ||= subscription.failed_queue.url
+    def failed_queue
+      @failed_queue ||= subscription.failed_queue
     end
 
-    def corrupt_queue_url
-      @corrupt_queue_url ||= subscription.corrupt_queue.url
+    def corrupt_queue
+      @corrupt_queue ||= subscription.corrupt_queue
     end
 
-    def slow_queue_url
-      @slow_queue_url ||= subscription.slow_queue.url
+    def slow_queue
+      @slow_queue ||= subscription.slow_queue
     end
 
     def subscription
-      @subscription ||= QueueSubscription.create(@topic_id)
+      QueueSubscription.create(aws_client, propono_config, topic_name)
     end
-
   end
 end