propono 1.0.0.rc2 → 1.0.0.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3871cae55ac78a6aca5d6af4da0a2cf01b55b615
-  data.tar.gz: 04d9247f52f05800498896ae54df812e359ea982
+  metadata.gz: 87002e015e85197a83721a25009e5a941535ac1a
+  data.tar.gz: b9e14c3b2b00450d0accb003e6a48907c6e74a5b
 SHA512:
-  metadata.gz: 9a0e0cc2ab9b18be7a1c1b1c737803362d79a9a777e8402656a1153f007ec12394dfff9bf3dc3eac4b61fdd1850f406f2fec0aa76d288ce20ab789ddb09bd5a2
-  data.tar.gz: e7bff5c5e17d08fb007578c7a1c981c53d5fd3e0c290a723a78de9d005cca0396e9d8abed128e421d06d0f50695bff650e01632b5102bbbae0f9ccf69f1f4d2c
+  metadata.gz: 0a8351c78c0b9c97aa88fcbdc617ebcbc6b858c9746e6d57c21e2e1d6d660018eb52de04de61bf3a8ba1deb11c034fd8a954a2252189f8faef42b9903ce718ef
+  data.tar.gz: 7f1f640ddff9e72c7aaf4142bb7dca54c976ff33e184e99d329ede85087651c23e7e2e512166304c9871a1fb6772d89e163da84b948bd84c16a8ff7557521949

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 1.0.0.rc3 / 2013-12-20
+* [FEATURE] Create failed and corrupt queues when subscribe.
+
 # 1.0.0.rc2 / 2013-12-15
 * [FEATURE] Make queue_suffix optional
 

data/lib/propono/components/queue_subscription.rb

@@ -4,7 +4,7 @@ module Propono
     include Sns
     include Sqs
 
-    attr_reader :topic_arn, :queue_name, :queue
+    attr_reader :topic_arn, :queue_name, :queue, :failed_queue, :corrupt_queue
 
     def self.create(topic_id, options = {})
       new(topic_id, options).tap do |subscription|
@@ -15,13 +15,15 @@ module Propono
     def initialize(topic_id, options = {})
       @topic_id = topic_id
       @suffixed_topic_id = "#{topic_id}#{Propono.config.queue_suffix}"
-      @queue_name = "#{Propono.config.application_name.gsub(" ", "_")}-#{@suffixed_topic_id}#{options[:queue_name_suffix]}"
+      @queue_name = "#{Propono.config.application_name.gsub(" ", "_")}-#{@suffixed_topic_id}"
     end
 
     def create
       raise ProponoError.new("topic_id is nil") unless @topic_id
       @topic = TopicCreator.find_or_create(@suffixed_topic_id)
       @queue = QueueCreator.find_or_create(queue_name)
+      @failed_queue = QueueCreator.find_or_create("#{queue_name}-failed")
+      @corrupt_queue = QueueCreator.find_or_create("#{queue_name}-corrupt")
       sns.subscribe(@topic.arn, @queue.arn, 'sqs')
       sqs.set_queue_attributes(@queue.url, "Policy", generate_policy)
     end

data/lib/propono/components/sqs_message.rb

@@ -2,17 +2,33 @@ module Propono
   class SqsMessage
     include Sqs
 
-    attr_reader :context, :message, :raw_message, :receipt_handle
+    attr_reader :context, :message, :raw_message, :receipt_handle, :failure_count
     def initialize(raw_message)
-      body = JSON.parse(raw_message["Body"])["Message"]
-      body = JSON.parse(body)
+      raw_body = raw_message["Body"]
+      @raw_body_json = JSON.parse(raw_body)
+      body = JSON.parse(@raw_body_json["Message"])
 
       @raw_message    = raw_message
       @context        = body.symbolize_keys
+      @failure_count  = context[:num_failures] || 0
       @message        = context.delete(:message)
       @receipt_handle = raw_message["receipt_handle"]
     end
 
+    def to_json_with_exception(exception)
+      message = @raw_body_json.dup
+      context = {}
+      context[:id] = @context[:id]
+      context[:message] = @message
+      context[:last_exception_message] = exception.message
+      context[:last_exception_stack_trace] = exception.backtrace
+      context[:last_exception_time] = Time.now
+      context[:num_failures] = failure_count + 1
+      context[:last_context] = @context
+      message['Message'] = context.to_json
+      JSON.pretty_generate(message)
+    end
+
     def ==(other)
       other.is_a?(SqsMessage) && other.receipt_handle == @receipt_handle
     end

data/lib/propono/configuration.rb

@@ -10,7 +10,8 @@ module Propono
       :application_name,
       :udp_host, :udp_port,
       :tcp_host, :tcp_port,
-      :logger
+      :logger,
+      :max_retries
     ]
     attr_writer *SETTINGS
 
@@ -18,6 +19,7 @@ module Propono
       self.logger = Propono::Logger.new
       self.queue_suffix = ""
       self.use_iam_profile = false
+      self.max_retries = 0
     end
 
     SETTINGS.each do |setting|

data/lib/propono/services/queue_listener.rb

@@ -1,4 +1,5 @@
 module Propono
+
   class QueueListener
     include Sqs
 
@@ -36,7 +37,7 @@ module Propono
       raise $!
     rescue
       Propono.config.logger.error "Unexpected error reading from queue #{queue_url}"
-      Propono.config.logger.error $!
+      Propono.config.logger.error $!, $!.backtrace
     end
 
     # The calls to delete_message are deliberately duplicated so
@@ -44,35 +45,42 @@ module Propono
     # has completed succesfully. We do *not* want to ensure that the
     # message is deleted regardless of what happens in this method.
     def process_raw_message(raw_sqs_message)
-      begin
-        sqs_message = SqsMessage.new(raw_sqs_message)
+      sqs_message = parse(raw_sqs_message)
+      unless sqs_message.nil?
         Propono.config.logger.info "Propono [#{sqs_message.context[:id]}]: Received from sqs."
-
-        begin
-          process_message(sqs_message)
-          delete_message(raw_sqs_message)
-        rescue
-          move_to_failed_queue(sqs_message)
-          delete_message(raw_sqs_message)
-        end
-      rescue
-        move_to_corrupt_queue(raw_sqs_message)
+        handle(sqs_message)
         delete_message(raw_sqs_message)
       end
     end
 
+    def parse(raw_sqs_message)
+      SqsMessage.new(raw_sqs_message)
+    rescue
+      Propono.config.logger.error "Error parsing message, moving to corrupt queue", $!, $!.backtrace
+      move_to_corrupt_queue(raw_sqs_message)
+      delete_message(raw_sqs_message)
+      nil
+    end
+
+    def handle(sqs_message)
+      process_message(sqs_message)
+    rescue => e
+      Propono.config.logger.error("Failed to handle message #{e.message} #{e.backtrace}")
+      requeue_message_on_failure(sqs_message, e)
+    end
+
     def process_message(sqs_message)
       @message_processor.call(sqs_message.message, sqs_message.context)
     end
 
     def move_to_corrupt_queue(raw_sqs_message)
-      QueueSubscription.create(@topic_id, queue_name_suffix: "-corrupt")
-      Propono.publish("#{@topic_id}-corrupt", raw_sqs_message)
+      sqs.send_message(corrupt_queue_url, raw_sqs_message["Body"])
     end
 
-    def move_to_failed_queue(sqs_message)
-      QueueSubscription.create(@topic_id, queue_name_suffix: "-failed")
-      Propono.publish("#{@topic_id}-failed", sqs_message.message, id: sqs_message.context[:id])
+    def requeue_message_on_failure(sqs_message, exception)
+      next_queue = (sqs_message.failure_count < Propono.config.max_retries) ? queue_url : failed_queue_url
+      Propono.config.logger.error "Error proessing message, moving to queue: #{next_queue}"
+      sqs.send_message(next_queue, sqs_message.to_json_with_exception(exception))
     end
 
     def delete_message(raw_sqs_message)
@@ -83,6 +91,14 @@ module Propono
       @queue_url ||= subscription.queue.url
     end
 
+    def failed_queue_url
+      @failed_queue_url ||= subscription.failed_queue.url
+    end
+
+    def corrupt_queue_url
+      @corrupt_queue_url ||= subscription.corrupt_queue.url
+    end
+
     def subscription
       @subscription ||= QueueSubscription.create(@topic_id)
     end

data/lib/propono/version.rb

@@ -1,3 +1,3 @@
 module Propono
-  VERSION = "1.0.0.rc2"
+  VERSION = "1.0.0.rc3"
 end

data/test/components/queue_subscription_test.rb

@@ -26,8 +26,12 @@ module Propono
 
       TopicCreator.stubs(find_or_create: Topic.new("1123"))
 
+      queue_name = subscription.send(:queue_name)
+      
       sqs = mock()
-      sqs.expects(:create_queue).with(subscription.send(:queue_name)).returns(mock(body: {'QueueUrl' => Fog::AWS::SQS::Mock::QueueUrl}))
+      sqs.expects(:create_queue).with(queue_name).returns(mock(body: {'QueueUrl' => Fog::AWS::SQS::Mock::QueueUrl}))
+      sqs.expects(:create_queue).with(queue_name + '-failed').returns(mock(body: {'QueueUrl' => Fog::AWS::SQS::Mock::QueueUrl}))
+      sqs.expects(:create_queue).with(queue_name + '-corrupt').returns(mock(body: {'QueueUrl' => Fog::AWS::SQS::Mock::QueueUrl}))
       QueueCreator.any_instance.stubs(sqs: sqs)
 
       subscription.create
@@ -81,11 +85,18 @@ module Propono
 
     def test_create_saves_queue
       queue = Queue.new(Fog::AWS::SQS::Mock::QueueUrl)
+      failed_queue = Queue.new(Fog::AWS::SQS::Mock::QueueUrl)
+      corrupt_queue = Queue.new(Fog::AWS::SQS::Mock::QueueUrl)
 
-      QueueCreator.expects(:find_or_create).returns(queue)
-      subscription = QueueSubscription.new("Some Topic")
+      QueueCreator.expects(:find_or_create).with('MyApp-SomeTopic-suf').returns(queue)
+      QueueCreator.expects(:find_or_create).with('MyApp-SomeTopic-suf-failed').returns(failed_queue)
+      QueueCreator.expects(:find_or_create).with('MyApp-SomeTopic-suf-corrupt').returns(corrupt_queue)
+      subscription = QueueSubscription.new("SomeTopic")
       subscription.create
+      
       assert_equal queue, subscription.queue
+      assert_equal failed_queue, subscription.failed_queue
+      assert_equal corrupt_queue, subscription.corrupt_queue
     end
 
     def test_create_raises_with_nil_topic

data/test/configuration_test.rb

@@ -105,6 +105,16 @@ module Propono
         Propono.config.application_name
       end
     end
+
+    def test_default_max_retries
+      assert_equal 0, Propono.config.max_retries
+    end
+    
+    def test_max_retries
+      val = 5
+      Propono.config.max_retries = val
+      assert_equal 5, Propono.config.max_retries
+    end
   end
 end
 

data/test/integration/sns_to_sqs_test.rb

@@ -38,5 +38,61 @@ module Propono
     ensure
       thread.terminate
     end
+    
+=begin
+    
+
+    def test_failed_messge_is_transferred_to_failed_channel
+      topic = "test-topic"
+      text = "This is my message #{DateTime.now} #{rand()}"
+      flunks = []
+      message_received = false
+    
+      Propono.subscribe_by_queue(topic)
+    
+      thread = Thread.new do
+        begin
+          Propono.listen_to_queue(topic) do |message, context|
+            raise StandardError.new 'BOOM'
+          end
+        rescue => e
+          flunks << e.message
+        ensure
+          thread.terminate
+        end
+      end
+      
+      failure_listener = Thread.new do
+        begin
+          Propono.listen_to_queue(topic, channel: :failed) do |message, context|
+            flunks << "Wrong message" unless message == text
+            flunks << "Wrong id" unless context[:id] =~ Regexp.new("[a-z0-9]{6}")
+            message_received = true
+          end
+        rescue => e
+          flunks << e.message
+        ensure
+          thread.terminate
+        end
+      end
+    
+      Thread.new do
+        sleep(1) while !message_received
+        sleep(5) # Make sure all the message deletion clear up in the thread has happened
+        thread.terminate
+        failure_listener.terminate
+      end
+    
+      sleep(1) # Make sure the listener has started
+    
+      Propono.publish(topic, text)
+      flunks << "Test Timeout" unless wait_for_thread(thread)
+      flunk(flunks.join("\n")) unless flunks.empty?
+    ensure
+      thread.terminate
+    end
+
+=end
+    
   end
 end

data/test/services/queue_listener_test.rb

@@ -25,6 +25,8 @@ module Propono
 
       @listener = QueueListener.new(@topic_id) {}
       @listener.stubs(sqs: @sqs)
+      
+      Propono.config.max_retries = 0
     end
 
     def test_listen_should_loop
@@ -129,16 +131,31 @@ module Propono
       @sqs.expects(:delete_message).with(queue_url, @receipt_handle1)
       @sqs.expects(:delete_message).with(queue_url, @receipt_handle2)
 
-      @listener = QueueListener.new(@topic_id) { raise StandardError.new("Test Error") }
+      exception = StandardError.new("Test Error")
+      @listener = QueueListener.new(@topic_id) { raise exception }
       @listener.stubs(queue_url: queue_url)
       @listener.stubs(sqs: @sqs)
+      @listener.stubs(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message1), exception)
+      @listener.stubs(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message2), exception)
+      @listener.send(:read_messages)
+    end
+    
+    def test_messages_are_retried_or_abandoned_on_failure
+      exception = StandardError.new("Test Error")
+      @listener = QueueListener.new(@topic_id) { raise exception }
+      @listener.expects(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message1), exception)
+      @listener.expects(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message2), exception)
+      @listener.stubs(sqs: @sqs)
       @listener.send(:read_messages)
     end
 
-    def test_messages_are_moved_to_failed_queue_if_there_is_an_exception
-      @listener = QueueListener.new(@topic_id) { raise StandardError.new("Test Error") }
-      @listener.expects(:move_to_failed_queue).with(SqsMessage.new(@sqs_message1))
-      @listener.expects(:move_to_failed_queue).with(SqsMessage.new(@sqs_message2))
+    def test_failed_on_moving_to_failed_queue_does_not_delete
+      exception = StandardError.new("Test Error")
+      @listener = QueueListener.new(@topic_id) { raise exception }
+      @listener.stubs(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message1), exception).raises(StandardError.new("failed to move"))
+      @listener.stubs(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message2), exception).raises(StandardError.new("failed to move"))
+      @listener.expects(:delete_message).with(@sqs_message1).never
+      @listener.expects(:delete_message).with(@sqs_message2).never
       @listener.stubs(sqs: @sqs)
       @listener.send(:read_messages)
     end
@@ -154,15 +171,29 @@ module Propono
       @listener.send(:read_messages)
     end
 
-    def test_move_to_failed_queue
-      QueueSubscription.expects(:create).with(@topic_id, queue_name_suffix: "-failed")
-      Propono.expects(:publish).with("#{@topic_id}-failed", @message1, id: @message1_id)
-      @listener.send(:move_to_failed_queue, SqsMessage.new(@sqs_message1))
+    def test_message_moved_to_failed_queue_if_there_is_an_exception_and_retry_count_is_zero
+      @sqs.expects(:send_message).with(regexp_matches(/https:\/\/queue.amazonaws.com\/[0-9]+\/MyApp-some-topic-failed/), anything)
+      @listener.send(:requeue_message_on_failure, SqsMessage.new(@sqs_message1), StandardError.new)
+    end
+    
+    def test_message_requeued_if_there_is_an_exception_but_failure_count_less_than_retry_count
+      Propono.config.max_retries = 5
+      message = SqsMessage.new(@sqs_message1)
+      message.stubs(failure_count: 4)
+      @sqs.expects(:send_message).with(regexp_matches(/https:\/\/queue.amazonaws.com\/[0-9]+\/MyApp-some-topic$/), anything)
+      @listener.send(:requeue_message_on_failure, message, StandardError.new)
+    end
+    
+    def test_message_requeued_if_there_is_an_exception_but_failure_count_exceeds_than_retry_count
+      Propono.config.max_retries = 5
+      message = SqsMessage.new(@sqs_message1)
+      message.stubs(failure_count: 5)
+      @sqs.expects(:send_message).with(regexp_matches(/https:\/\/queue.amazonaws.com\/[0-9]+\/MyApp-some-topic-failed/), anything)
+      @listener.send(:requeue_message_on_failure, message, StandardError.new)
     end
 
     def test_move_to_corrupt_queue
-      QueueSubscription.expects(:create).with(@topic_id, queue_name_suffix: "-corrupt")
-      Propono.expects(:publish).with("#{@topic_id}-corrupt", @sqs_message1)
+      @sqs.expects(:send_message).with(regexp_matches(/https:\/\/queue.amazonaws.com\/[0-9]+\/MyApp-some-topic-corrupt/), anything)
       @listener.send(:move_to_corrupt_queue, @sqs_message1)
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: propono
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc2
+  version: 1.0.0.rc3
 platform: ruby
 authors:
 - MalcyL
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-16 00:00:00.000000000 Z
+date: 2013-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog