propono 0.11.1 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8193ab42cb462b70c436b83459abb2df0203f6be
-  data.tar.gz: 225ec5fc62711fee56fe6520ba4bd745fb8a5aca
+  metadata.gz: 86ff2b93684d9aa043023ff05f81ccebcc6e108c
+  data.tar.gz: 7da5b3c1461cb3602e56dff51bcd97343e776262
 SHA512:
-  metadata.gz: 75c925d5b89a45630c70e9eb35e558db179eb77caf8a6c243dd9ce827da1e390e825a8205f5df9563e5ab5ab1b510f2d31fea1a0683188cdfdc64e85d102e51e
-  data.tar.gz: 7ccad01b84956d6a6a3dc22475fc26b0a839f7de0a1a866c655d06da75d300116d2f48c658f3821dc4d804441058c07e79ab3fbe5467abe90738951fc20aeaa6
+  metadata.gz: 38bd2565e6e0a0b9eb328d329b8d890ed13def10331792b5dcae86eafbc8535143faf7734a3607f45290ace56e1fc72d8e7559ed4d894348bb66aec12636758b
+  data.tar.gz: f2f63928f0f4b565636abf7504706b08dfead8cbe0d71b75946289824ff79992bae05bb526ff5d3e977da3735fec7577a7952d3ab5fb7391aa131d3885f4e49a

data/CHANGELOG.md

@@ -1,47 +1,39 @@
-# 0.11.1 / 2013-12-09
+# 1.0.0.rc1 / 2013-12-15
+* [FEATURE] Improve transactional handling of messages.
+* [FEATURE] Add failed/corrupt queues.
 
+# 0.11.1 / 2013-12-09
 * [BUGFIX] Re raise 403 forbidden excetion instead of continuing.
 
 # 0.11.0 / 2013-12-03
-
 * [FEATURE] Add support for IAM profiles for AWS auth
 
 # 0.10.0 / 2013-12-03
-
 * [FEATURE] Add queue_suffix config variable
 
 # 0.9.1 / Unreleased
-
 * [FEATURE] Propono will raise exceptions if the message processing fails
 
 # 0.9.0 / Unreleased
-
 * [FEATURE] Add message ids that track throughout Propono
 
 # 0.8.2 / 2013-11-01
-
 * [BUGFIX] Replace thread library with standard ruby threads to fix Unicorn problems.
 
 # 0.8.1 / 2013-11-01
-
 * [FEATURE] Log all messages published from Propono.
 
 # 0.8.0 / 2013-11-01
-
 * [FEATURE] SNS publish now delegates to a thread pool. The SNS response can be accessed via a future.
 
 # 0.7.0 / 2013-10-23
-
 * [FEATURE] Add TCP publish and listen methods.
 
 # 0.6.3 / 2013-10-20
-
 * [FEATURE] Catch all StandardError exceptions for UDP publishes.
 
 # 0.6.2 / 2013-10-20
-
 * [BUGFIX] Fixed integration tests that sometimes failed due to shared UDP ports or slow SQS subscriptions.
 
 # 0.6.1 / 2013-10-20
-
 * [BUGFIX] Added `require 'json'` to udp_listener.rb

data/lib/propono.rb

@@ -15,6 +15,7 @@ require "propono/components/queue"
 require "propono/components/topic"
 require "propono/components/post_subscription"
 require "propono/components/queue_subscription"
+require "propono/components/sqs_message"
 
 require "propono/services/publisher"
 require "propono/services/queue_creator"

data/lib/propono/components/queue_subscription.rb

@@ -4,17 +4,18 @@ module Propono
     include Sns
     include Sqs
 
-    attr_reader :topic_arn, :queue
+    attr_reader :topic_arn, :queue_name, :queue
 
-    def self.create(topic_id)
-      new(topic_id).tap do |subscription|
+    def self.create(topic_id, options = {})
+      new(topic_id, options).tap do |subscription|
         subscription.create
       end
     end
 
-    def initialize(topic_id)
+    def initialize(topic_id, options = {})
       @topic_id = topic_id
       @suffixed_topic_id = "#{topic_id}#{Propono.config.queue_suffix}"
+      @queue_name = "#{Propono.config.application_name.gsub(" ", "_")}-#{@suffixed_topic_id}#{options[:queue_name_suffix]}"
     end
 
     def create
@@ -25,10 +26,6 @@ module Propono
       sqs.set_queue_attributes(@queue.url, "Policy", generate_policy)
     end
 
-    def queue_name
-      @queue_name ||= "#{Propono.config.application_name.gsub(" ", "_")}-#{@suffixed_topic_id}"
-    end
-
     private
 
     def generate_policy

data/lib/propono/components/sns.rb

@@ -5,9 +5,7 @@ module Propono
     private
 
     def sns
-      @sns ||= Fog::AWS::SNS.new(
-        Propono.aws_options
-      )
+      @sns ||= Fog::AWS::SNS.new(Propono.aws_options)
     end
   end
 end

data/lib/propono/components/sqs.rb

@@ -5,9 +5,7 @@ module Propono
     private
 
     def sqs
-      @sqs ||= Fog::AWS::SQS.new(
-        Propono.aws_options
-      )
+      @sqs ||= Fog::AWS::SQS.new(Propono.aws_options)
     end
   end
 end

data/lib/propono/components/sqs_message.rb

@@ -0,0 +1,20 @@
+module Propono
+  class SqsMessage
+    include Sqs
+
+    attr_reader :context, :message, :raw_message, :receipt_handle
+    def initialize(raw_message)
+      body = JSON.parse(raw_message["Body"])["Message"]
+      body = JSON.parse(body)
+
+      @raw_message    = raw_message
+      @context        = body.symbolize_keys
+      @message        = context.delete(:message)
+      @receipt_handle = raw_message["receipt_handle"]
+    end
+
+    def ==(other)
+      other.is_a?(SqsMessage) && other.receipt_handle == @receipt_handle
+    end
+  end
+end

data/lib/propono/services/queue_listener.rb

@@ -1,6 +1,5 @@
 module Propono
   class QueueListener
-
     include Sqs
 
     def self.listen(topic_id, &message_processor)
@@ -29,10 +28,10 @@ module Propono
       if messages.empty?
         false
       else
-        messages.each { |msg| process_sqs_message(msg) }
+        messages.each { |msg| process_raw_message(msg) }
       end
     rescue Excon::Errors::Forbidden
-      Propono.config.logger.error "Forbidden error caught and re raised. #{queue_url}"
+      Propono.config.logger.error "Forbidden error caught and re-raised. #{queue_url}"
       Propono.config.logger.error $!
       raise $!
     rescue
@@ -40,22 +39,44 @@ module Propono
       Propono.config.logger.error $!
     end
 
-    def process_sqs_message(sqs_message)
-      body = JSON.parse(sqs_message["Body"])["Message"]
-
-      # Legacy syntax is covered in the rescue statement
-      # This begin/rescue dance and the rescue block will be removed in v1.
+    # The calls to delete_message are deliberately duplicated so
+    # as to ensure the message is only deleted if the preceeding line
+    # has completed succesfully. We do *not* want to ensure that the
+    # message is deleted regardless of what happens in this method.
+    def process_raw_message(raw_sqs_message)
       begin
-        body = JSON.parse(body)
-        context = body.symbolize_keys
-        message = context.delete(:message)
-        Propono.config.logger.info "Propono [#{context[:id]}]: Received from sqs."
-        @message_processor.call(message, context)
-      rescue JSON::ParserError, TypeError
-        Propono.config.logger.info("Sending and recieving messages without ids is deprecated")
-        @message_processor.call(body)
+        sqs_message = SqsMessage.new(raw_sqs_message)
+        Propono.config.logger.info "Propono [#{sqs_message.context[:id]}]: Received from sqs."
+
+        begin
+          process_message(sqs_message)
+          delete_message(raw_sqs_message)
+        rescue
+          move_to_failed_queue(sqs_message)
+          delete_message(raw_sqs_message)
+        end
+      rescue
+        move_to_corrupt_queue(raw_sqs_message)
+        delete_message(raw_sqs_message)
       end
-      sqs.delete_message(queue_url, sqs_message['ReceiptHandle'])
+    end
+
+    def process_message(sqs_message)
+      @message_processor.call(sqs_message.message, sqs_message.context)
+    end
+
+    def move_to_corrupt_queue(raw_sqs_message)
+      QueueSubscription.create(@topic_id, queue_name_suffix: "-corrupt")
+      Propono.publish("#{@topic_id}-corrupt", raw_sqs_message)
+    end
+
+    def move_to_failed_queue(sqs_message)
+      QueueSubscription.create(@topic_id, queue_name_suffix: "-failed")
+      Propono.publish("#{@topic_id}-failed", sqs_message.message, id: sqs_message.context[:id])
+    end
+
+    def delete_message(raw_sqs_message)
+      sqs.delete_message(queue_url, raw_sqs_message['ReceiptHandle'])
     end
 
     def queue_url

data/lib/propono/version.rb

@@ -1,3 +1,3 @@
 module Propono
-  VERSION = "0.11.1"
+  VERSION = "1.0.0.rc1"
 end

data/test/integration/integration_test.rb

@@ -16,9 +16,9 @@ module Propono
       end
     end
 
-    # Wait a max of 30secs before failing the test
+    # Wait a max of 20secs before failing the test
     def wait_for_thread(thread)
-      300.times do |x|
+      200.times do |x|
         return true unless thread.alive?
         sleep(0.1)
       end

data/test/integration/sns_to_sqs_test.rb

@@ -4,8 +4,9 @@ module Propono
   class SnsToSqsTest < IntegrationTest
     def test_the_message_gets_there
       topic = "test-topic"
-      text = "This is my message"
+      text = "This is my message #{DateTime.now} #{rand()}"
       flunks = []
+      message_received = false
 
       Propono.subscribe_by_queue(topic)
 
@@ -14,7 +15,7 @@ module Propono
           Propono.listen_to_queue(topic) do |message, context|
             flunks << "Wrong message" unless message == text
             flunks << "Wrong id" unless context[:id] =~ Regexp.new("[a-z0-9]{6}")
-            break
+            message_received = true
           end
         rescue => e
           flunks << e.message
@@ -23,7 +24,13 @@ module Propono
         end
       end
 
-      sleep(2) # Make sure the listener has started
+      Thread.new do
+        sleep(1) while !message_received
+        sleep(5) # Make sure all the message deletion clear up in the thread has happened
+        thread.terminate
+      end
+
+      sleep(1) # Make sure the listener has started
 
       Propono.publish(topic, text)
       flunks << "Test Timeout" unless wait_for_thread(thread)

data/test/integration/tcp_to_sqs_test.rb

@@ -4,8 +4,9 @@ module Propono
   class UdpToSqsTest < IntegrationTest
     def test_the_message_gets_there
       topic = "test-topic"
-      message = "This is my message"
+      message = "This is my message #{DateTime.now} #{rand()}"
       flunks = []
+      message_received = false
 
       Propono.config.tcp_host = "localhost"
       Propono.config.tcp_port = 20009
@@ -17,6 +18,7 @@ module Propono
           Propono.listen_to_queue(topic) do |sqs_message|
             flunks << "Wrong message" unless message == sqs_message
             sqs_thread.terminate
+            message_received = true
           end
         rescue => e
           flunks << e.message
@@ -25,13 +27,19 @@ module Propono
         end
       end
 
+      Thread.new do
+        sleep(1) while !message_received
+        sleep(5) # Make sure all the message deletion clear up in the thread has happened
+        sqs_thread.terminate
+      end
+
       tcp_thread = Thread.new do
         Propono.listen_to_tcp do |tcp_topic, tcp_message|
           Propono.publish(tcp_topic, tcp_message)
           tcp_thread.terminate
         end
       end
-      sleep(2) # Make sure the listener has started
+      sleep(1) # Make sure the listener has started
 
       Propono.publish(topic, message, protocol: :tcp)
       flunks << "Test Timeout" unless wait_for_thread(tcp_thread) && wait_for_thread(sqs_thread)

data/test/integration/udp_proxy_test.rb

@@ -4,8 +4,9 @@ module Propono
   class UdpProxyTest < IntegrationTest
     def test_the_message_gets_there
       topic = "test-topic"
-      text = "This is my message"
+      text = "This is my message #{DateTime.now} #{rand()}"
       flunks = []
+      message_received = false
 
       Propono.config.udp_port = 20001
 
@@ -16,7 +17,7 @@ module Propono
           Propono.listen_to_queue(topic) do |message, context|
             flunks << "Wrong message" unless text == message
             flunks << "Wrong id" unless context[:id] =~ Regexp.new("[a-z0-9]{6}-[a-z0-9]{6}")
-            break
+            message_received = true
           end
         rescue => e
           flunks << e.message
@@ -25,11 +26,17 @@ module Propono
         end
       end
 
+      Thread.new do
+        sleep(1) while !message_received
+        sleep(5) # Make sure all the message deletion clear up in the thread has happened
+        sqs_thread.terminate
+      end
+
       udp_thread = Thread.new do
         Propono.proxy_udp
       end
 
-      sleep(2) # Make sure the proxy has started
+      sleep(1) # Make sure the proxy has started
 
       Propono::Publisher.publish(topic, text, protocol: :udp)
       flunks << "Test timeout" unless wait_for_thread(sqs_thread)

data/test/integration/udp_to_sqs_test.rb

@@ -4,8 +4,9 @@ module Propono
   class UdpToSqsTest < IntegrationTest
     def test_the_message_gets_there
       topic = "test-topic"
-      message = "This is my message"
+      message = "This is my message #{DateTime.now} #{rand()}"
       flunks = []
+      message_received = false
 
       Propono.config.udp_port = 20002
 
@@ -16,6 +17,7 @@ module Propono
           Propono.listen_to_queue(topic) do |sqs_message|
             assert_equal message, sqs_message
             sqs_thread.terminate
+            message_received = true
           end
         rescue => e
           flunks << e.message
@@ -24,6 +26,12 @@ module Propono
         end
       end
 
+      Thread.new do
+        sleep(1) while !message_received
+        sleep(5) # Make sure all the message deletion clear up in the thread has happened
+        sqs_thread.terminate
+      end
+
       udp_thread = Thread.new do
         Propono.listen_to_udp do |udp_topic, udp_message|
           Propono.publish(udp_topic, udp_message)
@@ -31,7 +39,7 @@ module Propono
         end
       end
 
-      sleep(2) # Make sure the listener has started
+      sleep(1) # Make sure the listener has started
 
       Propono.publish(topic, message, protocol: :udp)
       flunks << "Test Timeout" unless wait_for_thread(udp_thread) && wait_for_thread(sqs_thread)

data/test/services/queue_listener_test.rb

@@ -57,14 +57,14 @@ module Propono
     end
 
     def test_read_messages_calls_process_message_for_each_msg
-      @listener.expects(:process_sqs_message).with(@sqs_message1)
-      @listener.expects(:process_sqs_message).with(@sqs_message2)
+      @listener.expects(:process_raw_message).with(@sqs_message1)
+      @listener.expects(:process_raw_message).with(@sqs_message2)
       @listener.send(:read_messages)
     end
 
     def test_read_messages_does_not_call_process_messages_if_there_are_none
       @sqs_response.stubs(body: {"Message" => []})
-      @listener.expects(:process_sqs_message).never
+      @listener.expects(:process_message).never
       @listener.send(:read_messages)
     end
 
@@ -79,7 +79,7 @@ module Propono
     def test_forbidden_error_is_logged_and_re_raised
       @listener.stubs(queue_url: "http://example.com")
       @sqs.stubs(:receive_message).raises(Excon::Errors::Forbidden.new(nil, nil, nil))
-      Propono.config.logger.expects(:error).with("Forbidden error caught and re raised. http://example.com")
+      Propono.config.logger.expects(:error).with("Forbidden error caught and re-raised. http://example.com")
       Propono.config.logger.expects(:error).with() {|x| x.is_a?(Excon::Errors::Forbidden)}
       assert_raises Excon::Errors::Forbidden do
         @listener.send(:read_messages)
@@ -123,50 +123,47 @@ module Propono
       @listener.send(:read_messages)
     end
 
-    def test_messages_are_not_deleted_if_there_is_an_exception
+    def test_messages_are_deleted_if_there_is_an_exception_processing
+      queue_url = "test-queue-url"
+
+      @sqs.expects(:delete_message).with(queue_url, @receipt_handle1)
+      @sqs.expects(:delete_message).with(queue_url, @receipt_handle2)
+
       @listener = QueueListener.new(@topic_id) { raise StandardError.new("Test Error") }
+      @listener.stubs(queue_url: queue_url)
       @listener.stubs(sqs: @sqs)
-      @sqs.expects(:delete_message).never
       @listener.send(:read_messages)
     end
-  end
-  class QueueListenerLegacySyntaxTest < Minitest::Test
-
-    def setup
-      super
-      @topic_id = "some-topic"
-
-      @receipt_handle1 = "test-receipt-handle1"
-      @receipt_handle2 = "test-receipt-handle2"
-      @message1 = {'cat' => "Foobar 123"}
-      @message2 = "qwertyuiop"
-      @sqs_message1 = { "ReceiptHandle" => @receipt_handle1, "Body" => {"Message" => @message1}.to_json}
-      @sqs_message2 = { "ReceiptHandle" => @receipt_handle2, "Body" => {"Message" => @message2}.to_json}
-      @messages = { "Message" => [ @sqs_message1, @sqs_message2 ] }
-      @sqs_response = mock().tap{|m|m.stubs(body: @messages)}
-      @sqs = mock()
-      @sqs.stubs(receive_message: @sqs_response)
-      @sqs.stubs(:delete_message)
 
-      @listener = QueueListener.new(@topic_id) {}
+    def test_messages_are_moved_to_failed_queue_if_there_is_an_exception
+      @listener = QueueListener.new(@topic_id) { raise StandardError.new("Test Error") }
+      @listener.expects(:move_to_failed_queue).with(SqsMessage.new(@sqs_message1))
+      @listener.expects(:move_to_failed_queue).with(SqsMessage.new(@sqs_message2))
       @listener.stubs(sqs: @sqs)
+      @listener.send(:read_messages)
     end
 
-    def test_old_syntax_has_deprecation_warning
-      Propono.config.logger.expects(:info).with("Sending and recieving messages without ids is deprecated")
-      @listener.stubs(sqs: @sqs)
+    def test_messages_are_moved_to_corrupt_queue_if_there_is_an_parsing_exception
+      sqs_message1 = "foobar"
+      sqs_message2 = "barfoo"
+      @messages["Message"][0] = sqs_message1
+      @messages["Message"][1] = sqs_message2
+
+      @listener.expects(:move_to_corrupt_queue).with(sqs_message1)
+      @listener.expects(:move_to_corrupt_queue).with(sqs_message2)
       @listener.send(:read_messages)
     end
 
-    def test_each_message_processor_is_yielded
-      messages_yielded = []
-      @listener = QueueListener.new(@topic_id) { |m| messages_yielded.push(m) }
-      @listener.stubs(sqs: @sqs)
-      @listener.send(:read_messages)
+    def test_move_to_failed_queue
+      QueueSubscription.expects(:create).with(@topic_id, queue_name_suffix: "-failed")
+      Propono.expects(:publish).with("#{@topic_id}-failed", @message1, id: @message1_id)
+      @listener.send(:move_to_failed_queue, SqsMessage.new(@sqs_message1))
+    end
 
-      assert_equal messages_yielded.size, 2
-      assert messages_yielded.include?(@message1)
-      assert messages_yielded.include?(@message2)
+    def test_move_to_corrupt_queue
+      QueueSubscription.expects(:create).with(@topic_id, queue_name_suffix: "-corrupt")
+      Propono.expects(:publish).with("#{@topic_id}-corrupt", @sqs_message1)
+      @listener.send(:move_to_corrupt_queue, @sqs_message1)
     end
   end
 end

data/test/services/subscriber_test.rb

@@ -5,7 +5,7 @@ module Propono
 
     def test_subscribe_by_queue_calls_queue_subscriber
       subscriber = QueueSubscription.new("topic")
-      QueueSubscription.expects(:new).with("topic").returns(subscriber)
+      QueueSubscription.expects(:new).with("topic", {}).returns(subscriber)
       QueueSubscription.any_instance.expects(:create)
       Subscriber.subscribe_by_queue("topic")
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: propono
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - MalcyL
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-09 00:00:00.000000000 Z
+date: 2013-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -118,6 +118,7 @@ files:
 - lib/propono/components/queue_subscription.rb
 - lib/propono/components/sns.rb
 - lib/propono/components/sqs.rb
+- lib/propono/components/sqs_message.rb
 - lib/propono/components/topic.rb
 - lib/propono/configuration.rb
 - lib/propono/helpers/hash.rb
@@ -172,9 +173,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.0.3