pwwka 0.8.0 → 0.9.0.RC1

data/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '2'
+services:
+  rabbit:
+    image: rabbitmq:3.5.6-management
+    ports:
+      - "10001:5672"
+      - "10002:15672"
+  resque:
+    image: redis:2.8.12
+    ports:
+      - "10003:6379"

data/lib/pwwka/configuration.rb

@@ -11,6 +11,7 @@ module Pwwka
     attr_accessor :options
     attr_accessor :send_message_resque_backoff_strategy
     attr_accessor :requeue_on_error
+    attr_writer   :keep_alive_on_handler_klass_exceptions
 
     def initialize
       @rabbit_mq_host        = nil
@@ -22,6 +23,11 @@ module Pwwka
                                                60,                 # quick interruption
                                                600, 600, 600] # longer-term outage?
       @requeue_on_error = false
+      @keep_alive_on_handler_klass_exceptions = false
+    end
+
+    def keep_alive_on_handler_klass_exceptions?
+      @keep_alive_on_handler_klass_exceptions
     end
 
     def payload_logging

data/lib/pwwka/receiver.rb

@@ -29,13 +29,22 @@ module Pwwka
             receiver.ack(delivery_info.delivery_tag)
             logf "Processed Message on %{queue_name} -> %{payload}, %{routing_key}", queue_name: queue_name, payload: payload, routing_key: delivery_info.routing_key
           rescue => e
+            error_options = {
+               queue_name: queue_name,
+                  payload: payload,
+              routing_key: delivery_info.routing_key,
+                exception: e
+            }
             if Pwwka.configuration.requeue_on_error && !delivery_info.redelivered
-              logf "Retrying an Error Processing Message on %{queue_name} -> %{payload}, %{routing_key}: %{exception}: %{backtrace}", queue_name: queue_name, payload: payload, routing_key: delivery_info.routing_key, exception: e, backtrace: e.backtrace.join(';'), at: :error
+              log_error "Retrying an Error Processing Message", error_options
               receiver.nack_requeue(delivery_info.delivery_tag)
             else
-              logf "Error Processing Message on %{queue_name} -> %{payload}, %{routing_key}: %{exception}: %{backtrace}", queue_name: queue_name, payload: payload, routing_key: delivery_info.routing_key, exception: e, backtrace: e.backtrace.join(';'), at: :error
+              log_error "Error Processing Message", error_options
               receiver.nack(delivery_info.delivery_tag)
             end
+            unless Pwwka.configuration.keep_alive_on_handler_klass_exceptions?
+              raise Interrupt,"Exiting due to exception #{e.inspect}"
+            end
           end
         end
       rescue Interrupt => _
@@ -77,5 +86,13 @@ module Pwwka
       channel_connector.connection_close
     end
 
+  private
+
+    def self.log_error(message,options)
+      options[:message] = message
+      options[:backtrace] = options.fetch(:exception).backtrace.join(';')
+      logf "%{message} on %{queue_name} -> %{payload}, %{routing_key}: %{exception}: %{backtrace}", options
+    end
+
   end
 end

data/lib/pwwka/test_handler.rb

@@ -8,6 +8,7 @@ module Pwwka
   # 2. Arrange for `test_teardown` to be called during teardown of your tests
   # 3. Use the method `pop_message` to examine the message on the queue
   class TestHandler
+    include Pwwka::Logging
 
     attr_reader :channel_connector
 

data/lib/pwwka/transmitter.rb

@@ -1,5 +1,6 @@
 begin  # optional dependency
   require 'resque'
+  require 'resque-retry'
 rescue LoadError
 end
 

data/lib/pwwka/version.rb

@@ -1,3 +1,3 @@
 module Pwwka
-  VERSION = '0.8.0'
+  VERSION = '0.9.0.RC1'
 end

data/pwwka.gemspec

@@ -26,4 +26,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency("rake")
   s.add_development_dependency("rspec")
   s.add_development_dependency("resque")
+  s.add_development_dependency("resque-retry")
+  s.add_development_dependency("simplecov")
+  s.add_development_dependency("resqutils")
 end

data/spec/integration/interrupted_receivers_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper.rb'
+require_relative "support/integration_test_setup"
+require_relative "support/logging_receiver"
+require_relative "support/integration_test_helpers"
+
+describe "receivers being interrupted", :integration do
+  include IntegrationTestHelpers
+
+  before do
+    @testing_setup = IntegrationTestSetup.new
+    setup_receivers
+  end
+
+  before :each do
+    WellBehavedReceiver.reset!
+  end
+
+  after do
+    @testing_setup.kill_threads_and_clear_queues
+  end
+
+  it "an error in one receiver doesn't prevent others from getting messages" do
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.foo")
+    allow_receivers_to_process_queues
+
+    expect(WellBehavedReceiver.messages_received.size).to eq(1)
+    expect(@testing_setup.threads[WellBehavedReceiver].alive?).to eq(true)
+    expect(@testing_setup.threads[InterruptingReceiver].alive?).to eq(false)
+  end
+
+  def setup_receivers
+    [
+      [InterruptingReceiver, "interrupting_receiver_pwwkatesting"],
+      [WellBehavedReceiver, "well_behaved_receiver_pwwkatesting"],
+    ].each do |(klass, queue_name)|
+      @testing_setup.make_queue_and_setup_receiver(klass,queue_name,"#")
+    end
+  end
+  class InterruptingReceiver
+    def self.handle!(delivery_info,properties,payload)
+      raise Interrupt,'simulated interrupt would realy be a signal'
+    end
+  end
+  class WellBehavedReceiver < LoggingReceiver
+  end
+end

data/spec/integration/send_and_receive_spec.rb

@@ -0,0 +1,98 @@
+require 'spec_helper.rb'
+require 'resqutils/spec/resque_helpers'
+
+require_relative "support/integration_test_setup"
+require_relative "support/logging_receiver"
+require_relative "support/integration_test_helpers"
+
+describe "sending and receiving messages", :integration do
+  include IntegrationTestHelpers
+  include Resqutils::Spec::ResqueHelpers
+
+  before do
+    @testing_setup = IntegrationTestSetup.new
+    [
+      [AllReceiver      , "all_receiver_pwwkatesting"       , "#"]                 ,
+      [FooReceiver      , "foo_receiver_pwwkatesting"       , "pwwka.testing.foo"] ,
+      [OtherFooReceiver , "other_foo_receiver_pwwkatesting" , "pwwka.testing.foo"] ,
+    ].each do |(klass, queue_name, routing_key)|
+      @testing_setup.make_queue_and_setup_receiver(klass,queue_name,routing_key)
+    end
+  end
+
+  before :each do
+    AllReceiver.reset!
+    FooReceiver.reset!
+    OtherFooReceiver.reset!
+    clear_queue(:delayed)
+  end
+
+  after do
+    @testing_setup.kill_threads_and_clear_queues
+  end
+
+  it "can send a message that gets routed to all receivers" do
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.foo")
+    allow_receivers_to_process_queues
+
+    expect(AllReceiver.messages_received.size).to eq(1)
+    expect(FooReceiver.messages_received.size).to eq(1)
+    expect(OtherFooReceiver.messages_received.size).to eq(1)
+    @testing_setup.queues.each do |queue|
+      expect(queue.message_count).to eq(0)
+    end
+  end
+
+  it "can send a message delayed" do
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.foo",
+                                      delayed: true,
+                                      delay_by: 5_000)
+    allow_receivers_to_process_queues(1_000)
+
+    expect(AllReceiver.messages_received.size).to eq(0)
+    expect(FooReceiver.messages_received.size).to eq(0)
+    expect(OtherFooReceiver.messages_received.size).to eq(0)
+
+    allow_receivers_to_process_queues(5_000)
+    expect(AllReceiver.messages_received.size).to eq(1)
+    expect(FooReceiver.messages_received.size).to eq(1)
+    expect(OtherFooReceiver.messages_received.size).to eq(1)
+  end
+
+  it "can send a message that is only delivered to some  handlers based on routing key" do
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.bar")
+    allow_receivers_to_process_queues
+
+    expect(AllReceiver.messages_received.size).to eq(1)
+    expect(FooReceiver.messages_received.size).to eq(0)
+    expect(OtherFooReceiver.messages_received.size).to eq(0)
+    @testing_setup.queues.each do |queue|
+      expect(queue.message_count).to eq(0)
+    end
+  end
+
+  it "can queue a job to send a message from a Resque job" do
+    Pwwka::Transmitter.send_message_async({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                          "pwwka.testing.bar")
+
+    allow_receivers_to_process_queues # not expecting anything to be processed
+
+    expect(AllReceiver.messages_received.size).to eq(0)
+
+    process_resque_job(Pwwka::SendMessageAsyncJob,:delayed)
+
+    allow_receivers_to_process_queues
+
+    expect(AllReceiver.messages_received.size).to eq(1)
+  end
+
+  class AllReceiver < LoggingReceiver
+  end
+  class FooReceiver < AllReceiver
+  end
+  class OtherFooReceiver < AllReceiver
+  end
+end

data/spec/integration/support/integration_test_helpers.rb

@@ -0,0 +1,5 @@
+module IntegrationTestHelpers
+  def allow_receivers_to_process_queues(ms_to_sleep = 1_000)
+    sleep (ms_to_sleep.to_f / 1_000.0)
+  end
+end

data/spec/integration/support/integration_test_setup.rb

@@ -0,0 +1,40 @@
+class IntegrationTestSetup
+
+  def threads
+    @threads ||= {}
+  end
+
+  def queues
+    @queues ||= []
+  end
+
+  def make_queue_and_setup_receiver(klass,queue_name,routing_key)
+    queue = channel.queue(queue_name, durable: true, arguments: {})
+    queue.bind(topic_exchange, routing_key: routing_key)
+    queues << queue
+    threads[klass] = Thread.new do
+      Pwwka::Receiver.subscribe(klass, queue_name, routing_key: routing_key)
+    end
+  end
+
+  def kill_threads_and_clear_queues
+    threads.each do |_,thread|
+      Thread.kill(thread)
+    end
+    queues.each do |queue|
+      queue.purge
+      queue.delete
+    end
+  end
+
+  def channel_connector
+    @channel_connector ||= Pwwka::ChannelConnector.new
+  end
+  def channel
+    channel_connector.channel
+  end
+  def topic_exchange
+    channel_connector.topic_exchange
+  end
+
+end

data/spec/integration/support/logging_receiver.rb

@@ -0,0 +1,10 @@
+class LoggingReceiver
+  def self.reset!; @messages_received = []; end
+  def self.messages_received; @messages_received ||= []; end
+
+  reset!
+
+  def self.handle!(delivery_info,properties,payload)
+    messages_received << [ delivery_info,properties,payload ]
+  end
+end

data/spec/integration/test_handler_spec.rb

@@ -0,0 +1,78 @@
+require 'spec_helper.rb'
+require_relative "support/integration_test_setup"
+require_relative "support/integration_test_helpers"
+
+describe "test handler for integration tests", :integration do
+  include IntegrationTestHelpers
+
+  subject(:test_handler) { Pwwka::TestHandler.new }
+  before do
+    test_handler.purge_test_queue
+    test_handler.test_setup
+    @testing_setup = IntegrationTestSetup.new
+    @logger = Pwwka.configuration.logger
+  end
+
+  after do
+    test_handler.test_teardown
+    @testing_setup.kill_threads_and_clear_queues
+    Pwwka.configuration.logger = @logger
+  end
+
+  it "allows introspecting messages that were sent" do
+    first_payload = { sample: "payload", has: { deeply: true, nested: 4 }}
+    second_payload = { other: :payload }
+
+    Pwwka::Transmitter.send_message!(first_payload,  "pwwka.testing.foo")
+    Pwwka::Transmitter.send_message!(second_payload, "pwwka.testing.bar")
+
+    first_message = test_handler.pop_message
+    expect(first_message.delivery_info).not_to be_nil
+    expect(first_message.properties).not_to be_nil
+    expect(first_message.payload).to eq(JSON.parse(first_payload.to_json))
+
+    second_message = test_handler.pop_message
+    expect(second_message.delivery_info).not_to be_nil
+    expect(second_message.properties).not_to be_nil
+    expect(second_message.payload).to eq(JSON.parse(second_payload.to_json))
+
+  end
+
+  it "get_topic_message_payload_for_tests" do
+    first_payload = { sample: "payload", has: { deeply: true, nested: 4 }}
+
+    stringio = StringIO.new
+    Pwwka.configuration.logger = Logger.new(stringio)
+    Pwwka::Transmitter.send_message!(first_payload,  "pwwka.testing.foo")
+
+    payload = test_handler.get_topic_message_payload_for_tests
+    expect(payload).to eq(JSON.parse(first_payload.to_json))
+    expect(stringio.string).to match(/get_topic_message_payload_for_tests is deprecated/)
+  end
+
+  it "get_topic_message_properties_for_tests" do
+    first_payload = { sample: "payload", has: { deeply: true, nested: 4 }}
+
+    stringio = StringIO.new
+    Pwwka.configuration.logger = Logger.new(stringio)
+    Pwwka::Transmitter.send_message!(first_payload,  "pwwka.testing.foo")
+
+    properties = test_handler.get_topic_message_properties_for_tests
+    expect(properties).to_not be_nil
+    expect(stringio.string).to match(/get_topic_message_properties_for_tests is deprecated/)
+  end
+
+  it "get_topic_message_delivery_info_for_tests" do
+    first_payload = { sample: "payload", has: { deeply: true, nested: 4 }}
+
+    stringio = StringIO.new
+    Pwwka.configuration.logger = Logger.new(stringio)
+    Pwwka::Transmitter.send_message!(first_payload,  "pwwka.testing.foo")
+
+    delivery_info = test_handler.get_topic_message_delivery_info_for_tests
+    expect(delivery_info).to_not be_nil
+    expect(stringio.string).to match(/get_topic_message_delivery_info_for_tests is deprecated/)
+  end
+
+
+end

data/spec/integration/unhandled_errors_in_receivers_spec.rb

@@ -0,0 +1,115 @@
+require 'spec_helper.rb'
+require_relative "support/integration_test_setup"
+require_relative "support/logging_receiver"
+require_relative "support/integration_test_helpers"
+
+describe "receivers with unhandled errors", :integration do
+  include IntegrationTestHelpers
+
+  before do
+    @testing_setup = IntegrationTestSetup.new
+    setup_receivers
+    Pwwka.configure do |c|
+      c.requeue_on_error = false
+      c.keep_alive_on_handler_klass_exceptions = false
+    end
+  end
+
+  before :each do
+    WellBehavedReceiver.reset!
+    ExceptionThrowingReceiver.reset!
+    IntermittentErrorReceiver.reset!
+  end
+
+  after do
+    @testing_setup.kill_threads_and_clear_queues
+  end
+
+  it "an error in one receiver doesn't prevent others from getting messages" do
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.foo")
+    allow_receivers_to_process_queues
+
+    expect(WellBehavedReceiver.messages_received.size).to eq(1)
+    expect(ExceptionThrowingReceiver.messages_received.size).to eq(1)
+  end
+
+  it "when configured to requeue failed messages, the message is requeued exactly once" do
+    Pwwka.configure do |c|
+      c.requeue_on_error = true
+      c.keep_alive_on_handler_klass_exceptions = true # only so we can check that the requeued message got sent; otherwise the receiver crashes and we can't test that
+    end
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.foo")
+    allow_receivers_to_process_queues
+
+    expect(WellBehavedReceiver.messages_received.size).to eq(1)
+    expect(ExceptionThrowingReceiver.messages_received.size).to eq(2)
+    expect(ExceptionThrowingReceiver.messages_received[1][0].redelivered).to eq(true)
+    expect(ExceptionThrowingReceiver.messages_received[1][2]).to eq(ExceptionThrowingReceiver.messages_received[0][2])
+  end
+
+  it "crashes the receiver that received an error" do
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.foo")
+    allow_receivers_to_process_queues
+
+    expect(@testing_setup.threads[ExceptionThrowingReceiver].alive?).to eq(false)
+  end
+
+  it "does not crash the receiver that received an error when we configure it not to" do
+    Pwwka.configure do |c|
+      c.keep_alive_on_handler_klass_exceptions = true
+    end
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.foo")
+    allow_receivers_to_process_queues
+
+    expect(@testing_setup.threads[ExceptionThrowingReceiver].alive?).to eq(true)
+  end
+
+  it "does not crash the receiver that successfully processed a message" do
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.foo")
+    allow_receivers_to_process_queues
+
+    expect(@testing_setup.threads[WellBehavedReceiver].alive?).to  eq(true)
+  end
+
+  it "crashes the receiver if it gets a failure that we retry" do
+    Pwwka.configure do |c|
+      c.requeue_on_error = true
+    end
+    Pwwka::Transmitter.send_message!({ sample: "payload", has: { deeply: true, nested: 4 }},
+                                     "pwwka.testing.foo")
+    allow_receivers_to_process_queues
+
+    expect(@testing_setup.threads[IntermittentErrorReceiver].alive?).to eq(false)
+  end
+
+  def setup_receivers
+    [
+      [ExceptionThrowingReceiver, "exception_throwing_receiver_pwwkatesting"],
+      [WellBehavedReceiver, "well_behaved_receiver_pwwkatesting"],
+      [IntermittentErrorReceiver, "intermittent_error_receiver_pwwkatesting"],
+    ].each do |(klass, queue_name)|
+      @testing_setup.make_queue_and_setup_receiver(klass,queue_name,"#")
+    end
+  end
+  class ExceptionThrowingReceiver < LoggingReceiver
+    def self.handle!(delivery_info,properties,payload)
+      super(delivery_info,properties,payload)
+      raise "OH NOES!"
+    end
+  end
+  class IntermittentErrorReceiver < LoggingReceiver
+    def self.handle!(delivery_info,properties,payload)
+      super(delivery_info,properties,payload)
+      unless delivery_info.redelivered
+        raise "OH NOES!"
+      end
+    end
+  end
+  class WellBehavedReceiver < LoggingReceiver
+  end
+end