beetle 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f2aa62352bfa22c0bcf8b02160027aed3429cd6
-  data.tar.gz: 491a52bbb628b89165e09671e505147d540844bf
+  metadata.gz: 0e984630440947f81643287a32934138fcdaee58
+  data.tar.gz: 421b42f6d56a837ba9e2a05a0d0f0087f8d5a737
 SHA512:
-  metadata.gz: 383565b61fe2689ac5767b668710762bda352a8c1c37253e351e7c5d527e97dbd240e5346403881cf3cf428b0b2137eea1fddb4cba15ea6eb3a2afb7f8f04205
-  data.tar.gz: 76fd7ccdb3f16680792695778685fca20102575b1c419ffd6e5fa3d4b48917de7abcbf808f109d2cb3fe70a76cd638d18fffd3165aa93fdefd984daec74722c3
+  metadata.gz: c91106551e7e66cf01cdf614130568abe45dd490bcf51c8e9669214c20af29d91a883c545fea13ae3555a2e8a5c5609784d7b7bbb7e01db733a4c06eefdee91f
+  data.tar.gz: 588b1921ed5a473953572fcf542c62a6a5344700eef6c50d821ee0430a3e38576ee740e0c3330295ff011a887aa63eebd7c56cc056f01a011a502cd4cd2adf46

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,13 @@
 = Release Notes
 
+== Version 0.4.3
+* fixed a race condition which could lead to duplicate message processing
+* fixed eventmachine shutdown sequence problem, which led to ACKs
+  occasionally being lost due to writing to a closed socket, which in
+  turn caused messages to be processed twice
+* stop_listening now always triggers the subscribe shutdown sequence
+  via a eventmachine timer callback, if the eventmachine reactor is running
+
 == Version 0.4.2
 * Fail hard on missing master file
 * Set message timestamp header

data/Rakefile

@@ -1,3 +1,4 @@
+require 'bundler/setup'
 require 'rake'
 require 'rake/testtask'
 require 'bundler/gem_tasks'

data/examples/consume_many_messages_and_shutdown_randomly.rb

@@ -0,0 +1,56 @@
+# consume_many_messages_and_shutdown_randomly.rb
+# this example excercises the shutdown sequence and tests whether
+# messages are handled more than once due to the shutdown
+#
+# ! check the examples/README.rdoc for information on starting your redis/rabbit !
+#
+# use it like so:
+#
+# while ruby consume_many_messages_and_shutdown_randomly.rb; do echo "no duplicate found yet"; done
+#
+# if the loop stops, a duplicate has been found.
+# you can stop this process by sending an interrupt signal
+
+trap("INT"){ puts "ignoring interrupt, please wait" }
+
+require "rubygems"
+require File.expand_path("../lib/beetle", File.dirname(__FILE__))
+
+# set Beetle log level to info, less noisy than debug
+Beetle.config.logger.level = Logger::INFO
+
+# setup client
+client = Beetle::Client.new
+client.register_queue(:test)
+client.register_message(:test)
+
+# create a redis instance with a different database
+redis = Redis.new(:db => 7)
+
+exit_code = 0
+
+# register our handler to the message, check out the message.rb for more stuff you can get from the message object
+client.register_handler(:test) do |message|
+  uuid = message.uuid
+  if redis.incr(uuid) > 1
+    exit_code = 1
+    puts "\n\nRECEIVED A MESSAGE twice: #{uuid}\n\n"
+    client.stop_listening
+  end
+end
+
+# start listening
+# this starts the event machine event loop using EM.run
+# the block passed to listen will be yielded as the last step of the setup process
+client.listen do
+  trap("TERM"){ client.stop_listening }
+  trap("INT"){ exit_code = 1; client.stop_listening }
+  # start a thread which randomly kills us
+  Thread.new do
+    sleep(2 + rand)
+    Process.kill("TERM", $$)
+  end
+  puts "trying to detect duplicates"
+end
+
+exit exit_code

data/examples/publish_many_messages.rb

@@ -0,0 +1,23 @@
+# publish_many_messages.rb
+# this script pbulishes ARGV[0] small test messages (or 100000 if no argument is provided)
+#
+# ! check the examples/README.rdoc for information on starting your redis/rabbit !
+#
+# start it with ruby publish_many_messages.rb 1000000
+
+require "rubygems"
+require File.expand_path("../lib/beetle", File.dirname(__FILE__))
+
+# set Beetle log level to info, less noisy than debug
+Beetle.config.logger.level = Logger::INFO
+
+# setup client
+client = Beetle::Client.new
+client.register_queue(:test)
+client.register_message(:test)
+
+# publish a lot of identical messages
+n = (ARGV[0] || 100000).to_i
+n.times{ client.publish(:test, 'x') }
+
+puts "published #{n} messages"

data/lib/beetle/client.rb

@@ -11,9 +11,9 @@ module Beetle
   # On the publisher side, publishing a message will ensure that the exchange it will be
   # sent to, and each of the queues bound to the exchange, will be created on demand. On
   # the subscriber side, exchanges, queues, bindings and queue subscriptions will be
-  # created when the application calls the listen method. An application can decide to
-  # subscribe to only a subset of the configured queues by passing a list of queue names
-  # to the listen method.
+  # created when the application calls the listen_queues method. An application can decide
+  # to subscribe to only a subset of the configured queues by passing a list of queue
+  # names to the listen method.
   #
   # The net effect of this strategy is that producers and consumers can be started in any
   # order, so that no message is lost if message producers are accidentally started before
@@ -216,14 +216,15 @@ module Beetle
       subscriber.listen_queues(queues, &block)
     end
 
-    # stops the eventmachine loop
+    # stops the subscriber by closing all channels and connections. note this an
+    # asynchronous operation due to the underlying eventmachine mechanism.
     def stop_listening
-      subscriber.stop!
+      @subscriber.stop! if @subscriber
     end
 
     # disconnects the publisher from all servers it's currently connected to
     def stop_publishing
-      publisher.stop
+      @publisher.stop if @publisher
     end
 
     # pause listening on a list of queues
@@ -267,8 +268,8 @@ module Beetle
     end
 
     def reset
-      stop_publishing if @publisher
-      stop_listening if @subscriber
+      stop_publishing
+      stop_listening
       config.reload
       load_brokers_from_config
     rescue Exception => e

data/lib/beetle/deduplication_store.rb

@@ -124,6 +124,12 @@ module Beetle
       with_failover { redis.setnx(key(msg_id, suffix), value) }
     end
 
+    # store some key/value pairs
+    def mset(msg_id, values)
+      values = values.inject([]){|a,(k,v)| a.concat([key(msg_id, k), v])}
+      with_failover { redis.mset(*values) }
+    end
+
     # store some key/value pairs if none of the given keys exist.
     def msetnx(msg_id, values)
       values = values.inject([]){|a,(k,v)| a.concat([key(msg_id, k), v])}
@@ -140,6 +146,12 @@ module Beetle
       with_failover { redis.get(key(msg_id, suffix)) }
     end
 
+    # retrieve the values with given <tt>suffixes</tt> for given <tt>msg_id</tt>. returns a list of strings.
+    def mget(msg_id, keys)
+      keys = keys.map{|suffix| key(msg_id, suffix)}
+      with_failover { redis.mget(*keys) }
+    end
+
     # delete key with given <tt>suffix</tt> for given <tt>msg_id</tt>.
     def del(msg_id, suffix)
       with_failover { redis.del(key(msg_id, suffix)) }

data/lib/beetle/message.rb

@@ -172,8 +172,7 @@ module Beetle
 
     # mark message handling complete in the deduplication store
     def completed!
-      @store.set(msg_id, :status, "completed")
-      timed_out!
+      @store.mset(msg_id, :status => "completed", :timeout => 0)
     end
 
     # whether we should wait before running the handler
@@ -237,6 +236,10 @@ module Beetle
       logger.debug "Beetle: deleted mutex: #{msg_id}"
     end
 
+    def fetch_status_delay_timeout_attempts_exceptions
+      @store.mget(msg_id, [:status, :delay, :timeout, :attempts, :exceptions])
+    end
+
     # process this message and do not allow any exception to escape to the caller
     def process(handler)
       logger.debug "Beetle: processing message #{msg_id}"
@@ -269,30 +272,33 @@ module Beetle
         run_handler(handler) == RC::HandlerCrash ? RC::AttemptsLimitReached : RC::OK
       elsif !key_exists?
         run_handler!(handler)
-      elsif completed?
-        ack!
-        RC::OK
-      elsif delayed?
-        logger.warn "Beetle: ignored delayed message (#{msg_id})!"
-        RC::Delayed
-      elsif !timed_out?
-        RC::HandlerNotYetTimedOut
-      elsif attempts_limit_reached?
-        completed!
-        ack!
-        logger.warn "Beetle: reached the handler execution attempts limit: #{attempts_limit} on #{msg_id}"
-        RC::AttemptsLimitReached
-      elsif exceptions_limit_reached?
-        completed!
-        ack!
-        logger.warn "Beetle: reached the handler exceptions limit: #{exceptions_limit} on #{msg_id}"
-        RC::ExceptionsLimitReached
       else
-        set_timeout!
-        if aquire_mutex!
-          run_handler!(handler)
+        status, delay, timeout, attempts, exceptions = fetch_status_delay_timeout_attempts_exceptions
+        if status == "completed"
+          ack!
+          RC::OK
+        elsif delay && delay.to_i > now
+          logger.warn "Beetle: ignored delayed message (#{msg_id})!"
+          RC::Delayed
+        elsif !(timeout && timeout.to_i < now)
+          RC::HandlerNotYetTimedOut
+        elsif attempts.to_i >= attempts_limit
+          completed!
+          ack!
+          logger.warn "Beetle: reached the handler execution attempts limit: #{attempts_limit} on #{msg_id}"
+          RC::AttemptsLimitReached
+        elsif exceptions.to_i > exceptions_limit
+          completed!
+          ack!
+          logger.warn "Beetle: reached the handler exceptions limit: #{exceptions_limit} on #{msg_id}"
+          RC::ExceptionsLimitReached
         else
-          RC::MutexLocked
+          set_timeout!
+          if aquire_mutex!
+            run_handler!(handler)
+          else
+            RC::MutexLocked
+          end
         end
       end
     end

data/lib/beetle/subscriber.rb

@@ -7,14 +7,13 @@ module Beetle
     # create a new subscriber instance
     def initialize(client, options = {}) #:nodoc:
       super
-      @status = :idle
-      @request_stop = false
       @servers.concat @client.additional_subscription_servers
       @handlers = {}
       @connections = {}
       @channels = {}
       @subscriptions = {}
       @listened_queues = []
+      @channels_closed = false
     end
 
     # the client calls this method to subscribe to a list of queues.
@@ -48,24 +47,19 @@ module Beetle
       end
     end
 
-    # closes all AMQP connections and stops the eventmachine loop
+    # closes all AMQP connections and stop the eventmachine loop.  note that the shutdown
+    # process is asynchronous. must not be called while a message handler is
+    # running. typically one would use <tt>EM.add_timer(0) { stop! }</tt> to ensure this.
     def stop! #:nodoc:
-      if @connections.empty?
-        EM.stop_event_loop
-      else
-        # Only kill connections if not currently processing a message
-        # otherwise messages can get ACKed after the connection is closed
-        # resulting in the ACK not being received and hence the
-        # message being re-delivered
-        if @status == :idle
-          server, connection = @connections.shift
-          logger.debug "Beetle: closing connection to #{server}"
-          connection.close { stop! }
-        else
-          # else ask for stop.  After processing the current message the
-          # stop will be re-attempted
-          @request_stop = true
+      if EM.reactor_running?
+        EM.add_timer(0) do
+          close_all_channels
+          close_all_connections
         end
+      else
+        # try to clean up as much a possible under the circumstances, by closing all connections
+        # this should a least close the sockets
+        close_connections_with_reactor_not_running
       end
     end
 
@@ -78,6 +72,38 @@ module Beetle
 
     private
 
+    # close all sockets.
+    def close_connections_with_reactor_not_running
+      @connections.each { |_, connection| connection.close }
+    ensure
+      @connections = {}
+      @channels = {}
+    end
+
+    # close all connections. this assumes the reactor is running
+    def close_all_connections
+      if @connections.empty?
+        EM.stop_event_loop
+      else
+        server, connection = @connections.shift
+        logger.debug "Beetle: closing connection to #{server}"
+        connection.close { close_all_connections }
+      end
+    end
+
+    # closes all channels. this needs to be the first action during a
+    # subscriber shutdown, so that susbscription callbacks can detect
+    # they should stop processing messages received from the prefetch
+    # queue.
+    def close_all_channels
+      return if @channels_closed
+      @channels.each do |server, channel|
+        logger.debug "Beetle: closing channel to server #{server}"
+        channel.close
+      end
+      @channels_closed = true
+    end
+
     def exchanges_for_queues(queues)
       @client.bindings.slice(*queues).map{|_, opts| opts.map{|opt| opt[:exchange]}}.flatten.uniq
     end
@@ -136,8 +162,11 @@ module Beetle
     def create_subscription_callback(queue_name, amqp_queue_name, handler, opts)
       server = @server
       lambda do |header, data|
+        if channel(server).closing?
+          logger.info "Beetle: ignoring message since channel to server #{server} already closed"
+          return
+        end
         begin
-          @status = :busy
           # logger.debug "Beetle: received message"
           processor = Handler.create(handler, opts)
           message_options = opts.merge(:server => server, :store => @client.deduplication_store)
@@ -167,10 +196,6 @@ module Beetle
         ensure
           # processing_completed swallows all exceptions, so we don't need to protect this call
           processor.processing_completed
-          @status = :idle
-          if @request_stop
-            stop!
-          end
         end
       end
     end

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

data/test/beetle/subscriber_test.rb

@@ -22,29 +22,32 @@ module Beetle
       assert_equal channel, @sub.send(:channel, "donald:1")
     end
 
-    test "stop! should close all amqp connections and then stop the event loop if no handler is currently running" do
-      connection1 = mock('con1')
+    test "stop! should close all amqp channels and connections and then stop the event loop if the reactor is running" do
+      connection1 = mock('conection1')
       connection1.expects(:close).yields
-      connection2 = mock('con2')
+      connection2 = mock('connection2')
       connection2.expects(:close).yields
-      @sub.instance_variable_set "@connections", [["server1", connection1], ["server2",connection2]]
+      channel1 = mock('channel1')
+      channel1.expects(:close)
+      channel2 = mock('channel2')
+      channel2.expects(:close)
+      @sub.instance_variable_set "@connections", [["server1", connection1], ["server2", connection2]]
+      @sub.instance_variable_set "@channels", {"server1" =>  channel1, "server2" => channel2}
+      EM.expects(:reactor_running?).returns(true)
       EM.expects(:stop_event_loop)
+      EM.expects(:add_timer).with(0).yields
       @sub.send(:stop!)
-      assert !@sub.instance_variable_get("@request_stop")
     end
 
-    test "stop! should not stop the event loop if a handler is currently running" do
-      @sub.instance_variable_set "@status", :busy
-      connection1 = mock('con1')
-      connection1.expects(:close).never
-      connection2 = mock('con2')
-      connection2.expects(:close).never
-      @sub.instance_variable_set "@connections", [["server1", connection1], ["server2",connection2]]
-      EM.expects(:stop_event_loop).never
+    test "stop! should close all connections if the reactor is not running" do
+      connection1 = mock('conection1')
+      connection1.expects(:close).yields
+      connection2 = mock('connection2')
+      connection2.expects(:close).yields
+      @sub.instance_variable_set "@connections", [["server1", connection1], ["server2", connection2]]
+      EM.expects(:reactor_running?).returns(false)
       @sub.send(:stop!)
-      assert @sub.instance_variable_get("@request_stop")
     end
-
   end
 
   class SubscriberPauseAndResumeTest < MiniTest::Unit::TestCase
@@ -219,13 +222,16 @@ module Beetle
   end
 
 
-  class DeadLetterrngCallBackExecutionTest < MiniTest::Unit::TestCase
+  class DeadLetteringCallBackExecutionTest < MiniTest::Unit::TestCase
     def setup
       @client = Client.new
       @client.config.dead_lettering_enabled = true
       @queue = "somequeue"
       @client.register_queue(@queue)
       @sub = @client.send(:subscriber)
+      mq = mock("MQ")
+      mq.expects(:closing?).returns(false)
+      @sub.expects(:channel).with(@sub.server).returns(mq)
       @exception = Exception.new "murks"
       @handler = Handler.create(lambda{|*args| raise @exception})
       # handler method 'processing_completed' should be called under all circumstances
@@ -257,42 +263,52 @@ module Beetle
       @sub = client.send(:subscriber)
       @exception = Exception.new "murks"
       @handler = Handler.create(lambda{|*args| raise @exception})
-      # handler method 'processing_completed' should be called under all circumstances
-      @handler.expects(:processing_completed).once
       @callback = @sub.send(:create_subscription_callback, "my myessage", @queue, @handler, :exceptions => 1)
     end
 
     test "exceptions raised from message processing should be ignored" do
+      @handler.expects(:processing_completed).once
       header = header_with_params({})
       Message.any_instance.expects(:process).raises(Exception.new("don't worry"))
+      channel = mock("MQ")
+      channel.expects(:closing?).returns(false)
+      @sub.expects(:channel).with(@sub.server).returns(channel)
       assert_nothing_raised { @callback.call(header, 'foo') }
     end
 
-    test "should call stop! if @request_stop has been set" do
+    test "callback should not process messages if the underlying channel has already been closed" do
+      @handler.expects(:processing_completed).never
       header = header_with_params({})
-      Message.any_instance.expects(:process).raises(Exception.new("don't worry"))
-      @sub.instance_variable_set("@request_stop", true)
-      @sub.expects(:stop!)
+      Message.any_instance.expects(:process).never
+      channel = mock("channel")
+      channel.expects(:closing?).returns(true)
+      @sub.expects(:channel).with(@sub.server).returns(channel)
       assert_nothing_raised { @callback.call(header, 'foo') }
     end
 
     test "should call reject on the message header when processing the handler returns true on reject?" do
+      @handler.expects(:processing_completed).once
       header = header_with_params({})
       result = mock("result")
       result.expects(:reject?).returns(true)
       Message.any_instance.expects(:process).returns(result)
       @sub.expects(:sleep).with(1)
+      mq = mock("MQ")
+      mq.expects(:closing?).returns(false)
+      @sub.expects(:channel).with(@sub.server).returns(mq)
       header.expects(:reject).with(:requeue => true)
       @callback.call(header, 'foo')
     end
 
     test "should sent a reply with status OK if the message reply_to header is set and processing the handler succeeds" do
+      @handler.expects(:processing_completed).once
       header = header_with_params(:reply_to => "tmp-queue")
       result = RC::OK
       Message.any_instance.expects(:process).returns(result)
       Message.any_instance.expects(:handler_result).returns("response-data")
       mq = mock("MQ")
-      @sub.expects(:channel).with(@sub.server).returns(mq)
+      mq.expects(:closing?).returns(false)
+      @sub.expects(:channel).with(@sub.server).returns(mq).twice
       exchange = mock("exchange")
       exchange.expects(:publish).with("response-data", :routing_key => "tmp-queue", :headers => {:status => "OK"}, :persistent => false)
       AMQP::Exchange.expects(:new).with(mq, :direct, "").returns(exchange)
@@ -300,12 +316,14 @@ module Beetle
     end
 
     test "should sent a reply with status FAILED if the message reply_to header is set and processing the handler fails" do
+      @handler.expects(:processing_completed).once
       header = header_with_params(:reply_to => "tmp-queue")
       result = RC::AttemptsLimitReached
       Message.any_instance.expects(:process).returns(result)
       Message.any_instance.expects(:handler_result).returns(nil)
       mq = mock("MQ")
-      @sub.expects(:channel).with(@sub.server).returns(mq)
+      mq.expects(:closing?).returns(false)
+      @sub.expects(:channel).with(@sub.server).returns(mq).twice
       exchange = mock("exchange")
       exchange.expects(:publish).with("", :routing_key => "tmp-queue", :headers => {:status => "FAILED"}, :persistent => false)
       AMQP::Exchange.expects(:new).with(mq, :direct, "").returns(exchange)
@@ -323,6 +341,9 @@ module Beetle
     test "subscribe should subscribe with a subscription callback created from the registered block and remember the subscription" do
       @client.register_queue(:some_queue, :exchange => "some_exchange", :key => "some_key")
       server = @sub.server
+      channel = mock("channel")
+      channel.expects(:closing?).returns(false)
+      @sub.expects(:channel).with(server).returns(channel)
       header = header_with_params({})
       header.expects(:ack)
       block_called = false

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-25 00:00:00.000000000 Z
+date: 2016-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuid4r
@@ -202,12 +202,14 @@ files:
 - bin/beetle
 - examples/README.rdoc
 - examples/attempts.rb
+- examples/consume_many_messages_and_shutdown_randomly.rb
 - examples/handler_class.rb
 - examples/handling_exceptions.rb
 - examples/multiple_exchanges.rb
 - examples/multiple_queues.rb
 - examples/nonexistent_server.rb
 - examples/pause_and_resume.rb
+- examples/publish_many_messages.rb
 - examples/redundant.rb
 - examples/rpc.rb
 - examples/simple.rb
@@ -284,7 +286,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.7
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 3
 summary: High Availability AMQP Messaging with Redundant Queues